From d1c3a0c5db11f71d199e2b1be83d59e5f90ae6f6 Mon Sep 17 00:00:00 2001 From: Danilo Calderini Date: Tue, 18 Apr 2023 17:38:54 +0200 Subject: [PATCH 01/25] importers, yield curve comparer and docu improved --- ifrs17/Import/Importers.ipynb | 26 +++++++-------- ifrs17/OverviewCalculationEngine.ipynb | 45 +++++++++++++++++++++++++- ifrs17/Utils/EqualityComparers.ipynb | 26 ++++----------- 3 files changed, 63 insertions(+), 34 deletions(-) diff --git a/ifrs17/Import/Importers.ipynb b/ifrs17/Import/Importers.ipynb index 5633d094..1aaa081f 100644 --- a/ifrs17/Import/Importers.ipynb +++ b/ifrs17/Import/Importers.ipynb @@ -508,7 +508,7 @@ "\n", "\nThe following methods are used in the different importers to compute the [IfrsVariables](../DataModel/DataStructure#ifrs-variable).", "\n", - "\nGetAllArgsAsync retrieves the partitions or Args that require computation. They are the union of the primary args (the one read from the main tab of the imported file) with the secondary args (senarios which depends on the best estimate data).", + "\nGetAllArgsAsync retrieves the partitions or Args that require computation. They are the union of the primary args (the one read from the main tab of the imported file) with the secondary args (scenarios which depends on the best estimate data).", "\n", "\nComputeAsync computes the IfrsVariables for a given partition (identified by its ImportArgs) and stores the results in a disposable workspace. This then serves as DataSource in the calculation of the secondary partitions (identified by the secondary args)." ], @@ -862,25 +862,25 @@ "\n var updatedCurrencies = toCommitYieldCurves.Select(x => x.Currency).Distinct();", "\n var dataNodesToUpdate = await workspace.Query().Where(x => updatedCurrencies.Contains(x.ContractualCurrency)).Select(x => x.SystemName).ToArrayAsync();", "\n var workspaceToCompute = Workspace.CreateNew();", - "\n workspaceToCompute.Initialize(x => x.FromSource(options.TargetDataSource));", + "\n workspaceToCompute.Initialize(x => x.FromSource(options.TargetDataSource).DisableInitialization().DisableInitialization());", "\n ", "\n foreach (var args in allArgs) {", "\n await CommitPartitionAsync(args, workspace, workspaceToCompute);", "\n var targetPartition = (Guid)(await options.TargetDataSource.Partition.GetKeyForInstanceAsync(args));", "\n var defaultPartition = (Guid)(await options.TargetDataSource.Partition.GetKeyForInstanceAsync(args with {Scenario = null}));", + "\n var previousPartition = (Guid)(await options.TargetDataSource.Partition.GetKeyForInstanceAsync(args with {Scenario = null, Year = args.Year-1, Month = 12}));", + "\n await options.TargetDataSource.Partition.SetAsync(null);", "\n if(ApplicationMessage.HasErrors()) return Activity.Finish().Merge(importLog);", "\n ", "\n // Avoid starting the computation if no best estimate cash flow has ever been imported ", - "\n if(args.Scenario == null) {", - "\n await options.TargetDataSource.Partition.SetAsync(null);", - "\n if(!(await options.TargetDataSource.Query().Where(x => x.Partition == targetPartition).Take(1).ToArrayAsync()).Any()) continue;", - "\n }", + "\n if(!(await options.TargetDataSource.Query().Where(x => x.Partition == defaultPartition).Take(1).ToArrayAsync()).Any() &&", + "\n !(await options.TargetDataSource.Query().Where(x => x.Partition == defaultPartition).Take(1).ToArrayAsync()).Any()) continue;", + "\n ", + "\n // Only nominals corresponding to the target data nodes are added to the workspace", + "\n var nominals = await options.TargetDataSource.Query().Where(x => dataNodesToUpdate.Contains(x.DataNode) && ", + "\n (x.Partition == targetPartition || x.Partition == defaultPartition || x.Partition == previousPartition)).ToArrayAsync();", + "\n if(nominals.Any()) await workspaceToCompute.UpdateAsync(nominals);", "\n", - "\n // Remove data nodes which are unaffected by the updated yield curves", - "\n // TODO : Reintroduce this functionality. Note all UpdateAsync/DeleteAsync performed to the workspaceToCompute are then trasferred to the DataSource.", - "\n // This is way this functionality should be written in a different way. ", - "\n // await workspaceToCompute.DeleteAsync( await workspaceToCompute.Query()", - "\n // .Where(x => !(dataNodesToUpdate.Contains(x.DataNode) && (x.Partition == targetPartition || x.Partition == defaultPartition))).ToArrayAsync() );", "\n", "\n importLog = importLog.Merge(await ComputeAsync(args, workspace, workspaceToCompute, false));", "\n if(importLog.Errors.Any()) return Activity.Finish().Merge(importLog);", @@ -1243,8 +1243,8 @@ "\n if(ApplicationMessage.HasErrors()) return Activity.Finish().Merge(log);", "\n ", "\n // Avoid starting the computation if no best estimate cash flow and actuals has ever been imported ", - "\n if(args.Scenario == null && !(await options.TargetDataSource.Query().Where(x => x.Partition == targetPartition).Take(1).ToArrayAsync()).Any() &&", - "\n !(await options.TargetDataSource.Query().Where(x => x.Partition == targetPartition).Take(1).ToArrayAsync()).Any()) continue;", + "\n if(!(await options.TargetDataSource.Query().Where(x => x.Partition == defaultPartition).Take(1).ToArrayAsync()).Any() &&", + "\n !(await options.TargetDataSource.Query().Where(x => x.Partition == defaultPartition).Take(1).ToArrayAsync()).Any()) continue;", "\n", "\n // Only nominals corresponding to the target data nodes are added to the workspace", "\n var nominals = await options.TargetDataSource.Query().Where(x => targetDataNodes.Contains(x.DataNode) && ", diff --git a/ifrs17/OverviewCalculationEngine.ipynb b/ifrs17/OverviewCalculationEngine.ipynb index 27e70a33..3535d7eb 100644 --- a/ifrs17/OverviewCalculationEngine.ipynb +++ b/ifrs17/OverviewCalculationEngine.ipynb @@ -214,8 +214,51 @@ "\n- Scenario and Sensitivity analysis", "\n- Run-off Projections (coming soon)", "\n ", + "\n" + ], + "metadata": {}, + "execution_count": 0, + "outputs": [] + }, + { + "cell_type": "markdown", + "source": [ + "## Scenario", + "\n", + "\nSensitivity Analysis (SA) is based on the comparison of different IFRS17 results, each of them labeled by a unique string called Scenario. ", + "\n", + "\nScenario is a Dimension and has to be known by the Platform in order to successfully import a file for that specific Scenario. See [DataModel](./DataModel/DataStructure#data-structure) for an overview of other types of Dimension and the [Initialization](../Ifrs17-template/Initialization/InitSystemorphRefDataToMemory#import-dimensions) to see how to import them.", + "\n", + "\nThe Best Estimate (BE) is the comparison base, and it does not need a column Scenario in the Main section. Alternatevely, the Scenario column can be left empty. ", "\n", - "\n For more information on the latest developments, please refer to our [GitHub](https://github.com/Systemorph/IFRS17CalculationEngine) project page. From there, you can get to know about future releases, place requests, track the current work and report issues." + "\n", + "\nScenarios and BE are stored in distinct Partitions to avoid Data contamination and only non redundant informations are saved." + ], + "metadata": {}, + "execution_count": 0, + "outputs": [] + }, + { + "cell_type": "markdown", + "source": [ + "", + "\n## Fall Back Principle", + "\n", + "\nUsually, only a subset of information changes for a given Scenario. Partial Scenario data is enriched with corresponding BE, called Fall Back Principle (FBP).", + "\nCombining FBP with optimization of latency time, memory usage and business standards inspired the features of the IFRS17 CalculationEngine:", + "\n- Minimun Interaction: importing one single file (Cashflow, Actual, YieldCurve, Data Node Parameter or Exchange Rate) for a specific Scenario creates automatically the corresponding Partition and data without any other action needed from the User.", + "\n- No Artifacts: if BE Transactional Data is not present, importing Scenario does not trigger IFRS17 calculations. Data is saved for succeeding calculations.", + "\n- No Memory Leak: if BE Transactional Data is imported or re-imported, all Scenario dependant Partitions will be updated as well. In order to avoid instability, IFRS17 CalculationEngine does not re-calculate Partitions that belong to subsequent years or months. " + ], + "metadata": {}, + "execution_count": 0, + "outputs": [] + }, + { + "cell_type": "markdown", + "source": [ + "For more information on the latest developments, please refer to our [GitHub](https://github.com/Systemorph/IFRS17CalculationEngine) project page. From there, you can get to know about future releases, place requests, track the current work and report issues.", + "\n " ], "metadata": {}, "execution_count": 0, diff --git a/ifrs17/Utils/EqualityComparers.ipynb b/ifrs17/Utils/EqualityComparers.ipynb index 7323fce0..376537a3 100644 --- a/ifrs17/Utils/EqualityComparers.ipynb +++ b/ifrs17/Utils/EqualityComparers.ipynb @@ -88,35 +88,21 @@ { "cell_type": "code", "source": [ - "using System.Reflection;", - "\nclass IfrsVariableComparer: IEqualityComparer", + "class IfrsVariableComparer: IEqualityComparer", "\n{", "\n private bool IgnoreValue;", - "\n private double precision;", - "\n private IfrsVariableComparer(bool ignoreValue, double precision)", + "\n private IfrsVariableComparer(bool ignoreValue)", "\n {", "\n IgnoreValue = ignoreValue;", - "\n this.precision = precision;", - "\n }", - "\n", - "\n // At this point it is cumbersome and invokes reflections. Will be simplified when we switch to the arrays - A.K.", - "\n private bool CompareValues(IfrsVariable x, IfrsVariable y){", - "\n var values = Enumerable.Range(0, 12).Select(x => x == 0 ? \"Value\" : \"Value\" + x.ToString());", - "\n foreach (var v in values){", - "\n var val1 = (double)x.GetType().GetProperty(v).GetValue(x, null);", - "\n var val2 = (double)y.GetType().GetProperty(v).GetValue(y, null);", - "\n if (Math.Abs(val1 - val2) > precision) return false;", - "\n }", - "\n return true;", "\n }", "\n", "\n public bool Equals(IfrsVariable x, IfrsVariable y) =>", "\n x.AccidentYear == y.AccidentYear && x.AmountType == y.AmountType && x.DataNode == y.DataNode && x.AocType == y.AocType && ", - "\n x.Novelty == y.Novelty && x.EstimateType == y.EstimateType && x.EconomicBasis == y.EconomicBasis && (IgnoreValue ? true : CompareValues(x, y)); ", + "\n x.Novelty == y.Novelty && x.EstimateType == y.EstimateType && x.EconomicBasis == y.EconomicBasis && (IgnoreValue ? true : Math.Abs(x.Value - y.Value) < Precision); ", "\n", "\n public int GetHashCode(IfrsVariable v) => 0;", "\n", - "\n public static IfrsVariableComparer Instance(bool ignoreValue = false, double precision = Precision) => new IfrsVariableComparer(ignoreValue, precision);", + "\n public static IfrsVariableComparer Instance(bool ignoreValue = false) => new IfrsVariableComparer(ignoreValue);", "\n}" ], "metadata": {}, @@ -136,11 +122,11 @@ "\n return true; ", "\n if (x == null || y == null)", "\n return false; ", - "\n if (!(x.Month == y.Month && x.Scenario == y.Scenario && x.Currency == y.Currency && x.Id == y.Id && x.Name == y.Name))", + "\n if (!(x.Scenario == y.Scenario && x.Currency == y.Currency && x.Name == y.Name))", "\n return false; ", "\n if (x.Year == y.Year)", "\n return x.Values.SequenceEqual(y.Values, YieldCurvePrecision); ", - "\n return x.Year > y.Year", + "\n return x.Year > y.Year", "\n ? x.Values.SequenceEqual(y.Values.Skip(x.Year - y.Year).ToArray(), YieldCurvePrecision)", "\n : x.Values.Skip(y.Year - x.Year).ToArray().SequenceEqual(y.Values, YieldCurvePrecision);", "\n }", From 649d62d78f1b148f90d12a4ee3ecd509955cf7f7 Mon Sep 17 00:00:00 2001 From: Danilo Calderini Date: Wed, 19 Apr 2023 16:06:58 +0200 Subject: [PATCH 02/25] bug for scenario cashflow and actuals --- .../Test/ScenarioDataImportTest.ipynb | 18 +++++++++--------- ifrs17/Constants/Validations.ipynb | 4 +++- ifrs17/Import/Importers.ipynb | 11 +++++++---- 3 files changed, 19 insertions(+), 14 deletions(-) diff --git a/ifrs17-template/Test/ScenarioDataImportTest.ipynb b/ifrs17-template/Test/ScenarioDataImportTest.ipynb index b0419970..96086175 100644 --- a/ifrs17-template/Test/ScenarioDataImportTest.ipynb +++ b/ifrs17-template/Test/ScenarioDataImportTest.ipynb @@ -74,7 +74,7 @@ "cell_type": "code", "source": [ "var argsBestEstimate = new ImportArgs (\"CH\", 2020, 12, Periodicity.Quarterly, null, ImportFormats.Cashflow);", - "\nvar argsScenario = new ImportArgs (\"CH\", 2020, 12, Periodicity.Quarterly, \"Test\", ImportFormats.Cashflow);" + "\nvar argsScenario = new ImportArgs (\"CH\", 2020, 12, Periodicity.Quarterly, \"MTUP10pct\", ImportFormats.Cashflow);" ], "metadata": {}, "execution_count": 0, @@ -187,7 +187,7 @@ "var cashflowsScenario = @\"", "\n@@Main", "\nReportingNode,Year,Month,Scenario", - "\nCH,2020,12,Test", + "\nCH,2020,12,MTUP10pct", "\n@@Cashflow", "\nDataNode,AmountType,EstimateType,AocType,Novelty,AccidentYear,Values0,Values1,Values2,Values3,Values4,Values5,Values6,Values7,Values8,Values9,Values10,Values11,Values12,Values13,Values14,Values15,Values16,Values17,Values18,Values19,Values20,Values21,Values22,Values23", "\nDT1.1,PR,BE,CL,C,,110,0,0,110,0,0,110,0,0,110,0,0,0,110,0,0,110,0,0,110,0,0,110,0", @@ -248,7 +248,7 @@ "var actualsScenario = @\"", "\n@@Main", "\nReportingNode,Year,Month,Scenario", - "\nCH,2020,12,Test", + "\nCH,2020,12,MTUP10pct", "\n@@Actual", "\nDataNode,AocType,AmountType,EstimateType,AccidentYear,Value", "\nDT1.1,CF,NIC,A,,-308\";" @@ -315,7 +315,7 @@ "var cashflowsScenarioUpdateCU = @\"", "\n@@Main", "\nReportingNode,Year,Month,Scenario", - "\nCH,2020,12,Test", + "\nCH,2020,12,MTUP10pct", "\n@@Cashflow", "\nDataNode,AmountType,EstimateType,AocType,Novelty,AccidentYear,Values0,Values1,Values2,Values3,Values4,Values5,Values6,Values7,Values8,Values9,Values10,Values11,Values12,Values13,Values14,Values15,Values16,Values17,Values18,Values19,Values20,Values21,Values22,Values23", "\nDT1.1,,CU,CL,C,,-17,-17,-17,-17,-17,-17,-17,-17,-17,-17,-17,-17,-17,-6.5,-6.5,-6.5,-6.5,-6.5,-6.5,-6.5,-6.5,-6.5,-7.5,-8.5\";" @@ -375,7 +375,7 @@ "var actualsScenarioUpdateACAAEA = @\"", "\n@@Main", "\nReportingNode,Year,Month,Scenario", - "\nCH,2020,12,Test", + "\nCH,2020,12,MTUP10pct", "\n@@Actual", "\nDataNode,AocType,AmountType,EstimateType,AccidentYear,Value", "\nDT1.1,CF,ACA,A,,-555", @@ -444,7 +444,7 @@ "var actualsScenarioNoACAAEA = @\"", "\n@@Main", "\nReportingNode,Year,Month,Scenario", - "\nCH,2020,12,Test", + "\nCH,2020,12,MTUP10pct", "\n@@Actual", "\nDataNode,AocType,AmountType,EstimateType,AccidentYear,Value", "\nDT1.1,CF,ACA,A,,0", @@ -500,7 +500,7 @@ "var cashflowsScenarioEqualToBestEstimate = @\"", "\n@@Main", "\nReportingNode,Year,Month,Scenario", - "\nCH,2020,12,Test", + "\nCH,2020,12,MTUP10pct", "\n@@Cashflow", "\nDataNode,AmountType,EstimateType,AocType,Novelty,AccidentYear,Values0,Values1,Values2,Values3,Values4,Values5,Values6,Values7,Values8,Values9,Values10,Values11,Values12,Values13,Values14,Values15,Values16,Values17,Values18,Values19,Values20,Values21,Values22,Values23", "\nDT1.1,PR,BE,CL,C,,100,0,0,100,0,0,100,0,0,100,0,0,0,100,0,0,100,0,0,100,0,0,100,0", @@ -518,7 +518,7 @@ "var actualsScenarioEqualToBestEstimate = @\"", "\n@@Main", "\nReportingNode,Year,Month,Scenario", - "\nCH,2020,12,Test", + "\nCH,2020,12,MTUP10pct", "\n@@Actual", "\nDataNode,AocType,AmountType,EstimateType,AccidentYear,Value", "\nDT1.1,CF,ACA,A,,-10", @@ -850,7 +850,7 @@ "var cashflowsScenarioNoCU = @\"", "\n@@Main", "\nReportingNode,Year,Month,Scenario", - "\nCH,2020,12,Test", + "\nCH,2020,12,MTUP10pct", "\n@@Cashflow", "\nDataNode,AmountType,EstimateType,AocType,Novelty,AccidentYear,Values0,Values1,Values2,Values3,Values4,Values5,Values6,Values7,Values8,Values9,Values10,Values11,Values12,Values13,Values14,Values15,Values16,Values17,Values18,Values19,Values20,Values21,Values22,Values23", "\nDT1.1,PR,BE,CL,C,,110,0,0,110,0,0,110,0,0,110,0,0,0,110,0,0,110,0,0,110,0,0,110,0", diff --git a/ifrs17/Constants/Validations.ipynb b/ifrs17/Constants/Validations.ipynb index b2684e17..bae2493e 100644 --- a/ifrs17/Constants/Validations.ipynb +++ b/ifrs17/Constants/Validations.ipynb @@ -70,7 +70,7 @@ "\n PartitionNotFound, ParsedPartitionNotFound, PartititionNameNotFound, PartitionTypeNotFound,", "\n // Dimensions", "\n AmountTypeNotFound, EstimateTypeNotFound, ReportingNodeNotFound, AocTypeMapNotFound, AocTypeNotFound, PortfolioGicNotFound, PortfolioGricNotFound, ", - "\n InvalidAmountTypeEstimateType, MultipleTechnicalMarginOpening,", + "\n InvalidAmountTypeEstimateType, MultipleTechnicalMarginOpening, DimensionNotFound, ", "\n // Exchange Rate", "\n ExchangeRateNotFound, ExchangeRateCurrency,", "\n // Data Note State", @@ -137,6 +137,8 @@ "\n (Error.PortfolioGricNotFound , 2) => $\"Portfolio {s[0]} assigned to Group of Reinsurance Contract {s[1]} does not exist.\",", "\n (Error.InvalidAmountTypeEstimateType , 2) => $\"Invalid combination of EstimateType {s[0]} and AmountType {s[1]}.\",", "\n (Error.MultipleTechnicalMarginOpening , 1) => $\"Multiple opening for techincal margin are not allowed for DataNode {s[0]}.\",", + "\n (Error.DimensionNotFound , 2) => $\"Property {0} has unknown value {1}.\",", + "\n ", "\n // Exchange Rate", "\n (Error.ExchangeRateNotFound , 2) => $\"Exchange Rate for {s[0]} {s[1]} is not present.\",", "\n (Error.ExchangeRateCurrency , 1) => $\"{s[0]} does not have any Exchange Rate defined.\", ", diff --git a/ifrs17/Import/Importers.ipynb b/ifrs17/Import/Importers.ipynb index 1aaa081f..b5986f2c 100644 --- a/ifrs17/Import/Importers.ipynb +++ b/ifrs17/Import/Importers.ipynb @@ -367,9 +367,11 @@ { "cell_type": "code", "source": [ - "public static void ValidateArgsForPeriod(this ImportArgs args) {", + "public async static void ValidateArgsForPeriod(this ImportArgs args, IDataSource targetDataSource) {", "\n if(args.Year == default(int)) ApplicationMessage.Log(Error.YearInMainNotFound);", "\n if(args.Month == default(int)) ApplicationMessage.Log(Error.MonthInMainNotFound);", + "\n var availableScenarios = await targetDataSource.Query().Select(x => x.SystemName).ToArrayAsync();", + "\n if(!(args.Scenario == default(string) || availableScenarios.Contains(args.Scenario))) ApplicationMessage.Log(Error.DimensionNotFound, \"Scenario\", args.Scenario);", "\n}" ], "metadata": {}, @@ -401,8 +403,9 @@ "\n break;", "\n }", "\n case nameof(PartitionByReportingNodeAndPeriod) : {", - "\n args.ValidateArgsForPeriod();", + "\n args.ValidateArgsForPeriod(dataSource);", "\n if(ApplicationMessage.HasErrors()) return;", + "\n", "\n await dataSource.UpdateAsync( new[]{ new PartitionByReportingNodeAndPeriod { ", "\n Id = (Guid)(await DataSource.Partition.GetKeyForInstanceAsync(args)),", "\n Year = args.Year,", @@ -832,7 +835,7 @@ "Import.DefineFormat(ImportFormats.YieldCurve, async (options, dataSet) => {", "\n Activity.Start();", "\n var primaryArgs = GetArgsFromMain(dataSet) with {ImportFormat = ImportFormats.YieldCurve};", - "\n primaryArgs.ValidateArgsForPeriod();", + "\n primaryArgs.ValidateArgsForPeriod(options.TargetDataSource);", "\n if(ApplicationMessage.HasErrors()) return Activity.Finish();", "\n var workspace = Workspace.CreateNew();", "\n workspace.Initialize(x => x.FromSource(options.TargetDataSource).DisableInitialization().DisableInitialization());", @@ -1213,7 +1216,7 @@ "Import.DefineFormat(ImportFormats.DataNodeParameter, async (options, dataSet) => {", "\n Activity.Start();", "\n var primaryArgs = GetArgsFromMain(dataSet) with {ImportFormat = ImportFormats.DataNodeParameter};", - "\n primaryArgs.ValidateArgsForPeriod();", + "\n primaryArgs.ValidateArgsForPeriod(options.TargetDataSource);", "\n if(ApplicationMessage.HasErrors()) return Activity.Finish();", "\n var workspace = Workspace.CreateNew();", "\n workspace.Initialize(x => x.FromSource(options.TargetDataSource).DisableInitialization().DisableInitialization());", From b84ba09e5353eac12541f2daaf1ce70efe2294b8 Mon Sep 17 00:00:00 2001 From: Danilo Calderini Date: Fri, 21 Apr 2023 16:13:00 +0200 Subject: [PATCH 03/25] first implementation --- ifrs17/DataModel/DataStructure.ipynb | 6 ++++-- ifrs17/OverviewCalculationEngine.ipynb | 11 ++++++++++- ifrs17/Utils/Queries.ipynb | 12 ++++++++++++ 3 files changed, 26 insertions(+), 3 deletions(-) diff --git a/ifrs17/DataModel/DataStructure.ipynb b/ifrs17/DataModel/DataStructure.ipynb index a81bfbf3..7af06b88 100644 --- a/ifrs17/DataModel/DataStructure.ipynb +++ b/ifrs17/DataModel/DataStructure.ipynb @@ -599,7 +599,9 @@ "source": [ "", "\n## Scenario", - "\nThe Scenario record holds the various scenarios for which calculations should also be performed." + "\nThe Scenario record holds the various scenarios used for Sensitivity Analysis (SA). ", + "\nThe default Scenario (i.e. the default one, with no-stress situations applied) is referred to as 'Best Estimate' (BE) and its identifier is a null string, allowing the input files to not specify any value.", + "\n" ], "metadata": {}, "execution_count": 0, @@ -617,7 +619,7 @@ { "cell_type": "markdown", "source": [ - "The default Scenario (i.e. the default one, with no-stress situations applied) is referred to as 'Best Estimate' and its identifier is a null string, allowing the input files to not specify any value." + "Records with importers that accept Scenario creates dedicated [Partitions](#partitions) to store the related data. Usually, only a subset of information changes for a given Scenario and therefore is completed by [Relaxed Queries](../Utils/Queries#relaxedqueries)." ], "metadata": {}, "execution_count": 0, diff --git a/ifrs17/OverviewCalculationEngine.ipynb b/ifrs17/OverviewCalculationEngine.ipynb index 27e70a33..8dbd1ef3 100644 --- a/ifrs17/OverviewCalculationEngine.ipynb +++ b/ifrs17/OverviewCalculationEngine.ipynb @@ -211,7 +211,7 @@ "\n- Timing consideration for Effective Actuals (Payables and Receivables mapped into Paid in Advance or Overdue Actuals)", "\n- Credit Default Risk for Reinsurance", "\n- Financial Performance (FP) and Other Comprehensive Income (OCI)", - "\n- Scenario and Sensitivity analysis", + "\n- [Scenario](./DataModel/DataStructure#scenario) and Sensitivity analysis", "\n- Run-off Projections (coming soon)", "\n ", "\n", @@ -220,6 +220,15 @@ "metadata": {}, "execution_count": 0, "outputs": [] + }, + { + "cell_type": "markdown", + "source": [ + "" + ], + "metadata": {}, + "execution_count": 0, + "outputs": [] } ] } \ No newline at end of file diff --git a/ifrs17/Utils/Queries.ipynb b/ifrs17/Utils/Queries.ipynb index e56cd20e..4dc9dfe7 100644 --- a/ifrs17/Utils/Queries.ipynb +++ b/ifrs17/Utils/Queries.ipynb @@ -25,6 +25,18 @@ "execution_count": 0, "outputs": [] }, + { + "cell_type": "markdown", + "source": [ + "", + "\n# Relaxed queries", + "\n", + "\nThe queries here collected retrieve data from a sorce of data, defined querySource . Priority is given to data from the same [Partition](../DataModel/DataStructure#partitions), which may be both BE or Scenario. If Scenario is missing, it falls back to the corresponding BE." + ], + "metadata": {}, + "execution_count": 0, + "outputs": [] + }, { "cell_type": "code", "source": [ From 102abb8e4d20da2eb73ffac60e2615b5df20719d Mon Sep 17 00:00:00 2001 From: Danilo Calderini Date: Fri, 21 Apr 2023 17:09:27 +0200 Subject: [PATCH 04/25] more documentation --- ifrs17/Import/ImportStorage.ipynb | 2 ++ ifrs17/Import/Importers.ipynb | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/ifrs17/Import/ImportStorage.ipynb b/ifrs17/Import/ImportStorage.ipynb index 58885725..e44ff388 100644 --- a/ifrs17/Import/ImportStorage.ipynb +++ b/ifrs17/Import/ImportStorage.ipynb @@ -37,6 +37,8 @@ "\n- [EstimateType](../DataModel/DataStructure)", "\n- [DataNodes](../DataModel/DataStructure)", "\n", + "\nThe data collection is done using [Relaxed Queries](../Utils/Queries#relaxedqueries) giving to the Import Storage to retrive information from different [Partitions](#partitions).", + "\n", "\nSuch storage is then passed to calculations defined in the corresponding Import Scopes:", "\n", "\n- [1ImportScope-Identities](./1ImportScope-Identities)", diff --git a/ifrs17/Import/Importers.ipynb b/ifrs17/Import/Importers.ipynb index 5633d094..789c6657 100644 --- a/ifrs17/Import/Importers.ipynb +++ b/ifrs17/Import/Importers.ipynb @@ -508,8 +508,8 @@ "\n", "\nThe following methods are used in the different importers to compute the [IfrsVariables](../DataModel/DataStructure#ifrs-variable).", "\n", - "\nGetAllArgsAsync retrieves the partitions or Args that require computation. They are the union of the primary args (the one read from the main tab of the imported file) with the secondary args (senarios which depends on the best estimate data).", - "\n", + "\nGetAllArgsAsync retrieves the partitions or Args that require computation. They are the union of the primary args (the one read from the main tab of the imported file) with the secondary args (scenarios which depends on the Best Estimate data).The implemented logic is different between Transactional Data (Openinig, Cashflow and Actuals) and Parameters (Yield Curves, DataNodeParameter).", + "\n ", "\nComputeAsync computes the IfrsVariables for a given partition (identified by its ImportArgs) and stores the results in a disposable workspace. This then serves as DataSource in the calculation of the secondary partitions (identified by the secondary args)." ], "metadata": {}, From 8bc79d6e2366121bd254b50c490c54c34d22ea90 Mon Sep 17 00:00:00 2001 From: Danilo Calderini Date: Mon, 24 Apr 2023 17:38:29 +0200 Subject: [PATCH 05/25] clean up --- ifrs17/OverviewCalculationEngine.ipynb | 34 -------------------------- 1 file changed, 34 deletions(-) diff --git a/ifrs17/OverviewCalculationEngine.ipynb b/ifrs17/OverviewCalculationEngine.ipynb index 3535d7eb..2d687361 100644 --- a/ifrs17/OverviewCalculationEngine.ipynb +++ b/ifrs17/OverviewCalculationEngine.ipynb @@ -220,40 +220,6 @@ "execution_count": 0, "outputs": [] }, - { - "cell_type": "markdown", - "source": [ - "## Scenario", - "\n", - "\nSensitivity Analysis (SA) is based on the comparison of different IFRS17 results, each of them labeled by a unique string called Scenario. ", - "\n", - "\nScenario is a Dimension and has to be known by the Platform in order to successfully import a file for that specific Scenario. See [DataModel](./DataModel/DataStructure#data-structure) for an overview of other types of Dimension and the [Initialization](../Ifrs17-template/Initialization/InitSystemorphRefDataToMemory#import-dimensions) to see how to import them.", - "\n", - "\nThe Best Estimate (BE) is the comparison base, and it does not need a column Scenario in the Main section. Alternatevely, the Scenario column can be left empty. ", - "\n", - "\n", - "\nScenarios and BE are stored in distinct Partitions to avoid Data contamination and only non redundant informations are saved." - ], - "metadata": {}, - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "source": [ - "", - "\n## Fall Back Principle", - "\n", - "\nUsually, only a subset of information changes for a given Scenario. Partial Scenario data is enriched with corresponding BE, called Fall Back Principle (FBP).", - "\nCombining FBP with optimization of latency time, memory usage and business standards inspired the features of the IFRS17 CalculationEngine:", - "\n- Minimun Interaction: importing one single file (Cashflow, Actual, YieldCurve, Data Node Parameter or Exchange Rate) for a specific Scenario creates automatically the corresponding Partition and data without any other action needed from the User.", - "\n- No Artifacts: if BE Transactional Data is not present, importing Scenario does not trigger IFRS17 calculations. Data is saved for succeeding calculations.", - "\n- No Memory Leak: if BE Transactional Data is imported or re-imported, all Scenario dependant Partitions will be updated as well. In order to avoid instability, IFRS17 CalculationEngine does not re-calculate Partitions that belong to subsequent years or months. " - ], - "metadata": {}, - "execution_count": 0, - "outputs": [] - }, { "cell_type": "markdown", "source": [ From c01acd264f2d3a1bf82dc0e11831021b89fe1c24 Mon Sep 17 00:00:00 2001 From: Danilo Calderini Date: Mon, 24 Apr 2023 21:10:35 +0200 Subject: [PATCH 06/25] test updated --- ifrs17/Test/EqualityComparerTest.ipynb | 2 +- ifrs17/Utils/EqualityComparers.ipynb | 25 +++++++++++++++++++------ 2 files changed, 20 insertions(+), 7 deletions(-) diff --git a/ifrs17/Test/EqualityComparerTest.ipynb b/ifrs17/Test/EqualityComparerTest.ipynb index 1faccfb1..4a200561 100644 --- a/ifrs17/Test/EqualityComparerTest.ipynb +++ b/ifrs17/Test/EqualityComparerTest.ipynb @@ -97,7 +97,7 @@ { "cell_type": "code", "source": [ - "comparer.Equals(yc,yc with { Month = 19}).Should().BeFalse()" + "comparer.Equals(yc,yc with { Month = 19}).Should().BeTrue()" ], "metadata": {}, "execution_count": 0, diff --git a/ifrs17/Utils/EqualityComparers.ipynb b/ifrs17/Utils/EqualityComparers.ipynb index 376537a3..e814bf23 100644 --- a/ifrs17/Utils/EqualityComparers.ipynb +++ b/ifrs17/Utils/EqualityComparers.ipynb @@ -91,18 +91,31 @@ "class IfrsVariableComparer: IEqualityComparer", "\n{", "\n private bool IgnoreValue;", - "\n private IfrsVariableComparer(bool ignoreValue)", - "\n {", - "\n IgnoreValue = ignoreValue;", - "\n }", + "\n private double precision;", + "\n", + "\n private IfrsVariableComparer(bool ignoreValue, double precision)", + "\n {", + "\n IgnoreValue = ignoreValue;", + "\n this.precision = precision;", + "\n }", + "\n ", + "\n // At this point it is cumbersome and invokes reflections. Will be simplified when we switch to the arrays - A.K.", + "\n private bool CompareValues(IfrsVariable x, IfrsVariable y){", + "\n var values = Enumerable.Range(0, 12).Select(x => x == 0 ? Value : Value + x.ToString());", + "\n foreach (var v in values){", + "\n var val1 = (double)x.GetType().GetProperty(v).GetValue(x, null);", + "\n var val2 = (double)y.GetType().GetProperty(v).GetValue(y, null);", + "\n if (Math.Abs(val1 - val2) > precision) return false;", + "\n }", + "\n return true;", "\n", "\n public bool Equals(IfrsVariable x, IfrsVariable y) =>", "\n x.AccidentYear == y.AccidentYear && x.AmountType == y.AmountType && x.DataNode == y.DataNode && x.AocType == y.AocType && ", - "\n x.Novelty == y.Novelty && x.EstimateType == y.EstimateType && x.EconomicBasis == y.EconomicBasis && (IgnoreValue ? true : Math.Abs(x.Value - y.Value) < Precision); ", + "\n x.Novelty == y.Novelty && x.EstimateType == y.EstimateType && x.EconomicBasis == y.EconomicBasis && (IgnoreValue ? true : CompareValues(x, y)); ", "\n", "\n public int GetHashCode(IfrsVariable v) => 0;", "\n", - "\n public static IfrsVariableComparer Instance(bool ignoreValue = false) => new IfrsVariableComparer(ignoreValue);", + "\n public static IfrsVariableComparer Instance(bool ignoreValue = false, double precision = Precision) => new IfrsVariableComparer(ignoreValue, precision);", "\n}" ], "metadata": {}, From 1f41275a89d03a72f432d5a87ae56aa3d1a1d95d Mon Sep 17 00:00:00 2001 From: Danilo Calderini Date: Mon, 24 Apr 2023 21:17:38 +0200 Subject: [PATCH 07/25] bug fix --- ifrs17/Utils/EqualityComparers.ipynb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ifrs17/Utils/EqualityComparers.ipynb b/ifrs17/Utils/EqualityComparers.ipynb index e814bf23..731e3725 100644 --- a/ifrs17/Utils/EqualityComparers.ipynb +++ b/ifrs17/Utils/EqualityComparers.ipynb @@ -92,7 +92,6 @@ "\n{", "\n private bool IgnoreValue;", "\n private double precision;", - "\n", "\n private IfrsVariableComparer(bool ignoreValue, double precision)", "\n {", "\n IgnoreValue = ignoreValue;", @@ -101,13 +100,14 @@ "\n ", "\n // At this point it is cumbersome and invokes reflections. Will be simplified when we switch to the arrays - A.K.", "\n private bool CompareValues(IfrsVariable x, IfrsVariable y){", - "\n var values = Enumerable.Range(0, 12).Select(x => x == 0 ? Value : Value + x.ToString());", + "\n var values = Enumerable.Range(0, 12).Select(x => x == 0 ? \"Value\" : \"Value\" + x.ToString());", "\n foreach (var v in values){", "\n var val1 = (double)x.GetType().GetProperty(v).GetValue(x, null);", "\n var val2 = (double)y.GetType().GetProperty(v).GetValue(y, null);", "\n if (Math.Abs(val1 - val2) > precision) return false;", "\n }", "\n return true;", + "\n }", "\n", "\n public bool Equals(IfrsVariable x, IfrsVariable y) =>", "\n x.AccidentYear == y.AccidentYear && x.AmountType == y.AmountType && x.DataNode == y.DataNode && x.AocType == y.AocType && ", From 9a77fad3013bd9eddc04c5ac82d7d4a8da19c14b Mon Sep 17 00:00:00 2001 From: Danilo Calderini Date: Mon, 24 Apr 2023 21:25:25 +0200 Subject: [PATCH 08/25] cleanup --- ifrs17/Utils/EqualityComparers.ipynb | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/ifrs17/Utils/EqualityComparers.ipynb b/ifrs17/Utils/EqualityComparers.ipynb index 731e3725..f0b30986 100644 --- a/ifrs17/Utils/EqualityComparers.ipynb +++ b/ifrs17/Utils/EqualityComparers.ipynb @@ -93,11 +93,11 @@ "\n private bool IgnoreValue;", "\n private double precision;", "\n private IfrsVariableComparer(bool ignoreValue, double precision)", - "\n {", - "\n IgnoreValue = ignoreValue;", - "\n this.precision = precision;", - "\n }", - "\n ", + "\n {", + "\n IgnoreValue = ignoreValue;", + "\n this.precision = precision;", + "\n }", + "\n", "\n // At this point it is cumbersome and invokes reflections. Will be simplified when we switch to the arrays - A.K.", "\n private bool CompareValues(IfrsVariable x, IfrsVariable y){", "\n var values = Enumerable.Range(0, 12).Select(x => x == 0 ? \"Value\" : \"Value\" + x.ToString());", @@ -106,7 +106,7 @@ "\n var val2 = (double)y.GetType().GetProperty(v).GetValue(y, null);", "\n if (Math.Abs(val1 - val2) > precision) return false;", "\n }", - "\n return true;", + "\n return true;", "\n }", "\n", "\n public bool Equals(IfrsVariable x, IfrsVariable y) =>", From 16149e7de24e511c7254da1574b055d117a9f080 Mon Sep 17 00:00:00 2001 From: Danilo Calderini Date: Tue, 25 Apr 2023 10:06:00 +0200 Subject: [PATCH 09/25] error message improved --- ifrs17/Constants/Validations.ipynb | 3 ++- ifrs17/Import/Importers.ipynb | 5 +++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/ifrs17/Constants/Validations.ipynb b/ifrs17/Constants/Validations.ipynb index bae2493e..048efc1d 100644 --- a/ifrs17/Constants/Validations.ipynb +++ b/ifrs17/Constants/Validations.ipynb @@ -70,7 +70,7 @@ "\n PartitionNotFound, ParsedPartitionNotFound, PartititionNameNotFound, PartitionTypeNotFound,", "\n // Dimensions", "\n AmountTypeNotFound, EstimateTypeNotFound, ReportingNodeNotFound, AocTypeMapNotFound, AocTypeNotFound, PortfolioGicNotFound, PortfolioGricNotFound, ", - "\n InvalidAmountTypeEstimateType, MultipleTechnicalMarginOpening, DimensionNotFound, ", + "\n InvalidAmountTypeEstimateType, MultipleTechnicalMarginOpening, DimensionNotFound, NoScenarioOpening,", "\n // Exchange Rate", "\n ExchangeRateNotFound, ExchangeRateCurrency,", "\n // Data Note State", @@ -138,6 +138,7 @@ "\n (Error.InvalidAmountTypeEstimateType , 2) => $\"Invalid combination of EstimateType {s[0]} and AmountType {s[1]}.\",", "\n (Error.MultipleTechnicalMarginOpening , 1) => $\"Multiple opening for techincal margin are not allowed for DataNode {s[0]}.\",", "\n (Error.DimensionNotFound , 2) => $\"Property {0} has unknown value {1}.\",", + "\n (Error.NoScenarioOpening , 0) => \"Only Best Estimate is valid Scenario for Openings\",", "\n ", "\n // Exchange Rate", "\n (Error.ExchangeRateNotFound , 2) => $\"Exchange Rate for {s[0]} {s[1]} is not present.\",", diff --git a/ifrs17/Import/Importers.ipynb b/ifrs17/Import/Importers.ipynb index d5211f39..f25e342e 100644 --- a/ifrs17/Import/Importers.ipynb +++ b/ifrs17/Import/Importers.ipynb @@ -511,8 +511,8 @@ "\n", "\nThe following methods are used in the different importers to compute the [IfrsVariables](../DataModel/DataStructure#ifrs-variable).", "\n", - "\nGetAllArgsAsync retrieves the partitions or Args that require computation. They are the union of the primary args (the one read from the main tab of the imported file) with the secondary args (scenarios which depends on the Best Estimate data).The implemented logic is different between Transactional Data (Openinig, Cashflow and Actuals) and Parameters (Yield Curves, DataNodeParameter).", - "\n ", + "\nGetAllArgsAsync retrieves the partitions or Args that require computation. They are the union of the primary args (the one read from the main tab of the imported file) with the secondary args (scenarios which depends on the best estimate data).", + "\n", "\nComputeAsync computes the IfrsVariables for a given partition (identified by its ImportArgs) and stores the results in a disposable workspace. This then serves as DataSource in the calculation of the secondary partitions (identified by the secondary args)." ], "metadata": {}, @@ -1619,6 +1619,7 @@ "Import.DefineFormat(ImportFormats.Opening, async (options, dataSet) => {", "\n Activity.Start();", "\n var primaryArgs = await GetArgsAndCommitPartitionAsync(dataSet, options.TargetDataSource) with {ImportFormat = ImportFormats.Opening};", + "\n if(primaryArgs.Scenario != default(string)) ApplicationMessage.Log(Error.NoScenarioOpening);", "\n if(Activity.HasErrors()) return Activity.Finish();", "\n", "\n var allArgs = await GetAllArgsAsync(primaryArgs, options.TargetDataSource, ImportFormats.Opening);", From 093db6a7adab563e392e0850d9c7f0050868c0e9 Mon Sep 17 00:00:00 2001 From: Danilo Calderini Date: Tue, 25 Apr 2023 10:57:21 +0200 Subject: [PATCH 10/25] bug yield curve importer --- ifrs17/Constants/Validations.ipynb | 3 ++- ifrs17/Import/Importers.ipynb | 21 +++++++++++++-------- 2 files changed, 15 insertions(+), 9 deletions(-) diff --git a/ifrs17/Constants/Validations.ipynb b/ifrs17/Constants/Validations.ipynb index bae2493e..048efc1d 100644 --- a/ifrs17/Constants/Validations.ipynb +++ b/ifrs17/Constants/Validations.ipynb @@ -70,7 +70,7 @@ "\n PartitionNotFound, ParsedPartitionNotFound, PartititionNameNotFound, PartitionTypeNotFound,", "\n // Dimensions", "\n AmountTypeNotFound, EstimateTypeNotFound, ReportingNodeNotFound, AocTypeMapNotFound, AocTypeNotFound, PortfolioGicNotFound, PortfolioGricNotFound, ", - "\n InvalidAmountTypeEstimateType, MultipleTechnicalMarginOpening, DimensionNotFound, ", + "\n InvalidAmountTypeEstimateType, MultipleTechnicalMarginOpening, DimensionNotFound, NoScenarioOpening,", "\n // Exchange Rate", "\n ExchangeRateNotFound, ExchangeRateCurrency,", "\n // Data Note State", @@ -138,6 +138,7 @@ "\n (Error.InvalidAmountTypeEstimateType , 2) => $\"Invalid combination of EstimateType {s[0]} and AmountType {s[1]}.\",", "\n (Error.MultipleTechnicalMarginOpening , 1) => $\"Multiple opening for techincal margin are not allowed for DataNode {s[0]}.\",", "\n (Error.DimensionNotFound , 2) => $\"Property {0} has unknown value {1}.\",", + "\n (Error.NoScenarioOpening , 0) => \"Only Best Estimate is valid Scenario for Openings\",", "\n ", "\n // Exchange Rate", "\n (Error.ExchangeRateNotFound , 2) => $\"Exchange Rate for {s[0]} {s[1]} is not present.\",", diff --git a/ifrs17/Import/Importers.ipynb b/ifrs17/Import/Importers.ipynb index d03f7b54..9753cfb8 100644 --- a/ifrs17/Import/Importers.ipynb +++ b/ifrs17/Import/Importers.ipynb @@ -843,14 +843,18 @@ "\n var committedYieldCurves = await options.TargetDataSource.Query().ToArrayAsync();", "\n var hasNameColumn = dataSet.Tables[ImportFormats.YieldCurve].Columns.Any(x => x.ColumnName == nameof(YieldCurve.Name));", "\n var importLog = await Import.FromDataSet(dataSet).WithType(", - "\n (dataset, datarow) => new YieldCurve {", - "\n Currency = datarow.Field(nameof(YieldCurve.Currency)),", - "\n Year = primaryArgs.Year,", - "\n Month = primaryArgs.Month, ", - "\n Scenario = primaryArgs.Scenario,", - "\n Values = datarow.Table.Columns.Where(c => c.ColumnName.StartsWith(nameof(YieldCurve.Values))).OrderBy(c => c.ColumnName.Length).ThenBy(c => c.ColumnName)", - "\n .Select(x => datarow.Field(x.ColumnName).CheckStringForExponentialAndConvertToDouble()).ToArray(),", - "\n Name = hasNameColumn ? datarow.Field(nameof(YieldCurve.Name)) : default(string)", + "\n (dataset, datarow) => {", + "\n var values = datarow.Table.Columns.Where(c => c.ColumnName.StartsWith(nameof(YieldCurve.Values))).OrderBy(c => c.ColumnName.Length).ThenBy(c => c.ColumnName)", + "\n .Select(x => datarow.Field(x.ColumnName).CheckStringForExponentialAndConvertToDouble()).ToArray().Prune();", + "\n if (values.Length == 0 ) return null;", + "\n return new YieldCurve {", + "\n Currency = datarow.Field(nameof(YieldCurve.Currency)),", + "\n Year = primaryArgs.Year,", + "\n Month = primaryArgs.Month, ", + "\n Scenario = primaryArgs.Scenario,", + "\n Values = values,", + "\n Name = hasNameColumn ? datarow.Field(nameof(YieldCurve.Name)) : default(string)", + "\n };", "\n }", "\n ).WithTarget(workspace).ExecuteAsync(); ", "\n", @@ -1619,6 +1623,7 @@ "Import.DefineFormat(ImportFormats.Opening, async (options, dataSet) => {", "\n Activity.Start();", "\n var primaryArgs = await GetArgsAndCommitPartitionAsync(dataSet, options.TargetDataSource) with {ImportFormat = ImportFormats.Opening};", + "\n if(primaryArgs.Scenario != default(string)) ApplicationMessage.Log(Error.NoScenarioOpening);", "\n if(Activity.HasErrors()) return Activity.Finish();", "\n", "\n var allArgs = await GetAllArgsAsync(primaryArgs, options.TargetDataSource, ImportFormats.Opening);", From e0608d8dc142e8d1a02c19dcdeb110f561d0d734 Mon Sep 17 00:00:00 2001 From: Danilo Calderini Date: Tue, 25 Apr 2023 11:16:52 +0200 Subject: [PATCH 11/25] code clean up --- ifrs17/OverviewCalculationEngine.ipynb | 12 ++---------- ifrs17/Utils/EqualityComparers.ipynb | 2 +- 2 files changed, 3 insertions(+), 11 deletions(-) diff --git a/ifrs17/OverviewCalculationEngine.ipynb b/ifrs17/OverviewCalculationEngine.ipynb index 2d687361..46e44c4c 100644 --- a/ifrs17/OverviewCalculationEngine.ipynb +++ b/ifrs17/OverviewCalculationEngine.ipynb @@ -213,22 +213,14 @@ "\n- Financial Performance (FP) and Other Comprehensive Income (OCI)", "\n- Scenario and Sensitivity analysis", "\n- Run-off Projections (coming soon)", + "\n", + "\nFor more information on the latest developments, please refer to our [GitHub](https://github.com/Systemorph/IFRS17CalculationEngine) project page. From there, you can get to know about future releases, place requests, track the current work and report issues.", "\n ", "\n" ], "metadata": {}, "execution_count": 0, "outputs": [] - }, - { - "cell_type": "markdown", - "source": [ - "For more information on the latest developments, please refer to our [GitHub](https://github.com/Systemorph/IFRS17CalculationEngine) project page. From there, you can get to know about future releases, place requests, track the current work and report issues.", - "\n " - ], - "metadata": {}, - "execution_count": 0, - "outputs": [] } ] } \ No newline at end of file diff --git a/ifrs17/Utils/EqualityComparers.ipynb b/ifrs17/Utils/EqualityComparers.ipynb index f0b30986..31c4359f 100644 --- a/ifrs17/Utils/EqualityComparers.ipynb +++ b/ifrs17/Utils/EqualityComparers.ipynb @@ -139,7 +139,7 @@ "\n return false; ", "\n if (x.Year == y.Year)", "\n return x.Values.SequenceEqual(y.Values, YieldCurvePrecision); ", - "\n return x.Year > y.Year", + "\n return x.Year > y.Year", "\n ? x.Values.SequenceEqual(y.Values.Skip(x.Year - y.Year).ToArray(), YieldCurvePrecision)", "\n : x.Values.Skip(y.Year - x.Year).ToArray().SequenceEqual(y.Values, YieldCurvePrecision);", "\n }", From 81c130c224b2cae51eb4ca0b383ab0ecbdc8f755 Mon Sep 17 00:00:00 2001 From: Danilo Calderini Date: Tue, 25 Apr 2023 14:22:39 +0200 Subject: [PATCH 12/25] comments resolved --- .../TransactionalData/Actuals_CH_2020_12.csv | 2 +- ifrs17/Constants/Validations.ipynb | 2 +- ifrs17/Import/Importers.ipynb | 19 +++++++++++-------- 3 files changed, 13 insertions(+), 10 deletions(-) diff --git a/ifrs17-template/Files/TransactionalData/Actuals_CH_2020_12.csv b/ifrs17-template/Files/TransactionalData/Actuals_CH_2020_12.csv index a7900401..a8d71374 100644 --- a/ifrs17-template/Files/TransactionalData/Actuals_CH_2020_12.csv +++ b/ifrs17-template/Files/TransactionalData/Actuals_CH_2020_12.csv @@ -1,5 +1,5 @@ @@Main,,,,, -ReportingNode,Year,Month,Scenario ,, +ReportingNode,Year,Month,Scenario CH,2020,12,,, @@Actual,,,,, DataNode,AocType,AmountType,EstimateType,AccidentYear,Value diff --git a/ifrs17/Constants/Validations.ipynb b/ifrs17/Constants/Validations.ipynb index 048efc1d..0152a7e0 100644 --- a/ifrs17/Constants/Validations.ipynb +++ b/ifrs17/Constants/Validations.ipynb @@ -137,7 +137,7 @@ "\n (Error.PortfolioGricNotFound , 2) => $\"Portfolio {s[0]} assigned to Group of Reinsurance Contract {s[1]} does not exist.\",", "\n (Error.InvalidAmountTypeEstimateType , 2) => $\"Invalid combination of EstimateType {s[0]} and AmountType {s[1]}.\",", "\n (Error.MultipleTechnicalMarginOpening , 1) => $\"Multiple opening for techincal margin are not allowed for DataNode {s[0]}.\",", - "\n (Error.DimensionNotFound , 2) => $\"Property {0} has unknown value {1}.\",", + "\n (Error.DimensionNotFound , 2) => $\"Column {0} has unknown value {1}.\",", "\n (Error.NoScenarioOpening , 0) => \"Only Best Estimate is valid Scenario for Openings\",", "\n ", "\n // Exchange Rate", diff --git a/ifrs17/Import/Importers.ipynb b/ifrs17/Import/Importers.ipynb index 9753cfb8..0687c90e 100644 --- a/ifrs17/Import/Importers.ipynb +++ b/ifrs17/Import/Importers.ipynb @@ -367,7 +367,7 @@ { "cell_type": "code", "source": [ - "public async static void ValidateArgsForPeriod(this ImportArgs args, IDataSource targetDataSource) {", + "public async static void ValidateArgsForPeriodAsync(this ImportArgs args, IDataSource targetDataSource) {", "\n if(args.Year == default(int)) ApplicationMessage.Log(Error.YearInMainNotFound);", "\n if(args.Month == default(int)) ApplicationMessage.Log(Error.MonthInMainNotFound);", "\n var availableScenarios = await targetDataSource.Query().Select(x => x.SystemName).ToArrayAsync();", @@ -403,7 +403,7 @@ "\n break;", "\n }", "\n case nameof(PartitionByReportingNodeAndPeriod) : {", - "\n args.ValidateArgsForPeriod(dataSource);", + "\n args.ValidateArgsForPeriodAsync(dataSource);", "\n if(ApplicationMessage.HasErrors()) return;", "\n", "\n await dataSource.UpdateAsync( new[]{ new PartitionByReportingNodeAndPeriod { ", @@ -835,7 +835,7 @@ "Import.DefineFormat(ImportFormats.YieldCurve, async (options, dataSet) => {", "\n Activity.Start();", "\n var primaryArgs = GetArgsFromMain(dataSet) with {ImportFormat = ImportFormats.YieldCurve};", - "\n primaryArgs.ValidateArgsForPeriod(options.TargetDataSource);", + "\n primaryArgs.ValidateArgsForPeriodAsync(options.TargetDataSource);", "\n if(ApplicationMessage.HasErrors()) return Activity.Finish();", "\n var workspace = Workspace.CreateNew();", "\n workspace.Initialize(x => x.FromSource(options.TargetDataSource).DisableInitialization().DisableInitialization());", @@ -879,14 +879,17 @@ "\n await options.TargetDataSource.Partition.SetAsync(null);", "\n if(ApplicationMessage.HasErrors()) return Activity.Finish().Merge(importLog);", "\n ", - "\n // Avoid starting the computation if no best estimate cash flow has ever been imported ", + "\n // Avoid starting the computation if no best estimate cash flow or actual has ever been imported ", "\n if(!(await options.TargetDataSource.Query().Where(x => x.Partition == defaultPartition).Take(1).ToArrayAsync()).Any() &&", "\n !(await options.TargetDataSource.Query().Where(x => x.Partition == defaultPartition).Take(1).ToArrayAsync()).Any()) continue;", "\n ", - "\n // Only nominals corresponding to the target data nodes are added to the workspace", - "\n var nominals = await options.TargetDataSource.Query().Where(x => dataNodesToUpdate.Contains(x.DataNode) && ", + "\n // Only rawvariables and ifrsvariables corresponding to the target data nodes are added to the workspace", + "\n var rawvariables = await options.TargetDataSource.Query().Where(x => dataNodesToUpdate.Contains(x.DataNode) && ", "\n (x.Partition == targetPartition || x.Partition == defaultPartition || x.Partition == previousPartition)).ToArrayAsync();", - "\n if(nominals.Any()) await workspaceToCompute.UpdateAsync(nominals);", + "\n if(rawvariables.Any()) await workspaceToCompute.UpdateAsync(rawvariables);", + "\n var ifrsvariables = await options.TargetDataSource.Query().Where(x => dataNodesToUpdate.Contains(x.DataNode) && ", + "\n (x.Partition == targetPartition || x.Partition == defaultPartition || x.Partition == previousPartition)).ToArrayAsync();", + "\n if(ifrsvariables.Any()) await workspaceToCompute.UpdateAsync(ifrsvariables);", "\n", "\n", "\n importLog = importLog.Merge(await ComputeAsync(args, workspace, workspaceToCompute, false));", @@ -1220,7 +1223,7 @@ "Import.DefineFormat(ImportFormats.DataNodeParameter, async (options, dataSet) => {", "\n Activity.Start();", "\n var primaryArgs = GetArgsFromMain(dataSet) with {ImportFormat = ImportFormats.DataNodeParameter};", - "\n primaryArgs.ValidateArgsForPeriod(options.TargetDataSource);", + "\n primaryArgs.ValidateArgsForPeriodAsync(options.TargetDataSource);", "\n if(ApplicationMessage.HasErrors()) return Activity.Finish();", "\n var workspace = Workspace.CreateNew();", "\n workspace.Initialize(x => x.FromSource(options.TargetDataSource).DisableInitialization().DisableInitialization());", From 421c9298584d500040c5e4c1392ecd1e64e08295 Mon Sep 17 00:00:00 2001 From: Danilo Calderini Date: Tue, 25 Apr 2023 15:58:40 +0200 Subject: [PATCH 13/25] revert to previous implementation --- ifrs17/Import/Importers.ipynb | 46 ++++++++++++++++------------------- 1 file changed, 21 insertions(+), 25 deletions(-) diff --git a/ifrs17/Import/Importers.ipynb b/ifrs17/Import/Importers.ipynb index 0687c90e..73557a44 100644 --- a/ifrs17/Import/Importers.ipynb +++ b/ifrs17/Import/Importers.ipynb @@ -838,11 +838,13 @@ "\n primaryArgs.ValidateArgsForPeriodAsync(options.TargetDataSource);", "\n if(ApplicationMessage.HasErrors()) return Activity.Finish();", "\n var workspace = Workspace.CreateNew();", - "\n workspace.Initialize(x => x.FromSource(options.TargetDataSource).DisableInitialization().DisableInitialization());", + "\n workspace.Initialize(x => x.FromSource(options.TargetDataSource))", + "\n .DisableInitialization()", + "\n .DisableInitialization());", "\n", "\n var committedYieldCurves = await options.TargetDataSource.Query().ToArrayAsync();", "\n var hasNameColumn = dataSet.Tables[ImportFormats.YieldCurve].Columns.Any(x => x.ColumnName == nameof(YieldCurve.Name));", - "\n var importLog = await Import.FromDataSet(dataSet).WithType(", + "\n var log = await Import.FromDataSet(dataSet).WithType(", "\n (dataset, datarow) => {", "\n var values = datarow.Table.Columns.Where(c => c.ColumnName.StartsWith(nameof(YieldCurve.Values))).OrderBy(c => c.ColumnName.Length).ThenBy(c => c.ColumnName)", "\n .Select(x => datarow.Field(x.ColumnName).CheckStringForExponentialAndConvertToDouble()).ToArray().Prune();", @@ -858,47 +860,41 @@ "\n }", "\n ).WithTarget(workspace).ExecuteAsync(); ", "\n", - "\n if(importLog.Errors.Any()) return Activity.Finish().Merge(importLog); ", + "\n if(log.Errors.Any()) return Activity.Finish().Merge(log); ", "\n var toCommitYieldCurves = (await workspace.Query().ToArrayAsync()).Except(committedYieldCurves, YieldCurveComparer.Instance());", "\n if (!toCommitYieldCurves.Any()) {", "\n ApplicationMessage.Log(Warning.VariablesAlreadyImported); ", - "\n return Activity.Finish().Merge(importLog);", + "\n return Activity.Finish().Merge(log);", "\n }", "\n", "\n var allArgs = await GetAllArgsAsync(primaryArgs, options.TargetDataSource, ImportFormats.YieldCurve);", "\n var updatedCurrencies = toCommitYieldCurves.Select(x => x.Currency).Distinct();", "\n var dataNodesToUpdate = await workspace.Query().Where(x => updatedCurrencies.Contains(x.ContractualCurrency)).Select(x => x.SystemName).ToArrayAsync();", "\n var workspaceToCompute = Workspace.CreateNew();", - "\n workspaceToCompute.Initialize(x => x.FromSource(options.TargetDataSource).DisableInitialization().DisableInitialization());", - "\n ", + "\n workspaceToCompute.Initialize(x => x.FromSource(options.TargetDataSource));", + "\n ApplicationMessage.Log(Warning.Generic, String.Join(\", \", dataNodesToUpdate));", "\n foreach (var args in allArgs) {", "\n await CommitPartitionAsync(args, workspace, workspaceToCompute);", "\n var targetPartition = (Guid)(await options.TargetDataSource.Partition.GetKeyForInstanceAsync(args));", "\n var defaultPartition = (Guid)(await options.TargetDataSource.Partition.GetKeyForInstanceAsync(args with {Scenario = null}));", - "\n var previousPartition = (Guid)(await options.TargetDataSource.Partition.GetKeyForInstanceAsync(args with {Scenario = null, Year = args.Year-1, Month = 12}));", - "\n await options.TargetDataSource.Partition.SetAsync(null);", - "\n if(ApplicationMessage.HasErrors()) return Activity.Finish().Merge(importLog);", + "\n if(ApplicationMessage.HasErrors()) return Activity.Finish().Merge(log);", "\n ", - "\n // Avoid starting the computation if no best estimate cash flow or actual has ever been imported ", - "\n if(!(await options.TargetDataSource.Query().Where(x => x.Partition == defaultPartition).Take(1).ToArrayAsync()).Any() &&", - "\n !(await options.TargetDataSource.Query().Where(x => x.Partition == defaultPartition).Take(1).ToArrayAsync()).Any()) continue;", + "\n // Avoid starting the computation if no best estimate cash flow has ever been imported ", + "\n if(!(await options.TargetDataSource.Query().Where(x => x.Partition == defaultPartition).Take(1).ToArrayAsync()).Any()) continue;", "\n ", - "\n // Only rawvariables and ifrsvariables corresponding to the target data nodes are added to the workspace", - "\n var rawvariables = await options.TargetDataSource.Query().Where(x => dataNodesToUpdate.Contains(x.DataNode) && ", - "\n (x.Partition == targetPartition || x.Partition == defaultPartition || x.Partition == previousPartition)).ToArrayAsync();", - "\n if(rawvariables.Any()) await workspaceToCompute.UpdateAsync(rawvariables);", - "\n var ifrsvariables = await options.TargetDataSource.Query().Where(x => dataNodesToUpdate.Contains(x.DataNode) && ", - "\n (x.Partition == targetPartition || x.Partition == defaultPartition || x.Partition == previousPartition)).ToArrayAsync();", - "\n if(ifrsvariables.Any()) await workspaceToCompute.UpdateAsync(ifrsvariables);", - "\n", + "\n // Remove data nodes which are unaffected by the updated yield curves\",", + "\n // TODO : Reintroduce this functionality. Note all UpdateAsync/DeleteAsync performed to the workspaceToCompute are then trasferred to the DataSource.\",", + "\n // This is way this functionality should be written in a different way. \",", + "\n // await workspaceToCompute.DeleteAsync( await workspaceToCompute.Query()\",", + "\n // .Where(x => !(dataNodesToUpdate.Contains(x.DataNode) && (x.Partition == targetPartition || x.Partition == defaultPartition))).ToArrayAsync() );\",", "\n", - "\n importLog = importLog.Merge(await ComputeAsync(args, workspace, workspaceToCompute, false));", - "\n if(importLog.Errors.Any()) return Activity.Finish().Merge(importLog);", + "\n log = log.Merge(await ComputeAsync(args, workspace, workspaceToCompute, false));", + "\n if(log.Errors.Any()) return Activity.Finish().Merge(log);", "\n }", "\n", "\n await workspaceToCompute.UpdateAsync(toCommitYieldCurves);", "\n await workspaceToCompute.CommitToTargetAsync(options.TargetDataSource);", - "\n return Activity.Finish().Merge(importLog);", + "\n return Activity.Finish().Merge(log);", "\n});" ], "metadata": {}, @@ -1252,8 +1248,8 @@ "\n await options.TargetDataSource.Partition.SetAsync(null);", "\n if(ApplicationMessage.HasErrors()) return Activity.Finish().Merge(log);", "\n ", - "\n // Avoid starting the computation if no best estimate cash flow and actuals has ever been imported ", - "\n if(!(await options.TargetDataSource.Query().Where(x => x.Partition == defaultPartition).Take(1).ToArrayAsync()).Any() &&", + "\n // Avoid starting the computation if no best estimate cash flow or actuals has ever been imported ", + "\n if(!(await options.TargetDataSource.Query().Where(x => x.Partition == defaultPartition).Take(1).ToArrayAsync()).Any() ||", "\n !(await options.TargetDataSource.Query().Where(x => x.Partition == defaultPartition).Take(1).ToArrayAsync()).Any()) continue;", "\n", "\n // Only nominals corresponding to the target data nodes are added to the workspace", From af955a8c7ecc5ffd19d68e745557b5377388c42c Mon Sep 17 00:00:00 2001 From: Danilo Calderini Date: Tue, 25 Apr 2023 16:13:10 +0200 Subject: [PATCH 14/25] more reverting --- ifrs17/Import/Importers.ipynb | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/ifrs17/Import/Importers.ipynb b/ifrs17/Import/Importers.ipynb index 73557a44..656ce2fa 100644 --- a/ifrs17/Import/Importers.ipynb +++ b/ifrs17/Import/Importers.ipynb @@ -838,7 +838,7 @@ "\n primaryArgs.ValidateArgsForPeriodAsync(options.TargetDataSource);", "\n if(ApplicationMessage.HasErrors()) return Activity.Finish();", "\n var workspace = Workspace.CreateNew();", - "\n workspace.Initialize(x => x.FromSource(options.TargetDataSource))", + "\n workspace.Initialize(x => x.FromSource(options.TargetDataSource)", "\n .DisableInitialization()", "\n .DisableInitialization());", "\n", @@ -872,12 +872,15 @@ "\n var dataNodesToUpdate = await workspace.Query().Where(x => updatedCurrencies.Contains(x.ContractualCurrency)).Select(x => x.SystemName).ToArrayAsync();", "\n var workspaceToCompute = Workspace.CreateNew();", "\n workspaceToCompute.Initialize(x => x.FromSource(options.TargetDataSource));", - "\n ApplicationMessage.Log(Warning.Generic, String.Join(\", \", dataNodesToUpdate));", "\n foreach (var args in allArgs) {", "\n await CommitPartitionAsync(args, workspace, workspaceToCompute);", "\n var targetPartition = (Guid)(await options.TargetDataSource.Partition.GetKeyForInstanceAsync(args));", "\n var defaultPartition = (Guid)(await options.TargetDataSource.Partition.GetKeyForInstanceAsync(args with {Scenario = null}));", "\n if(ApplicationMessage.HasErrors()) return Activity.Finish().Merge(log);", + "\n if(args.Scenario == null) {", + "\n await options.TargetDataSource.Partition.SetAsync(null);", + "\n if(!(await options.TargetDataSource.Query().Where(x => x.Partition == targetPartition).Take(1).ToArrayAsync()).Any()) continue;", + "\n }", "\n ", "\n // Avoid starting the computation if no best estimate cash flow has ever been imported ", "\n if(!(await options.TargetDataSource.Query().Where(x => x.Partition == defaultPartition).Take(1).ToArrayAsync()).Any()) continue;", From fb5d85b1b0371bf02762489500ad031cdd36e307 Mon Sep 17 00:00:00 2001 From: Danilo Calderini Date: Tue, 25 Apr 2023 16:17:31 +0200 Subject: [PATCH 15/25] code clean up --- ifrs17/Import/Importers.ipynb | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/ifrs17/Import/Importers.ipynb b/ifrs17/Import/Importers.ipynb index 656ce2fa..34732fe6 100644 --- a/ifrs17/Import/Importers.ipynb +++ b/ifrs17/Import/Importers.ipynb @@ -877,14 +877,13 @@ "\n var targetPartition = (Guid)(await options.TargetDataSource.Partition.GetKeyForInstanceAsync(args));", "\n var defaultPartition = (Guid)(await options.TargetDataSource.Partition.GetKeyForInstanceAsync(args with {Scenario = null}));", "\n if(ApplicationMessage.HasErrors()) return Activity.Finish().Merge(log);", + "\n ", + "\n // Avoid starting the computation if no best estimate cash flow has ever been imported ", "\n if(args.Scenario == null) {", "\n await options.TargetDataSource.Partition.SetAsync(null);", "\n if(!(await options.TargetDataSource.Query().Where(x => x.Partition == targetPartition).Take(1).ToArrayAsync()).Any()) continue;", "\n }", "\n ", - "\n // Avoid starting the computation if no best estimate cash flow has ever been imported ", - "\n if(!(await options.TargetDataSource.Query().Where(x => x.Partition == defaultPartition).Take(1).ToArrayAsync()).Any()) continue;", - "\n ", "\n // Remove data nodes which are unaffected by the updated yield curves\",", "\n // TODO : Reintroduce this functionality. Note all UpdateAsync/DeleteAsync performed to the workspaceToCompute are then trasferred to the DataSource.\",", "\n // This is way this functionality should be written in a different way. \",", From b489592fa22d0be04311b10c493bbdc7e88cc04d Mon Sep 17 00:00:00 2001 From: Danilo Calderini Date: Wed, 26 Apr 2023 16:20:35 +0200 Subject: [PATCH 16/25] more documentation --- ifrs17/DataModel/DataStructure.ipynb | 82 +++++++++++++++++++++----- ifrs17/Import/Importers.ipynb | 75 ++++++++++++----------- ifrs17/OverviewCalculationEngine.ipynb | 11 +--- ifrs17/Report/ReportStorage.ipynb | 19 +++++- ifrs17/Utils/Queries.ipynb | 2 +- 5 files changed, 124 insertions(+), 65 deletions(-) diff --git a/ifrs17/DataModel/DataStructure.ipynb b/ifrs17/DataModel/DataStructure.ipynb index 72a96f17..3703f857 100644 --- a/ifrs17/DataModel/DataStructure.ipynb +++ b/ifrs17/DataModel/DataStructure.ipynb @@ -599,8 +599,8 @@ "source": [ "", "\n## Scenario", - "\nThe Scenario record holds the various scenarios used for Sensitivity Analysis (SA). ", - "\nThe default Scenario (i.e. the default one, with no-stress situations applied) is referred to as 'Best Estimate' (BE) and its identifier is a null string, allowing the input files to not specify any value.", + "\nThe Scenario record holds the various Scenarios used for [Sensitivity analysis](../Report/ReportStorage#sensitivity). ", + "\nThe default Scenario (i.e. the default one, with no-stress situations applied) is referred to as 'Best Estimate' and its identifier is a null string, allowing the input files to not specify any value.", "\n" ], "metadata": {}, @@ -619,7 +619,14 @@ { "cell_type": "markdown", "source": [ - "Records with importers that accept Scenario creates dedicated [Partitions](#partitions) to store the related data. Usually, only a subset of information changes for a given Scenario and therefore is completed by [Relaxed Queries](../Utils/Queries#relaxedqueries)." + "The User can create any Scenario by providing a SystemName and a DisplayName to the Scenario section in Dimension.csv file and uploading it as done in the initialization of the Database in several projects.", + "\n", + "\nImporting a single file with Scenario (cfr [Scenario Implementation](../Import/Importers#scenario-implementation)) triggers parallel calculations in dedicated [Partitions](#partitions) to store the related data.", + "\n", + "\nUsually, only a subset of information changes for a given Scenario and therefore is completed using unpertubed data from the corresponding Best Estimate using [Relaxed Queries](../Utils/Queries#relaxedqueries).", + "\n", + "\nThe User is free of importing any number of files for the same Scenario, Year and Month. Nevertheless, adding distinct source of pertubation makes Sensitivity analysis more complex. We recommends to avoid this approach.", + "\n" ], "metadata": {}, "execution_count": 0, @@ -1260,7 +1267,9 @@ { "cell_type": "markdown", "source": [ - "The partition PartitionByReportingNodeAndPeriod is a further partition of the IfrsPartition sets. In particular, it defines sets for the data pertaining to a certain [Reporting Node](#reporting-node), [Scenario](#scenario), year and month. The value of the Month is the last month of the reporting period to which the data belongs to. " + "The partition PartitionByReportingNodeAndPeriod is a further partition of the IfrsPartition sets. In particular, it defines sets for the data pertaining to a certain [Reporting Node](#reporting-node), [Scenario](#scenario), year and month. The value of the Month is the last month of the reporting period to which the data belongs to. ", + "\n", + "\nThis Partition is the fundamental property used by the Calculation engine to recognise results belonging to different Scenario. It is used while importing new files to trigger calculations by the [Relaxed Queries](../Utils/Queries#relaxedqueries) and in the Report to perform [Sensitivity analysis](./Report/ReportStorage#sensitivity)." ], "metadata": {}, "execution_count": 0, @@ -1723,14 +1732,6 @@ "\n [PartitionKey(typeof(PartitionByReportingNodeAndPeriod))]", "\n public Guid Partition { get; init; }", "\n ", - "\n [Conversion(typeof(PrimitiveArrayConverter))]", - "\n public double[] Values {get; set;}", - "\n ", - "\n [NotVisible] ", - "\n [Dimension(typeof(EstimateType))]", - "\n [IdentityProperty]", - "\n public string EstimateType { get; init; }", - "\n ", "\n [NotVisible] ", "\n [Dimension(typeof(AmountType))]", "\n [IdentityProperty]", @@ -1749,7 +1750,7 @@ { "cell_type": "markdown", "source": [ - "Basically, RawVariable is defined for a certain Reporting Node, Scenario, Year, Month, Amount Type and Estimate Type." + "Basically, RawVariable is defined for a certain Reporting Node, Scenario, Year, Month, Amount Type and Calculation Type." ], "metadata": {}, "execution_count": 0, @@ -1758,7 +1759,16 @@ { "cell_type": "code", "source": [ - "public record RawVariable : BaseDataRecord {}" + "public record RawVariable : BaseDataRecord", + "\n{", + "\n [Conversion(typeof(PrimitiveArrayConverter))]", + "\n public double[] Values { get; init; }", + "\n ", + "\n [NotVisible]", + "\n [Dimension(typeof(EstimateType))]", + "\n [IdentityProperty]", + "\n public string EstimateType { get; init; }", + "\n}" ], "metadata": {}, "execution_count": 0, @@ -1788,6 +1798,25 @@ "source": [ "public record IfrsVariable : BaseDataRecord", "\n{", + "\n public double Value { get; init; }", + "\n public double Value1 { get; init; }", + "\n public double Value2 { get; init; }", + "\n public double Value3 { get; init; }", + "\n public double Value4 { get; init; }", + "\n public double Value5 { get; init; }", + "\n public double Value6 { get; init; }", + "\n public double Value7 { get; init; }", + "\n public double Value8 { get; init; }", + "\n public double Value9 { get; init; }", + "\n public double Value10 { get; init; }", + "\n public double Value11 { get; init; }", + "\n public double Value12 { get; init; }", + "\n ", + "\n [NotVisible] ", + "\n [Dimension(typeof(EstimateType))]", + "\n [IdentityProperty]", + "\n public string EstimateType { get; init; }", + "\n ", "\n [NotVisible] ", "\n [Dimension(typeof(EconomicBasis))]", "\n [IdentityProperty]", @@ -1949,10 +1978,10 @@ "\n ", "\n [NotVisible]", "\n [NoArithmetics(ArithmeticOperation.Scale)]", - "\n [Dimension(typeof(ProjectionConfiguration), nameof(Projection))]", + "\n [Dimension(typeof(int), nameof(Projection))]", "\n [IdentityProperty]", "\n //[AggregateBy]", - "\n public string Projection { get; init;}", + "\n public int Projection { get; init;}", "\n ", "\n [NotVisible]", "\n [Dimension(typeof(VariableType))]", @@ -2007,6 +2036,27 @@ "\n EconomicBasis = rv.EconomicBasis;", "\n Value = rv.Value;", "\n }", + "\n public ReportVariable(DataNodeData dn, IfrsVariable iv){", + "\n FunctionalCurrency = dn.FunctionalCurrency;", + "\n ContractualCurrency = dn.ContractualCurrency;", + "\n GroupOfContract = dn.DataNode;", + "\n Portfolio = dn.Portfolio;", + "\n LineOfBusiness = dn.LineOfBusiness;", + "\n LiabilityType = dn.LiabilityType;", + "\n InitialProfitability = dn.Profitability;", + "\n ValuationApproach = dn.ValuationApproach;", + "\n AnnualCohort = dn.AnnualCohort;", + "\n OciType = dn.OciType;", + "\n Partner = dn.Partner;", + "\n IsReinsurance = dn.IsReinsurance;", + "\n AccidentYear = iv.AccidentYear ?? default;", + "\n VariableType = iv.AocType;", + "\n Novelty = iv.Novelty;", + "\n AmountType = iv.AmountType;", + "\n EstimateType = iv.EstimateType;", + "\n EconomicBasis = iv.EconomicBasis;", + "\n Value = iv.Value;", + "\n }", "\n}" ], "metadata": {}, diff --git a/ifrs17/Import/Importers.ipynb b/ifrs17/Import/Importers.ipynb index e1fa2d16..7412913d 100644 --- a/ifrs17/Import/Importers.ipynb +++ b/ifrs17/Import/Importers.ipynb @@ -36,6 +36,30 @@ "execution_count": 0, "outputs": [] }, + { + "cell_type": "markdown", + "source": [ + "", + "\n# Scenario implementation", + "\nThe following Importers support Scenario:", + "\n - Cashflow", + "\n - Actual", + "\n - Opening", + "\n - Yield Curve", + "\n - Data Node Parameter", + "\n ", + "\nImporting Scenario data trigger calculation using [Relaxed Queries](../Utils/Queries#relaxedqueries), implying that the Scenario data depends on Best Estimate, which is supposed to be already imported. ", + "\n", + "\nRe-importing Scenario will trigger calculation for the Scenario Partition only.", + "\n", + "\nRe-importing Best Estimate will trigger calculations for the Best Estimate Partition and for all Scenario that depends on the mentioned Best Estimate, but only for the same Year and Month. Therefore, outdated Best Estimate and Scenario data may be taken into account for following Periods. Reimporting following Periods inputs will trigger the calculations needed to update.", + "\n", + "\nThe Calculation Engine inquires the Database for dependent Partitions in [GetAllArgsAsync](#allargs): the informations collected in each Args identify a specific Partition. Retriving the necessary data is done in the [Import Storage](./ImportStorage#import-storage), one for each Partitions to avoid data leakage." + ], + "metadata": {}, + "execution_count": 0, + "outputs": [] + }, { "cell_type": "markdown", "source": [ @@ -507,7 +531,8 @@ { "cell_type": "markdown", "source": [ - "## Calculate IFRS Variables", + "", + "\n## Calculate IFRS Variables", "\n", "\nThe following methods are used in the different importers to compute the [IfrsVariables](../DataModel/DataStructure#ifrs-variable).", "\n", @@ -585,14 +610,13 @@ "\n", "\n if(storage.DefaultPartition != storage.TargetPartition) {", "\n var bestEstimateIvs = await workspaceToCompute.LoadPartitionedDataAsync(storage.DefaultPartition);", - "\n ivs = ivs.Where(iv => iv.Values.Any(y => Math.Abs(y) >= Precision)).ToArray()", - "\n .Except(bestEstimateIvs, IfrsVariableComparer.Instance(ignoreValues: false))", - "\n .Concat(bestEstimateIvs.Intersect(ivs.Where(iv => iv.Values.All(y => Math.Abs(y) < Precision)).ToArray(), IfrsVariableComparer.Instance(ignoreValues: true))", - "\n .Select(x => x with {Values = Enumerable.Repeat(0d, x.Values.Length).ToArray(), Partition = storage.TargetPartition}).ToArray());", + "\n ivs = ivs.Where(iv => Math.Abs(iv.Value) >= Precision).ToArray()", + "\n .Except(bestEstimateIvs, IfrsVariableComparer.Instance(ignoreValue: false))", + "\n .Concat(ivs.Where(x => Math.Abs(x.Value) < Precision).Intersect(bestEstimateIvs, EqualityComparer.Instance).Select(x => x with {Value = 0.0}).ToArray());", "\n }", "\n", "\n workspace.Reset(x => x.ResetType());", - "\n await workspace.UpdateAsync(ivs.Where(x => storage.DefaultPartition != storage.TargetPartition || x.Values.Any(v => Math.Abs(v) >= Precision)));", + "\n await workspace.UpdateAsync(ivs.Where(x => storage.DefaultPartition != storage.TargetPartition || Math.Abs(x.Value) >= Precision));", "\n await workspace.CommitToAsync(workspaceToCompute, storage.TargetPartition, snapshot : true, ", "\n filter : x => storage.EstimateTypesByImportFormat[args.ImportFormat].Contains(x.EstimateType) ", "\n && storage.DataNodesByImportScope[ImportScope.Primary].Contains(x.DataNode));", @@ -849,6 +873,7 @@ "\n (dataset, datarow) => {", "\n var values = datarow.Table.Columns.Where(c => c.ColumnName.StartsWith(nameof(YieldCurve.Values))).OrderBy(c => c.ColumnName.Length).ThenBy(c => c.ColumnName)", "\n .Select(x => datarow.Field(x.ColumnName).CheckStringForExponentialAndConvertToDouble()).ToArray().Prune();", + "\n if (values.Length == 0 ) return null;", "\n return new YieldCurve {", "\n Currency = datarow.Field(nameof(YieldCurve.Currency)),", "\n Year = primaryArgs.Year,", @@ -1251,7 +1276,8 @@ "\n if(ApplicationMessage.HasErrors()) return Activity.Finish().Merge(log);", "\n ", "\n // Avoid starting the computation if no best estimate cash flow or actuals has ever been imported ", - "\n if(!(await options.TargetDataSource.Query().Where(x => x.Partition == defaultPartition).Take(1).ToArrayAsync()).Any()) continue;", + "\n if(!(await options.TargetDataSource.Query().Where(x => x.Partition == defaultPartition).Take(1).ToArrayAsync()).Any() ||", + "\n !(await options.TargetDataSource.Query().Where(x => x.Partition == defaultPartition).Take(1).ToArrayAsync()).Any()) continue;", "\n", "\n // Only nominals corresponding to the target data nodes are added to the workspace", "\n var nominals = await options.TargetDataSource.Query().Where(x => targetDataNodes.Contains(x.DataNode) && ", @@ -1449,9 +1475,6 @@ "\n return null;", "\n }", "\n", - "\n var currentPeriodValue = GetSign(ImportFormats.Actual, ", - "\n (aocType, valueType.AmountType, valueType.EstimateType, dataNodeData.IsReinsurance), ", - "\n parsingStorage.HierarchyCache) * datarow.Field(\"Value\").CheckStringForExponentialAndConvertToDouble();", "\n var item = new IfrsVariable {", "\n DataNode = dataNode,", "\n AocType = aocType,", @@ -1460,7 +1483,7 @@ "\n AmountType = valueType.AmountType,", "\n EstimateType = valueType.EstimateType,", "\n Partition = parsingStorage.TargetPartitionByReportingNodeAndPeriod.Id,", - "\n Values = SetProjectionValue(currentPeriodValue)", + "\n Value = GetSign(ImportFormats.Actual, (aocType, valueType.AmountType, valueType.EstimateType, dataNodeData.IsReinsurance), parsingStorage.HierarchyCache) * datarow.Field(nameof(IfrsVariable.Value)).CheckStringForExponentialAndConvertToDouble()", "\n };", "\n return item;", "\n }, ImportFormats.Actual", @@ -1492,17 +1515,10 @@ "\n", "\n var workspaceToCompute = Workspace.CreateNew();", "\n workspaceToCompute.Initialize(x => x.FromSource(options.TargetDataSource));", - "\n", - "\n if (Debug.Enable) ", - "\n {", - "\n var partition = (Guid)(await workspace.Partition.GetKeyForInstanceAsync(primaryArgs));", - "\n await workspace.CommitToAsync(workspaceToCompute, partition, snapshot : true);", - "\n } ", - "\n else ", - "\n foreach (var args in allArgs) {", - "\n log = log.Merge(await ComputeAsync(args, workspace, workspaceToCompute, false)); ", - "\n if(log.Errors.Any()) return Activity.Finish().Merge(log);", - "\n }", + "\n foreach (var args in allArgs) {", + "\n log = log.Merge(await ComputeAsync(args, workspace, workspaceToCompute, false)); ", + "\n if(log.Errors.Any()) return Activity.Finish().Merge(log);", + "\n }", "\n await workspaceToCompute.CommitToTargetAsync(options.TargetDataSource);", "\n return Activity.Finish().Merge(log);", "\n});" @@ -1554,10 +1570,6 @@ "\n ", "\n parsingStorage.ValidateEstimateTypeAndAmountType(estimateType, amountType);", "\n ", - "\n var currentPeriodValue = GetSign(importFormat, ", - "\n (aocStep.AocType, amountType, estimateType, parsingStorage.IsDataNodeReinsurance(dataNode)), ", - "\n parsingStorage.HierarchyCache) * datarow.Field(\"Value\")", - "\n .CheckStringForExponentialAndConvertToDouble();", "\n var iv = new IfrsVariable {", "\n DataNode = dataNode,", "\n AocType = aocStep.AocType,", @@ -1567,7 +1579,7 @@ "\n EstimateType = estimateType,", "\n EconomicBasis = economicBasis,", "\n Partition = parsingStorage.TargetPartitionByReportingNodeAndPeriod.Id,", - "\n Values = SetProjectionValue(currentPeriodValue)", + "\n Value = GetSign(importFormat, (aocStep.AocType, amountType, estimateType, parsingStorage.IsDataNodeReinsurance(dataNode)), parsingStorage.HierarchyCache) * datarow.Field(nameof(IfrsVariable.Value)).CheckStringForExponentialAndConvertToDouble()", "\n };", "\n return iv;", "\n }, importFormat // This should indicate the table name, not the input format", @@ -1662,15 +1674,6 @@ "metadata": {}, "execution_count": 0, "outputs": [] - }, - { - "cell_type": "code", - "source": [ - "" - ], - "metadata": {}, - "execution_count": 0, - "outputs": [] } ] } \ No newline at end of file diff --git a/ifrs17/OverviewCalculationEngine.ipynb b/ifrs17/OverviewCalculationEngine.ipynb index f102496e..4d6f7489 100644 --- a/ifrs17/OverviewCalculationEngine.ipynb +++ b/ifrs17/OverviewCalculationEngine.ipynb @@ -211,7 +211,7 @@ "\n- Timing consideration for Effective Actuals (Payables and Receivables mapped into Paid in Advance or Overdue Actuals)", "\n- Credit Default Risk for Reinsurance", "\n- Financial Performance (FP) and Other Comprehensive Income (OCI)", - "\n- [Scenario](./DataModel/DataStructure#scenario) and Sensitivity analysis", + "\n- [Scenario](./DataModel/DataStructure#scenario) and [Sensitivity analysis](./Report/ReportStorage#sensitivity)", "\n- Run-off Projections (coming soon)", "\n", "\nFor more information on the latest developments, please refer to our [GitHub](https://github.com/Systemorph/IFRS17CalculationEngine) project page. From there, you can get to know about future releases, place requests, track the current work and report issues.", @@ -221,15 +221,6 @@ "metadata": {}, "execution_count": 0, "outputs": [] - }, - { - "cell_type": "markdown", - "source": [ - "" - ], - "metadata": {}, - "execution_count": 0, - "outputs": [] } ] } \ No newline at end of file diff --git a/ifrs17/Report/ReportStorage.ipynb b/ifrs17/Report/ReportStorage.ipynb index e2082099..62c4b43d 100644 --- a/ifrs17/Report/ReportStorage.ipynb +++ b/ifrs17/Report/ReportStorage.ipynb @@ -41,6 +41,21 @@ "execution_count": 0, "outputs": [] }, + { + "cell_type": "markdown", + "source": [ + "", + "\n# Sensitivity Analysis", + "\n", + "\nSensitivity Analysis is the comparison of Best Estimate results with a Scenario ones. In IFRS17 Calculation Engine the comparison is done putting together the corresponding figures in the various Reports provided. The keyword to use is Report.Scenario, which can be set to four main values:", + "\n - null (default): the Report use only Best Estimate results", + "\n - all: the Report shows values of Best Estimate and all Scenarios side by side", + "\n - delta: the Report shows values of Best Estimate and the difference between it and each Scenario." + ], + "metadata": {}, + "execution_count": 0, + "outputs": [] + }, { "cell_type": "markdown", "source": [ @@ -75,8 +90,8 @@ "\n public string FunctionalCurrency { get; init; }", "\n", "\n [NotAggregated]", - "\n [Dimension(typeof(ProjectionConfiguration), nameof(Projection))]", - "\n public string Projection { get; init; }", + "\n [Dimension(typeof(int), nameof(Projection))]", + "\n public int Projection { get; init; }", "\n", "\n [Dimension(typeof(LiabilityType))]", "\n public string LiabilityType { get; init; }", diff --git a/ifrs17/Utils/Queries.ipynb b/ifrs17/Utils/Queries.ipynb index 4dc9dfe7..727d0a0c 100644 --- a/ifrs17/Utils/Queries.ipynb +++ b/ifrs17/Utils/Queries.ipynb @@ -31,7 +31,7 @@ "", "\n# Relaxed queries", "\n", - "\nThe queries here collected retrieve data from a sorce of data, defined querySource . Priority is given to data from the same [Partition](../DataModel/DataStructure#partitions), which may be both BE or Scenario. If Scenario is missing, it falls back to the corresponding BE." + "\nThe queries here collected retrieve data from a sorce of data, defined querySource . Priority is given to data from the same [Partition](../DataModel/DataStructure#partitions), which may be BE or Scenario. If Scenario data are missing, the query returns to the Best Estimate Data for the same Period or the previous." ], "metadata": {}, "execution_count": 0, From 29d6e55ad70812c2b696d444311401019f660a13 Mon Sep 17 00:00:00 2001 From: Danilo Calderini Date: Wed, 26 Apr 2023 16:38:45 +0200 Subject: [PATCH 17/25] fixing merge conflict - DataStructure --- ifrs17/DataModel/DataStructure.ipynb | 67 ++++++---------------------- 1 file changed, 13 insertions(+), 54 deletions(-) diff --git a/ifrs17/DataModel/DataStructure.ipynb b/ifrs17/DataModel/DataStructure.ipynb index 3703f857..a4ac50b8 100644 --- a/ifrs17/DataModel/DataStructure.ipynb +++ b/ifrs17/DataModel/DataStructure.ipynb @@ -1727,6 +1727,14 @@ "\n [Key]", "\n [NotVisible] ", "\n public Guid Id { get; init; }", + "\n", + "\n [Conversion(typeof(PrimitiveArrayConverter))]", + "\n public double[] Values {get; set;}", + "\n ", + "\n [NotVisible] ", + "\n [Dimension(typeof(EstimateType))]", + "\n [IdentityProperty]", + "\n public string EstimateType { get; init; }", "\n ", "\n [NotVisible]", "\n [PartitionKey(typeof(PartitionByReportingNodeAndPeriod))]", @@ -1750,7 +1758,7 @@ { "cell_type": "markdown", "source": [ - "Basically, RawVariable is defined for a certain Reporting Node, Scenario, Year, Month, Amount Type and Calculation Type." + "Basically, RawVariable is defined for a certain Reporting Node, Scenario, Year, Month, Amount Type and Estimate Type." ], "metadata": {}, "execution_count": 0, @@ -1759,16 +1767,7 @@ { "cell_type": "code", "source": [ - "public record RawVariable : BaseDataRecord", - "\n{", - "\n [Conversion(typeof(PrimitiveArrayConverter))]", - "\n public double[] Values { get; init; }", - "\n ", - "\n [NotVisible]", - "\n [Dimension(typeof(EstimateType))]", - "\n [IdentityProperty]", - "\n public string EstimateType { get; init; }", - "\n}" + "public record RawVariable : BaseDataRecord {}" ], "metadata": {}, "execution_count": 0, @@ -1797,26 +1796,7 @@ "cell_type": "code", "source": [ "public record IfrsVariable : BaseDataRecord", - "\n{", - "\n public double Value { get; init; }", - "\n public double Value1 { get; init; }", - "\n public double Value2 { get; init; }", - "\n public double Value3 { get; init; }", - "\n public double Value4 { get; init; }", - "\n public double Value5 { get; init; }", - "\n public double Value6 { get; init; }", - "\n public double Value7 { get; init; }", - "\n public double Value8 { get; init; }", - "\n public double Value9 { get; init; }", - "\n public double Value10 { get; init; }", - "\n public double Value11 { get; init; }", - "\n public double Value12 { get; init; }", - "\n ", - "\n [NotVisible] ", - "\n [Dimension(typeof(EstimateType))]", - "\n [IdentityProperty]", - "\n public string EstimateType { get; init; }", - "\n ", + "\n{ ", "\n [NotVisible] ", "\n [Dimension(typeof(EconomicBasis))]", "\n [IdentityProperty]", @@ -1978,10 +1958,10 @@ "\n ", "\n [NotVisible]", "\n [NoArithmetics(ArithmeticOperation.Scale)]", - "\n [Dimension(typeof(int), nameof(Projection))]", + "\n [Dimension(typeof(int), nameof(ProjectionConfiguration))]", "\n [IdentityProperty]", "\n //[AggregateBy]", - "\n public int Projection { get; init;}", + "\n public string Projection { get; init;}", "\n ", "\n [NotVisible]", "\n [Dimension(typeof(VariableType))]", @@ -2036,27 +2016,6 @@ "\n EconomicBasis = rv.EconomicBasis;", "\n Value = rv.Value;", "\n }", - "\n public ReportVariable(DataNodeData dn, IfrsVariable iv){", - "\n FunctionalCurrency = dn.FunctionalCurrency;", - "\n ContractualCurrency = dn.ContractualCurrency;", - "\n GroupOfContract = dn.DataNode;", - "\n Portfolio = dn.Portfolio;", - "\n LineOfBusiness = dn.LineOfBusiness;", - "\n LiabilityType = dn.LiabilityType;", - "\n InitialProfitability = dn.Profitability;", - "\n ValuationApproach = dn.ValuationApproach;", - "\n AnnualCohort = dn.AnnualCohort;", - "\n OciType = dn.OciType;", - "\n Partner = dn.Partner;", - "\n IsReinsurance = dn.IsReinsurance;", - "\n AccidentYear = iv.AccidentYear ?? default;", - "\n VariableType = iv.AocType;", - "\n Novelty = iv.Novelty;", - "\n AmountType = iv.AmountType;", - "\n EstimateType = iv.EstimateType;", - "\n EconomicBasis = iv.EconomicBasis;", - "\n Value = iv.Value;", - "\n }", "\n}" ], "metadata": {}, From 977f4c09509631466db7e842eb99b7406984a06b Mon Sep 17 00:00:00 2001 From: Danilo Calderini Date: Wed, 26 Apr 2023 16:43:05 +0200 Subject: [PATCH 18/25] fixing merge conflict - DataStructure 2 --- ifrs17/DataModel/DataStructure.ipynb | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/ifrs17/DataModel/DataStructure.ipynb b/ifrs17/DataModel/DataStructure.ipynb index a4ac50b8..eb0b210c 100644 --- a/ifrs17/DataModel/DataStructure.ipynb +++ b/ifrs17/DataModel/DataStructure.ipynb @@ -1727,7 +1727,11 @@ "\n [Key]", "\n [NotVisible] ", "\n public Guid Id { get; init; }", - "\n", + "\n ", + "\n [NotVisible]", + "\n [PartitionKey(typeof(PartitionByReportingNodeAndPeriod))]", + "\n public Guid Partition { get; init; }", + "\n ", "\n [Conversion(typeof(PrimitiveArrayConverter))]", "\n public double[] Values {get; set;}", "\n ", @@ -1736,10 +1740,6 @@ "\n [IdentityProperty]", "\n public string EstimateType { get; init; }", "\n ", - "\n [NotVisible]", - "\n [PartitionKey(typeof(PartitionByReportingNodeAndPeriod))]", - "\n public Guid Partition { get; init; }", - "\n ", "\n [NotVisible] ", "\n [Dimension(typeof(AmountType))]", "\n [IdentityProperty]", @@ -1796,7 +1796,7 @@ "cell_type": "code", "source": [ "public record IfrsVariable : BaseDataRecord", - "\n{ ", + "\n{", "\n [NotVisible] ", "\n [Dimension(typeof(EconomicBasis))]", "\n [IdentityProperty]", @@ -1958,7 +1958,7 @@ "\n ", "\n [NotVisible]", "\n [NoArithmetics(ArithmeticOperation.Scale)]", - "\n [Dimension(typeof(int), nameof(ProjectionConfiguration))]", + "\n [Dimension(typeof(ProjectionConfiguration), nameof(Projection))]", "\n [IdentityProperty]", "\n //[AggregateBy]", "\n public string Projection { get; init;}", From 6eda6ece45fae96cd7a7622fe45a61286d3a519c Mon Sep 17 00:00:00 2001 From: Danilo Calderini Date: Wed, 26 Apr 2023 17:12:36 +0200 Subject: [PATCH 19/25] fixing merge conflict - Importer --- ifrs17/Import/Importers.ipynb | 43 ++++++++++++++++++++++------------- 1 file changed, 27 insertions(+), 16 deletions(-) diff --git a/ifrs17/Import/Importers.ipynb b/ifrs17/Import/Importers.ipynb index 7412913d..bc1ea17f 100644 --- a/ifrs17/Import/Importers.ipynb +++ b/ifrs17/Import/Importers.ipynb @@ -610,13 +610,14 @@ "\n", "\n if(storage.DefaultPartition != storage.TargetPartition) {", "\n var bestEstimateIvs = await workspaceToCompute.LoadPartitionedDataAsync(storage.DefaultPartition);", - "\n ivs = ivs.Where(iv => Math.Abs(iv.Value) >= Precision).ToArray()", - "\n .Except(bestEstimateIvs, IfrsVariableComparer.Instance(ignoreValue: false))", - "\n .Concat(ivs.Where(x => Math.Abs(x.Value) < Precision).Intersect(bestEstimateIvs, EqualityComparer.Instance).Select(x => x with {Value = 0.0}).ToArray());", + "\n ivs = ivs.Where(iv => iv.Values.Any(y => Math.Abs(y) >= Precision)).ToArray()", + "\n .Except(bestEstimateIvs, IfrsVariableComparer.Instance(ignoreValues: false))", + "\n .Concat(bestEstimateIvs.Intersect(ivs.Where(iv => iv.Values.All(y => Math.Abs(y) < Precision)).ToArray(), IfrsVariableComparer.Instance(ignoreValues: true))", + "\n .Select(x => x with {Values = Enumerable.Repeat(0d, x.Values.Length).ToArray(), Partition = storage.TargetPartition}).ToArray());", "\n }", "\n", "\n workspace.Reset(x => x.ResetType());", - "\n await workspace.UpdateAsync(ivs.Where(x => storage.DefaultPartition != storage.TargetPartition || Math.Abs(x.Value) >= Precision));", + "\n await workspace.UpdateAsync(ivs.Where(x => storage.DefaultPartition != storage.TargetPartition || x.Values.Any(v => Math.Abs(v) >= Precision)));", "\n await workspace.CommitToAsync(workspaceToCompute, storage.TargetPartition, snapshot : true, ", "\n filter : x => storage.EstimateTypesByImportFormat[args.ImportFormat].Contains(x.EstimateType) ", "\n && storage.DataNodesByImportScope[ImportScope.Primary].Contains(x.DataNode));", @@ -873,7 +874,7 @@ "\n (dataset, datarow) => {", "\n var values = datarow.Table.Columns.Where(c => c.ColumnName.StartsWith(nameof(YieldCurve.Values))).OrderBy(c => c.ColumnName.Length).ThenBy(c => c.ColumnName)", "\n .Select(x => datarow.Field(x.ColumnName).CheckStringForExponentialAndConvertToDouble()).ToArray().Prune();", - "\n if (values.Length == 0 ) return null;", + "\n if (!values.Any()) return null;", "\n return new YieldCurve {", "\n Currency = datarow.Field(nameof(YieldCurve.Currency)),", "\n Year = primaryArgs.Year,", @@ -904,10 +905,7 @@ "\n if(ApplicationMessage.HasErrors()) return Activity.Finish().Merge(log);", "\n ", "\n // Avoid starting the computation if no best estimate cash flow has ever been imported ", - "\n if(args.Scenario == null) {", - "\n await options.TargetDataSource.Partition.SetAsync(null);", - "\n if(!(await options.TargetDataSource.Query().Where(x => x.Partition == targetPartition).Take(1).ToArrayAsync()).Any()) continue;", - "\n }", + "\n if(!(await options.TargetDataSource.Query().Where(x => x.Partition == defaultPartition).Take(1).ToArrayAsync()).Any()) continue;", "\n ", "\n // Remove data nodes which are unaffected by the updated yield curves\",", "\n // TODO : Reintroduce this functionality. Note all UpdateAsync/DeleteAsync performed to the workspaceToCompute are then trasferred to the DataSource.\",", @@ -1475,6 +1473,9 @@ "\n return null;", "\n }", "\n", + "\n var currentPeriodValue = GetSign(ImportFormats.Actual, ", + "\n (aocType, valueType.AmountType, valueType.EstimateType, dataNodeData.IsReinsurance), ", + "\n parsingStorage.HierarchyCache) * datarow.Field(\"Value\").CheckStringForExponentialAndConvertToDouble();", "\n var item = new IfrsVariable {", "\n DataNode = dataNode,", "\n AocType = aocType,", @@ -1483,7 +1484,7 @@ "\n AmountType = valueType.AmountType,", "\n EstimateType = valueType.EstimateType,", "\n Partition = parsingStorage.TargetPartitionByReportingNodeAndPeriod.Id,", - "\n Value = GetSign(ImportFormats.Actual, (aocType, valueType.AmountType, valueType.EstimateType, dataNodeData.IsReinsurance), parsingStorage.HierarchyCache) * datarow.Field(nameof(IfrsVariable.Value)).CheckStringForExponentialAndConvertToDouble()", + "\n Values = SetProjectionValue(currentPeriodValue)", "\n };", "\n return item;", "\n }, ImportFormats.Actual", @@ -1515,10 +1516,17 @@ "\n", "\n var workspaceToCompute = Workspace.CreateNew();", "\n workspaceToCompute.Initialize(x => x.FromSource(options.TargetDataSource));", - "\n foreach (var args in allArgs) {", - "\n log = log.Merge(await ComputeAsync(args, workspace, workspaceToCompute, false)); ", - "\n if(log.Errors.Any()) return Activity.Finish().Merge(log);", - "\n }", + "\n", + "\n if (Debug.Enable) ", + "\n {", + "\n var partition = (Guid)(await workspace.Partition.GetKeyForInstanceAsync(primaryArgs));", + "\n await workspace.CommitToAsync(workspaceToCompute, partition, snapshot : true);", + "\n } ", + "\n else ", + "\n foreach (var args in allArgs) {", + "\n log = log.Merge(await ComputeAsync(args, workspace, workspaceToCompute, false)); ", + "\n if(log.Errors.Any()) return Activity.Finish().Merge(log);", + "\n }", "\n await workspaceToCompute.CommitToTargetAsync(options.TargetDataSource);", "\n return Activity.Finish().Merge(log);", "\n});" @@ -1569,7 +1577,10 @@ "\n : null;", "\n ", "\n parsingStorage.ValidateEstimateTypeAndAmountType(estimateType, amountType);", - "\n ", + "\n var currentPeriodValue = GetSign(importFormat, ", + "\n (aocStep.AocType, amountType, estimateType, parsingStorage.IsDataNodeReinsurance(dataNode)), ", + "\n parsingStorage.HierarchyCache) * datarow.Field(\"Value\")", + "\n .CheckStringForExponentialAndConvertToDouble();", "\n var iv = new IfrsVariable {", "\n DataNode = dataNode,", "\n AocType = aocStep.AocType,", @@ -1579,7 +1590,7 @@ "\n EstimateType = estimateType,", "\n EconomicBasis = economicBasis,", "\n Partition = parsingStorage.TargetPartitionByReportingNodeAndPeriod.Id,", - "\n Value = GetSign(importFormat, (aocStep.AocType, amountType, estimateType, parsingStorage.IsDataNodeReinsurance(dataNode)), parsingStorage.HierarchyCache) * datarow.Field(nameof(IfrsVariable.Value)).CheckStringForExponentialAndConvertToDouble()", + "\n Values = SetProjectionValue(currentPeriodValue)", "\n };", "\n return iv;", "\n }, importFormat // This should indicate the table name, not the input format", From eaf4e978da0a63209fdb76f7cd850de1dc01b473 Mon Sep 17 00:00:00 2001 From: Danilo Calderini Date: Wed, 26 Apr 2023 17:19:07 +0200 Subject: [PATCH 20/25] fixing merge conflict - Importer 2 --- ifrs17/Import/Importers.ipynb | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/ifrs17/Import/Importers.ipynb b/ifrs17/Import/Importers.ipynb index bc1ea17f..7c8113d0 100644 --- a/ifrs17/Import/Importers.ipynb +++ b/ifrs17/Import/Importers.ipynb @@ -905,13 +905,15 @@ "\n if(ApplicationMessage.HasErrors()) return Activity.Finish().Merge(log);", "\n ", "\n // Avoid starting the computation if no best estimate cash flow has ever been imported ", - "\n if(!(await options.TargetDataSource.Query().Where(x => x.Partition == defaultPartition).Take(1).ToArrayAsync()).Any()) continue;", - "\n ", - "\n // Remove data nodes which are unaffected by the updated yield curves\",", - "\n // TODO : Reintroduce this functionality. Note all UpdateAsync/DeleteAsync performed to the workspaceToCompute are then trasferred to the DataSource.\",", - "\n // This is way this functionality should be written in a different way. \",", - "\n // await workspaceToCompute.DeleteAsync( await workspaceToCompute.Query()\",", - "\n // .Where(x => !(dataNodesToUpdate.Contains(x.DataNode) && (x.Partition == targetPartition || x.Partition == defaultPartition))).ToArrayAsync() );\",", + "\n if(args.Scenario == null) {", + "\n await options.TargetDataSource.Partition.SetAsync(null);", + "\n if(!(await options.TargetDataSource.Query().Where(x => x.Partition == targetPartition).Take(1).ToArrayAsync()).Any()) continue;", + "\n }", + "\n // Remove data nodes which are unaffected by the updated yield curves", + "\n // TODO : Reintroduce this functionality. Note all UpdateAsync/DeleteAsync performed to the workspaceToCompute are then trasferred to the DataSource.", + "\n // This is way this functionality should be written in a different way. ", + "\n // await workspaceToCompute.DeleteAsync( await workspaceToCompute.Query()", + "\n // .Where(x => !(dataNodesToUpdate.Contains(x.DataNode) && (x.Partition == targetPartition || x.Partition == defaultPartition))).ToArrayAsync() );", "\n", "\n log = log.Merge(await ComputeAsync(args, workspace, workspaceToCompute, false));", "\n if(log.Errors.Any()) return Activity.Finish().Merge(log);", @@ -1274,8 +1276,7 @@ "\n if(ApplicationMessage.HasErrors()) return Activity.Finish().Merge(log);", "\n ", "\n // Avoid starting the computation if no best estimate cash flow or actuals has ever been imported ", - "\n if(!(await options.TargetDataSource.Query().Where(x => x.Partition == defaultPartition).Take(1).ToArrayAsync()).Any() ||", - "\n !(await options.TargetDataSource.Query().Where(x => x.Partition == defaultPartition).Take(1).ToArrayAsync()).Any()) continue;", + "\n if(!(await options.TargetDataSource.Query().Where(x => x.Partition == defaultPartition).Take(1).ToArrayAsync()).Any()) continue;", "\n", "\n // Only nominals corresponding to the target data nodes are added to the workspace", "\n var nominals = await options.TargetDataSource.Query().Where(x => targetDataNodes.Contains(x.DataNode) && ", @@ -1577,6 +1578,7 @@ "\n : null;", "\n ", "\n parsingStorage.ValidateEstimateTypeAndAmountType(estimateType, amountType);", + "\n ", "\n var currentPeriodValue = GetSign(importFormat, ", "\n (aocStep.AocType, amountType, estimateType, parsingStorage.IsDataNodeReinsurance(dataNode)), ", "\n parsingStorage.HierarchyCache) * datarow.Field(\"Value\")", From 88d891e910734b98a43d0ed6aff382f62a394b74 Mon Sep 17 00:00:00 2001 From: Davide Colleoni Date: Thu, 27 Apr 2023 13:26:11 +0200 Subject: [PATCH 21/25] clean up + dataStructure and part of importers --- ifrs17/DataModel/DataStructure.ipynb | 23 +- ifrs17/Import/Importers.ipynb | 21 +- ifrs17/Import/Importers.ipynb.orig | 1653 -------------------------- 3 files changed, 34 insertions(+), 1663 deletions(-) delete mode 100644 ifrs17/Import/Importers.ipynb.orig diff --git a/ifrs17/DataModel/DataStructure.ipynb b/ifrs17/DataModel/DataStructure.ipynb index eb0b210c..7bdfc99c 100644 --- a/ifrs17/DataModel/DataStructure.ipynb +++ b/ifrs17/DataModel/DataStructure.ipynb @@ -600,8 +600,8 @@ "", "\n## Scenario", "\nThe Scenario record holds the various Scenarios used for [Sensitivity analysis](../Report/ReportStorage#sensitivity). ", - "\nThe default Scenario (i.e. the default one, with no-stress situations applied) is referred to as 'Best Estimate' and its identifier is a null string, allowing the input files to not specify any value.", - "\n" + "\n", + "\nAny Scenario-case can be defined with this record by providing a SystemName and a DisaplyName. During data collection phase the Scenario column of the [main](../Import/Importers#parse-the-main-tab) section of the input can be populated with the SystemName of the desired Scenario. The default Scenario (i.e. the default one, with no-stress situations applied) is referred to as 'Best Estimate' and its identifier is a null string, allowing the input files to not specify any value under the column Scenario." ], "metadata": {}, "execution_count": 0, @@ -619,14 +619,23 @@ { "cell_type": "markdown", "source": [ - "The User can create any Scenario by providing a SystemName and a DisplayName to the Scenario section in Dimension.csv file and uploading it as done in the initialization of the Database in several projects.", + "### Scenario and inputs", + "\n", + "\nUnder a particular Scenario, several data types can be imported (e.g. nominal cash flows, parameters, yield-curvers, etc.). However, we suggest to arrange a Scenario for every desired type of stress in order to facilitate the analysis of the results.", + "\n", + "\n### Dependecy with Best Estimate scenario: same period", + "\n", + "\nWhen a file is imported for a certain Scenario, the calculation engine integrates the set of inputs taking the remaining from the best estimate scenario. In this way the user is not required to input all data for each Scenario calculation but only the file with the stressed input. This is achieved through our system of [Relaxed Queries](../Utils/Queries#relaxedqueries). The assumption here is that the Best Estimate scenario is the first to be imported, and the stressed scenarios follow. ", "\n", - "\nImporting a single file with Scenario (cfr [Scenario Implementation](../Import/Importers#scenario-implementation)) triggers parallel calculations in dedicated [Partitions](#partitions) to store the related data.", + "\nThe dependency that each scenario has to the Best Estimate scenario is considered in case re-import. In this case, one import automatically triggers calculation in several partitions allowing all dependant cases to be updated. ", + "\nThe only exception to this concerns dependency in time. When figures for periods $P$ and $P+1$ are imported for all scenarios and a new input for Best Estimate period $P$ is provided, only the scenarios for period $P$ are automatically updated. A manual update of period $P+1$ is then required to update the figures of $P+1$. We consider this case of restating previous periods particularly sensitive and defer to the user the resposability to keep the results up to date. ", "\n", - "\nUsually, only a subset of information changes for a given Scenario and therefore is completed using unpertubed data from the corresponding Best Estimate using [Relaxed Queries](../Utils/Queries#relaxedqueries).", + "\n### Dependecy with Best Estimate scenario: across periods", "\n", - "\nThe User is free of importing any number of files for the same Scenario, Year and Month. Nevertheless, adding distinct source of pertubation makes Sensitivity analysis more complex. We recommends to avoid this approach.", - "\n" + "\nWhen a stress scenario is imported for a period $P$ and a previous period $P-1$ is available, the End of Period values ($P-1$) of the Best Estimate scenario is considered as Beginning of Period ($P$) for the stressed and no-stress scenarios. This allows to:", + "\n1. all scenarios must not be calculated every period,", + "\n2. new scenarios can be added in a production environment,", + "\n3. in each period the scenario depends only on the perturbation provided in the period and is not applied on top of the previous period perturbation." ], "metadata": {}, "execution_count": 0, diff --git a/ifrs17/Import/Importers.ipynb b/ifrs17/Import/Importers.ipynb index 7c8113d0..e1cc9cc1 100644 --- a/ifrs17/Import/Importers.ipynb +++ b/ifrs17/Import/Importers.ipynb @@ -349,7 +349,16 @@ "source": [ "## Parse the Main Tab", "\n", - "\nThe main table of our custom import formats contains the information which are required to identify the data partition depending on the ImportFormat. These information are temporarily stored in [Args](../DataModel/DataStructure#args) and used in the next methods.", + "\nThe main table of our custom import formats contains the information which are required to identify the data partition depending on the ImportFormat.", + "\nThe columns contained in the main table for the different ImportFormat are:", + "\n1. Yield Curve : Year, Month, Scenario;", + "\n2. Data Node : ReportingNode;", + "\n3. Nominal Cashflow, Actual, Opening, Simple Value, Data Node State, Data Node Parameter : ReportingNode, Year, Month, Scenario.", + "\n", + "\nNote that the Scenario column is not available for Data Node. Data Node can be created regardeless of the Scenario and controlling whether they are active or not in a particular scenario can be achieved through the import of Data Node State (which allows the specification of a Scenario)", + "\nIn addition, the Scenario column (for the ImportFormats that expect it) is not required as its value can be left empty in the case of Best Estimate scenario (in this case the entire column can be missing). ", + "\n", + "\nAfter having parsed the main table, these information are temporarily stored in [Args](../DataModel/DataStructure#args) and used in the next methods.", "\n", "\nGetArgsFromMain performes basic valiadations on the existance of the main tab. Then reads the reporting node, year, month, and scenario and return an ImportArgs with the results. If any of these information a default value is returned and will be validated in the following methods. " ], @@ -534,9 +543,15 @@ "", "\n## Calculate IFRS Variables", "\n", - "\nThe following methods are used in the different importers to compute the [IfrsVariables](../DataModel/DataStructure#ifrs-variable).", + "\nThe following methods are used in the all importers that compute [IfrsVariables](../DataModel/DataStructure#ifrs-variable):", + "\n- Yield Curve,", + "\n- Data Node State,", + "\n- Data Node Parameter,", + "\n- Nominal Cashflow,", + "\n- Actual,", + "\n- Opening.", "\n", - "\nGetAllArgsAsync retrieves the partitions or Args that require computation. They are the union of the primary args (the one read from the main tab of the imported file) with the secondary args (scenarios which depends on the best estimate data).", + "\nGetAllArgsAsync retrieves the partitions or Args that require computation. They are the union of the so called **primary args** which is read from the main tab of the imported file, with the **secondary args** corresponding to all scenarios which depends on the imported data.", "\n", "\nComputeAsync computes the IfrsVariables for a given partition (identified by its ImportArgs) and stores the results in a disposable workspace. This then serves as DataSource in the calculation of the secondary partitions (identified by the secondary args)." ], diff --git a/ifrs17/Import/Importers.ipynb.orig b/ifrs17/Import/Importers.ipynb.orig deleted file mode 100644 index 1bb51e85..00000000 --- a/ifrs17/Import/Importers.ipynb.orig +++ /dev/null @@ -1,1653 +0,0 @@ -{ - "metadata": { - "authors": [], - "id": "-WSDeQyVl0Cc-rDGmPI73w", - "kernelspec": { - "display_name": "Formula Framework", - "language": "C#", - "name": "C#" - }, - "language_info": { - "file_extension": ".cs", - "mimetype": "text/plain", - "name": "C#" - } - }, - "nbformat": 4, - "nbformat_minor": 5, - "cells": [ - { - "cell_type": "markdown", - "source": [ - "", - "\n

Importer Methods

", - "\n" - ], - "metadata": {}, - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "source": [ - "#!import \"6ImportScope-Compute\"" - ], - "metadata": {}, - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "source": [ - "# Parsing Storage", - "\n", - "\nThe Parsing storage collects data required for the import of a file. ", - "\nSuch storage is then passed to parsing format definitions to map the content of the file into variables." - ], - "metadata": {}, - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "source": [ - "public class ParsingStorage", - "\n{", - "\n private readonly IDataSource dataSource;", - "\n private readonly IWorkspace workspace;", - "\n private readonly ImportArgs args;", - "\n ", - "\n //Hierarchy Cache", - "\n public Systemorph.Vertex.Hierarchies.IHierarchicalDimensionCache HierarchyCache;", - "\n ", - "\n public ReportingNode ReportingNode { get; protected set; }", - "\n ", - "\n public Dictionary DataNodeDataBySystemName;", - "\n ", - "\n // Dimensions", - "\n public Dictionary EstimateType;", - "\n public Dictionary AmountType; ", - "\n public HashSet MandatoryAocSteps;", - "\n public HashSet AocTypeMap;", - "\n private HashSet estimateTypes;", - "\n private HashSet amountTypes;", - "\n private Dictionary> amountTypesByEstimateType => GetAmountTypesByEstimateType(HierarchyCache);", - "\n public HashSet TechnicalMarginEstimateTypes => GetTechnicalMarginEstimateType(); ", - "\n public Dictionary> DimensionsWithExternalId;", - "\n public Dictionary> SingleDataNodeParametersByGoc { get; private set; }", - "\n", - "\n // Partitions", - "\n public PartitionByReportingNode TargetPartitionByReportingNode;", - "\n public PartitionByReportingNodeAndPeriod TargetPartitionByReportingNodeAndPeriod;", - "\n ", - "\n //Constructor", - "\n public ParsingStorage(ImportArgs args, IDataSource dataSource, IWorkspace workspace)", - "\n {", - "\n this.args = args;", - "\n this.dataSource = dataSource;", - "\n this.workspace = workspace;", - "\n }", - "\n ", - "\n // Initialize", - "\n public async Task InitializeAsync()", - "\n {", - "\n //Partition Workspace and DataSource", - "\n TargetPartitionByReportingNode = (await workspace.Query().Where(p => p.ReportingNode == args.ReportingNode).ToArrayAsync()).SingleOrDefault(); ", - "\n ", - "\n if(TargetPartitionByReportingNode == null) ", - "\n { ApplicationMessage.Log(Error.ParsedPartitionNotFound, args.ReportingNode); return; } ", - "\n ", - "\n await workspace.Partition.SetAsync(TargetPartitionByReportingNode.Id);", - "\n await dataSource.Partition.SetAsync(TargetPartitionByReportingNode.Id);", - "\n ", - "\n if(args.ImportFormat == ImportFormats.Cashflow || args.ImportFormat == ImportFormats.Actual || ", - "\n args.ImportFormat == ImportFormats.SimpleValue || args.ImportFormat == ImportFormats.Opening)", - "\n {", - "\n TargetPartitionByReportingNodeAndPeriod = (await workspace.Query()", - "\n .Where(p => p.ReportingNode == args.ReportingNode &&", - "\n p.Year == args.Year &&", - "\n p.Month == args.Month &&", - "\n p.Scenario == args.Scenario).ToArrayAsync()).SingleOrDefault();", - "\n ", - "\n if(TargetPartitionByReportingNodeAndPeriod == null) ", - "\n { ApplicationMessage.Log(Error.ParsedPartitionNotFound, args.ReportingNode, args.Year.ToString(), args.Month.ToString(), args.Scenario); return; } ", - "\n ", - "\n await workspace.Partition.SetAsync(TargetPartitionByReportingNodeAndPeriod.Id);", - "\n await dataSource.Partition.SetAsync(TargetPartitionByReportingNodeAndPeriod.Id);", - "\n ", - "\n //Clean up the workspace", - "\n await workspace.DeleteAsync( await workspace.Query().ToArrayAsync() );", - "\n await workspace.DeleteAsync( await workspace.Query().ToArrayAsync() );", - "\n }", - "\n ", - "\n var reportingNodes = (await dataSource.Query().Where(x => x.SystemName == args.ReportingNode).ToArrayAsync());", - "\n if(!reportingNodes.Any()) { ApplicationMessage.Log(Error.ReportingNodeNotFound, args.ReportingNode); return; }", - "\n ReportingNode = reportingNodes.First();", - "\n", - "\n var aocConfigurationByAocStep = await dataSource.LoadAocStepConfigurationAsync(args.Year, args.Month);", - "\n MandatoryAocSteps = aocConfigurationByAocStep.Where(x => x.DataType == DataType.Mandatory).Select(x => new AocStep(x.AocType, x.Novelty)).ToHashSet();", - "\n AocTypeMap = args.ImportFormat switch {", - "\n ImportFormats.Cashflow => aocConfigurationByAocStep.Where(x => x.InputSource.Contains(InputSource.Cashflow) &&", - "\n !new DataType[]{DataType.Calculated, DataType.CalculatedTelescopic}.Contains(x.DataType) )", - "\n .GroupBy(x => new AocStep(x.AocType, x.Novelty), (k,v) => k).ToHashSet(),", - "\n ImportFormats.Actual => aocConfigurationByAocStep.Where(x => x.InputSource.Contains(InputSource.Actual) &&", - "\n !new DataType[]{DataType.Calculated, DataType.CalculatedTelescopic}.Contains(x.DataType) && ", - "\n new AocStep(x.AocType, x.Novelty) != new AocStep(AocTypes.BOP, Novelties.I))", - "\n .GroupBy(x => new AocStep(x.AocType, x.Novelty), (k,v) => k).ToHashSet(),", - "\n ImportFormats.Opening => aocConfigurationByAocStep.Where(x => x.InputSource.Contains(InputSource.Opening) && x.DataType == DataType.Optional).GroupBy(x => new AocStep(x.AocType, x.Novelty), (k,v) => k).ToHashSet(),", - "\n ImportFormats.SimpleValue => aocConfigurationByAocStep.GroupBy(x => new AocStep(x.AocType, x.Novelty), (k,v) => k).Concat((await dataSource.Query().ToArrayAsync())", - "\n .Select(vt => new AocStep(vt.SystemName,null))).ToHashSet(),", - "\n _ => Enumerable.Empty().ToHashSet(),", - "\n };", - "\n DataNodeDataBySystemName = args.ImportFormat == ImportFormats.Opening ", - "\n ? (await LoadDataNodesAsync(dataSource, args)).Where(kvp => kvp.Value.Year == args.Year).ToDictionary(kvp => kvp.Key, kvp => kvp.Value)", - "\n : await LoadDataNodesAsync(dataSource, args);", - "\n", - "\n SingleDataNodeParametersByGoc = await dataSource.LoadSingleDataNodeParametersAsync(args);", - "\n", - "\n // Dimensions", - "\n EstimateType = (await dataSource.Query().ToArrayAsync()).ToDictionary(x => x.SystemName);", - "\n AmountType = (await dataSource.Query().Where(x =>!(x is DeferrableAmountType)).ToArrayAsync()).ToDictionary(x => x.SystemName);", - "\n amountTypes = (await dataSource.Query().ToArrayAsync()).Select(at => at.SystemName).ToHashSet();", - "\n estimateTypes = args.ImportFormat switch {", - "\n ImportFormats.SimpleValue => (await dataSource.Query().ToArrayAsync()).Select(et => et.SystemName).ToHashSet(),", - "\n ImportFormats.Opening => (await dataSource.Query().Where(et => et.StructureType == StructureType.AoC).ToArrayAsync())", - "\n .Where(et => et.InputSource.Contains(InputSource.Opening)) //This Contains overload cannot be used in DB", - "\n .Select(et => et.SystemName).ToHashSet(),", - "\n _ => Enumerable.Empty().ToHashSet(),", - "\n };", - "\n ", - "\n ", - "\n // DimensionsWithExternalId", - "\n DimensionsWithExternalId = new Dictionary>()", - "\n {", - "\n { typeof(AmountType), await GetDimensionWithExternalIdDictionaryAsync() },", - "\n { typeof(EstimateType), await GetDimensionWithExternalIdDictionaryAsync() }", - "\n };", - "\n ", - "\n //Hierarchy Cache", - "\n HierarchyCache = workspace.ToHierarchicalDimensionCache();", - "\n await HierarchyCache.InitializeAsync();", - "\n }", - "\n ", - "\n public async Task> GetDimensionWithExternalIdDictionaryAsync () where T : KeyedOrderedDimension", - "\n {", - "\n var dict = new Dictionary();", - "\n var items = await dataSource.Query().ToArrayAsync();", - "\n foreach (var item in items) {", - "\n dict.TryAdd(item.SystemName, item.SystemName);", - "\n if(typeof(T).IsAssignableTo(typeof(KeyedOrderedDimensionWithExternalId))) {", - "\n var externalIds = (string[])(typeof(T).GetProperty(nameof(KeyedOrderedDimensionWithExternalId.ExternalId)).GetValue(item));", - "\n if(externalIds == null) continue;", - "\n foreach (var extId in externalIds) ", - "\n dict.TryAdd(extId, item.SystemName);", - "\n }", - "\n }", - "\n return dict;", - "\n }", - "\n ", - "\n // Getters", - "\n public bool IsDataNodeReinsurance(string goc) => DataNodeDataBySystemName[goc].IsReinsurance;", - "\n public bool IsValidDataNode(string goc) => DataNodeDataBySystemName.ContainsKey(goc);", - "\n", - "\n public CashFlowPeriodicity GetCashFlowPeriodicity(string goc) {", - "\n if(!SingleDataNodeParametersByGoc.TryGetValue(goc, out var inner)) ", - "\n return CashFlowPeriodicity.Monthly;", - "\n return inner[CurrentPeriod].CashFlowPeriodicity; ", - "\n }", - "\n", - "\n public InterpolationMethod GetInterpolationMethod(string goc) {", - "\n if(!SingleDataNodeParametersByGoc.TryGetValue(goc, out var inner))", - "\n return InterpolationMethod.NotApplicable;", - "\n return inner[CurrentPeriod].InterpolationMethod; ", - "\n }", - "\n", - "\n // Validations", - "\n public string ValidateEstimateType(string et, string goc) {", - "\n var allowedEstimateTypes = estimateTypes;", - "\n if (DataNodeDataBySystemName.TryGetValue(goc, out var dataNodeData) && dataNodeData.LiabilityType == LiabilityTypes.LIC)", - "\n estimateTypes.ExceptWith(TechnicalMarginEstimateTypes);", - "\n if(!allowedEstimateTypes.Contains(et))", - "\n ApplicationMessage.Log(Error.EstimateTypeNotFound, et);", - "\n return et;", - "\n }", - "\n ", - "\n public string ValidateAmountType(string at) {", - "\n if (at != null && !amountTypes.Contains(at))", - "\n ApplicationMessage.Log(Error.AmountTypeNotFound, at);", - "\n return at;", - "\n }", - "\n ", - "\n public AocStep ValidateAocStep(AocStep aoc) {", - "\n if (!AocTypeMap.Contains(aoc))", - "\n ApplicationMessage.Log(Error.AocTypeMapNotFound, aoc.AocType, aoc.Novelty);", - "\n return aoc;", - "\n }", - "\n ", - "\n public string ValidateDataNode(string goc, string importFormat) {", - "\n if (!DataNodeDataBySystemName.ContainsKey(goc))", - "\n {", - "\n if( importFormat == ImportFormats.Opening )", - "\n ApplicationMessage.Log(Error.InvalidDataNodeForOpening, goc);", - "\n else", - "\n ApplicationMessage.Log(Error.InvalidDataNode, goc);", - "\n }", - "\n return goc;", - "\n }", - "\n ", - "\n public void ValidateEstimateTypeAndAmountType(string estimateType, string amountType){", - "\n if (amountTypesByEstimateType.TryGetValue(estimateType, out var ats) && ats.Any() && !ats.Contains(amountType))", - "\n ApplicationMessage.Log(Error.InvalidAmountTypeEstimateType, estimateType, amountType);", - "\n }", - "\n}" - ], - "metadata": {}, - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "source": [ - "# Basics", - "\n", - "\nBasic methods to delete variable and update variables to the data source are defined." - ], - "metadata": {}, - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "source": [ - "## Clean the Target QuerySource" - ], - "metadata": {}, - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "source": [ - "public static async Task CleanAsync (this IDataSource dataSource, Guid partitionId = default, Expression> filter = null) where T : class, IPartitioned", - "\n{", - "\n var loadData = partitionId != (Guid)default", - "\n ? await dataSource.Query().Where(x => x.Partition == partitionId ).Where(filter?? (Expression>)(x => true)).ToListAsync()", - "\n : await dataSource.Query().Where(filter ?? (Expression>)(x => true)).ToListAsync();", - "\n await dataSource.DeleteAsync(loadData);", - "\n}" - ], - "metadata": {}, - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "source": [ - "## Update the Target DataSource", - "\n", - "\nSource and target data space are of type IDataSource to allow update and commit to a Workspace or to a DataSource." - ], - "metadata": {}, - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "source": [ - "public static async Task CommitToAsync (this IDataSource source, IDataSource target, Guid partitionId = default, bool snapshot = true, Expression> filter = null) ", - "\nwhere TData : class, IPartitioned", - "\nwhere TPartition : IfrsPartition", - "\n{", - "\n if(partitionId != (Guid)default) {", - "\n await target.Partition.SetAsync(partitionId);", - "\n await source.Partition.SetAsync(partitionId);", - "\n }", - "\n if(snapshot) await CleanAsync(target, partitionId, filter);", - "\n await target.UpdateAsync( await source.Query().ToArrayAsync() );", - "\n await target.CommitAsync();", - "\n}" - ], - "metadata": {}, - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "source": [ - "# Import helpers" - ], - "metadata": {}, - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "source": [ - "## Parse the Main Tab", - "\n", - "\nThe main table of our custom import formats contains the information which are required to identify the data partition depending on the ImportFormat. These information are temporarily stored in [Args](../DataModel/DataStructure#args) and used in the next methods.", - "\n", - "\nGetArgsFromMain performes basic valiadations on the existance of the main tab. Then reads the reporting node, year, month, and scenario and return an ImportArgs with the results. If any of these information a default value is returned and will be validated in the following methods. " - ], - "metadata": {}, - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "source": [ - "public ImportArgs GetArgsFromMain(IDataSet dataSet) {", - "\n var mainTab = dataSet.Tables[Main];", - "\n if(mainTab == null) ApplicationMessage.Log(Error.NoMainTab);", - "\n if(!mainTab.Rows.Any()) ApplicationMessage.Log(Error.IncompleteMainTab);", - "\n if(ApplicationMessage.HasErrors()) return null;", - "\n", - "\n var main = mainTab.Rows.First();", - "\n var reportingNode = mainTab.Columns.Any(x => x.ColumnName == nameof(Args.ReportingNode)) && main[nameof(Args.ReportingNode)] != null ? (string)main[nameof(ReportingNode)] : default(string);", - "\n var scenario = mainTab.Columns.Any(x => x.ColumnName == nameof(Args.Scenario)) && main[nameof(Args.Scenario)] != null ? (string)main[nameof(Scenario)] : default(string);", - "\n var year = mainTab.Columns.Any(x => x.ColumnName == nameof(Args.Year)) && main[nameof(Args.Year)] != null ? (int)Convert.ChangeType(main[nameof(Args.Year)], typeof(int)) : default(int);", - "\n var month = mainTab.Columns.Any(x => x.ColumnName == nameof(Args.Month)) && main[nameof(Args.Month)] != null ? (int)Convert.ChangeType(main[nameof(Args.Month)], typeof(int)) : default(int);", - "\n", - "\n return new ImportArgs(reportingNode, year, month, default(Periodicity), scenario, default(string));", - "\n}" - ], - "metadata": {}, - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "source": [ - "A validation method which checks year and month. Log errors if any is missing. " - ], - "metadata": {}, - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "source": [ - "public async static void ValidateArgsForPeriod(this ImportArgs args, IDataSource targetDataSource) {", - "\n if(args.Year == default(int)) ApplicationMessage.Log(Error.YearInMainNotFound);", - "\n if(args.Month == default(int)) ApplicationMessage.Log(Error.MonthInMainNotFound);", - "\n var availableScenarios = await targetDataSource.Query().Select(x => x.SystemName).ToArrayAsync();", - "\n if(!(args.Scenario == default(string) || availableScenarios.Contains(args.Scenario))) ApplicationMessage.Log(Error.DimensionNotFound, \"Scenario\", args.Scenario);", - "\n}" - ], - "metadata": {}, - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "source": [ - "## Create Partition", - "\n", - "\nThese are the methods used to create the partition if not already existing in the DataSource. " - ], - "metadata": {}, - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "source": [ - "public async Task CommitPartitionAsync(ImportArgs args, params IDataSource[] dataSources)", - "\n{", - "\n foreach (var dataSource in dataSources) {", - "\n switch(typeof(IPartition).Name) {", - "\n case nameof(PartitionByReportingNode) : {", - "\n await dataSource.UpdateAsync( new[] { new PartitionByReportingNode { ", - "\n Id = (Guid)(await DataSource.Partition.GetKeyForInstanceAsync(args)),", - "\n ReportingNode = args.ReportingNode } } );", - "\n break;", - "\n }", - "\n case nameof(PartitionByReportingNodeAndPeriod) : {", - "\n args.ValidateArgsForPeriod(dataSource);", - "\n if(ApplicationMessage.HasErrors()) return;", - "\n", - "\n await dataSource.UpdateAsync( new[]{ new PartitionByReportingNodeAndPeriod { ", - "\n Id = (Guid)(await DataSource.Partition.GetKeyForInstanceAsync(args)),", - "\n Year = args.Year,", - "\n Month = args.Month,", - "\n ReportingNode = args.ReportingNode, ", - "\n Scenario = args.Scenario } } );", - "\n break;", - "\n }", - "\n default : {", - "\n ApplicationMessage.Log(Error.PartitionTypeNotFound, typeof(IPartition).Name); ", - "\n return;", - "\n }", - "\n }", - "\n await dataSource.CommitAsync();", - "\n }", - "\n}" - ], - "metadata": {}, - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "source": [ - "## Get Args and create Partition", - "\n", - "\nThis is the main method to get Args which reference a specific data partition. All validations are triggered after parsing and if the partition is not already existing in the DataSource it is created. " - ], - "metadata": {}, - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "source": [ - "public async Task GetArgsAndCommitPartitionAsync(IDataSet dataSet, IDataSource targetDataSource)", - "\n{", - "\n var args = GetArgsFromMain(dataSet);", - "\n if(ApplicationMessage.HasErrors()) return null;", - "\n if(args.ReportingNode == default(string)) { ApplicationMessage.Log(Error.ReportingNodeInMainNotFound); return null; }", - "\n await CommitPartitionAsync(args, targetDataSource);", - "\n return args;", - "\n}" - ], - "metadata": {}, - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "source": [ - "## Data Node Factory" - ], - "metadata": {}, - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "source": [ - "public async Task DataNodeFactoryAsync(IDataSet dataSet, string tableName, ImportArgs args, IDataSource targetDataSource)", - "\n{", - "\n var partition = (await DataSource.Query().Where(p => p.ReportingNode == args.ReportingNode).ToArrayAsync()).FirstOrDefault();", - "\n if(partition == null) { ApplicationMessage.Log(Error.ParsedPartitionNotFound); return; }", - "\n ", - "\n var table = dataSet.Tables[tableName];", - "\n ", - "\n var dataNodesImported = table.Rows.Select(x => x.Field(nameof(RawVariable.DataNode))).ToHashSet();", - "\n var dataNodesDefined = await targetDataSource.Query().Where(x => dataNodesImported.Contains(x.SystemName)).ToArrayAsync();", - "\n var dataNodeStatesDefined = await targetDataSource.Query().Select(x => x.DataNode).ToArrayAsync();", - "\n var dataNodeParametersDefined = await targetDataSource.Query().Select(x => x.DataNode).ToArrayAsync(); ", - "\n var dataNodeStatesUndefined = dataNodesImported.Where(x => x != null && !dataNodeStatesDefined.Contains(x)).ToHashSet();", - "\n var dataNodeSingleParametersUndefined = dataNodesImported.Where(x => x != null &&", - "\n !dataNodeParametersDefined.Contains(x) && ", - "\n dataNodesDefined.SingleOrDefault(y => y.SystemName == x) is GroupOfInsuranceContract).ToHashSet();", - "\n if ((dataNodeStatesUndefined?.Any() ?? false))", - "\n await targetDataSource.UpdateAsync( dataNodeStatesUndefined.Select(x => ", - "\n new DataNodeState {DataNode = x, ", - "\n Year = args.Year, ", - "\n Month = DefaultDataNodeActivationMonth, ", - "\n State = State.Active, ", - "\n Partition = partition.Id})", - "\n .ToArray() );", - "\n if ((dataNodeSingleParametersUndefined?.Any() ?? false))", - "\n await targetDataSource.UpdateAsync( dataNodeSingleParametersUndefined.Select(x => ", - "\n new SingleDataNodeParameter {DataNode = x, ", - "\n Year = args.Year, ", - "\n Month = DefaultDataNodeActivationMonth, ", - "\n PremiumAllocation = DefaultPremiumExperienceAdjustmentFactor, ", - "\n Partition = partition.Id})", - "\n .ToArray() );", - "\n await targetDataSource.CommitAsync();", - "\n}" - ], - "metadata": {}, - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "source": [ - "## Calculate IFRS Variables", - "\n", - "\nThe following methods are used in the different importers to compute the [IfrsVariables](../DataModel/DataStructure#ifrs-variable).", - "\n", -<<<<<<< HEAD - "\nGetAllArgsAsync retrieves the partitions or Args that require computation. They are the union of the primary args (the one read from the main tab of the imported file) with the secondary args (scenarios which depends on the Best Estimate data).The implemented logic is different between Transactional Data (Openinig, Cashflow and Actuals) and Parameters (Yield Curves, DataNodeParameter).", - "\n ", -======= - "\nGetAllArgsAsync retrieves the partitions or Args that require computation. They are the union of the primary args (the one read from the main tab of the imported file) with the secondary args (scenarios which depends on the best estimate data).", - "\n", ->>>>>>> Scenario_Improvments - "\nComputeAsync computes the IfrsVariables for a given partition (identified by its ImportArgs) and stores the results in a disposable workspace. This then serves as DataSource in the calculation of the secondary partitions (identified by the secondary args)." - ], - "metadata": {}, - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "source": [ - "public async Task GetAllArgsAsync(ImportArgs args, IDataSource dataSource, string format)", - "\n{", - "\n ImportArgs[] allArgs;", - "\n switch(format)", - "\n {", - "\n case ImportFormats.Cashflow:", - "\n case ImportFormats.Actual: ", - "\n case ImportFormats.Opening:", - "\n {", - "\n if(args.Scenario != null) return args.RepeatOnce().ToArray();", - "\n var secondaryArgs = await dataSource.Query() ", - "\n .Where(x => x.ReportingNode == args.ReportingNode && x.Year == args.Year && x.Month == args.Month && x.Scenario != null)", - "\n .Select(x => new ImportArgs(x.ReportingNode, x.Year, x.Month, default(Periodicity), x.Scenario, format)).ToArrayAsync();", - "\n ", - "\n if(secondaryArgs.Any()) ApplicationMessage.Log(Warning.ScenarioReCalculations, String.Join(\", \", secondaryArgs.Select(x => x.Scenario)));", - "\n allArgs = args.RepeatOnce().Concat(secondaryArgs).ToArray();", - "\n break;", - "\n }", - "\n default: ", - "\n {", - "\n if(args.Scenario == null) {", - "\n var scenarios = format switch {", - "\n ImportFormats.YieldCurve => await dataSource.Query().Where(x => x.Year == args.Year && x.Month == args.Month && x.Scenario != null).Select(x => x.Scenario).Distinct().ToArrayAsync(),", - "\n ImportFormats.DataNodeParameter => await dataSource.Query().Where(x => x.Year == args.Year && x.Month == args.Month && x.Scenario != null).Select(x => x.Scenario).Distinct().ToArrayAsync()", - "\n };", - "\n var targetPartitions = await dataSource.Query()", - "\n .Where(x => x.Year == args.Year && x.Month == args.Month && !scenarios.Contains(x.Scenario)).OrderBy(x => x.Scenario).ToArrayAsync();", - "\n var targetScenarios = targetPartitions.Where(x => x.Scenario != null).Select(x => x.Scenario);", - "\n if(targetScenarios.Any()) ApplicationMessage.Log(Warning.ScenarioReCalculations, String.Join(\", \", targetScenarios));", - "\n allArgs = targetPartitions.Select(x => new ImportArgs(x.ReportingNode, x.Year, x.Month, default(Periodicity), x.Scenario, ImportFormats.Cashflow)).ToArray();", - "\n }", - "\n else {", - "\n allArgs = (await dataSource.Query()", - "\n .Where(x => x.Year == args.Year && x.Month == args.Month && x.Scenario == null).ToArrayAsync())", - "\n .Select(x => new ImportArgs(x.ReportingNode, x.Year, x.Month, default(Periodicity), args.Scenario, ImportFormats.Cashflow)).ToArray();", - "\n }", - "\n break;", - "\n }", - "\n }", - "\n return allArgs.Where(x => (!Scenarios.EnableScenario && x.Scenario == null) || Scenarios.EnableScenario).ToArray();", - "\n}" - ], - "metadata": {}, - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "source": [ - "public async Task ComputeAsync(ImportArgs args, IWorkspace workspace, IWorkspace workspaceToCompute, bool saveRawVariables)", - "\n{", - "\n Activity.Start();", - "\n var storage = new ImportStorage(args, workspaceToCompute, workspace);", - "\n await storage.InitializeAsync();", - "\n if(Activity.HasErrors()) return Activity.Finish();", - "\n ", - "\n var universe = Scopes.ForStorage(storage).ToScope();", - "\n var identities = universe.GetScopes(storage.DataNodesByImportScope[ImportScope.Primary]).SelectMany(s => s.Identities);", - "\n var ivs = universe.GetScopes(identities).SelectMany(x => x.CalculatedIfrsVariables);", - "\n if(Activity.HasErrors()) return Activity.Finish();", - "\n", - "\n if(storage.DefaultPartition != storage.TargetPartition) {", - "\n var bestEstimateIvs = await workspaceToCompute.LoadPartitionedDataAsync(storage.DefaultPartition);", - "\n ivs = ivs.Where(iv => Math.Abs(iv.Value) >= Precision).ToArray()", - "\n .Except(bestEstimateIvs, IfrsVariableComparer.Instance(ignoreValue: false))", - "\n .Concat(ivs.Where(x => Math.Abs(x.Value) < Precision).Intersect(bestEstimateIvs, EqualityComparer.Instance).Select(x => x with {Value = 0.0}).ToArray());", - "\n }", - "\n", - "\n workspace.Reset(x => x.ResetType());", - "\n await workspace.UpdateAsync(ivs.Where(x => storage.DefaultPartition != storage.TargetPartition || Math.Abs(x.Value) >= Precision));", - "\n await workspace.CommitToAsync(workspaceToCompute, storage.TargetPartition, snapshot : true, ", - "\n filter : x => storage.EstimateTypesByImportFormat[args.ImportFormat].Contains(x.EstimateType) ", - "\n && storage.DataNodesByImportScope[ImportScope.Primary].Contains(x.DataNode));", - "\n if(saveRawVariables) {", - "\n if(args.Scenario == null) await workspace.DeleteAsync(await workspace.Query().Where(rv => rv.Values.Sum(x => Math.Abs(x)) < Precision).ToArrayAsync());", - "\n await workspace.CommitToAsync(workspaceToCompute, storage.TargetPartition, snapshot : true, ", - "\n filter : x => storage.DataNodesByImportScope[ImportScope.Primary].Contains(x.DataNode)", - "\n );", - "\n }", - "\n return Activity.Finish();", - "\n}" - ], - "metadata": {}, - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "source": [ - "# Validations" - ], - "metadata": {}, - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "source": [ - "## Validation for Active Data Node States" - ], - "metadata": {}, - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "source": [ - "public async static Task ValidateForDataNodeStateActiveAsync(this IWorkspace workspace, Dictionary dataNodes) where T : BaseDataRecord", - "\n{ ", - "\n foreach(var item in (await workspace.Query().ToArrayAsync()).GroupBy(x => x.DataNode))", - "\n if(!dataNodes.ContainsKey(item.First().DataNode))", - "\n ApplicationMessage.Log(Error.InactiveDataNodeState, item.First().DataNode);", - "\n}", - "\n" - ], - "metadata": {}, - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "source": [ - "## Validate for Data Node States Logic" - ], - "metadata": {}, - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "source": [ - "public async static Task ValidateDataNodeStatesAsync(this IWorkspace workspace, Dictionary persistentDataNodeByDataNode)", - "\n{", - "\n foreach(var importedDataNodeState in await workspace.Query().ToArrayAsync())", - "\n {", - "\n if(persistentDataNodeByDataNode.TryGetValue(importedDataNodeState.DataNode, out var currentPersistentDataNode))", - "\n {", - "\n if(importedDataNodeState.State < currentPersistentDataNode.State)", - "\n ApplicationMessage.Log(Error.ChangeDataNodeState, importedDataNodeState.DataNode, ", - "\n currentPersistentDataNode.State.ToString(), ", - "\n importedDataNodeState.State.ToString());", - "\n", - "\n if(importedDataNodeState.State == currentPersistentDataNode.State)", - "\n await workspace.DeleteAsync(importedDataNodeState);", - "\n }", - "\n }", - "\n}" - ], - "metadata": {}, - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "source": [ - "## Validate for Mandatory Aoc Steps" - ], - "metadata": {}, - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "source": [ - "using static Systemorph.Vertex.Equality.IdentityPropertyExtensions;", - "\npublic async static Task ValidateForMandatoryAocSteps(this IWorkspace workspace, IDataSet dataSet, HashSet mandatoryAocSteps)", - "\n{ ", - "\n var ignoreProperties = new[]{nameof(AocType), nameof(Novelty)};", - "\n var missingAocStepsByIdentityProperties = (await workspace.Query().ToListAsync())", - "\n .GroupBy(x => x.ToIdentityString(ignoreProperties),", - "\n x => new AocStep(x.AocType, x.Novelty),", - "\n (properties, parsedAocSteps) => (properties, mandatoryAocSteps.Except(parsedAocSteps))", - "\n );", - "\n foreach((var properties, var missingSteps) in missingAocStepsByIdentityProperties) ", - "\n foreach(var missingStep in missingSteps) ApplicationMessage.Log(Warning.MandatoryAocStepMissing, missingStep.AocType, missingStep.Novelty, properties);", - "\n}" - ], - "metadata": {}, - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "source": [ - "# Analysis of Change Configuration" - ], - "metadata": {}, - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "source": [ - "The [Analysis of Change configuration](../DataModel/DataStructure#aoc-step-configuration) is parsed from the input file and complemented with defaults to allow for an easy insertion of new AoC Steps.", - "\n", - "\nAfter having checked that the [AocTypes](../DataModel/DataStructure#aoc-variable-type) loaded in the target DataSource are including all the compulsory ones, default configurations are generated on the basis of the AocTypes ordering. ", - "\n", - "\nThe following categories have been identified based on the *Order* of the novel AoC Step:", - "\n", - "\n**Category** $$\\hspace{2.8cm}$$ **Default added with same configuration of**", - "\n| | |", - "\n|-----------------------|-------------------------------------------------------|", - "\n| Order < RCU $$\\phantom{.......................}$$ | MC with Novelty I |", - "\n| RCU < Order < CF | RCU with Novelty I |", - "\n| IA < Order < YCU | AU with both Novelty I and N |", - "\n| CRU < Order < WO | EV with Novelty I and N |", - "\n| WO < Order < CL | WO with Novelty C (only for Import Source = Actual) |", - "\n", - "\nThe new AoC Configurations are created with the same order of the AoC Types. " - ], - "metadata": {}, - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "source": [ - "Import.DefineFormat(ImportFormats.AocConfiguration, async (options, dataSet) => {", - "\n Activity.Start();", - "\n var workspace = Workspace.CreateNew();", - "\n workspace.InitializeFrom(options.TargetDataSource);", - "\n", - "\n var aocTypes = await options.TargetDataSource.Query().OrderBy(x => x.Order).ToArrayAsync();", - "\n var aocTypesCompulsory = typeof(AocTypes).GetFields().Select(x => (string)x.Name);", - "\n if(aocTypesCompulsory.Where(x => !aocTypes.Select(x => x.SystemName).Contains(x)).Any()) {", - "\n ApplicationMessage.Log(Error.AocTypeCompulsoryNotFound);", - "\n return Activity.Finish();", - "\n }", - "\n ", - "\n var logConfig = await Import.FromDataSet(dataSet).WithType().WithTarget(workspace).ExecuteAsync();", - "\n if(logConfig.Errors.Any()) return Activity.Finish().Merge(logConfig); ", - "\n", - "\n var orderByName = aocTypes.ToDictionary(x => x.SystemName, x => x.Order);", - "\n var aocConfigs = (await workspace.Query().ToArrayAsync())", - "\n .GroupBy(x => (x.AocType, x.Novelty))", - "\n .Select(y => y.OrderByDescending(x => x.Year).ThenByDescending(x => x.Month).FirstOrDefault())", - "\n .ToDictionary(x => (x.AocType, x.Novelty));", - "\n var aocOrder = aocConfigs.ToDictionary(x => x.Key, x => x.Value.Order);", - "\n var newAocTypes = orderByName.Keys.Where(x => !aocConfigs.Keys.Contains((x, Novelties.I)) && ", - "\n !aocConfigs.Keys.Contains((x, Novelties.N)) && ", - "\n !aocConfigs.Keys.Contains((x, Novelties.C)) && ", - "\n !aocTypes.Any(y => y.Parent == x) &&", - "\n !aocTypesCompulsory.Contains(x)).ToArray();", - "\n", - "\n foreach(var newAocType in newAocTypes) {", - "\n if(orderByName[newAocType] < orderByName[AocTypes.RCU])", - "\n {", - "\n var step = (AocTypes.MC, Novelties.I);", - "\n await workspace.UpdateAsync( aocConfigs[step] with { AocType = newAocType, DataType = DataType.Optional, Order = ++aocOrder[step] }); ", - "\n }", - "\n else if(orderByName[newAocType] > orderByName[AocTypes.RCU] && orderByName[newAocType] < orderByName[AocTypes.CF]) ", - "\n {", - "\n var step = (AocTypes.RCU, Novelties.I);", - "\n await workspace.UpdateAsync( aocConfigs[step] with { AocType = newAocType, DataType = DataType.Optional, Order = ++aocOrder[step] });", - "\n }", - "\n else if(orderByName[newAocType] > orderByName[AocTypes.IA] && orderByName[newAocType] < orderByName[AocTypes.YCU]) ", - "\n {", - "\n foreach (var novelty in new[]{Novelties.I, Novelties.N}) {", - "\n var step = (AocTypes.AU, novelty);", - "\n var order = orderByName[newAocType] < orderByName[AocTypes.AU]? ++aocOrder[(AocTypes.IA, novelty)] : ++aocOrder[(AocTypes.AU, novelty)];", - "\n await workspace.UpdateAsync( aocConfigs[step] with { AocType = newAocType, DataType = DataType.Optional, Order = order } );", - "\n }", - "\n }", - "\n else if(orderByName[newAocType] > orderByName[AocTypes.CRU] && orderByName[newAocType] < orderByName[AocTypes.WO])", - "\n {", - "\n var stepI = (AocTypes.EV, Novelties.I);", - "\n var orderI = orderByName[newAocType] < orderByName[AocTypes.EV]? ++aocOrder[(AocTypes.CRU, Novelties.I)] : ++aocOrder[(AocTypes.EV, Novelties.I)];", - "\n await workspace.UpdateAsync( aocConfigs[stepI] with { AocType = newAocType, DataType = DataType.Optional, Order = orderI } );", - "\n", - "\n var stepN = (AocTypes.EV, Novelties.N);", - "\n var orderN = orderByName[newAocType] < orderByName[AocTypes.EV]? ++aocOrder[(AocTypes.AU, Novelties.N)] : ++aocOrder[(AocTypes.EV, Novelties.N)];", - "\n await workspace.UpdateAsync( aocConfigs[stepN] with { AocType = newAocType, DataType = DataType.Optional, Order = orderN } );", - "\n }", - "\n else if(orderByName[newAocType] > orderByName[AocTypes.WO] && orderByName[newAocType] < orderByName[AocTypes.CL])", - "\n {", - "\n var step = (AocTypes.WO, Novelties.C);", - "\n await workspace.UpdateAsync( aocConfigs[step] with { AocType = newAocType, DataType = DataType.Optional, Order = ++aocOrder[step] } );", - "\n }", - "\n else", - "\n ApplicationMessage.Log(Error.AocTypePositionNotSupported);", - "\n };", - "\n", - "\n var aocConfigsFinal = await workspace.Query().ToArrayAsync();", - "\n if(aocConfigsFinal.GroupBy(x => x.Order).Any(x => x.Count() > 1))", - "\n ApplicationMessage.Log(Error.AocConfigurationOrderNotUnique);", - "\n", - "\n await workspace.CommitToTargetAsync(options.TargetDataSource);", - "\n return Activity.Finish().Merge(logConfig); ", - "\n});" - ], - "metadata": {}, - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "source": [ - "# Yield Curves ", - "\n", - "\n[Yield curves](../DataModel/DataStructure#yield-curve) are imported for a specific year, month, and scenario (not required if best estimate). This information is collected in the main section of the file. ", - "\n", - "\nYield curves to be imported are compared against those already present in the DataSource and only the new ones are stored." - ], - "metadata": {}, - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "source": [ - "Import.DefineFormat(ImportFormats.YieldCurve, async (options, dataSet) => {", - "\n Activity.Start();", - "\n var primaryArgs = GetArgsFromMain(dataSet) with {ImportFormat = ImportFormats.YieldCurve};", - "\n primaryArgs.ValidateArgsForPeriod(options.TargetDataSource);", - "\n if(ApplicationMessage.HasErrors()) return Activity.Finish();", - "\n var workspace = Workspace.CreateNew();", - "\n workspace.Initialize(x => x.FromSource(options.TargetDataSource).DisableInitialization().DisableInitialization());", - "\n", - "\n var committedYieldCurves = await options.TargetDataSource.Query().ToArrayAsync();", - "\n var hasNameColumn = dataSet.Tables[ImportFormats.YieldCurve].Columns.Any(x => x.ColumnName == nameof(YieldCurve.Name));", - "\n var importLog = await Import.FromDataSet(dataSet).WithType(", - "\n (dataset, datarow) => new YieldCurve {", - "\n Currency = datarow.Field(nameof(YieldCurve.Currency)),", - "\n Year = primaryArgs.Year,", - "\n Month = primaryArgs.Month, ", - "\n Scenario = primaryArgs.Scenario,", - "\n Values = datarow.Table.Columns.Where(c => c.ColumnName.StartsWith(nameof(YieldCurve.Values))).OrderBy(c => c.ColumnName.Length).ThenBy(c => c.ColumnName)", - "\n .Select(x => datarow.Field(x.ColumnName).CheckStringForExponentialAndConvertToDouble()).ToArray(),", - "\n Name = hasNameColumn ? datarow.Field(nameof(YieldCurve.Name)) : default(string)", - "\n }", - "\n ).WithTarget(workspace).ExecuteAsync(); ", - "\n", - "\n if(importLog.Errors.Any()) return Activity.Finish().Merge(importLog); ", - "\n var toCommitYieldCurves = (await workspace.Query().ToArrayAsync()).Except(committedYieldCurves, YieldCurveComparer.Instance());", - "\n if (!toCommitYieldCurves.Any()) {", - "\n ApplicationMessage.Log(Warning.VariablesAlreadyImported); ", - "\n return Activity.Finish().Merge(importLog);", - "\n }", - "\n", - "\n var allArgs = await GetAllArgsAsync(primaryArgs, options.TargetDataSource, ImportFormats.YieldCurve);", - "\n var updatedCurrencies = toCommitYieldCurves.Select(x => x.Currency).Distinct();", - "\n var dataNodesToUpdate = await workspace.Query().Where(x => updatedCurrencies.Contains(x.ContractualCurrency)).Select(x => x.SystemName).ToArrayAsync();", - "\n var workspaceToCompute = Workspace.CreateNew();", - "\n workspaceToCompute.Initialize(x => x.FromSource(options.TargetDataSource).DisableInitialization().DisableInitialization());", - "\n ", - "\n foreach (var args in allArgs) {", - "\n await CommitPartitionAsync(args, workspace, workspaceToCompute);", - "\n var targetPartition = (Guid)(await options.TargetDataSource.Partition.GetKeyForInstanceAsync(args));", - "\n var defaultPartition = (Guid)(await options.TargetDataSource.Partition.GetKeyForInstanceAsync(args with {Scenario = null}));", - "\n var previousPartition = (Guid)(await options.TargetDataSource.Partition.GetKeyForInstanceAsync(args with {Scenario = null, Year = args.Year-1, Month = 12}));", - "\n await options.TargetDataSource.Partition.SetAsync(null);", - "\n if(ApplicationMessage.HasErrors()) return Activity.Finish().Merge(importLog);", - "\n ", - "\n // Avoid starting the computation if no best estimate cash flow has ever been imported ", - "\n if(!(await options.TargetDataSource.Query().Where(x => x.Partition == defaultPartition).Take(1).ToArrayAsync()).Any() &&", - "\n !(await options.TargetDataSource.Query().Where(x => x.Partition == defaultPartition).Take(1).ToArrayAsync()).Any()) continue;", - "\n ", - "\n // Only nominals corresponding to the target data nodes are added to the workspace", - "\n var nominals = await options.TargetDataSource.Query().Where(x => dataNodesToUpdate.Contains(x.DataNode) && ", - "\n (x.Partition == targetPartition || x.Partition == defaultPartition || x.Partition == previousPartition)).ToArrayAsync();", - "\n if(nominals.Any()) await workspaceToCompute.UpdateAsync(nominals);", - "\n", - "\n", - "\n importLog = importLog.Merge(await ComputeAsync(args, workspace, workspaceToCompute, false));", - "\n if(importLog.Errors.Any()) return Activity.Finish().Merge(importLog);", - "\n }", - "\n", - "\n await workspaceToCompute.UpdateAsync(toCommitYieldCurves);", - "\n await workspaceToCompute.CommitToTargetAsync(options.TargetDataSource);", - "\n return Activity.Finish().Merge(importLog);", - "\n});" - ], - "metadata": {}, - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "source": [ - "# Data Nodes" - ], - "metadata": {}, - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "source": [ - "", - "\n", - "\n[Portfolios](../DataModel/DataStructure#portfolios) and [Group of Contracts](../DataModel/DataStructure#group-of-contracts) are imported in the same file. " - ], - "metadata": {}, - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "source": [ - "public async Task UploadDataNodesToWorkspaceAsync(IDataSet dataSet, IWorkspace workspace, IDataSource targetDataSource)", - "\n{", - "\n workspace.Reset(x => x.ResetInitializationRules().ResetCurrentPartitions());", - "\n workspace.Initialize(x => x.FromSource(targetDataSource)", - "\n .DisableInitialization()", - "\n .DisableInitialization()", - "\n .DisableInitialization()", - "\n .DisableInitialization());", - "\n ", - "\n Activity.Start();", - "\n var args = await GetArgsAndCommitPartitionAsync(dataSet, targetDataSource);", - "\n if(Activity.HasErrors()) return Activity.Finish();", - "\n ", - "\n var storage = new ParsingStorage(args, targetDataSource, workspace);", - "\n await storage.InitializeAsync();", - "\n if(Activity.HasErrors()) return Activity.Finish();", - "\n ", - "\n var importLogPortfolios = await Import.FromDataSet(dataSet)", - "\n .WithType((dataset, datarow) => ", - "\n new InsurancePortfolio { SystemName = datarow.Field(nameof(DataNode.SystemName)),", - "\n DisplayName = datarow.Field(nameof(DataNode.DisplayName)),", - "\n Partition = storage.TargetPartitionByReportingNode.Id,", - "\n ContractualCurrency = datarow.Field(nameof(DataNode.ContractualCurrency)),", - "\n FunctionalCurrency = storage.ReportingNode.Currency,", - "\n LineOfBusiness = datarow.Field(nameof(DataNode.LineOfBusiness)),", - "\n ValuationApproach = datarow.Field(nameof(DataNode.ValuationApproach)),", - "\n OciType = datarow.Field(nameof(DataNode.OciType)) })", - "\n .WithType((dataset, datarow) =>", - "\n new ReinsurancePortfolio { SystemName = datarow.Field(nameof(DataNode.SystemName)),", - "\n DisplayName = datarow.Field(nameof(DataNode.DisplayName)),", - "\n Partition = storage.TargetPartitionByReportingNode.Id,", - "\n ContractualCurrency = datarow.Field(nameof(DataNode.ContractualCurrency)),", - "\n FunctionalCurrency = storage.ReportingNode.Currency,", - "\n LineOfBusiness = datarow.Field(nameof(DataNode.LineOfBusiness)),", - "\n ValuationApproach = datarow.Field(nameof(DataNode.ValuationApproach)),", - "\n OciType = datarow.Field(nameof(DataNode.OciType)) })", - "\n .WithTarget(workspace)", - "\n .ExecuteAsync();", - "\n ", - "\n var portfolios = await workspace.Query().ToDictionaryAsync(x => x.SystemName);", - "\n", - "\n var yieldCurveColumnGroupOfInsuranceContract = dataSet.Tables.Contains(nameof(GroupOfInsuranceContract)) && dataSet.Tables[nameof(GroupOfInsuranceContract)].Columns.Any(x => x.ColumnName == nameof(GroupOfInsuranceContract.YieldCurveName));", - "\n var yieldCurveColumnGroupOfReinsuranceContract = dataSet.Tables.Contains(nameof(GroupOfReinsuranceContract)) && dataSet.Tables[nameof(GroupOfReinsuranceContract)].Columns.Any(x => x.ColumnName == nameof(GroupOfReinsuranceContract.YieldCurveName));", - "\n", - "\n var importLogGroupOfContracts = await Import.FromDataSet(dataSet)", - "\n .WithType((dataset, datarow) => {", - "\n var gicSystemName = datarow.Field(nameof(DataNode.SystemName));", - "\n var pf = datarow.Field(nameof(InsurancePortfolio));", - "\n if(!portfolios.TryGetValue(pf, out var portfolioData)) {", - "\n ApplicationMessage.Log(Error.PortfolioGicNotFound, pf, gicSystemName);", - "\n return null;", - "\n }", - "\n var gic = new GroupOfInsuranceContract { SystemName = gicSystemName,", - "\n DisplayName = datarow.Field(nameof(DataNode.DisplayName)),", - "\n Partition = storage.TargetPartitionByReportingNode.Id,", - "\n ContractualCurrency = portfolioData.ContractualCurrency,", - "\n FunctionalCurrency = portfolioData.FunctionalCurrency,", - "\n LineOfBusiness = portfolioData.LineOfBusiness,", - "\n ValuationApproach = portfolioData.ValuationApproach,", - "\n OciType = portfolioData.OciType,", - "\n AnnualCohort = Convert.ToInt32(datarow.Field(nameof(GroupOfContract.AnnualCohort))),", - "\n LiabilityType = datarow.Field(nameof(GroupOfContract.LiabilityType)),", - "\n Profitability = datarow.Field(nameof(GroupOfContract.Profitability)),", - "\n Portfolio = pf,", - "\n YieldCurveName = yieldCurveColumnGroupOfInsuranceContract", - "\n ? datarow.Field(nameof(GroupOfContract.YieldCurveName)) ", - "\n : (string)null };", - "\n return ExtendGroupOfContract(gic, datarow);", - "\n })", - "\n .WithType((dataset, datarow) => {", - "\n var gricSystemName = datarow.Field(nameof(DataNode.SystemName));", - "\n var pf = datarow.Field(nameof(ReinsurancePortfolio));", - "\n if(!portfolios.TryGetValue(pf, out var portfolioData)) {", - "\n ApplicationMessage.Log(Error.PortfolioGicNotFound, pf, gricSystemName);", - "\n return null;", - "\n }", - "\n var gric = new GroupOfReinsuranceContract { SystemName = gricSystemName,", - "\n DisplayName = datarow.Field(nameof(DataNode.DisplayName)),", - "\n Partition = storage.TargetPartitionByReportingNode.Id,", - "\n ContractualCurrency = portfolioData.ContractualCurrency,", - "\n FunctionalCurrency = portfolioData.FunctionalCurrency,", - "\n LineOfBusiness = portfolioData.LineOfBusiness,", - "\n ValuationApproach = portfolioData.ValuationApproach,", - "\n OciType = portfolioData.OciType,", - "\n AnnualCohort = Convert.ToInt32(datarow.Field(nameof(GroupOfContract.AnnualCohort))),", - "\n LiabilityType = datarow.Field(nameof(GroupOfContract.LiabilityType)),", - "\n Profitability = datarow.Field(nameof(GroupOfContract.Profitability)),", - "\n Portfolio = pf,", - "\n Partner = datarow.Field(nameof(GroupOfContract.Partner)),", - "\n YieldCurveName = yieldCurveColumnGroupOfReinsuranceContract", - "\n ? datarow.Field(nameof(GroupOfContract.YieldCurveName)) ", - "\n : (string)null };", - "\n return ExtendGroupOfContract(gric, datarow);", - "\n })", - "\n .WithTarget(workspace)", - "\n .ExecuteAsync();", - "\n ", - "\n return Activity.Finish().Merge(importLogPortfolios).Merge(importLogGroupOfContracts);", - "\n}" - ], - "metadata": {}, - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "source": [ - "Import.DefineFormat(ImportFormats.DataNode, async (options, dataSet) => {", - "\n var workspace = Workspace.CreateNew();", - "\n var log = await UploadDataNodesToWorkspaceAsync(dataSet, workspace, options.TargetDataSource);", - "\n var partition = (Guid)workspace.Partition.GetCurrent(nameof(PartitionByReportingNode));", - "\n await workspace.CommitToAsync(options.TargetDataSource, partition);", - "\n await workspace.CommitToAsync(options.TargetDataSource, partition);", - "\n await workspace.CommitToAsync(options.TargetDataSource, partition);", - "\n await workspace.CommitToAsync(options.TargetDataSource, partition);", - "\n return log;", - "\n});" - ], - "metadata": {}, - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "source": [ - "# Data Node State" - ], - "metadata": {}, - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "source": [ - "public async Task UploadDataNodeStateToWorkspaceAsync(IDataSet dataSet, IWorkspace workspace, IDataSource targetDataSource)", - "\n{", - "\n workspace.Reset(x => x.ResetInitializationRules().ResetCurrentPartitions());", - "\n workspace.Initialize(x => x.FromSource(targetDataSource)", - "\n .DisableInitialization()", - "\n .DisableInitialization()", - "\n .DisableInitialization());", - "\n Activity.Start();", - "\n var args = await GetArgsAndCommitPartitionAsync(dataSet, targetDataSource) with {ImportFormat = ImportFormats.DataNodeState};", - "\n if(Activity.HasErrors()) return Activity.Finish();", - "\n ", - "\n var storage = new ParsingStorage(args, targetDataSource, workspace);", - "\n await storage.InitializeAsync();", - "\n if(Activity.HasErrors()) return Activity.Finish();", - "\n", - "\n var importLog = await Import.FromDataSet(dataSet).WithType(", - "\n (dataset, datarow) => new DataNodeState {", - "\n DataNode = datarow.Field(nameof(DataNodeState.DataNode)),", - "\n State = (State)Enum.Parse(typeof(State), datarow.Field(nameof(DataNodeState.State))),", - "\n Year = args.Year,", - "\n Month = args.Month,", - "\n Partition = storage.TargetPartitionByReportingNode.Id", - "\n }", - "\n ).WithTarget(workspace).ExecuteAsync();", - "\n", - "\n await workspace.ValidateDataNodeStatesAsync(storage.DataNodeDataBySystemName);", - "\n return Activity.Finish().Merge(importLog);", - "\n}" - ], - "metadata": {}, - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "source": [ - "Import.DefineFormat(ImportFormats.DataNodeState, async (options, dataSet) => {", - "\n var workspace = Workspace.CreateNew();", - "\n var log = await UploadDataNodeStateToWorkspaceAsync(dataSet, workspace, options.TargetDataSource);", - "\n await workspace.CommitToAsync(options.TargetDataSource, (Guid)workspace.Partition.GetCurrent(nameof(PartitionByReportingNode)), snapshot: false); ", - "\n return log;", - "\n});" - ], - "metadata": {}, - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "source": [ - "# Data Node Parameters" - ], - "metadata": {}, - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "source": [ - "public async Task UploadDataNodeParameterToWorkspaceAsync(IDataSet dataSet, ImportArgs args, IWorkspace workspace, IDataSource targetDataSource)", - "\n{", - "\n Activity.Start();", - "\n var storage = new ParsingStorage(args, targetDataSource, workspace);", - "\n await storage.InitializeAsync();", - "\n if(Activity.HasErrors()) return Activity.Finish();", - "\n var singleDataNode = new List();", - "\n var interDataNode = new List<(string,string)>();", - "\n", - "\n var hasCashFlowPeriodicityColumn = dataSet.Tables[nameof(SingleDataNodeParameter)].Columns.Any(x => x.ColumnName == nameof(SingleDataNodeParameter.CashFlowPeriodicity));", - "\n var hasInterpolationMethodColumn = dataSet.Tables[nameof(SingleDataNodeParameter)].Columns.Any(x => x.ColumnName == nameof(SingleDataNodeParameter.InterpolationMethod));", - "\n", - "\n var importLog = await Import.FromDataSet(dataSet)", - "\n .WithType( (dataset, datarow) => {", - "\n", - "\n //read and validate DataNodes", - "\n var dataNode = datarow.Field(nameof(DataNode));", - "\n if(!storage.IsValidDataNode(dataNode)) { ApplicationMessage.Log(Error.InvalidDataNode, dataNode); return null; }", - "\n", - "\n //check for duplicates", - "\n if(singleDataNode.Contains(dataNode)) { ApplicationMessage.Log(Error.DuplicateSingleDataNode, dataNode); return null; }", - "\n singleDataNode.Add(dataNode);", - "\n ", - "\n CashFlowPeriodicity periodicity = default;", - "\n if (hasCashFlowPeriodicityColumn)", - "\n if ( Enum.TryParse(datarow.Field(nameof(SingleDataNodeParameter.CashFlowPeriodicity)), out CashFlowPeriodicity cfp))", - "\n periodicity = cfp;", - "\n else { ApplicationMessage.Log(Error.InvalidCashFlowPeriodicity, dataNode); return null; }", - "\n", - "\n InterpolationMethod interpolationMethod = default;", - "\n if(hasInterpolationMethodColumn)", - "\n {", - "\n var interpolationMethodInput = datarow.Field(nameof(SingleDataNodeParameter.InterpolationMethod));", - "\n if ( Enum.TryParse(interpolationMethodInput, out InterpolationMethod ipm)) ", - "\n interpolationMethod = ipm;", - "\n else if ( !(periodicity == (CashFlowPeriodicity)default && string.IsNullOrEmpty(interpolationMethodInput)) ) { ApplicationMessage.Log(Error.InvalidInterpolationMethod, dataNode); return null; }", - "\n }", - "\n", - "\n //Instantiate SingleDataNodeParameter", - "\n return new SingleDataNodeParameter {", - "\n Year = args.Year,", - "\n Month = args.Month,", - "\n Scenario = args.Scenario,", - "\n Partition = storage.TargetPartitionByReportingNode.Id,", - "\n DataNode = dataNode,", - "\n CashFlowPeriodicity = periodicity,", - "\n InterpolationMethod = interpolationMethod,", - "\n PremiumAllocation = (datarow.Field(nameof(SingleDataNodeParameter.PremiumAllocation)))", - "\n .ToString().CheckStringForExponentialAndConvertToDouble(),", - "\n };", - "\n })", - "\n .WithType( (dataset, datarow) => {", - "\n", - "\n //read and validate DataNodes", - "\n var dataNode = datarow.Field(nameof(InterDataNodeParameter.DataNode));", - "\n if(!storage.IsValidDataNode(dataNode)) { ApplicationMessage.Log(Error.InvalidDataNode, dataNode); return null; }", - "\n", - "\n var linkedDataNode = datarow.Field(nameof(InterDataNodeParameter.LinkedDataNode));", - "\n if(!storage.IsValidDataNode(linkedDataNode)) { ApplicationMessage.Log(Error.InvalidDataNode, linkedDataNode); return null; }", - "\n var dataNodes = new string[]{dataNode, linkedDataNode}.OrderBy(x => x).ToArray();", - "\n", - "\n //validate ReinsuranceGross Link", - "\n var isDn1Reinsurance = storage.IsDataNodeReinsurance(dataNodes[0]);", - "\n var isDn2Reinsurance = storage.IsDataNodeReinsurance(dataNodes[1]);", - "\n var isGrossReinsuranceLink = (isDn1Reinsurance && !isDn2Reinsurance) != (!isDn1Reinsurance && isDn2Reinsurance);", - "\n var reinsCov = (datarow.Field(nameof(InterDataNodeParameter.ReinsuranceCoverage)))", - "\n .ToString().CheckStringForExponentialAndConvertToDouble();", - "\n if(!isGrossReinsuranceLink && Math.Abs(reinsCov) > Precision )", - "\n ApplicationMessage.Log(Error.ReinsuranceCoverageDataNode, dataNodes[0], dataNodes[1]); // TODO: is this error or warning?", - "\n", - "\n //check for duplicates", - "\n if(interDataNode.Contains((dataNodes[0], dataNodes[1])) || interDataNode.Contains((dataNodes[1], dataNodes[0])))", - "\n ApplicationMessage.Log(Error.DuplicateInterDataNode, dataNodes[0], dataNodes[1]); // TODO: is this error or warning?", - "\n", - "\n interDataNode.Add((dataNodes[0], dataNodes[1])); ", - "\n //Instantiate InterDataNodeParameter", - "\n return new InterDataNodeParameter {", - "\n Year = args.Year,", - "\n Month = args.Month,", - "\n Scenario = args.Scenario,", - "\n Partition = storage.TargetPartitionByReportingNode.Id,", - "\n DataNode = dataNodes[0],", - "\n LinkedDataNode = dataNodes[1],", - "\n ReinsuranceCoverage = reinsCov,", - "\n };", - "\n })", - "\n .WithTarget(workspace)", - "\n .ExecuteAsync();", - "\n ", - "\n return Activity.Finish().Merge(importLog);", - "\n}" - ], - "metadata": {}, - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "source": [ - "Import.DefineFormat(ImportFormats.DataNodeParameter, async (options, dataSet) => {", - "\n Activity.Start();", - "\n var primaryArgs = GetArgsFromMain(dataSet) with {ImportFormat = ImportFormats.DataNodeParameter};", - "\n primaryArgs.ValidateArgsForPeriod(options.TargetDataSource);", - "\n if(ApplicationMessage.HasErrors()) return Activity.Finish();", - "\n var workspace = Workspace.CreateNew();", - "\n workspace.Initialize(x => x.FromSource(options.TargetDataSource).DisableInitialization().DisableInitialization());", - "\n", - "\n var committedParameters = await options.TargetDataSource.Query().ToArrayAsync();", - "\n var log = await UploadDataNodeParameterToWorkspaceAsync(dataSet, primaryArgs, workspace, options.TargetDataSource); ", - "\n", - "\n if(log.Errors.Any()) return Activity.Finish().Merge(log); ", - "\n var toCommitParameters = (await workspace.Query().ToArrayAsync()).Except(committedParameters, ParametersComparer.Instance());", - "\n if (!toCommitParameters.Any()) {", - "\n ApplicationMessage.Log(Warning.VariablesAlreadyImported); ", - "\n return Activity.Finish().Merge(log);", - "\n }", - "\n", - "\n var allArgs = await GetAllArgsAsync(primaryArgs, options.TargetDataSource, ImportFormats.DataNodeParameter);", - "\n var targetDataNodes = toCommitParameters.Select(x => x.DataNode)", - "\n .Concat(toCommitParameters.Where(x => x is InterDataNodeParameter).Select(x => ((InterDataNodeParameter)x).LinkedDataNode)).ToHashSet();", - "\n var workspaceToCompute = Workspace.CreateNew();", - "\n workspaceToCompute.Initialize(x => x.FromSource(options.TargetDataSource).DisableInitialization());", - "\n", - "\n foreach (var args in allArgs) { ", - "\n await CommitPartitionAsync(args, workspace, workspaceToCompute);", - "\n var targetPartition = (Guid)(await options.TargetDataSource.Partition.GetKeyForInstanceAsync(args));", - "\n var defaultPartition = (Guid)(await options.TargetDataSource.Partition.GetKeyForInstanceAsync(args with {Scenario = null}));", - "\n var previousPartition = (Guid)(await options.TargetDataSource.Partition.GetKeyForInstanceAsync(args with {Scenario = null, Year = args.Year-1, Month = 12}));", - "\n await options.TargetDataSource.Partition.SetAsync(null);", - "\n if(ApplicationMessage.HasErrors()) return Activity.Finish().Merge(log);", - "\n ", - "\n // Avoid starting the computation if no best estimate cash flow and actuals has ever been imported ", - "\n if(!(await options.TargetDataSource.Query().Where(x => x.Partition == defaultPartition).Take(1).ToArrayAsync()).Any() &&", - "\n !(await options.TargetDataSource.Query().Where(x => x.Partition == defaultPartition).Take(1).ToArrayAsync()).Any()) continue;", - "\n", - "\n // Only nominals corresponding to the target data nodes are added to the workspace", - "\n var nominals = await options.TargetDataSource.Query().Where(x => targetDataNodes.Contains(x.DataNode) && ", - "\n (x.Partition == targetPartition || x.Partition == defaultPartition || x.Partition == previousPartition)).ToArrayAsync();", - "\n if(nominals.Any()) await workspaceToCompute.UpdateAsync(nominals);", - "\n", - "\n log = log.Merge(await ComputeAsync(args, workspace, workspaceToCompute, false));", - "\n if(log.Errors.Any()) return Activity.Finish().Merge(log);", - "\n }", - "\n", - "\n await workspaceToCompute.UpdateAsync(toCommitParameters);", - "\n await workspaceToCompute.CommitToTargetAsync(options.TargetDataSource);", - "\n return Activity.Finish().Merge(log);", - "\n});" - ], - "metadata": {}, - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "source": [ - "# Cashflows", - "\n", - "\nVariables are created upon import of Cash flow file.", - "\nCash flows are firstly mapped into [RawVariables](../DataModel/DataStructure#raw-variables). These are then used as input for the calculation of the IFRS 17 business logic which computes [IfrsVariables](../DataModel/DataStructure#ifrs-variable).", - "\n", - "\nSome computed variables depend on both cash flow and actual input, requiring recalculation at each new import.", - "\nIn order to improve performance and maximize computational efficiency, we only (re)compute the set of variables that are expected to change given the underlying business logic." - ], - "metadata": {}, - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "source": [ - "public async Task ParseCashflowsToWorkspaceAsync(IDataSet dataSet, ImportArgs args, IWorkspace workspace, IDataSource targetDataSource)", - "\n{", - "\n workspace.Reset(x => x.ResetInitializationRules().ResetCurrentPartitions());", - "\n workspace.Initialize(x => x.FromSource(targetDataSource).DisableInitialization().DisableInitialization());", - "\n ", - "\n Activity.Start();", - "\n var parsingStorage = new ParsingStorage(args, targetDataSource, workspace);", - "\n await parsingStorage.InitializeAsync();", - "\n if(Activity.HasErrors()) return Activity.Finish();", - "\n ", - "\n var hasAccidentYearColumn = dataSet.Tables[ImportFormats.Cashflow].Columns.Any(x => x.ColumnName == nameof(RawVariable.AccidentYear));", - "\n ", - "\n var importLog = await Import.FromDataSet(dataSet)", - "\n .WithType ( (dataset, datarow) => {", - "\n var aocType = datarow.Field(nameof(RawVariable.AocType));", - "\n var novelty = datarow.Field(nameof(RawVariable.Novelty));", - "\n var dataNode = datarow.Field(nameof(DataNode));", - "\n ", - "\n if(!parsingStorage.DataNodeDataBySystemName.TryGetValue(dataNode, out var dataNodeData)) {", - "\n ApplicationMessage.Log(Error.InvalidDataNode, dataNode);", - "\n return null;", - "\n }", - "\n ", - "\n // Error if AocType is not present in the mapping", - "\n if(!parsingStorage.AocTypeMap.Contains(new AocStep(aocType, novelty))) {", - "\n ApplicationMessage.Log(Error.AocTypeMapNotFound, aocType, novelty);", - "\n return null;", - "\n }", - "\n ", - "\n // Filter out cash flows for DataNode that were created in the past and are still active and come with AocType = BOPI", - "\n if(dataNodeData.Year < args.Year && aocType == AocTypes.BOP && novelty == Novelties.I) {", - "\n ApplicationMessage.Log(Warning.ActiveDataNodeWithCashflowBOPI, dataNode);", - "\n return null;", - "\n }", - "\n", - "\n (string AmountType, string EstimateType) valueType = datarow.ParseAmountAndEstimateType(ImportFormats.Cashflow, parsingStorage.DimensionsWithExternalId, parsingStorage.EstimateType, parsingStorage.AmountType);", - "\n var values = datarow.Table.Columns.Where(c => c.ColumnName.StartsWith(nameof(RawVariable.Values))).OrderBy(c => c.ColumnName.Length).ThenBy(c => c.ColumnName)", - "\n .Select(x => datarow.Field(x.ColumnName).CheckStringForExponentialAndConvertToDouble()).ToArray();", - "\n ", - "\n // Filter out empty raw variables for AocStep \\not\\in MandatoryAocSteps", - "\n if(args.Scenario == null) {", - "\n values = values.Prune();", - "\n if(values.Length == 0 && !parsingStorage.MandatoryAocSteps.Contains(new AocStep(aocType, novelty))) return null;", - "\n }", - "\n ", - "\n var item = new RawVariable {", - "\n DataNode = dataNode,", - "\n AocType = aocType,", - "\n Novelty = novelty,", - "\n AmountType = valueType.AmountType,", - "\n EstimateType = valueType.EstimateType,", - "\n AccidentYear = hasAccidentYearColumn && Int32.TryParse((datarow.Field(nameof(RawVariable.AccidentYear))), out var accidentYear)", - "\n ? accidentYear", - "\n : (int?)null,", - "\n Partition = parsingStorage.TargetPartitionByReportingNodeAndPeriod.Id,", - "\n Values = Multiply(GetSign(ImportFormats.Cashflow, (aocType, valueType.AmountType, valueType.EstimateType, dataNodeData.IsReinsurance), parsingStorage.HierarchyCache), values)", - "\n .Interpolate(parsingStorage.GetCashFlowPeriodicity(dataNode), parsingStorage.GetInterpolationMethod(dataNode))", - "\n };", - "\n return item;", - "\n }, ImportFormats.Cashflow", - "\n ).WithTarget(workspace).ExecuteAsync();", - "\n ", - "\n await workspace.ValidateForMandatoryAocSteps(dataSet, parsingStorage.MandatoryAocSteps);", - "\n await workspace.ValidateForDataNodeStateActiveAsync(parsingStorage.DataNodeDataBySystemName);", - "\n return Activity.Finish().Merge(importLog);", - "\n}" - ], - "metadata": {}, - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "source": [ - "Import.DefineFormat(ImportFormats.Cashflow, async (options, dataSet) => {", - "\n Activity.Start();", - "\n var primaryArgs = await GetArgsAndCommitPartitionAsync(dataSet, options.TargetDataSource) with {ImportFormat = ImportFormats.Cashflow};", - "\n if(Activity.HasErrors()) return Activity.Finish();", - "\n", - "\n var allArgs = await GetAllArgsAsync(primaryArgs, options.TargetDataSource, ImportFormats.Cashflow);", - "\n await DataNodeFactoryAsync(dataSet, ImportFormats.Cashflow, primaryArgs, options.TargetDataSource);", - "\n if(Activity.HasErrors()) return Activity.Finish();", - "\n ", - "\n var workspace = Workspace.CreateNew();", - "\n var log = await ParseCashflowsToWorkspaceAsync(dataSet, primaryArgs, workspace, options.TargetDataSource);", - "\n if(log.Errors.Any()) return Activity.Finish().Merge(log);", - "\n", - "\n var workspaceToCompute = Workspace.CreateNew();", - "\n workspaceToCompute.Initialize(x => x.FromSource(options.TargetDataSource));", - "\n if (Debug.Enable) ", - "\n {", - "\n if(primaryArgs.Scenario == null) ", - "\n await workspace.DeleteAsync(await workspace.Query().Where(rv => rv.Values.Sum(x => Math.Abs(x)) < Precision).ToArrayAsync());", - "\n var partition = (Guid)(await workspace.Partition.GetKeyForInstanceAsync(primaryArgs));", - "\n await workspace.CommitToAsync(workspaceToCompute, partition, snapshot : true);", - "\n } ", - "\n else ", - "\n foreach (var args in allArgs) {", - "\n log = log.Merge(await ComputeAsync(args, workspace, workspaceToCompute, args == primaryArgs));", - "\n if(log.Errors.Any()) return Activity.Finish().Merge(log);", - "\n }", - "\n", - "\n", - "\n await workspaceToCompute.CommitToTargetAsync(options.TargetDataSource);", - "\n return Activity.Finish().Merge(log); ", - "\n});" - ], - "metadata": {}, - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "source": [ - "# Actuals", - "\n", - "\nVariables are created upon import of Actual file.", - "\nActuals are directly mapped into [IfrsVariables](../DataModel/DataStructure#ifrs-variable).", - "\n", - "\n", - "\n", - "\nSome computed variables depend on both cash flow and actual input, requiring recalculation at each new import.", - "\nIn order to improve performance and maximize computational efficiency, we only (re)compute the set of variables that are expected to change given the underlying business logic." - ], - "metadata": {}, - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "source": [ - "public async Task ParseActualsToWorkspaceAsync(IDataSet dataSet, ImportArgs args, IWorkspace workspace, IDataSource targetDataSource)", - "\n{", - "\n workspace.Reset(x => x.ResetInitializationRules().ResetCurrentPartitions());", - "\n workspace.Initialize(x => x.FromSource(targetDataSource)", - "\n .DisableInitialization()", - "\n .DisableInitialization());", - "\n ", - "\n Activity.Start();", - "\n var parsingStorage = new ParsingStorage(args, targetDataSource, workspace);", - "\n await parsingStorage.InitializeAsync();", - "\n if(Activity.HasErrors()) return Activity.Finish();", - "\n", - "\n var importLog = await Import.FromDataSet(dataSet)", - "\n .WithType ( (dataset, datarow) => {", - "\n var dataNode = datarow.Field(nameof(DataNode));", - "\n if(!parsingStorage.DataNodeDataBySystemName.TryGetValue(dataNode, out var dataNodeData)) {", - "\n ApplicationMessage.Log(Error.InvalidDataNode, dataNode);", - "\n return null;", - "\n }", - "\n ", - "\n (string AmountType, string EstimateType) valueType = datarow.ParseAmountAndEstimateType(ImportFormats.Actual, parsingStorage.DimensionsWithExternalId, parsingStorage.EstimateType, parsingStorage.AmountType);", - "\n var isStdActual = valueType.EstimateType == EstimateTypes.A;", - "\n", - "\n var aocType = datarow.Field(nameof(IfrsVariable.AocType));", - "\n if((!isStdActual && aocType != AocTypes.CF && aocType != AocTypes.WO) || (isStdActual && aocType != AocTypes.CF) ) {", - "\n ApplicationMessage.Log(Error.AocTypeNotValid, aocType);", - "\n return null;", - "\n }", - "\n", - "\n var item = new IfrsVariable {", - "\n DataNode = dataNode,", - "\n AocType = aocType,", - "\n Novelty = Novelties.C,", - "\n AccidentYear = Int32.TryParse((datarow.Field(nameof(IfrsVariable.AccidentYear))), out var tempAccYear)? tempAccYear : (int?)null,", - "\n AmountType = valueType.AmountType,", - "\n EstimateType = valueType.EstimateType,", - "\n Partition = parsingStorage.TargetPartitionByReportingNodeAndPeriod.Id,", - "\n Value = GetSign(ImportFormats.Actual, (aocType, valueType.AmountType, valueType.EstimateType, dataNodeData.IsReinsurance), parsingStorage.HierarchyCache) * datarow.Field(nameof(IfrsVariable.Value)).CheckStringForExponentialAndConvertToDouble()", - "\n };", - "\n return item;", - "\n }, ImportFormats.Actual", - "\n ).WithTarget(workspace).ExecuteAsync();", - "\n ", - "\n await workspace.ValidateForDataNodeStateActiveAsync(parsingStorage.DataNodeDataBySystemName);", - "\n return Activity.Finish().Merge(importLog);", - "\n}" - ], - "metadata": {}, - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "source": [ - "Import.DefineFormat(ImportFormats.Actual, async (options, dataSet) => {", - "\n Activity.Start();", - "\n var primaryArgs = await GetArgsAndCommitPartitionAsync(dataSet, options.TargetDataSource) with {ImportFormat = ImportFormats.Actual};", - "\n if(Activity.HasErrors()) return Activity.Finish();", - "\n", - "\n var allArgs = await GetAllArgsAsync(primaryArgs, options.TargetDataSource, ImportFormats.Actual);", - "\n await DataNodeFactoryAsync(dataSet, ImportFormats.Actual, primaryArgs, options.TargetDataSource);", - "\n if(Activity.HasErrors()) return Activity.Finish();", - "\n", - "\n var workspace = Workspace.CreateNew();", - "\n var log = await ParseActualsToWorkspaceAsync(dataSet, primaryArgs, workspace, options.TargetDataSource);", - "\n if(log.Errors.Any()) return Activity.Finish().Merge(log);", - "\n", - "\n var workspaceToCompute = Workspace.CreateNew();", - "\n workspaceToCompute.Initialize(x => x.FromSource(options.TargetDataSource));", - "\n foreach (var args in allArgs) {", - "\n log = log.Merge(await ComputeAsync(args, workspace, workspaceToCompute, false)); ", - "\n if(log.Errors.Any()) return Activity.Finish().Merge(log);", - "\n }", - "\n await workspaceToCompute.CommitToTargetAsync(options.TargetDataSource);", - "\n return Activity.Finish().Merge(log);", - "\n});" - ], - "metadata": {}, - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "source": [ - "# Simple Value", - "\n", - "\nSimple Value format imports [IfrsVariables](../DataModel/DataStructure#ifrs-variable) computed by an independent tool.", - "\nIn this case our IFRS 17 calculation is not applied and variables are stored in the Database for being consumed in reports with our powerful reporting tooling." - ], - "metadata": {}, - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "source": [ - "public async Task ParseSimpleValueToWorkspaceAsync(IDataSet dataSet, ImportArgs args, IWorkspace workspace, IDataSource targetDataSource)", - "\n{", - "\n workspace.Reset(x => x.ResetInitializationRules().ResetCurrentPartitions());", - "\n workspace.Initialize(x => x.FromSource(targetDataSource)", - "\n .DisableInitialization()", - "\n .DisableInitialization());", - "\n ", - "\n Activity.Start();", - "\n var importFormat = args.ImportFormat;", - "\n var parsingStorage = new ParsingStorage(args, targetDataSource, workspace);", - "\n await parsingStorage.InitializeAsync();", - "\n if(Activity.HasErrors()) return Activity.Finish(); ", - "\n", - "\n var importLog = await Import.FromDataSet(dataSet)", - "\n .WithType ( (dataset, datarow) => {", - "\n var dataNode = parsingStorage.ValidateDataNode(datarow.Field(nameof(DataNode)),importFormat);", - "\n var amountType = parsingStorage.ValidateAmountType(datarow.Field(nameof(IfrsVariable.AmountType)));", - "\n var estimateType = parsingStorage.ValidateEstimateType(datarow.Field(nameof(IfrsVariable.EstimateType)), dataNode); //TODO LIC/LRC dependence", - "\n var aocStep = importFormat == ImportFormats.SimpleValue ", - "\n ? parsingStorage.ValidateAocStep(new AocStep (datarow.Field(nameof(IfrsVariable.AocType)), ", - "\n datarow.Field(nameof(IfrsVariable.Novelty))))", - "\n : new AocStep(AocTypes.BOP, Novelties.I);", - "\n var economicBasis = importFormat == ImportFormats.SimpleValue ", - "\n ? datarow.Field(nameof(IfrsVariable.EconomicBasis)) ", - "\n : null;", - "\n ", - "\n parsingStorage.ValidateEstimateTypeAndAmountType(estimateType, amountType);", - "\n ", - "\n var iv = new IfrsVariable {", - "\n DataNode = dataNode,", - "\n AocType = aocStep.AocType,", - "\n Novelty = aocStep.Novelty,", - "\n AccidentYear = Int32.TryParse((datarow.Field(nameof(IfrsVariable.AccidentYear))), out var accidentYear) ? accidentYear : (int?)null,", - "\n AmountType = amountType,", - "\n EstimateType = estimateType,", - "\n EconomicBasis = economicBasis,", - "\n Partition = parsingStorage.TargetPartitionByReportingNodeAndPeriod.Id,", - "\n Value = GetSign(importFormat, (aocStep.AocType, amountType, estimateType, parsingStorage.IsDataNodeReinsurance(dataNode)), parsingStorage.HierarchyCache) * datarow.Field(nameof(IfrsVariable.Value)).CheckStringForExponentialAndConvertToDouble()", - "\n };", - "\n return iv;", - "\n }, importFormat // This should indicate the table name, not the input format", - "\n ).WithTarget(workspace).ExecuteAsync();", - "\n ", - "\n // Checking if there are inconsistencies in the TechnicalMarginEstimateTypes --> double entries in the steps where we expect to have unique values", - "\n var invalidVariables = await workspace.Query()", - "\n .Where(iv => parsingStorage.TechnicalMarginEstimateTypes.Contains(iv.EstimateType))", - "\n .Where(iv => iv.AocType == AocTypes.BOP || iv.AocType == AocTypes.EOP || iv.AocType == AocTypes.AM || iv.AocType == AocTypes.EA)", - "\n .GroupBy(iv => new {iv.DataNode, iv.AocType, iv.Novelty})", - "\n .Where(g => g.Count() > 1)", - "\n .Select(g => g.Key)", - "\n .ToArrayAsync();", - "\n ", - "\n foreach (var iv in invalidVariables)", - "\n ApplicationMessage.Log(Error.MultipleTechnicalMarginOpening, $\"{iv.DataNode},{iv.AocType},{iv.Novelty}\");", - "\n ", - "\n await workspace.ValidateForDataNodeStateActiveAsync(parsingStorage.DataNodeDataBySystemName);", - "\n return Activity.Finish().Merge(importLog);", - "\n}" - ], - "metadata": {}, - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "source": [ - "Import.DefineFormat(ImportFormats.SimpleValue, async (options, dataSet) => {", - "\n Activity.Start();", - "\n var args = await GetArgsAndCommitPartitionAsync(dataSet, options.TargetDataSource) with {ImportFormat = ImportFormats.SimpleValue};", - "\n if(Activity.HasErrors()) return Activity.Finish();", - "\n await DataNodeFactoryAsync(dataSet, ImportFormats.SimpleValue, args, options.TargetDataSource);", - "\n if(Activity.HasErrors()) return Activity.Finish();", - "\n", - "\n var workspace = Workspace.CreateNew();", - "\n var parsingLog = await ParseSimpleValueToWorkspaceAsync(dataSet, args, workspace, options.TargetDataSource);", - "\n if(parsingLog.Errors.Any()) return Activity.Finish().Merge(parsingLog);", - "\n", - "\n workspace.Query().Select(v => new {v.DataNode, v.AccidentYear}).Distinct();", - "\n", - "\n var targetDataNodes = workspace.Query().Select(v => v.DataNode).Distinct().ToArray();", - "\n await workspace.CommitToAsync(", - "\n options.TargetDataSource, (Guid)(await DataSource.Partition.GetKeyForInstanceAsync(args)),", - "\n snapshot : true, filter : x => targetDataNodes.Contains(x.DataNode));", - "\n return Activity.Finish().Merge(parsingLog);", - "\n});" - ], - "metadata": {}, - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "source": [ - "# Opening", - "\n", - "\nOpening format imports [IfrsVariables](../DataModel/DataStructure#ifrs-variable) for the first period transition. It allows to import in force opening values (AoC Step BOP and novelty I) for EstimateTypes C, L, LR, AA, OA, DA." - ], - "metadata": {}, - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "source": [ - "Import.DefineFormat(ImportFormats.Opening, async (options, dataSet) => {", - "\n Activity.Start();", - "\n var primaryArgs = await GetArgsAndCommitPartitionAsync(dataSet, options.TargetDataSource) with {ImportFormat = ImportFormats.Opening};", - "\n if(Activity.HasErrors()) return Activity.Finish();", - "\n", - "\n var allArgs = await GetAllArgsAsync(primaryArgs, options.TargetDataSource, ImportFormats.Opening);", - "\n await DataNodeFactoryAsync(dataSet, ImportFormats.Opening, primaryArgs, options.TargetDataSource);", - "\n if(Activity.HasErrors()) return Activity.Finish();", - "\n", - "\n var workspace = Workspace.CreateNew();", - "\n var log = await ParseSimpleValueToWorkspaceAsync(dataSet, primaryArgs, workspace, options.TargetDataSource);", - "\n if(log.Errors.Any()) return Activity.Finish().Merge(log);", - "\n", - "\n var workspaceToCompute = Workspace.CreateNew();", - "\n workspaceToCompute.Initialize(x => x.FromSource(options.TargetDataSource));", - "\n foreach (var args in allArgs) {", - "\n log = log.Merge(await ComputeAsync(args, workspace, workspaceToCompute, false)); ", - "\n if(log.Errors.Any()) return Activity.Finish().Merge(log);", - "\n }", - "\n await workspaceToCompute.CommitToTargetAsync(options.TargetDataSource, x => x.SnapshotMode());", - "\n return Activity.Finish().Merge(log);", - "\n", - "\n})" - ], - "metadata": {}, - "execution_count": 0, - "outputs": [] - } - ] -} \ No newline at end of file From f6ead65416cf0ea2ffce1b5f19a4944bb3490362 Mon Sep 17 00:00:00 2001 From: Davide Colleoni Date: Thu, 27 Apr 2023 17:07:50 +0200 Subject: [PATCH 22/25] some more --- ifrs17/DataModel/DataStructure.ipynb | 2 +- ifrs17/Import/ImportStorage.ipynb | 5 +++- ifrs17/Import/Importers.ipynb | 43 +++++++++------------------- ifrs17/Report/ReportStorage.ipynb | 4 +-- ifrs17/Utils/Queries.ipynb | 27 +++++++++-------- 5 files changed, 34 insertions(+), 47 deletions(-) diff --git a/ifrs17/DataModel/DataStructure.ipynb b/ifrs17/DataModel/DataStructure.ipynb index 7bdfc99c..652e186e 100644 --- a/ifrs17/DataModel/DataStructure.ipynb +++ b/ifrs17/DataModel/DataStructure.ipynb @@ -627,7 +627,7 @@ "\n", "\nWhen a file is imported for a certain Scenario, the calculation engine integrates the set of inputs taking the remaining from the best estimate scenario. In this way the user is not required to input all data for each Scenario calculation but only the file with the stressed input. This is achieved through our system of [Relaxed Queries](../Utils/Queries#relaxedqueries). The assumption here is that the Best Estimate scenario is the first to be imported, and the stressed scenarios follow. ", "\n", - "\nThe dependency that each scenario has to the Best Estimate scenario is considered in case re-import. In this case, one import automatically triggers calculation in several partitions allowing all dependant cases to be updated. ", + "\nThe dependency that each scenario has to the Best Estimate scenario is considered in case re-import. In this case, one import automatically triggers calculation in several partitions allowing all dependant cases to be updated. For more details on how the calculation is performed refer to [Calculate IFRS Variables: for all scenarios](../Import/Importers#calculate-ifrs-variables-for-all-scenarios).", "\nThe only exception to this concerns dependency in time. When figures for periods $P$ and $P+1$ are imported for all scenarios and a new input for Best Estimate period $P$ is provided, only the scenarios for period $P$ are automatically updated. A manual update of period $P+1$ is then required to update the figures of $P+1$. We consider this case of restating previous periods particularly sensitive and defer to the user the resposability to keep the results up to date. ", "\n", "\n### Dependecy with Best Estimate scenario: across periods", diff --git a/ifrs17/Import/ImportStorage.ipynb b/ifrs17/Import/ImportStorage.ipynb index 79df39fe..7c276817 100644 --- a/ifrs17/Import/ImportStorage.ipynb +++ b/ifrs17/Import/ImportStorage.ipynb @@ -37,7 +37,10 @@ "\n- [EstimateType](../DataModel/DataStructure)", "\n- [DataNodes](../DataModel/DataStructure)", "\n", - "\nThe data collection is done using [Relaxed Queries](../Utils/Queries#relaxedqueries) giving to the Import Storage to retrive information from different [Partitions](#partitions).", + "\nThe data required for the calculations is retrieved by queries of [parameters](../Utils/Queries#current-and-previous-parameters) and [transactional data](../Utils/Queries#relaxedqueries) and stored in the storage to be available to the Import Scopes.", + "\nIn particular, for Best Estimate scenario the value belonging to the partition with Scenario Best Estimate is used for current and previous period value. For every scenario $S$ every query to parameters prioritize the data belonging to the partition with Scenario $S$ for the current period, if available, else the value belonging to the Best Estimate scenario is used. Previous period parameter from the Best Estimate scenario is always considered. ", + "\n", + "\nFor Scenario calculations, the transactional data (RawVariable and IfrsVariable) stored in the ImportStorage is the union of the variables provided for the Scenario and the variables provided for the Best Estimate. In case of intersection betweeen the two sets, priority is given to the values belonging to the scenario. This allows users to input only the variables for scenario that differ from the Best Estimate scenario. For example, if a scenario affects only the Risk Adjustment variables, the nominal cashflow can contain only the Risk Adjustment. If other variables are provided they are stored in the data source and used in calculation. Note that if a scenario requires a variable provided in the Best Estimate scenario to have 0 value, then it must be input in the file with 0 value. Omitting this variable from the input results in the calculation engine picking up the value provided in the Best Estiamte scenario. This applies to nominal cash flow, actuals, and opening. ", "\n", "\nSuch storage is then passed to calculations defined in the corresponding Import Scopes:", "\n", diff --git a/ifrs17/Import/Importers.ipynb b/ifrs17/Import/Importers.ipynb index e1cc9cc1..1050e749 100644 --- a/ifrs17/Import/Importers.ipynb +++ b/ifrs17/Import/Importers.ipynb @@ -36,30 +36,6 @@ "execution_count": 0, "outputs": [] }, - { - "cell_type": "markdown", - "source": [ - "", - "\n# Scenario implementation", - "\nThe following Importers support Scenario:", - "\n - Cashflow", - "\n - Actual", - "\n - Opening", - "\n - Yield Curve", - "\n - Data Node Parameter", - "\n ", - "\nImporting Scenario data trigger calculation using [Relaxed Queries](../Utils/Queries#relaxedqueries), implying that the Scenario data depends on Best Estimate, which is supposed to be already imported. ", - "\n", - "\nRe-importing Scenario will trigger calculation for the Scenario Partition only.", - "\n", - "\nRe-importing Best Estimate will trigger calculations for the Best Estimate Partition and for all Scenario that depends on the mentioned Best Estimate, but only for the same Year and Month. Therefore, outdated Best Estimate and Scenario data may be taken into account for following Periods. Reimporting following Periods inputs will trigger the calculations needed to update.", - "\n", - "\nThe Calculation Engine inquires the Database for dependent Partitions in [GetAllArgsAsync](#allargs): the informations collected in each Args identify a specific Partition. Retriving the necessary data is done in the [Import Storage](./ImportStorage#import-storage), one for each Partitions to avoid data leakage." - ], - "metadata": {}, - "execution_count": 0, - "outputs": [] - }, { "cell_type": "markdown", "source": [ @@ -540,8 +516,7 @@ { "cell_type": "markdown", "source": [ - "", - "\n## Calculate IFRS Variables", + "## Calculate IFRS Variables: for all scenarios", "\n", "\nThe following methods are used in the all importers that compute [IfrsVariables](../DataModel/DataStructure#ifrs-variable):", "\n- Yield Curve,", @@ -550,10 +525,20 @@ "\n- Nominal Cashflow,", "\n- Actual,", "\n- Opening.", + "\n

", + "\n", + "\nGetAllArgsAsync retrieves the partitions or Args that require computation. This method is relevant in the re-calculation of Scenario as it keeps track of the dependencies between Best Estimate scenario and stressed scenarios. The partition to be recomputed are defined as the union of the so called **primary args** which is read from the main tab of the imported file, with the **secondary args** corresponding to all scenarios which depends on the imported data.", + "\n", + "\nIn the case of Yield Curve the **secondary args** correspond to all Reporting Nodes that use one of the currencies present in the imported file. Recomputed scenarios are all existing if the input is for Best Estimate, and only the specified scenario when it is present in the input. ", + "\nYear, and Month of the recomputed partitions correspond to the value input int the file. ", + "\n", + "\nFor all other import formats the **secondary args** correspond to all scenarios if the input is for Best Estimate, and only to the scenario provided in input if present. ", + "\nReportingNode, Year, and Month of the recomputed partitions correspond to the value input int the file. ", + "\n

", "\n", - "\nGetAllArgsAsync retrieves the partitions or Args that require computation. They are the union of the so called **primary args** which is read from the main tab of the imported file, with the **secondary args** corresponding to all scenarios which depends on the imported data.", + "\nComputeAsync triggers computations of the IfrsVariables for a given partition (identified by its ImportArgs) and stores the results in a disposable workspace. This then serves as DataSource in the calculation of the secondary partitions (identified by the secondary args). The calculations is performed through ImportScopes (one example is [present value](2ImportScope-PresentValue)) with the use of the [ImportStorage](ImportStorage). ", "\n", - "\nComputeAsync computes the IfrsVariables for a given partition (identified by its ImportArgs) and stores the results in a disposable workspace. This then serves as DataSource in the calculation of the secondary partitions (identified by the secondary args)." + "\nIn the case of Scenario calculation the [ImportStorage](ImportStorage) combines the inputs with all information present in the data source for that scenario. In case some information has not been provided a default fall-back logic retries the missing information from the Best Estimate scenario by applying relaxed queries for both [parameters](../Utils/Queries#current-and-previous-parameters) and [transactional data](../Utils/Queries#relaxedqueries). " ], "metadata": {}, "execution_count": 0, @@ -1557,7 +1542,7 @@ "# Simple Value", "\n", "\nSimple Value format imports [IfrsVariables](../DataModel/DataStructure#ifrs-variable) computed by an independent tool.", - "\nIn this case our IFRS 17 calculation is not applied and variables are stored in the Database for being consumed in reports with our powerful reporting tooling." + "\nIn this case our IFRS 17 calculation is not applied and variables are stored in the Database for being consumed in reports with our powerful reporting tooling. Because there is no computation for this import format, values for scenarios must be imported using the Simple Value format specifing in the main table the name of the Scenario to which the variable belongs to. Analogously to the other import formats only the variables that change values with respect to the Best Estimate scenario should be entered. Note that a value going from not 0 to 0 value is also change that should be input. " ], "metadata": {}, "execution_count": 0, diff --git a/ifrs17/Report/ReportStorage.ipynb b/ifrs17/Report/ReportStorage.ipynb index 62c4b43d..71f7fbb9 100644 --- a/ifrs17/Report/ReportStorage.ipynb +++ b/ifrs17/Report/ReportStorage.ipynb @@ -90,8 +90,8 @@ "\n public string FunctionalCurrency { get; init; }", "\n", "\n [NotAggregated]", - "\n [Dimension(typeof(int), nameof(Projection))]", - "\n public int Projection { get; init; }", + "\n [Dimension(typeof(ProjectionConfiguration), nameof(Projection))]", + "\n public string Projection { get; init; }", "\n", "\n [Dimension(typeof(LiabilityType))]", "\n public string LiabilityType { get; init; }", diff --git a/ifrs17/Utils/Queries.ipynb b/ifrs17/Utils/Queries.ipynb index 727d0a0c..d0ddc0c8 100644 --- a/ifrs17/Utils/Queries.ipynb +++ b/ifrs17/Utils/Queries.ipynb @@ -25,18 +25,6 @@ "execution_count": 0, "outputs": [] }, - { - "cell_type": "markdown", - "source": [ - "", - "\n# Relaxed queries", - "\n", - "\nThe queries here collected retrieve data from a sorce of data, defined querySource . Priority is given to data from the same [Partition](../DataModel/DataStructure#partitions), which may be BE or Scenario. If Scenario data are missing, the query returns to the Best Estimate Data for the same Period or the previous." - ], - "metadata": {}, - "execution_count": 0, - "outputs": [] - }, { "cell_type": "code", "source": [ @@ -88,7 +76,15 @@ { "cell_type": "markdown", "source": [ - "# Current and Previous Parameters" + "# Current and Previous Parameters", + "\n", + "\nFor every parameter with a notion of Year and Month the following query methods provide the calculation engine with the current period value and with the previous period value. ", + "\n", + "\nThe current period is defined by the main table of the input file which triggers calculation. If no data is present for this period the value considered as current period is the last value provided in time. ", + "\n", + "\nIn the Year-to-Date view the previous period corresponds to end of previous year provided in the main table. A similar relaxed mechanism is applied for this query. If the value for the previous period is present it is returned. If the value it is not present the last value provided in time prior to the end of previous year is returned. ", + "\n", + "\nIn the case of Scenario calculations which are not Best Estimate, the data provided by the query for the current period belongs to the selected Scenario. If no data is available for the selected scenario (not even belonging to periods prior the current period), the previous period value for Best Estimate scenario is returned. In the case of the previous period value for a partition with Scenario not Best Estimate, the Best Estimate scenario for previous period is returned. In case this is not present in the data source, the current value for Best Estimate scenario is returned. " ], "metadata": {}, "execution_count": 0, @@ -435,7 +431,10 @@ { "cell_type": "markdown", "source": [ - "# Data Variables" + "# Data Variables", + "\n", + "\nThe following methods query the data source for RawVariable and IfrsVariable. ", + "\nWhen Scenario is not Best Estimate, the result of the query to the partition with the desired Scenario is provided if not empty. In the case this set is empty the result of the query to the Best Estimate partition is provided instead. " ], "metadata": {}, "execution_count": 0, From 9eb4a31a868f95b51415a0f457472b84fc82c9b8 Mon Sep 17 00:00:00 2001 From: Davide Colleoni Date: Thu, 27 Apr 2023 17:30:26 +0200 Subject: [PATCH 23/25] more --- ifrs17/OverviewCalculationEngine.ipynb | 2 +- ifrs17/Report/ReportStorage.ipynb | 17 +---------------- 2 files changed, 2 insertions(+), 17 deletions(-) diff --git a/ifrs17/OverviewCalculationEngine.ipynb b/ifrs17/OverviewCalculationEngine.ipynb index 4d6f7489..46e44c4c 100644 --- a/ifrs17/OverviewCalculationEngine.ipynb +++ b/ifrs17/OverviewCalculationEngine.ipynb @@ -211,7 +211,7 @@ "\n- Timing consideration for Effective Actuals (Payables and Receivables mapped into Paid in Advance or Overdue Actuals)", "\n- Credit Default Risk for Reinsurance", "\n- Financial Performance (FP) and Other Comprehensive Income (OCI)", - "\n- [Scenario](./DataModel/DataStructure#scenario) and [Sensitivity analysis](./Report/ReportStorage#sensitivity)", + "\n- Scenario and Sensitivity analysis", "\n- Run-off Projections (coming soon)", "\n", "\nFor more information on the latest developments, please refer to our [GitHub](https://github.com/Systemorph/IFRS17CalculationEngine) project page. From there, you can get to know about future releases, place requests, track the current work and report issues.", diff --git a/ifrs17/Report/ReportStorage.ipynb b/ifrs17/Report/ReportStorage.ipynb index 71f7fbb9..21d8424e 100644 --- a/ifrs17/Report/ReportStorage.ipynb +++ b/ifrs17/Report/ReportStorage.ipynb @@ -41,21 +41,6 @@ "execution_count": 0, "outputs": [] }, - { - "cell_type": "markdown", - "source": [ - "", - "\n# Sensitivity Analysis", - "\n", - "\nSensitivity Analysis is the comparison of Best Estimate results with a Scenario ones. In IFRS17 Calculation Engine the comparison is done putting together the corresponding figures in the various Reports provided. The keyword to use is Report.Scenario, which can be set to four main values:", - "\n - null (default): the Report use only Best Estimate results", - "\n - all: the Report shows values of Best Estimate and all Scenarios side by side", - "\n - delta: the Report shows values of Best Estimate and the difference between it and each Scenario." - ], - "metadata": {}, - "execution_count": 0, - "outputs": [] - }, { "cell_type": "markdown", "source": [ @@ -256,7 +241,7 @@ "\n }", "\n", "\n public async Task> GetScenariosAsync(string scenario) => ", - "\n scenario == \"Delta\" || scenario == \"All\" ", + "\n scenario == Scenarios.Delta || scenario == Scenarios.All", "\n ? (await workspace.Query().Select(x => x.Scenario).ToArrayAsync()).ToHashSet()", "\n : scenario.RepeatOnce().ToHashSet();", "\n}" From de3b5032b49deb2c468a53609b5b9999082a6ea7 Mon Sep 17 00:00:00 2001 From: Davide Colleoni Date: Thu, 27 Apr 2023 23:23:49 +0200 Subject: [PATCH 24/25] refinement --- ifrs17-template/Report/Reports.ipynb | 7 +++++-- ifrs17/DataModel/DataStructure.ipynb | 17 ++++++++--------- ifrs17/Import/ImportStorage.ipynb | 6 +++--- ifrs17/Import/Importers.ipynb | 28 +++++++++++++++------------- ifrs17/Utils/Queries.ipynb | 2 +- 5 files changed, 32 insertions(+), 28 deletions(-) diff --git a/ifrs17-template/Report/Reports.ipynb b/ifrs17-template/Report/Reports.ipynb index b9dc29aa..58e9479a 100644 --- a/ifrs17-template/Report/Reports.ipynb +++ b/ifrs17-template/Report/Reports.ipynb @@ -84,7 +84,9 @@ "\nThe granularity of the reported figures can be modified by changing the Column Slices options.", "\nFor example one can add \"GroupOfContract\" to separate the contributions of the individual Group of Contracts.", "\n", - "\nAggregated values are displayed when the data has a finer granularity than the one selected by the report slice options." + "\nAggregated values are displayed when the data has a finer granularity than the one selected by the report slice options.", + "\n", + "\nThe Best Estimate scenario is selected by default. You can select other scenarios through the Scenario field. Possible options includes 'All' and 'Delta'. Select ColumnSlice Scenario if you chose one of these. " ], "metadata": {}, "execution_count": 0, @@ -97,7 +99,8 @@ "\npv.ReportingNode = \"CH\";", "\npv.ReportingPeriod = (2021, 3);", "\npv.CurrencyType = CurrencyType.Contractual;", - "\npv.ColumnSlices = new string[]{};//\"GroupOfContract\", \"AmountType\"", + "\npv.ColumnSlices = new string[]{\"Scenario\"};//\"GroupOfContract\", \"AmountType\", \"Scenario\"", + "\npv.Scenario = null; //\"All\";", "\npv.DataFilter = null; //new [] {(\"GroupOfContract\", \"DT1.2\"),(\"LiabilityType\", \"LIC\") };", "\n(await pv.ToReportAsync)" ], diff --git a/ifrs17/DataModel/DataStructure.ipynb b/ifrs17/DataModel/DataStructure.ipynb index 652e186e..8f8950cb 100644 --- a/ifrs17/DataModel/DataStructure.ipynb +++ b/ifrs17/DataModel/DataStructure.ipynb @@ -599,9 +599,9 @@ "source": [ "", "\n## Scenario", - "\nThe Scenario record holds the various Scenarios used for [Sensitivity analysis](../Report/ReportStorage#sensitivity). ", + "\nThe Scenario record holds the various Scenarios used for Sensitivity analysis. ", "\n", - "\nAny Scenario-case can be defined with this record by providing a SystemName and a DisaplyName. During data collection phase the Scenario column of the [main](../Import/Importers#parse-the-main-tab) section of the input can be populated with the SystemName of the desired Scenario. The default Scenario (i.e. the default one, with no-stress situations applied) is referred to as 'Best Estimate' and its identifier is a null string, allowing the input files to not specify any value under the column Scenario." + "\nAny Scenario-case can be defined with this record by providing a SystemName and a DisplayName. During data collection phase the Scenario column of the [main](../Import/Importers#parse-the-main-tab) table of the input can be populated with the SystemName of the desired Scenario. The default Scenario (i.e. the default one, with no-stress situations applied) is referred to as 'Best Estimate' and its identifier is a null string, allowing the input files to not specify any value under the column Scenario or to lack the column itself." ], "metadata": {}, "execution_count": 0, @@ -625,16 +625,17 @@ "\n", "\n### Dependecy with Best Estimate scenario: same period", "\n", - "\nWhen a file is imported for a certain Scenario, the calculation engine integrates the set of inputs taking the remaining from the best estimate scenario. In this way the user is not required to input all data for each Scenario calculation but only the file with the stressed input. This is achieved through our system of [Relaxed Queries](../Utils/Queries#relaxedqueries). The assumption here is that the Best Estimate scenario is the first to be imported, and the stressed scenarios follow. ", + "\nWhen a file is imported for a certain Scenario, the calculation engine integrates the set of inputs taking the remaining from the best estimate scenario. In this way the user is not required to input all data for each Scenario calculation but only the file with the stressed input. This is achieved through our system of [Relaxed Queries](../Utils/Queries). The assumption here is that the Best Estimate scenario is the first to be imported, and the stressed scenarios follow. ", "\n", - "\nThe dependency that each scenario has to the Best Estimate scenario is considered in case re-import. In this case, one import automatically triggers calculation in several partitions allowing all dependant cases to be updated. For more details on how the calculation is performed refer to [Calculate IFRS Variables: for all scenarios](../Import/Importers#calculate-ifrs-variables-for-all-scenarios).", - "\nThe only exception to this concerns dependency in time. When figures for periods $P$ and $P+1$ are imported for all scenarios and a new input for Best Estimate period $P$ is provided, only the scenarios for period $P$ are automatically updated. A manual update of period $P+1$ is then required to update the figures of $P+1$. We consider this case of restating previous periods particularly sensitive and defer to the user the resposability to keep the results up to date. ", + "\nThe dependency that each scenario has to the Best Estimate scenario is considered in case of re-import. In this case, one import automatically triggers calculation in several partitions allowing all dependant cases to be updated. For more details on how the calculation is performed refer to [Calculate IFRS Variables: for all scenarios](../Import/Importers#calculate-ifrs-variables-for-all-scenarios).", + "\n", + "\nThe only exception to this is applied to time dependency. When figures for periods $P$ and $P+1$ are imported for all scenarios and a new input for Best Estimate period $P$ is provided, only the scenarios for period $P$ are automatically updated. A manual update of period $P+1$ is then required to update the figures of $P+1$. We consider the case of restating previous periods particularly sensitive and defer to the user the resposability to ensure that all results are up to date. ", "\n", "\n### Dependecy with Best Estimate scenario: across periods", "\n", "\nWhen a stress scenario is imported for a period $P$ and a previous period $P-1$ is available, the End of Period values ($P-1$) of the Best Estimate scenario is considered as Beginning of Period ($P$) for the stressed and no-stress scenarios. This allows to:", "\n1. all scenarios must not be calculated every period,", - "\n2. new scenarios can be added in a production environment,", + "\n2. new scenarios can be added at any time in a production environment,", "\n3. in each period the scenario depends only on the perturbation provided in the period and is not applied on top of the previous period perturbation." ], "metadata": {}, @@ -1276,9 +1277,7 @@ { "cell_type": "markdown", "source": [ - "The partition PartitionByReportingNodeAndPeriod is a further partition of the IfrsPartition sets. In particular, it defines sets for the data pertaining to a certain [Reporting Node](#reporting-node), [Scenario](#scenario), year and month. The value of the Month is the last month of the reporting period to which the data belongs to. ", - "\n", - "\nThis Partition is the fundamental property used by the Calculation engine to recognise results belonging to different Scenario. It is used while importing new files to trigger calculations by the [Relaxed Queries](../Utils/Queries#relaxedqueries) and in the Report to perform [Sensitivity analysis](./Report/ReportStorage#sensitivity)." + "The partition PartitionByReportingNodeAndPeriod is a further partition of the IfrsPartition sets. In particular, it defines sets for the data pertaining to a certain [Reporting Node](#reporting-node), [Scenario](#scenario), year and month. The value of the Month is the last month of the reporting period to which the data belongs to. " ], "metadata": {}, "execution_count": 0, diff --git a/ifrs17/Import/ImportStorage.ipynb b/ifrs17/Import/ImportStorage.ipynb index 7c276817..c95bd8e4 100644 --- a/ifrs17/Import/ImportStorage.ipynb +++ b/ifrs17/Import/ImportStorage.ipynb @@ -37,10 +37,10 @@ "\n- [EstimateType](../DataModel/DataStructure)", "\n- [DataNodes](../DataModel/DataStructure)", "\n", - "\nThe data required for the calculations is retrieved by queries of [parameters](../Utils/Queries#current-and-previous-parameters) and [transactional data](../Utils/Queries#relaxedqueries) and stored in the storage to be available to the Import Scopes.", - "\nIn particular, for Best Estimate scenario the value belonging to the partition with Scenario Best Estimate is used for current and previous period value. For every scenario $S$ every query to parameters prioritize the data belonging to the partition with Scenario $S$ for the current period, if available, else the value belonging to the Best Estimate scenario is used. Previous period parameter from the Best Estimate scenario is always considered. ", + "\nThe data required for the calculations is retrieved by queries of [parameters](../Utils/Queries#current-and-previous-parameters) and [transactional data](../Utils/Queries#data-variables) and stored in the storage to be available for the Import Scopes.", + "\nIn particular, for Best Estimate scenario the value belonging to the partition with Scenario Best Estimate is used for current and previous period value. For every scenario $S$ every query to parameters prioritize the data belonging to the partition with Scenario $S$ for the current period, if not present the value belonging to the Best Estimate scenario is used. Previous period parameter is always retrieved from the Best Estimate scenario. ", "\n", - "\nFor Scenario calculations, the transactional data (RawVariable and IfrsVariable) stored in the ImportStorage is the union of the variables provided for the Scenario and the variables provided for the Best Estimate. In case of intersection betweeen the two sets, priority is given to the values belonging to the scenario. This allows users to input only the variables for scenario that differ from the Best Estimate scenario. For example, if a scenario affects only the Risk Adjustment variables, the nominal cashflow can contain only the Risk Adjustment. If other variables are provided they are stored in the data source and used in calculation. Note that if a scenario requires a variable provided in the Best Estimate scenario to have 0 value, then it must be input in the file with 0 value. Omitting this variable from the input results in the calculation engine picking up the value provided in the Best Estiamte scenario. This applies to nominal cash flow, actuals, and opening. ", + "\nFor Scenario calculations, the transactional data (RawVariable and IfrsVariable) stored in the ImportStorage is the union of the variables provided for the Scenario and the variables provided for the Best Estimate. In case of intersection betweeen the two sets, priority is given to the values belonging to the scenario. This allows users to input only the variables for scenario that differ from the Best Estimate scenario. For example, if a scenario affects only the Risk Adjustment variables, the nominal cashflow input might contain only the Risk Adjustment. If other variables are provided they are equally considered, stored in the data source and used in calculation. Note that if a scenario requires a variable provided under the Best Estimate scenario to have 0 value, then it must be input in the scenario file with 0 value. Omitting this variable from the input results in the calculation engine picking up the value provided in the Best Estimate scenario. This applies to nominal cash flow, actuals, and opening. ", "\n", "\nSuch storage is then passed to calculations defined in the corresponding Import Scopes:", "\n", diff --git a/ifrs17/Import/Importers.ipynb b/ifrs17/Import/Importers.ipynb index 1050e749..72f89742 100644 --- a/ifrs17/Import/Importers.ipynb +++ b/ifrs17/Import/Importers.ipynb @@ -326,17 +326,17 @@ "## Parse the Main Tab", "\n", "\nThe main table of our custom import formats contains the information which are required to identify the data partition depending on the ImportFormat.", - "\nThe columns contained in the main table for the different ImportFormat are:", + "\nThe columns contained in the main table for the different ImportFormats are:", "\n1. Yield Curve : Year, Month, Scenario;", "\n2. Data Node : ReportingNode;", "\n3. Nominal Cashflow, Actual, Opening, Simple Value, Data Node State, Data Node Parameter : ReportingNode, Year, Month, Scenario.", "\n", - "\nNote that the Scenario column is not available for Data Node. Data Node can be created regardeless of the Scenario and controlling whether they are active or not in a particular scenario can be achieved through the import of Data Node State (which allows the specification of a Scenario)", - "\nIn addition, the Scenario column (for the ImportFormats that expect it) is not required as its value can be left empty in the case of Best Estimate scenario (in this case the entire column can be missing). ", + "\nNote that the Scenario column is not available for Data Node. Data Node can be created regardless of the Scenario and controlling whether they are active or not in a particular scenario can be achieved through the import of Data Node State (which allows the specification of a Scenario).", + "\nIn addition, the Scenario column (for the ImportFormats that expect it) is not required, as its value can be left empty in the case of Best Estimate scenario (in this case the entire column can be missing). ", "\n", "\nAfter having parsed the main table, these information are temporarily stored in [Args](../DataModel/DataStructure#args) and used in the next methods.", "\n", - "\nGetArgsFromMain performes basic valiadations on the existance of the main tab. Then reads the reporting node, year, month, and scenario and return an ImportArgs with the results. If any of these information a default value is returned and will be validated in the following methods. " + "\nGetArgsFromMain performes basic validations on the existance of the main tab. Then reads the reporting node, year, month, and scenario and returns an ImportArgs with the results. If any of these information is missing a default value is returned and will be validated in the following methods. " ], "metadata": {}, "execution_count": 0, @@ -518,7 +518,7 @@ "source": [ "## Calculate IFRS Variables: for all scenarios", "\n", - "\nThe following methods are used in the all importers that compute [IfrsVariables](../DataModel/DataStructure#ifrs-variable):", + "\nThe following methods are used in all importers that compute [IfrsVariables](../DataModel/DataStructure#ifrs-variable):", "\n- Yield Curve,", "\n- Data Node State,", "\n- Data Node Parameter,", @@ -527,18 +527,20 @@ "\n- Opening.", "\n

", "\n", - "\nGetAllArgsAsync retrieves the partitions or Args that require computation. This method is relevant in the re-calculation of Scenario as it keeps track of the dependencies between Best Estimate scenario and stressed scenarios. The partition to be recomputed are defined as the union of the so called **primary args** which is read from the main tab of the imported file, with the **secondary args** corresponding to all scenarios which depends on the imported data.", + "\nGetAllArgsAsync retrieves the partitions or Args that require computation. This method is relevant to the re-calculation of Scenarios as it keeps track of the dependencies between Best Estimate scenario and stressed scenarios. The partition to be recomputed are defined as the union of the so called **primary args** which is read from the main table of the imported file, with the **secondary args** corresponding to all scenarios which depend on the imported data.", "\n", - "\nIn the case of Yield Curve the **secondary args** correspond to all Reporting Nodes that use one of the currencies present in the imported file. Recomputed scenarios are all existing if the input is for Best Estimate, and only the specified scenario when it is present in the input. ", - "\nYear, and Month of the recomputed partitions correspond to the value input int the file. ", + "\nIn the case of Yield Curve the **secondary args** correspond to the partition relative to all Reporting Nodes that use one of the currencies present in the imported file. Only the partitions relative to the specified scenario is recomputed when the input is for a specific scenario. Instead, all existing scenarios expect for those with a perturbed Yield Curve data explictly provided as input are recomputed if the input is for Best Estimate scenario. Year, and Month of the recomputed partitions match the value input in the main table of the imported file. ", "\n", - "\nFor all other import formats the **secondary args** correspond to all scenarios if the input is for Best Estimate, and only to the scenario provided in input if present. ", - "\nReportingNode, Year, and Month of the recomputed partitions correspond to the value input int the file. ", + "\nIn the case of Data Node Parameters and State only the **secondary args** correspond to the partition for the specified scenario if the input is for a specific scenario. When input for Best Estimate scenario is imported all scenarios that do not have perturbed DataNodeParameter or DataNodeState values are recomputed. ", + "\nReportingvNode, Year, and Month of the recomputed partitions correspond to the value input int the file.", + "\n", + "\nFor all other import formats the **secondary args** correspond to the partition for the specified scenario if the input is for a specific scenario and to all scenarios if the input is for Best Estimate scenario. ", + "\nReporting Node, Year, and Month of the recomputed partitions correspond to the value input int the file. ", "\n

", "\n", - "\nComputeAsync triggers computations of the IfrsVariables for a given partition (identified by its ImportArgs) and stores the results in a disposable workspace. This then serves as DataSource in the calculation of the secondary partitions (identified by the secondary args). The calculations is performed through ImportScopes (one example is [present value](2ImportScope-PresentValue)) with the use of the [ImportStorage](ImportStorage). ", + "\nComputeAsync triggers computations of the Ifrs Variables for a given partition (identified by its ImportArgs) and stores the results in a disposable workspace. This then serves as DataSource in the calculation of the secondary partitions (identified by the secondary args). The calculations is performed through ImportScopes (one example is [present value](2ImportScope-PresentValue)) with the use of the [ImportStorage](ImportStorage). ", "\n", - "\nIn the case of Scenario calculation the [ImportStorage](ImportStorage) combines the inputs with all information present in the data source for that scenario. In case some information has not been provided a default fall-back logic retries the missing information from the Best Estimate scenario by applying relaxed queries for both [parameters](../Utils/Queries#current-and-previous-parameters) and [transactional data](../Utils/Queries#relaxedqueries). " + "\nIn the case of Scenario calculation the [ImportStorage](ImportStorage) combines the inputs with all information present in the data source for that scenario. In case some information has not been provided for the specified scenario a default fall-back logic retrieves the missing information from the Best Estimate scenario by applying relaxed queries for both [parameters](../Utils/Queries#current-and-previous-parameters) and [transactional data](../Utils/Queries#relaxedqueries). " ], "metadata": {}, "execution_count": 0, @@ -1542,7 +1544,7 @@ "# Simple Value", "\n", "\nSimple Value format imports [IfrsVariables](../DataModel/DataStructure#ifrs-variable) computed by an independent tool.", - "\nIn this case our IFRS 17 calculation is not applied and variables are stored in the Database for being consumed in reports with our powerful reporting tooling. Because there is no computation for this import format, values for scenarios must be imported using the Simple Value format specifing in the main table the name of the Scenario to which the variable belongs to. Analogously to the other import formats only the variables that change values with respect to the Best Estimate scenario should be entered. Note that a value going from not 0 to 0 value is also change that should be input. " + "\nIn this case our IFRS 17 calculation is not applied and variables are stored in the Database for being consumed in reports with our powerful reporting tooling. Because there is no computation for this import format, values for scenarios must be imported using the Simple Value format specifing the Scenario in the main table. Analogously to the other import formats, only the variables that change values with respect to the Best Estimate scenario should be input. Note that a variable with a value different form zero in the Best Estimate scenario should be input with value zero for the scenario if it should not be shown in the report." ], "metadata": {}, "execution_count": 0, diff --git a/ifrs17/Utils/Queries.ipynb b/ifrs17/Utils/Queries.ipynb index d0ddc0c8..ec0643a4 100644 --- a/ifrs17/Utils/Queries.ipynb +++ b/ifrs17/Utils/Queries.ipynb @@ -434,7 +434,7 @@ "# Data Variables", "\n", "\nThe following methods query the data source for RawVariable and IfrsVariable. ", - "\nWhen Scenario is not Best Estimate, the result of the query to the partition with the desired Scenario is provided if not empty. In the case this set is empty the result of the query to the Best Estimate partition is provided instead. " + "\nWhen Scenario is not Best Estimate, the result of the query to the partition with the desired Scenario is provided if not empty. In the case this set is empty the result of the query for the Best Estimate scenario is provided instead. " ], "metadata": {}, "execution_count": 0, From 9aabe539616a62cbd1e8fe6fb9ce0fdafddcd6a8 Mon Sep 17 00:00:00 2001 From: Davide Colleoni Date: Thu, 27 Apr 2023 23:47:10 +0200 Subject: [PATCH 25/25] refinements --- ifrs17/DataModel/DataStructure.ipynb | 12 ++++++------ ifrs17/Import/ImportStorage.ipynb | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/ifrs17/DataModel/DataStructure.ipynb b/ifrs17/DataModel/DataStructure.ipynb index 8f8950cb..670d3565 100644 --- a/ifrs17/DataModel/DataStructure.ipynb +++ b/ifrs17/DataModel/DataStructure.ipynb @@ -621,20 +621,20 @@ "source": [ "### Scenario and inputs", "\n", - "\nUnder a particular Scenario, several data types can be imported (e.g. nominal cash flows, parameters, yield-curvers, etc.). However, we suggest to arrange a Scenario for every desired type of stress in order to facilitate the analysis of the results.", + "\nUnder a particular Scenario, several data types can be imported (e.g. nominal cash flows, parameters, yield-curvers, etc.). However, we suggest arranging a different Scenario for every desired type of stress to facilitate the analysis of the results.", "\n", "\n### Dependecy with Best Estimate scenario: same period", "\n", - "\nWhen a file is imported for a certain Scenario, the calculation engine integrates the set of inputs taking the remaining from the best estimate scenario. In this way the user is not required to input all data for each Scenario calculation but only the file with the stressed input. This is achieved through our system of [Relaxed Queries](../Utils/Queries). The assumption here is that the Best Estimate scenario is the first to be imported, and the stressed scenarios follow. ", + "\nWhen a file is imported for a specific Scenario, the calculation engine integrates the set of inputs taking the remaining from the best estimate scenario. In this way the user is not required to input again all data for each Scenario calculation but only the file with the stressed input. This is achieved through our system of [Relaxed Queries](../Utils/Queries). The assumption here is that the Best Estimate scenario is the first to be imported, and the stressed scenarios follow. ", "\n", - "\nThe dependency that each scenario has to the Best Estimate scenario is considered in case of re-import. In this case, one import automatically triggers calculation in several partitions allowing all dependant cases to be updated. For more details on how the calculation is performed refer to [Calculate IFRS Variables: for all scenarios](../Import/Importers#calculate-ifrs-variables-for-all-scenarios).", + "\nIn the case of re-import, the Engine considers the dependency between each scenario and the Best Estimate scenario. In this case, one import automatically triggers calculation in several partitions allowing all dependant cases to be updated. For more details on how the calculation is performed refer to [Calculate IFRS Variables: for all scenarios](../Import/Importers#calculate-ifrs-variables-for-all-scenarios).", "\n", - "\nThe only exception to this is applied to time dependency. When figures for periods $P$ and $P+1$ are imported for all scenarios and a new input for Best Estimate period $P$ is provided, only the scenarios for period $P$ are automatically updated. A manual update of period $P+1$ is then required to update the figures of $P+1$. We consider the case of restating previous periods particularly sensitive and defer to the user the resposability to ensure that all results are up to date. ", + "\nThe only exception to this is applied to time dependency. When the user imports figures for periods $P$ and $P+1$ are imported for all scenarios and a new input for Best Estimate period $P$ is provided, only the scenarios for period $P$ are automatically updated. A manual update of period $P+1$ is then required to update the figures of $P+1$. We consider the case of restating previous periods particularly sensitive and defer to the user the resposability to ensure that all results are up to date. ", "\n", "\n### Dependecy with Best Estimate scenario: across periods", "\n", - "\nWhen a stress scenario is imported for a period $P$ and a previous period $P-1$ is available, the End of Period values ($P-1$) of the Best Estimate scenario is considered as Beginning of Period ($P$) for the stressed and no-stress scenarios. This allows to:", - "\n1. all scenarios must not be calculated every period,", + "\nWhen a stress scenario is imported for a period $P$ and a previous period $P-1$ is available, the End of Period values ($P-1$) of the Best Estimate scenario is considered as Beginning of Period ($P$) for the stressed and no-stress scenarios. This implies that:", + "\n1. scenarios can be occasionally not be calculated without impacting their calculation in future periods,", "\n2. new scenarios can be added at any time in a production environment,", "\n3. in each period the scenario depends only on the perturbation provided in the period and is not applied on top of the previous period perturbation." ], diff --git a/ifrs17/Import/ImportStorage.ipynb b/ifrs17/Import/ImportStorage.ipynb index c95bd8e4..f566d2c0 100644 --- a/ifrs17/Import/ImportStorage.ipynb +++ b/ifrs17/Import/ImportStorage.ipynb @@ -38,7 +38,7 @@ "\n- [DataNodes](../DataModel/DataStructure)", "\n", "\nThe data required for the calculations is retrieved by queries of [parameters](../Utils/Queries#current-and-previous-parameters) and [transactional data](../Utils/Queries#data-variables) and stored in the storage to be available for the Import Scopes.", - "\nIn particular, for Best Estimate scenario the value belonging to the partition with Scenario Best Estimate is used for current and previous period value. For every scenario $S$ every query to parameters prioritize the data belonging to the partition with Scenario $S$ for the current period, if not present the value belonging to the Best Estimate scenario is used. Previous period parameter is always retrieved from the Best Estimate scenario. ", + "\nThe calculation of a Best Estimate scenario is perfomerd using parameters from the data partition with Scenario Best Estimate (for both current and previous period values). For every scenario $S$ every query to parameters prioritize the data belonging to the partition with Scenario $S$ for the current period, if not present the value belonging to the Best Estimate scenario is used. Previous period parameter is always retrieved from the Best Estimate scenario. ", "\n", "\nFor Scenario calculations, the transactional data (RawVariable and IfrsVariable) stored in the ImportStorage is the union of the variables provided for the Scenario and the variables provided for the Best Estimate. In case of intersection betweeen the two sets, priority is given to the values belonging to the scenario. This allows users to input only the variables for scenario that differ from the Best Estimate scenario. For example, if a scenario affects only the Risk Adjustment variables, the nominal cashflow input might contain only the Risk Adjustment. If other variables are provided they are equally considered, stored in the data source and used in calculation. Note that if a scenario requires a variable provided under the Best Estimate scenario to have 0 value, then it must be input in the scenario file with 0 value. Omitting this variable from the input results in the calculation engine picking up the value provided in the Best Estimate scenario. This applies to nominal cash flow, actuals, and opening. ", "\n",