Adding mandatory AoC step exception in the filter

ifrs17/Constants/Validations.ipynb

@@ -51,7 +51,7 @@
             "source": [
                 "public enum Warning {",
                 "\n    // Import",
-                "\n    ActiveDataNodeWithCashflowBOPI, VariablesAlreadyImported, VariablesAlreadyCalculated, ScenarioReCalculations,",
+                "\n    ActiveDataNodeWithCashflowBOPI, VariablesAlreadyImported, VariablesAlreadyCalculated, ScenarioReCalculations, MandatoryAocStepMissing,",
                 "\n    // Default",
                 "\n    Generic",
                 "\n}; "
@@ -185,6 +185,7 @@
                 "\n    // Import",
                 "\n    (Warning.ActiveDataNodeWithCashflowBOPI , 1) => $\"Cash flow with AoC Type: {AocTypes.BOP} and Novelty: {Novelties.I} for Group of Contract {s[0]} is not allowed because previous period data are available.\",",
                 "\n    (Warning.VariablesAlreadyImported       , 0) => $\"The import of the current file does not contain any new data. Hence, no data will be saved or calculations will be performed.\",",
+                "\n    (Warning.MandatoryAocStepMissing        , 3) => $\"The AoC step ({s[0]}, {s[1]}) is not imported for ({s[2]}).\",",
                 "\n    (Warning.ScenarioReCalculations         , 1) => $\"The present Best Estimate import makes the result of dependent Scenarios out of date. Hence, the following Scenarios are re-calculated: {s[0]}.\", ",
                 "\n    // Default",
                 "\n    (Warning.Generic                        , _) => $\"{s[0]}\",",

ifrs17/Import/Importers.ipynb

@@ -69,6 +69,7 @@
                 "\n    // Dimensions",
                 "\n    public Dictionary<string, EstimateType> EstimateType;",
                 "\n    public Dictionary<string, AmountType> AmountType; ",
+                "\n    public HashSet<AocStep> MandatoryAocSteps;",
                 "\n    public HashSet<AocStep> AocTypeMap;",
                 "\n    private HashSet<string> estimateTypes;",
                 "\n    private HashSet<string> amountTypes;",
@@ -125,6 +126,7 @@
                 "\n        ReportingNode = reportingNodes.First();",
                 "\n",
                 "\n        var aocConfigurationByAocStep = await dataSource.LoadAocStepConfigurationAsync(args.Year, args.Month);",
+                "\n        MandatoryAocSteps = aocConfigurationByAocStep.Where(x => x.DataType == DataType.Mandatory).Select(x => new AocStep(x.AocType, x.Novelty)).ToHashSet();",
                 "\n        AocTypeMap = args.ImportFormat switch {",
                 "\n                ImportFormats.Cashflow => aocConfigurationByAocStep.Where(x => x.InputSource.Contains(InputSource.Cashflow) &&",
                 "\n                                                                                !new DataType[]{DataType.Calculated, DataType.CalculatedTelescopic}.Contains(x.DataType) )",
@@ -471,30 +473,6 @@
             "execution_count": 0,
             "outputs": []
         },
-        {
-            "cell_type": "markdown",
-            "source": [
-                "## Validation for Active Data Node States"
-            ],
-            "metadata": {},
-            "execution_count": 0,
-            "outputs": []
-        },
-        {
-            "cell_type": "code",
-            "source": [
-                "public async Task ValidateForDataNodeStateActiveAsync<T>(IWorkspace workspace, Dictionary<string, DataNodeData> dataNodes) where T : BaseDataRecord",
-                "\n{   ",
-                "\n    foreach(var item in (await workspace.Query<T>().ToArrayAsync()).GroupBy(x => x.DataNode))",
-                "\n        if(!dataNodes.ContainsKey(item.First().DataNode))",
-                "\n            ApplicationMessage.Log(Error.InactiveDataNodeState, item.First().DataNode);",
-                "\n}",
-                "\n"
-            ],
-            "metadata": {},
-            "execution_count": 0,
-            "outputs": []
-        },
         {
             "cell_type": "markdown",
             "source": [
@@ -593,6 +571,102 @@
             "execution_count": 0,
             "outputs": []
         },
+        {
+            "cell_type": "markdown",
+            "source": [
+                "# Validations"
+            ],
+            "metadata": {},
+            "execution_count": 0,
+            "outputs": []
+        },
+        {
+            "cell_type": "markdown",
+            "source": [
+                "## Validation for Active Data Node States"
+            ],
+            "metadata": {},
+            "execution_count": 0,
+            "outputs": []
+        },
+        {
+            "cell_type": "code",
+            "source": [
+                "public async static Task ValidateForDataNodeStateActiveAsync<T>(this IWorkspace workspace, Dictionary<string, DataNodeData> dataNodes) where T : BaseDataRecord",
+                "\n{   ",
+                "\n    foreach(var item in (await workspace.Query<T>().ToArrayAsync()).GroupBy(x => x.DataNode))",
+                "\n        if(!dataNodes.ContainsKey(item.First().DataNode))",
+                "\n            ApplicationMessage.Log(Error.InactiveDataNodeState, item.First().DataNode);",
+                "\n}",
+                "\n"
+            ],
+            "metadata": {},
+            "execution_count": 0,
+            "outputs": []
+        },
+        {
+            "cell_type": "markdown",
+            "source": [
+                "## Validate for Data Node States Logic"
+            ],
+            "metadata": {},
+            "execution_count": 0,
+            "outputs": []
+        },
+        {
+            "cell_type": "code",
+            "source": [
+                "public async static Task ValidateDataNodeStatesAsync(this IWorkspace workspace, Dictionary<string, DataNodeData> persistentDataNodeByDataNode)",
+                "\n{",
+                "\n    foreach(var importedDataNodeState in await workspace.Query<DataNodeState>().ToArrayAsync())",
+                "\n    {",
+                "\n        if(persistentDataNodeByDataNode.TryGetValue(importedDataNodeState.DataNode, out var currentPersistentDataNode))",
+                "\n        {",
+                "\n            if(importedDataNodeState.State < currentPersistentDataNode.State)",
+                "\n                ApplicationMessage.Log(Error.ChangeDataNodeState, importedDataNodeState.DataNode, ",
+                "\n                                                                  currentPersistentDataNode.State.ToString(), ",
+                "\n                                                                  importedDataNodeState.State.ToString());",
+                "\n",
+                "\n            if(importedDataNodeState.State == currentPersistentDataNode.State)",
+                "\n                await workspace.DeleteAsync<DataNodeState>(importedDataNodeState);",
+                "\n        }",
+                "\n    }",
+                "\n}"
+            ],
+            "metadata": {},
+            "execution_count": 0,
+            "outputs": []
+        },
+        {
+            "cell_type": "markdown",
+            "source": [
+                "## Validate for Mandatory Aoc Steps"
+            ],
+            "metadata": {},
+            "execution_count": 0,
+            "outputs": []
+        },
+        {
+            "cell_type": "code",
+            "source": [
+                "using static Systemorph.Vertex.Equality.IdentityPropertyExtensions;",
+                "\npublic async static Task ValidateForMandatoryAocSteps(this IWorkspace workspace, IDataSet dataSet, HashSet<AocStep> mandatoryAocSteps)",
+                "\n{   ",
+                "\n    var excludedProperties = new[]{nameof(AocType), nameof(Novelty)};",
+                "\n    var includingProperties = typeof(RawVariable).GetIdentityProperties().Select(x=>x.Name).Except(excludedProperties).ToArray();",
+                "\n    var missingAocStepsByIdentityProperties = (await workspace.Query<RawVariable>().ToListAsync())",
+                "\n                     .GroupBy(x => x.ToStringWith(properties: includingProperties),",
+                "\n                              x => new AocStep(x.AocType, x.Novelty),",
+                "\n                              (properties, parsedAocSteps) => (properties, mandatoryAocSteps.Except(parsedAocSteps))",
+                "\n                             );",
+                "\n    foreach((var properties, var missingSteps) in missingAocStepsByIdentityProperties) ",
+                "\n        foreach(var missingStep in missingSteps)  ApplicationMessage.Log(Warning.MandatoryAocStepMissing, missingStep.AocType, missingStep.Novelty, properties);",
+                "\n}"
+            ],
+            "metadata": {},
+            "execution_count": 0,
+            "outputs": []
+        },
         {
             "cell_type": "markdown",
             "source": [
@@ -946,22 +1020,7 @@
         {
             "cell_type": "code",
             "source": [
-                "public async static Task ValidateDataNodeStatesAsync(this IWorkspace workspace, Dictionary<string, DataNodeData> persistentDataNodeByDataNode)",
-                "\n{",
-                "\n    foreach(var importedDataNodeState in await workspace.Query<DataNodeState>().ToArrayAsync())",
-                "\n    {",
-                "\n        if(persistentDataNodeByDataNode.TryGetValue(importedDataNodeState.DataNode, out var currentPersistentDataNode))",
-                "\n        {",
-                "\n            if(importedDataNodeState.State < currentPersistentDataNode.State)",
-                "\n                ApplicationMessage.Log(Error.ChangeDataNodeState, importedDataNodeState.DataNode, ",
-                "\n                                                                  currentPersistentDataNode.State.ToString(), ",
-                "\n                                                                  importedDataNodeState.State.ToString());",
-                "\n",
-                "\n            if(importedDataNodeState.State == currentPersistentDataNode.State)",
-                "\n                await workspace.DeleteAsync<DataNodeState>(importedDataNodeState);",
-                "\n        }",
-                "\n    }",
-                "\n}"
+                ""
             ],
             "metadata": {},
             "execution_count": 0,
@@ -1187,11 +1246,10 @@
                 "\n            var values = datarow.Table.Columns.Where(c => c.ColumnName.StartsWith(nameof(RawVariable.Values))).OrderBy(c => c.ColumnName.Length).ThenBy(c => c.ColumnName)",
                 "\n                                .Select(x => datarow.Field<string>(x.ColumnName).CheckStringForExponentialAndConvertToDouble()).ToArray();",
                 "\n            ",
-                "\n            // Filter out empty raw variables for AocType != CL",
-                "\n            //TODO: extend this check for all mandatory step and not just for CL",
+                "\n            // Filter out empty raw variables for AocStep \\not\\in MandatoryAocSteps",
                 "\n            if(args.Scenario == null) {",
                 "\n                values = values.Prune();",
-                "\n                if(values.Length == 0 && aocType != AocTypes.CL) return null;",
+                "\n                if(values.Length == 0 && !parsingStorage.MandatoryAocSteps.Contains(new AocStep(aocType, novelty))) return null;",
                 "\n            }",
                 "\n                ",
                 "\n            var item = new RawVariable {",
@@ -1210,7 +1268,8 @@
                 "\n        }, ImportFormats.Cashflow",
                 "\n    ).WithTarget(workspace).ExecuteAsync();",
                 "\n    ",
-                "\n    await ValidateForDataNodeStateActiveAsync<RawVariable>(workspace, parsingStorage.DataNodeDataBySystemName);",
+                "\n    await workspace.ValidateForMandatoryAocSteps(dataSet, parsingStorage.MandatoryAocSteps);",
+                "\n    await workspace.ValidateForDataNodeStateActiveAsync<RawVariable>(parsingStorage.DataNodeDataBySystemName);",
                 "\n    return Activity.Finish().Merge(importLog);",
                 "\n}"
             ],
@@ -1311,7 +1370,7 @@
                 "\n        }, ImportFormats.Actual",
                 "\n    ).WithTarget(workspace).ExecuteAsync();",
                 "\n    ",
-                "\n    await ValidateForDataNodeStateActiveAsync<IfrsVariable>(workspace, parsingStorage.DataNodeDataBySystemName);",
+                "\n    await workspace.ValidateForDataNodeStateActiveAsync<IfrsVariable>(parsingStorage.DataNodeDataBySystemName);",
                 "\n    return Activity.Finish().Merge(importLog);",
                 "\n}"
             ],
@@ -1419,7 +1478,7 @@
                 "\n    foreach (var iv in invalidVariables)",
                 "\n        ApplicationMessage.Log(Error.MultipleTechnicalMarginOpening, $\"{iv.DataNode},{iv.AocType},{iv.Novelty}\");",
                 "\n    ",
-                "\n    await ValidateForDataNodeStateActiveAsync<IfrsVariable>(workspace, parsingStorage.DataNodeDataBySystemName);",
+                "\n    await workspace.ValidateForDataNodeStateActiveAsync<IfrsVariable>(parsingStorage.DataNodeDataBySystemName);",
                 "\n    targetPartitionByReportingNodeAndPeriodId = parsingStorage.TargetPartitionByReportingNodeAndPeriod.Id;",
                 "\n    return Activity.Finish().Merge(importLog);",
                 "\n}"

ifrs17/Utils/Extensions.ipynb

@@ -207,12 +207,12 @@
             "cell_type": "code",
             "source": [
                 "using System.Text;",
-                "\npublic static string ToIdentityString<T>(this T v) where T : class",
+                "\npublic static string ToIdentityString<T>(this T v, params string[] excludingProperties) where T : class",
                 "\n{",
                 "\n   StringBuilder sb = new StringBuilder();",
                 "\n   var propertyInfos = v.GetType()",
                 "\n                        .GetProperties()",
-                "\n                        .Where(x => Attribute.IsDefined(x, typeof(IdentityPropertyAttribute)))",
+                "\n                        .Where(x => Attribute.IsDefined(x, typeof(IdentityPropertyAttribute)) && !excludingProperties.Contains(x.Name))",
                 "\n                        .OrderByDescending(x => x.PropertyType.Name).ThenByDescending(x => x.Name)",
                 "\n                        .Select(x => sb.Append(x.Name).Append(\":\").Append(v.GetType().GetProperty(x.Name)?.GetValue(v, null)).Append(\", \")).ToArray();",
                 "\n   return propertyInfos.Count() == 0? v.ToString() : propertyInfos.Select(p => p.ToString()).ToArray().Last();",
@@ -222,6 +222,29 @@
             "execution_count": 0,
             "outputs": []
         },
+        {
+            "cell_type": "markdown",
+            "source": [
+                "# ToString with including properties"
+            ],
+            "metadata": {},
+            "execution_count": 0,
+            "outputs": []
+        },
+        {
+            "cell_type": "code",
+            "source": [
+                "public static string ToStringWith<T>(this T variable, string[] properties) where T : BaseVariableIdentity",
+                "\n{",
+                "\n    var propertiesRead = variable.ToString().Split('{', '}')[1].Split(',');",
+                "\n    var propertiesFiltered = propertiesRead.Where(x=>  properties.Contains(x.Split('=')[0].Trim()));",
+                "\n    return string.Join(\", \", propertiesFiltered).Trim();",
+                "\n}"
+            ],
+            "metadata": {},
+            "execution_count": 0,
+            "outputs": []
+        },
         {
             "cell_type": "code",
             "source": [