From 55988ad28b90ada3a6f5270972e9ab730fa3e6bd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Melvyn=20La=C3=AFly?= Date: Sat, 4 Jun 2022 18:25:40 +0200 Subject: [PATCH 1/4] Avoid risk of out of bound exception when getting a csv column name + self documentation --- src/Csv/CsvInference.fs | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/src/Csv/CsvInference.fs b/src/Csv/CsvInference.fs index afc089381..201bb8f6c 100644 --- a/src/Csv/CsvInference.fs +++ b/src/Csv/CsvInference.fs @@ -263,7 +263,8 @@ let internal parseHeaders headers numberOfColumns schema unitsOfMeasureProvider match parseResult with | SchemaParseResult.Name name -> makeUnique name, None | SchemaParseResult.NameAndUnit (name, unit) -> - // store the original header because the inferred type might not support units of measure + // store the original header because the inferred type might not support units of measure. + // format: schemaDefinition \n schemaName (makeUnique item) + "\n" + (makeUnique name), Some unit | SchemaParseResult.Full prop -> let prop = { prop with Name = makeUnique prop.Name } @@ -367,6 +368,14 @@ let internal getFields preferOptionals inferedType schema = match Array.get schema index with | Some prop -> prop | None -> + let schemaCompleteDefinition, schemaName = + let split = field.Name.Split('\n') + + if split.Length > 1 then + split.[0], split.[1] + else + field.Name, field.Name + match field.Type with | InferedType.Primitive (typ, unit, optional) -> @@ -394,15 +403,14 @@ let internal getFields preferOptionals inferedType schema = match unit with | Some unit -> if StructuralInference.supportsUnitsOfMeasure typ then - typ, Some unit, field.Name.Split('\n').[1] + typ, Some unit, schemaName else - typ, None, field.Name.Split('\n').[0] - | _ -> typ, None, field.Name.Split('\n').[0] + typ, None, schemaCompleteDefinition + | _ -> typ, None, schemaCompleteDefinition PrimitiveInferedProperty.Create(name, typ, typWrapper, unit) - | _ -> - PrimitiveInferedProperty.Create(field.Name.Split('\n').[0], typeof, preferOptionals, None)) + | _ -> PrimitiveInferedProperty.Create(schemaCompleteDefinition, typeof, preferOptionals, None)) | _ -> failwithf "inferFields: Expected record type, got %A" inferedType From eaf940ad57ac294c8030e25fd4c7dc4376f9d7ef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Melvyn=20La=C3=AFly?= Date: Sat, 4 Jun 2022 12:54:33 +0200 Subject: [PATCH 2/4] Move TypeWrapper definition --- src/CommonRuntime/StructuralTypes.fs | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/src/CommonRuntime/StructuralTypes.fs b/src/CommonRuntime/StructuralTypes.fs index 7263c2976..80bcfa49f 100644 --- a/src/CommonRuntime/StructuralTypes.fs +++ b/src/CommonRuntime/StructuralTypes.fs @@ -192,6 +192,15 @@ type Bit = Bit // ------------------------------------------------------------------------------------------------ +/// Represents a transformation of a type +[] +type TypeWrapper = + /// No transformation will be made to the type + | None + /// The type T will be converter to type T option + | Option + /// The type T will be converter to type Nullable + | Nullable /// Represents type information about a primitive property (used mainly in the CSV provider) /// This type captures the type, unit of measure and handling of missing values (if we /// infer that the value may be missing, we can generate option or nullable) @@ -219,12 +228,3 @@ type PrimitiveInferedProperty = static member Create(name, typ, optional, unit) = PrimitiveInferedProperty.Create(name, typ, (if optional then TypeWrapper.Option else TypeWrapper.None), unit) -/// Represents a transformation of a type -[] -type TypeWrapper = - /// No transformation will be made to the type - | None - /// The type T will be converter to type T option - | Option - /// The type T will be converter to type Nullable - | Nullable From d79c9bc66bd277e97bb00b5d5a861657a4ea13ac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Melvyn=20La=C3=AFly?= Date: Sat, 4 Jun 2022 13:00:43 +0200 Subject: [PATCH 3/4] Introduce a PrimitiveInferedValue to replace PrimitiveInferedProperty when we don't have a property name --- .../ConversionsGenerator.fs | 6 ++-- src/CommonProviderImplementation/Helpers.fs | 4 +-- src/CommonRuntime/StructuralTypes.fs | 31 +++++++++++++------ src/Json/JsonConversionsGenerator.fs | 5 ++- src/Json/JsonGenerator.fs | 2 +- .../InferenceTests.fs | 16 +++++----- tests/FSharp.Data.Tests/CsvProvider.fs | 2 +- 7 files changed, 40 insertions(+), 26 deletions(-) diff --git a/src/CommonProviderImplementation/ConversionsGenerator.fs b/src/CommonProviderImplementation/ConversionsGenerator.fs index 7cfb8d3a4..3a5d56fd4 100644 --- a/src/CommonProviderImplementation/ConversionsGenerator.fs +++ b/src/CommonProviderImplementation/ConversionsGenerator.fs @@ -1,4 +1,4 @@ -// Copyright 2011-2015, Tomas Petricek (http://tomasp.net), Gustavo Guerra (http://functionalflow.co.uk), and other contributors +// Copyright 2011-2015, Tomas Petricek (http://tomasp.net), Gustavo Guerra (http://functionalflow.co.uk), and other contributors // Licensed under the Apache License, Version 2.0, see LICENSE.md in this project // // Conversions from string to various primitive types @@ -68,6 +68,8 @@ let getBackConversionQuotation missingValuesStr cultureStr typ value : Expr and converts it to /// an expression of other type - the type is specified by `field` let convertStringValue missingValuesStr cultureStr (field: PrimitiveInferedProperty) = + let fieldName = field.Name + let field = field.Value let returnType = match field.TypeWrapper with @@ -92,7 +94,7 @@ let convertStringValue missingValuesStr cultureStr (field: PrimitiveInferedPrope let varExpr = Expr.Cast(Expr.Var var) let body = - typeof?GetNonOptionalValue field.RuntimeType (field.Name, convert varExpr, varExpr) + typeof?GetNonOptionalValue field.RuntimeType (fieldName, convert varExpr, varExpr) Expr.Let(var, value, body) | TypeWrapper.Option -> convert value diff --git a/src/CommonProviderImplementation/Helpers.fs b/src/CommonProviderImplementation/Helpers.fs index 72a61745a..a4f1d7163 100644 --- a/src/CommonProviderImplementation/Helpers.fs +++ b/src/CommonProviderImplementation/Helpers.fs @@ -22,9 +22,9 @@ open ProviderImplementation.ProvidedTypes // ---------------------------------------------------------------------------------------------- [] -module internal PrimitiveInferedPropertyExtensions = +module internal PrimitiveInferedValueExtensions = - type PrimitiveInferedProperty with + type PrimitiveInferedValue with member x.TypeWithMeasure = match x.UnitOfMeasure with diff --git a/src/CommonRuntime/StructuralTypes.fs b/src/CommonRuntime/StructuralTypes.fs index 80bcfa49f..62f25ca7d 100644 --- a/src/CommonRuntime/StructuralTypes.fs +++ b/src/CommonRuntime/StructuralTypes.fs @@ -201,16 +201,18 @@ type TypeWrapper = | Option /// The type T will be converter to type Nullable | Nullable -/// Represents type information about a primitive property (used mainly in the CSV provider) + static member FromOption optional = + if optional then TypeWrapper.Option else TypeWrapper.None + +/// Represents type information about a primitive value (used mainly in the CSV provider) /// This type captures the type, unit of measure and handling of missing values (if we /// infer that the value may be missing, we can generate option or nullable) -type PrimitiveInferedProperty = - { Name: string - InferedType: Type +type PrimitiveInferedValue = + { InferedType: Type RuntimeType: Type UnitOfMeasure: Type option TypeWrapper: TypeWrapper } - static member Create(name, typ, typWrapper, unit) = + static member Create(typ, typWrapper, unit) = let runtimeTyp = if typ = typeof then typeof @@ -219,12 +221,23 @@ type PrimitiveInferedProperty = else typ - { Name = name - InferedType = typ + { InferedType = typ RuntimeType = runtimeTyp UnitOfMeasure = unit TypeWrapper = typWrapper } - static member Create(name, typ, optional, unit) = - PrimitiveInferedProperty.Create(name, typ, (if optional then TypeWrapper.Option else TypeWrapper.None), unit) + static member Create(typ, optional, unit) = + PrimitiveInferedValue.Create(typ, TypeWrapper.FromOption optional, unit) + +/// Represents type information about a primitive property (used mainly in the CSV provider) +/// This type captures the type, unit of measure and handling of missing values (if we +/// infer that the value may be missing, we can generate option or nullable) +type PrimitiveInferedProperty = + { Name: string + Value: PrimitiveInferedValue } + static member Create(name, typ, (typWrapper: TypeWrapper), unit) = + { Name = name + Value = PrimitiveInferedValue.Create(typ, typWrapper, unit) } + static member Create(name, typ, optional, unit) = + PrimitiveInferedProperty.Create(name, typ, TypeWrapper.FromOption optional, unit) diff --git a/src/Json/JsonConversionsGenerator.fs b/src/Json/JsonConversionsGenerator.fs index 6656ec389..3805e702d 100644 --- a/src/Json/JsonConversionsGenerator.fs +++ b/src/Json/JsonConversionsGenerator.fs @@ -1,4 +1,4 @@ -// ---------------------------------------------------------------------------------------------- +// ---------------------------------------------------------------------------------------------- // Conversions from string to various primitive types // ---------------------------------------------------------------------------------------------- @@ -48,10 +48,9 @@ type JsonConversionCallingType = /// Creates a function that takes Expr and converts it to /// an expression of other type - the type is specified by `field` -let convertJsonValue missingValuesStr cultureStr canPassAllConversionCallingTypes (field: PrimitiveInferedProperty) = +let convertJsonValue missingValuesStr cultureStr canPassAllConversionCallingTypes (field: PrimitiveInferedValue) = assert (field.TypeWithMeasure = field.RuntimeType) - assert (field.Name = "") let returnType = match field.TypeWrapper with diff --git a/src/Json/JsonGenerator.fs b/src/Json/JsonGenerator.fs index b85c767ee..225bf41ff 100644 --- a/src/Json/JsonGenerator.fs +++ b/src/Json/JsonGenerator.fs @@ -300,7 +300,7 @@ module JsonTypeBuilder = | InferedType.Primitive (inferedType, unit, optional) -> let typ, conv, conversionCallingType = - PrimitiveInferedProperty.Create("", inferedType, optional, unit) + PrimitiveInferedValue.Create(inferedType, optional, unit) |> convertJsonValue "" ctx.CultureStr canPassAllConversionCallingTypes { ConvertedType = typ diff --git a/tests/FSharp.Data.DesignTime.Tests/InferenceTests.fs b/tests/FSharp.Data.DesignTime.Tests/InferenceTests.fs index 198f16149..b0155a858 100644 --- a/tests/FSharp.Data.DesignTime.Tests/InferenceTests.fs +++ b/tests/FSharp.Data.DesignTime.Tests/InferenceTests.fs @@ -295,8 +295,8 @@ let ``Infers units of measure correctly``() = ||> CsvInference.getFields false |> List.map (fun field -> field.Name, - field.RuntimeType, - prettyTypeName field.TypeWithMeasure) + field.Value.RuntimeType, + prettyTypeName field.Value.TypeWithMeasure) let propString = "String(metre)" , typeof , "string" let propFloat = "Float" , typeof , "float" @@ -319,9 +319,9 @@ let ``Inference schema override by column name``() = ||> CsvInference.getFields false |> List.map (fun field -> field.Name, - field.RuntimeType, - prettyTypeName field.TypeWithMeasure, - field.TypeWrapper) + field.Value.RuntimeType, + prettyTypeName field.Value.TypeWithMeasure, + field.Value.TypeWrapper) let col1 = "A" , typeof , "int", TypeWrapper.None let col2 = "B" , typeof, "decimal" , TypeWrapper.Nullable @@ -342,9 +342,9 @@ let ``Inference schema override by parameter``() = ||> CsvInference.getFields false |> List.map (fun field -> field.Name, - field.RuntimeType, - prettyTypeName field.TypeWithMeasure, - field.TypeWrapper) + field.Value.RuntimeType, + prettyTypeName field.Value.TypeWithMeasure, + field.Value.TypeWrapper) let col1 = "Column1" , typeof, "float" , TypeWrapper.None let col2 = "Foo" , typeof , "int" , TypeWrapper.None diff --git a/tests/FSharp.Data.Tests/CsvProvider.fs b/tests/FSharp.Data.Tests/CsvProvider.fs index 16c5c6571..326f813bb 100644 --- a/tests/FSharp.Data.Tests/CsvProvider.fs +++ b/tests/FSharp.Data.Tests/CsvProvider.fs @@ -635,5 +635,5 @@ let ``InferColumnTypes shall infer empty string as Double``() = let csv = CsvFile.Load(Path.Combine(__SOURCE_DIRECTORY__, "Data/emptyMissingValue.csv")) let types = csv.InferColumnTypes(2,[|""|],System.Globalization.CultureInfo.GetCultureInfo(""), null, false, false) let expected = "Double" - let actual = types.[3].InferedType.Name + let actual = types.[3].Value.InferedType.Name actual |> should equal expected From abd0d2274a8aa7d754af14ee7c90c31ae23bab05 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Melvyn=20La=C3=AFly?= Date: Sun, 12 Jun 2022 15:33:33 +0200 Subject: [PATCH 4/4] No longer display expected error message in passing xsd validation test --- tests/FSharp.Data.Tests/XmlProvider.fs | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/tests/FSharp.Data.Tests/XmlProvider.fs b/tests/FSharp.Data.Tests/XmlProvider.fs index cd9354f88..a4174608b 100644 --- a/tests/FSharp.Data.Tests/XmlProvider.fs +++ b/tests/FSharp.Data.Tests/XmlProvider.fs @@ -1145,13 +1145,14 @@ type SimpleTypes = XmlProvider open System.Xml.Schema -let isValid xmlSchemaSet = +let isValid xmlSchemaSet displayErrorMessage = fun (xml: XElement) -> try XDocument(xml).Validate(xmlSchemaSet, validationEventHandler = null) true with :? XmlSchemaException as e -> - printfn "%s/n%O" e.Message xml + if displayErrorMessage then + printfn "%s/n%O" e.Message xml false [] @@ -1187,7 +1188,7 @@ let ``simple types are formatted properly``() = double = System.Double.PositiveInfinity) let schema = SimpleTypes.GetSchema() - let isValid = isValid schema + let isValid = isValid schema true isValid simpleValues.XElement |> should equal true isValid minValues.XElement |> should equal true isValid maxValues.XElement |> should equal true @@ -1205,15 +1206,17 @@ let ``time is omitted when zero``() = decimal = 0M, double = System.Double.NaN) - let isValid = isValid schema + let isValidWithMsg = isValid schema true + // Don't display the error message each time when we expect to see it: + let isValidWithoutMsg = isValid schema false let validXml = System.DateTime(2018, 8, 29) |> simpleValues - isValid validXml.XElement |> should equal true + isValidWithMsg validXml.XElement |> should equal true validXml.XElement.Attribute(XName.Get "date").Value |> should equal "2018-08-29" let invalidXml = System.DateTime(2018, 8, 29, 5, 30, 56) |> simpleValues - isValid invalidXml.XElement |> should equal false + isValidWithoutMsg invalidXml.XElement |> should equal false invalidXml.XElement.Attribute(XName.Get "date").Value |> should equal "2018-08-29T05:30:56.0000000"