From 610ce53d0b7036a0480e314e15cbe582f3006534 Mon Sep 17 00:00:00 2001 From: nikoyak <99891966+nikoyak@users.noreply.github.com> Date: Fri, 18 Feb 2022 17:32:10 +0300 Subject: [PATCH 1/3] Added InferDictionariesFromRecords flag to JSON TP JSON TP always projects json record fields into the properties of generated types. Sometimes it is not desirable. The added flag will allow to infer dictionaries from the records if the names of all the fields of similar records in the provider's training data are inferred to the same non-string primitive type (by type inference rules). The flag is not set by default for backwards compatibility. --- src/Json/JsonGenerator.fs | 128 +++++++++++++++--- src/Json/JsonProvider.fs | 9 +- src/Json/JsonRuntime.fs | 60 ++++++++ src/Test.fsx | 3 +- .../SignatureTestCases.config | 44 +++--- .../TypeProviderInstantiation.fs | 12 +- ...on,Dates.json,False,,,True,False.expected} | 0 ...Inference.json,False,,,True,False.expected | 96 +++++++++++++ ...yInference.json,False,,,True,True.expected | 125 +++++++++++++++++ ...leNested.json,False,,,True,False.expected} | 0 ...on,Empty.json,False,,,True,False.expected} | 0 ...n,GitHub.json,False,,,True,False.expected} | 0 ...n,Nested.json,False,,,True,False.expected} | 0 ...onValues.json,False,,,True,False.expected} | 0 ...n,Simple.json,False,,,True,False.expected} | 0 ...pleArray.json,False,,,True,False.expected} | 0 ...imeSpans.json,False,,,True,False.expected} | 0 ...terSample.json,True,,,True,False.expected} | 0 ...terStream.json,True,,,True,False.expected} | 0 ...ference.json,False,,,False,False.expected} | 0 ...nference.json,False,,,True,False.expected} | 0 ...indinium.json,False,,,True,False.expected} | 0 ...WikiData.json,False,,,True,False.expected} | 0 ...json,False,WorldBank,,True,False.expected} | 0 ...contacts.json,False,,,True,False.expected} | 0 ...ptionals.json,False,,,True,False.expected} | 0 ...projects.json,False,,,True,False.expected} | 0 ...n,reddit.json,False,,,True,False.expected} | 0 ...pics.json,True,Topic,,True,False.expected} | 0 .../Data/DictionaryInference.json | 10 ++ tests/FSharp.Data.Tests/JsonProvider.fs | 20 +++ 31 files changed, 459 insertions(+), 48 deletions(-) rename tests/FSharp.Data.DesignTime.Tests/expected/{Json,Dates.json,False,,,True.expected => Json,Dates.json,False,,,True,False.expected} (100%) create mode 100644 tests/FSharp.Data.DesignTime.Tests/expected/Json,DictionaryInference.json,False,,,True,False.expected create mode 100644 tests/FSharp.Data.DesignTime.Tests/expected/Json,DictionaryInference.json,False,,,True,True.expected rename tests/FSharp.Data.DesignTime.Tests/expected/{Json,DoubleNested.json,False,,,True.expected => Json,DoubleNested.json,False,,,True,False.expected} (100%) rename tests/FSharp.Data.DesignTime.Tests/expected/{Json,Empty.json,False,,,True.expected => Json,Empty.json,False,,,True,False.expected} (100%) rename tests/FSharp.Data.DesignTime.Tests/expected/{Json,GitHub.json,False,,,True.expected => Json,GitHub.json,False,,,True,False.expected} (100%) rename tests/FSharp.Data.DesignTime.Tests/expected/{Json,Nested.json,False,,,True.expected => Json,Nested.json,False,,,True,False.expected} (100%) rename tests/FSharp.Data.DesignTime.Tests/expected/{Json,OptionValues.json,False,,,True.expected => Json,OptionValues.json,False,,,True,False.expected} (100%) rename tests/FSharp.Data.DesignTime.Tests/expected/{Json,Simple.json,False,,,True.expected => Json,Simple.json,False,,,True,False.expected} (100%) rename tests/FSharp.Data.DesignTime.Tests/expected/{Json,SimpleArray.json,False,,,True.expected => Json,SimpleArray.json,False,,,True,False.expected} (100%) rename tests/FSharp.Data.DesignTime.Tests/expected/{Json,TimeSpans.json,False,,,True.expected => Json,TimeSpans.json,False,,,True,False.expected} (100%) rename tests/FSharp.Data.DesignTime.Tests/expected/{Json,TwitterSample.json,True,,,True.expected => Json,TwitterSample.json,True,,,True,False.expected} (100%) rename tests/FSharp.Data.DesignTime.Tests/expected/{Json,TwitterStream.json,True,,,True.expected => Json,TwitterStream.json,True,,,True,False.expected} (100%) rename tests/FSharp.Data.DesignTime.Tests/expected/{Json,TypeInference.json,False,,,False.expected => Json,TypeInference.json,False,,,False,False.expected} (100%) rename tests/FSharp.Data.DesignTime.Tests/expected/{Json,TypeInference.json,False,,,True.expected => Json,TypeInference.json,False,,,True,False.expected} (100%) rename tests/FSharp.Data.DesignTime.Tests/expected/{Json,Vindinium.json,False,,,True.expected => Json,Vindinium.json,False,,,True,False.expected} (100%) rename tests/FSharp.Data.DesignTime.Tests/expected/{Json,WikiData.json,False,,,True.expected => Json,WikiData.json,False,,,True,False.expected} (100%) rename tests/FSharp.Data.DesignTime.Tests/expected/{Json,WorldBank.json,False,WorldBank,,True.expected => Json,WorldBank.json,False,WorldBank,,True,False.expected} (100%) rename tests/FSharp.Data.DesignTime.Tests/expected/{Json,contacts.json,False,,,True.expected => Json,contacts.json,False,,,True,False.expected} (100%) rename tests/FSharp.Data.DesignTime.Tests/expected/{Json,optionals.json,False,,,True.expected => Json,optionals.json,False,,,True,False.expected} (100%) rename tests/FSharp.Data.DesignTime.Tests/expected/{Json,projects.json,False,,,True.expected => Json,projects.json,False,,,True,False.expected} (100%) rename tests/FSharp.Data.DesignTime.Tests/expected/{Json,reddit.json,False,,,True.expected => Json,reddit.json,False,,,True,False.expected} (100%) rename tests/FSharp.Data.DesignTime.Tests/expected/{Json,topics.json,True,Topic,,True.expected => Json,topics.json,True,Topic,,True,False.expected} (100%) create mode 100644 tests/FSharp.Data.Tests/Data/DictionaryInference.json diff --git a/src/Json/JsonGenerator.fs b/src/Json/JsonGenerator.fs index c6df93f59..7ed9ac924 100644 --- a/src/Json/JsonGenerator.fs +++ b/src/Json/JsonGenerator.fs @@ -26,21 +26,24 @@ type internal JsonGenerationContext = JsonValueType : Type JsonRuntimeType : Type TypeCache : Dictionary + InferDictionariesFromRecords: bool GenerateConstructors : bool } - static member Create(cultureStr, tpType, ?uniqueNiceName, ?typeCache) = + static member Create(cultureStr, tpType, ?uniqueNiceName, ?typeCache, ?inferDictionariesFromRecords) = let uniqueNiceName = defaultArg uniqueNiceName (NameUtils.uniqueGenerator NameUtils.nicePascalName) let typeCache = defaultArg typeCache (Dictionary()) - JsonGenerationContext.Create(cultureStr, tpType, uniqueNiceName, typeCache, true) + let inferDictionariesFromRecords = defaultArg inferDictionariesFromRecords false + JsonGenerationContext.Create(cultureStr, tpType, uniqueNiceName, typeCache, inferDictionariesFromRecords, true) - static member Create(cultureStr, tpType, uniqueNiceName, typeCache, generateConstructors) = + static member Create(cultureStr, tpType, uniqueNiceName, typeCache, inferDictionariesFromRecords, generateConstructors) = { CultureStr = cultureStr TypeProviderType = tpType UniqueNiceName = uniqueNiceName IJsonDocumentType = typeof JsonValueType = typeof JsonRuntimeType = typeof - TypeCache = typeCache + TypeCache = typeCache + InferDictionariesFromRecords = inferDictionariesFromRecords GenerateConstructors = generateConstructors } member x.MakeOptionType(typ:Type) = typedefof>.MakeGenericType typ @@ -291,8 +294,94 @@ module JsonTypeBuilder = let makeUnique = NameUtils.uniqueGenerator NameUtils.nicePascalName makeUnique "JsonValue" |> ignore - // Add all record fields as properties - let members = + let inferedKeyValueType = + let aggr = List.fold (StructuralInference.subtypeInfered false) InferedType.Top + let dropRecordName infType = + match infType with + | InferedType.Record (_, fields, opt) -> InferedType.Record (None, fields, opt) + | _ -> infType + + if not <| ctx.InferDictionariesFromRecords + then None + else + let infType = + [for prop in props -> StructuralInference.getInferedTypeFromString (TextRuntime.GetCulture ctx.CultureStr) prop.Name None] + |> aggr + match infType with + | InferedType.Primitive (typ = typ) when typ <> typeof -> + let inferValueType = ([for prop in props -> prop.Type |> dropRecordName] |> aggr).DropOptionality () + (infType, inferValueType) |> Some + | _ -> None + + match inferedKeyValueType with + | Some (inferedKeyType, inferedValueType) -> + // Add all record fields as dictionary items + let valueName = name + "Value" + + let keyResult = generateJsonType ctx (*canPassAllConversionCallingTypes*)false (*optionalityHandledByParent*)true "" inferedKeyType + let valueResult = generateJsonType ctx (*canPassAllConversionCallingTypes*)false (*optionalityHandledByParent*)true valueName inferedValueType + let valueConvertedTypeErased = valueResult.ConvertedTypeErased ctx + let valueOptionalType = typedefof<_ option>.MakeGenericType([|valueResult.ConvertedType|]) + + let tupleType = Microsoft.FSharp.Reflection.FSharpType.MakeTupleType([|keyResult.ConvertedType; valueResult.ConvertedType|]) + let itemsSeqType = typedefof<_ seq>.MakeGenericType([|tupleType|]) + + //let itemsSeqType = typedefof>.MakeGenericType([|valueResult.ConvertedType|]) + + let itemsGetter = fun (Singleton jDoc) -> + ctx.JsonRuntimeType?ConvertRecordToDictionary (keyResult.ConvertedType, valueConvertedTypeErased) (jDoc, keyResult.ConverterFunc ctx, valueResult.ConverterFunc ctx) + + let keysGetter = fun (Singleton jDoc) -> + ctx.JsonRuntimeType?GetKeysFromInferedDictionary (keyResult.ConvertedType) (jDoc, keyResult.ConverterFunc ctx) + + let valuesGetter = fun (Singleton jDoc) -> + ctx.JsonRuntimeType?GetValuesFromInferedDictionary (valueConvertedTypeErased) (jDoc, valueResult.ConverterFunc ctx) + + let (|Doubleton|) = function [f; s] -> f, s | _ -> failwith "Parameter mismatch" + + let itemGetter = fun (Doubleton (jDoc, key)) -> + ctx.JsonRuntimeType?GetValueByKeyFromInferedDictionary (keyResult.ConvertedType, valueConvertedTypeErased) (jDoc, keyResult.ConverterFunc ctx, valueResult.ConverterFunc ctx, key) + + let tryFindCode = fun (Doubleton (jDoc, key)) -> + ctx.JsonRuntimeType?TryGetValueByKeyFromInferedDictionary (keyResult.ConvertedType, valueConvertedTypeErased) (jDoc, keyResult.ConverterFunc ctx, valueResult.ConverterFunc ctx, key) + + let containsKeyCode = fun (Doubleton (jDoc, key)) -> + ctx.JsonRuntimeType?InferedDictionaryContainsKey (keyResult.ConvertedType) (jDoc, keyResult.ConverterFunc ctx, key) + + let countGetter = fun (Singleton jDoc) -> + <@@ JsonRuntime.GetRecordProperties(%%jDoc).Length @@> + + let isEmptyGetter = fun (Singleton jDoc) -> + <@@ JsonRuntime.GetRecordProperties(%%jDoc).Length = 0 @@> + + [ + ProvidedProperty("Items", itemsSeqType, getterCode = itemsGetter) + ProvidedProperty("Keys", keyResult.ConvertedType.MakeArrayType(), getterCode = keysGetter) + ProvidedProperty("Values", valueResult.ConvertedType.MakeArrayType(), getterCode = valuesGetter) + ProvidedProperty("Item", valueResult.ConvertedType, getterCode = itemGetter, indexParameters = [ProvidedParameter("key", keyResult.ConvertedType)]) + ProvidedProperty("Count", typeof, getterCode = countGetter) + ProvidedProperty("IsEmpty", typeof, getterCode = isEmptyGetter) ] + |> objectTy.AddMembers + [ + ProvidedMethod("TryFind", [ProvidedParameter("key", keyResult.ConvertedType)], valueOptionalType, tryFindCode) + ProvidedMethod("ContainsKey", [ProvidedParameter("key", keyResult.ConvertedType)], typeof, containsKeyCode) ] + |> objectTy.AddMembers + if ctx.GenerateConstructors then + objectTy.AddMember <| + ProvidedConstructor([ProvidedParameter("items", itemsSeqType)], invokeCode = fun args -> + let kvSeq = args.Head + let conv (value: Expr) = + let value = ProviderHelpers.some keyResult.ConvertedType value + ConversionsGenerator.getBackConversionQuotation "" ctx.CultureStr keyResult.ConvertedType value :> Expr + let convFunc = + ReflectionHelpers.makeDelegate conv keyResult.ConvertedType + + let cultureStr = ctx.CultureStr + ctx.JsonRuntimeType?CreateRecordFromDictionary (keyResult.ConvertedType, valueConvertedTypeErased) (kvSeq, cultureStr, convFunc) ) + () + | None -> + // Add all record fields as properties + let members = [for prop in props -> let propResult = generateJsonType ctx (*canPassAllConversionCallingTypes*)true (*optionalityHandledByParent*)true "" prop.Type @@ -336,29 +425,30 @@ module JsonTypeBuilder = let name = makeUnique prop.Name prop.Name, - ProvidedProperty(name, convertedType, getterCode = getter), + [ProvidedProperty(name, convertedType, getterCode = getter)], ProvidedParameter(NameUtils.niceCamelName name, replaceJDocWithJValue ctx convertedType) ] - let names, properties, parameters = List.unzip3 members - objectTy.AddMembers properties - - if ctx.GenerateConstructors then + let names, properties, parameters = List.unzip3 members + let properties = properties |> List.concat + objectTy.AddMembers properties + if ctx.GenerateConstructors then objectTy.AddMember <| - ProvidedConstructor(parameters, invokeCode = fun args -> - let properties = - Expr.NewArray(typeof, - args - |> List.mapi (fun i a -> Expr.NewTuple [ Expr.Value names.[i]; Expr.Coerce(a, typeof) ])) - let cultureStr = ctx.CultureStr - <@@ JsonRuntime.CreateRecord(%%properties, cultureStr) @@>) + ProvidedConstructor(parameters, invokeCode = fun args -> + let properties = + Expr.NewArray(typeof, + args + |> List.mapi (fun i a -> Expr.NewTuple [ Expr.Value names.[i]; Expr.Coerce(a, typeof) ])) + let cultureStr = ctx.CultureStr + <@@ JsonRuntime.CreateRecord(%%properties, cultureStr) @@>) + () + if ctx.GenerateConstructors then objectTy.AddMember <| ProvidedConstructor( [ProvidedParameter("jsonValue", ctx.JsonValueType)], invokeCode = fun (Singleton arg) -> <@@ JsonDocument.Create((%%arg:JsonValue), "") @@> ) - objectTy | InferedType.Collection (_, types) -> getOrCreateType ctx inferedType <| fun () -> diff --git a/src/Json/JsonProvider.fs b/src/Json/JsonProvider.fs index ab8b98bf6..74e3da112 100644 --- a/src/Json/JsonProvider.fs +++ b/src/Json/JsonProvider.fs @@ -39,6 +39,7 @@ type public JsonProvider(cfg:TypeProviderConfig) as this = let resolutionFolder = args.[5] :?> string let resource = args.[6] :?> string let inferTypesFromValues = args.[7] :?> bool + let inferDictionariesFromRecords = args.[8] :?> bool let cultureInfo = TextRuntime.GetCulture cultureStr @@ -58,7 +59,7 @@ type public JsonProvider(cfg:TypeProviderConfig) as this = using (IO.logTime "TypeGeneration" sample) <| fun _ -> - let ctx = JsonGenerationContext.Create(cultureStr, tpType) + let ctx = JsonGenerationContext.Create(cultureStr, tpType, ?inferDictionariesFromRecords = Some inferDictionariesFromRecords) let result = JsonTypeBuilder.generateJsonType ctx (*canPassAllConversionCallingTypes*)false (*optionalityHandledByParent*)false rootName inferedType { GeneratedType = tpType @@ -89,7 +90,8 @@ type public JsonProvider(cfg:TypeProviderConfig) as this = ProvidedStaticParameter("Encoding", typeof, parameterDefaultValue = "") ProvidedStaticParameter("ResolutionFolder", typeof, parameterDefaultValue = "") ProvidedStaticParameter("EmbeddedResource", typeof, parameterDefaultValue = "") - ProvidedStaticParameter("InferTypesFromValues", typeof, parameterDefaultValue = true) ] + ProvidedStaticParameter("InferTypesFromValues", typeof, parameterDefaultValue = true) + ProvidedStaticParameter("InferDictionariesFromRecords", typeof, parameterDefaultValue = false) ] let helpText = """Typed representation of a JSON document. @@ -102,7 +104,8 @@ type public JsonProvider(cfg:TypeProviderConfig) as this = When specified, the type provider first attempts to load the sample from the specified resource (e.g. 'MyCompany.MyAssembly, resource_name.json'). This is useful when exposing types generated by the type provider. If true, turns on additional type inference from values. - (e.g. type inference infers string values such as "123" as ints and values constrained to 0 and 1 as booleans.)""" + (e.g. type inference infers string values such as "123" as ints and values constrained to 0 and 1 as booleans.) + If true, json record is considered as a dictionary, if the names of all the its fields are infered (by type inference rules) into the same non-string primitive type.""" do jsonProvTy.AddXmlDoc helpText do jsonProvTy.DefineStaticParameters(parameters, buildTypes) diff --git a/src/Json/JsonRuntime.fs b/src/Json/JsonRuntime.fs index 152be9f84..9d4cbd788 100644 --- a/src/Json/JsonRuntime.fs +++ b/src/Json/JsonRuntime.fs @@ -157,6 +157,56 @@ type JsonRuntime = | JsonValue.Null -> [| |] | x -> failwithf "Expecting an array at '%s', got %s" (doc.Path()) <| x.ToString(JsonSaveOptions.DisableFormatting) + /// Get properties of the record + static member GetRecordProperties(doc:IJsonDocument) = + match doc.JsonValue with + | JsonValue.Record items -> items + | JsonValue.Null -> [||] + | x -> failwithf "Expecting a record at '%s', got %s" (doc.Path()) <| x.ToString(JsonSaveOptions.DisableFormatting) + + /// Converts JSON record to dictionary + static member ConvertRecordToDictionary<'Key, 'Value when 'Key: equality>(doc:IJsonDocument, mappingKey:Func, mappingValue:Func) = + JsonRuntime.GetRecordProperties(doc) + |> Seq.map (fun (k, v) -> + let key = doc.CreateNew(JsonValue.String k, k) |> mappingKey.Invoke + let value = doc.CreateNew(v, k) |> mappingValue.Invoke + key, value) + + + /// Get a value by the key from infered dictionary + static member InferedDictionaryContainsKey<'Key when 'Key: equality>(doc:IJsonDocument, mappingKey:Func, key: 'Key) = + let finder (k, _) = + (doc.CreateNew(JsonValue.String k, k) |> mappingKey.Invoke) = key + (JsonRuntime.GetRecordProperties(doc) |> Array.tryFind finder).IsSome + + /// Try get a value by the key from infered dictionary + static member TryGetValueByKeyFromInferedDictionary<'Key, 'Value when 'Key: equality>(doc:IJsonDocument, mappingKey:Func, mappingValue:Func, key: 'Key) = + let picker (k, v) = + if (doc.CreateNew(JsonValue.String k, k) |> mappingKey.Invoke) = key then + doc.CreateNew(v, k) |> mappingValue.Invoke |> Some + else + None + JsonRuntime.GetRecordProperties(doc) |> Array.tryPick picker + + /// Get a value by the key from infered dictionary + static member GetValueByKeyFromInferedDictionary<'Key, 'Value when 'Key: equality>(doc:IJsonDocument, mappingKey:Func, mappingValue:Func, key: 'Key) = + match JsonRuntime.TryGetValueByKeyFromInferedDictionary(doc, mappingKey, mappingValue, key) with + | Some value -> value + | _ -> key + |> sprintf "The given key '%A' was not present in the dictionary." + |> System.Collections.Generic.KeyNotFoundException + |> raise + + /// Get keys from infered dictionary + static member GetKeysFromInferedDictionary<'Key when 'Key: equality>(doc:IJsonDocument, mappingKey:Func) = + JsonRuntime.GetRecordProperties(doc) + |> Array.map (fun (k, _) -> doc.CreateNew(JsonValue.String k, k) |> mappingKey.Invoke) + + /// Get values from infered dictionary + static member GetValuesFromInferedDictionary<'Value>(doc:IJsonDocument, mappingValue:Func) = + JsonRuntime.GetRecordProperties(doc) + |> Array.map (fun (k, v) -> doc.CreateNew(v, k) |> mappingValue.Invoke) + /// Get optional json property static member TryGetPropertyUnpacked(doc:IJsonDocument, name) = doc.JsonValue.TryGetProperty(name) @@ -301,6 +351,16 @@ type JsonRuntime = |> JsonValue.Record JsonDocument.Create(json, "") + // Creates a JsonValue.Record from key*value seq and wraps it in a json document + static member CreateRecordFromDictionary<'Key, 'Value when 'Key: equality>(keyValuePairs: ('Key * 'Value) seq, cultureStr, mappingKeyBack: Func<'Key, string>) = + let cultureInfo = TextRuntime.GetCulture cultureStr + let json = + keyValuePairs + |> Seq.map (fun (k, v) -> (k |> mappingKeyBack.Invoke), JsonRuntime.ToJsonValue cultureInfo (v :> obj)) + |> Seq.toArray + |> JsonValue.Record + JsonDocument.Create(json, "") + /// Creates a scalar JsonValue.Array and wraps it in a json document static member CreateArray(elements:obj[], cultureStr) = let cultureInfo = TextRuntime.GetCulture cultureStr diff --git a/src/Test.fsx b/src/Test.fsx index 10955cca0..ef59cb462 100644 --- a/src/Test.fsx +++ b/src/Test.fsx @@ -66,7 +66,8 @@ Json { Sample = "optionals.json" Encoding = "" ResolutionFolder = "" EmbeddedResource = "" - InferTypesFromValues = true } + InferTypesFromValues = true + InferDictionariesFromRecords = false } |> dumpAll Xml { Sample = "JsonInXml.xml" diff --git a/tests/FSharp.Data.DesignTime.Tests/SignatureTestCases.config b/tests/FSharp.Data.DesignTime.Tests/SignatureTestCases.config index a7b3894c1..47a3ae15b 100644 --- a/tests/FSharp.Data.DesignTime.Tests/SignatureTestCases.config +++ b/tests/FSharp.Data.DesignTime.Tests/SignatureTestCases.config @@ -35,27 +35,29 @@ Xml,TimeSpans.xml,false,false,,true, Xml,,false,false,,false,po.xsd Xml,,false,false,,false,homonim.xsd Xml,,false,false,,false,IncludeFromWeb.xsd -Json,WorldBank.json,false,WorldBank,,true -Json,TwitterStream.json,true,,,true -Json,TwitterSample.json,true,,,true -Json,OptionValues.json,false,,,true -Json,SimpleArray.json,false,,,true -Json,DoubleNested.json,false,,,true -Json,Nested.json,false,,,true -Json,Simple.json,false,,,true -Json,WikiData.json,false,,,true -Json,Empty.json,false,,,true -Json,projects.json,false,,,true -Json,Dates.json,false,,,true -Json,GitHub.json,false,,,true -Json,topics.json,true,Topic,,true -Json,Vindinium.json,false,,,true -Json,contacts.json,false,,,true -Json,optionals.json,false,,,true -Json,reddit.json,false,,,true -Json,TypeInference.json,false,,,true -Json,TypeInference.json,false,,,false -Json,TimeSpans.json,false,,,true +Json,WorldBank.json,false,WorldBank,,true,false,false +Json,TwitterStream.json,true,,,true,false +Json,TwitterSample.json,true,,,true,false +Json,OptionValues.json,false,,,true,false +Json,SimpleArray.json,false,,,true,false +Json,DoubleNested.json,false,,,true,false +Json,Nested.json,false,,,true,false +Json,Simple.json,false,,,true,false +Json,WikiData.json,false,,,true,false +Json,Empty.json,false,,,true,false +Json,projects.json,false,,,true,false +Json,Dates.json,false,,,true,false +Json,GitHub.json,false,,,true,false +Json,topics.json,true,Topic,,true,false +Json,Vindinium.json,false,,,true,false +Json,contacts.json,false,,,true,false +Json,optionals.json,false,,,true,false +Json,reddit.json,false,,,true,false +Json,TypeInference.json,false,,,true,false +Json,TypeInference.json,false,,,false,false +Json,TimeSpans.json,false,,,true,false +Json,DictionaryInference.json,false,,,true,false +Json,DictionaryInference.json,false,,,true,true Html,MarketDepth.htm,false,false, Html,MarketDepth.htm,true,false, Html,SimpleHtmlTablesWithTr.html,false,false, diff --git a/tests/FSharp.Data.DesignTime.Tests/TypeProviderInstantiation.fs b/tests/FSharp.Data.DesignTime.Tests/TypeProviderInstantiation.fs index 41fc8bbe8..f09a4cbcc 100644 --- a/tests/FSharp.Data.DesignTime.Tests/TypeProviderInstantiation.fs +++ b/tests/FSharp.Data.DesignTime.Tests/TypeProviderInstantiation.fs @@ -44,7 +44,8 @@ type JsonProviderArgs = Encoding : string ResolutionFolder : string EmbeddedResource : string - InferTypesFromValues : bool } + InferTypesFromValues : bool + InferDictionariesFromRecords : bool } type HtmlProviderArgs = { Sample : string @@ -110,7 +111,8 @@ type TypeProviderInstantiation = box x.Encoding box x.ResolutionFolder box x.EmbeddedResource - box x.InferTypesFromValues |] + box x.InferTypesFromValues + box x.InferDictionariesFromRecords |] | Html x -> (fun cfg -> new HtmlProvider(cfg) :> TypeProviderForNamespaces), [| box x.Sample @@ -155,7 +157,8 @@ type TypeProviderInstantiation = x.SampleIsList.ToString() x.RootName x.Culture - x.InferTypesFromValues.ToString() ] + x.InferTypesFromValues.ToString() + x.InferDictionariesFromRecords.ToString() ] | Html x -> ["Html" x.Sample @@ -223,7 +226,8 @@ type TypeProviderInstantiation = Encoding = "" ResolutionFolder = "" EmbeddedResource = "" - InferTypesFromValues = args.[5] |> bool.Parse } + InferTypesFromValues = args.[5] |> bool.Parse + InferDictionariesFromRecords = args.[6] |> bool.Parse } | "Html" -> Html { Sample = args.[1] PreferOptionals = args.[2] |> bool.Parse diff --git a/tests/FSharp.Data.DesignTime.Tests/expected/Json,Dates.json,False,,,True.expected b/tests/FSharp.Data.DesignTime.Tests/expected/Json,Dates.json,False,,,True,False.expected similarity index 100% rename from tests/FSharp.Data.DesignTime.Tests/expected/Json,Dates.json,False,,,True.expected rename to tests/FSharp.Data.DesignTime.Tests/expected/Json,Dates.json,False,,,True,False.expected diff --git a/tests/FSharp.Data.DesignTime.Tests/expected/Json,DictionaryInference.json,False,,,True,False.expected b/tests/FSharp.Data.DesignTime.Tests/expected/Json,DictionaryInference.json,False,,,True,False.expected new file mode 100644 index 000000000..9b488861b --- /dev/null +++ b/tests/FSharp.Data.DesignTime.Tests/expected/Json,DictionaryInference.json,False,,,True,False.expected @@ -0,0 +1,96 @@ +class JsonProvider : obj + static member AsyncGetSamples: () -> JsonProvider+JsonProvider+Root[] async + let f = new Func<_,_>(fun (t:TextReader) -> JsonRuntime.ConvertArray(JsonDocument.Create(t), new Func<_,_>(id)))) + TextRuntime.AsyncMap((IO.asyncReadTextAtRuntimeWithDesignTimeRules "" "" "JSON" "" "DictionaryInference.json"), f) + + static member AsyncLoad: uri:string -> JsonProvider+JsonProvider+Root[] async + let f = new Func<_,_>(fun (t:TextReader) -> JsonRuntime.ConvertArray(JsonDocument.Create(t), new Func<_,_>(id)))) + TextRuntime.AsyncMap((IO.asyncReadTextAtRuntime false "" "" "JSON" "" uri), f) + + static member GetSamples: () -> JsonProvider+JsonProvider+Root[] + JsonRuntime.ConvertArray(JsonDocument.Create(FSharpAsync.RunSynchronously((IO.asyncReadTextAtRuntimeWithDesignTimeRules "" "" "JSON" "" "DictionaryInference.json"))), new Func<_,_>(id))) + + static member Load: stream:System.IO.Stream -> JsonProvider+JsonProvider+Root[] + JsonRuntime.ConvertArray(JsonDocument.Create(((new StreamReader(stream)) :> TextReader)), new Func<_,_>(id))) + + static member Load: reader:System.IO.TextReader -> JsonProvider+JsonProvider+Root[] + JsonRuntime.ConvertArray(JsonDocument.Create(reader), new Func<_,_>(id))) + + static member Load: uri:string -> JsonProvider+JsonProvider+Root[] + JsonRuntime.ConvertArray(JsonDocument.Create(FSharpAsync.RunSynchronously((IO.asyncReadTextAtRuntime false "" "" "JSON" "" uri))), new Func<_,_>(id))) + + static member Load: value:JsonValue -> JsonProvider+JsonProvider+Root[] + JsonRuntime.ConvertArray(JsonDocument.Create(value, ""), new Func<_,_>(id))) + + static member Parse: text:string -> JsonProvider+JsonProvider+Root[] + JsonRuntime.ConvertArray(JsonDocument.Create(((new StringReader(text)) :> TextReader)), new Func<_,_>(id))) + + static member ParseList: text:string -> JsonProvider+JsonProvider+JsonProvider+Root[][] + JsonRuntime.ConvertArray(JsonDocument.CreateList(((new StringReader(text)) :> TextReader)), new Func<_,_>(id))) + + +class JsonProvider+Root : FDR.BaseTypes.IJsonDocument + new : rec:JsonProvider+Rec -> rec2:JsonProvider+Rec2 -> JsonProvider+Root + JsonRuntime.CreateRecord([| ("rec", + (rec :> obj)) + ("rec2", + (rec2 :> obj)) |], "") + + new : jsonValue:JsonValue -> JsonProvider+Root + JsonDocument.Create(jsonValue, "") + + member Rec: JsonProvider+Rec with get + JsonRuntime.GetPropertyPacked(this, "rec") + + member Rec2: JsonProvider+Rec2 with get + JsonRuntime.GetPropertyPacked(this, "rec2") + + +class JsonProvider+Rec : FDR.BaseTypes.IJsonDocument + new : 0:int -> 1:int option -> JsonProvider+Rec + JsonRuntime.CreateRecord([| ("0", + (0 :> obj)) + ("1", + (1 :> obj)) |], "") + + new : jsonValue:JsonValue -> JsonProvider+Rec + JsonDocument.Create(jsonValue, "") + + member 0: int with get + let value = JsonRuntime.TryGetPropertyUnpackedWithPath(this, "0") + JsonRuntime.GetNonOptionalValue(value.Path, JsonRuntime.ConvertInteger("", value.JsonOpt), value.JsonOpt) + + member 1: int option with get + JsonRuntime.ConvertInteger("", JsonRuntime.TryGetPropertyUnpacked(this, "1")) + + +class JsonProvider+Rec2 : FDR.BaseTypes.IJsonDocument + new : 0:JsonProvider+0 option -> 1:JsonProvider+0 -> JsonProvider+Rec2 + JsonRuntime.CreateRecord([| ("0", + (0 :> obj)) + ("1", + (1 :> obj)) |], "") + + new : jsonValue:JsonValue -> JsonProvider+Rec2 + JsonDocument.Create(jsonValue, "") + + member 0: JsonProvider+0 option with get + JsonRuntime.TryGetPropertyPacked(this, "0") + + member 1: JsonProvider+0 with get + JsonRuntime.GetPropertyPacked(this, "1") + + +class JsonProvider+0 : FDR.BaseTypes.IJsonDocument + new : a:int -> JsonProvider+0 + JsonRuntime.CreateRecord([| ("a", + (a :> obj)) |], "") + + new : jsonValue:JsonValue -> JsonProvider+0 + JsonDocument.Create(jsonValue, "") + + member A: int with get + let value = JsonRuntime.TryGetPropertyUnpackedWithPath(this, "a") + JsonRuntime.GetNonOptionalValue(value.Path, JsonRuntime.ConvertInteger("", value.JsonOpt), value.JsonOpt) + + diff --git a/tests/FSharp.Data.DesignTime.Tests/expected/Json,DictionaryInference.json,False,,,True,True.expected b/tests/FSharp.Data.DesignTime.Tests/expected/Json,DictionaryInference.json,False,,,True,True.expected new file mode 100644 index 000000000..c27719b9d --- /dev/null +++ b/tests/FSharp.Data.DesignTime.Tests/expected/Json,DictionaryInference.json,False,,,True,True.expected @@ -0,0 +1,125 @@ +class JsonProvider : obj + static member AsyncGetSamples: () -> JsonProvider+JsonProvider+Root[] async + let f = new Func<_,_>(fun (t:TextReader) -> JsonRuntime.ConvertArray(JsonDocument.Create(t), new Func<_,_>(id)))) + TextRuntime.AsyncMap((IO.asyncReadTextAtRuntimeWithDesignTimeRules "" "" "JSON" "" "DictionaryInference.json"), f) + + static member AsyncLoad: uri:string -> JsonProvider+JsonProvider+Root[] async + let f = new Func<_,_>(fun (t:TextReader) -> JsonRuntime.ConvertArray(JsonDocument.Create(t), new Func<_,_>(id)))) + TextRuntime.AsyncMap((IO.asyncReadTextAtRuntime false "" "" "JSON" "" uri), f) + + static member GetSamples: () -> JsonProvider+JsonProvider+Root[] + JsonRuntime.ConvertArray(JsonDocument.Create(FSharpAsync.RunSynchronously((IO.asyncReadTextAtRuntimeWithDesignTimeRules "" "" "JSON" "" "DictionaryInference.json"))), new Func<_,_>(id))) + + static member Load: stream:System.IO.Stream -> JsonProvider+JsonProvider+Root[] + JsonRuntime.ConvertArray(JsonDocument.Create(((new StreamReader(stream)) :> TextReader)), new Func<_,_>(id))) + + static member Load: reader:System.IO.TextReader -> JsonProvider+JsonProvider+Root[] + JsonRuntime.ConvertArray(JsonDocument.Create(reader), new Func<_,_>(id))) + + static member Load: uri:string -> JsonProvider+JsonProvider+Root[] + JsonRuntime.ConvertArray(JsonDocument.Create(FSharpAsync.RunSynchronously((IO.asyncReadTextAtRuntime false "" "" "JSON" "" uri))), new Func<_,_>(id))) + + static member Load: value:JsonValue -> JsonProvider+JsonProvider+Root[] + JsonRuntime.ConvertArray(JsonDocument.Create(value, ""), new Func<_,_>(id))) + + static member Parse: text:string -> JsonProvider+JsonProvider+Root[] + JsonRuntime.ConvertArray(JsonDocument.Create(((new StringReader(text)) :> TextReader)), new Func<_,_>(id))) + + static member ParseList: text:string -> JsonProvider+JsonProvider+JsonProvider+Root[][] + JsonRuntime.ConvertArray(JsonDocument.CreateList(((new StringReader(text)) :> TextReader)), new Func<_,_>(id))) + + +class JsonProvider+Root : FDR.BaseTypes.IJsonDocument + new : rec:JsonProvider+Rec -> rec2:JsonProvider+Rec2 -> JsonProvider+Root + JsonRuntime.CreateRecord([| ("rec", + (rec :> obj)) + ("rec2", + (rec2 :> obj)) |], "") + + new : jsonValue:JsonValue -> JsonProvider+Root + JsonDocument.Create(jsonValue, "") + + member Rec: JsonProvider+Rec with get + JsonRuntime.GetPropertyPacked(this, "rec") + + member Rec2: JsonProvider+Rec2 with get + JsonRuntime.GetPropertyPacked(this, "rec2") + + +class JsonProvider+Rec : FDR.BaseTypes.IJsonDocument + new : items:bool * int seq -> JsonProvider+Rec + JsonRuntime.CreateRecordFromDictionary(items, "", new Func<_,_>(fun (t:bool) -> TextRuntime.ConvertBooleanBack(Some t, false))) + + new : jsonValue:JsonValue -> JsonProvider+Rec + JsonDocument.Create(jsonValue, "") + + member ContainsKey: key:bool -> bool + JsonRuntime.InferedDictionaryContainsKey(this, new Func<_,_>(fun (t:IJsonDocument) -> JsonRuntime.GetNonOptionalValue(t.Path(), JsonRuntime.ConvertBoolean(Some t.JsonValue), Some t.JsonValue)), key) + + member Count: int with get + JsonRuntime.GetRecordProperties(this).Length + + member IsEmpty: bool with get + (Operators.op_Equality JsonRuntime.GetRecordProperties(this).Length 0) + + member Item: int with get + JsonRuntime.GetValueByKeyFromInferedDictionary(this, new Func<_,_>(fun (t:IJsonDocument) -> JsonRuntime.GetNonOptionalValue(t.Path(), JsonRuntime.ConvertBoolean(Some t.JsonValue), Some t.JsonValue)), new Func<_,_>(fun (t:IJsonDocument) -> JsonRuntime.GetNonOptionalValue(t.Path(), JsonRuntime.ConvertInteger("", Some t.JsonValue), Some t.JsonValue)), key) + + member Items: bool * int seq with get + JsonRuntime.ConvertRecordToDictionary(this, new Func<_,_>(fun (t:IJsonDocument) -> JsonRuntime.GetNonOptionalValue(t.Path(), JsonRuntime.ConvertBoolean(Some t.JsonValue), Some t.JsonValue)), new Func<_,_>(fun (t:IJsonDocument) -> JsonRuntime.GetNonOptionalValue(t.Path(), JsonRuntime.ConvertInteger("", Some t.JsonValue), Some t.JsonValue))) + + member Keys: bool[] with get + JsonRuntime.GetKeysFromInferedDictionary(this, new Func<_,_>(fun (t:IJsonDocument) -> JsonRuntime.GetNonOptionalValue(t.Path(), JsonRuntime.ConvertBoolean(Some t.JsonValue), Some t.JsonValue))) + + member TryFind: key:bool -> int option + JsonRuntime.TryGetValueByKeyFromInferedDictionary(this, new Func<_,_>(fun (t:IJsonDocument) -> JsonRuntime.GetNonOptionalValue(t.Path(), JsonRuntime.ConvertBoolean(Some t.JsonValue), Some t.JsonValue)), new Func<_,_>(fun (t:IJsonDocument) -> JsonRuntime.GetNonOptionalValue(t.Path(), JsonRuntime.ConvertInteger("", Some t.JsonValue), Some t.JsonValue)), key) + + member Values: int[] with get + JsonRuntime.GetValuesFromInferedDictionary(this, new Func<_,_>(fun (t:IJsonDocument) -> JsonRuntime.GetNonOptionalValue(t.Path(), JsonRuntime.ConvertInteger("", Some t.JsonValue), Some t.JsonValue))) + + +class JsonProvider+Rec2 : FDR.BaseTypes.IJsonDocument + new : items:bool * JsonProvider+Rec2Value seq -> JsonProvider+Rec2 + JsonRuntime.CreateRecordFromDictionary(items, "", new Func<_,_>(fun (t:bool) -> TextRuntime.ConvertBooleanBack(Some t, false))) + + new : jsonValue:JsonValue -> JsonProvider+Rec2 + JsonDocument.Create(jsonValue, "") + + member ContainsKey: key:bool -> bool + JsonRuntime.InferedDictionaryContainsKey(this, new Func<_,_>(fun (t:IJsonDocument) -> JsonRuntime.GetNonOptionalValue(t.Path(), JsonRuntime.ConvertBoolean(Some t.JsonValue), Some t.JsonValue)), key) + + member Count: int with get + JsonRuntime.GetRecordProperties(this).Length + + member IsEmpty: bool with get + (Operators.op_Equality JsonRuntime.GetRecordProperties(this).Length 0) + + member Item: JsonProvider+Rec2Value with get + JsonRuntime.GetValueByKeyFromInferedDictionary(this, new Func<_,_>(fun (t:IJsonDocument) -> JsonRuntime.GetNonOptionalValue(t.Path(), JsonRuntime.ConvertBoolean(Some t.JsonValue), Some t.JsonValue)), new Func<_,_>(id)), key) + + member Items: bool * JsonProvider+Rec2Value seq with get + JsonRuntime.ConvertRecordToDictionary(this, new Func<_,_>(fun (t:IJsonDocument) -> JsonRuntime.GetNonOptionalValue(t.Path(), JsonRuntime.ConvertBoolean(Some t.JsonValue), Some t.JsonValue)), new Func<_,_>(id))) + + member Keys: bool[] with get + JsonRuntime.GetKeysFromInferedDictionary(this, new Func<_,_>(fun (t:IJsonDocument) -> JsonRuntime.GetNonOptionalValue(t.Path(), JsonRuntime.ConvertBoolean(Some t.JsonValue), Some t.JsonValue))) + + member TryFind: key:bool -> JsonProvider+Rec2Value option + JsonRuntime.TryGetValueByKeyFromInferedDictionary(this, new Func<_,_>(fun (t:IJsonDocument) -> JsonRuntime.GetNonOptionalValue(t.Path(), JsonRuntime.ConvertBoolean(Some t.JsonValue), Some t.JsonValue)), new Func<_,_>(id)), key) + + member Values: JsonProvider+JsonProvider+Rec2Value[] with get + JsonRuntime.GetValuesFromInferedDictionary(this, new Func<_,_>(id))) + + +class JsonProvider+Rec2Value : FDR.BaseTypes.IJsonDocument + new : a:int -> JsonProvider+Rec2Value + JsonRuntime.CreateRecord([| ("a", + (a :> obj)) |], "") + + new : jsonValue:JsonValue -> JsonProvider+Rec2Value + JsonDocument.Create(jsonValue, "") + + member A: int with get + let value = JsonRuntime.TryGetPropertyUnpackedWithPath(this, "a") + JsonRuntime.GetNonOptionalValue(value.Path, JsonRuntime.ConvertInteger("", value.JsonOpt), value.JsonOpt) + + diff --git a/tests/FSharp.Data.DesignTime.Tests/expected/Json,DoubleNested.json,False,,,True.expected b/tests/FSharp.Data.DesignTime.Tests/expected/Json,DoubleNested.json,False,,,True,False.expected similarity index 100% rename from tests/FSharp.Data.DesignTime.Tests/expected/Json,DoubleNested.json,False,,,True.expected rename to tests/FSharp.Data.DesignTime.Tests/expected/Json,DoubleNested.json,False,,,True,False.expected diff --git a/tests/FSharp.Data.DesignTime.Tests/expected/Json,Empty.json,False,,,True.expected b/tests/FSharp.Data.DesignTime.Tests/expected/Json,Empty.json,False,,,True,False.expected similarity index 100% rename from tests/FSharp.Data.DesignTime.Tests/expected/Json,Empty.json,False,,,True.expected rename to tests/FSharp.Data.DesignTime.Tests/expected/Json,Empty.json,False,,,True,False.expected diff --git a/tests/FSharp.Data.DesignTime.Tests/expected/Json,GitHub.json,False,,,True.expected b/tests/FSharp.Data.DesignTime.Tests/expected/Json,GitHub.json,False,,,True,False.expected similarity index 100% rename from tests/FSharp.Data.DesignTime.Tests/expected/Json,GitHub.json,False,,,True.expected rename to tests/FSharp.Data.DesignTime.Tests/expected/Json,GitHub.json,False,,,True,False.expected diff --git a/tests/FSharp.Data.DesignTime.Tests/expected/Json,Nested.json,False,,,True.expected b/tests/FSharp.Data.DesignTime.Tests/expected/Json,Nested.json,False,,,True,False.expected similarity index 100% rename from tests/FSharp.Data.DesignTime.Tests/expected/Json,Nested.json,False,,,True.expected rename to tests/FSharp.Data.DesignTime.Tests/expected/Json,Nested.json,False,,,True,False.expected diff --git a/tests/FSharp.Data.DesignTime.Tests/expected/Json,OptionValues.json,False,,,True.expected b/tests/FSharp.Data.DesignTime.Tests/expected/Json,OptionValues.json,False,,,True,False.expected similarity index 100% rename from tests/FSharp.Data.DesignTime.Tests/expected/Json,OptionValues.json,False,,,True.expected rename to tests/FSharp.Data.DesignTime.Tests/expected/Json,OptionValues.json,False,,,True,False.expected diff --git a/tests/FSharp.Data.DesignTime.Tests/expected/Json,Simple.json,False,,,True.expected b/tests/FSharp.Data.DesignTime.Tests/expected/Json,Simple.json,False,,,True,False.expected similarity index 100% rename from tests/FSharp.Data.DesignTime.Tests/expected/Json,Simple.json,False,,,True.expected rename to tests/FSharp.Data.DesignTime.Tests/expected/Json,Simple.json,False,,,True,False.expected diff --git a/tests/FSharp.Data.DesignTime.Tests/expected/Json,SimpleArray.json,False,,,True.expected b/tests/FSharp.Data.DesignTime.Tests/expected/Json,SimpleArray.json,False,,,True,False.expected similarity index 100% rename from tests/FSharp.Data.DesignTime.Tests/expected/Json,SimpleArray.json,False,,,True.expected rename to tests/FSharp.Data.DesignTime.Tests/expected/Json,SimpleArray.json,False,,,True,False.expected diff --git a/tests/FSharp.Data.DesignTime.Tests/expected/Json,TimeSpans.json,False,,,True.expected b/tests/FSharp.Data.DesignTime.Tests/expected/Json,TimeSpans.json,False,,,True,False.expected similarity index 100% rename from tests/FSharp.Data.DesignTime.Tests/expected/Json,TimeSpans.json,False,,,True.expected rename to tests/FSharp.Data.DesignTime.Tests/expected/Json,TimeSpans.json,False,,,True,False.expected diff --git a/tests/FSharp.Data.DesignTime.Tests/expected/Json,TwitterSample.json,True,,,True.expected b/tests/FSharp.Data.DesignTime.Tests/expected/Json,TwitterSample.json,True,,,True,False.expected similarity index 100% rename from tests/FSharp.Data.DesignTime.Tests/expected/Json,TwitterSample.json,True,,,True.expected rename to tests/FSharp.Data.DesignTime.Tests/expected/Json,TwitterSample.json,True,,,True,False.expected diff --git a/tests/FSharp.Data.DesignTime.Tests/expected/Json,TwitterStream.json,True,,,True.expected b/tests/FSharp.Data.DesignTime.Tests/expected/Json,TwitterStream.json,True,,,True,False.expected similarity index 100% rename from tests/FSharp.Data.DesignTime.Tests/expected/Json,TwitterStream.json,True,,,True.expected rename to tests/FSharp.Data.DesignTime.Tests/expected/Json,TwitterStream.json,True,,,True,False.expected diff --git a/tests/FSharp.Data.DesignTime.Tests/expected/Json,TypeInference.json,False,,,False.expected b/tests/FSharp.Data.DesignTime.Tests/expected/Json,TypeInference.json,False,,,False,False.expected similarity index 100% rename from tests/FSharp.Data.DesignTime.Tests/expected/Json,TypeInference.json,False,,,False.expected rename to tests/FSharp.Data.DesignTime.Tests/expected/Json,TypeInference.json,False,,,False,False.expected diff --git a/tests/FSharp.Data.DesignTime.Tests/expected/Json,TypeInference.json,False,,,True.expected b/tests/FSharp.Data.DesignTime.Tests/expected/Json,TypeInference.json,False,,,True,False.expected similarity index 100% rename from tests/FSharp.Data.DesignTime.Tests/expected/Json,TypeInference.json,False,,,True.expected rename to tests/FSharp.Data.DesignTime.Tests/expected/Json,TypeInference.json,False,,,True,False.expected diff --git a/tests/FSharp.Data.DesignTime.Tests/expected/Json,Vindinium.json,False,,,True.expected b/tests/FSharp.Data.DesignTime.Tests/expected/Json,Vindinium.json,False,,,True,False.expected similarity index 100% rename from tests/FSharp.Data.DesignTime.Tests/expected/Json,Vindinium.json,False,,,True.expected rename to tests/FSharp.Data.DesignTime.Tests/expected/Json,Vindinium.json,False,,,True,False.expected diff --git a/tests/FSharp.Data.DesignTime.Tests/expected/Json,WikiData.json,False,,,True.expected b/tests/FSharp.Data.DesignTime.Tests/expected/Json,WikiData.json,False,,,True,False.expected similarity index 100% rename from tests/FSharp.Data.DesignTime.Tests/expected/Json,WikiData.json,False,,,True.expected rename to tests/FSharp.Data.DesignTime.Tests/expected/Json,WikiData.json,False,,,True,False.expected diff --git a/tests/FSharp.Data.DesignTime.Tests/expected/Json,WorldBank.json,False,WorldBank,,True.expected b/tests/FSharp.Data.DesignTime.Tests/expected/Json,WorldBank.json,False,WorldBank,,True,False.expected similarity index 100% rename from tests/FSharp.Data.DesignTime.Tests/expected/Json,WorldBank.json,False,WorldBank,,True.expected rename to tests/FSharp.Data.DesignTime.Tests/expected/Json,WorldBank.json,False,WorldBank,,True,False.expected diff --git a/tests/FSharp.Data.DesignTime.Tests/expected/Json,contacts.json,False,,,True.expected b/tests/FSharp.Data.DesignTime.Tests/expected/Json,contacts.json,False,,,True,False.expected similarity index 100% rename from tests/FSharp.Data.DesignTime.Tests/expected/Json,contacts.json,False,,,True.expected rename to tests/FSharp.Data.DesignTime.Tests/expected/Json,contacts.json,False,,,True,False.expected diff --git a/tests/FSharp.Data.DesignTime.Tests/expected/Json,optionals.json,False,,,True.expected b/tests/FSharp.Data.DesignTime.Tests/expected/Json,optionals.json,False,,,True,False.expected similarity index 100% rename from tests/FSharp.Data.DesignTime.Tests/expected/Json,optionals.json,False,,,True.expected rename to tests/FSharp.Data.DesignTime.Tests/expected/Json,optionals.json,False,,,True,False.expected diff --git a/tests/FSharp.Data.DesignTime.Tests/expected/Json,projects.json,False,,,True.expected b/tests/FSharp.Data.DesignTime.Tests/expected/Json,projects.json,False,,,True,False.expected similarity index 100% rename from tests/FSharp.Data.DesignTime.Tests/expected/Json,projects.json,False,,,True.expected rename to tests/FSharp.Data.DesignTime.Tests/expected/Json,projects.json,False,,,True,False.expected diff --git a/tests/FSharp.Data.DesignTime.Tests/expected/Json,reddit.json,False,,,True.expected b/tests/FSharp.Data.DesignTime.Tests/expected/Json,reddit.json,False,,,True,False.expected similarity index 100% rename from tests/FSharp.Data.DesignTime.Tests/expected/Json,reddit.json,False,,,True.expected rename to tests/FSharp.Data.DesignTime.Tests/expected/Json,reddit.json,False,,,True,False.expected diff --git a/tests/FSharp.Data.DesignTime.Tests/expected/Json,topics.json,True,Topic,,True.expected b/tests/FSharp.Data.DesignTime.Tests/expected/Json,topics.json,True,Topic,,True,False.expected similarity index 100% rename from tests/FSharp.Data.DesignTime.Tests/expected/Json,topics.json,True,Topic,,True.expected rename to tests/FSharp.Data.DesignTime.Tests/expected/Json,topics.json,True,Topic,,True,False.expected diff --git a/tests/FSharp.Data.Tests/Data/DictionaryInference.json b/tests/FSharp.Data.Tests/Data/DictionaryInference.json new file mode 100644 index 000000000..b78d6851d --- /dev/null +++ b/tests/FSharp.Data.Tests/Data/DictionaryInference.json @@ -0,0 +1,10 @@ +[ + { + "rec": { "0": "111", "1": "222" }, + "rec2": { "0": { "a": "10" }, "1": { "a": "20" } } + }, + { + "rec": { "0": "333" }, + "rec2": { "1": { "a": "30" }, "1": { "a": "40" } } + } +] \ No newline at end of file diff --git a/tests/FSharp.Data.Tests/JsonProvider.fs b/tests/FSharp.Data.Tests/JsonProvider.fs index 96a4fe042..82d005d22 100644 --- a/tests/FSharp.Data.Tests/JsonProvider.fs +++ b/tests/FSharp.Data.Tests/JsonProvider.fs @@ -757,3 +757,23 @@ let ``Can load different nested payloads`` () = payload2.User |> should equal "alice" payload2.Role |> should equal "admin" payload2.RegisteredSince |> should equal (DateTime(2021, 11, 1)) + + +[] +let ``Can control dictionary inference`` () = + let notinferred = JsonProvider<"Data/DictionaryInference.json", InferDictionariesFromRecords=false>.GetSamples().[0] + + notinferred.Rec.``0`` |> should equal 111 + notinferred.Rec.``1`` |> should equal (Some 222) + + let inferred = JsonProvider<"Data/DictionaryInference.json", InferDictionariesFromRecords=true>.GetSamples().[0] + + inferred.Rec.Count |> should equal 2 + inferred.Rec.IsEmpty |> should equal false + + inferred.Rec.ContainsKey(false) |> should equal true + inferred.Rec.[true] |> should equal 222 + inferred.Rec.TryFind(true) |> should equal <| Some 222 + + inferred.Rec.Values |> should equal [|111; 222|] + inferred.Rec.Keys |> should equal [|false; true|] \ No newline at end of file From 24e49d710d18762efb7821fe9f1866adde26b611 Mon Sep 17 00:00:00 2001 From: nikoyak <99891966+nikoyak@users.noreply.github.com> Date: Sat, 19 Feb 2022 16:16:42 +0300 Subject: [PATCH 2/3] Minor refactorings - removed extra pipe ! added missing newline at end of the files + code reusing --- src/Json/JsonGenerator.fs | 7 ++----- tests/FSharp.Data.Tests/Data/DictionaryInference.json | 2 +- tests/FSharp.Data.Tests/JsonProvider.fs | 2 +- 3 files changed, 4 insertions(+), 7 deletions(-) diff --git a/src/Json/JsonGenerator.fs b/src/Json/JsonGenerator.fs index 7ed9ac924..6060cce97 100644 --- a/src/Json/JsonGenerator.fs +++ b/src/Json/JsonGenerator.fs @@ -301,7 +301,7 @@ module JsonTypeBuilder = | InferedType.Record (_, fields, opt) -> InferedType.Record (None, fields, opt) | _ -> infType - if not <| ctx.InferDictionariesFromRecords + if not ctx.InferDictionariesFromRecords then None else let infType = @@ -321,13 +321,10 @@ module JsonTypeBuilder = let keyResult = generateJsonType ctx (*canPassAllConversionCallingTypes*)false (*optionalityHandledByParent*)true "" inferedKeyType let valueResult = generateJsonType ctx (*canPassAllConversionCallingTypes*)false (*optionalityHandledByParent*)true valueName inferedValueType let valueConvertedTypeErased = valueResult.ConvertedTypeErased ctx - let valueOptionalType = typedefof<_ option>.MakeGenericType([|valueResult.ConvertedType|]) let tupleType = Microsoft.FSharp.Reflection.FSharpType.MakeTupleType([|keyResult.ConvertedType; valueResult.ConvertedType|]) let itemsSeqType = typedefof<_ seq>.MakeGenericType([|tupleType|]) - //let itemsSeqType = typedefof>.MakeGenericType([|valueResult.ConvertedType|]) - let itemsGetter = fun (Singleton jDoc) -> ctx.JsonRuntimeType?ConvertRecordToDictionary (keyResult.ConvertedType, valueConvertedTypeErased) (jDoc, keyResult.ConverterFunc ctx, valueResult.ConverterFunc ctx) @@ -363,7 +360,7 @@ module JsonTypeBuilder = ProvidedProperty("IsEmpty", typeof, getterCode = isEmptyGetter) ] |> objectTy.AddMembers [ - ProvidedMethod("TryFind", [ProvidedParameter("key", keyResult.ConvertedType)], valueOptionalType, tryFindCode) + ProvidedMethod("TryFind", [ProvidedParameter("key", keyResult.ConvertedType)], valueResult.ConvertedType |> ctx.MakeOptionType, tryFindCode) ProvidedMethod("ContainsKey", [ProvidedParameter("key", keyResult.ConvertedType)], typeof, containsKeyCode) ] |> objectTy.AddMembers if ctx.GenerateConstructors then diff --git a/tests/FSharp.Data.Tests/Data/DictionaryInference.json b/tests/FSharp.Data.Tests/Data/DictionaryInference.json index b78d6851d..7419b320c 100644 --- a/tests/FSharp.Data.Tests/Data/DictionaryInference.json +++ b/tests/FSharp.Data.Tests/Data/DictionaryInference.json @@ -7,4 +7,4 @@ "rec": { "0": "333" }, "rec2": { "1": { "a": "30" }, "1": { "a": "40" } } } -] \ No newline at end of file +] diff --git a/tests/FSharp.Data.Tests/JsonProvider.fs b/tests/FSharp.Data.Tests/JsonProvider.fs index 82d005d22..7fb509ba6 100644 --- a/tests/FSharp.Data.Tests/JsonProvider.fs +++ b/tests/FSharp.Data.Tests/JsonProvider.fs @@ -776,4 +776,4 @@ let ``Can control dictionary inference`` () = inferred.Rec.TryFind(true) |> should equal <| Some 222 inferred.Rec.Values |> should equal [|111; 222|] - inferred.Rec.Keys |> should equal [|false; true|] \ No newline at end of file + inferred.Rec.Keys |> should equal [|false; true|] From 61d3cc275d118ddc08ec87f40fb2ffabf89029f3 Mon Sep 17 00:00:00 2001 From: nikoyak <99891966+nikoyak@users.noreply.github.com> Date: Fri, 25 Feb 2022 00:05:52 +0300 Subject: [PATCH 3/3] Changed based on PR#1430 reviews - dropped overused lambda expressions * renamed the InferDictionariesFromRecords flag into PreferDictionaries --- src/Json/JsonGenerator.fs | 32 +++++++++---------- src/Json/JsonProvider.fs | 8 ++--- src/Test.fsx | 2 +- .../TypeProviderInstantiation.fs | 8 ++--- tests/FSharp.Data.Tests/JsonProvider.fs | 4 +-- 5 files changed, 27 insertions(+), 27 deletions(-) diff --git a/src/Json/JsonGenerator.fs b/src/Json/JsonGenerator.fs index 6060cce97..94a464cd3 100644 --- a/src/Json/JsonGenerator.fs +++ b/src/Json/JsonGenerator.fs @@ -26,16 +26,16 @@ type internal JsonGenerationContext = JsonValueType : Type JsonRuntimeType : Type TypeCache : Dictionary - InferDictionariesFromRecords: bool + PreferDictionaries: bool GenerateConstructors : bool } - static member Create(cultureStr, tpType, ?uniqueNiceName, ?typeCache, ?inferDictionariesFromRecords) = + static member Create(cultureStr, tpType, ?uniqueNiceName, ?typeCache, ?preferDictionaries) = let uniqueNiceName = defaultArg uniqueNiceName (NameUtils.uniqueGenerator NameUtils.nicePascalName) let typeCache = defaultArg typeCache (Dictionary()) - let inferDictionariesFromRecords = defaultArg inferDictionariesFromRecords false - JsonGenerationContext.Create(cultureStr, tpType, uniqueNiceName, typeCache, inferDictionariesFromRecords, true) + let preferDictionaries = defaultArg preferDictionaries false + JsonGenerationContext.Create(cultureStr, tpType, uniqueNiceName, typeCache, preferDictionaries, true) - static member Create(cultureStr, tpType, uniqueNiceName, typeCache, inferDictionariesFromRecords, generateConstructors) = + static member Create(cultureStr, tpType, uniqueNiceName, typeCache, preferDictionaries, generateConstructors) = { CultureStr = cultureStr TypeProviderType = tpType UniqueNiceName = uniqueNiceName @@ -43,7 +43,7 @@ type internal JsonGenerationContext = JsonValueType = typeof JsonRuntimeType = typeof TypeCache = typeCache - InferDictionariesFromRecords = inferDictionariesFromRecords + PreferDictionaries = preferDictionaries GenerateConstructors = generateConstructors } member x.MakeOptionType(typ:Type) = typedefof>.MakeGenericType typ @@ -301,7 +301,7 @@ module JsonTypeBuilder = | InferedType.Record (_, fields, opt) -> InferedType.Record (None, fields, opt) | _ -> infType - if not ctx.InferDictionariesFromRecords + if not ctx.PreferDictionaries then None else let infType = @@ -325,30 +325,30 @@ module JsonTypeBuilder = let tupleType = Microsoft.FSharp.Reflection.FSharpType.MakeTupleType([|keyResult.ConvertedType; valueResult.ConvertedType|]) let itemsSeqType = typedefof<_ seq>.MakeGenericType([|tupleType|]) - let itemsGetter = fun (Singleton jDoc) -> + let itemsGetter (Singleton jDoc) = ctx.JsonRuntimeType?ConvertRecordToDictionary (keyResult.ConvertedType, valueConvertedTypeErased) (jDoc, keyResult.ConverterFunc ctx, valueResult.ConverterFunc ctx) - let keysGetter = fun (Singleton jDoc) -> + let keysGetter (Singleton jDoc) = ctx.JsonRuntimeType?GetKeysFromInferedDictionary (keyResult.ConvertedType) (jDoc, keyResult.ConverterFunc ctx) - let valuesGetter = fun (Singleton jDoc) -> + let valuesGetter (Singleton jDoc) = ctx.JsonRuntimeType?GetValuesFromInferedDictionary (valueConvertedTypeErased) (jDoc, valueResult.ConverterFunc ctx) let (|Doubleton|) = function [f; s] -> f, s | _ -> failwith "Parameter mismatch" - let itemGetter = fun (Doubleton (jDoc, key)) -> + let itemGetter (Doubleton (jDoc, key)) = ctx.JsonRuntimeType?GetValueByKeyFromInferedDictionary (keyResult.ConvertedType, valueConvertedTypeErased) (jDoc, keyResult.ConverterFunc ctx, valueResult.ConverterFunc ctx, key) - let tryFindCode = fun (Doubleton (jDoc, key)) -> + let tryFindCode (Doubleton (jDoc, key)) = ctx.JsonRuntimeType?TryGetValueByKeyFromInferedDictionary (keyResult.ConvertedType, valueConvertedTypeErased) (jDoc, keyResult.ConverterFunc ctx, valueResult.ConverterFunc ctx, key) - let containsKeyCode = fun (Doubleton (jDoc, key)) -> + let containsKeyCode (Doubleton (jDoc, key)) = ctx.JsonRuntimeType?InferedDictionaryContainsKey (keyResult.ConvertedType) (jDoc, keyResult.ConverterFunc ctx, key) - let countGetter = fun (Singleton jDoc) -> + let countGetter (Singleton jDoc) = <@@ JsonRuntime.GetRecordProperties(%%jDoc).Length @@> - let isEmptyGetter = fun (Singleton jDoc) -> + let isEmptyGetter (Singleton jDoc) = <@@ JsonRuntime.GetRecordProperties(%%jDoc).Length = 0 @@> [ @@ -385,7 +385,7 @@ module JsonTypeBuilder = let propName = prop.Name let optionalityHandledByProperty = propResult.ConversionCallingType <> JsonDocument - let getter = fun (Singleton jDoc) -> + let getter (Singleton jDoc) = if optionalityHandledByProperty then diff --git a/src/Json/JsonProvider.fs b/src/Json/JsonProvider.fs index 74e3da112..935df2205 100644 --- a/src/Json/JsonProvider.fs +++ b/src/Json/JsonProvider.fs @@ -39,7 +39,7 @@ type public JsonProvider(cfg:TypeProviderConfig) as this = let resolutionFolder = args.[5] :?> string let resource = args.[6] :?> string let inferTypesFromValues = args.[7] :?> bool - let inferDictionariesFromRecords = args.[8] :?> bool + let preferDictionaries = args.[8] :?> bool let cultureInfo = TextRuntime.GetCulture cultureStr @@ -59,7 +59,7 @@ type public JsonProvider(cfg:TypeProviderConfig) as this = using (IO.logTime "TypeGeneration" sample) <| fun _ -> - let ctx = JsonGenerationContext.Create(cultureStr, tpType, ?inferDictionariesFromRecords = Some inferDictionariesFromRecords) + let ctx = JsonGenerationContext.Create(cultureStr, tpType, ?preferDictionaries = Some preferDictionaries) let result = JsonTypeBuilder.generateJsonType ctx (*canPassAllConversionCallingTypes*)false (*optionalityHandledByParent*)false rootName inferedType { GeneratedType = tpType @@ -91,7 +91,7 @@ type public JsonProvider(cfg:TypeProviderConfig) as this = ProvidedStaticParameter("ResolutionFolder", typeof, parameterDefaultValue = "") ProvidedStaticParameter("EmbeddedResource", typeof, parameterDefaultValue = "") ProvidedStaticParameter("InferTypesFromValues", typeof, parameterDefaultValue = true) - ProvidedStaticParameter("InferDictionariesFromRecords", typeof, parameterDefaultValue = false) ] + ProvidedStaticParameter("PreferDictionaries", typeof, parameterDefaultValue = false) ] let helpText = """Typed representation of a JSON document. @@ -105,7 +105,7 @@ type public JsonProvider(cfg:TypeProviderConfig) as this = (e.g. 'MyCompany.MyAssembly, resource_name.json'). This is useful when exposing types generated by the type provider. If true, turns on additional type inference from values. (e.g. type inference infers string values such as "123" as ints and values constrained to 0 and 1 as booleans.) - If true, json record is considered as a dictionary, if the names of all the its fields are infered (by type inference rules) into the same non-string primitive type.""" + If true, json record is considered as a dictionary, if the names of all the its fields are infered (by type inference rules) into the same non-string primitive type.""" do jsonProvTy.AddXmlDoc helpText do jsonProvTy.DefineStaticParameters(parameters, buildTypes) diff --git a/src/Test.fsx b/src/Test.fsx index ef59cb462..bd7ef9f05 100644 --- a/src/Test.fsx +++ b/src/Test.fsx @@ -67,7 +67,7 @@ Json { Sample = "optionals.json" ResolutionFolder = "" EmbeddedResource = "" InferTypesFromValues = true - InferDictionariesFromRecords = false } + PreferDictionaries = false } |> dumpAll Xml { Sample = "JsonInXml.xml" diff --git a/tests/FSharp.Data.DesignTime.Tests/TypeProviderInstantiation.fs b/tests/FSharp.Data.DesignTime.Tests/TypeProviderInstantiation.fs index f09a4cbcc..485ff887f 100644 --- a/tests/FSharp.Data.DesignTime.Tests/TypeProviderInstantiation.fs +++ b/tests/FSharp.Data.DesignTime.Tests/TypeProviderInstantiation.fs @@ -45,7 +45,7 @@ type JsonProviderArgs = ResolutionFolder : string EmbeddedResource : string InferTypesFromValues : bool - InferDictionariesFromRecords : bool } + PreferDictionaries : bool } type HtmlProviderArgs = { Sample : string @@ -112,7 +112,7 @@ type TypeProviderInstantiation = box x.ResolutionFolder box x.EmbeddedResource box x.InferTypesFromValues - box x.InferDictionariesFromRecords |] + box x.PreferDictionaries |] | Html x -> (fun cfg -> new HtmlProvider(cfg) :> TypeProviderForNamespaces), [| box x.Sample @@ -158,7 +158,7 @@ type TypeProviderInstantiation = x.RootName x.Culture x.InferTypesFromValues.ToString() - x.InferDictionariesFromRecords.ToString() ] + x.PreferDictionaries.ToString() ] | Html x -> ["Html" x.Sample @@ -227,7 +227,7 @@ type TypeProviderInstantiation = ResolutionFolder = "" EmbeddedResource = "" InferTypesFromValues = args.[5] |> bool.Parse - InferDictionariesFromRecords = args.[6] |> bool.Parse } + PreferDictionaries = args.[6] |> bool.Parse } | "Html" -> Html { Sample = args.[1] PreferOptionals = args.[2] |> bool.Parse diff --git a/tests/FSharp.Data.Tests/JsonProvider.fs b/tests/FSharp.Data.Tests/JsonProvider.fs index 7fb509ba6..b3dd00e8d 100644 --- a/tests/FSharp.Data.Tests/JsonProvider.fs +++ b/tests/FSharp.Data.Tests/JsonProvider.fs @@ -761,12 +761,12 @@ let ``Can load different nested payloads`` () = [] let ``Can control dictionary inference`` () = - let notinferred = JsonProvider<"Data/DictionaryInference.json", InferDictionariesFromRecords=false>.GetSamples().[0] + let notinferred = JsonProvider<"Data/DictionaryInference.json", PreferDictionaries=false>.GetSamples().[0] notinferred.Rec.``0`` |> should equal 111 notinferred.Rec.``1`` |> should equal (Some 222) - let inferred = JsonProvider<"Data/DictionaryInference.json", InferDictionariesFromRecords=true>.GetSamples().[0] + let inferred = JsonProvider<"Data/DictionaryInference.json", PreferDictionaries=true>.GetSamples().[0] inferred.Rec.Count |> should equal 2 inferred.Rec.IsEmpty |> should equal false