diff --git a/.fantomasignore b/.fantomasignore deleted file mode 100644 index cfdc4db2d..000000000 --- a/.fantomasignore +++ /dev/null @@ -1,2 +0,0 @@ -paket-files/ -tests/ diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 37251f12f..62a9917a7 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -32,22 +32,14 @@ Type providers consist of two components: CSV provider, this component does the type inference and generates types (that are mapped to runtime components by the compiler). -We need _runtime components_ for .NET Standard 2.0 (netstandard2.0). We also need a _design time_ -component, to be able to host the type providers in .NET Core-based tooling. +We need a _runtime component_ for .NET Standard 2.0 (netstandard2.0). We also need a _design time_ +component for each, to be able to host the type provider in .NET Core-based tooling. -The _core_ runtime components are the following projects. No type providers are activated if you reference these: - - * **FSharp.Data.Http** - * **FSharp.Data.Csv.Core** - * **FSharp.Data.Html.Core** - * **FSharp.Data.Json.Core** - * **FSharp.Data.Xml.Core** - -The _enhanced_ runtime component that mentions the associated the design-time component is in the following project: +The _runtime_ components are in the following project: * **FSharp.Data** -The design-time component is the following project: +The _design time_ components are in the following project: * **FSharp.Data.DesignTime** diff --git a/Directory.Build.props b/Directory.Build.props index 03c303658..3459447c8 100644 --- a/Directory.Build.props +++ b/Directory.Build.props @@ -11,13 +11,5 @@ https://fsprojects.github.io/FSharp.Data https://raw.githubusercontent.com/fsprojects/FSharp.Data/master/docs/img/logo.png git - true - $(MSBuildThisFileDirectory)src\keyfile.snk - false - $(AllowedOutputExtensionsInPackageBuildOutputFolder);.pdb - true - true - true - Embedded diff --git a/FSharp.Data.sln b/FSharp.Data.sln index 019afa28d..3f7d10b36 100755 --- a/FSharp.Data.sln +++ b/FSharp.Data.sln @@ -1,7 +1,7 @@  Microsoft Visual Studio Solution File, Format Version 12.00 -# Visual Studio Version 17 -VisualStudioVersion = 17.4.32814.64 +# Visual Studio 15 +VisualStudioVersion = 15.0.26730.16 MinimumVisualStudioVersion = 10.0.40219.1 Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "docs", "docs", "{7E48C05A-8C42-4871-A618-180BEEF696AA}" ProjectSection(SolutionItems) = preProject @@ -53,17 +53,17 @@ EndProject Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "ja", "ja", "{CA5C64CE-E68C-4320-A00A-E67CA883B9D2}" ProjectSection(SolutionItems) = preProject docs\ja\contributing.md = docs\ja\contributing.md + docs\ja\fsharpdata.md = docs\ja\fsharpdata.md + docs\ja\index.md = docs\ja\index.md docs\ja\library\CsvFile.fsx = docs\ja\library\CsvFile.fsx docs\ja\library\CsvProvider.fsx = docs\ja\library\CsvProvider.fsx - docs\ja\fsharpdata.md = docs\ja\fsharpdata.md docs\ja\library\Http.fsx = docs\ja\library\Http.fsx - docs\ja\index.md = docs\ja\index.md docs\ja\library\JsonProvider.fsx = docs\ja\library\JsonProvider.fsx - docs\ja\tutorials\JsonToXml.fsx = docs\ja\tutorials\JsonToXml.fsx docs\ja\library\JsonValue.fsx = docs\ja\library\JsonValue.fsx - docs\tools\templates\ja\template.cshtml = docs\tools\templates\ja\template.cshtml docs\ja\library\WorldBank.fsx = docs\ja\library\WorldBank.fsx docs\ja\library\XmlProvider.fsx = docs\ja\library\XmlProvider.fsx + docs\ja\tutorials\JsonToXml.fsx = docs\ja\tutorials\JsonToXml.fsx + docs\tools\templates\ja\template.cshtml = docs\tools\templates\ja\template.cshtml EndProjectSection EndProject Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution Items", "{36E72EB1-5847-4B38-8993-B951648CB0D9}" @@ -77,32 +77,21 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution EndProjectSection EndProject Project("{6EC3EE1D-3C4E-46DD-8F32-0CC8E7565705}") = "FSharp.Data", "src\FSharp.Data\FSharp.Data.fsproj", "{6EBFDE55-9687-40A9-8C1A-6E204ECB117F}" + ProjectSection(ProjectDependencies) = postProject + {B85F245B-3FB9-4253-8251-16F98F05B6EC} = {B85F245B-3FB9-4253-8251-16F98F05B6EC} + EndProjectSection EndProject -Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "tests", "tests", "{1F33D53A-C007-408C-AF6C-B7D62288F941}" -EndProject -Project("{6EC3EE1D-3C4E-46DD-8F32-0CC8E7565705}") = "FSharp.Data.Reference.Tests", "tests\FSharp.Data.Reference.Tests\FSharp.Data.Reference.Tests.fsproj", "{DE36F8D0-7895-4ABD-9755-921F1BEAD3C9}" -EndProject -Project("{6EC3EE1D-3C4E-46DD-8F32-0CC8E7565705}") = "FSharp.Data.Core.Tests", "tests\FSharp.Data.Core.Tests\FSharp.Data.Core.Tests.fsproj", "{1746A3E0-32A2-49A7-9C8A-A0BCB52683B0}" -EndProject -Project("{6EC3EE1D-3C4E-46DD-8F32-0CC8E7565705}") = "FSharp.Data.DesignTime.Tests", "tests\FSharp.Data.DesignTime.Tests\FSharp.Data.DesignTime.Tests.fsproj", "{A5B31ACC-56FB-4EC2-917F-BEB3754EF9AC}" -EndProject -Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "FSharp.Data.Core.Tests.CSharp", "tests\FSharp.Data.Core.Tests.CSharp\FSharp.Data.Core.Tests.CSharp.csproj", "{290FED0C-D7C8-486F-AACF-3D7A1304C863}" -EndProject -Project("{6EC3EE1D-3C4E-46DD-8F32-0CC8E7565705}") = "FSharp.Data.Json.Core", "src\FSharp.Data.Json.Core\FSharp.Data.Json.Core.fsproj", "{DAEBFBCF-84CD-40BB-B8F6-99B1A9C4641F}" -EndProject -Project("{6EC3EE1D-3C4E-46DD-8F32-0CC8E7565705}") = "FSharp.Data.Tests", "tests\FSharp.Data.Tests\FSharp.Data.Tests.fsproj", "{750148EC-6A05-421D-96A4-E5AC9D18AF58}" -EndProject -Project("{6EC3EE1D-3C4E-46DD-8F32-0CC8E7565705}") = "FSharp.Data.DesignTime", "src\FSharp.Data.DesignTime\FSharp.Data.DesignTime.fsproj", "{44E0DF97-D8FD-4805-8A84-B888D9589C8A}" +Project("{6EC3EE1D-3C4E-46DD-8F32-0CC8E7565705}") = "FSharp.Data.DesignTime", "src\FSharp.Data.DesignTime\FSharp.Data.DesignTime.fsproj", "{B85F245B-3FB9-4253-8251-16F98F05B6EC}" EndProject -Project("{6EC3EE1D-3C4E-46DD-8F32-0CC8E7565705}") = "FSharp.Data.Html.Core", "src\FSharp.Data.Html.Core\FSharp.Data.Html.Core.fsproj", "{E91BF68E-257C-43E6-BDE9-672D4E3BFAFA}" +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "tests", "tests", "{1F33D53A-C007-408C-AF6C-B7D62288F941}" EndProject -Project("{6EC3EE1D-3C4E-46DD-8F32-0CC8E7565705}") = "FSharp.Data.Http", "src\FSharp.Data.Http\FSharp.Data.Http.fsproj", "{29EDED03-D2D6-415C-A17D-00806C52035A}" +Project("{F2A71F9B-5D33-465A-A702-920D77279786}") = "FSharp.Data.Reference.Tests", "tests\FSharp.Data.Reference.Tests\FSharp.Data.Reference.Tests.fsproj", "{DE36F8D0-7895-4ABD-9755-921F1BEAD3C9}" EndProject -Project("{6EC3EE1D-3C4E-46DD-8F32-0CC8E7565705}") = "FSharp.Data.Xml.Core", "src\FSharp.Data.Xml.Core\FSharp.Data.Xml.Core.fsproj", "{1ECEFFEE-1040-40ED-9EB5-CE720A33058D}" +Project("{F2A71F9B-5D33-465A-A702-920D77279786}") = "FSharp.Data.Tests", "tests\FSharp.Data.Tests\FSharp.Data.Tests.fsproj", "{1746A3E0-32A2-49A7-9C8A-A0BCB52683B0}" EndProject -Project("{6EC3EE1D-3C4E-46DD-8F32-0CC8E7565705}") = "FSharp.Data.Csv.Core", "src\FSharp.Data.Csv.Core\FSharp.Data.Csv.Core.fsproj", "{0A1B8B61-268D-4061-B567-A47141C608E4}" +Project("{F2A71F9B-5D33-465A-A702-920D77279786}") = "FSharp.Data.DesignTime.Tests", "tests\FSharp.Data.DesignTime.Tests\FSharp.Data.DesignTime.Tests.fsproj", "{A5B31ACC-56FB-4EC2-917F-BEB3754EF9AC}" EndProject -Project("{6EC3EE1D-3C4E-46DD-8F32-0CC8E7565705}") = "FSharp.Data.WorldBank.Core", "src\FSharp.Data.WorldBank.Core\FSharp.Data.WorldBank.Core.fsproj", "{A69D007B-EAF0-4866-A8B4-A2EDF2614E56}" +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "FSharp.Data.Tests.CSharp", "tests\FSharp.Data.Tests.CSharp\FSharp.Data.Tests.CSharp.csproj", "{290FED0C-D7C8-486F-AACF-3D7A1304C863}" EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution @@ -114,6 +103,10 @@ Global {6EBFDE55-9687-40A9-8C1A-6E204ECB117F}.Debug|Any CPU.Build.0 = Debug|Any CPU {6EBFDE55-9687-40A9-8C1A-6E204ECB117F}.Release|Any CPU.ActiveCfg = Release|Any CPU {6EBFDE55-9687-40A9-8C1A-6E204ECB117F}.Release|Any CPU.Build.0 = Release|Any CPU + {B85F245B-3FB9-4253-8251-16F98F05B6EC}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {B85F245B-3FB9-4253-8251-16F98F05B6EC}.Debug|Any CPU.Build.0 = Debug|Any CPU + {B85F245B-3FB9-4253-8251-16F98F05B6EC}.Release|Any CPU.ActiveCfg = Release|Any CPU + {B85F245B-3FB9-4253-8251-16F98F05B6EC}.Release|Any CPU.Build.0 = Release|Any CPU {DE36F8D0-7895-4ABD-9755-921F1BEAD3C9}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {DE36F8D0-7895-4ABD-9755-921F1BEAD3C9}.Debug|Any CPU.Build.0 = Debug|Any CPU {DE36F8D0-7895-4ABD-9755-921F1BEAD3C9}.Release|Any CPU.ActiveCfg = Release|Any CPU @@ -130,50 +123,17 @@ Global {290FED0C-D7C8-486F-AACF-3D7A1304C863}.Debug|Any CPU.Build.0 = Debug|Any CPU {290FED0C-D7C8-486F-AACF-3D7A1304C863}.Release|Any CPU.ActiveCfg = Release|Any CPU {290FED0C-D7C8-486F-AACF-3D7A1304C863}.Release|Any CPU.Build.0 = Release|Any CPU - {DAEBFBCF-84CD-40BB-B8F6-99B1A9C4641F}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {DAEBFBCF-84CD-40BB-B8F6-99B1A9C4641F}.Debug|Any CPU.Build.0 = Debug|Any CPU - {DAEBFBCF-84CD-40BB-B8F6-99B1A9C4641F}.Release|Any CPU.ActiveCfg = Release|Any CPU - {DAEBFBCF-84CD-40BB-B8F6-99B1A9C4641F}.Release|Any CPU.Build.0 = Release|Any CPU - {750148EC-6A05-421D-96A4-E5AC9D18AF58}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {750148EC-6A05-421D-96A4-E5AC9D18AF58}.Debug|Any CPU.Build.0 = Debug|Any CPU - {750148EC-6A05-421D-96A4-E5AC9D18AF58}.Release|Any CPU.ActiveCfg = Release|Any CPU - {750148EC-6A05-421D-96A4-E5AC9D18AF58}.Release|Any CPU.Build.0 = Release|Any CPU - {44E0DF97-D8FD-4805-8A84-B888D9589C8A}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {44E0DF97-D8FD-4805-8A84-B888D9589C8A}.Debug|Any CPU.Build.0 = Debug|Any CPU - {44E0DF97-D8FD-4805-8A84-B888D9589C8A}.Release|Any CPU.ActiveCfg = Release|Any CPU - {44E0DF97-D8FD-4805-8A84-B888D9589C8A}.Release|Any CPU.Build.0 = Release|Any CPU - {E91BF68E-257C-43E6-BDE9-672D4E3BFAFA}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {E91BF68E-257C-43E6-BDE9-672D4E3BFAFA}.Debug|Any CPU.Build.0 = Debug|Any CPU - {E91BF68E-257C-43E6-BDE9-672D4E3BFAFA}.Release|Any CPU.ActiveCfg = Release|Any CPU - {E91BF68E-257C-43E6-BDE9-672D4E3BFAFA}.Release|Any CPU.Build.0 = Release|Any CPU - {29EDED03-D2D6-415C-A17D-00806C52035A}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {29EDED03-D2D6-415C-A17D-00806C52035A}.Debug|Any CPU.Build.0 = Debug|Any CPU - {29EDED03-D2D6-415C-A17D-00806C52035A}.Release|Any CPU.ActiveCfg = Release|Any CPU - {29EDED03-D2D6-415C-A17D-00806C52035A}.Release|Any CPU.Build.0 = Release|Any CPU - {1ECEFFEE-1040-40ED-9EB5-CE720A33058D}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {1ECEFFEE-1040-40ED-9EB5-CE720A33058D}.Debug|Any CPU.Build.0 = Debug|Any CPU - {1ECEFFEE-1040-40ED-9EB5-CE720A33058D}.Release|Any CPU.ActiveCfg = Release|Any CPU - {1ECEFFEE-1040-40ED-9EB5-CE720A33058D}.Release|Any CPU.Build.0 = Release|Any CPU - {0A1B8B61-268D-4061-B567-A47141C608E4}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {0A1B8B61-268D-4061-B567-A47141C608E4}.Debug|Any CPU.Build.0 = Debug|Any CPU - {0A1B8B61-268D-4061-B567-A47141C608E4}.Release|Any CPU.ActiveCfg = Release|Any CPU - {0A1B8B61-268D-4061-B567-A47141C608E4}.Release|Any CPU.Build.0 = Release|Any CPU - {A69D007B-EAF0-4866-A8B4-A2EDF2614E56}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {A69D007B-EAF0-4866-A8B4-A2EDF2614E56}.Debug|Any CPU.Build.0 = Debug|Any CPU - {A69D007B-EAF0-4866-A8B4-A2EDF2614E56}.Release|Any CPU.ActiveCfg = Release|Any CPU - {A69D007B-EAF0-4866-A8B4-A2EDF2614E56}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE EndGlobalSection + GlobalSection(ExtensibilityGlobals) = postSolution + SolutionGuid = {34E4FA16-B344-439D-A789-1E8355E5659F} + EndGlobalSection GlobalSection(NestedProjects) = preSolution {DE36F8D0-7895-4ABD-9755-921F1BEAD3C9} = {1F33D53A-C007-408C-AF6C-B7D62288F941} {1746A3E0-32A2-49A7-9C8A-A0BCB52683B0} = {1F33D53A-C007-408C-AF6C-B7D62288F941} {A5B31ACC-56FB-4EC2-917F-BEB3754EF9AC} = {1F33D53A-C007-408C-AF6C-B7D62288F941} {290FED0C-D7C8-486F-AACF-3D7A1304C863} = {1F33D53A-C007-408C-AF6C-B7D62288F941} - {750148EC-6A05-421D-96A4-E5AC9D18AF58} = {1F33D53A-C007-408C-AF6C-B7D62288F941} - EndGlobalSection - GlobalSection(ExtensibilityGlobals) = postSolution - SolutionGuid = {34E4FA16-B344-439D-A789-1E8355E5659F} EndGlobalSection EndGlobal diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md index 21da41fe3..d09668a20 100644 --- a/RELEASE_NOTES.md +++ b/RELEASE_NOTES.md @@ -1,13 +1,3 @@ -### 5.0.1 - Aug 15 2022 - -* There are now multiple packages - * FSharp.Data -- includes everything - * FSharp.Data.Http -- http types/helpers - * FSharp.Data.Csv.Core -- csv types/helpers - * FSharp.Data.Json.Core -- json types/helpers - * FSharp.Data.Html.Core -- html types/helpers - * FSharp.Data.Xml.Core -- xml types/helpers - ### 4.2.10 - Aug 12 2022 * Implement "inline schemas": ability to add type hints into the type providers' source documents by @mlaily in https://github.com/fsprojects/FSharp.Data/pull/1447 diff --git a/build.fsx b/build.fsx index 46d0a3f5f..0eead60f0 100644 --- a/build.fsx +++ b/build.fsx @@ -36,15 +36,9 @@ let summary = "Library of F# type providers and data access tools" let description = """ - The FSharp.Data packages contain type providers and utilities to access - common data formats (CSV, HTML, JSON and XML in your F# applications and scripts. - - * FSharp.Data -- includes everything - * FSharp.Data.Http -- http types/helpers - * FSharp.Data.Csv.Core -- csv types/helpers - * FSharp.Data.Json.Core -- json types/helpers - * FSharp.Data.Html.Core -- html types/helpers - * FSharp.Data.Xml.Core -- xml types/helpers""" + The FSharp.Data package contains type providers and utilities to access + common data formats (CSV, HTML, JSON and XML in your F# applications and scripts. It also + contains helpers for parsing CSV, HTML and JSON files and for sending HTTP requests.""" let tags = "F# fsharp data typeprovider WorldBank CSV HTML CSS JSON XML HTTP linqpad-samples" @@ -126,9 +120,9 @@ Target.create "RunTests" (fun _ -> "FSharp.Data.sln" |> DotNet.test setParams) // -------------------------------------------------------------------------------------- -// Build packages +// Build a NuGet package -Target.create "Pack" (fun _ -> +Target.create "NuGet" (fun _ -> // Format the release notes let releaseNotes = release.Notes |> String.concat "\n" @@ -142,7 +136,12 @@ Target.create "Pack" (fun _ -> ("PackageLicenseExpression", license) ("PackageReleaseNotes", releaseNotes) ("Summary", summary) - ("PackageDescription", description) ] + ("PackageDescription", description) + ("EnableSourceLink", "true") + ("PublishRepositoryUrl", "true") + ("EmbedUntrackedSources", "true") + ("IncludeSymbols", "true") + ("SymbolPackageFormat", "snupkg") ] DotNet.pack (fun p -> @@ -150,7 +149,7 @@ Target.create "Pack" (fun _ -> Configuration = DotNet.BuildConfiguration.Release OutputPath = Some "bin" MSBuildParams = { p.MSBuildParams with Properties = properties } }) - "FSharp.Data.sln") + "src/FSharp.Data/FSharp.Data.fsproj") // -------------------------------------------------------------------------------------- // Generate the documentation @@ -179,7 +178,7 @@ Target.create "Help" (fun _ -> printfn " * Build" printfn " * RunTests" printfn " * GenerateDocs" - printfn " * Pack (creates package only, doesn't publish)" + printfn " * NuGet (creates package only, doesn't publish)" printfn " * All (calls previous 5)" printfn "" printfn " Other targets:" @@ -229,7 +228,7 @@ Target.create "All" ignore ==> "GenerateDocs" ==> "All" -"Build" ==> "Pack" ==> "All" +"Build" ==> "NuGet" ==> "All" "Build" ==> "All" "Build" ==> "RunTests" ==> "All" diff --git a/docs/library/CsvFile.fsx b/docs/library/CsvFile.fsx index 70bc22bf1..ef3316694 100644 --- a/docs/library/CsvFile.fsx +++ b/docs/library/CsvFile.fsx @@ -6,20 +6,18 @@ index: 2 --- *) (*** condition: prepare ***) -#r "../../src/FSharp.Data/bin/Release/netstandard2.0/FSharp.Data.Http.dll" -#r "../../src/FSharp.Data/bin/Release/netstandard2.0/FSharp.Data.Csv.Core.dll" #r "../../src/FSharp.Data/bin/Release/netstandard2.0/FSharp.Data.dll" (*** condition: fsx ***) #if FSX #r "nuget: FSharp.Data,{{fsdocs-package-version}}" -#endif +#endif // FSX (*** condition: ipynb ***) #if IPYNB #r "nuget: FSharp.Data,{{fsdocs-package-version}}" Formatter.SetPreferredMimeTypesFor(typeof, "text/plain") -Formatter.Register(fun (x: obj) (writer: TextWriter) -> fprintfn writer "%120A" x) -#endif +Formatter.Register(fun (x:obj) (writer: TextWriter) -> fprintfn writer "%120A" x ) +#endif // IPYNB (** [![Binder](../img/badge-binder.svg)](https://mybinder.org/v2/gh/fsprojects/FSharp.Data/gh-pages?filepath={{fsdocs-source-basename}}.ipynb)  [![Script](../img/badge-script.svg)]({{root}}/{{fsdocs-source-basename}}.fsx)  @@ -50,14 +48,12 @@ points to a live CSV file on the Yahoo finance web site: *) // Download the stock prices -let msft = - CsvFile - .Load(__SOURCE_DIRECTORY__ + "/../data/MSFT.csv") - .Cache() +let msft = CsvFile.Load(__SOURCE_DIRECTORY__ + "/../data/MSFT.csv").Cache() // Print the prices in the HLOC format for row in msft.Rows |> Seq.truncate 10 do - printfn "HLOC: (%s, %s, %s)" (row.GetColumn "High") (row.GetColumn "Low") (row.GetColumn "Date") + printfn "HLOC: (%s, %s, %s)" + (row.GetColumn "High") (row.GetColumn "Low") (row.GetColumn "Date") (*** include-fsi-merged-output ***) (** @@ -96,7 +92,8 @@ The following example shows how to process the sample previous CSV sample using open FSharp.Data.CsvExtensions for row in msft.Rows |> Seq.truncate 10 do - printfn "HLOC: (%f, %M, %O)" (row.["High"].AsFloat()) (row?Low.AsDecimal()) (row?Date.AsDateTime()) + printfn "HLOC: (%f, %M, %O)" + (row.["High"].AsFloat()) (row?Low.AsDecimal()) (row?Date.AsDateTime()) (*** include-fsi-merged-output ***) @@ -111,8 +108,7 @@ separator and quote characters when saving. *) // Saving the first 10 stock prices where the closing price is higher than the opening price in TSV format: -msft - .Filter(fun row -> row?Close.AsFloat() > row?Open.AsFloat()) +msft.Filter(fun row -> row?Close.AsFloat() > row?Open.AsFloat()) .Truncate(10) .SaveToString('\t') diff --git a/docs/library/CsvProvider.fsx b/docs/library/CsvProvider.fsx index d62078e90..059fff799 100644 --- a/docs/library/CsvProvider.fsx +++ b/docs/library/CsvProvider.fsx @@ -6,20 +6,18 @@ index: 1 --- *) (*** condition: prepare ***) -#r "../../src/FSharp.Data/bin/Release/netstandard2.0/FSharp.Data.Http.dll" -#r "../../src/FSharp.Data/bin/Release/netstandard2.0/FSharp.Data.Csv.Core.dll" #r "../../src/FSharp.Data/bin/Release/netstandard2.0/FSharp.Data.dll" (*** condition: fsx ***) #if FSX #r "nuget: FSharp.Data,{{fsdocs-package-version}}" -#endif +#endif // FSX (*** condition: ipynb ***) #if IPYNB #r "nuget: FSharp.Data,{{fsdocs-package-version}}" Formatter.SetPreferredMimeTypesFor(typeof, "text/plain") -Formatter.Register(fun (x: obj) (writer: TextWriter) -> fprintfn writer "%120A" x) -#endif +Formatter.Register(fun (x:obj) (writer: TextWriter) -> fprintfn writer "%120A" x ) +#endif // IPYNB (** [![Binder](../img/badge-binder.svg)](https://mybinder.org/v2/gh/fsprojects/FSharp.Data/gh-pages?filepath={{fsdocs-source-basename}}.ipynb)  @@ -86,10 +84,7 @@ The following sample calls the `Load` method with an URL that points to a live C *) // Download the stock prices -let msft = - Stocks - .Load(__SOURCE_DIRECTORY__ + "/../data/MSFT.csv") - .Cache() +let msft = Stocks.Load(__SOURCE_DIRECTORY__ + "/../data/MSFT.csv").Cache() // Look at the most recent row. Note the 'Date' property // is of type 'DateTime' and 'Open' has a type 'decimal' @@ -99,7 +94,7 @@ let lastOpen = firstRow.Open // Print the first 10 prices in the HLOC format for row in msft.Rows |> Seq.truncate 10 do - printfn "HLOC: (%A, %A, %A, %A)" row.High row.Low row.Open row.Close + printfn "HLOC: (%A, %A, %A, %A)" row.High row.Low row.Open row.Close (*** include-fsi-merged-output ***) @@ -132,8 +127,7 @@ a static argument. Also note that in this case we're using the same data at runt so we use the `GetSample` method instead of calling `Load` and passing the same parameter again. *) -let small = - CsvProvider<"../data/SmallTest.csv", ResolutionFolder=ResolutionFolder>.GetSample () +let small = CsvProvider<"../data/SmallTest.csv", ResolutionFolder=ResolutionFolder>.GetSample() (*** include-fsi-merged-output ***) @@ -141,8 +135,7 @@ let small = We can also use the default constructor instead of the `GetSample` static method: *) -let small2 = - new CsvProvider<"../data/SmallTest.csv", ResolutionFolder=ResolutionFolder>() +let small2 = new CsvProvider<"../data/SmallTest.csv", ResolutionFolder=ResolutionFolder>() (*** include-fsi-merged-output ***) @@ -158,10 +151,9 @@ following simple calculation: open FSharp.Data.UnitSystems.SI.UnitNames for row in small.Rows do - let speed = row.Distance / row.Time - - if speed > 15.0M then - printfn "%s (%A m/s)" row.Name speed + let speed = row.Distance / row.Time + if speed > 15.0M then + printfn "%s (%A m/s)" row.Name speed (*** include-fsi-merged-output ***) @@ -186,8 +178,8 @@ type AirQuality = CsvProvider<"../data/AirQuality.csv", ";", ResolutionFolder=Re let airQuality = new AirQuality() for row in airQuality.Rows |> Seq.truncate 10 do - if row.Month > 6 then - printfn "Temp: %i Ozone: %f " row.Temp row.Ozone + if row.Month > 6 then + printfn "Temp: %i Ozone: %f " row.Temp row.Ozone (*** include-fsi-merged-output ***) @@ -203,21 +195,18 @@ we also set `IgnoreErrors` static parameter to `true` so that lines with incorre are automatically skipped (the sample file ([`data/MortalityNY.csv`](../data/MortalityNY.tsv)) contains additional unstructured data at the end): *) -let mortalityNy = - CsvProvider<"../data/MortalityNY.tsv", IgnoreErrors=true, ResolutionFolder=ResolutionFolder>.GetSample () +let mortalityNy = CsvProvider<"../data/MortalityNY.tsv", IgnoreErrors=true, ResolutionFolder=ResolutionFolder>.GetSample() // Find the name of a cause based on code // (Pedal cyclist injured in an accident) -let cause = - mortalityNy.Rows - |> Seq.find (fun r -> r.``Cause of death Code`` = "V13.4") +let cause = mortalityNy.Rows |> Seq.find (fun r -> + r.``Cause of death Code`` = "V13.4") // Print the number of injured cyclists printfn "CAUSE: %s" cause.``Cause of death`` - for r in mortalityNy.Rows do - if r.``Cause of death Code`` = "V13.4" then - printfn "%s (%d cases)" r.County r.Count + if r.``Cause of death Code`` = "V13.4" then + printfn "%s (%d cases)" r.County r.Count (*** include-fsi-merged-output ***) @@ -240,9 +229,8 @@ the `MissingValues` static parameter of `CsvProvider` as a comma-separated strin For example, to ignore `this` and `that` we could do: *) -CsvProvider<"X,Y,Z\nthis,that,1.0", MissingValues="this,that"> - .GetSample() - .Rows +CsvProvider<"X,Y,Z\nthis,that,1.0", + MissingValues="this,that">.GetSample().Rows (*** include-fsi-merged-output ***) (** @@ -253,11 +241,11 @@ each row, then remove missing values and then use the standard `Seq.average` fun open System let mean = - airQuality.Rows - |> Seq.toArray - |> Array.map (fun row -> row.Ozone) - |> Array.filter (fun elem -> not (Double.IsNaN elem)) - |> Array.average + airQuality.Rows + |> Seq.toArray + |> Array.map (fun row -> row.Ozone) + |> Array.filter (fun elem -> not (Double.IsNaN elem)) + |> Array.average (*** include-fsi-merged-output ***) @@ -332,12 +320,15 @@ consider that row as a data row. In that case, the columns will be named `Column names are overridden using the `Schema` parameter. Note that you can override only the name in the `Schema` parameter and still have the provider infer the type for you. Example: *) -type OneTwoThree = CsvProvider<"1,2,3", HasHeaders=false, Schema="Duration (float),foo,float option"> +type OneTwoThree = + CsvProvider<"1,2,3", HasHeaders = false, Schema = "Duration (float),foo,float option"> let csv = OneTwoThree.GetSample() - for row in csv.Rows do - printfn "%f %d %f" (row.Duration / 1.0) row.Foo (defaultArg row.Column3 1.0) + printfn "%f %d %f" + (row.Duration/1.0) + row.Foo + (defaultArg row.Column3 1.0) (*** include-fsi-merged-output ***) @@ -351,12 +342,14 @@ the other columns blank in the schema (you also don't need to add all the traili *) type Titanic1 = - CsvProvider<"../data/Titanic.csv", Schema=",,Passenger Class,,,float", ResolutionFolder=ResolutionFolder> + CsvProvider<"../data/Titanic.csv", + Schema=",,Passenger Class,,,float", + ResolutionFolder=ResolutionFolder> let titanic1 = Titanic1.GetSample() - for row in titanic1.Rows |> Seq.truncate 10 do - printfn "%s Class = %d Fare = %g" row.Name row.``Passenger Class`` row.Fare + printfn "%s Class = %d Fare = %g" + row.Name row.``Passenger Class`` row.Fare (*** include-fsi-merged-output ***) @@ -366,12 +359,14 @@ Alternatively, you can rename and override the type of any column by name instea *) type Titanic2 = - CsvProvider<"../data/Titanic.csv", Schema="Fare=float,PClass->Passenger Class", ResolutionFolder=ResolutionFolder> + CsvProvider<"../data/Titanic.csv", + Schema="Fare=float,PClass->Passenger Class", + ResolutionFolder=ResolutionFolder> let titanic2 = Titanic2.GetSample() - for row in titanic2.Rows |> Seq.truncate 10 do - printfn "%s Class = %d Fare = %g" row.Name row.``Passenger Class`` row.Fare + printfn "%s Class = %d Fare = %g" + row.Name row.``Passenger Class`` row.Fare (*** include-fsi-merged-output ***) @@ -390,11 +385,11 @@ the `Save` method. You can also use the `SaveToString()` to get the output direc // Saving the first 10 rows that don't have missing values to a new csv file airQuality - .Filter(fun row -> - not (Double.IsNaN row.Ozone) - && not (Double.IsNaN row.``Solar.R``)) - .Truncate(10) - .SaveToString() + .Filter(fun row -> + not (Double.IsNaN row.Ozone) && + not (Double.IsNaN row.``Solar.R``)) + .Truncate(10) + .SaveToString() (*** include-fsi-merged-output ***) @@ -405,7 +400,10 @@ It's also possible to transform the columns themselves by using `Map` and the co *) let doubleOzone = - airQuality.Map(fun row -> AirQuality.Row(row.Ozone * 2.0, row.``Solar.R``, row.Wind, row.Temp, row.Month, row.Day)) + airQuality.Map(fun row -> + AirQuality.Row + ( row.Ozone * 2.0, row.``Solar.R``, + row.Wind, row.Temp, row.Month, row.Day)) (*** include-fsi-merged-output ***) @@ -416,12 +414,12 @@ You can also append new rows, either by creating them directly as in the previou *) let newRows = - AirQuality.ParseRows( - """41;190;7.4;67;5;1 - 36;118;8;72;5;2""" - ) + AirQuality.ParseRows + ("""41;190;7.4;67;5;1 + 36;118;8;72;5;2""") -let airQualityWithExtraRows = airQuality.Append newRows +let airQualityWithExtraRows = + airQuality.Append newRows (*** include-fsi-merged-output ***) @@ -431,11 +429,13 @@ It's even possible to create csv files without parsing at all: *) -type MyCsvType = CsvProvider +type MyCsvType = + CsvProvider let myRows = - [ MyCsvType.Row(1, "a", None) - MyCsvType.Row(2, "B", Some System.DateTime.Now) ] + [ MyCsvType.Row(1, "a", None) + MyCsvType.Row(2, "B", Some System.DateTime.Now) ] let myCsv = new MyCsvType(myRows) myCsv.SaveToString() diff --git a/docs/library/HtmlCssSelectors.fsx b/docs/library/HtmlCssSelectors.fsx index a93bb1450..06c68177e 100644 --- a/docs/library/HtmlCssSelectors.fsx +++ b/docs/library/HtmlCssSelectors.fsx @@ -6,20 +6,18 @@ index: 4 --- *) (*** condition: prepare ***) -#r "../../src/FSharp.Data/bin/Release/netstandard2.0/FSharp.Data.Http.dll" -#r "../../src/FSharp.Data/bin/Release/netstandard2.0/FSharp.Data.Csv.Core.dll" -#r "../../src/FSharp.Data/bin/Release/netstandard2.0/FSharp.Data.Html.Core.dll" +#r "../../src/FSharp.Data/bin/Release/netstandard2.0/FSharp.Data.dll" (*** condition: fsx ***) #if FSX #r "nuget: FSharp.Data,{{fsdocs-package-version}}" -#endif +#endif // FSX (*** condition: ipynb ***) #if IPYNB #r "nuget: FSharp.Data,{{fsdocs-package-version}}" Formatter.SetPreferredMimeTypesFor(typeof, "text/plain") -Formatter.Register(fun (x: obj) (writer: TextWriter) -> fprintfn writer "%120A" x) -#endif +Formatter.Register(fun (x:obj) (writer: TextWriter) -> fprintfn writer "%120A" x ) +#endif // IPYNB (** [![Binder](../img/badge-binder.svg)](https://mybinder.org/v2/gh/fsprojects/FSharp.Data/gh-pages?filepath={{fsdocs-source-basename}}.ipynb)  [![Script](../img/badge-script.svg)]({{root}}/{{fsdocs-source-basename}}.fsx)  @@ -54,15 +52,11 @@ Then we can , for example, use the direct descendants selector to select another id `ires`. The CSS selector to do so is `div#search > div#ires`: *) let links = - doc.CssSelect("div#search > div#ires div.g > div.s div.kv cite") - |> List.map (fun n -> - match n.InnerText() with - | t when - (t.StartsWith("https://") - || t.StartsWith("http://")) - -> - t - | t -> "http://" + t) + doc.CssSelect("div#search > div#ires div.g > div.s div.kv cite") + |> List.map (fun n -> + match n.InnerText() with + | t when (t.StartsWith("https://") || t.StartsWith("http://"))-> t + | t -> "http://" + t ) (*** include-fsi-merged-output ***) @@ -93,8 +87,8 @@ let doc2 = HtmlDocument.Load(fsys) let books = doc2.CssSelect("div.g h3.r a") - |> List.map (fun a -> a.InnerText().Trim(), a.AttributeValue("href")) - |> List.filter (fun (title, href) -> title.Contains("F#")) + |> List.map(fun a -> a.InnerText().Trim(), a.AttributeValue("href")) + |> List.filter(fun (title, href) -> title.Contains("F#")) (*** include-fsi-merged-output ***) @@ -109,9 +103,7 @@ You can also refer to the table below for a complete list of supported selectors Finds all links with an english hreflang attribute. *) -let englishDoc = - HtmlDocument.Parse( - """ +let englishDoc = HtmlDocument.Parse(""" @@ -119,10 +111,10 @@ let englishDoc = Some other text will not be outlined - """ - ) + """) -let englishLinks = englishDoc.CssSelect("a[hreflang|=en]") +let englishLinks = + englishDoc.CssSelect("a[hreflang|=en]") (*** include-fsi-merged-output ***) (** @@ -130,9 +122,7 @@ let englishLinks = englishDoc.CssSelect("a[hreflang|=en]") Finds all inputs with a name containing "man". This includes results where "man" is a substring: *) -let manDoc = - HtmlDocument.Parse( - """ +let manDoc = HtmlDocument.Parse(""" @@ -144,10 +134,10 @@ let manDoc = - """ - ) + """) -let manElems = manDoc.CssSelect("input[name*='man']") +let manElems = + manDoc.CssSelect("input[name*='man']") (*** include-fsi-merged-output ***) (** @@ -155,7 +145,8 @@ let manElems = manDoc.CssSelect("input[name*='man']") Finds all inputs with a name containing the word "man". This requires a whitespace around the word: *) -let manWordElems = manDoc.CssSelect("input[name~='man']") +let manWordElems = + manDoc.CssSelect("input[name~='man']") (*** include-fsi-merged-output ***) @@ -164,7 +155,8 @@ let manWordElems = manDoc.CssSelect("input[name~='man']") Finds all inputs with a name ending with "man". *) -let manEndElemes = manDoc.CssSelect("input[name$='man']") +let manEndElemes = + manDoc.CssSelect("input[name$='man']") (*** include-fsi-merged-output ***) @@ -174,7 +166,8 @@ let manEndElemes = manDoc.CssSelect("input[name$='man']") Finds all inputs with a name equal to "man". *) -let manEqElemes = manDoc.CssSelect("input[name='man']") +let manEqElemes = + manDoc.CssSelect("input[name='man']") (*** include-fsi-merged-output ***) @@ -183,7 +176,8 @@ let manEqElemes = manDoc.CssSelect("input[name='man']") Finds all inputs with a name different to "man". *) -let notManElems = manDoc.CssSelect("input[name!='man']") +let notManElems = + manDoc.CssSelect("input[name!='man']") (*** include-fsi-merged-output ***) @@ -193,7 +187,8 @@ let notManElems = manDoc.CssSelect("input[name!='man']") Finds all inputs with a name starting with "man". *) -let manStartElems = manDoc.CssSelect("input[name^='man']") +let manStartElems = + manDoc.CssSelect("input[name^='man']") (*** include-fsi-merged-output ***) @@ -203,9 +198,7 @@ let manStartElems = manDoc.CssSelect("input[name^='man']") There are some syntax shortcuts to find forms controls. *) -let htmlForm = - HtmlDocument.Parse( - """ +let htmlForm = HtmlDocument.Parse(""" @@ -228,8 +221,7 @@ let htmlForm = - """ - ) + """) (** You can use `:prop` to find CSS elements with the specified value of the `type` attribute @@ -339,3 +331,4 @@ Selector name|Status|specification [1] :root Selector seems to be useless in our case because with the HTML parser the root is always the html node. *) + diff --git a/docs/library/HtmlParser.fsx b/docs/library/HtmlParser.fsx index 78668932a..28f2872f5 100644 --- a/docs/library/HtmlParser.fsx +++ b/docs/library/HtmlParser.fsx @@ -6,20 +6,18 @@ index: 3 --- *) (*** condition: prepare ***) -#r "../../src/FSharp.Data/bin/Release/netstandard2.0/FSharp.Data.Http.dll" -#r "../../src/FSharp.Data/bin/Release/netstandard2.0/FSharp.Data.Csv.Core.dll" -#r "../../src/FSharp.Data/bin/Release/netstandard2.0/FSharp.Data.Html.Core.dll" +#r "../../src/FSharp.Data/bin/Release/netstandard2.0/FSharp.Data.dll" (*** condition: fsx ***) #if FSX #r "nuget: FSharp.Data,{{fsdocs-package-version}}" -#endif +#endif // FSX (*** condition: ipynb ***) #if IPYNB #r "nuget: FSharp.Data,{{fsdocs-package-version}}" Formatter.SetPreferredMimeTypesFor(typeof, "text/plain") -Formatter.Register(fun (x: obj) (writer: TextWriter) -> fprintfn writer "%120A" x) -#endif +Formatter.Register(fun (x:obj) (writer: TextWriter) -> fprintfn writer "%120A" x ) +#endif // IPYNB (** [![Binder](../img/badge-binder.svg)](https://mybinder.org/v2/gh/fsprojects/FSharp.Data/gh-pages?filepath={{fsdocs-source-basename}}.ipynb)  [![Script](../img/badge-script.svg)]({{root}}/{{fsdocs-source-basename}}.fsx)  @@ -59,10 +57,11 @@ which in this case is the url that the search result is pointing to, and additio we are looking at. *) let links = - results.Descendants [ "a" ] + results.Descendants ["a"] |> Seq.choose (fun x -> - x.TryGetAttribute("href") - |> Option.map (fun a -> x.InnerText(), a.Value())) + x.TryGetAttribute("href") + |> Option.map (fun a -> x.InnerText(), a.Value()) + ) |> Seq.truncate 10 |> Seq.toList @@ -79,13 +78,8 @@ and `Seq.map`. let searchResults = links |> List.filter (fun (name, url) -> - name <> "Cached" - && name <> "Similar" - && url.StartsWith("/url?")) - |> List.map (fun (name, url) -> - name, - url - .Substring(0, url.IndexOf("&sa=")) - .Replace("/url?q=", "")) + name <> "Cached" && name <> "Similar" && url.StartsWith("/url?")) + |> List.map (fun (name, url) -> name, url.Substring(0, url.IndexOf("&sa=")).Replace("/url?q=", "")) (*** include-fsi-merged-output ***) + diff --git a/docs/library/HtmlProvider.fsx b/docs/library/HtmlProvider.fsx index 532baee1c..488d23baa 100644 --- a/docs/library/HtmlProvider.fsx +++ b/docs/library/HtmlProvider.fsx @@ -6,21 +6,18 @@ index: 2 --- *) (*** condition: prepare ***) -#r "../../src/FSharp.Data/bin/Release/netstandard2.0/FSharp.Data.Http.dll" -#r "../../src/FSharp.Data/bin/Release/netstandard2.0/FSharp.Data.Csv.Core.dll" -#r "../../src/FSharp.Data/bin/Release/netstandard2.0/FSharp.Data.Html.Core.dll" #r "../../src/FSharp.Data/bin/Release/netstandard2.0/FSharp.Data.dll" (*** condition: fsx ***) #if FSX #r "nuget: FSharp.Data,{{fsdocs-package-version}}" -#endif +#endif // FSX (*** condition: ipynb ***) #if IPYNB #r "nuget: FSharp.Data,{{fsdocs-package-version}}" Formatter.SetPreferredMimeTypesFor(typeof, "text/plain") -Formatter.Register(fun (x: obj) (writer: TextWriter) -> fprintfn writer "%120A" x) -#endif +Formatter.Register(fun (x:obj) (writer: TextWriter) -> fprintfn writer "%120A" x ) +#endif // IPYNB (** [![Binder](../img/badge-binder.svg)](https://mybinder.org/v2/gh/fsprojects/FSharp.Data/gh-pages?filepath={{fsdocs-source-basename}}.ipynb)  [![Script](../img/badge-script.svg)]({{root}}/{{fsdocs-source-basename}}.fsx)  @@ -61,10 +58,9 @@ first row is headers. (This behaviour is likely to get smarter in later releases *) [] -let F1_2017_URL = - "https://en.wikipedia.org/wiki/2017_FIA_Formula_One_World_Championship" +let ResolutionFolder = __SOURCE_DIRECTORY__ -type F1_2017 = HtmlProvider +type F1_2017 = HtmlProvider<"../data/2017_F1.htm", ResolutionFolder=ResolutionFolder> (** The generated type provides a type space of tables that it has managed to parse out of the given HTML Document. @@ -73,8 +69,13 @@ entities exist then the table will simply be named `Tablexx` where xx is the pos The `Load` method allows reading the data from a file or web resource. We could also have used a web URL instead of a local file in the sample parameter of the type provider. The following sample calls the `Load` method with an URL that points to a live version of the same page on wikipedia. *) +let url = + "https://en.wikipedia.org/wiki/" + + "2017_FIA_Formula_One_World_Championship" + // Download the latest market depth information -let f1Calendar = F1_2017.Load(F1_2017_URL).Tables.``Season calendarEdit`` +let f1Calendar = + F1_2017.Load(url).Tables.``Season calendar`` // Look at the most recent row. Note the 'Date' property // is of type 'DateTime' and 'Open' has a type 'decimal' @@ -85,7 +86,8 @@ let date = firstRow.Date // Print the bid / offer volumes for each row for row in f1Calendar.Rows do - printfn "Race, round %A is hosted at %A on %A" row.Round row.``Grand Prix`` row.Date + printfn "Race, round %A is hosted at %A on %A" + row.Round row.``Grand Prix`` row.Date (*** include-fsi-merged-output ***) @@ -111,30 +113,24 @@ Note that we're using the live URL as the sample, so we can just use the default // Configure the type provider -type NugetStats = HtmlProvider<"https://www.nuget.org/packages/FSharp.Data"> +type NugetStats = + HtmlProvider<"https://www.nuget.org/packages/FSharp.Data"> // load the live package stats for FSharp.Data let rawStats = NugetStats().Tables.Table4 // helper function to analyze version numbers from nuget -let getMinorVersion (v: string) = - System - .Text - .RegularExpressions - .Regex( - @"\d.\d" - ) - .Match( - v - ) - .Value +let getMinorVersion (v:string) = + System.Text.RegularExpressions.Regex(@"\d.\d").Match(v).Value // group by minor version and calculate download count let stats = - rawStats.Rows - |> Seq.groupBy (fun r -> getMinorVersion r.Version) - |> Seq.map (fun (k, xs) -> k, xs |> Seq.sumBy (fun x -> x.Downloads)) - |> Seq.toArray + rawStats.Rows + |> Seq.groupBy (fun r -> + getMinorVersion r.Version) + |> Seq.map (fun (k, xs) -> + k, xs |> Seq.sumBy (fun x -> x.Downloads)) + |> Seq.toArray (*** include-fsi-merged-output ***) @@ -147,23 +143,22 @@ This sample shows some more screen scraping from Wikipedia: *) (*** define-output:doctorWhoChart ***) -[] -let DrWho = - "https://en.wikipedia.org/wiki/List_of_Doctor_Who_episodes_(1963%E2%80%931989)" +let [] DrWho = + "https://en.wikipedia.org/wiki/List_of_Doctor_Who_episodes_(1963%E2%80%931989)" let doctorWho = new HtmlProvider() // Get the average number of viewers for each doctor's series run let viewersByDoctor = - doctorWho.Tables.``Season 1 (1963-1964) Edit``.Rows - |> Seq.groupBy (fun season -> season.``Directed by``) - |> Seq.map (fun (doctor, seasons) -> - let averaged = - seasons - |> Seq.averageBy (fun season -> season.``UK viewers (millions)``) - - doctor, averaged) - |> Seq.toArray + doctorWho.Tables.``Season 1 (1963-1964) Edit``.Rows + |> Seq.groupBy (fun season -> season.``Directed by``) + |> Seq.map (fun (doctor, seasons) -> + let averaged = + seasons + |> Seq.averageBy (fun season -> + season.``UK viewers (millions)``) + doctor, averaged) + |> Seq.toArray (*** include-fsi-merged-output ***) diff --git a/docs/library/Http.fsx b/docs/library/Http.fsx index a69fecd1a..31f92da29 100644 --- a/docs/library/Http.fsx +++ b/docs/library/Http.fsx @@ -6,18 +6,18 @@ index: 1 --- *) (*** condition: prepare ***) -#r "../../src/FSharp.Data/bin/Release/netstandard2.0/FSharp.Data.Http.dll" +#r "../../src/FSharp.Data/bin/Release/netstandard2.0/FSharp.Data.dll" (*** condition: fsx ***) #if FSX #r "nuget: FSharp.Data,{{fsdocs-package-version}}" -#endif +#endif // FSX (*** condition: ipynb ***) #if IPYNB #r "nuget: FSharp.Data,{{fsdocs-package-version}}" Formatter.SetPreferredMimeTypesFor(typeof, "text/plain") -Formatter.Register(fun (x: obj) (writer: TextWriter) -> fprintfn writer "%120A" x) -#endif +Formatter.Register(fun (x:obj) (writer: TextWriter) -> fprintfn writer "%120A" x ) +#endif // IPYNB (** [![Binder](../img/badge-binder.svg)](https://mybinder.org/v2/gh/fsprojects/FSharp.Data/gh-pages?filepath={{fsdocs-source-basename}}.ipynb)  [![Script](../img/badge-script.svg)]({{root}}/{{fsdocs-source-basename}}.fsx)  @@ -56,10 +56,8 @@ can use `cref:M:FSharp.Data.Http.RequestString` and `cref:M:FSharp.Data.Http.Asy Http.RequestString("http://tomasp.net") // Download web site asynchronously -async { - let! html = Http.AsyncRequestString("http://tomasp.net") - printfn "%d" html.Length -} +async { let! html = Http.AsyncRequestString("http://tomasp.net") + printfn "%d" html.Length } |> Async.Start (*** include-fsi-merged-output ***) @@ -76,7 +74,9 @@ can pass them using the optional parameter `query`. The following example also e specifies the GET method, but it will be set automatically for you if you omit it: *) -Http.RequestString("http://httpbin.org/get", query = [ "test", "foo" ], httpMethod = "GET") +Http.RequestString + ( "http://httpbin.org/get", + query=["test", "foo"], httpMethod="GET" ) (*** include-fsi-merged-output ***) @@ -96,12 +96,10 @@ provide your API key: let apiKey = "" // Run the HTTP web request -Http.RequestString( - "http://api.themoviedb.org/3/search/movie", - httpMethod = "GET", - query = [ "api_key", apiKey; "query", "batman" ], - headers = [ "Accept", "application/json" ] -) +Http.RequestString + ( "http://api.themoviedb.org/3/search/movie", httpMethod = "GET", + query = [ "api_key", apiKey; "query", "batman" ], + headers = [ "Accept", "application/json" ]) (** The library supports a simple and unchecked string based API (used in the previous example), @@ -117,11 +115,10 @@ open FSharp.Data.HttpRequestHeaders (*** do-not-eval ***) // Run the HTTP web request -Http.RequestString( - "http://api.themoviedb.org/3/search/movie", - query = [ "api_key", apiKey; "query", "batman" ], - headers = [ Accept HttpContentTypes.Json ] -) +Http.RequestString + ( "http://api.themoviedb.org/3/search/movie", + query = [ "api_key", apiKey; "query", "batman" ], + headers = [ Accept HttpContentTypes.Json ]) (** ## Getting extra information @@ -146,8 +143,7 @@ instead of the `cref:M:FSharp.Data.Http.RequestString` method: *) -let response = - Http.Request("http://api.themoviedb.org/3/search/movie", silentHttpErrors = true) +let response = Http.Request("http://api.themoviedb.org/3/search/movie", silentHttpErrors = true) // Examine information about the response response.Headers @@ -172,7 +168,7 @@ The following example uses the [httpbin.org](http://httpbin.org) service which returns the request details: *) -Http.RequestString("http://httpbin.org/post", body = FormValues [ "test", "foo" ]) +Http.RequestString("http://httpbin.org/post", body = FormValues ["test", "foo"]) (*** include-fsi-merged-output ***) @@ -182,11 +178,10 @@ or `application/octet-stream`, depending on which kind of `HttpRequestBody` you this behaviour by adding `content-type` to the list of headers using the optional argument `headers`: *) -Http.RequestString( - "http://httpbin.org/post", +Http.RequestString + ( "http://httpbin.org/post", headers = [ ContentType HttpContentTypes.Json ], - body = TextRequest """ {"test": 42} """ -) + body = TextRequest """ {"test": 42} """) (*** include-fsi-merged-output ***) @@ -203,8 +198,8 @@ The following is an old sample showing how this is set. // Build URL with documentation for a given class let msdnUrl className = - let root = "http://msdn.microsoft.com" - sprintf "%s/en-gb/library/%s.aspx" root className + let root = "http://msdn.microsoft.com" + sprintf "%s/en-gb/library/%s.aspx" root className // Get the page and search for F# code let docInCSharp = Http.RequestString(msdnUrl "system.web.httprequest") @@ -214,19 +209,16 @@ open System.Net let cc = CookieContainer() // Send a request to switch the language -Http.RequestString( - msdnUrl "system.datetime", - query = - [ "cs-save-lang", "1" - "cs-lang", "fsharp" ], - cookieContainer = cc -) -|> ignore +Http.RequestString + ( msdnUrl "system.datetime", + query = ["cs-save-lang", "1"; "cs-lang","fsharp"], + cookieContainer = cc) |> ignore // Request the documentation again & search for F# let docInFSharp = - Http.RequestString(msdnUrl "system.web.httprequest", cookieContainer = cc) - + Http.RequestString + ( msdnUrl "system.web.httprequest", + cookieContainer = cc ) docInFSharp.Contains "F#" (** @@ -238,10 +230,11 @@ The `cref:M:FSharp.Data.Http.RequestString` method will always return the respon *) let logoUrl = "https://raw.github.com/fsharp/FSharp.Data/master/misc/logo.png" - match Http.Request(logoUrl).Body with -| Text text -> printfn "Got text content: %s" text -| Binary bytes -> printfn "Got %d bytes of binary content" bytes.Length +| Text text -> + printfn "Got text content: %s" text +| Binary bytes -> + printfn "Got %d bytes of binary content" bytes.Length (** ## Customizing the HTTP request @@ -261,16 +254,14 @@ Assuming the certificate is stored in `myCertificate.pfx`: open System.Security.Cryptography.X509Certificates // Load the certificate from local file -let clientCert = new X509Certificate2(".\myCertificate.pfx", "password") +let clientCert = + new X509Certificate2(".\myCertificate.pfx", "password") // Send the request with certificate -Http.Request( - "http://yourprotectedresouce.com/data", - customizeHttpRequest = - fun req -> - req.ClientCertificates.Add(clientCert) |> ignore - req -) +Http.Request + ( "http://yourprotectedresouce.com/data", + customizeHttpRequest = fun req -> + req.ClientCertificates.Add(clientCert) |> ignore; req) (** ## Handling multipart form data @@ -286,14 +277,14 @@ uploads of arbitrary size. let largeFilePath = "//path/to/large/file.mp4" let data = System.IO.File.OpenRead(largeFilePath) :> System.IO.Stream -Http.Request( - "http://endpoint/for/multipart/data", - body = - Multipart( - boundary = "define a custom boundary here", // this is used to separate the items you're streaming - parts = [ MultipartItem("formFieldName", System.IO.Path.GetFileName(largeFilePath), data) ] - ) -) +Http.Request + ( "http://endpoint/for/multipart/data", + body = Multipart( + boundary = "define a custom boundary here", // this is used to separate the items you're streaming + parts = [ + MultipartItem("formFieldName", System.IO.Path.GetFileName(largeFilePath), data) + ] + )) (** ## Related articles diff --git a/docs/library/JsonProvider.fsx b/docs/library/JsonProvider.fsx index a49a3aa9e..6f1a261d7 100644 --- a/docs/library/JsonProvider.fsx +++ b/docs/library/JsonProvider.fsx @@ -6,20 +6,18 @@ index: 3 --- *) (*** condition: prepare ***) -#r "../../src/FSharp.Data/bin/Release/netstandard2.0/FSharp.Data.Http.dll" -#r "../../src/FSharp.Data/bin/Release/netstandard2.0/FSharp.Data.Json.Core.dll" #r "../../src/FSharp.Data/bin/Release/netstandard2.0/FSharp.Data.dll" (*** condition: fsx ***) #if FSX #r "nuget: FSharp.Data,{{fsdocs-package-version}}" -#endif +#endif // FSX (*** condition: ipynb ***) #if IPYNB #r "nuget: FSharp.Data,{{fsdocs-package-version}}" Formatter.SetPreferredMimeTypesFor(typeof, "text/plain") -Formatter.Register(fun (x: obj) (writer: TextWriter) -> fprintfn writer "%120A" x) -#endif +Formatter.Register(fun (x:obj) (writer: TextWriter) -> fprintfn writer "%120A" x ) +#endif // IPYNB (** [![Binder](../img/badge-binder.svg)](https://mybinder.org/v2/gh/fsprojects/FSharp.Data/gh-pages?filepath={{fsdocs-source-basename}}.ipynb)  [![Script](../img/badge-script.svg)]({{root}}/{{fsdocs-source-basename}}.fsx)  @@ -126,15 +124,14 @@ following example uses two records - one with `name` and `age` and the second wi If we want to just use the same text used for the schema at runtime, we can use the `GetSamples` method: *) -type People = - JsonProvider<""" +type People = JsonProvider<""" [ { "name":"John", "age":94 }, { "name":"Tomas" } ] """> for item in People.GetSamples() do - printf "%s " item.Name - item.Age |> Option.iter (printf "(%d)") - printfn "" + printf "%s " item.Name + item.Age |> Option.iter (printf "(%d)") + printfn "" (*** include-fsi-merged-output ***) @@ -153,10 +150,10 @@ as follows: type Values = JsonProvider<""" [{"value":94 }, {"value":"Tomas" }] """> for item in Values.GetSamples() do - match item.Value.Number, item.Value.String with - | Some num, _ -> printfn "Numeric: %d" num - | _, Some str -> printfn "Text: %s" str - | _ -> printfn "Some other value!" + match item.Value.Number, item.Value.String with + | Some num, _ -> printfn "Numeric: %d" num + | _, Some str -> printfn "Text: %s" str + | _ -> printfn "Some other value!" (*** include-fsi-merged-output ***) @@ -177,8 +174,7 @@ Applied to the previous example this would be: *) -type People2 = - JsonProvider<""" +type People2 = JsonProvider<""" [ { "name":"John", "age":94 }, { "name":"Tomas" } ] """, SampleIsList=true> @@ -208,12 +204,12 @@ Let's consider an example where this can be useful: *) type AmbiguousEntity = - JsonProvider - + """, + SampleIsList = true> let code = (AmbiguousEntity.GetSamples()[1]).Code let length = (AmbiguousEntity.GetSamples()[1]).Length @@ -230,12 +226,13 @@ Now let's enable inline schemas: open FSharp.Data.Runtime.StructuralInference type AmbiguousEntity2 = - JsonProvider", "length":"typeof>" } { "code":"123", "length":"42" } { "code":"4E5", "length":"1.83" } - """, SampleIsList=true, InferenceMode=InferenceMode.ValuesAndInlineSchemasOverrides> - + """, + SampleIsList = true, + InferenceMode = InferenceMode.ValuesAndInlineSchemasOverrides> let code2 = (AmbiguousEntity2.GetSamples()[1]).Code let length2 = (AmbiguousEntity2.GetSamples()[1]).Length @@ -263,7 +260,7 @@ Inline schemas also enable support for units of measure. In the previous example, the `Length` property is now inferred as a `float` with the `metre` unit of measure (from the default SI units). -Warning: units of measures are discarded when merged with types without a unit or with a different unit. +Warning: units of measures are discarded when merged with types without a unit or with a different unit. As mentioned previously, with the `ValuesAndInlineSchemasHints` inference mode, inline schemas types are merged with other inferred types with the same precedence. Since values-inferred types never have units, inline-schemas-inferred types will lose their @@ -307,10 +304,11 @@ let doc = WorldBank.GetSample() (** Note that we can also load the data directly from the web both in the `Load` method and in the type provider sample parameter, and there's an asynchronous `AsyncLoad` method available too: *) let wbReq = - "http://api.worldbank.org/country/cz/indicator/" - + "GC.DOD.TOTL.GD.ZS?format=json" + "http://api.worldbank.org/country/cz/indicator/" + + "GC.DOD.TOTL.GD.ZS?format=json" -let docAsync = WorldBank.AsyncLoad(wbReq) +let docAsync = + WorldBank.AsyncLoad(wbReq) (*** include-fsi-merged-output ***) @@ -322,12 +320,13 @@ provider infers that there is only one record and one array. We can print the da // Print general information let info = doc.Record -printfn "Showing page %d of %d. Total records %d" info.Page info.Pages info.Total +printfn "Showing page %d of %d. Total records %d" + info.Page info.Pages info.Total // Print all data points for record in doc.Array do - record.Value - |> Option.iter (fun value -> printfn "%d: %f" record.Date value) + record.Value |> Option.iter (fun value -> + printfn "%d: %f" record.Date value) (*** include-fsi-merged-output ***) @@ -349,13 +348,11 @@ provider that the sample is actually a _list of samples_: *) type Tweet = JsonProvider<"../data/TwitterStream.json", SampleIsList=true, ResolutionFolder=ResolutionFolder> - -let text = (*[omit:(omitted)]*) - """ {"in_reply_to_status_id_str":null,"text":"\u5927\u91d1\u6255\u3063\u3066\u904a\u3070\u3057\u3066\u3082\u3089\u3046\u3002\u3082\u3046\u3053\u306e\u4e0a\u306a\u3044\u8d05\u6ca2\u3002\u3067\u3082\uff0c\u5b9f\u969b\u306b\u306f\u305d\u306e\u8d05\u6ca2\u306e\u672c\u8cea\u3092\u6e80\u55ab\u3067\u304d\u308b\u4eba\u306f\u9650\u3089\u308c\u3066\u308b\u3002\u305d\u3053\u306b\u76ee\u306b\u898b\u3048\u306a\u3044\u968e\u5c64\u304c\u3042\u308b\u3068\u304a\u3082\u3046\u3002","in_reply_to_user_id_str":null,"retweet_count":0,"geo":null,"source":"web","retweeted":false,"truncated":false,"id_str":"263290764686155776","entities":{"user_mentions":[],"hashtags":[],"urls":[]},"in_reply_to_user_id":null,"in_reply_to_status_id":null,"place":null,"coordinates":null,"in_reply_to_screen_name":null,"created_at":"Tue Oct 30 14:46:24 +0000 2012","user":{"notifications":null,"contributors_enabled":false,"time_zone":"Tokyo","profile_background_color":"FFFFFF","location":"Kodaira Tokyo Japan","profile_background_tile":false,"profile_image_url_https":"https:\/\/si0.twimg.com\/profile_images\/1172376796\/70768_100000537851636_3599485_q_normal.jpg","default_profile_image":false,"follow_request_sent":null,"profile_sidebar_fill_color":"17451B","description":"KS(Green62)\/WasedaUniv.(Schl Adv Sci\/Eng)\/SynBio\/ChronoBio\/iGEM2010-2012\/Travel\/Airplane\/ \u5bfa\u30fb\u5ead\u3081\u3050\u308a","favourites_count":17,"screen_name":"Merlin_wand","profile_sidebar_border_color":"000000","id_str":"94788486","verified":false,"lang":"ja","statuses_count":8641,"profile_use_background_image":true,"protected":false,"profile_image_url":"http:\/\/a0.twimg.com\/profile_images\/1172376796\/70768_100000537851636_3599485_q_normal.jpg","listed_count":31,"geo_enabled":true,"created_at":"Sat Dec 05 13:07:32 +0000 2009","profile_text_color":"000000","name":"Marin","profile_background_image_url":"http:\/\/a0.twimg.com\/profile_background_images\/612807391\/twitter_free1.br.jpg","friends_count":629,"url":null,"id":94788486,"is_translator":false,"default_profile":false,"following":null,"profile_background_image_url_https":"https:\/\/si0.twimg.com\/profile_background_images\/612807391\/twitter_free1.br.jpg","utc_offset":32400,"profile_link_color":"ADADAD","followers_count":426},"id":263290764686155776,"contributors":null,"favorited":false} """ (*[/omit]*) - +let text = (*[omit:(omitted)]*)""" {"in_reply_to_status_id_str":null,"text":"\u5927\u91d1\u6255\u3063\u3066\u904a\u3070\u3057\u3066\u3082\u3089\u3046\u3002\u3082\u3046\u3053\u306e\u4e0a\u306a\u3044\u8d05\u6ca2\u3002\u3067\u3082\uff0c\u5b9f\u969b\u306b\u306f\u305d\u306e\u8d05\u6ca2\u306e\u672c\u8cea\u3092\u6e80\u55ab\u3067\u304d\u308b\u4eba\u306f\u9650\u3089\u308c\u3066\u308b\u3002\u305d\u3053\u306b\u76ee\u306b\u898b\u3048\u306a\u3044\u968e\u5c64\u304c\u3042\u308b\u3068\u304a\u3082\u3046\u3002","in_reply_to_user_id_str":null,"retweet_count":0,"geo":null,"source":"web","retweeted":false,"truncated":false,"id_str":"263290764686155776","entities":{"user_mentions":[],"hashtags":[],"urls":[]},"in_reply_to_user_id":null,"in_reply_to_status_id":null,"place":null,"coordinates":null,"in_reply_to_screen_name":null,"created_at":"Tue Oct 30 14:46:24 +0000 2012","user":{"notifications":null,"contributors_enabled":false,"time_zone":"Tokyo","profile_background_color":"FFFFFF","location":"Kodaira Tokyo Japan","profile_background_tile":false,"profile_image_url_https":"https:\/\/si0.twimg.com\/profile_images\/1172376796\/70768_100000537851636_3599485_q_normal.jpg","default_profile_image":false,"follow_request_sent":null,"profile_sidebar_fill_color":"17451B","description":"KS(Green62)\/WasedaUniv.(Schl Adv Sci\/Eng)\/SynBio\/ChronoBio\/iGEM2010-2012\/Travel\/Airplane\/ \u5bfa\u30fb\u5ead\u3081\u3050\u308a","favourites_count":17,"screen_name":"Merlin_wand","profile_sidebar_border_color":"000000","id_str":"94788486","verified":false,"lang":"ja","statuses_count":8641,"profile_use_background_image":true,"protected":false,"profile_image_url":"http:\/\/a0.twimg.com\/profile_images\/1172376796\/70768_100000537851636_3599485_q_normal.jpg","listed_count":31,"geo_enabled":true,"created_at":"Sat Dec 05 13:07:32 +0000 2009","profile_text_color":"000000","name":"Marin","profile_background_image_url":"http:\/\/a0.twimg.com\/profile_background_images\/612807391\/twitter_free1.br.jpg","friends_count":629,"url":null,"id":94788486,"is_translator":false,"default_profile":false,"following":null,"profile_background_image_url_https":"https:\/\/si0.twimg.com\/profile_background_images\/612807391\/twitter_free1.br.jpg","utc_offset":32400,"profile_link_color":"ADADAD","followers_count":426},"id":263290764686155776,"contributors":null,"favorited":false} """(*[/omit]*) let tweet = Tweet.Parse(text) -printfn "%s (retweeted %d times)\n:%s" tweet.User.Value.Name tweet.RetweetCount.Value tweet.Text.Value +printfn "%s (retweeted %d times)\n:%s" + tweet.User.Value.Name tweet.RetweetCount.Value tweet.Text.Value (*** include-fsi-merged-output ***) @@ -396,8 +393,7 @@ we need to post a JSON value similar to this: *) [] -let issueSample = - """ +let issueSample = """ { "title": "Found a bug", "body": "I'm having a problem with this.", @@ -422,14 +418,12 @@ create an instance, and send a POST request: type GitHubIssue = JsonProvider let newIssue = - GitHubIssue.Issue( - "Test issue", - "This is a test issue created in FSharp.Data documentation", - assignee = "", - labels = [||], - milestone = 0 - ) - + GitHubIssue.Issue + ( "Test issue", + "This is a test issue created in FSharp.Data documentation", + assignee = "", + labels = [| |], + milestone = 0) newIssue.JsonValue.Request "https://api.github.com/repos/fsharp/FSharp.Data/issues" (** @@ -448,8 +442,9 @@ static parameter `EmbeddedResource` (don't forget then to [include the file](htt project file). If you are building a library `MyLib.dll`, you can write: *) -type WB = - JsonProvider<"../data/WorldBank.json", EmbeddedResource="MyLib, MyLib.data.worldbank.json", ResolutionFolder=ResolutionFolder> +type WB = JsonProvider<"../data/WorldBank.json", + EmbeddedResource="MyLib, MyLib.data.worldbank.json", + ResolutionFolder=ResolutionFolder> (** You still need to specify the local path, but this is only used when compiling `MyLib.dll`. diff --git a/docs/library/JsonValue.fsx b/docs/library/JsonValue.fsx index 7d6092286..7a0451a29 100644 --- a/docs/library/JsonValue.fsx +++ b/docs/library/JsonValue.fsx @@ -6,20 +6,18 @@ index: 5 --- *) (*** condition: prepare ***) -#r "../../src/FSharp.Data/bin/Release/netstandard2.0/FSharp.Data.Http.dll" -#r "../../src/FSharp.Data/bin/Release/netstandard2.0/FSharp.Data.Json.Core.dll" #r "../../src/FSharp.Data/bin/Release/netstandard2.0/FSharp.Data.dll" (*** condition: fsx ***) #if FSX #r "nuget: FSharp.Data,{{fsdocs-package-version}}" -#endif +#endif // FSX (*** condition: ipynb ***) #if IPYNB #r "nuget: FSharp.Data,{{fsdocs-package-version}}" Formatter.SetPreferredMimeTypesFor(typeof, "text/plain") -Formatter.Register(fun (x: obj) (writer: TextWriter) -> fprintfn writer "%120A" x) -#endif +Formatter.Register(fun (x:obj) (writer: TextWriter) -> fprintfn writer "%120A" x ) +#endif // IPYNB (** [![Binder](../img/badge-binder.svg)](https://mybinder.org/v2/gh/fsprojects/FSharp.Data/gh-pages?filepath={{fsdocs-source-basename}}.ipynb)  [![Script](../img/badge-script.svg)]({{root}}/{{fsdocs-source-basename}}.fsx)  @@ -47,11 +45,9 @@ to parse strings formatted using JSON as follows: *) let info = - JsonValue.Parse( - """ + JsonValue.Parse(""" { "name": "Tomas", "born": 1985, - "siblings": [ "Jan", "Alexander" ] } """ - ) + "siblings": [ "Jan", "Alexander" ] } """) (*** include-fsi-merged-output ***) @@ -107,7 +103,7 @@ printfn "%s (%d)" (info?name.AsString()) (info?born.AsInteger()) // Print names of all siblings for sib in info?siblings do - printfn "%s" (sib.AsString()) + printfn "%s" (sib.AsString()) (*** include-fsi-merged-output ***) @@ -146,10 +142,11 @@ let value = JsonValue.Load(__SOURCE_DIRECTORY__ + "../../data/WorldBank.json") asynchronous version available too: *) let wbReq = - "http://api.worldbank.org/country/cz/indicator/" - + "GC.DOD.TOTL.GD.ZS?format=json" + "http://api.worldbank.org/country/cz/indicator/" + + "GC.DOD.TOTL.GD.ZS?format=json" -let valueAsync = JsonValue.AsyncLoad(wbReq) +let valueAsync = + JsonValue.AsyncLoad(wbReq) (*** include-fsi-merged-output ***) @@ -162,12 +159,15 @@ match value with | JsonValue.Array [| info; data |] -> // Print overall information let page, pages, total = info?page, info?pages, info?total - printfn "Showing page %d of %d. Total records %d" (page.AsInteger()) (pages.AsInteger()) (total.AsInteger()) + printfn + "Showing page %d of %d. Total records %d" + (page.AsInteger()) (pages.AsInteger()) (total.AsInteger()) // Print every non-null data point for record in data do - if record?value <> JsonValue.Null then - printfn "%d: %f" (record?date.AsInteger()) (record?value.AsFloat()) + if record?value <> JsonValue.Null then + printfn "%d: %f" (record?date.AsInteger()) + (record?value.AsFloat()) | _ -> printfn "failed" (*** include-fsi-merged-output ***) diff --git a/docs/library/WorldBank.fsx b/docs/library/WorldBank.fsx index d6ce9936f..fa2afe250 100644 --- a/docs/library/WorldBank.fsx +++ b/docs/library/WorldBank.fsx @@ -6,20 +6,18 @@ index: 5 --- *) (*** condition: prepare ***) -#r "../../src/FSharp.Data/bin/Release/netstandard2.0/FSharp.Data.Http.dll" -#r "../../src/FSharp.Data/bin/Release/netstandard2.0/FSharp.Data.WorldBank.Core.dll" #r "../../src/FSharp.Data/bin/Release/netstandard2.0/FSharp.Data.dll" (*** condition: fsx ***) #if FSX #r "nuget: FSharp.Data,{{fsdocs-package-version}}" -#endif +#endif // FSX (*** condition: ipynb ***) #if IPYNB #r "nuget: FSharp.Data,{{fsdocs-package-version}}" Formatter.SetPreferredMimeTypesFor(typeof, "text/plain") -Formatter.Register(fun (x: obj) (writer: TextWriter) -> fprintfn writer "%120A" x) -#endif +Formatter.Register(fun (x:obj) (writer: TextWriter) -> fprintfn writer "%120A" x ) +#endif // IPYNB (** [![Binder](../img/badge-binder.svg)](https://mybinder.org/v2/gh/fsprojects/FSharp.Data/gh-pages?filepath={{fsdocs-source-basename}}.ipynb)  [![Script](../img/badge-script.svg)]({{root}}/{{fsdocs-source-basename}}.fsx)  @@ -56,7 +54,9 @@ open FSharp.Data let data = WorldBankData.GetDataContext() -data.Countries.``United Kingdom``.Indicators.``Gross capital formation (% of GDP)`` +data + .Countries.``United Kingdom`` + .Indicators.``Gross capital formation (% of GDP)`` |> Seq.maxBy fst (*** include-fsi-merged-output ***) @@ -103,20 +103,20 @@ let wb = WorldBank.GetDataContext() // Create a list of countries to process let countries = - [| wb.Countries.``Arab World`` - wb.Countries.``European Union`` - wb.Countries.Australia - wb.Countries.Brazil - wb.Countries.Canada - wb.Countries.Chile - wb.Countries.``Czech Republic`` - wb.Countries.Denmark - wb.Countries.France - wb.Countries.Greece - wb.Countries.``Low income`` - wb.Countries.``High income`` - wb.Countries.``United Kingdom`` - wb.Countries.``United States`` |] + [| wb.Countries.``Arab World`` + wb.Countries.``European Union`` + wb.Countries.Australia + wb.Countries.Brazil + wb.Countries.Canada + wb.Countries.Chile + wb.Countries.``Czech Republic`` + wb.Countries.Denmark + wb.Countries.France + wb.Countries.Greece + wb.Countries.``Low income`` + wb.Countries.``High income`` + wb.Countries.``United Kingdom`` + wb.Countries.``United States`` |] (** To download the information in parallel, we can create a list of asynchronous @@ -124,7 +124,8 @@ computations, compose them using `Async.Parallel` and then run the (single) obta computation to perform all the downloads: *) -[ for c in countries -> c.Indicators.``Gross capital formation (% of GDP)`` ] +[ for c in countries -> + c.Indicators.``Gross capital formation (% of GDP)`` ] |> Async.Parallel |> Async.RunSynchronously diff --git a/docs/library/XmlProvider.fsx b/docs/library/XmlProvider.fsx index a0f24d825..bd71471d2 100644 --- a/docs/library/XmlProvider.fsx +++ b/docs/library/XmlProvider.fsx @@ -6,21 +6,18 @@ index: 4 --- *) (*** condition: prepare ***) -#r "../../src/FSharp.Data/bin/Release/netstandard2.0/FSharp.Data.Http.dll" -#r "../../src/FSharp.Data/bin/Release/netstandard2.0/FSharp.Data.Csv.Core.dll" -#r "../../src/FSharp.Data/bin/Release/netstandard2.0/FSharp.Data.Xml.Core.dll" #r "../../src/FSharp.Data/bin/Release/netstandard2.0/FSharp.Data.dll" (*** condition: fsx ***) #if FSX #r "nuget: FSharp.Data,{{fsdocs-package-version}}" -#endif +#endif // FSX (*** condition: ipynb ***) #if IPYNB #r "nuget: FSharp.Data,{{fsdocs-package-version}}" Formatter.SetPreferredMimeTypesFor(typeof, "text/plain") -Formatter.Register(fun (x: obj) (writer: TextWriter) -> fprintfn writer "%120A" x) -#endif +Formatter.Register(fun (x:obj) (writer: TextWriter) -> fprintfn writer "%120A" x ) +#endif // IPYNB (** [![Binder](../img/badge-binder.svg)](https://mybinder.org/v2/gh/fsprojects.github.io/FSharp.Data/main?filepath={{fsdocs-source-basename}}.ipynb)  [![Script](../img/badge-script.svg)]({{root}}/{{fsdocs-source-basename}}.fsx)  @@ -51,7 +48,6 @@ is located in the `../../bin` directory, we can load it in F# Interactive as fol `XDocument` type internally): *) #r "System.Xml.Linq.dll" - open FSharp.Data (** @@ -103,9 +99,7 @@ all, what if a node contains some value, but also has some attributes? *) type Detailed = XmlProvider<"""Karl Popper"""> - -let info = - Detailed.Parse("""Thomas Kuhn""") +let info = Detailed.Parse("""Thomas Kuhn""") printfn "%s (full=%b)" info.Name.Value info.Name.Full @@ -128,7 +122,7 @@ contains multiple `` nodes (note that if we leave out the parameter to th type Test = XmlProvider<"13"> for v in Test.GetSample().Values do - printfn "%d" v + printfn "%d" v (** The type provider generates a property `Values` that returns an array with the @@ -151,12 +145,12 @@ Let's consider an example where this can be useful: *) type AmbiguousEntity = - XmlProvider - """, SampleIsList=true> - + """, + SampleIsList = true> let code = (AmbiguousEntity.GetSamples()[1]).Code let length = (AmbiguousEntity.GetSamples()[1]).Length @@ -173,12 +167,13 @@ Now let's enable inline schemas: open FSharp.Data.Runtime.StructuralInference type AmbiguousEntity2 = - XmlProvider - """, SampleIsList=true, InferenceMode=InferenceMode.ValuesAndInlineSchemasOverrides> - + """, + SampleIsList = true, + InferenceMode = InferenceMode.ValuesAndInlineSchemasOverrides> let code2 = (AmbiguousEntity2.GetSamples()[1]).Code let length2 = (AmbiguousEntity2.GetSamples()[1]).Length @@ -204,7 +199,7 @@ Inline schemas also enable support for units of measure. In the previous example, the `Length` property is now inferred as a `float` with the `metre` unit of measure (from the default SI units). -Warning: units of measures are discarded when merged with types without a unit or with a different unit. +Warning: units of measures are discarded when merged with types without a unit or with a different unit. As mentioned previously, with the `ValuesAndInlineSchemasHints` inference mode, inline schemas types are merged with other inferred types with the same precedence. Since values-inferred types never have units, inline-schemas-inferred types will lose their @@ -230,8 +225,7 @@ At runtime, we use the generated type provider to parse the following string one of the `author` nodes also contains a `died` attribute): *) -let authors = - """ +let authors = """ @@ -251,7 +245,6 @@ type Authors = XmlProvider<"../data/Writers.xml", ResolutionFolder=ResolutionFol let topic = Authors.Parse(authors) printfn "%s" topic.Topic - for author in topic.Authors do printf " - %s" author.Name author.Born |> Option.iter (printf " (%d)") @@ -304,12 +297,11 @@ that takes `Html.Div` and acts as follows: *) /// Prints the content of a
element -let rec printDiv (div: Html.Div) = - div.Spans |> Seq.iter (printfn "%s") - div.Divs |> Seq.iter printDiv - - if div.Spans.Length = 0 && div.Divs.Length = 0 then - div.Value |> Option.iter (printfn "%s") +let rec printDiv (div:Html.Div) = + div.Spans |> Seq.iter (printfn "%s") + div.Divs |> Seq.iter printDiv + if div.Spans.Length = 0 && div.Divs.Length = 0 then + div.Value |> Option.iter (printfn "%s") // Print the root
element with all children printDiv html @@ -363,7 +355,7 @@ let data = Census.Load("https://api.census.gov/data.xml") let apiLinks = data.Datasets - |> Array.map (fun ds -> ds.Title, ds.Distribution.AccessUrl) + |> Array.map (fun ds -> ds.Title,ds.Distribution.AccessUrl) |> Array.truncate 10 (*** include-fsi-merged-output ***) @@ -384,19 +376,16 @@ and then post each link over (for example) a message queue. This is where `AsyncLoad` comes into play: *) -let enqueue (title, apiUrl) = - // do the real message enqueueing here instead of - printfn "%s -> %s" title apiUrl +let enqueue (title,apiUrl) = + // do the real message enqueueing here instead of + printfn "%s -> %s" title apiUrl // helper task which gets scheduled on some background thread somewhere... -let cacheJanitor () = - async { - let! reloadData = Census.AsyncLoad("https://api.census.gov/data.xml") - - reloadData.Datasets - |> Array.map (fun ds -> ds.Title, ds.Distribution.AccessUrl) - |> Array.iter enqueue - } +let cacheJanitor() = async { + let! reloadData = Census.AsyncLoad("https://api.census.gov/data.xml") + reloadData.Datasets |> Array.map (fun ds -> ds.Title,ds.Distribution.AccessUrl) + |> Array.iter enqueue +} (*** include-fsi-merged-output ***) @@ -430,7 +419,7 @@ printfn "%s" blog.Channel.Title // Get all item nodes and print title with link for item in blog.Channel.Items do - printfn " - %s (%s)" item.Title item.Link + printfn " - %s (%s)" item.Title item.Link (*** include-fsi-merged-output ***) @@ -443,8 +432,7 @@ Consider the problem of flattening a data set. Let's say you have xml data that *) [] -let customersXmlSample = - """ +let customersXmlSample = """ @@ -467,8 +455,7 @@ and you want to transform it into something like this: *) [] -let orderLinesXmlSample = - """ +let orderLinesXmlSample = """ @@ -484,11 +471,15 @@ type InputXml = XmlProvider type OutputXml = XmlProvider let orderLines = - OutputXml.OrderLines - [| for customer in InputXml.GetSample().Customers do - for order in customer.Orders do - for line in order.OrderLines do - yield OutputXml.OrderLine(customer.Name, order.Number, line.Item, line.Quantity) |] + OutputXml.OrderLines [| + for customer in InputXml.GetSample().Customers do + for order in customer.Orders do + for line in order.OrderLines do + yield OutputXml.OrderLine + ( customer.Name, + order.Number, + line.Item, + line.Quantity ) |] (*** include-fsi-merged-output ***) @@ -501,8 +492,7 @@ The value of the parameter can be either the name of a schema file or plain text like in the following example: *) -type Person = - XmlProvider @@ -515,9 +505,7 @@ type Person = """> -let turing = - Person.Parse - """ +let turing = Person.Parse """ Turing 1912-06-23 @@ -541,7 +529,7 @@ When the file includes other schema files, the `ResolutionFolder` parameter can The uri may also refer to online resources: *) -type RssXsd = XmlProvider +type RssXsd = XmlProvider (** @@ -549,8 +537,7 @@ The schema is expected to define a root element (a global element with complex t In case of multiple root elements: *) -type TwoRoots = - XmlProvider @@ -573,15 +560,15 @@ the provided type has an optional property for each alternative: *) let e1 = TwoRoots.Parse "" - match e1.Root1, e1.Root2 with -| Some x, None -> printfn "Foo = %s and Fow = %A" x.Foo x.Fow +| Some x, None -> + printfn "Foo = %s and Fow = %A" x.Foo x.Fow | _ -> failwith "Unexpected" let e2 = TwoRoots.Parse "" - match e2.Root1, e2.Root2 with -| None, Some x -> printfn "Bar = %s and Baz = %O" x.Bar x.Baz +| None, Some x -> + printfn "Bar = %s and Baz = %O" x.Bar x.Baz | _ -> failwith "Unexpected" (*** include-fsi-merged-output ***) @@ -596,8 +583,7 @@ The following xsd defines `foo` as a sequence made of an arbitrary number of `bar` elements followed by a single `baz` element. *) -type FooSequence = - XmlProvider @@ -614,9 +600,7 @@ type FooSequence = here a valid xml element is parsed as an instance of the provided type, with two properties corresponding to `bar`and `baz` elements, where the former is an array in order to hold multiple elements: *) -let fooSequence = - FooSequence.Parse - """ +let fooSequence = FooSequence.Parse """ 42 43 @@ -630,8 +614,7 @@ printfn "%d" fooSequence.Baz.Year // 1957 (** Instead of a sequence we may have a `choice`: *) -type FooChoice = - XmlProvider @@ -653,15 +636,12 @@ in type providers) but also preferred because it improves discoverability: intellisense can show both alternatives. There is a lack of precision but this is not the main goal. *) -let fooChoice = - FooChoice.Parse - """ +let fooChoice = FooChoice.Parse """ 1957-08-13 """ printfn "%d items" fooChoice.Bars.Length // 0 items - match fooChoice.Baz with | Some date -> printfn "%d" date.Year // 1957 | None -> () @@ -679,8 +659,7 @@ XML Schema provides various extensibility mechanisms. The following example is a terse summary mixing substitution groups with abstract recursive definitions. *) -type Prop = - XmlProvider @@ -694,9 +673,7 @@ type Prop = """> -let formula = - Prop.Parse - """ +let formula = Prop.Parse """ p1 diff --git a/docs/tutorials/JsonAnonymizer.fsx b/docs/tutorials/JsonAnonymizer.fsx index f7e1096b2..0f76f40c1 100644 --- a/docs/tutorials/JsonAnonymizer.fsx +++ b/docs/tutorials/JsonAnonymizer.fsx @@ -6,19 +6,18 @@ index: 1 --- *) (*** condition: prepare ***) -#r "../../src/FSharp.Data/bin/Release/netstandard2.0/FSharp.Data.Http.dll" -#r "../../src/FSharp.Data/bin/Release/netstandard2.0/FSharp.Data.Json.Core.dll" +#r "../../src/FSharp.Data/bin/Release/netstandard2.0/FSharp.Data.dll" (*** condition: fsx ***) #if FSX #r "nuget: FSharp.Data,{{fsdocs-package-version}}" -#endif +#endif // FSX (*** condition: ipynb ***) #if IPYNB #r "nuget: FSharp.Data,{{fsdocs-package-version}}" Formatter.SetPreferredMimeTypesFor(typeof, "text/plain") -Formatter.Register(fun (x: obj) (writer: TextWriter) -> fprintfn writer "%120A" x) -#endif +Formatter.Register(fun (x:obj) (writer: TextWriter) -> fprintfn writer "%120A" x ) +#endif // IPYNB (** # Anonymizing JSON @@ -49,112 +48,86 @@ open FSharp.Data type JsonAnonymizer(?propertiesToSkip, ?valuesToSkip) = - let propertiesToSkip = Set.ofList (defaultArg propertiesToSkip []) - let valuesToSkip = Set.ofList (defaultArg valuesToSkip []) - - let rng = Random() - - let digits = [| '0' .. '9' |] - let lowerLetters = [| 'a' .. 'z' |] - let upperLetters = [| 'A' .. 'Z' |] - - let getRandomChar (c: char) = - if Char.IsDigit c then - digits.[rng.Next(10)] - elif Char.IsLetter c then - if Char.IsLower c then - lowerLetters.[rng.Next(26)] + let propertiesToSkip = Set.ofList (defaultArg propertiesToSkip []) + let valuesToSkip = Set.ofList (defaultArg valuesToSkip []) + + let rng = Random() + + let digits = [| '0' .. '9' |] + let lowerLetters = [| 'a' .. 'z' |] + let upperLetters = [| 'A' .. 'Z' |] + + let getRandomChar (c:char) = + if Char.IsDigit c then digits.[rng.Next(10)] + elif Char.IsLetter c then + if Char.IsLower c + then lowerLetters.[rng.Next(26)] + else upperLetters.[rng.Next(26)] + else c + + let randomize (str:string) = + String(str.ToCharArray() |> Array.map getRandomChar) + + let isType testType typ = + match typ with + | Runtime.StructuralTypes.InferedType.Primitive (typ, _, _, _) -> typ = testType + | _ -> false + + let rec anonymize json = + match json with + | JsonValue.String s when valuesToSkip.Contains s -> json + | JsonValue.String s -> + let typ = + Runtime.StructuralInference.inferPrimitiveType + Runtime.StructuralInference.defaultUnitsOfMeasureProvider + Runtime.StructuralInference.InferenceMode'.ValuesOnly + CultureInfo.InvariantCulture s None + + ( if typ |> isType typeof then Guid.NewGuid().ToString() + elif typ |> isType typeof || + typ |> isType typeof then s + elif typ |> isType typeof then s else - upperLetters.[rng.Next(26)] - else - c - - let randomize (str: string) = - String(str.ToCharArray() |> Array.map getRandomChar) - - let isType testType typ = - match typ with - | Runtime.StructuralTypes.InferedType.Primitive (typ, _, _, _) -> typ = testType - | _ -> false - - let rec anonymize json = - match json with - | JsonValue.String s when valuesToSkip.Contains s -> json - | JsonValue.String s -> - let typ = - Runtime.StructuralInference.inferPrimitiveType - Runtime.StructuralInference.defaultUnitsOfMeasureProvider - Runtime.StructuralInference.InferenceMode'.ValuesOnly - CultureInfo.InvariantCulture - s - None - - (if typ |> isType typeof then - Guid.NewGuid().ToString() - elif typ |> isType typeof - || typ |> isType typeof then - s - elif typ |> isType typeof then - s - else - let prefix, s = - if s.StartsWith "http://" then - "http://", s.Substring("http://".Length) - elif s.StartsWith "https://" then - "https://", s.Substring("https://".Length) - else - "", s - - prefix + randomize s) - |> JsonValue.String - | JsonValue.Number d -> - let typ = - Runtime.StructuralInference.inferPrimitiveType - Runtime.StructuralInference.defaultUnitsOfMeasureProvider - Runtime.StructuralInference.InferenceMode'.ValuesOnly - CultureInfo.InvariantCulture - (d.ToString()) - None - - if typ |> isType typeof - || typ |> isType typeof then - json - else - d.ToString() - |> randomize - |> Decimal.Parse - |> JsonValue.Number - | JsonValue.Float f -> - f.ToString() - |> randomize - |> Double.Parse - |> JsonValue.Float - | JsonValue.Boolean _ - | JsonValue.Null -> json - | JsonValue.Record props -> - props - |> Array.map (fun (key, value) -> - let newValue = - if propertiesToSkip.Contains key then - value - else - anonymize value - - key, newValue) - |> JsonValue.Record - | JsonValue.Array array -> array |> Array.map anonymize |> JsonValue.Array - - member _.Anonymize json = anonymize json - -let json = - JsonValue.Load( - __SOURCE_DIRECTORY__ - + "../../data/TwitterStream.json" - ) - + let prefix, s = + if s.StartsWith "http://" then + "http://", s.Substring("http://".Length) + elif s.StartsWith "https://" then + "https://", s.Substring("https://".Length) + else "", s + prefix + randomize s ) + |> JsonValue.String + | JsonValue.Number d -> + let typ = + Runtime.StructuralInference.inferPrimitiveType + Runtime.StructuralInference.defaultUnitsOfMeasureProvider + Runtime.StructuralInference.InferenceMode'.ValuesOnly + CultureInfo.InvariantCulture (d.ToString()) None + if typ |> isType typeof || + typ |> isType typeof then json + else d.ToString() |> randomize |> Decimal.Parse |> JsonValue.Number + | JsonValue.Float f -> + f.ToString() + |> randomize + |> Double.Parse + |> JsonValue.Float + | JsonValue.Boolean _ | JsonValue.Null -> json + | JsonValue.Record props -> + props + |> Array.map (fun (key, value) -> + let newValue = if propertiesToSkip.Contains key then value else anonymize value + key, newValue) + |> JsonValue.Record + | JsonValue.Array array -> + array + |> Array.map anonymize + |> JsonValue.Array + + member __.Anonymize json = anonymize json + +let json = JsonValue.Load (__SOURCE_DIRECTORY__ + "../../data/TwitterStream.json") printfn "%O" json -let anonymizedJson = (JsonAnonymizer [ "lang" ]).Anonymize json +let anonymizedJson = (JsonAnonymizer ["lang"]).Anonymize json printfn "%O" anonymizedJson (** diff --git a/docs/tutorials/JsonToXml.fsx b/docs/tutorials/JsonToXml.fsx index 88e55b78d..7862b442c 100644 --- a/docs/tutorials/JsonToXml.fsx +++ b/docs/tutorials/JsonToXml.fsx @@ -6,21 +6,18 @@ index: 2 --- *) (*** condition: prepare ***) -#r "../../src/FSharp.Data/bin/Release/netstandard2.0/FSharp.Data.Http.dll" -#r "../../src/FSharp.Data/bin/Release/netstandard2.0/FSharp.Data.Csv.Core.dll" -#r "../../src/FSharp.Data/bin/Release/netstandard2.0/FSharp.Data.Json.Core.dll" -#r "../../src/FSharp.Data/bin/Release/netstandard2.0/FSharp.Data.Xml.Core.dll" +#r "../../src/FSharp.Data/bin/Release/netstandard2.0/FSharp.Data.dll" (*** condition: fsx ***) #if FSX #r "nuget: FSharp.Data,{{fsdocs-package-version}}" -#endif +#endif // FSX (*** condition: ipynb ***) #if IPYNB #r "nuget: FSharp.Data,{{fsdocs-package-version}}" Formatter.SetPreferredMimeTypesFor(typeof, "text/plain") -Formatter.Register(fun (x: obj) (writer: TextWriter) -> fprintfn writer "%120A" x) -#endif +Formatter.Register(fun (x:obj) (writer: TextWriter) -> fprintfn writer "%120A" x ) +#endif // IPYNB (** # Converting between JSON and XML @@ -50,7 +47,6 @@ We will be using the LINQ to XML API (available in `System.Xml.Linq.dll`) and th *) #r "System.Xml.Linq.dll" - open System.Xml.Linq open FSharp.Data @@ -96,32 +92,33 @@ attributes to a corresponding JSON type: *) /// Creates a JSON representation of a XML element -let rec fromXml (xml: XElement) = - - // Create a collection of key/value pairs for all attributes - let attrs = - [ for attr in xml.Attributes() -> (attr.Name.LocalName, JsonValue.String attr.Value) ] - - // Function that turns a collection of XElement values - // into an array of JsonValue (using fromXml recursively) - let createArray xelems = - [| for xelem in xelems -> fromXml xelem |] - |> JsonValue.Array - - // Group child elements by their name and then turn all single- - // element groups into a record (recursively) and all multi- - // element groups into a JSON array using createArray - let children = - xml.Elements() - |> Seq.groupBy (fun x -> x.Name.LocalName) - |> Seq.map (fun (key, childs) -> - match Seq.toList childs with - | [ child ] -> key, fromXml child - | children -> key + "s", createArray children) - - // Concatenate elements produced for child elements & attributes - Array.append (Array.ofList attrs) (Array.ofSeq children) - |> JsonValue.Record +let rec fromXml (xml:XElement) = + + // Create a collection of key/value pairs for all attributes + let attrs = + [ for attr in xml.Attributes() -> + (attr.Name.LocalName, JsonValue.String attr.Value) ] + + // Function that turns a collection of XElement values + // into an array of JsonValue (using fromXml recursively) + let createArray xelems = + [| for xelem in xelems -> fromXml xelem |] + |> JsonValue.Array + + // Group child elements by their name and then turn all single- + // element groups into a record (recursively) and all multi- + // element groups into a JSON array using createArray + let children = + xml.Elements() + |> Seq.groupBy (fun x -> x.Name.LocalName) + |> Seq.map (fun (key, childs) -> + match Seq.toList childs with + | [child] -> key, fromXml child + | children -> key + "s", createArray children ) + + // Concatenate elements produced for child elements & attributes + Array.append (Array.ofList attrs) (Array.ofSeq children) + |> JsonValue.Record (** @@ -158,48 +155,44 @@ and record construct nested element(s) or attribute: /// Creates an XML representation of a JSON value (works /// only when the top-level value is an object or an array) -let toXml (x: JsonValue) = - // Helper functions for constructing XML - // attributes and XML elements - let attr name value = - XAttribute(XName.Get name, value) :> XObject - - let elem name (value: obj) = - XElement(XName.Get name, value) :> XObject - - // Inner recursive function that implements the conversion - let rec toXml = - function - // Primitive values are returned as objects - | JsonValue.Null -> null - | JsonValue.Boolean b -> b :> obj - | JsonValue.Number number -> number :> obj - | JsonValue.Float number -> number :> obj - | JsonValue.String s -> s :> obj - - // JSON object becomes a collection of XML - // attributes (for primitives) or child elements - | JsonValue.Record properties -> - properties - |> Array.map (fun (key, value) -> - match value with - | JsonValue.String s -> attr key s - | JsonValue.Boolean b -> attr key b - | JsonValue.Number n -> attr key n - | JsonValue.Float n -> attr key n - | _ -> elem key (toXml value)) - :> obj - - // JSON array is turned into a - // sequence of elements - | JsonValue.Array elements -> - elements - |> Array.map (fun item -> elem "item" (toXml item)) - :> obj - - // Perform the conversion and cast the result to sequence - // of objects (may fail for unexpected inputs!) - (toXml x) :?> XObject seq +let toXml(x:JsonValue) = + // Helper functions for constructing XML + // attributes and XML elements + let attr name value = + XAttribute(XName.Get name, value) :> XObject + let elem name (value:obj) = + XElement(XName.Get name, value) :> XObject + + // Inner recursive function that implements the conversion + let rec toXml = function + // Primitive values are returned as objects + | JsonValue.Null -> null + | JsonValue.Boolean b -> b :> obj + | JsonValue.Number number -> number :> obj + | JsonValue.Float number -> number :> obj + | JsonValue.String s -> s :> obj + + // JSON object becomes a collection of XML + // attributes (for primitives) or child elements + | JsonValue.Record properties -> + properties + |> Array.map (fun (key, value) -> + match value with + | JsonValue.String s -> attr key s + | JsonValue.Boolean b -> attr key b + | JsonValue.Number n -> attr key n + | JsonValue.Float n -> attr key n + | _ -> elem key (toXml value)) :> obj + + // JSON array is turned into a + // sequence of elements + | JsonValue.Array elements -> + elements |> Array.map (fun item -> + elem "item" (toXml item)) :> obj + + // Perform the conversion and cast the result to sequence + // of objects (may fail for unexpected inputs!) + (toXml x) :?> XObject seq (** diff --git a/nuget/FSharp.Data.nuspec b/nuget/FSharp.Data.nuspec new file mode 100755 index 000000000..f8c36e670 --- /dev/null +++ b/nuget/FSharp.Data.nuspec @@ -0,0 +1,39 @@ + + + + @project@ + FSharp.Data + @build.number@ + @authors@ + @authors@ + Apache-2.0 + https://fsprojects.github.io/FSharp.Data + https://raw.github.com/fsharp/FSharp.Data/master/misc/logo.png + false + @summary@ + @description@ + @releaseNotes@ + Copyright 2021 + @tags@ + + + + + + + + + + + + + + + + + + + + + + diff --git a/nuget/linqpad-samples/GitHub.linq b/nuget/linqpad-samples/GitHub.linq new file mode 100644 index 000000000..2bd722d42 --- /dev/null +++ b/nuget/linqpad-samples/GitHub.linq @@ -0,0 +1,19 @@ + + FSharp.Core, Version=4.3.0.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a + <ProgramFilesX86>\Reference Assemblies\Microsoft\FSharp\.NETFramework\v4.0\4.3.0.0\FSharp.Core.dll + FSharp.Data + + +open FSharp.Data + +// Beware of rate limiting while running this sample: https://developer.github.com/v3/#rate-limiting +type GitHub = JsonProvider<"https://api.github.com/repos/fsharp/FSharp.Data/issues"> + +let topRecentlyUpdatedIssues = + GitHub.GetSamples() + |> Seq.filter (fun issue -> issue.State = "open") + |> Seq.sortBy (fun issue -> System.DateTime.Now - issue.UpdatedAt) + |> Seq.truncate 5 + +for issue in topRecentlyUpdatedIssues do + printfn "#%d %s" issue.Number issue.Title \ No newline at end of file diff --git a/nuget/linqpad-samples/Jira.linq b/nuget/linqpad-samples/Jira.linq new file mode 100644 index 000000000..b6569d997 --- /dev/null +++ b/nuget/linqpad-samples/Jira.linq @@ -0,0 +1,15 @@ + + FSharp.Core, Version=4.3.0.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a + <ProgramFilesX86>\Reference Assemblies\Microsoft\FSharp\.NETFramework\v4.0\4.3.0.0\FSharp.Core.dll + FSharp.Data + + +open FSharp.Data + +[] +let apiUrl = "https://jira.atlassian.com/rest/api/2/search?filter=-4" // all issues +type Jira = JsonProvider +let jira = Jira.Load(apiUrl) + +let tickets = jira.Issues |> Array.map (fun ticket -> (ticket.Id, ticket.Fields.Summary)) +tickets |> Dump \ No newline at end of file diff --git a/nuget/linqpad-samples/TypeInference.linq b/nuget/linqpad-samples/TypeInference.linq new file mode 100644 index 000000000..193198cd4 --- /dev/null +++ b/nuget/linqpad-samples/TypeInference.linq @@ -0,0 +1,27 @@ + + FSharp.Core, Version=4.3.0.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a + <ProgramFilesX86>\Reference Assemblies\Microsoft\FSharp\.NETFramework\v4.0\4.3.0.0\FSharp.Core.dll + FSharp.Data + + +open FSharp.Data + +// The JsonProvider<...> takes one static parameter of type string. The parameter can be either a sample string or a sample file +// (relative to the current folder or online accessible via http or https). It is not likely that this could lead to ambiguities. +type Simple = JsonProvider<""" { "name":"John", "age":94 } """> +let simple = Simple.Parse(""" { "name":"Tomas", "age":4 } """) +simple.Age |> Dump +simple.Name |> Dump + +// A list may mix integers and floats. When the sample is a collection, the type provider generates a type +// that can be used to store all values in the sample. In this case, the resulting type is decimal, because one of the values is not an integer +type Numbers = JsonProvider<""" [1, 2, 3, 3.14] """> +Numbers.Parse(""" [1.2, 45.1, 98.2, 5] """) |> Dump + +// Other primitive types cannot be combined into a single type. For example, if the list contains numbers and strings. +// In this case, the provider generates two methods that can be used to get values that match one of the types: +type Mixed = JsonProvider<""" [1, 2, "hello", "world"] """> +let mixed = Mixed.Parse(""" [4, 5, "hello", "world" ] """) + +mixed.Numbers |> Seq.sum |> Dump +mixed.Strings |> String.concat ", " |> Dump \ No newline at end of file diff --git a/nuget/publish.cmd b/nuget/publish.cmd new file mode 100644 index 000000000..6a30dc533 --- /dev/null +++ b/nuget/publish.cmd @@ -0,0 +1 @@ +@for %%f in (..\bin\*.nupkg) do @..\packages\NuGet.CommandLine\tools\NuGet.exe push %%f \ No newline at end of file diff --git a/src/AssemblyInfo.Csv.Core.fs b/src/AssemblyInfo.Csv.Core.fs deleted file mode 100644 index c0ef17e0b..000000000 --- a/src/AssemblyInfo.Csv.Core.fs +++ /dev/null @@ -1,27 +0,0 @@ -// Auto-Generated by FAKE; do not edit -namespace System - -open System.Reflection - -[] -[] -[] -[] -[] -do () - -module internal AssemblyVersionInformation = - [] - let AssemblyTitle = "FSharp.Data.Csv.Core" - - [] - let AssemblyProduct = "FSharp.Data" - - [] - let AssemblyDescription = "Library of F# type providers and data access tools" - - [] - let AssemblyVersion = "5.0.1.0" - - [] - let AssemblyFileVersion = "5.0.1.0" diff --git a/src/AssemblyInfo.DesignTime.fs b/src/AssemblyInfo.DesignTime.fs index f870a91a5..8e2005f92 100644 --- a/src/AssemblyInfo.DesignTime.fs +++ b/src/AssemblyInfo.DesignTime.fs @@ -1,27 +1,17 @@ // Auto-Generated by FAKE; do not edit namespace System - open System.Reflection [] [] [] -[] -[] +[] +[] do () module internal AssemblyVersionInformation = - [] - let AssemblyTitle = "FSharp.Data.DesignTime" - - [] - let AssemblyProduct = "FSharp.Data" - - [] - let AssemblyDescription = "Library of F# type providers and data access tools" - - [] - let AssemblyVersion = "5.0.1.0" - - [] - let AssemblyFileVersion = "5.0.1.0" + let [] AssemblyTitle = "FSharp.Data.DesignTime" + let [] AssemblyProduct = "FSharp.Data" + let [] AssemblyDescription = "Library of F# type providers and data access tools" + let [] AssemblyVersion = "4.2.8.0" + let [] AssemblyFileVersion = "4.2.8.0" diff --git a/src/AssemblyInfo.Html.Core.fs b/src/AssemblyInfo.Html.Core.fs deleted file mode 100644 index bd3974889..000000000 --- a/src/AssemblyInfo.Html.Core.fs +++ /dev/null @@ -1,27 +0,0 @@ -// Auto-Generated by FAKE; do not edit -namespace System - -open System.Reflection - -[] -[] -[] -[] -[] -do () - -module internal AssemblyVersionInformation = - [] - let AssemblyTitle = "FSharp.Data.Html.Core" - - [] - let AssemblyProduct = "FSharp.Data" - - [] - let AssemblyDescription = "Library of F# type providers and data access tools" - - [] - let AssemblyVersion = "5.0.1.0" - - [] - let AssemblyFileVersion = "5.0.1.0" diff --git a/src/AssemblyInfo.Http.fs b/src/AssemblyInfo.Http.fs deleted file mode 100644 index e502ea06a..000000000 --- a/src/AssemblyInfo.Http.fs +++ /dev/null @@ -1,27 +0,0 @@ -// Auto-Generated by FAKE; do not edit -namespace System - -open System.Reflection - -[] -[] -[] -[] -[] -do () - -module internal AssemblyVersionInformation = - [] - let AssemblyTitle = "FSharp.Data.Http" - - [] - let AssemblyProduct = "FSharp.Data" - - [] - let AssemblyDescription = "Library of F# type providers and data access tools" - - [] - let AssemblyVersion = "5.0.1.0" - - [] - let AssemblyFileVersion = "5.0.1.0" diff --git a/src/AssemblyInfo.Json.Core.fs b/src/AssemblyInfo.Json.Core.fs deleted file mode 100644 index 2b5815538..000000000 --- a/src/AssemblyInfo.Json.Core.fs +++ /dev/null @@ -1,27 +0,0 @@ -// Auto-Generated by FAKE; do not edit -namespace System - -open System.Reflection - -[] -[] -[] -[] -[] -do () - -module internal AssemblyVersionInformation = - [] - let AssemblyTitle = "FSharp.Data.Json.Core" - - [] - let AssemblyProduct = "FSharp.Data" - - [] - let AssemblyDescription = "Library of F# type providers and data access tools" - - [] - let AssemblyVersion = "5.0.1.0" - - [] - let AssemblyFileVersion = "5.0.1.0" diff --git a/src/AssemblyInfo.WorldBank.Core.fs b/src/AssemblyInfo.WorldBank.Core.fs deleted file mode 100644 index 0c7839464..000000000 --- a/src/AssemblyInfo.WorldBank.Core.fs +++ /dev/null @@ -1,27 +0,0 @@ -// Auto-Generated by FAKE; do not edit -namespace System - -open System.Reflection - -[] -[] -[] -[] -[] -do () - -module internal AssemblyVersionInformation = - [] - let AssemblyTitle = "FSharp.Data.WorldBank.Core" - - [] - let AssemblyProduct = "FSharp.Data" - - [] - let AssemblyDescription = "Library of F# type providers and data access tools" - - [] - let AssemblyVersion = "5.0.1.0" - - [] - let AssemblyFileVersion = "5.0.1.0" diff --git a/src/AssemblyInfo.Xml.Core.fs b/src/AssemblyInfo.Xml.Core.fs deleted file mode 100644 index a77ba79eb..000000000 --- a/src/AssemblyInfo.Xml.Core.fs +++ /dev/null @@ -1,27 +0,0 @@ -// Auto-Generated by FAKE; do not edit -namespace System - -open System.Reflection - -[] -[] -[] -[] -[] -do () - -module internal AssemblyVersionInformation = - [] - let AssemblyTitle = "FSharp.Data.Xml.Core" - - [] - let AssemblyProduct = "FSharp.Data" - - [] - let AssemblyDescription = "Library of F# type providers and data access tools" - - [] - let AssemblyVersion = "5.0.1.0" - - [] - let AssemblyFileVersion = "5.0.1.0" diff --git a/src/AssemblyInfo.fs b/src/AssemblyInfo.fs index 1490e0647..baa16c53a 100644 --- a/src/AssemblyInfo.fs +++ b/src/AssemblyInfo.fs @@ -1,27 +1,17 @@ // Auto-Generated by FAKE; do not edit namespace System - open System.Reflection [] [] [] -[] -[] +[] +[] do () module internal AssemblyVersionInformation = - [] - let AssemblyTitle = "FSharp.Data" - - [] - let AssemblyProduct = "FSharp.Data" - - [] - let AssemblyDescription = "Library of F# type providers and data access tools" - - [] - let AssemblyVersion = "5.0.1.0" - - [] - let AssemblyFileVersion = "5.0.1.0" + let [] AssemblyTitle = "FSharp.Data" + let [] AssemblyProduct = "FSharp.Data" + let [] AssemblyDescription = "Library of F# type providers and data access tools" + let [] AssemblyVersion = "4.2.8.0" + let [] AssemblyFileVersion = "4.2.8.0" diff --git a/src/CommonProviderImplementation/ConversionsGenerator.fs b/src/CommonProviderImplementation/ConversionsGenerator.fs index 3a9af5722..3a5d56fd4 100644 --- a/src/CommonProviderImplementation/ConversionsGenerator.fs +++ b/src/CommonProviderImplementation/ConversionsGenerator.fs @@ -67,7 +67,7 @@ let getBackConversionQuotation missingValuesStr cultureStr typ value : Expr and converts it to /// an expression of other type - the type is specified by `field` -let internal convertStringValue missingValuesStr cultureStr (field: PrimitiveInferedProperty) = +let convertStringValue missingValuesStr cultureStr (field: PrimitiveInferedProperty) = let fieldName = field.Name let field = field.Value diff --git a/src/CommonProviderImplementation/Helpers.fs b/src/CommonProviderImplementation/Helpers.fs index 1ad38bd5e..a4f1d7163 100644 --- a/src/CommonProviderImplementation/Helpers.fs +++ b/src/CommonProviderImplementation/Helpers.fs @@ -103,21 +103,21 @@ type DisposableTypeProviderForNamespaces(config, ?assemblyReplacementMap) as x = use _holder = logTime "DisposingEvent" (sprintf "%O [%d]" x id) dispose None) - member _.Id = id + member __.Id = id - member _.SetFileToWatch(fullTypeName, path) = + member __.SetFileToWatch(fullTypeName, path) = lock filesToWatch (fun () -> filesToWatch.[fullTypeName] <- path) - member _.GetFileToWath(fullTypeName) = + member __.GetFileToWath(fullTypeName) = lock filesToWatch (fun () -> match filesToWatch.TryGetValue(fullTypeName) with | true, path -> Some path | _ -> None) - member _.AddDisposeAction action = + member __.AddDisposeAction action = lock disposeActions (fun () -> disposeActions.Add action) - member _.InvalidateOneType typeName = + member __.InvalidateOneType typeName = (use _holder = logTime "InvalidateOneType" (sprintf "%s in %O [%d]" typeName x id) dispose (Some typeName) log (sprintf "Calling invalidate for %O [%d]" x id)) @@ -743,6 +743,5 @@ module internal ProviderHelpers = spec.GeneratedType) -[] -[] +[] do () diff --git a/src/CommonRuntime/Caching.fs b/src/CommonRuntime/Caching.fs index 37afa2a48..9bc9c85a3 100644 --- a/src/CommonRuntime/Caching.fs +++ b/src/CommonRuntime/Caching.fs @@ -1,5 +1,5 @@ -/// Implements caching using in-memory and local file system -module internal FSharp.Data.Runtime.Caching +/// Implements caching using in-memory and local file system +module FSharp.Data.Runtime.Caching open System open System.Collections.Concurrent @@ -34,7 +34,7 @@ let createInMemoryCache (expiration: TimeSpan) = } { new ICache<_, _> with - member _.Set(key, value) = + member __.Set(key, value) = dict.[key] <- (value, DateTime.UtcNow) invalidationFunction key |> Async.Start @@ -47,7 +47,7 @@ let createInMemoryCache (expiration: TimeSpan) = Some value | _ -> None - member _.Remove(key) = + member __.Remove(key) = match dict.TryRemove(key) with | true, _ -> log (sprintf "Explicitly removed from cache: %O" key) | _ -> () } @@ -89,7 +89,7 @@ let createInternetFileCache prefix expiration = let cache = { new ICache with - member _.Set(key, value) = + member __.Set(key, value) = let cacheFile = cacheFile key try @@ -101,7 +101,7 @@ let createInternetFileCache prefix expiration = e.Message ) - member _.TryRetrieve(key, ?extendCacheExpiration) = + member __.TryRetrieve(key, ?extendCacheExpiration) = if extendCacheExpiration = Some true then failwith "Not implemented" @@ -119,7 +119,7 @@ let createInternetFileCache prefix expiration = Debug.WriteLine("Caching: Failed to read file {0} with an exception: {1}", cacheFile, e.Message) None - member _.Remove(key) = + member __.Remove(key) = let cacheFile = cacheFile key try diff --git a/src/CommonRuntime/IO.fs b/src/CommonRuntime/IO.fs index b91d07420..e4c08cbc5 100644 --- a/src/CommonRuntime/IO.fs +++ b/src/CommonRuntime/IO.fs @@ -1,4 +1,4 @@ -/// Helper functions called from the generated code for working with files +/// Helper functions called from the generated code for working with files module FSharp.Data.Runtime.IO open System @@ -113,7 +113,7 @@ let internal logTime category (instance: string) = s.Start() { new IDisposable with - member _.Dispose() = + member __.Dispose() = s.Stop() Interlocked.Decrement &indentation |> ignore log (sprintf "Finished %s [%dms]" category s.ElapsedMilliseconds) @@ -126,11 +126,11 @@ let internal logTime category (instance: string) = let internal dummyDisposable = { new IDisposable with - member _.Dispose() = () } + member __.Dispose() = () } -let internal log (_: string) = () -let internal logWithStackTrace (_: string) = () -let internal logTime (_: string) (_: string) = dummyDisposable +let inline internal log (_: string) = () +let inline internal logWithStackTrace (_: string) = () +let inline internal logTime (_: string) (_: string) = dummyDisposable #endif @@ -165,9 +165,9 @@ type private FileWatcher(path) = watcher.Renamed.Add(checkForChanges "renamed") watcher.Deleted.Add(checkForChanges "deleted") - member _.Subscribe(name, action) = subscriptions.Add(name, action) + member __.Subscribe(name, action) = subscriptions.Add(name, action) - member _.Unsubscribe(name) = + member __.Unsubscribe(name) = if subscriptions.Remove(name) then log (sprintf "Unsubscribed %s from %s watcher" name path) @@ -205,7 +205,7 @@ let watchForChanges path (owner, onChange) = watcher) { new IDisposable with - member _.Dispose() = + member __.Dispose() = lock watchers (fun () -> if watcher.Unsubscribe(owner) then watchers.Remove(path) |> ignore) } diff --git a/src/CommonRuntime/NameUtils.fs b/src/CommonRuntime/NameUtils.fs index 1d494b771..5462007b0 100644 --- a/src/CommonRuntime/NameUtils.fs +++ b/src/CommonRuntime/NameUtils.fs @@ -2,6 +2,7 @@ module FSharp.Data.Runtime.NameUtils open System +open System.Globalization open System.Collections.Generic open FSharp.Data.Runtime diff --git a/src/CommonRuntime/StructuralInference.fs b/src/CommonRuntime/StructuralInference.fs index e2f2c3904..cbd3941e8 100644 --- a/src/CommonRuntime/StructuralInference.fs +++ b/src/CommonRuntime/StructuralInference.fs @@ -1,8 +1,6 @@ /// Implements type inference for unstructured documents like XML or JSON module FSharp.Data.Runtime.StructuralInference -#nowarn "44" - open System open System.Diagnostics open System.Collections.Generic @@ -28,7 +26,6 @@ type InferenceMode = | ValuesAndInlineSchemasOverrides = 4 /// This is the internal DU representing all the valid cases we support, mapped from the public InferenceMode. -[] type InferenceMode' = | NoInference /// Backward compatible mode. @@ -53,9 +50,8 @@ type InferenceMode' = | InferenceMode.ValuesAndInlineSchemasOverrides -> InferenceMode'.ValuesAndInlineSchemasOverrides | _ -> failwithf "Unexpected inference mode value %A" inferenceMode -[] -let asOption inp = - match inp with +let asOption = + function | true, x -> Some x | false, _ -> None @@ -97,14 +93,12 @@ let private primitiveTypes = @ numericTypes /// Checks whether a type supports unit of measure -[] let supportsUnitsOfMeasure typ = List.exists ((=) typ) numericTypes /// Returns a tag of a type - a tag represents a 'kind' of type /// (essentially it describes the different bottom types we have) -[] -let typeTag inferredType = - match inferredType with +let typeTag = + function | InferedType.Record (name = n) -> InferedTypeTag.Record n | InferedType.Collection _ -> InferedTypeTag.Collection | InferedType.Null @@ -220,7 +214,7 @@ let private (|SubtypePrimitives|_|) allowEmptyValues = /// The contract that should hold about the function is that given two types with the /// same `InferedTypeTag`, the result also has the same `InferedTypeTag`. /// -let rec internal subtypeInfered allowEmptyValues ot1 ot2 = +let rec subtypeInfered allowEmptyValues ot1 ot2 = match ot1, ot2 with // Subtype of matching types or one of equal types | SubtypePrimitives allowEmptyValues t -> InferedType.Primitive t @@ -361,7 +355,7 @@ and private unionCollectionTypes allowEmptyValues cases1 cases2 = tag, (m, t) | _ -> failwith "unionCollectionTypes: pairBy returned None, None") -and internal unionCollectionOrder order1 order2 = +and unionCollectionOrder order1 order2 = order1 @ (order2 |> List.filter (fun x -> not (List.exists ((=) x) order1))) @@ -369,7 +363,7 @@ and internal unionCollectionOrder order1 order2 = /// Get the union of record types (merge their properties) /// This matches the corresponding members and marks them as `Optional` /// if one may be missing. It also returns subtype of their types. -and internal unionRecordTypes allowEmptyValues t1 t2 = +and unionRecordTypes allowEmptyValues t1 t2 = List.pairBy (fun (p: InferedProperty) -> p.Name) t1 t2 |> List.map (fun (name, fst, snd) -> match fst, snd with @@ -387,7 +381,7 @@ and internal unionRecordTypes allowEmptyValues t1 t2 = /// Infer the type of the collection based on multiple sample types /// (group the types by tag, count their multiplicity) -let internal inferCollectionType allowEmptyValues types = +let inferCollectionType allowEmptyValues types = let groupedTypes = types |> Seq.groupBy typeTag @@ -398,13 +392,11 @@ let internal inferCollectionType allowEmptyValues types = InferedType.Collection(List.map fst groupedTypes, Map.ofList groupedTypes) -[] type IUnitsOfMeasureProvider = abstract SI: str: string -> System.Type abstract Product: measure1: System.Type * measure2: System.Type -> System.Type abstract Inverse: denominator: System.Type -> System.Type -[] let defaultUnitsOfMeasureProvider = { new IUnitsOfMeasureProvider with member x.SI(_) : Type = null @@ -416,7 +408,6 @@ let private uomTransformations = [ "³"; "^3" ], (fun (provider: IUnitsOfMeasureProvider) t -> provider.Product(provider.Product(t, t), t)) [ "^-1" ], (fun (provider: IUnitsOfMeasureProvider) t -> provider.Inverse(t)) ] -[] let parseUnitOfMeasure (provider: IUnitsOfMeasureProvider) (str: string) = let unit = uomTransformations @@ -441,7 +432,6 @@ let parseUnitOfMeasure (provider: IUnitsOfMeasureProvider) (str: string) = /// The inferred types may be set explicitly via inline schemas. /// This table specifies the mapping from (the names that users can use) to (the types used). -[] let nameToType = [ "int", (typeof, TypeWrapper.None) "int64", (typeof, TypeWrapper.None) @@ -475,7 +465,6 @@ let private validInlineSchema = /// This can be of the form: type|measure|type<measure> /// type{measure} is also supported to ease definition in xml values. /// -[] let parseTypeAndUnit unitsOfMeasureProvider (nameToType: IDictionary) str = let m = typeAndUnitRegex.Value.Match(str) @@ -527,7 +516,6 @@ module private Helpers = /// with the desiredUnit applied, /// or a value parsed from an inline schema. /// (For inline schemas, the unit parsed from the schema takes precedence over desiredUnit when present) -[] let inferPrimitiveType (unitsOfMeasureProvider: IUnitsOfMeasureProvider) (inferenceMode: InferenceMode') @@ -627,6 +615,5 @@ let inferPrimitiveType |> Option.defaultValue fallbackType /// Infers the type of a simple string value -[] let getInferedTypeFromString unitsOfMeasureProvider inferenceMode cultureInfo value unit = inferPrimitiveType unitsOfMeasureProvider inferenceMode cultureInfo value unit diff --git a/src/CommonRuntime/StructuralTypes.fs b/src/CommonRuntime/StructuralTypes.fs index 2da962f37..11483deb1 100644 --- a/src/CommonRuntime/StructuralTypes.fs +++ b/src/CommonRuntime/StructuralTypes.fs @@ -11,7 +11,6 @@ open FSharp.Data.Runtime /// /// Types that represent the result of static type inference. /// -[] type InferedProperty = { Name: string mutable Type: InferedType } @@ -20,7 +19,6 @@ type InferedProperty = /// For heterogeneous types (types that have multiple possible forms /// such as differently named XML nodes or records and arrays mixed together) /// this type represents the number of occurrences of individual forms -[] type InferedMultiplicity = | Single | OptionalSingle @@ -29,7 +27,6 @@ type InferedMultiplicity = /// For heterogeneous types, this represents the tag that defines the form /// (that is either primitive type, collection, named record etc.) [] -[] type InferedTypeTag = // Unknown type | Null @@ -64,7 +61,6 @@ type InferedTypeTag = /// we would lose information about multiplicity and so we would not be able /// to generate nicer types! [] -[] type InferedType = | Primitive of typ: Type * unit: option * optional: bool * shouldOverrideOnMerge: bool | Record of name: string option * fields: InferedProperty list * optional: bool @@ -145,7 +141,7 @@ type InferedType = // ------------------------------------------------------------------------------------------------ // Additional operations for working with the inferred representation -type internal InferedTypeTag with +type InferedTypeTag with member x.NiceName = match x with | Null -> failwith "Null nodes should be skipped" @@ -188,24 +184,20 @@ type internal InferedTypeTag with /// Dummy type to represent that only "0" was found. /// Will be generated as 'int', unless it's converted to Bit. -[] type Bit0 = Bit0 /// Dummy type to represent that only "1" was found /// Will be generated as 'int', unless it's converted to Bit -[] type Bit1 = Bit1 /// Dummy type to represent that only one of "0" and "1" were found /// Will be generated as a 'bool', unless it's converted to another numerical type -[] type Bit = Bit // ------------------------------------------------------------------------------------------------ /// Represents a transformation of a type [] -[] type TypeWrapper = /// No transformation will be made to the type | None @@ -219,7 +211,7 @@ type TypeWrapper = /// Represents type information about a primitive value (used mainly in the CSV provider) /// This type captures the type, unit of measure and handling of missing values (if we /// infer that the value may be missing, we can generate option or nullable) -type internal PrimitiveInferedValue = +type PrimitiveInferedValue = { InferedType: Type RuntimeType: Type UnitOfMeasure: Type option @@ -244,7 +236,7 @@ type internal PrimitiveInferedValue = /// Represents type information about a primitive property (used mainly in the CSV provider) /// This type captures the type, unit of measure and handling of missing values (if we /// infer that the value may be missing, we can generate option or nullable) -type internal PrimitiveInferedProperty = +type PrimitiveInferedProperty = { Name: string Value: PrimitiveInferedValue } static member Create(name, typ, (typWrapper: TypeWrapper), unit) = diff --git a/src/CommonRuntime/TextRuntime.fs b/src/CommonRuntime/TextRuntime.fs index 178cac246..3d06bc8ff 100644 --- a/src/CommonRuntime/TextRuntime.fs +++ b/src/CommonRuntime/TextRuntime.fs @@ -1,16 +1,16 @@ -namespace FSharp.Data.Runtime +namespace FSharp.Data.Runtime open System -open System.Collections.Generic open System.Globalization open FSharp.Data +open FSharp.Data.Runtime /// Static helper methods called from the generated code for working with text type TextRuntime = [] [] - static val mutable private cultureInfoCache: Dictionary + static val mutable private cultureInfoCache: Collections.Generic.Dictionary /// Returns CultureInfo matching the specified culture string /// (or InvariantCulture if the argument is null or empty) @@ -21,7 +21,7 @@ type TextRuntime = let mutable cache = TextRuntime.cultureInfoCache if cache = null then - cache <- Dictionary() + cache <- Collections.Generic.Dictionary() TextRuntime.cultureInfoCache <- cache match cache.TryGetValue cultureStr with diff --git a/src/Csv/CsvFile.fs b/src/Csv/CsvFile.fs index 3161d19e2..e8767fad1 100644 --- a/src/Csv/CsvFile.fs +++ b/src/Csv/CsvFile.fs @@ -17,21 +17,21 @@ open System.Text type CsvRow(parent: CsvFile, columns: string[]) = /// The columns of the row - member _.Columns = columns + member __.Columns = columns /// Gets a column by index - member _.GetColumn index = columns.[index] + member __.GetColumn index = columns.[index] /// Gets a column by name - member _.GetColumn columnName = + member __.GetColumn columnName = columns.[parent.GetColumnIndex columnName] /// Gets a column by index - member _.Item + member __.Item with get index = columns.[index] /// Gets a column by name - member _.Item + member __.Item with get columnName = columns.[parent.GetColumnIndex columnName] /// @@ -74,10 +74,10 @@ and CsvFile | None -> [] |> dict /// Returns the index of the column with the given name - member _.GetColumnIndex columnName = headerDic.[columnName] + member __.GetColumnIndex columnName = headerDic.[columnName] /// Returns the index of the column with the given name, or returns None if no column is found - member _.TryGetColumnIndex columnName = + member __.TryGetColumnIndex columnName = match headerDic.TryGetValue columnName with | true, index -> Some index | false, _ -> None diff --git a/src/Csv/CsvInference.fs b/src/Csv/CsvInference.fs index d228e03a0..afee7a315 100644 --- a/src/Csv/CsvInference.fs +++ b/src/Csv/CsvInference.fs @@ -3,6 +3,7 @@ module FSharp.Data.Runtime.CsvInference open System open System.IO +open System.Runtime.InteropServices open System.Text.RegularExpressions open FSharp.Data open FSharp.Data.Runtime @@ -414,7 +415,7 @@ type CsvFile with /// - Assumes all columns can have missing values /// - when set to true, inference will prefer to use the option type instead of nullable types, double.NaN or "" for missing values /// - optional function to resolve Units of Measure - member internal x.InferColumnTypes + member x.InferColumnTypes ( inferRows, missingValues, diff --git a/src/Csv/CsvRuntime.fs b/src/Csv/CsvRuntime.fs index e62249ee0..89a6899aa 100644 --- a/src/Csv/CsvRuntime.fs +++ b/src/Csv/CsvRuntime.fs @@ -256,22 +256,22 @@ type CsvFile<'RowType> ) = /// The rows with data - member _.Rows = rows + member __.Rows = rows /// The names of the columns - member _.Headers = headers + member __.Headers = headers /// The number of columns - member _.NumberOfColumns = numberOfColumns + member __.NumberOfColumns = numberOfColumns /// The character(s) used as column separator(s) - member _.Separators = separators + member __.Separators = separators /// The quotation mark use for surrounding values containing separator chars - member _.Quote = quote + member __.Quote = quote interface IDisposable with - member _.Dispose() = disposer.Dispose() + member __.Dispose() = disposer.Dispose() /// [] diff --git a/src/FSharp.Data.Csv.Core/FSharp.Data.Csv.Core.fsproj b/src/FSharp.Data.Csv.Core/FSharp.Data.Csv.Core.fsproj deleted file mode 100644 index 5e3bbbabf..000000000 --- a/src/FSharp.Data.Csv.Core/FSharp.Data.Csv.Core.fsproj +++ /dev/null @@ -1,29 +0,0 @@ - - - - Library - netstandard2.0 - $(OtherFlags) --warnon:1182 --nowarn:10001 --nowarn:44 - true - false - logo.png - - true - - - - - - - - - - - - - - - - - - diff --git a/src/FSharp.Data.Csv.Core/InternalsVisibleTo.fs b/src/FSharp.Data.Csv.Core/InternalsVisibleTo.fs deleted file mode 100644 index 04aff9db8..000000000 --- a/src/FSharp.Data.Csv.Core/InternalsVisibleTo.fs +++ /dev/null @@ -1,10 +0,0 @@ -namespace FSharp.Data - -open System.Runtime.CompilerServices - -[] -[] -[] -[] -[] -do () diff --git a/src/FSharp.Data.Csv.Core/paket.references b/src/FSharp.Data.Csv.Core/paket.references deleted file mode 100644 index c89b441a0..000000000 --- a/src/FSharp.Data.Csv.Core/paket.references +++ /dev/null @@ -1,2 +0,0 @@ -Microsoft.SourceLink.GitHub -FSharp.Core diff --git a/src/FSharp.Data.DesignTime/FSharp.Data.DesignTime.fsproj b/src/FSharp.Data.DesignTime/FSharp.Data.DesignTime.fsproj index aa6e5fd5a..26ca346ef 100755 --- a/src/FSharp.Data.DesignTime/FSharp.Data.DesignTime.fsproj +++ b/src/FSharp.Data.DesignTime/FSharp.Data.DesignTime.fsproj @@ -1,16 +1,27 @@ - false netstandard2.0 IS_DESIGNTIME;NO_GENERATIVE;$(DefineConstants) - $(OtherFlags) --warnon:1182 --nowarn:44 + --warnon:1182 false false + true + ..\keyfile.snk + false true + + + + + + + + + ProvidedTypes.fsi @@ -21,29 +32,41 @@ + + + + + + + + + + + + + + + + + + + + - - - - - - - - - + diff --git a/src/FSharp.Data.Html.Core/FSharp.Data.Html.Core.fsproj b/src/FSharp.Data.Html.Core/FSharp.Data.Html.Core.fsproj deleted file mode 100644 index 2814db6e4..000000000 --- a/src/FSharp.Data.Html.Core/FSharp.Data.Html.Core.fsproj +++ /dev/null @@ -1,35 +0,0 @@ - - - - Library - netstandard2.0 - $(OtherFlags) --warnon:1182 --nowarn:10001 --nowarn:44 - true - false - logo.png - - true - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/src/FSharp.Data.Html.Core/InternalsVisibleTo.fs b/src/FSharp.Data.Html.Core/InternalsVisibleTo.fs deleted file mode 100644 index 73b5e6d63..000000000 --- a/src/FSharp.Data.Html.Core/InternalsVisibleTo.fs +++ /dev/null @@ -1,9 +0,0 @@ -namespace FSharp.Data - -open System.Runtime.CompilerServices - -[] -[] -[] -[] -do () diff --git a/src/FSharp.Data.Html.Core/paket.references b/src/FSharp.Data.Html.Core/paket.references deleted file mode 100644 index c89b441a0..000000000 --- a/src/FSharp.Data.Html.Core/paket.references +++ /dev/null @@ -1,2 +0,0 @@ -Microsoft.SourceLink.GitHub -FSharp.Core diff --git a/src/FSharp.Data.Http/FSharp.Data.Http.fsproj b/src/FSharp.Data.Http/FSharp.Data.Http.fsproj deleted file mode 100644 index d03aa1b2b..000000000 --- a/src/FSharp.Data.Http/FSharp.Data.Http.fsproj +++ /dev/null @@ -1,31 +0,0 @@ - - - - Library - netstandard2.0 - $(OtherFlags) --warnon:1182 --nowarn:10001 --nowarn:44 - true - false - logo.png - - true - - - - - - - - - - - - - - - - - - - - diff --git a/src/FSharp.Data.Http/InternalsVisibleTo.fs b/src/FSharp.Data.Http/InternalsVisibleTo.fs deleted file mode 100644 index b514a2b6b..000000000 --- a/src/FSharp.Data.Http/InternalsVisibleTo.fs +++ /dev/null @@ -1,14 +0,0 @@ -namespace FSharp.Data - -open System.Runtime.CompilerServices - -[] -[] -[] -[] -[] -[] -[] -[] -[] -do () diff --git a/src/FSharp.Data.Http/paket.references b/src/FSharp.Data.Http/paket.references deleted file mode 100644 index c89b441a0..000000000 --- a/src/FSharp.Data.Http/paket.references +++ /dev/null @@ -1,2 +0,0 @@ -Microsoft.SourceLink.GitHub -FSharp.Core diff --git a/src/FSharp.Data.Json.Core/FSharp.Data.Json.Core.fsproj b/src/FSharp.Data.Json.Core/FSharp.Data.Json.Core.fsproj deleted file mode 100644 index 0b1911d8d..000000000 --- a/src/FSharp.Data.Json.Core/FSharp.Data.Json.Core.fsproj +++ /dev/null @@ -1,31 +0,0 @@ - - - - Library - netstandard2.0 - $(OtherFlags) --warnon:1182 --nowarn:10001 --nowarn:44 - true - false - logo.png - - true - - - - - - - - - - - - - - - - - - - - diff --git a/src/FSharp.Data.Json.Core/InternalsVisibleTo.fs b/src/FSharp.Data.Json.Core/InternalsVisibleTo.fs deleted file mode 100644 index a59db0192..000000000 --- a/src/FSharp.Data.Json.Core/InternalsVisibleTo.fs +++ /dev/null @@ -1,11 +0,0 @@ -namespace FSharp.Data - -open System.Runtime.CompilerServices - -[] -[] -[] -[] -[] -[] -do () diff --git a/src/FSharp.Data.Json.Core/paket.references b/src/FSharp.Data.Json.Core/paket.references deleted file mode 100644 index c89b441a0..000000000 --- a/src/FSharp.Data.Json.Core/paket.references +++ /dev/null @@ -1,2 +0,0 @@ -Microsoft.SourceLink.GitHub -FSharp.Core diff --git a/src/FSharp.Data.WorldBank.Core/FSharp.Data.WorldBank.Core.fsproj b/src/FSharp.Data.WorldBank.Core/FSharp.Data.WorldBank.Core.fsproj deleted file mode 100644 index ebd2c67dc..000000000 --- a/src/FSharp.Data.WorldBank.Core/FSharp.Data.WorldBank.Core.fsproj +++ /dev/null @@ -1,27 +0,0 @@ - - - - Library - netstandard2.0 - $(OtherFlags) --warnon:1182 --nowarn:10001 - true - false - logo.png - - true - - - - - - - - - - - - - - - - diff --git a/src/FSharp.Data.WorldBank.Core/InternalsVisibleTo.fs b/src/FSharp.Data.WorldBank.Core/InternalsVisibleTo.fs deleted file mode 100644 index 73b5e6d63..000000000 --- a/src/FSharp.Data.WorldBank.Core/InternalsVisibleTo.fs +++ /dev/null @@ -1,9 +0,0 @@ -namespace FSharp.Data - -open System.Runtime.CompilerServices - -[] -[] -[] -[] -do () diff --git a/src/FSharp.Data.WorldBank.Core/paket.references b/src/FSharp.Data.WorldBank.Core/paket.references deleted file mode 100644 index c89b441a0..000000000 --- a/src/FSharp.Data.WorldBank.Core/paket.references +++ /dev/null @@ -1,2 +0,0 @@ -Microsoft.SourceLink.GitHub -FSharp.Core diff --git a/src/FSharp.Data.Xml.Core/FSharp.Data.Xml.Core.fsproj b/src/FSharp.Data.Xml.Core/FSharp.Data.Xml.Core.fsproj deleted file mode 100644 index 68cacad38..000000000 --- a/src/FSharp.Data.Xml.Core/FSharp.Data.Xml.Core.fsproj +++ /dev/null @@ -1,30 +0,0 @@ - - - - Library - netstandard2.0 - $(OtherFlags) --warnon:1182 --nowarn:10001 --nowarn:44 - true - false - logo.png - - true - - - - - - - - - - - - - - - - - - - diff --git a/src/FSharp.Data.Xml.Core/InternalsVisibleTo.fs b/src/FSharp.Data.Xml.Core/InternalsVisibleTo.fs deleted file mode 100644 index 73b5e6d63..000000000 --- a/src/FSharp.Data.Xml.Core/InternalsVisibleTo.fs +++ /dev/null @@ -1,9 +0,0 @@ -namespace FSharp.Data - -open System.Runtime.CompilerServices - -[] -[] -[] -[] -do () diff --git a/src/FSharp.Data.Xml.Core/paket.references b/src/FSharp.Data.Xml.Core/paket.references deleted file mode 100644 index c89b441a0..000000000 --- a/src/FSharp.Data.Xml.Core/paket.references +++ /dev/null @@ -1,2 +0,0 @@ -Microsoft.SourceLink.GitHub -FSharp.Core diff --git a/src/FSharp.Data/FSharp.Data.fsproj b/src/FSharp.Data/FSharp.Data.fsproj index 797a49377..174700c5c 100755 --- a/src/FSharp.Data/FSharp.Data.fsproj +++ b/src/FSharp.Data/FSharp.Data.fsproj @@ -3,34 +3,58 @@ Library netstandard2.0 - $(OtherFlags) --warnon:1182 --nowarn:10001 + --warnon:1182 true false logo.png typeproviders typeproviders + false + true + ..\keyfile.snk true + + + + + + + + + + + + + + + + + + + + + + + + + + + - + true all - - - - - - - + diff --git a/src/Html/HtmlActivePatterns.fs b/src/Html/HtmlActivePatterns.fs index f370f66c3..c2b3cd45a 100644 --- a/src/Html/HtmlActivePatterns.fs +++ b/src/Html/HtmlActivePatterns.fs @@ -1,6 +1,5 @@ -namespace FSharp.Data +namespace FSharp.Data -[] module HtmlActivePatterns = let (|HtmlElement|HtmlText|HtmlComment|HtmlCData|) (node: HtmlNode) = match node with diff --git a/src/Html/HtmlCssSelectors.fs b/src/Html/HtmlCssSelectors.fs index dc40698ff..93ec31aaf 100644 --- a/src/Html/HtmlCssSelectors.fs +++ b/src/Html/HtmlCssSelectors.fs @@ -1,9 +1,7 @@ -namespace FSharp.Data +namespace FSharp.Data open System -#nowarn "26" - module internal HtmlCssSelectors = type SelectorToken = diff --git a/src/Html/HtmlInference.fs b/src/Html/HtmlInference.fs index 03db0ffcb..8882a4736 100644 --- a/src/Html/HtmlInference.fs +++ b/src/Html/HtmlInference.fs @@ -7,14 +7,14 @@ open FSharp.Data.Runtime open FSharp.Data.Runtime.StructuralInference open FSharp.Data.Runtime.StructuralTypes -type internal Parameters = +type Parameters = { MissingValues: string[] CultureInfo: CultureInfo UnitsOfMeasureProvider: IUnitsOfMeasureProvider PreferOptionals: bool InferenceMode: InferenceMode' } -let internal inferColumns parameters (headerNamesAndUnits: _[]) rows = +let inferColumns parameters (headerNamesAndUnits: _[]) rows = let inferRows = 0 let schema = Array.init headerNamesAndUnits.Length (fun _ -> None) @@ -32,7 +32,7 @@ let internal inferColumns parameters (headerNamesAndUnits: _[]) rows = parameters.PreferOptionals parameters.UnitsOfMeasureProvider -let internal inferHeaders parameters (rows: string[][]) = +let inferHeaders parameters (rows: string[][]) = if rows.Length <= 2 then false, None, None, None //Not enough info to infer anything, assume first row data else @@ -51,7 +51,7 @@ let internal inferHeaders parameters (rows: string[][]) = let headerNames, units = Array.unzip headerNamesAndUnits true, Some headerNames, Some units, Some dataRowsType -let internal inferListType parameters (values: string[]) = +let inferListType parameters (values: string[]) = if values.Length > 0 then let inferedtype value = diff --git a/src/Html/HtmlNode.fs b/src/Html/HtmlNode.fs deleted file mode 100644 index 115b63ee6..000000000 --- a/src/Html/HtmlNode.fs +++ /dev/null @@ -1,233 +0,0 @@ -#nowarn "10001" -namespace FSharp.Data - -open System -open System.ComponentModel -open System.Text - -// -------------------------------------------------------------------------------------- - -/// Represents an HTML attribute. The name is always normalized to lowercase -/// -/// Contains the primary types for the FSharp.Data package. -/// -/// -type HtmlAttribute = - - internal - | HtmlAttribute of name: string * value: string - - /// - /// Creates an html attribute - /// - /// The name of the attribute - /// The value of the attribute - static member New(name: string, value: string) = - HtmlAttribute(name.ToLowerInvariant(), value) - -[] -[] -/// Represents an HTML node. The names of elements are always normalized to lowercase -type HtmlNode = - - internal - | HtmlElement of name: string * attributes: HtmlAttribute list * elements: HtmlNode list - | HtmlText of content: string - | HtmlComment of content: string - | HtmlCData of content: string - - /// - /// Creates an html element - /// - /// The name of the element - static member NewElement(name: string) = - HtmlElement(name.ToLowerInvariant(), [], []) - - /// - /// Creates an html element - /// - /// The name of the element - /// The HtmlAttribute(s) of the element - static member NewElement(name: string, attrs: seq<_>) = - let attrs = attrs |> Seq.map HtmlAttribute.New |> Seq.toList - HtmlElement(name.ToLowerInvariant(), attrs, []) - - /// - /// Creates an html element - /// - /// The name of the element - /// The children elements of this element - static member NewElement(name: string, children: seq<_>) = - HtmlElement(name.ToLowerInvariant(), [], List.ofSeq children) - - - /// - /// Creates an html element - /// - /// The name of the element - /// The HtmlAttribute(s) of the element - /// The children elements of this element - static member NewElement(name: string, attrs: seq<_>, children: seq<_>) = - let attrs = attrs |> Seq.map HtmlAttribute.New |> Seq.toList - HtmlElement(name.ToLowerInvariant(), attrs, List.ofSeq children) - - /// - /// Creates a text content element - /// - /// The actual content - static member NewText content = HtmlText(content) - - /// - /// Creates a comment element - /// - /// The actual content - static member NewComment content = HtmlComment(content) - - /// - /// Creates a CData element - /// - /// The actual content - static member NewCData content = HtmlCData(content) - - override x.ToString() = - let isVoidElement = - let set = - [| "area" - "base" - "br" - "col" - "command" - "embed" - "hr" - "img" - "input" - "keygen" - "link" - "meta" - "param" - "source" - "track" - "wbr" |] - |> Set.ofArray - - fun name -> Set.contains name set - - let rec serialize (sb: StringBuilder) indentation canAddNewLine html = - let append (str: string) = sb.Append str |> ignore - - let appendEndTag name = - append "" - - let newLine plus = - sb.AppendLine() |> ignore - String(' ', indentation + plus) |> append - - match html with - | HtmlElement (name, attributes, elements) -> - let onlyText = - elements - |> List.forall (function - | HtmlText _ -> true - | _ -> false) - - if canAddNewLine && not onlyText then newLine 0 - append "<" - append name - - for HtmlAttribute (name, value) in attributes do - append " " - append name - append "=\"" - append value - append "\"" - - if isVoidElement name then - append " />" - elif elements.IsEmpty then - append ">" - appendEndTag name - else - append ">" - if not onlyText then newLine 2 - let mutable canAddNewLine = false - - for element in elements do - serialize sb (indentation + 2) canAddNewLine element - canAddNewLine <- true - - if not onlyText then newLine 0 - appendEndTag name - | HtmlText str -> append str - | HtmlComment str -> - append "" - | HtmlCData str -> - append "" - - let sb = StringBuilder() - serialize sb 0 false x |> ignore - sb.ToString() - - /// - [] - [] - member x._Print = - let str = x.ToString() - - if str.Length > 512 then - str.Substring(0, 509) + "..." - else - str - -[] -/// Represents an HTML document -type HtmlDocument = - internal - | HtmlDocument of docType: string * elements: HtmlNode list - - /// - /// Creates an html document - /// - /// The document type specifier string - /// The child elements of this document - static member New(docType, children: seq<_>) = - HtmlDocument(docType, List.ofSeq children) - - /// - /// Creates an html document - /// - /// The child elements of this document - static member New(children: seq<_>) = HtmlDocument("", List.ofSeq children) - - override x.ToString() = - match x with - | HtmlDocument (docType, elements) -> - (if String.IsNullOrEmpty docType then - "" - else - "" + Environment.NewLine) - + (elements - |> List.map (fun x -> x.ToString()) - |> String.Concat) - - /// - [] - [] - member x._Print = - let str = x.ToString() - - if str.Length > 512 then - str.Substring(0, 509) + "..." - else - str diff --git a/src/Html/HtmlOperations.fs b/src/Html/HtmlOperations.fs index df79bd85b..75b9e5a09 100644 --- a/src/Html/HtmlOperations.fs +++ b/src/Html/HtmlOperations.fs @@ -49,13 +49,13 @@ module HtmlNode = /// Gets the given nodes name let name n = match n with - | HtmlNode.HtmlElement (name = name) -> name + | HtmlElement (name = name) -> name | _ -> "" /// Gets all of the nodes immediately under this node let elements n = match n with - | HtmlNode.HtmlElement (elements = elements) -> elements + | HtmlElement (elements = elements) -> elements | _ -> [] /// @@ -193,7 +193,7 @@ module HtmlNode = /// Gets all of the attributes of this node let attributes n = match n with - | HtmlNode.HtmlElement (attributes = attributes) -> attributes + | HtmlElement (attributes = attributes) -> attributes | _ -> [] /// @@ -259,16 +259,16 @@ module HtmlNode = let private innerTextExcluding' recurse exclusions n = let rec innerText' n = match n with - | HtmlNode.HtmlElement (name, _, content) when List.forall ((<>) name) exclusions -> + | HtmlElement (name, _, content) when List.forall ((<>) name) exclusions -> seq { for e in content do match e with - | HtmlNode.HtmlText (text) -> yield text - | HtmlNode.HtmlComment (_) -> yield "" + | HtmlText (text) -> yield text + | HtmlComment (_) -> yield "" | elem -> if recurse then yield innerText' elem else yield "" } |> String.Concat - | HtmlNode.HtmlText (text) -> text + | HtmlText (text) -> text | _ -> "" innerText' n diff --git a/src/Html/HtmlParser.fs b/src/Html/HtmlParser.fs index 1c9f4519d..61990ba30 100644 --- a/src/Html/HtmlParser.fs +++ b/src/Html/HtmlParser.fs @@ -2,6 +2,7 @@ namespace FSharp.Data open System +open System.ComponentModel open System.IO open System.Text open System.Text.RegularExpressions @@ -10,6 +11,233 @@ open FSharp.Data.Runtime open System.Runtime.InteropServices open System.Collections.Generic +// -------------------------------------------------------------------------------------- + +/// Represents an HTML attribute. The name is always normalized to lowercase +/// +/// Contains the primary types for the FSharp.Data package. +/// +/// +type HtmlAttribute = + + internal + | HtmlAttribute of name: string * value: string + + /// + /// Creates an html attribute + /// + /// The name of the attribute + /// The value of the attribute + static member New(name: string, value: string) = + HtmlAttribute(name.ToLowerInvariant(), value) + +[] +/// Represents an HTML node. The names of elements are always normalized to lowercase +type HtmlNode = + + internal + | HtmlElement of name: string * attributes: HtmlAttribute list * elements: HtmlNode list + | HtmlText of content: string + | HtmlComment of content: string + | HtmlCData of content: string + + /// + /// Creates an html element + /// + /// The name of the element + static member NewElement(name: string) = + HtmlElement(name.ToLowerInvariant(), [], []) + + /// + /// Creates an html element + /// + /// The name of the element + /// The HtmlAttribute(s) of the element + static member NewElement(name: string, attrs: seq<_>) = + let attrs = attrs |> Seq.map HtmlAttribute.New |> Seq.toList + HtmlElement(name.ToLowerInvariant(), attrs, []) + + /// + /// Creates an html element + /// + /// The name of the element + /// The children elements of this element + static member NewElement(name: string, children: seq<_>) = + HtmlElement(name.ToLowerInvariant(), [], List.ofSeq children) + + + /// + /// Creates an html element + /// + /// The name of the element + /// The HtmlAttribute(s) of the element + /// The children elements of this element + static member NewElement(name: string, attrs: seq<_>, children: seq<_>) = + let attrs = attrs |> Seq.map HtmlAttribute.New |> Seq.toList + HtmlElement(name.ToLowerInvariant(), attrs, List.ofSeq children) + + /// + /// Creates a text content element + /// + /// The actual content + static member NewText content = HtmlText(content) + + /// + /// Creates a comment element + /// + /// The actual content + static member NewComment content = HtmlComment(content) + + /// + /// Creates a CData element + /// + /// The actual content + static member NewCData content = HtmlCData(content) + + override x.ToString() = + let isVoidElement = + let set = + [| "area" + "base" + "br" + "col" + "command" + "embed" + "hr" + "img" + "input" + "keygen" + "link" + "meta" + "param" + "source" + "track" + "wbr" |] + |> Set.ofArray + + fun name -> Set.contains name set + + let rec serialize (sb: StringBuilder) indentation canAddNewLine html = + let append (str: string) = sb.Append str |> ignore + + let appendEndTag name = + append "" + + let newLine plus = + sb.AppendLine() |> ignore + String(' ', indentation + plus) |> append + + match html with + | HtmlElement (name, attributes, elements) -> + let onlyText = + elements + |> List.forall (function + | HtmlText _ -> true + | _ -> false) + + if canAddNewLine && not onlyText then newLine 0 + append "<" + append name + + for HtmlAttribute (name, value) in attributes do + append " " + append name + append "=\"" + append value + append "\"" + + if isVoidElement name then + append " />" + elif elements.IsEmpty then + append ">" + appendEndTag name + else + append ">" + if not onlyText then newLine 2 + let mutable canAddNewLine = false + + for element in elements do + serialize sb (indentation + 2) canAddNewLine element + canAddNewLine <- true + + if not onlyText then newLine 0 + appendEndTag name + | HtmlText str -> append str + | HtmlComment str -> + append "" + | HtmlCData str -> + append "" + + let sb = StringBuilder() + serialize sb 0 false x |> ignore + sb.ToString() + + /// + [] + [] + member x._Print = + let str = x.ToString() + + if str.Length > 512 then + str.Substring(0, 509) + "..." + else + str + +[] +/// Represents an HTML document +type HtmlDocument = + internal + | HtmlDocument of docType: string * elements: HtmlNode list + + /// + /// Creates an html document + /// + /// The document type specifier string + /// The child elements of this document + static member New(docType, children: seq<_>) = + HtmlDocument(docType, List.ofSeq children) + + /// + /// Creates an html document + /// + /// The child elements of this document + static member New(children: seq<_>) = HtmlDocument("", List.ofSeq children) + + override x.ToString() = + match x with + | HtmlDocument (docType, elements) -> + (if String.IsNullOrEmpty docType then + "" + else + "" + Environment.NewLine) + + (elements + |> List.map (fun x -> x.ToString()) + |> String.Concat) + + /// + [] + [] + member x._Print = + let str = x.ToString() + + if str.Length > 512 then + str.Substring(0, 509) + "..." + else + str + + // -------------------------------------------------------------------------------------- module private TextParser = @@ -93,20 +321,20 @@ module internal HtmlParser = | CDATAMode -> "cdata" type HtmlState = - { mutable Attributes: (CharList * CharList) list - mutable CurrentTag: CharList - mutable Content: CharList - mutable HasFormattedParent: bool - mutable InsertionMode: InsertionMode - mutable Tokens: HtmlToken list + { Attributes: (CharList * CharList) list ref + CurrentTag: CharList ref + Content: CharList ref + HasFormattedParent: bool ref + InsertionMode: InsertionMode ref + Tokens: HtmlToken list ref Reader: TextReader } static member Create(reader: TextReader) = - { Attributes = [] - CurrentTag = CharList.Empty - Content = CharList.Empty - HasFormattedParent = false - InsertionMode = DefaultMode - Tokens = [] + { Attributes = ref [] + CurrentTag = ref CharList.Empty + Content = ref CharList.Empty + HasFormattedParent = ref false + InsertionMode = ref DefaultMode + Tokens = ref [] Reader = reader } member x.Pop() = x.Reader.Read() |> ignore @@ -116,28 +344,30 @@ module internal HtmlParser = [| 0 .. (count - 1) |] |> Array.map (fun _ -> x.Reader.ReadChar()) - member x.Contents = x.Content.ToString() - member x.ContentLength = x.Content.Length + member x.Contents = (!x.Content).ToString() + member x.ContentLength = (!x.Content).Length member x.NewAttribute() = - x.Attributes <- (CharList.Empty, CharList.Empty) :: x.Attributes + x.Attributes + := (CharList.Empty, CharList.Empty) + :: (!x.Attributes) member x.ConsAttrName() = - match x.Attributes with + match !x.Attributes with | [] -> x.NewAttribute() x.ConsAttrName() | (h, _) :: _ -> h.Cons(Char.ToLowerInvariant(x.Reader.ReadChar())) - member x.CurrentTagName() = x.CurrentTag.ToString().Trim() + member x.CurrentTagName() = (!x.CurrentTag).ToString().Trim() member x.CurrentAttrName() = - match x.Attributes with + match !x.Attributes with | [] -> String.Empty | (h, _) :: _ -> h.ToString() member x.ConsAttrValue(c) = - match x.Attributes with + match !x.Attributes with | [] -> x.NewAttribute() x.ConsAttrValue(c) @@ -146,7 +376,7 @@ module internal HtmlParser = member x.ConsAttrValue() = x.ConsAttrValue(x.Reader.ReadChar()) member x.GetAttributes() = - x.Attributes + !x.Attributes |> List.choose (fun (key, value) -> if key.Length > 0 then Some @@ -156,12 +386,12 @@ module internal HtmlParser = |> List.rev member x.EmitSelfClosingTag() = - let name = x.CurrentTag.ToString().Trim() + let name = (!x.CurrentTag).ToString().Trim() let result = Tag(true, name, x.GetAttributes()) - x.CurrentTag <- CharList.Empty - x.InsertionMode <- DefaultMode - x.Attributes <- [] - x.Tokens <- result :: x.Tokens + x.CurrentTag := CharList.Empty + x.InsertionMode := DefaultMode + x.Attributes := [] + x.Tokens := result :: !x.Tokens member x.IsFormattedTag = match x.CurrentTagName().ToLower() with @@ -175,7 +405,7 @@ module internal HtmlParser = | _ -> false member x.EmitTag(isEnd) = - let name = x.CurrentTag.ToString().Trim() + let name = (!x.CurrentTag).ToString().Trim() let result = if isEnd then @@ -190,37 +420,38 @@ module internal HtmlParser = // pre is the only default formatted tag, nested pres are not // allowed in the spec. if x.IsFormattedTag then - x.HasFormattedParent <- not isEnd + x.HasFormattedParent := not isEnd else - x.HasFormattedParent <- x.HasFormattedParent || x.IsFormattedTag + x.HasFormattedParent + := !x.HasFormattedParent || x.IsFormattedTag - x.InsertionMode <- - if x.IsScriptTag && (not isEnd) then - ScriptMode - else - DefaultMode + x.InsertionMode + := if x.IsScriptTag && (not isEnd) then + ScriptMode + else + DefaultMode - x.CurrentTag <- CharList.Empty - x.Attributes <- [] - x.Tokens <- result :: x.Tokens + x.CurrentTag := CharList.Empty + x.Attributes := [] + x.Tokens := result :: !x.Tokens member x.EmitToAttributeValue() = - assert (x.InsertionMode = InsertionMode.CharRefMode) - let content = x.Content.ToString() |> HtmlCharRefs.substitute + assert (!x.InsertionMode = InsertionMode.CharRefMode) + let content = (!x.Content).ToString() |> HtmlCharRefs.substitute for c in content.ToCharArray() do x.ConsAttrValue c - x.Content <- CharList.Empty - x.InsertionMode <- DefaultMode + x.Content := CharList.Empty + x.InsertionMode := DefaultMode member x.Emit() : unit = let result = - let content = x.Content.ToString() + let content = (!x.Content).ToString() - match x.InsertionMode with + match !x.InsertionMode with | DefaultMode -> - if x.HasFormattedParent then + if !x.HasFormattedParent then Text content else let normalizedContent = wsRegex.Value.Replace(content, " ") @@ -235,24 +466,24 @@ module internal HtmlParser = | DocTypeMode -> DocType content | CDATAMode -> CData(content.Replace("", "")) - x.Content <- CharList.Empty - x.InsertionMode <- DefaultMode + x.Content := CharList.Empty + x.InsertionMode := DefaultMode match result with | Text t when String.IsNullOrEmpty(t) -> () - | _ -> x.Tokens <- result :: x.Tokens + | _ -> x.Tokens := result :: !x.Tokens - member x.Cons() = x.Content.Cons(x.Reader.ReadChar()) - member x.Cons(char) = x.Content.Cons(char) - member x.Cons(char) = Array.iter (x.Content.Cons) char + member x.Cons() = (!x.Content).Cons(x.Reader.ReadChar()) + member x.Cons(char) = (!x.Content).Cons(char) + member x.Cons(char) = Array.iter ((!x.Content).Cons) char member x.Cons(char: string) = x.Cons(char.ToCharArray()) member x.ConsTag() = match x.Reader.ReadChar() with | TextParser.Whitespace _ -> () - | a -> x.CurrentTag.Cons(Char.ToLowerInvariant a) + | a -> (!x.CurrentTag).Cons(Char.ToLowerInvariant a) - member x.ClearContent() = x.Content.Clear() + member x.ClearContent() = (!x.Content).Clear() // Tokenises a stream into a sequence of HTML tokens. let private tokenise reader = @@ -266,15 +497,15 @@ module internal HtmlParser = else state.Pop() tagOpen state - | TextParser.EndOfFile _ -> state.Tokens <- EOF :: state.Tokens + | TextParser.EndOfFile _ -> state.Tokens := EOF :: !state.Tokens | '&' -> if state.ContentLength > 0 then state.Emit() else - state.InsertionMode <- CharRefMode + state.InsertionMode := CharRefMode charRef state | _ -> - match state.InsertionMode with + match !state.InsertionMode with | DefaultMode -> state.Cons() data state @@ -538,7 +769,7 @@ module internal HtmlParser = | '>' -> state.Cons([| '<'; '/' |]) state.Cons(state.CurrentTagName()) - state.CurrentTag.Clear() + (!state.CurrentTag).Clear() script state | TextParser.Letter _ -> state.ConsTag() @@ -595,7 +826,7 @@ module internal HtmlParser = | _ -> state.Cons([| '<'; '/' |]) state.Cons(state.CurrentTagName()) - state.CurrentTag.Clear() + (!state.CurrentTag).Clear() script state and charRef state = @@ -632,7 +863,7 @@ module internal HtmlParser = and bogusComment state = let rec bogusComment' (state: HtmlState) = let exitBogusComment state = - state.InsertionMode <- CommentMode + state.InsertionMode := CommentMode state.Emit() match state.Peek() with @@ -664,10 +895,10 @@ module internal HtmlParser = cData (i + 1) state | '>' when i = 2 -> state.Cons() - state.InsertionMode <- CDATAMode + state.InsertionMode := CDATAMode state.Emit() | TextParser.EndOfFile _ -> - state.InsertionMode <- CDATAMode + state.InsertionMode := CDATAMode state.Emit() | _ -> state.Cons() @@ -677,7 +908,7 @@ module internal HtmlParser = match state.Peek() with | '>' -> state.Pop() - state.InsertionMode <- DocTypeMode + state.InsertionMode := DocTypeMode state.Emit() | _ -> state.Cons() @@ -689,7 +920,7 @@ module internal HtmlParser = state.Pop() commentEndDash state | TextParser.EndOfFile _ -> - state.InsertionMode <- CommentMode + state.InsertionMode := CommentMode state.Emit() | _ -> state.Cons() @@ -701,7 +932,7 @@ module internal HtmlParser = state.Pop() commentEndState state | TextParser.EndOfFile _ -> - state.InsertionMode <- CommentMode + state.InsertionMode := CommentMode state.Emit() | _ -> state.Cons() @@ -711,10 +942,10 @@ module internal HtmlParser = match state.Peek() with | '>' -> state.Pop() - state.InsertionMode <- CommentMode + state.InsertionMode := CommentMode state.Emit() | TextParser.EndOfFile _ -> - state.InsertionMode <- CommentMode + state.InsertionMode := CommentMode state.Emit() | _ -> state.Cons() @@ -840,7 +1071,7 @@ module internal HtmlParser = state.EmitTag(false) | '&' -> assert (state.ContentLength = 0) - state.InsertionMode <- InsertionMode.CharRefMode + state.InsertionMode := InsertionMode.CharRefMode attributeValueUnquotedCharRef [ '/'; '>' ] state | _ -> state.ConsAttrValue() @@ -862,7 +1093,7 @@ module internal HtmlParser = afterAttributeValueQuoted state | '&' -> assert (state.ContentLength = 0) - state.InsertionMode <- InsertionMode.CharRefMode + state.InsertionMode := InsertionMode.CharRefMode attributeValueQuotedCharRef quote state | _ -> state.ConsAttrValue() @@ -916,13 +1147,13 @@ module internal HtmlParser = state.NewAttribute() attributeName state - let mutable next = state.Reader.Peek() + let next = ref (state.Reader.Peek()) - while next <> -1 do + while !next <> -1 do data state - next <- state.Reader.Peek() + next := state.Reader.Peek() - state.Tokens |> List.rev + !state.Tokens |> List.rev let private parse reader = let canNotHaveChildren (name: string) = @@ -1013,19 +1244,19 @@ module internal HtmlParser = (dt, tokens, content) else let _, elements, expectedTagEnd, parentTagName, name, attributes = callstack.Pop() - let e = HtmlNode.HtmlElement(name, attributes, content) + let e = HtmlElement(name, attributes, content) parse' dt (e :: elements) expectedTagEnd parentTagName tokens match tokens with | DocType dt :: rest -> parse' (dt.Trim()) elements expectedTagEnd parentTagName rest | Tag (_, "br", []) :: rest -> - let t = HtmlNode.HtmlText Environment.NewLine + let t = HtmlText Environment.NewLine parse' docType (t :: elements) expectedTagEnd parentTagName rest | Tag (true, name, attributes) :: rest -> - let e = HtmlNode.HtmlElement(name, attributes, []) + let e = HtmlElement(name, attributes, []) parse' docType (e :: elements) expectedTagEnd parentTagName rest | Tag (false, name, attributes) :: rest when canNotHaveChildren name -> - let e = HtmlNode.HtmlElement(name, attributes, []) + let e = HtmlElement(name, attributes, []) parse' docType (e :: elements) expectedTagEnd parentTagName rest | Tag (_, name, _) :: _ when isImplicitlyClosedByStartTag expectedTagEnd name -> // insert missing or when starting new row/cell/header @@ -1060,20 +1291,20 @@ module internal HtmlParser = // ignore this token parse' docType elements expectedTagEnd parentTagName rest else - let t = HtmlNode.HtmlText(a + b) + let t = HtmlText(a + b) parse' docType (t :: elements) expectedTagEnd parentTagName rest | Text cont :: rest -> if cont = "" then // ignore this token parse' docType elements expectedTagEnd parentTagName rest else - let t = HtmlNode.HtmlText cont + let t = HtmlText cont parse' docType (t :: elements) expectedTagEnd parentTagName rest | Comment cont :: rest -> - let c = HtmlNode.HtmlComment cont + let c = HtmlComment cont parse' docType (c :: elements) expectedTagEnd parentTagName rest | CData cont :: rest -> - let c = HtmlNode.HtmlCData cont + let c = HtmlCData cont parse' docType (c :: elements) expectedTagEnd parentTagName rest | EOF :: _ -> recursiveReturn (docType, [], List.rev elements) | [] -> recursiveReturn (docType, [], List.rev elements) @@ -1095,46 +1326,44 @@ module internal HtmlParser = /// All br tags will be replaced by newlines let parseFragment reader = parse reader |> snd -[] -module HtmlAutoOpens = - // -------------------------------------------------------------------------------------- - - type HtmlDocument with - - /// Parses the specified HTML string - static member Parse(text) = - use reader = new StringReader(text) - HtmlParser.parseDocument reader - - /// Loads HTML from the specified stream - static member Load(stream: Stream) = - use reader = new StreamReader(stream) - HtmlParser.parseDocument reader - - /// Loads HTML from the specified reader - static member Load(reader: TextReader) = HtmlParser.parseDocument reader - - /// Loads HTML from the specified uri asynchronously - static member AsyncLoad(uri: string, [] ?encoding) = - async { - let encoding = defaultArg encoding Encoding.UTF8 - let! reader = IO.asyncReadTextAtRuntime false "" "" "HTML" encoding.WebName uri - return HtmlParser.parseDocument reader - } - - /// Loads HTML from the specified uri - static member Load(uri: string, [] ?encoding) = - HtmlDocument.AsyncLoad(uri, ?encoding = encoding) - |> Async.RunSynchronously - - type HtmlNode with - - /// Parses the specified HTML string to a list of HTML nodes - static member Parse(text) = - use reader = new StringReader(text) - HtmlParser.parseFragment reader - - /// Parses the specified HTML string to a list of HTML nodes - static member ParseRooted(rootName, text) = - use reader = new StringReader(text) - HtmlNode.HtmlElement(rootName, [], HtmlParser.parseFragment reader) +// -------------------------------------------------------------------------------------- + +type HtmlDocument with + + /// Parses the specified HTML string + static member Parse(text) = + use reader = new StringReader(text) + HtmlParser.parseDocument reader + + /// Loads HTML from the specified stream + static member Load(stream: Stream) = + use reader = new StreamReader(stream) + HtmlParser.parseDocument reader + + /// Loads HTML from the specified reader + static member Load(reader: TextReader) = HtmlParser.parseDocument reader + + /// Loads HTML from the specified uri asynchronously + static member AsyncLoad(uri: string, [] ?encoding) = + async { + let encoding = defaultArg encoding Encoding.UTF8 + let! reader = IO.asyncReadTextAtRuntime false "" "" "HTML" encoding.WebName uri + return HtmlParser.parseDocument reader + } + + /// Loads HTML from the specified uri + static member Load(uri: string, [] ?encoding) = + HtmlDocument.AsyncLoad(uri, ?encoding = encoding) + |> Async.RunSynchronously + +type HtmlNode with + + /// Parses the specified HTML string to a list of HTML nodes + static member Parse(text) = + use reader = new StringReader(text) + HtmlParser.parseFragment reader + + /// Parses the specified HTML string to a list of HTML nodes + static member ParseRooted(rootName, text) = + use reader = new StringReader(text) + HtmlElement(rootName, [], HtmlParser.parseFragment reader) diff --git a/src/Html/HtmlRuntime.fs b/src/Html/HtmlRuntime.fs index e8394059c..8ddaa8290 100644 --- a/src/Html/HtmlRuntime.fs +++ b/src/Html/HtmlRuntime.fs @@ -1,8 +1,10 @@ -namespace FSharp.Data.Runtime +namespace FSharp.Data.Runtime open System open System.Globalization +open System.IO open System.Text +open System.Text.RegularExpressions open FSharp.Data open FSharp.Data.HtmlExtensions open FSharp.Data.Runtime @@ -30,34 +32,17 @@ type HtmlTableCell = | Cell (_, d) -> d /// Representation of an HTML table cell -type HtmlTable - internal - ( - name: string, - headerNamesAndUnits: (string * Type option)[] option, // always set at designtime, never at runtime - inferedProperties: PrimitiveInferedProperty list option, // sometimes set at designtime, never at runtime - hasHeaders: bool option, // always set at designtime, never at runtime - rows: string[][], - html: HtmlNode - ) = - member _.Name = name - - // always set at designtime, never at runtime - member internal _.HeaderNamesAndUnits = headerNamesAndUnits - - // sometimes set at designtime, never at runtime - member internal _.InferedProperties = inferedProperties - - member _.HasHeaders = hasHeaders // always set at designtime, never at runtime - - member _.Rows = rows - - member _.Html = html - - override _.ToString() = +type HtmlTable = + { Name: string + HeaderNamesAndUnits: (string * Type option)[] option // always set at designtime, never at runtime + InferedProperties: PrimitiveInferedProperty list option // sometimes set at designtime, never at runtime + HasHeaders: bool option // always set at designtime, never at runtime + Rows: string[][] + Html: HtmlNode } + override x.ToString() = let sb = StringBuilder() - sb.AppendLine name |> ignore - let data = array2D rows + sb.AppendLine x.Name |> ignore + let data = array2D x.Rows let rows = data.GetLength(0) let columns = data.GetLength(1) let widths = Array.zeroCreate columns @@ -106,11 +91,10 @@ type HtmlDefinitionList = sb.ToString() /// Representation of an HTML table, list, or definition list -type HtmlObjectDescription = +type HtmlObject = | Table of HtmlTable | List of HtmlList | DefinitionList of HtmlDefinitionList - member x.Name = match x with | Table t -> t.Name @@ -297,13 +281,13 @@ module HtmlRuntime = | HtmlElement ("th", _, contents) -> Cell(true, getContents contents) | _ -> Empty - let mutable col_i = colindex + let col_i = ref colindex - while col_i < res.[rowindex].Length - && res.[rowindex].[col_i] <> Empty do - col_i <- col_i + 1 + while !col_i < res.[rowindex].Length + && res.[rowindex].[!col_i] <> Empty do + incr (col_i) - for j in [ col_i .. (col_i + colSpan cell - 1) ] do + for j in [ !col_i .. (!col_i + colSpan cell - 1) ] do for i in [ rowindex .. (rowindex + rowSpan cell - 1) ] do if i < rows.Length && j < numberOfColumns then res.[i].[j] <- data @@ -357,7 +341,12 @@ module HtmlRuntime = let rows = res |> Array.map (Array.map (fun x -> x.Data)) - HtmlTable(name, headerNamesAndUnits, inferedProperties, hasHeaders, rows, table) + { Name = name + HeaderNamesAndUnits = headerNamesAndUnits + InferedProperties = inferedProperties + HasHeaders = hasHeaders + Rows = rows + Html = table } |> Some let private parseList makeUnique index (list: HtmlNode, parents: HtmlNode list) = @@ -418,7 +407,7 @@ module HtmlRuntime = Html = definitionList } |> Some - let internal getTables inferenceParameters includeLayoutTables (doc: HtmlDocument) = + let getTables inferenceParameters includeLayoutTables (doc: HtmlDocument) = let tableElements = doc.DescendantsWithPath "table" |> List.ofSeq let tableElements = @@ -450,7 +439,7 @@ module HtmlRuntime = |> List.mapi (parseDefinitionList (NameUtils.uniqueGenerator id)) |> List.choose id - let internal getHtmlObjects inferenceParameters includeLayoutTables (doc: HtmlDocument) = + let getHtmlObjects inferenceParameters includeLayoutTables (doc: HtmlDocument) = Seq.concat [ doc |> getTables inferenceParameters includeLayoutTables @@ -473,7 +462,7 @@ open FSharp.Data.Runtime /// Underlying representation of the root types generated by HtmlProvider type HtmlDocument internal (doc, tables, lists, definitionLists) = - member _.Html = doc + member __.Html = doc /// [] @@ -510,7 +499,7 @@ type HtmlDocument internal (doc, tables, lists, definitionLists) = 10001, IsHidden = true, IsError = false)>] - member _.GetTable(id: string) = tables |> Map.find id + member __.GetTable(id: string) = tables |> Map.find id /// [] @@ -518,7 +507,7 @@ type HtmlDocument internal (doc, tables, lists, definitionLists) = 10001, IsHidden = true, IsError = false)>] - member _.GetList(id: string) = lists |> Map.find id + member __.GetList(id: string) = lists |> Map.find id /// [] @@ -526,15 +515,15 @@ type HtmlDocument internal (doc, tables, lists, definitionLists) = 10001, IsHidden = true, IsError = false)>] - member _.GetDefinitionList(id: string) = definitionLists |> Map.find id + member __.GetDefinitionList(id: string) = definitionLists |> Map.find id /// Underlying representation of table types generated by HtmlProvider type HtmlTable<'RowType> internal (name: string, headers: string[] option, values: 'RowType[], html: HtmlNode) = - member _.Name = name - member _.Headers = headers - member _.Rows = values - member _.Html = html + member __.Name = name + member __.Headers = headers + member __.Rows = values + member __.Html = html /// [] @@ -556,9 +545,9 @@ type HtmlTable<'RowType> internal (name: string, headers: string[] option, value /// Underlying representation of list types generated by HtmlProvider type HtmlList<'ItemType> internal (name: string, values: 'ItemType[], html) = - member _.Name = name - member _.Values = values - member _.Html = html + member __.Name = name + member __.Values = values + member __.Html = html [] [ and converts it to /// an expression of other type - the type is specified by `field` -let internal convertJsonValue - missingValuesStr - cultureStr - canPassAllConversionCallingTypes - (field: PrimitiveInferedValue) - = +let convertJsonValue missingValuesStr cultureStr canPassAllConversionCallingTypes (field: PrimitiveInferedValue) = let returnType = diff --git a/src/Json/JsonDocument.fs b/src/Json/JsonDocument.fs deleted file mode 100644 index 61447c204..000000000 --- a/src/Json/JsonDocument.fs +++ /dev/null @@ -1,97 +0,0 @@ -// -------------------------------------------------------------------------------------- -// JSON type provider - methods that are called from the generated erased code -// -------------------------------------------------------------------------------------- -namespace FSharp.Data.Runtime.BaseTypes - -open System.ComponentModel -open System.IO -open FSharp.Data - -#nowarn "10001" - -/// -type IJsonDocument = - abstract JsonValue: JsonValue - - [] - [] - abstract Path: unit -> string - - [] - [] - abstract CreateNew: value: JsonValue * pathIncrement: string -> IJsonDocument - -/// Underlying representation of types generated by JsonProvider -/// -/// Contains the runtime base types used by generated row types for FSharp.Data type providers. -/// -[] -type JsonDocument = - - private - { - /// - Json: JsonValue - /// - Path: string - } - - interface IJsonDocument with - member x.JsonValue = x.Json - member x.Path() = x.Path - - member x.CreateNew(value, pathIncrement) = - JsonDocument.Create(value, x.Path + pathIncrement) - - /// The underlying JsonValue - member x.JsonValue = x.Json - - /// - [] - [] - override x.ToString() = x.JsonValue.ToString() - - /// - [] - [] - static member Create(value, path) = - { Json = value; Path = path } :> IJsonDocument - - /// - [] - [] - static member Create(reader: TextReader) = - use reader = reader - let text = reader.ReadToEnd() - let value = JsonValue.Parse(text) - JsonDocument.Create(value, "") - - /// - [] - [] - static member CreateList(reader: TextReader) = - use reader = reader - let text = reader.ReadToEnd() - - match JsonValue.ParseMultiple(text) |> Seq.toArray with - | [| JsonValue.Array array |] -> array - | array -> array - |> Array.mapi (fun i value -> JsonDocument.Create(value, "[" + (string i) + "]")) diff --git a/src/Json/JsonInference.fs b/src/Json/JsonInference.fs index 1699e66b9..f42e41da9 100644 --- a/src/Json/JsonInference.fs +++ b/src/Json/JsonInference.fs @@ -14,7 +14,7 @@ open FSharp.Data.Runtime.StructuralInference /// functionality is handled in `StructureInference` (most notably, by /// `inferCollectionType` and various functions to find common subtype), so /// here we just need to infer types of primitive JSON values. -let rec internal inferType unitsOfMeasureProvider inferenceMode cultureInfo parentName json = +let rec inferType unitsOfMeasureProvider inferenceMode cultureInfo parentName json = let inline inRangeDecimal lo hi (v: decimal) : bool = (v >= decimal lo) && (v <= decimal hi) let inline inRangeFloat lo hi (v: float) : bool = (v >= float lo) && (v <= float hi) let inline isIntegerDecimal (v: decimal) : bool = Math.Round v = v diff --git a/src/Json/JsonRuntime.fs b/src/Json/JsonRuntime.fs index 385dca19a..0c121bd9a 100644 --- a/src/Json/JsonRuntime.fs +++ b/src/Json/JsonRuntime.fs @@ -1,12 +1,110 @@ // -------------------------------------------------------------------------------------- // JSON type provider - methods that are called from the generated erased code // -------------------------------------------------------------------------------------- +namespace FSharp.Data.Runtime.BaseTypes + +open System.ComponentModel +open System.IO +open FSharp.Data +open FSharp.Data.Runtime + +#nowarn "10001" + +/// +type IJsonDocument = + abstract JsonValue: JsonValue + + [] + [] + abstract Path: unit -> string + + [] + [] + abstract CreateNew: value: JsonValue * pathIncrement: string -> IJsonDocument + +/// Underlying representation of types generated by JsonProvider +/// +/// Contains the runtime base types used by generated row types for FSharp.Data type providers. +/// +[] +type JsonDocument = + + private + { + /// + Json: JsonValue + /// + Path: string + } + + interface IJsonDocument with + member x.JsonValue = x.Json + member x.Path() = x.Path + + member x.CreateNew(value, pathIncrement) = + JsonDocument.Create(value, x.Path + pathIncrement) + + /// The underlying JsonValue + member x.JsonValue = x.Json + + /// + [] + [] + override x.ToString() = x.JsonValue.ToString() + + /// + [] + [] + static member Create(value, path) = + { Json = value; Path = path } :> IJsonDocument + + /// + [] + [] + static member Create(reader: TextReader) = + use reader = reader + let text = reader.ReadToEnd() + let value = JsonValue.Parse(text) + JsonDocument.Create(value, "") + + /// + [] + [] + static member CreateList(reader: TextReader) = + use reader = reader + let text = reader.ReadToEnd() + + match JsonValue.ParseMultiple(text) |> Seq.toArray with + | [| JsonValue.Array array |] -> array + | array -> array + |> Array.mapi (fun i value -> JsonDocument.Create(value, "[" + (string i) + "]")) + +// -------------------------------------------------------------------------------------- namespace FSharp.Data.Runtime open System open System.Globalization open FSharp.Data +open FSharp.Data.JsonExtensions open FSharp.Data.Runtime open FSharp.Data.Runtime.BaseTypes open FSharp.Data.Runtime.StructuralTypes diff --git a/src/Runtime.fs b/src/Runtime.fs index 5ab8e0003..739899daa 100644 --- a/src/Runtime.fs +++ b/src/Runtime.fs @@ -1,8 +1,8 @@ -namespace global +namespace global open System.Runtime.CompilerServices open FSharp.Core.CompilerServices [] -[] +[] do () diff --git a/src/SetupTesting.fsx b/src/SetupTesting.fsx index 7118966d2..9ce599560 100644 --- a/src/SetupTesting.fsx +++ b/src/SetupTesting.fsx @@ -4,55 +4,39 @@ open System open System.IO open System.Xml.Linq -let generateSetupScript dir proj = +let generateSetupScript dir proj = let getElemName name = XName.Get name - let getElemValue name (parent: XElement) = + let getElemValue name (parent:XElement) = let elem = parent.Element(getElemName name) - - if elem = null || String.IsNullOrEmpty elem.Value then - None - else - Some(elem.Value) - - let getAttrValue name (elem: XElement) = + if elem = null || String.IsNullOrEmpty elem.Value then None else Some(elem.Value) + + let getAttrValue name (elem:XElement) = let attr = elem.Attribute(XName.Get name) - - if attr = null || String.IsNullOrEmpty attr.Value then - None - else - Some(attr.Value) + if attr = null || String.IsNullOrEmpty attr.Value then None else Some(attr.Value) let (|??) (option1: 'a Option) option2 = - if option1.IsSome then - option1 - else - option2 + if option1.IsSome then option1 else option2 let fsProjFile = Path.Combine(dir, proj + ".fsproj") let fsProjXml = XDocument.Load fsProjFile - let refs = + let refs = fsProjXml.Document.Descendants(getElemName "Reference") - |> Seq.choose (fun elem -> - getElemValue "HintPath" elem - |?? getAttrValue "Include" elem) + |> Seq.choose (fun elem -> getElemValue "HintPath" elem |?? getAttrValue "Include" elem) |> Seq.map (fun ref -> ref.Replace(@"\", @"\\").Split(',').[0]) - |> Seq.filter (fun ref -> - ref <> "mscorlib" - && ref <> "FSharp.Core" - && not (ref.EndsWith "FSharp.Core.dll")) + |> Seq.filter (fun ref -> ref <> "mscorlib" && ref <> "FSharp.Core" && not (ref.EndsWith "FSharp.Core.dll")) |> Seq.map (fun ref -> "#r \"" + ref + "\"") |> Seq.toList - let fsFiles = + let fsFiles = fsProjXml.Document.Descendants(getElemName "Compile") |> Seq.choose (fun elem -> getAttrValue "Include" elem) |> Seq.filter (Path.GetExtension >> (<>) ".fsi") |> Seq.filter (Path.GetFileName >> (<>) "Test.fs") |> Seq.map (fun path -> "#load \"" + path.Replace(@"\", @"\\") + "\"") |> Seq.toList - + let tempFile = Path.Combine(dir, "__setup__" + proj + "__.fsx") - File.WriteAllLines(tempFile, refs @ fsFiles) + File.WriteAllLines(tempFile, refs @ fsFiles) diff --git a/src/Test.fsx b/src/Test.fsx index d1314bb87..bd7ef9f05 100644 --- a/src/Test.fsx +++ b/src/Test.fsx @@ -2,7 +2,6 @@ let dir = __SOURCE_DIRECTORY__ + "/FSharp.Data.DesignTime" let proj = "FSharp.Data.DesignTime" SetupTesting.generateSetupScript (__SOURCE_DIRECTORY__ + "/FSharp.Data.DesignTime") "FSharp.Data.DesignTime" - #load "FSharp.Data.DesignTime/__setup__FSharp.Data.DesignTime__.fsx" #load "../paket-files/fsprojects/FSharp.TypeProviders.SDK/src/ProvidedTypesTesting.fs" #load "../tests/FSharp.Data.DesignTime.Tests/TypeProviderInstantiation.fs" @@ -15,44 +14,25 @@ open FSharp.Data open FSharp.Data.Runtime let (++) a b = Path.Combine(a, b) - -let resolutionFolder = - __SOURCE_DIRECTORY__ - ++ ".." - ++ "tests" - ++ "FSharp.Data.Tests" - ++ "Data" - -let outputFolder = - __SOURCE_DIRECTORY__ - ++ ".." - ++ "tests" - ++ "FSharp.Data.DesignTime.Tests" - ++ "expected" - +let resolutionFolder = __SOURCE_DIRECTORY__ ++ ".." ++ "tests" ++ "FSharp.Data.Tests" ++ "Data" +let outputFolder = __SOURCE_DIRECTORY__ ++ ".." ++ "tests" ++ "FSharp.Data.DesignTime.Tests" ++ "expected" let assemblyName = "FSharp.Data.dll" -let dump signatureOnly ignoreOutput saveToFileSystem (inst: TypeProviderInstantiation) = +let dump signatureOnly ignoreOutput platform saveToFileSystem (inst:TypeProviderInstantiation) = let root = __SOURCE_DIRECTORY__ ++ ".." ++ "bin" - let runtimeAssembly = root ++ "netstandard2.0" ++ assemblyName - let runtimeAssemblyRefs = TypeProviderInstantiation.GetRuntimeAssemblyRefs() - - inst.Dump( - resolutionFolder, - (if saveToFileSystem then - outputFolder - else - ""), - runtimeAssembly, - runtimeAssemblyRefs, - signatureOnly, - ignoreOutput - ) + let runtimeAssembly = + match platform with + | NetStandard20 -> root ++ "netstandard2.0" ++ assemblyName + let runtimeAssemblyRefs = TypeProviderInstantiation.GetRuntimeAssemblyRefs platform + inst.Dump(resolutionFolder, (if saveToFileSystem then outputFolder else ""), runtimeAssembly, runtimeAssemblyRefs, signatureOnly, ignoreOutput) |> Console.WriteLine -let dumpAll inst = dump false false false inst +let dumpAll inst = + dump false false NetStandard20 false inst +// dump false false NetStandard16 false inst +// dump false false NetStandard20 false inst -let parameters: HtmlInference.Parameters = +let parameters : HtmlInference.Parameters = { MissingValues = TextConversions.DefaultMissingValues CultureInfo = CultureInfo.InvariantCulture UnitsOfMeasureProvider = StructuralInference.defaultUnitsOfMeasureProvider @@ -60,7 +40,7 @@ let parameters: HtmlInference.Parameters = let includeLayout = false -let printTable tableName (url: string) = +let printTable tableName (url:string) = url |> HtmlDocument.Load |> HtmlRuntime.getTables (Some parameters) includeLayout @@ -69,31 +49,28 @@ let printTable tableName (url: string) = printTable "Overview" "https://en.wikipedia.org/wiki/List_of_Doctor_Who_serials" -Html - { Sample = "doctor_who3.html" - PreferOptionals = false - IncludeLayoutTables = false - MissingValues = "NaN,NA,N/A,#N/A,:,-,TBA,TBD" - Culture = "" - Encoding = "" - ResolutionFolder = "" - EmbeddedResource = "" } +Html { Sample = "doctor_who3.html" + PreferOptionals = false + IncludeLayoutTables = false + MissingValues = "NaN,NA,N/A,#N/A,:,-,TBA,TBD" + Culture = "" + Encoding = "" + ResolutionFolder = "" + EmbeddedResource = "" } |> dumpAll -Json - { Sample = "optionals.json" - SampleIsList = false - RootName = "" - Culture = "" - Encoding = "" - ResolutionFolder = "" - EmbeddedResource = "" - InferTypesFromValues = true - PreferDictionaries = false } +Json { Sample = "optionals.json" + SampleIsList = false + RootName = "" + Culture = "" + Encoding = "" + ResolutionFolder = "" + EmbeddedResource = "" + InferTypesFromValues = true + PreferDictionaries = false } |> dumpAll -Xml - { Sample = "JsonInXml.xml" +Xml { Sample = "JsonInXml.xml" SampleIsList = true Global = false Culture = "" @@ -104,8 +81,7 @@ Xml Schema = "" } |> dumpAll -Csv - { Sample = "AirQuality.csv" +Csv { Sample = "AirQuality.csv" Separators = ";" InferRows = Int32.MaxValue Schema = "" @@ -124,13 +100,9 @@ Csv |> dumpAll let testCases = - __SOURCE_DIRECTORY__ - ++ ".." - ++ "tests" - ++ "FSharp.Data.DesignTime.Tests" - ++ "SignatureTestCases.config" + __SOURCE_DIRECTORY__ ++ ".." ++ "tests" ++ "FSharp.Data.DesignTime.Tests" ++ "SignatureTestCases.config" |> File.ReadAllLines |> Array.map (TypeProviderInstantiation.Parse >> snd) for testCase in testCases do - dump false false true testCase + dump false false NetStandard20 true testCase diff --git a/src/WorldBank/WorldBankRuntime.fs b/src/WorldBank/WorldBankRuntime.fs index 766e1fc1b..91e7b5379 100644 --- a/src/WorldBank/WorldBankRuntime.fs +++ b/src/WorldBank/WorldBankRuntime.fs @@ -292,63 +292,59 @@ type Indicator internal (connection: ServiceConnection, countryOrRegionCode: str let dataDict = lazy (dict data) /// Get the code for the country or region of the indicator - member _.Code = countryOrRegionCode + member x.Code = countryOrRegionCode /// Get the code for the indicator - member _.IndicatorCode = indicatorCode + member x.IndicatorCode = indicatorCode /// Get the name of the indicator - member _.Name = connection.IndicatorsIndexed.[indicatorCode].Name + member x.Name = connection.IndicatorsIndexed.[indicatorCode].Name /// Get the source of the indicator - member _.Source = connection.IndicatorsIndexed.[indicatorCode].Source + member x.Source = connection.IndicatorsIndexed.[indicatorCode].Source /// Get the description of the indicator - member _.Description = connection.IndicatorsIndexed.[indicatorCode].Description + member x.Description = connection.IndicatorsIndexed.[indicatorCode].Description /// Get the indicator value for the given year. If there's no data for that year, NaN is returned - member _.Item + member x.Item with get year = match dataDict.Force().TryGetValue year with | true, value -> value | _ -> Double.NaN /// Get the indicator value for the given year, if present - member _.TryGetValueAt year = + member x.TryGetValueAt year = match dataDict.Force().TryGetValue year with | true, value -> Some value | _ -> None /// Get the years for which the indicator has values - member _.Years = dataDict.Force().Keys + member x.Years = dataDict.Force().Keys /// Get the values for the indicator (without years) - member _.Values = dataDict.Force().Values + member x.Values = dataDict.Force().Values interface seq with - member _.GetEnumerator() = data.GetEnumerator() + member x.GetEnumerator() = data.GetEnumerator() interface IEnumerable with - member _.GetEnumerator() = (data.GetEnumerator() :> _) + member x.GetEnumerator() = (data.GetEnumerator() :> _) /// Metadata for an Indicator [] [] type IndicatorDescription internal (connection: ServiceConnection, topicCode: string, indicatorCode: string) = /// Get the code for the topic of the indicator - member _.Code = topicCode - + member x.Code = topicCode /// Get the code for the indicator - member _.IndicatorCode = indicatorCode - + member x.IndicatorCode = indicatorCode /// Get the name of the indicator - member _.Name = connection.IndicatorsIndexed.[indicatorCode].Name - + member x.Name = connection.IndicatorsIndexed.[indicatorCode].Name /// Get the source of the indicator - member _.Source = connection.IndicatorsIndexed.[indicatorCode].Source - + member x.Source = connection.IndicatorsIndexed.[indicatorCode].Source /// Get the description of the indicator - member _.Description = connection.IndicatorsIndexed.[indicatorCode].Description + member x.Description = connection.IndicatorsIndexed.[indicatorCode].Description /// type IIndicators = @@ -361,17 +357,17 @@ type Indicators internal (connection: ServiceConnection, countryOrRegionCode) = seq { for indicator in connection.Indicators -> Indicator(connection, countryOrRegionCode, indicator.Id) } interface IIndicators with - member _.GetIndicator(indicatorCode) = + member x.GetIndicator(indicatorCode) = Indicator(connection, countryOrRegionCode, indicatorCode) - member _.AsyncGetIndicator(indicatorCode) = + member x.AsyncGetIndicator(indicatorCode) = async { return Indicator(connection, countryOrRegionCode, indicatorCode) } interface seq with - member _.GetEnumerator() = indicators.GetEnumerator() + member x.GetEnumerator() = indicators.GetEnumerator() interface IEnumerable with - member _.GetEnumerator() = indicators.GetEnumerator() :> _ + member x.GetEnumerator() = indicators.GetEnumerator() :> _ /// type IIndicatorsDescriptions = @@ -386,14 +382,14 @@ type IndicatorsDescriptions internal (connection: ServiceConnection, topicCode) } interface IIndicatorsDescriptions with - member _.GetIndicator(indicatorCode) = + member x.GetIndicator(indicatorCode) = IndicatorDescription(connection, topicCode, indicatorCode) interface seq with - member _.GetEnumerator() = indicatorsDescriptions.GetEnumerator() + member x.GetEnumerator() = indicatorsDescriptions.GetEnumerator() interface IEnumerable with - member _.GetEnumerator() = + member x.GetEnumerator() = indicatorsDescriptions.GetEnumerator() :> _ /// @@ -405,21 +401,17 @@ type ICountry = [] type Country internal (connection: ServiceConnection, countryCode: string) = let indicators = new Indicators(connection, countryCode) - /// Get the WorldBank code of the country - member _.Code = countryCode - + member x.Code = countryCode /// Get the name of the country - member _.Name = connection.CountriesIndexed.[countryCode].Name - + member x.Name = connection.CountriesIndexed.[countryCode].Name /// Get the capital city of the country - member _.CapitalCity = connection.CountriesIndexed.[countryCode].CapitalCity - + member x.CapitalCity = connection.CountriesIndexed.[countryCode].CapitalCity /// Get the region of the country - member _.Region = connection.CountriesIndexed.[countryCode].Region + member x.Region = connection.CountriesIndexed.[countryCode].Region interface ICountry with - member _.GetIndicators() = indicators + member x.GetIndicators() = indicators /// type ICountryCollection = @@ -440,13 +432,13 @@ type CountryCollection<'T when 'T :> Country> internal (connection: ServiceConne } interface seq<'T> with - member _.GetEnumerator() = items.GetEnumerator() + member x.GetEnumerator() = items.GetEnumerator() interface IEnumerable with - member _.GetEnumerator() = (items :> IEnumerable).GetEnumerator() + member x.GetEnumerator() = (items :> IEnumerable).GetEnumerator() interface ICountryCollection with - member _.GetCountry(countryCode (*this parameter is only here to help FunScript*) , _countryName) = + member x.GetCountry(countryCode (*this parameter is only here to help FunScript*) , _countryName) = Country(connection, countryCode) /// @@ -460,15 +452,15 @@ type IRegion = type Region internal (connection: ServiceConnection, regionCode: string) = let indicators = new Indicators(connection, regionCode) /// Get the WorldBank code for the region - member _.RegionCode = regionCode + member x.RegionCode = regionCode /// Get the name of the region - member _.Name = connection.RegionsIndexed.[regionCode] + member x.Name = connection.RegionsIndexed.[regionCode] interface IRegion with - member _.GetCountries() = + member x.GetCountries() = CountryCollection(connection, Some regionCode) - member _.GetIndicators() = indicators + member x.GetIndicators() = indicators /// type IRegionCollection = @@ -480,13 +472,13 @@ type RegionCollection<'T when 'T :> Region> internal (connection: ServiceConnect seq { for (code, _) in connection.Regions -> Region(connection, code) :?> 'T } interface seq<'T> with - member _.GetEnumerator() = items.GetEnumerator() + member x.GetEnumerator() = items.GetEnumerator() interface IEnumerable with - member _.GetEnumerator() = (items :> IEnumerable).GetEnumerator() + member x.GetEnumerator() = (items :> IEnumerable).GetEnumerator() interface IRegionCollection with - member _.GetRegion(regionCode) = Region(connection, regionCode) + member x.GetRegion(regionCode) = Region(connection, regionCode) /// type ITopic = @@ -497,18 +489,15 @@ type ITopic = [] type Topic internal (connection: ServiceConnection, topicCode: string) = let indicatorsDescriptions = new IndicatorsDescriptions(connection, topicCode) - /// Get the WorldBank code of the topic - member _.Code = topicCode - + member x.Code = topicCode /// Get the name of the topic - member _.Name = connection.TopicsIndexed.[topicCode].Name - + member x.Name = connection.TopicsIndexed.[topicCode].Name /// Get the description of the topic - member _.Description = connection.TopicsIndexed.[topicCode].Description + member x.Description = connection.TopicsIndexed.[topicCode].Description interface ITopic with - member _.GetIndicators() = indicatorsDescriptions + member x.GetIndicators() = indicatorsDescriptions /// type ITopicCollection = @@ -520,13 +509,13 @@ type TopicCollection<'T when 'T :> Topic> internal (connection: ServiceConnectio seq { for topic in connection.Topics -> Topic(connection, topic.Id) :?> 'T } interface seq<'T> with - member _.GetEnumerator() = items.GetEnumerator() + member x.GetEnumerator() = items.GetEnumerator() interface IEnumerable with - member _.GetEnumerator() = (items :> IEnumerable).GetEnumerator() + member x.GetEnumerator() = (items :> IEnumerable).GetEnumerator() interface ITopicCollection with - member _.GetTopic(topicCode) = Topic(connection, topicCode) + member x.GetTopic(topicCode) = Topic(connection, topicCode) /// type IWorldBankData = @@ -544,8 +533,8 @@ type WorldBankData(serviceUrl: string, sources: string) = let connection = new ServiceConnection(restCache, serviceUrl, sources) interface IWorldBankData with - member _.GetCountries() = + member x.GetCountries() = CountryCollection(connection, None) :> seq<_> - member _.GetRegions() = RegionCollection(connection) :> seq<_> - member _.GetTopics() = TopicCollection(connection) :> seq<_> + member x.GetRegions() = RegionCollection(connection) :> seq<_> + member x.GetTopics() = TopicCollection(connection) :> seq<_> diff --git a/src/Xml/XmlExtensions.fs b/src/Xml/XmlExtensions.fs deleted file mode 100644 index 4ea19829d..000000000 --- a/src/Xml/XmlExtensions.fs +++ /dev/null @@ -1,68 +0,0 @@ -// -------------------------------------------------------------------------------------- -// XML type provider - methods & types used by the generated erased code -// -------------------------------------------------------------------------------------- -namespace FSharp.Data - -open System.Xml.Linq -open System.Runtime.InteropServices - -// XElementExtensions is not a static class with C#-style extension methods because that would -// force to reference System.Xml.Linq.dll everytime you reference FSharp.Data, even when not using -// any of the XML parts -[] -/// Extension methods for XElement -module XElementExtensions = - - type XElement with - - /// Sends the XML to the specified uri. Defaults to a POST request. - member x.Request(uri: string, [] ?httpMethod, [] ?headers: seq<_>) = - let httpMethod = defaultArg httpMethod HttpMethod.Post - let headers = defaultArg (Option.map List.ofSeq headers) [] - - let headers = - if - headers - |> List.exists (fst >> (=) (fst (HttpRequestHeaders.UserAgent ""))) - then - headers - else - HttpRequestHeaders.UserAgent "FSharp.Data XML Type Provider" - :: headers - - let headers = - HttpRequestHeaders.ContentType HttpContentTypes.Xml - :: headers - - Http.Request( - uri, - body = TextRequest(x.ToString(SaveOptions.DisableFormatting)), - headers = headers, - httpMethod = httpMethod - ) - - /// Sends the XML to the specified uri. Defaults to a POST request. - member x.RequestAsync(uri: string, [] ?httpMethod, [] ?headers: seq<_>) = - let httpMethod = defaultArg httpMethod HttpMethod.Post - let headers = defaultArg (Option.map List.ofSeq headers) [] - - let headers = - if - headers - |> List.exists (fst >> (=) (fst (HttpRequestHeaders.UserAgent ""))) - then - headers - else - HttpRequestHeaders.UserAgent "FSharp.Data XML Type Provider" - :: headers - - let headers = - HttpRequestHeaders.ContentType HttpContentTypes.Xml - :: headers - - Http.AsyncRequest( - uri, - body = TextRequest(x.ToString(SaveOptions.DisableFormatting)), - headers = headers, - httpMethod = httpMethod - ) diff --git a/src/Xml/XmlInference.fs b/src/Xml/XmlInference.fs index fc63fd9a0..6fb83c285 100644 --- a/src/Xml/XmlInference.fs +++ b/src/Xml/XmlInference.fs @@ -2,7 +2,7 @@ // Implements type inference for XML // -------------------------------------------------------------------------------------- -module internal ProviderImplementation.XmlInference +module ProviderImplementation.XmlInference open System open System.Xml.Linq @@ -12,10 +12,6 @@ open FSharp.Data.Runtime open FSharp.Data.Runtime.StructuralInference open FSharp.Data.Runtime.StructuralTypes -/// Takes a map and succeeds if it is empty -let (|EmptyMap|_|) result (map: Map<_, _>) = - if map.IsEmpty then Some result else None - // The type of XML element is always a non-optional record with a field // for every attribute. If it has some content, then it also // contains a special field named "" which is either a collection diff --git a/src/Xml/XmlRuntime.fs b/src/Xml/XmlRuntime.fs index fef40a02e..3e26cb746 100644 --- a/src/Xml/XmlRuntime.fs +++ b/src/Xml/XmlRuntime.fs @@ -1,9 +1,77 @@ // -------------------------------------------------------------------------------------- // XML type provider - methods & types used by the generated erased code // -------------------------------------------------------------------------------------- +namespace FSharp.Data + +open System.Xml.Linq +open System.Runtime.InteropServices + +// XElementExtensions is not a static class with C#-style extension methods because that would +// force to reference System.Xml.Linq.dll everytime you reference FSharp.Data, even when not using +// any of the XML parts +[] +/// Extension methods for XElement +module XElementExtensions = + + type XElement with + + /// Sends the XML to the specified uri. Defaults to a POST request. + member x.Request(uri: string, [] ?httpMethod, [] ?headers: seq<_>) = + let httpMethod = defaultArg httpMethod HttpMethod.Post + let headers = defaultArg (Option.map List.ofSeq headers) [] + + let headers = + if + headers + |> List.exists (fst >> (=) (fst (HttpRequestHeaders.UserAgent ""))) + then + headers + else + HttpRequestHeaders.UserAgent "FSharp.Data XML Type Provider" + :: headers + + let headers = + HttpRequestHeaders.ContentType HttpContentTypes.Xml + :: headers + + Http.Request( + uri, + body = TextRequest(x.ToString(SaveOptions.DisableFormatting)), + headers = headers, + httpMethod = httpMethod + ) + + /// Sends the XML to the specified uri. Defaults to a POST request. + member x.RequestAsync(uri: string, [] ?httpMethod, [] ?headers: seq<_>) = + let httpMethod = defaultArg httpMethod HttpMethod.Post + let headers = defaultArg (Option.map List.ofSeq headers) [] + + let headers = + if + headers + |> List.exists (fst >> (=) (fst (HttpRequestHeaders.UserAgent ""))) + then + headers + else + HttpRequestHeaders.UserAgent "FSharp.Data XML Type Provider" + :: headers + + let headers = + HttpRequestHeaders.ContentType HttpContentTypes.Xml + :: headers + + Http.AsyncRequest( + uri, + body = TextRequest(x.ToString(SaveOptions.DisableFormatting)), + headers = headers, + httpMethod = httpMethod + ) + +// -------------------------------------------------------------------------------------- namespace FSharp.Data.Runtime.BaseTypes +open System open System.ComponentModel open System.IO open System.Xml.Linq diff --git a/src/Xml/XsdInference.fs b/src/Xml/XsdInference.fs index 81fc8c739..5f073db5e 100644 --- a/src/Xml/XsdInference.fs +++ b/src/Xml/XsdInference.fs @@ -208,7 +208,7 @@ module XsdParsing = /// Element definitions in a schema are mapped to InferedType instances -module internal XsdInference = +module XsdInference = open XsdModel open FSharp.Data.Runtime.StructuralTypes diff --git a/tests/FSharp.Data.Core.Tests/FSharp.Data.Core.Tests.fsproj b/tests/FSharp.Data.Core.Tests/FSharp.Data.Core.Tests.fsproj deleted file mode 100644 index 48a896c14..000000000 --- a/tests/FSharp.Data.Core.Tests/FSharp.Data.Core.Tests.fsproj +++ /dev/null @@ -1,43 +0,0 @@ - - - - net6.0 - false - - true - true - false - $(OtherFlags) --warnon:1182 --nowarn:44 - - true - - - - - - - PreserveNewest - - - - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/tests/FSharp.Data.Core.Tests/Program.fs b/tests/FSharp.Data.Core.Tests/Program.fs deleted file mode 100644 index 96a10a38d..000000000 --- a/tests/FSharp.Data.Core.Tests/Program.fs +++ /dev/null @@ -1,6 +0,0 @@ -open System - -[] -let main _argv = - printfn "Dotnet Core NUnit Tests..." - 0 diff --git a/tests/FSharp.Data.Core.Tests/paket.references b/tests/FSharp.Data.Core.Tests/paket.references deleted file mode 100644 index aaa9d970b..000000000 --- a/tests/FSharp.Data.Core.Tests/paket.references +++ /dev/null @@ -1,8 +0,0 @@ -group Test - - Microsoft.NET.Test.Sdk - NUnit - NUnit3TestAdapter - FsUnit - FsCheck - GitHubActionsTestLogger diff --git a/tests/FSharp.Data.DesignTime.Tests/FSharp.Data.DesignTime.Tests.fsproj b/tests/FSharp.Data.DesignTime.Tests/FSharp.Data.DesignTime.Tests.fsproj index b12489268..cbecb419b 100644 --- a/tests/FSharp.Data.DesignTime.Tests/FSharp.Data.DesignTime.Tests.fsproj +++ b/tests/FSharp.Data.DesignTime.Tests/FSharp.Data.DesignTime.Tests.fsproj @@ -7,7 +7,6 @@ true false - $(OtherFlags) --nowarn:44 diff --git a/tests/FSharp.Data.DesignTime.Tests/InferenceTests.fs b/tests/FSharp.Data.DesignTime.Tests/InferenceTests.fs index 43d9b1ccd..72a589f9a 100644 --- a/tests/FSharp.Data.DesignTime.Tests/InferenceTests.fs +++ b/tests/FSharp.Data.DesignTime.Tests/InferenceTests.fs @@ -3,9 +3,6 @@ module FSharp.Data.DesignTime.Tests.InferenceTests open FsUnit open System open System.Globalization -open System.Xml -open System.Xml.Linq -open System.Xml.Schema open NUnit.Framework open FSharp.Data open FSharp.Data.Runtime @@ -15,18 +12,19 @@ open FSharp.Data.Runtime.StructuralInference open ProviderImplementation /// A collection containing just one type -let internal SimpleCollection typ = +let SimpleCollection typ = InferedType.Collection([ typeTag typ], Map.ofSeq [typeTag typ, (InferedMultiplicity.Multiple, typ)]) let culture = TextRuntime.GetCulture "" -let internal inferenceMode = InferenceMode'.ValuesOnly -let internal unitsOfMeasureProvider = ProviderHelpers.unitsOfMeasureProvider +let inferenceMode = InferenceMode'.ValuesOnly +let unitsOfMeasureProvider = ProviderHelpers.unitsOfMeasureProvider -let internal inferType (csv:CsvFile) inferRows missingValues cultureInfo schema assumeMissingValues preferOptionals = +let inferType (csv:CsvFile) inferRows missingValues cultureInfo schema assumeMissingValues preferOptionals = let headerNamesAndUnits, schema = parseHeaders csv.Headers csv.NumberOfColumns schema unitsOfMeasureProvider inferType headerNamesAndUnits schema (csv.Rows |> Seq.map (fun x -> x.Columns)) inferRows missingValues inferenceMode cultureInfo assumeMissingValues preferOptionals unitsOfMeasureProvider -let internal toRecord fields = InferedType.Record(None, fields, false) +let toRecord fields = InferedType.Record(None, fields, false) + [] let ``List.pairBy helper function works``() = @@ -385,6 +383,7 @@ let ``Inference with % suffix``() = let expected = toRecord [ propFloat ; propInteger ] actual |> should equal expected + [] let ``Inference with $ prefix``() = let source = CsvFile.Parse("float,integer\n$2.0,$2\n$4.0,$3\n") @@ -394,21 +393,27 @@ let ``Inference with $ prefix``() = let expected = toRecord [ propFloat ; propInteger ] actual |> should equal expected -let internal getInferedTypeFromSamples samples = + +open System.Xml +open System.Xml.Linq +open System.Xml.Schema + +let getInferedTypeFromSamples samples = let culture = System.Globalization.CultureInfo.InvariantCulture samples |> Array.map XElement.Parse |> XmlInference.inferType unitsOfMeasureProvider inferenceMode culture false false |> Seq.fold (subtypeInfered false) InferedType.Top -let internal getInferedTypeFromSchema xsd = + +let getInferedTypeFromSchema xsd = xsd |> XmlSchema.parseSchema "" |> XsdParsing.getElements |> List.ofSeq |> XsdInference.inferElements -let internal isValid xsd = +let isValid xsd = let xmlSchemaSet = XmlSchema.parseSchema "" xsd fun xml -> let settings = XmlReaderSettings(ValidationType = ValidationType.Schema) @@ -421,7 +426,8 @@ let internal isValid xsd = printfn "%s/n%s" e.Message xml false -let internal getInferedTypes xsd xmlSamples = + +let getInferedTypes xsd xmlSamples = //printfn "%s/n---------------------------------------------" xsd let isValid = isValid xsd for xml in xmlSamples do @@ -434,11 +440,14 @@ let internal getInferedTypes xsd xmlSamples = //printfn "%A" inferedTypeFromSamples inferedTypeFromSchema, inferedTypeFromSamples -let internal check xsd xmlSamples = + + +let check xsd xmlSamples = //printfn "checking schema and samples" let inferedTypeFromSchema, inferedTypeFromSamples = getInferedTypes xsd xmlSamples inferedTypeFromSchema |> should equal inferedTypeFromSamples + [] let ``at least one global complex element is needed``() = let xsd = """ @@ -497,7 +506,6 @@ let ``recursive schemas don't cause loops``() = """ let inferedTypeFromSchema = getInferedTypeFromSchema xsd - inferedTypeFromSchema |> ignore //printfn "%A" inferedTypeFromSchema let xsd = """] -let main _argv = +let main argv = printfn "Dotnet Core NUnit Tests..." - 0 + 0 \ No newline at end of file diff --git a/tests/FSharp.Data.DesignTime.Tests/SignatureTestCases.config b/tests/FSharp.Data.DesignTime.Tests/SignatureTestCases.config index 6f6688a33..63a6c5092 100644 --- a/tests/FSharp.Data.DesignTime.Tests/SignatureTestCases.config +++ b/tests/FSharp.Data.DesignTime.Tests/SignatureTestCases.config @@ -207,4 +207,5 @@ Html,doctor_who3.html,false,false, Html,SimpleHtmlLists.html,false,false, Html,EmptyDefinitionLists.html,false,false, Html,zoopla.html,false,false, -Html,zoopla2.html,false,false, \ No newline at end of file +Html,zoopla2.html,false,false, +WorldBank,World Development Indicators;Global Financial Development,true diff --git a/tests/FSharp.Data.DesignTime.Tests/SignatureTests.fs b/tests/FSharp.Data.DesignTime.Tests/SignatureTests.fs index 5636dcbc5..9347bd109 100644 --- a/tests/FSharp.Data.DesignTime.Tests/SignatureTests.fs +++ b/tests/FSharp.Data.DesignTime.Tests/SignatureTests.fs @@ -10,44 +10,52 @@ let (++) a b = Path.Combine(a, b) let sourceDirectory = __SOURCE_DIRECTORY__ -let testCases = +let testCasesTuple = sourceDirectory ++ "SignatureTestCases.config" |> File.ReadAllLines // "No data is available for encoding 932. For information on defining a custom encoding, see the documentation for the Encoding.RegisterProvider method." |> Array.filter (fun line -> not (line.Contains ("cp932.csv"))) + |> Array.map TypeProviderInstantiation.Parse + +let testCases = + testCasesTuple + // These WorldBank tests nearly always need updating. COmment out this line if you want to go through the process of + // updating them. + |> Array.filter (snd >> function | WorldBank _ -> false | _ -> true) + |> Array.map snd let expectedDirectory = sourceDirectory ++ "expected" let resolutionFolder = sourceDirectory ++ ".." ++ "FSharp.Data.Tests" ++ "Data" +let assemblyName = "FSharp.Data.dll" +let netstandard2RuntimeAssembly = sourceDirectory ++ ".." ++ ".." ++ "src" ++ "FSharp.Data" ++ "bin" ++ + #if DEBUG + "Debug" + #else + "Release" + #endif + ++ "netstandard2.0" ++ assemblyName -#if DEBUG -let build = "Debug" -#else -let build = "Release" -#endif - -let netstandard2RuntimeAssembly = sourceDirectory ++ ".." ++ ".." ++ "src" ++ "FSharp.Data" ++ "bin" ++ build ++ "netstandard2.0" ++ "FSharp.Data.dll" +let getRuntimeRefs platform = TypeProviderInstantiation.GetRuntimeAssemblyRefs platform let normalize (str:string) = str.Replace("\r\n", "\n").Replace("\r", "\n").Replace("@\"\"", "\"\"") [] [] -let ``Validate signature didn't change `` (testCaseSpec: string) = - let _, testCase = TypeProviderInstantiation.Parse testCaseSpec - let path = testCase.ExpectedPath expectedDirectory - let expected = path |> File.ReadAllText |> normalize - let assemblyRefs = TypeProviderInstantiation.GetRuntimeAssemblyRefs() - let outputRaw = testCase.Dump (resolutionFolder, "", netstandard2RuntimeAssembly, assemblyRefs, signatureOnly=false, ignoreOutput=false) - let output = outputRaw |> normalize - if output <> expected then - printfn "Obtained Signature:\n%s" outputRaw - File.WriteAllText(testCase.ExpectedPath expectedDirectory + ".obtained", outputRaw) - output |> should equal expected +let ``Validate signature didn't change `` (testCase:TypeProviderInstantiation) = + let path = testCase.ExpectedPath expectedDirectory + let expected = path |> File.ReadAllText |> normalize + let assemblyRefs = getRuntimeRefs NetStandard20 + printfn "assemblyRefs = %A" assemblyRefs + let outputRaw = testCase.Dump (resolutionFolder, "", netstandard2RuntimeAssembly, assemblyRefs, signatureOnly=false, ignoreOutput=false) + let output = outputRaw |> normalize + if output <> expected then + printfn "Obtained Signature:\n%s" outputRaw + File.WriteAllText(testCase.ExpectedPath expectedDirectory + ".obtained", outputRaw) + output |> should equal expected [] [] -let ``Generating expressions works in netstandard2.0 `` (testCaseSpec: string) = - let _, testCase = TypeProviderInstantiation.Parse testCaseSpec - let assemblyRefs = TypeProviderInstantiation.GetRuntimeAssemblyRefs() - testCase.Dump(resolutionFolder, "", netstandard2RuntimeAssembly, assemblyRefs, signatureOnly=false, ignoreOutput=true) |> ignore +let ``Generating expressions works in netstandard2.0 `` (testCase:TypeProviderInstantiation) = + testCase.Dump(resolutionFolder, "", netstandard2RuntimeAssembly, (getRuntimeRefs NetStandard20), signatureOnly=false, ignoreOutput=true) |> ignore diff --git a/tests/FSharp.Data.DesignTime.Tests/TypeProviderInstantiation.fs b/tests/FSharp.Data.DesignTime.Tests/TypeProviderInstantiation.fs index b632fe664..422e0fdde 100644 --- a/tests/FSharp.Data.DesignTime.Tests/TypeProviderInstantiation.fs +++ b/tests/FSharp.Data.DesignTime.Tests/TypeProviderInstantiation.fs @@ -8,7 +8,7 @@ open ProviderImplementation.ProvidedTypesTesting open FSharp.Data.Runtime open FSharp.Data.Runtime.StructuralInference -type internal CsvProviderArgs = +type CsvProviderArgs = { Sample : string Separators : string InferRows : int @@ -26,7 +26,7 @@ type internal CsvProviderArgs = ResolutionFolder : string EmbeddedResource : string } -type internal XmlProviderArgs = +type XmlProviderArgs = { Sample : string SampleIsList : bool Global : bool @@ -38,7 +38,7 @@ type internal XmlProviderArgs = Schema : string InferenceMode: InferenceMode } -type internal JsonProviderArgs = +type JsonProviderArgs = { Sample : string SampleIsList : bool RootName : string @@ -50,7 +50,7 @@ type internal JsonProviderArgs = PreferDictionaries : bool InferenceMode: InferenceMode } -type internal HtmlProviderArgs = +type HtmlProviderArgs = { Sample : string PreferOptionals : bool IncludeLayoutTables : bool @@ -60,11 +60,13 @@ type internal HtmlProviderArgs = ResolutionFolder : string EmbeddedResource : string } -type internal WorldBankProviderArgs = +type WorldBankProviderArgs = { Sources : string Asynchronous : bool } -type internal TypeProviderInstantiation = +type Platform = NetStandard20 + +type TypeProviderInstantiation = | Csv of CsvProviderArgs | Xml of XmlProviderArgs | Json of JsonProviderArgs @@ -249,23 +251,7 @@ type internal TypeProviderInstantiation = Asynchronous = args.[2] |> bool.Parse } | _ -> failwithf "Unknown: %s" args.[0] - static member GetRuntimeAssemblyRefs () = - let (++) a b = Path.Combine(a, b) - #if DEBUG - let build = "Debug" - #else - let build = "Release" - #endif - - let extraDlls = - [ "FSharp.Data.Http" - "FSharp.Data.Csv.Core" - "FSharp.Data.Html.Core" - "FSharp.Data.Xml.Core" - "FSharp.Data.Json.Core" - "FSharp.Data.WorldBank.Core" ] - let extraRefs = - [ for j in extraDlls do - __SOURCE_DIRECTORY__ ++ ".." ++ ".." ++ "src" ++ "FSharp.Data" ++ "bin" ++ build ++ "netstandard2.0" ++ (j + ".dll") ] - extraRefs @ Targets.DotNetStandard20FSharpRefs() + static member GetRuntimeAssemblyRefs platform = + match platform with + | NetStandard20 -> Targets.DotNetStandard20FSharpRefs() diff --git a/tests/FSharp.Data.Reference.Tests/FSharp.Data.Reference.Tests.fsproj b/tests/FSharp.Data.Reference.Tests/FSharp.Data.Reference.Tests.fsproj index 532d035a1..ad64764f6 100644 --- a/tests/FSharp.Data.Reference.Tests/FSharp.Data.Reference.Tests.fsproj +++ b/tests/FSharp.Data.Reference.Tests/FSharp.Data.Reference.Tests.fsproj @@ -7,7 +7,6 @@ true false - $(OtherFlags) --warnon:1182 --nowarn:44 diff --git a/tests/FSharp.Data.Reference.Tests/Program.fs b/tests/FSharp.Data.Reference.Tests/Program.fs index 96a10a38d..c94e2a499 100644 --- a/tests/FSharp.Data.Reference.Tests/Program.fs +++ b/tests/FSharp.Data.Reference.Tests/Program.fs @@ -1,6 +1,6 @@ -open System +open System [] -let main _argv = +let main argv = printfn "Dotnet Core NUnit Tests..." - 0 + 0 \ No newline at end of file diff --git a/tests/FSharp.Data.Core.Tests.CSharp/CsvExtensionsTests.cs b/tests/FSharp.Data.Tests.CSharp/CsvExtensionsTests.cs similarity index 100% rename from tests/FSharp.Data.Core.Tests.CSharp/CsvExtensionsTests.cs rename to tests/FSharp.Data.Tests.CSharp/CsvExtensionsTests.cs diff --git a/tests/FSharp.Data.Core.Tests.CSharp/FSharp.Data.Core.Tests.CSharp.csproj b/tests/FSharp.Data.Tests.CSharp/FSharp.Data.Tests.CSharp.csproj similarity index 61% rename from tests/FSharp.Data.Core.Tests.CSharp/FSharp.Data.Core.Tests.CSharp.csproj rename to tests/FSharp.Data.Tests.CSharp/FSharp.Data.Tests.CSharp.csproj index 6a2b87bfd..5159e9b6e 100644 --- a/tests/FSharp.Data.Core.Tests.CSharp/FSharp.Data.Core.Tests.CSharp.csproj +++ b/tests/FSharp.Data.Tests.CSharp/FSharp.Data.Tests.CSharp.csproj @@ -7,7 +7,6 @@ true true false - $(OtherFlags) --warnon:1182 --nowarn:44 @@ -20,12 +19,7 @@ - - - - - - + diff --git a/tests/FSharp.Data.Core.Tests.CSharp/HtmlExtensionsTests.cs b/tests/FSharp.Data.Tests.CSharp/HtmlExtensionsTests.cs similarity index 100% rename from tests/FSharp.Data.Core.Tests.CSharp/HtmlExtensionsTests.cs rename to tests/FSharp.Data.Tests.CSharp/HtmlExtensionsTests.cs diff --git a/tests/FSharp.Data.Core.Tests.CSharp/JsonExtensionsTests.cs b/tests/FSharp.Data.Tests.CSharp/JsonExtensionsTests.cs similarity index 100% rename from tests/FSharp.Data.Core.Tests.CSharp/JsonExtensionsTests.cs rename to tests/FSharp.Data.Tests.CSharp/JsonExtensionsTests.cs diff --git a/tests/FSharp.Data.Core.Tests.CSharp/Properties/AssemblyInfo.cs b/tests/FSharp.Data.Tests.CSharp/Properties/AssemblyInfo.cs similarity index 100% rename from tests/FSharp.Data.Core.Tests.CSharp/Properties/AssemblyInfo.cs rename to tests/FSharp.Data.Tests.CSharp/Properties/AssemblyInfo.cs diff --git a/tests/FSharp.Data.Core.Tests.CSharp/paket.references b/tests/FSharp.Data.Tests.CSharp/paket.references similarity index 100% rename from tests/FSharp.Data.Core.Tests.CSharp/paket.references rename to tests/FSharp.Data.Tests.CSharp/paket.references diff --git a/tests/FSharp.Data.Tests/CsvProvider.fs b/tests/FSharp.Data.Tests/CsvProvider.fs index 673d456aa..36a69d037 100644 --- a/tests/FSharp.Data.Tests/CsvProvider.fs +++ b/tests/FSharp.Data.Tests/CsvProvider.fs @@ -6,8 +6,8 @@ open System open System.IO open FSharp.Data.UnitSystems.SI.UnitNames open FSharp.Data -open FSharp.Data.Runtime open FSharp.Data.Runtime.CsvInference +open FSharp.Data.Runtime open System.Globalization let [] simpleCsv = """ @@ -72,14 +72,14 @@ let ``Inference of numbers with empty values`` () = let row = rows.[0] - let _f1:float = row.Float1 - let _f2:float = row.Float2 - let _f3:float = row.Float3 - let _f4:float = row.Float4 - let _i:Nullable = row.Int - let _f5:float = row.Float5 - let _f6:float = row.Float6 - let _d:option = row.Date + let f1:float = row.Float1 + let f2:float = row.Float2 + let f3:float = row.Float3 + let f4:float = row.Float4 + let i:Nullable = row.Int + let f5:float = row.Float5 + let f6:float = row.Float6 + let d:option = row.Date let expected = 1.0, 1.0, 1.0, 1.0, Nullable(), Double.NaN, Double.NaN, (None: DateTime option) let actual = row.Float1, row.Float2, row.Float3, row.Float4, row.Int, row.Float5, row.Float6, row.Date @@ -343,9 +343,9 @@ let ``Uses UTF8 for sample file when encoding not specified``() = [] let ``Disposing CsvProvider shouldn't throw``() = - let _csv = + let csv = use csv = CsvProvider<"Data/TabSeparated.csv", HasHeaders=false>.GetSample() - csv.Rows |> Seq.iter (fun _ -> ()) + csv.Rows |> Seq.iter (fun x -> ()) () [] diff --git a/tests/FSharp.Data.Core.Tests/CsvReader.fs b/tests/FSharp.Data.Tests/CsvReader.fs similarity index 100% rename from tests/FSharp.Data.Core.Tests/CsvReader.fs rename to tests/FSharp.Data.Tests/CsvReader.fs diff --git a/tests/FSharp.Data.Tests/FSharp.Data.Tests.fsproj b/tests/FSharp.Data.Tests/FSharp.Data.Tests.fsproj index 69617490b..0b65efeeb 100755 --- a/tests/FSharp.Data.Tests/FSharp.Data.Tests.fsproj +++ b/tests/FSharp.Data.Tests/FSharp.Data.Tests.fsproj @@ -9,7 +9,6 @@ false true - $(OtherFlags) --warnon:1182 --nowarn:44 @@ -18,10 +17,21 @@ PreserveNewest + + + + + + + + + + + diff --git a/tests/FSharp.Data.Core.Tests/HtmlCharRefs.fs b/tests/FSharp.Data.Tests/HtmlCharRefs.fs similarity index 92% rename from tests/FSharp.Data.Core.Tests/HtmlCharRefs.fs rename to tests/FSharp.Data.Tests/HtmlCharRefs.fs index 6be3f42c7..de6d09a12 100644 --- a/tests/FSharp.Data.Core.Tests/HtmlCharRefs.fs +++ b/tests/FSharp.Data.Tests/HtmlCharRefs.fs @@ -1,4 +1,4 @@ -module FSharp.Data.Tests.HtmlCharRefs +module FSharp.Data.Tests.HtmlCharRefs open NUnit.Framework open FsUnit @@ -8,7 +8,7 @@ open FSharp.Data.HtmlNode open FSharp.Data.JsonExtensions let charRefsTestCases = - JsonValue.Load(__SOURCE_DIRECTORY__ + "/../FSharp.Data.Tests/Data/charrefs.json")?items.AsArray() + JsonValue.Load(__SOURCE_DIRECTORY__ + "/Data/charrefs.json")?items.AsArray() |> Array.map (fun x -> [| x?key.AsString(); x?characters.AsString() |]) [] diff --git a/tests/FSharp.Data.Core.Tests/HtmlCssSelectors.fs b/tests/FSharp.Data.Tests/HtmlCssSelectors.fs similarity index 100% rename from tests/FSharp.Data.Core.Tests/HtmlCssSelectors.fs rename to tests/FSharp.Data.Tests/HtmlCssSelectors.fs diff --git a/tests/FSharp.Data.Core.Tests/HtmlOperations.fs b/tests/FSharp.Data.Tests/HtmlOperations.fs similarity index 99% rename from tests/FSharp.Data.Core.Tests/HtmlOperations.fs rename to tests/FSharp.Data.Tests/HtmlOperations.fs index b8cb4223d..925e6b4e2 100644 --- a/tests/FSharp.Data.Core.Tests/HtmlOperations.fs +++ b/tests/FSharp.Data.Tests/HtmlOperations.fs @@ -1,4 +1,4 @@ -module FSharp.Data.Tests.HtmlOperations +module FSharp.Data.Tests.HtmlOperations open NUnit.Framework open FsUnit diff --git a/tests/FSharp.Data.Core.Tests/HtmlParser.fs b/tests/FSharp.Data.Tests/HtmlParser.fs similarity index 98% rename from tests/FSharp.Data.Core.Tests/HtmlParser.fs rename to tests/FSharp.Data.Tests/HtmlParser.fs index fe9a26fc2..b22feb528 100644 --- a/tests/FSharp.Data.Core.Tests/HtmlParser.fs +++ b/tests/FSharp.Data.Tests/HtmlParser.fs @@ -589,7 +589,7 @@ let ``Can handle html without html tag``() = [] let ``Can find header when nested in a div``() = let tables = - HtmlDocument.Load "wimbledon_wikipedia.html" + HtmlDocument.Load "Data/wimbledon_wikipedia.html" |> getTables false |> List.map (fun t -> t.Name, t) |> Map.ofList @@ -599,7 +599,7 @@ let ``Can find header when nested in a div``() = [] let ``Can parse tables imdb chart``() = - let imdb = HtmlDocument.Load "imdb_chart.htm" + let imdb = HtmlDocument.Load "Data/imdb_chart.htm" let tables = imdb |> getTables false tables.Length |> should equal 2 tables.[0].Name |> should equal "Top 250" @@ -608,12 +608,12 @@ let ``Can parse tables imdb chart``() = [] let ``Can parse tables ebay cars``() = - let _ebay = HtmlDocument.Load "ebay_cars.htm" + let ebay = HtmlDocument.Load "Data/ebay_cars.htm" true |> should equal true [] let ``Does not crash when parsing us presidents``() = - let _table = HtmlDocument.Load "us_presidents_wikipedia.html" |> getTables false + let table = HtmlDocument.Load "Data/us_presidents_wikipedia.html" |> getTables false true |> should equal true [] @@ -859,33 +859,33 @@ let ``Drops whitespace outside pre``() = [] let ``Can parse national rail mobile site correctly``() = - HtmlDocument.Load "UKDepartures.html" + HtmlDocument.Load "Data/UKDepartures.html" |> HtmlDocument.descendantsNamed false [ "li" ] |> Seq.length |> should equal 68 - HtmlDocument.Load "UKLiveProgress.html" + HtmlDocument.Load "Data/UKLiveProgress.html" |> HtmlDocument.descendantsNamed false [ "li" ] |> Seq.length |> should equal 15 - HtmlDocument.Load "UKDepartures.html" + HtmlDocument.Load "Data/UKDepartures.html" |> HtmlDocument.descendantsNamed false [ "li"; "hr" ] |> Seq.length |> should equal 69 - HtmlDocument.Load "UKLiveProgress.html" + HtmlDocument.Load "Data/UKLiveProgress.html" |> HtmlDocument.descendantsNamed false [ "li"; "hr" ] |> Seq.length |> should equal 17 [] let ``Can parse old zoopla site correctly``() = - HtmlDocument.Load "zoopla.html" + HtmlDocument.Load "Data/zoopla.html" |> HtmlDocument.descendants false (fun x -> HtmlNode.hasName "li" x && HtmlNode.hasAttribute "itemtype" "http://schema.org/Place" x) |> Seq.length |> should equal 100 [] let ``Can parse new zoopla site correctly``() = - HtmlDocument.Load "zoopla2.html" + HtmlDocument.Load "Data/zoopla2.html" |> HtmlDocument.descendants false (fun x -> HtmlNode.hasName "li" x && HtmlNode.hasAttribute "itemtype" "http://schema.org/Residence" x) |> Seq.length |> should equal 10 diff --git a/tests/FSharp.Data.Core.Tests/Http.fs b/tests/FSharp.Data.Tests/Http.fs similarity index 99% rename from tests/FSharp.Data.Core.Tests/Http.fs rename to tests/FSharp.Data.Tests/Http.fs index f2223a5ca..30b240e7a 100644 --- a/tests/FSharp.Data.Core.Tests/Http.fs +++ b/tests/FSharp.Data.Tests/Http.fs @@ -224,14 +224,14 @@ let testFormDataSizesInBytes = [ [] let testFormDataBodySize (size: int) = use localServer = startHttpLocalServer() - let bodyString = seq {for _i in 0..size -> "x\n"} |> String.concat "" + let bodyString = seq {for i in 0..size -> "x\n"} |> String.concat "" let body = FormValues([("input", bodyString)]) Assert.DoesNotThrowAsync(fun () -> Http.AsyncRequest (url= localServer.BaseAddress + "/200", httpMethod="POST", body=body, timeout = 10000) |> Async.Ignore |> Async.StartAsTask :> _) [] let testMultipartFormDataBodySize (size: int) = use localServer = startHttpLocalServer() - let bodyString = seq {for _i in 0..size -> "x\n"} |> String.concat "" + let bodyString = seq {for i in 0..size -> "x\n"} |> String.concat "" let multipartItem = [ MultipartItem("input", "input.txt", new MemoryStream(Encoding.UTF8.GetBytes(bodyString)) :> Stream) ] let body = Multipart(Guid.NewGuid().ToString(), multipartItem) diff --git a/tests/FSharp.Data.Core.Tests/JsonConversions.fs b/tests/FSharp.Data.Tests/JsonConversions.fs similarity index 95% rename from tests/FSharp.Data.Core.Tests/JsonConversions.fs rename to tests/FSharp.Data.Tests/JsonConversions.fs index ef8a8917f..43c87bbdf 100644 --- a/tests/FSharp.Data.Core.Tests/JsonConversions.fs +++ b/tests/FSharp.Data.Tests/JsonConversions.fs @@ -1,4 +1,4 @@ -module FSharp.Data.Tests.JsonConversions +module FSharp.Data.Tests.JsonConversions open NUnit.Framework open FsUnit diff --git a/tests/FSharp.Data.Core.Tests/JsonParserProperties.fs b/tests/FSharp.Data.Tests/JsonParserProperties.fs similarity index 97% rename from tests/FSharp.Data.Core.Tests/JsonParserProperties.fs rename to tests/FSharp.Data.Tests/JsonParserProperties.fs index 7fcda85be..c50b003b6 100644 --- a/tests/FSharp.Data.Core.Tests/JsonParserProperties.fs +++ b/tests/FSharp.Data.Tests/JsonParserProperties.fs @@ -11,16 +11,16 @@ let rec isValidJsonString s = match s with | [] -> true | ['\\'] -> false - | '"' :: _t -> false - | h :: _t when Globalization.CharUnicodeInfo.GetUnicodeCategory h + | '"' :: t -> false + | h :: t when Globalization.CharUnicodeInfo.GetUnicodeCategory h = Globalization.UnicodeCategory.Control -> false | '\\' :: 'u' :: d1 :: d2 :: d3 :: d4 :: t when [d1;d2;d3;d4] |> Seq.forall (fun c -> (Char.IsDigit c || Text.RegularExpressions.Regex("[a-fA-F]").IsMatch(c.ToString()))) -> isValidJsonString t - | '\\' :: i :: _t when escaped |> (not << Set.contains i) -> false + | '\\' :: i :: t when escaped |> (not << Set.contains i) -> false | '\\' :: i :: t when escaped |> Set.contains i -> isValidJsonString t - | _h :: t -> isValidJsonString t + | h :: t -> isValidJsonString t let validJsonStringGen = Arb.generate diff --git a/tests/FSharp.Data.Core.Tests/JsonValue.fs b/tests/FSharp.Data.Tests/JsonValue.fs similarity index 99% rename from tests/FSharp.Data.Core.Tests/JsonValue.fs rename to tests/FSharp.Data.Tests/JsonValue.fs index 1e0ee911a..2c240fe65 100644 --- a/tests/FSharp.Data.Core.Tests/JsonValue.fs +++ b/tests/FSharp.Data.Tests/JsonValue.fs @@ -1,4 +1,4 @@ -module FSharp.Data.Tests.JsonValue +module FSharp.Data.Tests.JsonValue open NUnit.Framework open System diff --git a/tests/FSharp.Data.Core.Tests/NameUtils.fs b/tests/FSharp.Data.Tests/NameUtils.fs similarity index 96% rename from tests/FSharp.Data.Core.Tests/NameUtils.fs rename to tests/FSharp.Data.Tests/NameUtils.fs index 9bb3b717c..5fc45ceab 100644 --- a/tests/FSharp.Data.Core.Tests/NameUtils.fs +++ b/tests/FSharp.Data.Tests/NameUtils.fs @@ -1,4 +1,4 @@ -// -------------------------------------------------------------------------------------- +// -------------------------------------------------------------------------------------- // Tests for a utility that generates nice PascalCase and camelCase names for members // -------------------------------------------------------------------------------------- @@ -87,7 +87,7 @@ let ``Handles long and ugly names`` () = [] let ``Unique generator generates unique names`` () = let gen = uniqueGenerator nicePascalName - let names = [ for _i in 0 .. 100 -> gen "test" ] + let names = [ for i in 0 .. 100 -> gen "test" ] Seq.length names |> should equal (Seq.length (set names)) [] diff --git a/tests/FSharp.Data.Tests/Program.fs b/tests/FSharp.Data.Tests/Program.fs index 96a10a38d..6329c59f8 100644 --- a/tests/FSharp.Data.Tests/Program.fs +++ b/tests/FSharp.Data.Tests/Program.fs @@ -1,6 +1,6 @@ open System [] -let main _argv = +let main argv = printfn "Dotnet Core NUnit Tests..." 0 diff --git a/tests/FSharp.Data.Core.Tests/TextConversions.fs b/tests/FSharp.Data.Tests/TextConversions.fs similarity index 98% rename from tests/FSharp.Data.Core.Tests/TextConversions.fs rename to tests/FSharp.Data.Tests/TextConversions.fs index 16d064145..5547ed822 100644 --- a/tests/FSharp.Data.Core.Tests/TextConversions.fs +++ b/tests/FSharp.Data.Tests/TextConversions.fs @@ -1,4 +1,4 @@ -module FSharp.Data.Tests.Conversions +module FSharp.Data.Tests.Conversions open NUnit.Framework open FsUnit diff --git a/tests/FSharp.Data.Tests/XmlProvider.fs b/tests/FSharp.Data.Tests/XmlProvider.fs index 199f66410..a13028d6e 100644 --- a/tests/FSharp.Data.Tests/XmlProvider.fs +++ b/tests/FSharp.Data.Tests/XmlProvider.fs @@ -151,10 +151,10 @@ type Html = XmlProvider<""" [] let ``Nested xml types compile when only used in type annotations``() = - let _divWorks (_div:Html.Div) = () - let _spanWorks (_span:Html.Span) = () - let _ulWorks (_ul:Html.Ul) = () - let _liWorks (_li:Html.Li) = () + let divWorks (div:Html.Div) = () + let spanWorks (span:Html.Span) = () + let ulWorks (ul:Html.Ul) = () + let liWorks (li:Html.Li) = () () [] @@ -168,8 +168,8 @@ let sameNameDifferentNamespace = """ [] let ``XML elements with same name in different namespaces``() = let xml = XmlProvider.GetSample() - let _b1 = xml.B - let _b2 = xml.B2 + let b1 = xml.B + let b2 = xml.B2 () []