Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
24 changes: 16 additions & 8 deletions build.fsx
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,7 @@ let release = ReleaseNotes.load "RELEASE_NOTES.md"
// --------------------------------------------------------------------------------------
// Generate assembly info files with the right version & up-to-date information

Target.create "AssemblyInfo" <| fun _ ->
Target.create "AssemblyInfo" (fun _ ->
for file in !! "src/AssemblyInfo*.fs" do
let replace (oldValue:string) newValue (str:string) = str.Replace(oldValue, newValue)
let title =
Expand All @@ -66,43 +66,49 @@ Target.create "AssemblyInfo" <| fun _ ->
AssemblyInfo.Description summary
AssemblyInfo.Version version
AssemblyInfo.FileVersion version]
)

// --------------------------------------------------------------------------------------
// Clean build results

Target.create "Clean" <| fun _ ->
Target.create "Clean" (fun _ ->
seq {
yield! !!"**/bin"
yield! !!"**/obj"
} |> Shell.cleanDirs
)

Target.create "CleanDocs" <| fun _ ->
Target.create "CleanDocs" (fun _ ->
Shell.cleanDirs ["docs/output"]
)

let internetCacheFolder = Environment.GetFolderPath(Environment.SpecialFolder.InternetCache)

Target.create "CleanInternetCaches" <| fun _ ->
Target.create "CleanInternetCaches" (fun _ ->
Shell.cleanDirs [ internetCacheFolder @@ "DesignTimeURIs"
internetCacheFolder @@ "WorldBankSchema"
internetCacheFolder @@ "WorldBankRuntime"]
)

// --------------------------------------------------------------------------------------
// Build library & test projects

Target.create "Build" <| fun _ ->
Target.create "Build" (fun _ ->
"FSharp.Data.sln"
|> DotNet.build (fun o ->
{ o with Configuration = DotNet.BuildConfiguration.Release })
)

Target.create "RunTests" <| fun _ ->
Target.create "RunTests" (fun _ ->
"FSharp.Data.sln"
|> DotNet.test (fun o ->
{ o with Configuration = DotNet.BuildConfiguration.Release })
)

// --------------------------------------------------------------------------------------
// Build a NuGet package

Target.create "NuGet" <| fun _ ->
Target.create "NuGet" (fun _ ->
// Format the release notes
let releaseNotes = release.Notes |> String.concat "\n"

Expand Down Expand Up @@ -131,6 +137,7 @@ Target.create "NuGet" <| fun _ ->
MSBuildParams = { p.MSBuildParams with Properties = properties}
}
) "src/FSharp.Data/FSharp.Data.fsproj"
)

// --------------------------------------------------------------------------------------
// Generate the documentation
Expand All @@ -142,7 +149,7 @@ Target.create "GenerateDocs" (fun _ ->
// --------------------------------------------------------------------------------------
// Help

Target.create "Help" <| fun _ ->
Target.create "Help" (fun _ ->
printfn ""
printfn " Please specify the target by calling 'build -t <Target>'"
printfn ""
Expand All @@ -156,6 +163,7 @@ Target.create "Help" <| fun _ ->
printfn " Other targets:"
printfn " * CleanInternetCaches"
printfn ""
)

Target.create "All" ignore

Expand Down
149 changes: 76 additions & 73 deletions src/CommonProviderImplementation/Helpers.fs

Large diffs are not rendered by default.

41 changes: 22 additions & 19 deletions src/CommonRuntime/IO.fs
Original file line number Diff line number Diff line change
Expand Up @@ -60,13 +60,14 @@ type internal UriResolver =
let private logLock = obj()
let mutable private indentation = 0

let private appendToLogMultiple logFile lines = lock logLock <| fun () ->
let path = __SOURCE_DIRECTORY__ + "/../../" + logFile
use stream = File.Open(path, FileMode.Append, FileAccess.Write, FileShare.ReadWrite)
use writer = new StreamWriter(stream)
for (line:string) in lines do
writer.WriteLine(line.Replace("\r", null).Replace("\n","\\n"))
writer.Flush()
let private appendToLogMultiple logFile lines =
lock logLock (fun () ->
let path = __SOURCE_DIRECTORY__ + "/../../" + logFile
use stream = File.Open(path, FileMode.Append, FileAccess.Write, FileShare.ReadWrite)
use writer = new StreamWriter(stream)
for (line:string) in lines do
writer.WriteLine(line.Replace("\r", null).Replace("\n","\\n"))
writer.Flush())

let private appendToLog logFile line =
appendToLogMultiple logFile [line]
Expand Down Expand Up @@ -168,7 +169,7 @@ let watchForChanges path (owner, onChange) =

let watcher =

lock watchers <| fun () ->
lock watchers (fun () ->

match watchers.TryGetValue(path) with
| true, watcher ->
Expand All @@ -184,12 +185,14 @@ let watchForChanges path (owner, onChange) =
watcher.Subscribe(owner, onChange)
watchers.Add(path, watcher)
watcher
)

{ new IDisposable with
member __.Dispose() =
lock watchers <| fun () ->
lock watchers (fun () ->
if watcher.Unsubscribe(owner) then
watchers.Remove(path) |> ignore
watchers.Remove(path) |> ignore
)
}

/// Opens a stream to the uri using the uriResolver resolution rules
Expand Down Expand Up @@ -221,21 +224,21 @@ let internal asyncRead (uriResolver:UriResolver) formatName encodingStr (uri:Uri
return new StreamReader(file, encoding) :> TextReader
}, Some path

let private withUri uri f =
let private withUri uri =
match Uri.TryCreate(uri, UriKind.RelativeOrAbsolute) with
| false, _ -> failwithf "Invalid uri: %s" uri
| true, uri -> f uri
| true, uri -> uri

/// Returns a TextReader for the uri using the runtime resolution rules
let asyncReadTextAtRuntime forFSI defaultResolutionFolder resolutionFolder formatName encodingStr uri =
withUri uri <| fun uri ->
let resolver = UriResolver.Create((if forFSI then RuntimeInFSI else Runtime),
defaultResolutionFolder, resolutionFolder)
asyncRead resolver formatName encodingStr uri |> fst
let uri = withUri uri
let resolver = UriResolver.Create((if forFSI then RuntimeInFSI else Runtime),
defaultResolutionFolder, resolutionFolder)
asyncRead resolver formatName encodingStr uri |> fst

/// Returns a TextReader for the uri using the designtime resolution rules
let asyncReadTextAtRuntimeWithDesignTimeRules defaultResolutionFolder resolutionFolder formatName encodingStr uri =
withUri uri <| fun uri ->
let resolver = UriResolver.Create(DesignTime, defaultResolutionFolder, resolutionFolder)
asyncRead resolver formatName encodingStr uri |> fst
let uri = withUri uri
let resolver = UriResolver.Create(DesignTime, defaultResolutionFolder, resolutionFolder)
asyncRead resolver formatName encodingStr uri |> fst

2 changes: 1 addition & 1 deletion src/CommonRuntime/StructuralInference.fs
Original file line number Diff line number Diff line change
Expand Up @@ -269,7 +269,7 @@ let inferPrimitiveType (cultureInfo:CultureInfo) (value : string) =
| Parse TextConversions.AsDateTimeOffset dateTimeOffset when not (isFakeDate dateTimeOffset.UtcDateTime value) -> typeof<DateTimeOffset>
| Parse TextConversions.AsDateTime date when not (isFakeDate date value) -> typeof<DateTime>
| Parse TextConversions.AsDecimal _ -> typeof<decimal>
| Parse (TextConversions.AsFloat [| |] (*useNoneForMissingValues*)false) _ -> typeof<float>
| Parse (TextConversions.AsFloat [| |] false) _ -> typeof<float>
| Parse asGuid _ -> typeof<Guid>
| _ -> typeof<string>

Expand Down
3 changes: 2 additions & 1 deletion src/CommonRuntime/StructuralTypes.fs
Original file line number Diff line number Diff line change
Expand Up @@ -88,7 +88,8 @@ type InferedType =
| Record(name, props, false) -> Record(name, props, true)
| Json(typ, false) -> Json(typ, true)
| Collection (order, types) ->
Collection (order, Map.map (fun _ (mult, typ) -> (if mult = Single then OptionalSingle else mult), typ) types)
let typesR = types |> Map.map (fun _ (mult, typ) -> (if mult = Single then OptionalSingle else mult), typ)
Collection (order, typesR)
| Top -> failwith "EnsuresHandlesMissingValues: unexpected InferedType.Top"

member x.DropOptionality() =
Expand Down
2 changes: 1 addition & 1 deletion src/CommonRuntime/TextRuntime.fs
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,7 @@ type TextRuntime =

static member ConvertFloat(cultureStr, missingValuesStr, text) =
text |> Option.bind (TextConversions.AsFloat (TextRuntime.GetMissingValues missingValuesStr)
(*useNoneForMissingValues*)true
true
(TextRuntime.GetCulture cultureStr))

static member ConvertBoolean(text) =
Expand Down
6 changes: 3 additions & 3 deletions src/Csv/CsvExtensions.fs
Original file line number Diff line number Diff line change
Expand Up @@ -9,8 +9,8 @@ open System.Runtime.InteropServices
open FSharp.Data
open FSharp.Data.Runtime

[<Extension>]
/// Extension methods with conversions from strings to other types
[<Extension>]
type StringExtensions =

[<Extension>]
Expand Down Expand Up @@ -38,7 +38,7 @@ type StringExtensions =
static member AsFloat(x:String, [<Optional>] ?cultureInfo, [<Optional>] ?missingValues) =
let cultureInfo = defaultArg cultureInfo CultureInfo.InvariantCulture
let missingValues = defaultArg missingValues TextConversions.DefaultMissingValues
match TextConversions.AsFloat missingValues (*useNoneForMissingValues*)false cultureInfo x with
match TextConversions.AsFloat missingValues false cultureInfo x with
| Some f -> f
| _ -> failwithf "Not a float: %s" x

Expand Down Expand Up @@ -75,8 +75,8 @@ type StringExtensions =
| Some g -> g
| _ -> failwithf "Not a guid: %s" x

[<CompilationRepresentation(CompilationRepresentationFlags.ModuleSuffix)>]
/// Provides the dynamic operator for getting column values by name from CSV rows
[<CompilationRepresentation(CompilationRepresentationFlags.ModuleSuffix)>]
module CsvExtensions =

/// Get the value of a column by name from a CSV row
Expand Down
18 changes: 12 additions & 6 deletions src/Csv/CsvGenerator.fs
Original file line number Diff line number Diff line change
Expand Up @@ -30,13 +30,16 @@ module internal CsvTypeBuilder =
let fields = inferredFields |> List.mapi (fun index field ->
let typ, typWithoutMeasure, conv, convBack = ConversionsGenerator.convertStringValue missingValuesStr cultureStr field
let propertyName = NameUtils.capitalizeFirstLetter field.Name
{ TypeForTuple = typWithoutMeasure
ProvidedProperty = ProvidedProperty(propertyName, typ, getterCode = fun (Singleton row) ->
let prop = ProvidedProperty(propertyName, typ, getterCode = fun (Singleton row) ->
match inferredFields with
| [ _ ] -> row
| _ -> Expr.TupleGet(row, index))
Convert = fun rowVarExpr -> conv <@ TextConversions.AsString((%%rowVarExpr:string[]).[index]) @>
ConvertBack = fun rowVarExpr -> convBack (match inferredFields with [ _ ] -> rowVarExpr | _ -> Expr.TupleGet(rowVarExpr, index))
let convert rowVarExpr = conv <@ TextConversions.AsString((%%rowVarExpr:string[]).[index]) @>
let convertBack rowVarExpr = convBack (match inferredFields with [ _ ] -> rowVarExpr | _ -> Expr.TupleGet(rowVarExpr, index))
{ TypeForTuple = typWithoutMeasure
ProvidedProperty = prop
Convert = convert
ConvertBack = convertBack
ProvidedParameter = ProvidedParameter(NameUtils.niceCamelName propertyName, typ) } )

// The erased row type will be a tuple of all the field types (without the units of measure). If there is a single column then it is just the column type.
Expand All @@ -48,10 +51,13 @@ module internal CsvTypeBuilder =
let rowType = ProvidedTypeDefinition("Row", Some rowErasedType, hideObjectMethods = true, nonNullable = true)

let ctor =
ProvidedConstructor([ for field in fields -> field.ProvidedParameter ], invokeCode = fun args ->
let parameters = [ for field in fields -> field.ProvidedParameter ]
let invoke args =
match args with
| [ arg ] -> arg
| _ -> Expr.NewTuple(args))
| _ -> Expr.NewTuple(args)
ProvidedConstructor(parameters, invokeCode = invoke)

rowType.AddMember ctor

// Each property of the generated row type will simply be a tuple get
Expand Down
12 changes: 6 additions & 6 deletions src/Csv/CsvInference.fs
Original file line number Diff line number Diff line change
Expand Up @@ -50,11 +50,11 @@ let private overrideByNameRegex = lazy Regex(@"^(?<name>.+)(->(?<newName>.+)(=(?

[<RequireQualifiedAccess>]
type private SchemaParseResult =
| Name of string
| NameAndUnit of string * Type
| Full of PrimitiveInferedProperty
| FullByName of PrimitiveInferedProperty * (*originalName*)string
| Rename of (*name*)string * (*originalName*)string
| Name of name: string
| NameAndUnit of name: string * unitOfMeasure: Type
| Full of property: PrimitiveInferedProperty
| FullByName of property: PrimitiveInferedProperty * originalName: string
| Rename of name: string * originalName: string

let private asOption = function true, x -> Some x | false, _ -> None

Expand Down Expand Up @@ -263,7 +263,7 @@ let internal inferType (headerNamesAndUnits:_[]) schema (rows:seq<_>) inferRows
// all the columns types are already set, so all the rows will be the same
types |> List.head
else
List.reduce (StructuralInference.subtypeInfered ((*allowEmptyValues*)not preferOptionals)) types
List.reduce (StructuralInference.subtypeInfered (not preferOptionals)) types

inferedType, schema

Expand Down
25 changes: 13 additions & 12 deletions src/Csv/CsvProvider.fs
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,8 @@ type public CsvProvider(cfg:TypeProviderConfig) as this =

let getSpec (extension:string) value =

use sampleCsv = using (IO.logTime "Parsing" sample) <| fun _ ->
use sampleCsv =
use _holder = IO.logTime "Parsing" sample
let separators =
if String.IsNullOrEmpty separators && extension.ToLowerInvariant() = ".tsv"
then "\t" else separators
Expand All @@ -72,11 +73,12 @@ type public CsvProvider(cfg:TypeProviderConfig) as this =

let separators = sampleCsv.Separators

let inferredFields = using (IO.logTime "Inference" sample) <| fun _ ->
let inferredFields =
use _holder = IO.logTime "Inference" sample
sampleCsv.InferColumnTypes(inferRows, TextRuntime.GetMissingValues missingValuesStr, TextRuntime.GetCulture cultureStr, schema,
assumeMissingValues, preferOptionals, ProviderHelpers.unitsOfMeasureProvider)

using (IO.logTime "TypeGeneration" sample) <| fun _ ->
use _holder = IO.logTime "TypeGeneration" sample

let csvType, csvErasedType, rowType, stringArrayToRow, rowToStringArray =
inferredFields
Expand All @@ -91,22 +93,21 @@ type public CsvProvider(cfg:TypeProviderConfig) as this =
| None -> <@@ None: string[] option @@>
| Some headers -> Expr.NewArray(typeof<string>, headers |> Array.map (fun h -> Expr.Value(h)) |> List.ofArray) |> (fun x-> <@@ Some (%%x : string[]) @@>)

let ctor =
ProvidedConstructor(
[ ProvidedParameter("rows", paramType) ],
invokeCode = (fun (Singleton paramValue) ->
let body = csvErasedType?CreateEmpty () (Expr.Var rowToStringArrayVar, paramValue, headers, sampleCsv.NumberOfColumns, separators, quote)
Expr.Let(rowToStringArrayVar, rowToStringArray, body)))
let ctorCode (Singleton paramValue: Expr list) =
let body = csvErasedType?CreateEmpty () (Expr.Var rowToStringArrayVar, paramValue, headers, sampleCsv.NumberOfColumns, separators, quote)
Expr.Let(rowToStringArrayVar, rowToStringArray, body)
let ctor = ProvidedConstructor([ ProvidedParameter("rows", paramType) ], invokeCode = ctorCode)
csvType.AddMember(ctor)

let parseRowsCode (Singleton text: Expr list) =
let body = csvErasedType?ParseRows () (text, Expr.Var stringArrayToRowVar, separators, quote, ignoreErrors)
Expr.Let(stringArrayToRowVar, stringArrayToRow, body)
let parseRows =
ProvidedMethod("ParseRows",
[ProvidedParameter("text", typeof<string>)],
rowType.MakeArrayType(),
isStatic = true,
invokeCode = fun (Singleton text) ->
let body = csvErasedType?ParseRows () (text, Expr.Var stringArrayToRowVar, separators, quote, ignoreErrors)
Expr.Let(stringArrayToRowVar, stringArrayToRow, body))
invokeCode = parseRowsCode)
csvType.AddMember parseRows

{ GeneratedType = csvType
Expand Down
3 changes: 2 additions & 1 deletion src/Html/HtmlGenerator.fs
Original file line number Diff line number Diff line change
Expand Up @@ -26,9 +26,10 @@ module internal HtmlGenerator =
let private getPropertyName = NameUtils.capitalizeFirstLetter

let private typeNameGenerator() =
NameUtils.uniqueGenerator <| fun s ->
NameUtils.uniqueGenerator (fun s ->
HtmlParser.invalidTypeNameRegex.Value.Replace(s, " ")
|> NameUtils.nicePascalName
)

let private createTableType getTableTypeName (inferenceParameters, missingValuesStr, cultureStr) (table:HtmlTable) =

Expand Down
10 changes: 6 additions & 4 deletions src/Html/HtmlProvider.fs
Original file line number Diff line number Diff line change
Expand Up @@ -36,10 +36,12 @@ type public HtmlProvider(cfg:TypeProviderConfig) as this =

let getSpec _ value =

let doc = using (IO.logTime "Parsing" sample) <| fun _ ->
let doc =
use _holder = IO.logTime "Parsing" sample
HtmlDocument.Parse value

let htmlType = using (IO.logTime "Inference" sample) <| fun _ ->
let htmlType =
use _holder = IO.logTime "Inference" sample
let inferenceParameters : HtmlInference.Parameters =
{ MissingValues = TextRuntime.GetMissingValues missingValuesStr
CultureInfo = TextRuntime.GetCulture cultureStr
Expand All @@ -49,7 +51,7 @@ type public HtmlProvider(cfg:TypeProviderConfig) as this =
|> HtmlRuntime.getHtmlObjects (Some inferenceParameters) includeLayoutTables
|> HtmlGenerator.generateTypes asm ns typeName (inferenceParameters, missingValuesStr, cultureStr)

using (IO.logTime "TypeGeneration" sample) <| fun _ ->
use _holder = IO.logTime "TypeGeneration" sample

{ GeneratedType = htmlType
RepresentationType = htmlType
Expand All @@ -59,7 +61,7 @@ type public HtmlProvider(cfg:TypeProviderConfig) as this =
CreateFromValue = None
}

generateType "HTML" (Sample sample) getSpec this cfg encodingStr resolutionFolder resource typeName (*maxNumberOfRows*)None
generateType "HTML" (Sample sample) getSpec this cfg encodingStr resolutionFolder resource typeName None

// Add static parameter that specifies the API we want to get (compile-time)
let parameters =
Expand Down
Loading