From 3b142fe35a52d028f8440a63d3116092e5731cca Mon Sep 17 00:00:00 2001 From: TIHan Date: Tue, 16 Apr 2019 11:02:38 -0700 Subject: [PATCH 1/5] Trying to build benchmarks --- benchmarks/CompilerServiceBenchmarks/Program.fs | 2 +- benchmarks/Directory.Build.props | 3 --- benchmarks/Directory.Build.targets | 3 --- 3 files changed, 1 insertion(+), 7 deletions(-) delete mode 100644 benchmarks/Directory.Build.props delete mode 100644 benchmarks/Directory.Build.targets diff --git a/benchmarks/CompilerServiceBenchmarks/Program.fs b/benchmarks/CompilerServiceBenchmarks/Program.fs index 386d8d95fb5..7056b293936 100644 --- a/benchmarks/CompilerServiceBenchmarks/Program.fs +++ b/benchmarks/CompilerServiceBenchmarks/Program.fs @@ -7,7 +7,7 @@ open FSharp.Compiler.Text open FSharp.Compiler.AbstractIL open FSharp.Compiler.AbstractIL.IL open FSharp.Compiler.AbstractIL.ILBinaryReader -open CodeAnalysis.Text +open Microsoft.CodeAnalysis.Text open BenchmarkDotNet.Attributes open BenchmarkDotNet.Running diff --git a/benchmarks/Directory.Build.props b/benchmarks/Directory.Build.props deleted file mode 100644 index bb8eac309b1..00000000000 --- a/benchmarks/Directory.Build.props +++ /dev/null @@ -1,3 +0,0 @@ - - - diff --git a/benchmarks/Directory.Build.targets b/benchmarks/Directory.Build.targets deleted file mode 100644 index ccd47cc0a9a..00000000000 --- a/benchmarks/Directory.Build.targets +++ /dev/null @@ -1,3 +0,0 @@ - - - From 3bdeff02ca03fdd2488dd6b3d467a2600f6fa714 Mon Sep 17 00:00:00 2001 From: TIHan Date: Wed, 17 Apr 2019 14:11:19 -0700 Subject: [PATCH 2/5] Benchmark testing 100 referenced projects in the compiler service --- .../CompilerServiceBenchmarks/Program.fs | 147 ++++++++++++++++-- 1 file changed, 131 insertions(+), 16 deletions(-) diff --git a/benchmarks/CompilerServiceBenchmarks/Program.fs b/benchmarks/CompilerServiceBenchmarks/Program.fs index 7056b293936..68bf04c2ea5 100644 --- a/benchmarks/CompilerServiceBenchmarks/Program.fs +++ b/benchmarks/CompilerServiceBenchmarks/Program.fs @@ -80,6 +80,48 @@ type SourceText with member this.ToFSharpSourceText() = SourceText.weakTable.GetValue(this, Runtime.CompilerServices.ConditionalWeakTable<_,_>.CreateValueCallback(SourceText.create)) +[] +module Helpers = + + let createProject name referencedProjects = + let tmpPath = Path.GetTempPath() + let file = Path.Combine(tmpPath, Path.ChangeExtension(name, ".fs")) + { + ProjectFileName = name + ProjectId = None + SourceFiles = [|file|] + OtherOptions = [|"--optimize+"|] + ReferencedProjects = + referencedProjects + |> List.map (fun x -> (x.ProjectFileName, x)) + |> Array.ofList + IsIncompleteTypeCheckEnvironment = false + UseScriptResolutionRules = false + LoadTime = DateTime() + UnresolvedReferences = None + OriginalLoadReferences = [] + ExtraProjectInfo = None + Stamp = None + } + + let generateSourceCode moduleName = + sprintf """ +module %s = + + type %s = + + val X : int + + val Y : int + + val Z : int + + let function%s (x: %s) = + let x = 1 + let y = 2 + let z = x + y + z""" moduleName moduleName moduleName moduleName + [] type CompilerService() = @@ -112,7 +154,7 @@ type CompilerService() = [] member __.Setup() = match checkerOpt with - | None -> checkerOpt <- Some(FSharpChecker.Create()) + | None -> checkerOpt <- Some(FSharpChecker.Create(projectCacheSize = 200)) | _ -> () match sourceOpt with @@ -127,18 +169,9 @@ type CompilerService() = |> Array.map (fun x -> (x.Location)) |> Some | _ -> () - - [] - member __.ParsingSetup() = - match checkerOpt with - | None -> failwith "no checker" - | Some(checker) -> - checker.InvalidateAll() - checker.ClearLanguageServiceRootCachesAndCollectAndFinalizeAllTransients() - checker.ParseFile("dummy.fs", SourceText.ofString "dummy", parsingOptions) |> Async.RunSynchronously |> ignore [] - member __.Parsing() = + member __.ParsingTypeCheckerFs() = match checkerOpt, sourceOpt with | None, _ -> failwith "no checker" | _, None -> failwith "no source" @@ -146,11 +179,15 @@ type CompilerService() = let results = checker.ParseFile("TypeChecker.fs", source.ToFSharpSourceText(), parsingOptions) |> Async.RunSynchronously if results.ParseHadErrors then failwithf "parse had errors: %A" results.Errors - [] - member __.ILReadingSetup() = - // With caching, performance increases an order of magnitude when re-reading an ILModuleReader. - // Clear it for benchmarking. - ClearAllILModuleReaderCache() + [] + member __.ParsingTypeCheckerFsSetup() = + match checkerOpt with + | None -> failwith "no checker" + | Some(checker) -> + checker.InvalidateAll() + checker.ClearLanguageServiceRootCachesAndCollectAndFinalizeAllTransients() + checker.ParseFile("dummy.fs", SourceText.ofString "dummy", parsingOptions) |> Async.RunSynchronously |> ignore + ClearAllILModuleReaderCache() [] member __.ILReading() = @@ -198,6 +235,84 @@ type CompilerService() = ) ) + [] + member __.ILReadingSetup() = + // With caching, performance increases an order of magnitude when re-reading an ILModuleReader. + // Clear it for benchmarking. + ClearAllILModuleReaderCache() + + member val TypeCheckFileWith100ReferencedProjectsOptions = + createProject "MainProject" + [ for i = 1 to 100 do + yield createProject ("ReferencedProject" + string i) [] + ] + + member this.TypeCheckFileWith100ReferencedProjectsRun() = + let options = this.TypeCheckFileWith100ReferencedProjectsOptions + let file = options.SourceFiles.[0] + + match checkerOpt with + | None -> failwith "no checker" + | Some checker -> + let parseResult, checkResult = + checker.ParseAndCheckFileInProject(file, 0, SourceText.ofString (File.ReadAllText(file)), (* set the stamp to 0L on each run so we don't evaluate the whole project again *) { options with Stamp = Some 0L }) + |> Async.RunSynchronously + + if parseResult.Errors.Length > 0 then + failwithf "%A" parseResult.Errors + + match checkResult with + | FSharpCheckFileAnswer.Aborted -> failwith "aborted" + | FSharpCheckFileAnswer.Succeeded checkFileResult -> + + if checkFileResult.Errors.Length > 0 then + failwithf "%A" checkFileResult.Errors + + [] + member this.TypeCheckFileWith100ReferencedProjectsSetup() = + this.TypeCheckFileWith100ReferencedProjectsOptions.SourceFiles + |> Seq.iter (fun file -> + File.WriteAllText(file, generateSourceCode (Path.GetFileNameWithoutExtension(file))) + ) + + this.TypeCheckFileWith100ReferencedProjectsOptions.ReferencedProjects + |> Seq.iter (fun (_, referencedProjectOptions) -> + referencedProjectOptions.SourceFiles + |> Seq.iter (fun file -> + File.WriteAllText(file, generateSourceCode (Path.GetFileNameWithoutExtension(file))) + ) + ) + + this.TypeCheckFileWith100ReferencedProjectsRun() + + [] + member this.TypeCheckFileWith100ReferencedProjects() = + // Because the checker's projectcachesize is set to 200, this should be fast. + // If set to 3, it will be almost as slow as re-evaluating all project and it's projects references on setup; this could be a bug or not what we want. + this.TypeCheckFileWith100ReferencedProjectsRun() + + [] + member this.TypeCheckFileWith100ReferencedProjectsCleanup() = + this.TypeCheckFileWith100ReferencedProjectsOptions.SourceFiles + |> Seq.iter (fun file -> + try File.Delete(file) with | _ -> () + ) + + this.TypeCheckFileWith100ReferencedProjectsOptions.ReferencedProjects + |> Seq.iter (fun (_, referencedProjectOptions) -> + referencedProjectOptions.SourceFiles + |> Seq.iter (fun file -> + try File.Delete(file) with | _ -> () + ) + ) + + match checkerOpt with + | None -> failwith "no checker" + | Some(checker) -> + checker.InvalidateAll() + checker.ClearLanguageServiceRootCachesAndCollectAndFinalizeAllTransients() + ClearAllILModuleReaderCache() + [] let main argv = let _ = BenchmarkRunner.Run() From 162f05306073954f34efba884e2be77c63be63ee Mon Sep 17 00:00:00 2001 From: TIHan Date: Wed, 17 Apr 2019 17:03:15 -0700 Subject: [PATCH 3/5] Fixed project reference bit --- .../CompilerServiceBenchmarks/Program.fs | 25 ++++++++++--------- 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/benchmarks/CompilerServiceBenchmarks/Program.fs b/benchmarks/CompilerServiceBenchmarks/Program.fs index 68bf04c2ea5..da8f7f753f6 100644 --- a/benchmarks/CompilerServiceBenchmarks/Program.fs +++ b/benchmarks/CompilerServiceBenchmarks/Program.fs @@ -87,10 +87,11 @@ module Helpers = let tmpPath = Path.GetTempPath() let file = Path.Combine(tmpPath, Path.ChangeExtension(name, ".fs")) { - ProjectFileName = name + ProjectFileName = Path.Combine(tmpPath, Path.ChangeExtension(name, ".dll")) ProjectId = None SourceFiles = [|file|] - OtherOptions = [|"--optimize+"|] + OtherOptions = + Array.append [|"--optimize+"; "--target:library" |] (referencedProjects |> Array.ofList |> Array.map (fun x -> "-r:" + x.ProjectFileName)) ReferencedProjects = referencedProjects |> List.map (fun x -> (x.ProjectFileName, x)) @@ -106,21 +107,21 @@ module Helpers = let generateSourceCode moduleName = sprintf """ -module %s = +module Benchmark.%s - type %s = +type %s = - val X : int + val X : int - val Y : int + val Y : int - val Z : int + val Z : int - let function%s (x: %s) = - let x = 1 - let y = 2 - let z = x + y - z""" moduleName moduleName moduleName moduleName +let function%s (x: %s) = + let x = 1 + let y = 2 + let z = x + y + z""" moduleName moduleName moduleName moduleName [] type CompilerService() = From 0d4963aa07dc95469ae13e93fec88be115a4646d Mon Sep 17 00:00:00 2001 From: TIHan Date: Wed, 17 Apr 2019 19:27:08 -0700 Subject: [PATCH 4/5] Change stamp --- benchmarks/CompilerServiceBenchmarks/Program.fs | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/benchmarks/CompilerServiceBenchmarks/Program.fs b/benchmarks/CompilerServiceBenchmarks/Program.fs index da8f7f753f6..37b926b01e7 100644 --- a/benchmarks/CompilerServiceBenchmarks/Program.fs +++ b/benchmarks/CompilerServiceBenchmarks/Program.fs @@ -83,7 +83,7 @@ type SourceText with [] module Helpers = - let createProject name referencedProjects = + let createProject name referencedProjects stamp = let tmpPath = Path.GetTempPath() let file = Path.Combine(tmpPath, Path.ChangeExtension(name, ".fs")) { @@ -102,7 +102,7 @@ module Helpers = UnresolvedReferences = None OriginalLoadReferences = [] ExtraProjectInfo = None - Stamp = None + Stamp = Some 0L (* set the stamp to 0L on each run so we don't evaluate the whole project again *) } let generateSourceCode moduleName = @@ -245,7 +245,8 @@ type CompilerService() = member val TypeCheckFileWith100ReferencedProjectsOptions = createProject "MainProject" [ for i = 1 to 100 do - yield createProject ("ReferencedProject" + string i) [] + yield + createProject ("ReferencedProject" + string i) [] ] member this.TypeCheckFileWith100ReferencedProjectsRun() = @@ -256,7 +257,7 @@ type CompilerService() = | None -> failwith "no checker" | Some checker -> let parseResult, checkResult = - checker.ParseAndCheckFileInProject(file, 0, SourceText.ofString (File.ReadAllText(file)), (* set the stamp to 0L on each run so we don't evaluate the whole project again *) { options with Stamp = Some 0L }) + checker.ParseAndCheckFileInProject(file, 0, SourceText.ofString (File.ReadAllText(file)), options) |> Async.RunSynchronously if parseResult.Errors.Length > 0 then From ac16b54a103568e8f5e19a6aec21f0485cbf6f03 Mon Sep 17 00:00:00 2001 From: TIHan Date: Wed, 17 Apr 2019 20:20:13 -0700 Subject: [PATCH 5/5] Fixed build --- benchmarks/CompilerServiceBenchmarks/Program.fs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/benchmarks/CompilerServiceBenchmarks/Program.fs b/benchmarks/CompilerServiceBenchmarks/Program.fs index 37b926b01e7..201856e3f98 100644 --- a/benchmarks/CompilerServiceBenchmarks/Program.fs +++ b/benchmarks/CompilerServiceBenchmarks/Program.fs @@ -83,7 +83,7 @@ type SourceText with [] module Helpers = - let createProject name referencedProjects stamp = + let createProject name referencedProjects = let tmpPath = Path.GetTempPath() let file = Path.Combine(tmpPath, Path.ChangeExtension(name, ".fs")) {