diff --git a/eng/Build.ps1 b/eng/Build.ps1 index 33b86aba6e0..f4b8149a169 100644 --- a/eng/Build.ps1 +++ b/eng/Build.ps1 @@ -286,11 +286,11 @@ function TestUsingMSBuild([string] $testProject, [string] $targetFramework, [str } function TestUsingXUnit([string] $testProject, [string] $targetFramework, [string]$testadapterpath) { - TestUsingMsBuild -testProject $testProject -targetFramework $targetFramework -testadapterpath $testadapterpath -noTestFilter $false + TestUsingMsBuild -testProject $testProject -targetFramework $targetFramework -testadapterpath $testadapterpath -noTestFilter $true } function TestUsingNUnit([string] $testProject, [string] $targetFramework, [string]$testadapterpath) { - TestUsingMsBuild -testProject $testProject -targetFramework $targetFramework -testadapterpath $testadapterpath -noTestFilter $true + TestUsingMsBuild -testProject $testProject -targetFramework $targetFramework -testadapterpath $testadapterpath -noTestFilter $false } function BuildCompiler() { diff --git a/tests/FSharp.Compiler.ComponentTests/FSharp.Compiler.ComponentTests.fsproj b/tests/FSharp.Compiler.ComponentTests/FSharp.Compiler.ComponentTests.fsproj index b82cc462d4c..f97a823591c 100644 --- a/tests/FSharp.Compiler.ComponentTests/FSharp.Compiler.ComponentTests.fsproj +++ b/tests/FSharp.Compiler.ComponentTests/FSharp.Compiler.ComponentTests.fsproj @@ -46,6 +46,7 @@ + diff --git a/tests/FSharp.Compiler.ComponentTests/Language/CompilerDirectiveTests.fs b/tests/FSharp.Compiler.ComponentTests/Language/CompilerDirectiveTests.fs index 32219920d93..0237a6d4d83 100644 --- a/tests/FSharp.Compiler.ComponentTests/Language/CompilerDirectiveTests.fs +++ b/tests/FSharp.Compiler.ComponentTests/Language/CompilerDirectiveTests.fs @@ -23,3 +23,13 @@ module ``Test Compiler Directives`` = """ |> compile |> shouldFail |> withSingleDiagnostic (Warning 213, Line 2, Col 1, Line 2, Col 10, "'' is not a valid assembly name") + +module ``Test compiler directives in FSI`` = + [] + let ``r# "" is invalid`` () = + Fsx""" +#r "" + """ |> ignoreWarnings + |> eval + |> shouldFail + |> withSingleDiagnostic (Error 2301, Line 2, Col 1, Line 2, Col 6, "'' is not a valid assembly name") \ No newline at end of file diff --git a/tests/FSharp.Compiler.ComponentTests/Scripting/Interactive.fs b/tests/FSharp.Compiler.ComponentTests/Scripting/Interactive.fs new file mode 100644 index 00000000000..a1eb8fa8842 --- /dev/null +++ b/tests/FSharp.Compiler.ComponentTests/Scripting/Interactive.fs @@ -0,0 +1,29 @@ +// Copyright (c) Microsoft Corporation. All Rights Reserved. See License.txt in the project root for license information. + +namespace FSharp.Compiler.ComponentTests.Scripting + +open Xunit +open FSharp.Test.Utilities.Compiler + +module ``Interactive tests`` = + [] + let ``Eval object value``() = + Fsx "1+1" + |> eval + |> shouldSucceed + |> withEvalTypeEquals typeof + |> withEvalValueEquals 2 + + +module ``External FSI tests`` = + [] + let ``Eval object value``() = + Fsx "1+1" + |> runFsi + |> shouldSucceed + + [] + let ``Invalid expression should fail``() = + Fsx "1+a" + |> runFsi + |> shouldFail \ No newline at end of file diff --git a/tests/FSharp.Test.Utilities/Compiler.fs b/tests/FSharp.Test.Utilities/Compiler.fs index d1312d19898..f63bf89e32b 100644 --- a/tests/FSharp.Test.Utilities/Compiler.fs +++ b/tests/FSharp.Test.Utilities/Compiler.fs @@ -2,6 +2,8 @@ namespace FSharp.Test.Utilities +open FSharp.Compiler.Interactive.Shell +open FSharp.Compiler.Scripting open FSharp.Compiler.SourceCodeServices open FSharp.Test.Utilities open FSharp.Test.Utilities.Assert @@ -68,17 +70,23 @@ module rec Compiler = Range: Range Message: string } + type EvalOutput = Result + type ExecutionOutput = { ExitCode: int StdOut: string StdErr: string } + type RunOutput = + | EvalOutput of EvalOutput + | ExecutionOutput of ExecutionOutput + type Output = { OutputPath: string option Dependencies: string list Adjust: int Diagnostics: ErrorInfo list - Output: ExecutionOutput option } + Output: RunOutput option } type TestResult = | Success of Output @@ -400,15 +408,71 @@ module rec Compiler = | None -> failwith "Compilation didn't produce any output. Unable to run. (did you forget to set output type to Exe?)" | Some p -> let (exitCode, output, errors) = CompilerAssert.ExecuteAndReturnResult (p, s.Dependencies, false) - let executionResult = { s with Output = Some { ExitCode = exitCode; StdOut = output; StdErr = errors } } + let executionResult = { s with Output = Some (ExecutionOutput { ExitCode = exitCode; StdOut = output; StdErr = errors }) } if exitCode = 0 then Success executionResult else Failure executionResult let compileAndRun = compile >> run - let compileExeAndRun = asExe >> compileAndRun + let private evalFSharp (fs: FSharpCompilationSource) : TestResult = + let source = getSource fs.Source + let options = fs.Options |> Array.ofList + + use script = new FSharpScript(additionalArgs=options) + + let ((evalresult: Result), (err: FSharpErrorInfo[])) = script.Eval(source) + + let diagnostics = err |> fromFSharpErrorInfo + + let result = + { OutputPath = None + Dependencies = [] + Adjust = 0 + Diagnostics = diagnostics + Output = Some(EvalOutput evalresult) } + + let (errors, warnings) = partitionErrors diagnostics + + let evalError = match evalresult with Ok _ -> false | _ -> true + + if evalError || errors.Length > 0 || (warnings.Length > 0 && not fs.IgnoreWarnings) then + Failure result + else + Success result + + let eval (cUnit: CompilationUnit) : TestResult = + match cUnit with + | FS fs -> evalFSharp fs + | _ -> failwith "Script evaluation is only supported for F#." + + let runFsi (cUnit: CompilationUnit) : TestResult = + match cUnit with + | FS fs -> + let source = getSource fs.Source + + let options = fs.Options |> Array.ofList + + let errors = CompilerAssert.RunScriptWithOptionsAndReturnResult options source + + let result = + { OutputPath = None + Dependencies = [] + Adjust = 0 + Diagnostics = [] + Output = None } + + if errors.Count > 0 then + let output = ExecutionOutput { + ExitCode = -1 + StdOut = String.Empty + StdErr = ((errors |> String.concat "\n").Replace("\r\n","\n")) } + Failure { result with Output = Some output } + else + Success result + | _ -> failwith "FSI running only supports F#." + let private createBaselineErrors (baseline: Baseline) actualErrors extension : unit = match baseline.SourceFilename with @@ -465,6 +529,7 @@ module rec Compiler = if not success then createBaselineErrors bsl actualIL "fs.il.err" Assert.Fail(errorMsg) + let verifyILBaseline (cUnit: CompilationUnit) : CompilationUnit = match cUnit with | FS fs -> @@ -541,7 +606,7 @@ module rec Compiler = let private assertResultsCategory (what: string) (selector: Output -> ErrorInfo list) (expected: ErrorInfo list) (result: TestResult) : TestResult = match result with - | Success r | Failure r -> + | Success r | Failure r -> assertErrors what r.Adjust (selector r) expected result @@ -605,7 +670,7 @@ module rec Compiler = result let withMessages (messages: string list) (result: TestResult) : TestResult = - checkErrorMessages messages (fun r -> r.Diagnostics) result + checkErrorMessages messages (fun r -> r.Diagnostics) result let withMessage (message: string) (result: TestResult) : TestResult = withMessages [message] result @@ -627,7 +692,10 @@ module rec Compiler = | Success r | Failure r -> match r.Output with | None -> failwith "Execution output is missing, cannot check exit code." - | Some o -> Assert.AreEqual(o.ExitCode, expectedExitCode, sprintf "Exit code was expected to be: %A, but got %A." expectedExitCode o.ExitCode) + | Some o -> + match o with + | ExecutionOutput e -> Assert.AreEqual(e.ExitCode, expectedExitCode, sprintf "Exit code was expected to be: %A, but got %A." expectedExitCode e.ExitCode) + | _ -> failwith "Cannot check exit code on this run result." result let private checkOutput (category: string) (substring: string) (selector: ExecutionOutput -> string) (result: TestResult) : TestResult = @@ -636,9 +704,12 @@ module rec Compiler = match r.Output with | None -> failwith (sprintf "Execution output is missing cannot check \"%A\"" category) | Some o -> - let where = selector o - if not (where.Contains(substring)) then - failwith (sprintf "\nThe following substring:\n %A\nwas not found in the %A\nOutput:\n %A" substring category where) + match o with + | ExecutionOutput e -> + let where = selector e + if not (where.Contains(substring)) then + failwith (sprintf "\nThe following substring:\n %A\nwas not found in the %A\nOutput:\n %A" substring category where) + | _ -> failwith "Cannot check output on this run result." result let withOutputContains (substring: string) (result: TestResult) : TestResult = @@ -649,3 +720,31 @@ module rec Compiler = let withStdErrContains (substring: string) (result: TestResult) : TestResult = checkOutput "STDERR" substring (fun o -> o.StdErr) result + + // TODO: probably needs a bit of simplification, + need to remove that pyramid of doom. + let private assertEvalOutput (selector: FsiValue -> 'T) (value: 'T) (result: TestResult) : TestResult = + match result with + | Success r | Failure r -> + match r.Output with + | None -> failwith "Execution output is missing cannot check value." + | Some o -> + match o with + | EvalOutput e -> + match e with + | Ok v -> + match v with + | None -> failwith "Cannot assert value of evaluation, since it is None." + | Some e -> Assert.AreEqual(value, (selector e)) + | Result.Error ex -> raise ex + | _ -> failwith "Only 'eval' output is supported." + result + + // TODO: Need to support for: + // STDIN, to test completions + // Contains + // Cancellation + let withEvalValueEquals (value: 'T) (result: TestResult) : TestResult = + assertEvalOutput (fun (x: FsiValue) -> x.ReflectionValue :?> 'T) value result + + let withEvalTypeEquals t (result: TestResult) : TestResult = + assertEvalOutput (fun (x: FsiValue) -> x.ReflectionType) t result \ No newline at end of file diff --git a/tests/FSharp.Test.Utilities/CompilerAssert.fs b/tests/FSharp.Test.Utilities/CompilerAssert.fs index 82599ffc142..843d6d58399 100644 --- a/tests/FSharp.Test.Utilities/CompilerAssert.fs +++ b/tests/FSharp.Test.Utilities/CompilerAssert.fs @@ -665,35 +665,39 @@ let main argv = 0""" static member CompileLibraryAndVerifyIL (source: string) (f: ILVerifier -> unit) = CompilerAssert.CompileLibraryAndVerifyILWithOptions [||] source f - static member RunScriptWithOptions options (source: string) (expectedErrorMessages: string list) = - lock gate <| fun () -> - // Intialize output and input streams - use inStream = new StringReader("") - use outStream = new StringWriter() - use errStream = new StringWriter() - - // Build command line arguments & start FSI session - let argv = [| "C:\\fsi.exe" |] - #if NETCOREAPP - let args = Array.append argv [|"--noninteractive"; "--targetprofile:netcore"|] - #else - let args = Array.append argv [|"--noninteractive"; "--targetprofile:mscorlib"|] - #endif - let allArgs = Array.append args options - - let fsiConfig = FsiEvaluationSession.GetDefaultConfiguration() - use fsiSession = FsiEvaluationSession.Create(fsiConfig, allArgs, inStream, outStream, errStream, collectible = true) - - let ch, errors = fsiSession.EvalInteractionNonThrowing source - - let errorMessages = ResizeArray() - errors - |> Seq.iter (fun error -> errorMessages.Add(error.Message)) + static member RunScriptWithOptionsAndReturnResult options (source: string) = + // Intialize output and input streams + use inStream = new StringReader("") + use outStream = new StringWriter() + use errStream = new StringWriter() + + // Build command line arguments & start FSI session + let argv = [| "C:\\fsi.exe" |] +#if NETCOREAPP + let args = Array.append argv [|"--noninteractive"; "--targetprofile:netcore"|] +#else + let args = Array.append argv [|"--noninteractive"; "--targetprofile:mscorlib"|] +#endif + let allArgs = Array.append args options - match ch with - | Choice2Of2 ex -> errorMessages.Add(ex.Message) - | _ -> () + let fsiConfig = FsiEvaluationSession.GetDefaultConfiguration() + use fsiSession = FsiEvaluationSession.Create(fsiConfig, allArgs, inStream, outStream, errStream, collectible = true) + let ch, errors = fsiSession.EvalInteractionNonThrowing source + + let errorMessages = ResizeArray() + errors + |> Seq.iter (fun error -> errorMessages.Add(error.Message)) + + match ch with + | Choice2Of2 ex -> errorMessages.Add(ex.Message) + | _ -> () + + errorMessages + + static member RunScriptWithOptions options (source: string) (expectedErrorMessages: string list) = + lock gate <| fun () -> + let errorMessages = CompilerAssert.RunScriptWithOptionsAndReturnResult options source if expectedErrorMessages.Length <> errorMessages.Count then Assert.Fail(sprintf "Expected error messages: %A \n\n Actual error messages: %A" expectedErrorMessages errorMessages) else diff --git a/tests/FSharp.Test.Utilities/FSharp.Test.Utilities.fsproj b/tests/FSharp.Test.Utilities/FSharp.Test.Utilities.fsproj index 3ea86adf999..0bfdf0255f4 100644 --- a/tests/FSharp.Test.Utilities/FSharp.Test.Utilities.fsproj +++ b/tests/FSharp.Test.Utilities/FSharp.Test.Utilities.fsproj @@ -32,6 +32,7 @@ + @@ -53,6 +54,7 @@ + Always diff --git a/tests/FSharp.Test.Utilities/ILChecker.fs b/tests/FSharp.Test.Utilities/ILChecker.fs index 91126acb5c8..bff010c5f43 100644 --- a/tests/FSharp.Test.Utilities/ILChecker.fs +++ b/tests/FSharp.Test.Utilities/ILChecker.fs @@ -134,7 +134,6 @@ module ILChecker = let verifyIL (dllFilePath: string) (expectedIL: string) = checkIL dllFilePath [expectedIL] - let verifyILAndReturnActual (dllFilePath: string) (expectedIL: string) = checkILAux' [] dllFilePath [expectedIL] let reassembleIL ilFilePath dllFilePath = let ilasmPath = config.ILASM