diff --git a/src/fsharp/CompilerConfig.fs b/src/fsharp/CompilerConfig.fs index c7dd623fad6..0bdc81698c5 100644 --- a/src/fsharp/CompilerConfig.fs +++ b/src/fsharp/CompilerConfig.fs @@ -270,6 +270,11 @@ type LStatus = | Unprocessed | Processed +type TokenizeOption = + | AndCompile + | Only + | Unfiltered + type PackageManagerLine = { Directive: Directive LineStatus: LStatus @@ -367,7 +372,7 @@ type TcConfigBuilder = mutable importAllReferencesOnly: bool mutable simulateException: string option mutable printAst: bool - mutable tokenizeOnly: bool + mutable tokenize: TokenizeOption mutable testInteractionParser: bool mutable reportNumDecls: bool mutable printSignature: bool @@ -573,7 +578,7 @@ type TcConfigBuilder = importAllReferencesOnly = false simulateException = None printAst = false - tokenizeOnly = false + tokenize = TokenizeOption.AndCompile testInteractionParser = false reportNumDecls = false printSignature = false @@ -960,7 +965,7 @@ type TcConfig private (data: TcConfigBuilder, validate: bool) = member x.simulateException = data.simulateException member x.printAst = data.printAst member x.targetFrameworkVersion = targetFrameworkVersionValue - member x.tokenizeOnly = data.tokenizeOnly + member x.tokenize = data.tokenize member x.testInteractionParser = data.testInteractionParser member x.reportNumDecls = data.reportNumDecls member x.printSignature = data.printSignature diff --git a/src/fsharp/CompilerConfig.fsi b/src/fsharp/CompilerConfig.fsi index 4c8831e6ba6..acb5f140b3d 100644 --- a/src/fsharp/CompilerConfig.fsi +++ b/src/fsharp/CompilerConfig.fsi @@ -121,6 +121,11 @@ type LStatus = | Unprocessed | Processed +type TokenizeOption = + | AndCompile + | Only + | Unfiltered + type PackageManagerLine = { Directive: Directive LineStatus: LStatus @@ -182,7 +187,7 @@ type TcConfigBuilder = mutable importAllReferencesOnly: bool mutable simulateException: string option mutable printAst: bool - mutable tokenizeOnly: bool + mutable tokenize: TokenizeOption mutable testInteractionParser: bool mutable reportNumDecls: bool mutable printSignature: bool @@ -374,7 +379,7 @@ type TcConfig = member importAllReferencesOnly: bool member simulateException: string option member printAst: bool - member tokenizeOnly: bool + member tokenize: TokenizeOption member testInteractionParser: bool member reportNumDecls: bool member printSignature: bool diff --git a/src/fsharp/CompilerOptions.fs b/src/fsharp/CompilerOptions.fs index b04c13031ab..51339cb8453 100644 --- a/src/fsharp/CompilerOptions.fs +++ b/src/fsharp/CompilerOptions.fs @@ -1139,9 +1139,14 @@ let internalFlags (tcConfigB:TcConfigBuilder) = CompilerOption ("tokenize", tagNone, - OptionUnit (fun () -> tcConfigB.tokenizeOnly <- true), + OptionUnit (fun () -> tcConfigB.tokenize <- TokenizeOption.Only), Some(InternalCommandLineOption("--tokenize", rangeCmdArgs)), None) + CompilerOption + ("tokenize-unfiltered", tagNone, + OptionUnit (fun () -> tcConfigB.tokenize <- TokenizeOption.Unfiltered), + Some(InternalCommandLineOption("--tokenize-unfiltered", rangeCmdArgs)), None) + CompilerOption ("testInteractionParser", tagNone, OptionUnit (fun () -> tcConfigB.testInteractionParser <- true), diff --git a/src/fsharp/ParseAndCheckInputs.fs b/src/fsharp/ParseAndCheckInputs.fs index 351561f585d..3fdcad9d519 100644 --- a/src/fsharp/ParseAndCheckInputs.fs +++ b/src/fsharp/ParseAndCheckInputs.fs @@ -289,11 +289,13 @@ let ParseInput (lexer, errorLogger: ErrorLogger, lexbuf: UnicodeLexing.Lexbuf, d let filteringErrorLogger = GetErrorLoggerFilteringByScopedPragmas(false, scopedPragmas, errorLogger) delayLogger.CommitDelayedDiagnostics filteringErrorLogger +type Tokenizer = unit -> Parser.token + // Show all tokens in the stream, for testing purposes -let ShowAllTokensAndExit (shortFilename, tokenizer: LexFilter.LexFilter, lexbuf: LexBuffer) = +let ShowAllTokensAndExit (shortFilename, tokenizer: Tokenizer, lexbuf: LexBuffer) = while true do printf "tokenize - getting one token from %s\n" shortFilename - let t = tokenizer.GetToken() + let t = tokenizer () printf "tokenize - got %s @ %a\n" (Parser.token_to_string t) outputRange lexbuf.LexemeRange match t with | Parser.EOF _ -> exit 0 @@ -301,11 +303,11 @@ let ShowAllTokensAndExit (shortFilename, tokenizer: LexFilter.LexFilter, lexbuf: if lexbuf.IsPastEndOfStream then printf "!!! at end of stream\n" // Test one of the parser entry points, just for testing purposes -let TestInteractionParserAndExit (tokenizer: LexFilter.LexFilter, lexbuf: LexBuffer) = +let TestInteractionParserAndExit (tokenizer: Tokenizer, lexbuf: LexBuffer) = while true do - match (Parser.interaction (fun _ -> tokenizer.GetToken()) lexbuf) with + match (Parser.interaction (fun _ -> tokenizer ()) lexbuf) with | ParsedScriptInteraction.Definitions(l, m) -> printfn "Parsed OK, got %d defs @ %a" l.Length outputRange m - | ParsedScriptInteraction.HashDirective (_, m) -> printfn "Parsed OK, got hash @ %a" outputRange m + | ParsedScriptInteraction.HashDirective(_, m) -> printfn "Parsed OK, got hash @ %a" outputRange m exit 0 // Report the statistics for testing purposes @@ -369,10 +371,14 @@ let ParseOneInputLexbuf (tcConfig: TcConfig, lexResourceManager, conditionalComp Lexhelp.usingLexbufForParsing (lexbuf, filename) (fun lexbuf -> // Set up the LexFilter over the token stream - let tokenizer = LexFilter.LexFilter(lightStatus, tcConfig.compilingFslib, Lexer.token lexargs skipWhitespaceTokens, lexbuf) + let tokenizer,tokenizeOnly = + match tcConfig.tokenize with + | Unfiltered -> (fun () -> Lexer.token lexargs skipWhitespaceTokens lexbuf), true + | Only -> LexFilter.LexFilter(lightStatus, tcConfig.compilingFslib, Lexer.token lexargs skipWhitespaceTokens, lexbuf).GetToken, true + | _ -> LexFilter.LexFilter(lightStatus, tcConfig.compilingFslib, Lexer.token lexargs skipWhitespaceTokens, lexbuf).GetToken, false // If '--tokenize' then show the tokens now and exit - if tcConfig.tokenizeOnly then + if tokenizeOnly then ShowAllTokensAndExit(shortFilename, tokenizer, lexbuf) // Test hook for one of the parser entry points @@ -380,7 +386,7 @@ let ParseOneInputLexbuf (tcConfig: TcConfig, lexResourceManager, conditionalComp TestInteractionParserAndExit (tokenizer, lexbuf) // Parse the input - let res = ParseInput((fun _ -> tokenizer.GetToken()), errorLogger, lexbuf, None, filename, isLastCompiland) + let res = ParseInput((fun _ -> tokenizer ()), errorLogger, lexbuf, None, filename, isLastCompiland) // Report the statistics for testing purposes if tcConfig.reportNumDecls then diff --git a/tests/fsharpqa/Source/CompilerOptions/fsc/tokenize/env.lst b/tests/fsharpqa/Source/CompilerOptions/fsc/tokenize/env.lst new file mode 100644 index 00000000000..5016280a55a --- /dev/null +++ b/tests/fsharpqa/Source/CompilerOptions/fsc/tokenize/env.lst @@ -0,0 +1,3 @@ +# Test tokenize outputs tokens + SOURCE=tokenize01.fs COMPILE_ONLY=1 SCFLAGS="--tokenize" + SOURCE=tokenize02.fs COMPILE_ONLY=1 SCFLAGS="--tokenize-unfiltered" \ No newline at end of file diff --git a/tests/fsharpqa/Source/CompilerOptions/fsc/tokenize/tokenize01.fs b/tests/fsharpqa/Source/CompilerOptions/fsc/tokenize/tokenize01.fs new file mode 100644 index 00000000000..48c56e06129 --- /dev/null +++ b/tests/fsharpqa/Source/CompilerOptions/fsc/tokenize/tokenize01.fs @@ -0,0 +1,60 @@ +// #NoMT #CompilerOptions +#light + +namespace N + module M = + let f x = () + f 10 + +//tokenize - got NAMESPACE +//tokenize - got IDENT +//tokenize - got OBLOCKBEGIN +//tokenize - got MODULE_COMING_SOON +//tokenize - got MODULE_COMING_SOON +//tokenize - got MODULE_COMING_SOON +//tokenize - got MODULE_COMING_SOON +//tokenize - got MODULE_COMING_SOON +//tokenize - got MODULE_COMING_SOON +//tokenize - got MODULE_IS_HERE +//tokenize - got IDENT +//tokenize - got EQUALS +//tokenize - got OBLOCKBEGIN +//tokenize - got OLET +//tokenize - got IDENT +//tokenize - got IDENT +//tokenize - got EQUALS +//tokenize - got OBLOCKBEGIN +//tokenize - got LPAREN +//tokenize - got RPAREN_COMING_SOON +//tokenize - got RPAREN_COMING_SOON +//tokenize - got RPAREN_COMING_SOON +//tokenize - got RPAREN_COMING_SOON +//tokenize - got RPAREN_COMING_SOON +//tokenize - got RPAREN_COMING_SOON +//tokenize - got RPAREN_IS_HERE +//tokenize - got OBLOCKEND_COMING_SOON +//tokenize - got OBLOCKEND_COMING_SOON +//tokenize - got OBLOCKEND_COMING_SOON +//tokenize - got OBLOCKEND_COMING_SOON +//tokenize - got OBLOCKEND_COMING_SOON +//tokenize - got OBLOCKEND_COMING_SOON +//tokenize - got OBLOCKEND_IS_HERE +//tokenize - got ODECLEND +//tokenize - got OBLOCKSEP +//tokenize - got IDENT +//tokenize - got INT32 +//tokenize - got OBLOCKEND_COMING_SOON +//tokenize - got OBLOCKEND_COMING_SOON +//tokenize - got OBLOCKEND_COMING_SOON +//tokenize - got OBLOCKEND_COMING_SOON +//tokenize - got OBLOCKEND_COMING_SOON +//tokenize - got OBLOCKEND_COMING_SOON +//tokenize - got OBLOCKEND_IS_HERE +//tokenize - got OBLOCKEND_COMING_SOON +//tokenize - got OBLOCKEND_COMING_SOON +//tokenize - got OBLOCKEND_COMING_SOON +//tokenize - got OBLOCKEND_COMING_SOON +//tokenize - got OBLOCKEND_COMING_SOON +//tokenize - got OBLOCKEND_COMING_SOON +//tokenize - got OBLOCKEND_IS_HERE +//tokenize - got EOF \ No newline at end of file diff --git a/tests/fsharpqa/Source/CompilerOptions/fsc/tokenize/tokenize02.fs b/tests/fsharpqa/Source/CompilerOptions/fsc/tokenize/tokenize02.fs new file mode 100644 index 00000000000..d139e72caee --- /dev/null +++ b/tests/fsharpqa/Source/CompilerOptions/fsc/tokenize/tokenize02.fs @@ -0,0 +1,22 @@ +// #NoMT #CompilerOptions +#light + +namespace N + module M = + let f x = () + f 10 + +//tokenize - got NAMESPACE +//tokenize - got IDENT +//tokenize - got MODULE +//tokenize - got IDENT +//tokenize - got EQUALS +//tokenize - got LET +//tokenize - got IDENT +//tokenize - got IDENT +//tokenize - got EQUALS +//tokenize - got LPAREN +//tokenize - got RPAREN +//tokenize - got IDENT +//tokenize - got INT32 +//tokenize - got EOF \ No newline at end of file diff --git a/tests/fsharpqa/Source/test.lst b/tests/fsharpqa/Source/test.lst index e4b9f292d8c..5a8307bdb9f 100644 --- a/tests/fsharpqa/Source/test.lst +++ b/tests/fsharpqa/Source/test.lst @@ -57,6 +57,7 @@ CompilerOptions01,NoMT CompilerOptions\fsc\subsystemversion CompilerOptions01,NoMT CompilerOptions\fsc\tailcalls CompilerOptions01,NoMT CompilerOptions\fsc\target CompilerOptions01,NoMT,NoHostedCompiler CompilerOptions\fsc\times +CompilerOptions01,NoMT,NoHostedCompiler CompilerOptions\fsc\tokenize CompilerOptions01,NoMT CompilerOptions\fsc\warn CompilerOptions01,NoMT CompilerOptions\fsc\warnaserror CompilerOptions01,NoMT CompilerOptions\fsc\warnon