Add FSC option for retrieving raw lexer tokens

src/fsharp/CompilerConfig.fs

@@ -270,6 +270,11 @@ type LStatus =
     | Unprocessed
     | Processed
 
+type TokenizeOption =
+    | AndCompile
+    | Only
+    | Unfiltered
+
 type PackageManagerLine =
     { Directive: Directive
       LineStatus: LStatus
@@ -367,7 +372,7 @@ type TcConfigBuilder =
       mutable importAllReferencesOnly: bool
       mutable simulateException: string option
       mutable printAst: bool
-      mutable tokenizeOnly: bool
+      mutable tokenize: TokenizeOption
       mutable testInteractionParser: bool
       mutable reportNumDecls: bool
       mutable printSignature: bool
@@ -573,7 +578,7 @@ type TcConfigBuilder =
           importAllReferencesOnly = false
           simulateException = None
           printAst = false
-          tokenizeOnly = false
+          tokenize = TokenizeOption.AndCompile
           testInteractionParser = false
           reportNumDecls = false
           printSignature = false
@@ -960,7 +965,7 @@ type TcConfig private (data: TcConfigBuilder, validate: bool) =
     member x.simulateException = data.simulateException
     member x.printAst = data.printAst
     member x.targetFrameworkVersion = targetFrameworkVersionValue
-    member x.tokenizeOnly = data.tokenizeOnly
+    member x.tokenize = data.tokenize
     member x.testInteractionParser = data.testInteractionParser
     member x.reportNumDecls = data.reportNumDecls
     member x.printSignature = data.printSignature

src/fsharp/CompilerConfig.fsi

@@ -121,6 +121,11 @@ type LStatus =
     | Unprocessed
     | Processed
 
+type TokenizeOption =
+    | AndCompile
+    | Only
+    | Unfiltered
+
 type PackageManagerLine =
     { Directive: Directive
       LineStatus: LStatus
@@ -182,7 +187,7 @@ type TcConfigBuilder =
       mutable importAllReferencesOnly: bool
       mutable simulateException: string option
       mutable printAst: bool
-      mutable tokenizeOnly: bool
+      mutable tokenize: TokenizeOption
       mutable testInteractionParser: bool
       mutable reportNumDecls: bool
       mutable printSignature: bool
@@ -374,7 +379,7 @@ type TcConfig =
     member importAllReferencesOnly: bool
     member simulateException: string option
     member printAst: bool
-    member tokenizeOnly: bool
+    member tokenize: TokenizeOption
     member testInteractionParser: bool
     member reportNumDecls: bool
     member printSignature: bool

src/fsharp/CompilerOptions.fs

@@ -1139,9 +1139,14 @@ let internalFlags (tcConfigB:TcConfigBuilder) =
 
     CompilerOption
        ("tokenize", tagNone,
-        OptionUnit (fun () -> tcConfigB.tokenizeOnly <- true),
+        OptionUnit (fun () -> tcConfigB.tokenize <- TokenizeOption.Only),
         Some(InternalCommandLineOption("--tokenize", rangeCmdArgs)), None)
 
+    CompilerOption
+       ("tokenize-unfiltered", tagNone,
+        OptionUnit (fun () -> tcConfigB.tokenize <- TokenizeOption.Unfiltered),
+        Some(InternalCommandLineOption("--tokenize-unfiltered", rangeCmdArgs)), None)
+
     CompilerOption
        ("testInteractionParser", tagNone,
         OptionUnit (fun () -> tcConfigB.testInteractionParser <- true),

src/fsharp/ParseAndCheckInputs.fs

@@ -289,23 +289,25 @@ let ParseInput (lexer, errorLogger: ErrorLogger, lexbuf: UnicodeLexing.Lexbuf, d
         let filteringErrorLogger = GetErrorLoggerFilteringByScopedPragmas(false, scopedPragmas, errorLogger)
         delayLogger.CommitDelayedDiagnostics filteringErrorLogger
 
+type Tokenizer = unit -> Parser.token
+
 // Show all tokens in the stream, for testing purposes
-let ShowAllTokensAndExit (shortFilename, tokenizer: LexFilter.LexFilter, lexbuf: LexBuffer<char>) =
+let ShowAllTokensAndExit (shortFilename, tokenizer: Tokenizer, lexbuf: LexBuffer<char>) =
     while true do
         printf "tokenize - getting one token from %s\n" shortFilename
-        let t = tokenizer.GetToken()
+        let t = tokenizer ()
         printf "tokenize - got %s @ %a\n" (Parser.token_to_string t) outputRange lexbuf.LexemeRange
         match t with
         | Parser.EOF _ -> exit 0
         | _ -> ()
         if lexbuf.IsPastEndOfStream then printf "!!! at end of stream\n"
 
 // Test one of the parser entry points, just for testing purposes
-let TestInteractionParserAndExit (tokenizer: LexFilter.LexFilter, lexbuf: LexBuffer<char>) =
+let TestInteractionParserAndExit (tokenizer: Tokenizer, lexbuf: LexBuffer<char>) =
     while true do
-        match (Parser.interaction (fun _ -> tokenizer.GetToken()) lexbuf) with
+        match (Parser.interaction (fun _ -> tokenizer ()) lexbuf) with
         | ParsedScriptInteraction.Definitions(l, m) -> printfn "Parsed OK, got %d defs @ %a" l.Length outputRange m
-        | ParsedScriptInteraction.HashDirective (_, m) -> printfn "Parsed OK, got hash @ %a" outputRange m
+        | ParsedScriptInteraction.HashDirective(_, m) -> printfn "Parsed OK, got hash @ %a" outputRange m
     exit 0
 
 // Report the statistics for testing purposes
@@ -369,18 +371,22 @@ let ParseOneInputLexbuf (tcConfig: TcConfig, lexResourceManager, conditionalComp
             Lexhelp.usingLexbufForParsing (lexbuf, filename) (fun lexbuf ->
 
                 // Set up the LexFilter over the token stream
-                let tokenizer = LexFilter.LexFilter(lightStatus, tcConfig.compilingFslib, Lexer.token lexargs skipWhitespaceTokens, lexbuf)
+                let tokenizer,tokenizeOnly =
+                    match tcConfig.tokenize with
+                    | Unfiltered -> (fun () -> Lexer.token lexargs skipWhitespaceTokens lexbuf), true
+                    | Only ->       LexFilter.LexFilter(lightStatus, tcConfig.compilingFslib, Lexer.token lexargs skipWhitespaceTokens, lexbuf).GetToken, true
+                    | _ ->          LexFilter.LexFilter(lightStatus, tcConfig.compilingFslib, Lexer.token lexargs skipWhitespaceTokens, lexbuf).GetToken, false
 
                 // If '--tokenize' then show the tokens now and exit
-                if tcConfig.tokenizeOnly then
+                if tokenizeOnly then
                     ShowAllTokensAndExit(shortFilename, tokenizer, lexbuf)
 
                 // Test hook for one of the parser entry points
                 if tcConfig.testInteractionParser then
                     TestInteractionParserAndExit (tokenizer, lexbuf)
 
                 // Parse the input
-                let res = ParseInput((fun _ -> tokenizer.GetToken()), errorLogger, lexbuf, None, filename, isLastCompiland)
+                let res = ParseInput((fun _ -> tokenizer ()), errorLogger, lexbuf, None, filename, isLastCompiland)
 
                 // Report the statistics for testing purposes
                 if tcConfig.reportNumDecls then

tests/fsharpqa/Source/CompilerOptions/fsc/tokenize/env.lst

@@ -0,0 +1,3 @@
+# Test tokenize outputs tokens
+	SOURCE=tokenize01.fs  COMPILE_ONLY=1  SCFLAGS="--tokenize"
+	SOURCE=tokenize02.fs  COMPILE_ONLY=1  SCFLAGS="--tokenize-unfiltered"

tests/fsharpqa/Source/CompilerOptions/fsc/tokenize/tokenize01.fs

@@ -0,0 +1,60 @@
+// #NoMT #CompilerOptions 
+#light
+
+namespace N
+   module M =
+      let f x = ()
+      f 10
+
+//<Expects status="success">tokenize - got NAMESPACE</Expects>
+//<Expects status="success">tokenize - got IDENT</Expects>
+//<Expects status="success">tokenize - got OBLOCKBEGIN</Expects>
+//<Expects status="success">tokenize - got MODULE_COMING_SOON</Expects>
+//<Expects status="success">tokenize - got MODULE_COMING_SOON</Expects>
+//<Expects status="success">tokenize - got MODULE_COMING_SOON</Expects>
+//<Expects status="success">tokenize - got MODULE_COMING_SOON</Expects>
+//<Expects status="success">tokenize - got MODULE_COMING_SOON</Expects>
+//<Expects status="success">tokenize - got MODULE_COMING_SOON</Expects>
+//<Expects status="success">tokenize - got MODULE_IS_HERE</Expects>
+//<Expects status="success">tokenize - got IDENT</Expects>
+//<Expects status="success">tokenize - got EQUALS</Expects>
+//<Expects status="success">tokenize - got OBLOCKBEGIN</Expects>
+//<Expects status="success">tokenize - got OLET</Expects>
+//<Expects status="success">tokenize - got IDENT</Expects>
+//<Expects status="success">tokenize - got IDENT</Expects>
+//<Expects status="success">tokenize - got EQUALS</Expects>
+//<Expects status="success">tokenize - got OBLOCKBEGIN</Expects>
+//<Expects status="success">tokenize - got LPAREN</Expects>
+//<Expects status="success">tokenize - got RPAREN_COMING_SOON</Expects>
+//<Expects status="success">tokenize - got RPAREN_COMING_SOON</Expects>
+//<Expects status="success">tokenize - got RPAREN_COMING_SOON</Expects>
+//<Expects status="success">tokenize - got RPAREN_COMING_SOON</Expects>
+//<Expects status="success">tokenize - got RPAREN_COMING_SOON</Expects>
+//<Expects status="success">tokenize - got RPAREN_COMING_SOON</Expects>
+//<Expects status="success">tokenize - got RPAREN_IS_HERE</Expects>
+//<Expects status="success">tokenize - got OBLOCKEND_COMING_SOON</Expects>
+//<Expects status="success">tokenize - got OBLOCKEND_COMING_SOON</Expects>
+//<Expects status="success">tokenize - got OBLOCKEND_COMING_SOON</Expects>
+//<Expects status="success">tokenize - got OBLOCKEND_COMING_SOON</Expects>
+//<Expects status="success">tokenize - got OBLOCKEND_COMING_SOON</Expects>
+//<Expects status="success">tokenize - got OBLOCKEND_COMING_SOON</Expects>
+//<Expects status="success">tokenize - got OBLOCKEND_IS_HERE</Expects>
+//<Expects status="success">tokenize - got ODECLEND</Expects>
+//<Expects status="success">tokenize - got OBLOCKSEP</Expects>
+//<Expects status="success">tokenize - got IDENT</Expects>
+//<Expects status="success">tokenize - got INT32</Expects>
+//<Expects status="success">tokenize - got OBLOCKEND_COMING_SOON</Expects>
+//<Expects status="success">tokenize - got OBLOCKEND_COMING_SOON</Expects>
+//<Expects status="success">tokenize - got OBLOCKEND_COMING_SOON</Expects>
+//<Expects status="success">tokenize - got OBLOCKEND_COMING_SOON</Expects>
+//<Expects status="success">tokenize - got OBLOCKEND_COMING_SOON</Expects>
+//<Expects status="success">tokenize - got OBLOCKEND_COMING_SOON</Expects>
+//<Expects status="success">tokenize - got OBLOCKEND_IS_HERE</Expects>
+//<Expects status="success">tokenize - got OBLOCKEND_COMING_SOON</Expects>
+//<Expects status="success">tokenize - got OBLOCKEND_COMING_SOON</Expects>
+//<Expects status="success">tokenize - got OBLOCKEND_COMING_SOON</Expects>
+//<Expects status="success">tokenize - got OBLOCKEND_COMING_SOON</Expects>
+//<Expects status="success">tokenize - got OBLOCKEND_COMING_SOON</Expects>
+//<Expects status="success">tokenize - got OBLOCKEND_COMING_SOON</Expects>
+//<Expects status="success">tokenize - got OBLOCKEND_IS_HERE</Expects>
+//<Expects status="success">tokenize - got EOF</Expects>

tests/fsharpqa/Source/CompilerOptions/fsc/tokenize/tokenize02.fs

@@ -0,0 +1,22 @@
+// #NoMT #CompilerOptions 
+#light
+
+namespace N
+   module M =
+      let f x = ()
+      f 10
+
+//<Expects status="success">tokenize - got NAMESPACE</Expects>
+//<Expects status="success">tokenize - got IDENT</Expects>
+//<Expects status="success">tokenize - got MODULE</Expects>
+//<Expects status="success">tokenize - got IDENT</Expects>
+//<Expects status="success">tokenize - got EQUALS</Expects>
+//<Expects status="success">tokenize - got LET</Expects>
+//<Expects status="success">tokenize - got IDENT</Expects>
+//<Expects status="success">tokenize - got IDENT</Expects>
+//<Expects status="success">tokenize - got EQUALS</Expects>
+//<Expects status="success">tokenize - got LPAREN</Expects>
+//<Expects status="success">tokenize - got RPAREN</Expects>
+//<Expects status="success">tokenize - got IDENT</Expects>
+//<Expects status="success">tokenize - got INT32</Expects>
+//<Expects status="success">tokenize - got EOF</Expects>

tests/fsharpqa/Source/test.lst

@@ -57,6 +57,7 @@ CompilerOptions01,NoMT				CompilerOptions\fsc\subsystemversion
 CompilerOptions01,NoMT				CompilerOptions\fsc\tailcalls
 CompilerOptions01,NoMT				CompilerOptions\fsc\target
 CompilerOptions01,NoMT,NoHostedCompiler				CompilerOptions\fsc\times
+CompilerOptions01,NoMT,NoHostedCompiler				CompilerOptions\fsc\tokenize
 CompilerOptions01,NoMT				CompilerOptions\fsc\warn
 CompilerOptions01,NoMT				CompilerOptions\fsc\warnaserror
 CompilerOptions01,NoMT				CompilerOptions\fsc\warnon