From 1386bf9289b5d90d04e2b79c10fa9c712feb59b8 Mon Sep 17 00:00:00 2001
From: Erik Schierboom <erik_schierboom@hotmail.com>
Date: Sat, 2 Jan 2021 14:33:55 +0100
Subject: [PATCH] Add option to display pre-lexfilter tokens

---
 src/fsharp/CompilerConfig.fs      |  3 +++
 src/fsharp/CompilerConfig.fsi     |  2 ++
 src/fsharp/CompilerOptions.fs     |  5 +++++
 src/fsharp/ParseAndCheckInputs.fs | 21 +++++++++++++++++++--
 4 files changed, 29 insertions(+), 2 deletions(-)

diff --git a/src/fsharp/CompilerConfig.fs b/src/fsharp/CompilerConfig.fs
index c544b626f75..bb1f699d7b6 100644
--- a/src/fsharp/CompilerConfig.fs
+++ b/src/fsharp/CompilerConfig.fs
@@ -364,6 +364,7 @@ type TcConfigBuilder =
       mutable simulateException: string option
       mutable printAst: bool
       mutable tokenizeOnly: bool
+      mutable preLexFilterTokenizeOnly: bool
       mutable testInteractionParser: bool
       mutable reportNumDecls: bool
       mutable printSignature: bool
@@ -532,6 +533,7 @@ type TcConfigBuilder =
           simulateException = None
           printAst = false
           tokenizeOnly = false
+          preLexFilterTokenizeOnly = false
           testInteractionParser = false
           reportNumDecls = false
           printSignature = false
@@ -935,6 +937,7 @@ type TcConfig private (data: TcConfigBuilder, validate: bool) =
     member x.printAst = data.printAst
     member x.targetFrameworkVersion = targetFrameworkVersionValue
     member x.tokenizeOnly = data.tokenizeOnly
+    member x.preLexFilterTokenizeOnly = data.preLexFilterTokenizeOnly
     member x.testInteractionParser = data.testInteractionParser
     member x.reportNumDecls = data.reportNumDecls
     member x.printSignature = data.printSignature
diff --git a/src/fsharp/CompilerConfig.fsi b/src/fsharp/CompilerConfig.fsi
index dc7894c2c41..d2f5b11fa32 100644
--- a/src/fsharp/CompilerConfig.fsi
+++ b/src/fsharp/CompilerConfig.fsi
@@ -182,6 +182,7 @@ type TcConfigBuilder =
       mutable simulateException: string option
       mutable printAst: bool
       mutable tokenizeOnly: bool
+      mutable preLexFilterTokenizeOnly: bool
       mutable testInteractionParser: bool
       mutable reportNumDecls: bool
       mutable printSignature: bool
@@ -363,6 +364,7 @@ type TcConfig =
     member simulateException: string option
     member printAst: bool
     member tokenizeOnly: bool
+    member preLexFilterTokenizeOnly: bool
     member testInteractionParser: bool
     member reportNumDecls: bool
     member printSignature: bool
diff --git a/src/fsharp/CompilerOptions.fs b/src/fsharp/CompilerOptions.fs
index e6e5d02feb9..3fbfe7e64f6 100644
--- a/src/fsharp/CompilerOptions.fs
+++ b/src/fsharp/CompilerOptions.fs
@@ -1135,6 +1135,11 @@ let internalFlags (tcConfigB:TcConfigBuilder) =
        ("tokenize", tagNone,
         OptionUnit (fun () -> tcConfigB.tokenizeOnly <- true),
         Some(InternalCommandLineOption("--tokenize", rangeCmdArgs)), None)
+
+    CompilerOption
+        ("pre-lexfilter-tokenize", tagNone,
+         OptionUnit (fun () -> tcConfigB.preLexFilterTokenizeOnly <- true),
+         Some(InternalCommandLineOption("--pre-lexfilter-tokenize", rangeCmdArgs)), None)
     
     CompilerOption
        ("testInteractionParser", tagNone,
diff --git a/src/fsharp/ParseAndCheckInputs.fs b/src/fsharp/ParseAndCheckInputs.fs
index f34f40e5c02..419e6cb17f8 100644
--- a/src/fsharp/ParseAndCheckInputs.fs
+++ b/src/fsharp/ParseAndCheckInputs.fs
@@ -287,6 +287,17 @@ let ParseInput (lexer, errorLogger: ErrorLogger, lexbuf: UnicodeLexing.Lexbuf, d
         let filteringErrorLogger = GetErrorLoggerFilteringByScopedPragmas(false, scopedPragmas, errorLogger)
         delayLogger.CommitDelayedDiagnostics filteringErrorLogger
 
+// Show all pre-lex filter tokens in the stream, for testing purposes
+let ShowAllPreLexFilterTokensAndExit (shortFilename, tokenizer: (LexBuffer<char> -> Parser.token), lexbuf: LexBuffer<char>) =
+    while true do 
+        printf "tokenize - getting one token from %s\n" shortFilename
+        let t = tokenizer(lexbuf)
+        printf "tokenize - got %s @ %a\n" (Parser.token_to_string t) outputRange lexbuf.LexemeRange
+        match t with
+        | Parser.EOF _ -> exit 0 
+        | _ -> ()
+        if lexbuf.IsPastEndOfStream then printf "!!! at end of stream\n"
+
 // Show all tokens in the stream, for testing purposes
 let ShowAllTokensAndExit (shortFilename, tokenizer: LexFilter.LexFilter, lexbuf: LexBuffer<char>) =
     while true do 
@@ -340,9 +351,15 @@ let ParseOneInputLexbuf (tcConfig: TcConfig, lexResourceManager, conditionalComp
 
         let input = 
             Lexhelp.usingLexbufForParsing (lexbuf, filename) (fun lexbuf ->
-                
+
+                let tokenizerPreLexFilter = Lexer.token lexargs skipWhitespaceTokens
+
+                // If '--pre-lexfilter-tokenize' then show the tokens pre-LexFilter and exit
+                if tcConfig.preLexFilterTokenizeOnly then 
+                    ShowAllPreLexFilterTokensAndExit(shortFilename, tokenizerPreLexFilter, lexbuf)
+
                 // Set up the LexFilter over the token stream
-                let tokenizer = LexFilter.LexFilter(lightStatus, tcConfig.compilingFslib, Lexer.token lexargs skipWhitespaceTokens, lexbuf)
+                let tokenizer = LexFilter.LexFilter(lightStatus, tcConfig.compilingFslib, tokenizerPreLexFilter, lexbuf)
 
                 // If '--tokenize' then show the tokens now and exit
                 if tcConfig.tokenizeOnly then