Skip to content

Add option to display pre-lexfilter tokens #10831

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
ErikSchierboom wants to merge 1 commit into from
Closed

Add option to display pre-lexfilter tokens #10831

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions src/fsharp/CompilerConfig.fs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -364,6 +364,7 @@ type TcConfigBuilder =
mutable simulateException: string option
mutable printAst: bool
mutable tokenizeOnly: bool
mutable preLexFilterTokenizeOnly: bool
mutable testInteractionParser: bool
mutable reportNumDecls: bool
mutable printSignature: bool
Expand Down Expand Up @@ -532,6 +533,7 @@ type TcConfigBuilder =
simulateException = None
printAst = false
tokenizeOnly = false
preLexFilterTokenizeOnly = false
testInteractionParser = false
reportNumDecls = false
printSignature = false
Expand Down Expand Up @@ -935,6 +937,7 @@ type TcConfig private (data: TcConfigBuilder, validate: bool) =
member x.printAst = data.printAst
member x.targetFrameworkVersion = targetFrameworkVersionValue
member x.tokenizeOnly = data.tokenizeOnly
member x.preLexFilterTokenizeOnly = data.preLexFilterTokenizeOnly
member x.testInteractionParser = data.testInteractionParser
member x.reportNumDecls = data.reportNumDecls
member x.printSignature = data.printSignature
Expand Down
2 changes: 2 additions & 0 deletions src/fsharp/CompilerConfig.fsi
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -182,6 +182,7 @@ type TcConfigBuilder =
mutable simulateException: string option
mutable printAst: bool
mutable tokenizeOnly: bool
mutable preLexFilterTokenizeOnly: bool
mutable testInteractionParser: bool
mutable reportNumDecls: bool
mutable printSignature: bool
Expand Down Expand Up @@ -363,6 +364,7 @@ type TcConfig =
member simulateException: string option
member printAst: bool
member tokenizeOnly: bool
member preLexFilterTokenizeOnly: bool
member testInteractionParser: bool
member reportNumDecls: bool
member printSignature: bool
Expand Down
5 changes: 5 additions & 0 deletions src/fsharp/CompilerOptions.fs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1135,6 +1135,11 @@ let internalFlags (tcConfigB:TcConfigBuilder) =
("tokenize", tagNone,
OptionUnit (fun () -> tcConfigB.tokenizeOnly <- true),
Some(InternalCommandLineOption("--tokenize", rangeCmdArgs)), None)

CompilerOption
("pre-lexfilter-tokenize", tagNone,
OptionUnit (fun () -> tcConfigB.preLexFilterTokenizeOnly <- true),
Some(InternalCommandLineOption("--pre-lexfilter-tokenize", rangeCmdArgs)), None)

CompilerOption
("testInteractionParser", tagNone,
Expand Down
21 changes: 19 additions & 2 deletions src/fsharp/ParseAndCheckInputs.fs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -287,6 +287,17 @@ let ParseInput (lexer, errorLogger: ErrorLogger, lexbuf: UnicodeLexing.Lexbuf, d
let filteringErrorLogger = GetErrorLoggerFilteringByScopedPragmas(false, scopedPragmas, errorLogger)
delayLogger.CommitDelayedDiagnostics filteringErrorLogger

// Show all pre-lex filter tokens in the stream, for testing purposes
let ShowAllPreLexFilterTokensAndExit (shortFilename, tokenizer: (LexBuffer<char> -> Parser.token), lexbuf: LexBuffer<char>) =
while true do
printf "tokenize - getting one token from %s\n" shortFilename
let t = tokenizer(lexbuf)
printf "tokenize - got %s @ %a\n" (Parser.token_to_string t) outputRange lexbuf.LexemeRange
match t with
| Parser.EOF _ -> exit 0
| _ -> ()
if lexbuf.IsPastEndOfStream then printf "!!! at end of stream\n"

// Show all tokens in the stream, for testing purposes
let ShowAllTokensAndExit (shortFilename, tokenizer: LexFilter.LexFilter, lexbuf: LexBuffer<char>) =
while true do
Expand Down Expand Up @@ -340,9 +351,15 @@ let ParseOneInputLexbuf (tcConfig: TcConfig, lexResourceManager, conditionalComp

let input =
Lexhelp.usingLexbufForParsing (lexbuf, filename) (fun lexbuf ->


let tokenizerPreLexFilter = Lexer.token lexargs skipWhitespaceTokens

// If '--pre-lexfilter-tokenize' then show the tokens pre-LexFilter and exit
if tcConfig.preLexFilterTokenizeOnly then
ShowAllPreLexFilterTokensAndExit(shortFilename, tokenizerPreLexFilter, lexbuf)

// Set up the LexFilter over the token stream
let tokenizer = LexFilter.LexFilter(lightStatus, tcConfig.compilingFslib, Lexer.token lexargs skipWhitespaceTokens, lexbuf)
let tokenizer = LexFilter.LexFilter(lightStatus, tcConfig.compilingFslib, tokenizerPreLexFilter, lexbuf)

// If '--tokenize' then show the tokens now and exit
if tcConfig.tokenizeOnly then
Expand Down