From d9c9a835a7e87b2eeab2d80f97a86238e39f4552 Mon Sep 17 00:00:00 2001 From: Sebastian Wilzbach Date: Mon, 19 Jun 2017 16:28:43 +0200 Subject: [PATCH 1/2] Allow configureable WhitespaceBehavior in getTokensForParser --- src/dparse/lexer.d | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/dparse/lexer.d b/src/dparse/lexer.d index dd0db6ad..01d9aae8 100644 --- a/src/dparse/lexer.d +++ b/src/dparse/lexer.d @@ -171,7 +171,7 @@ public struct LexerConfig { string fileName; StringBehavior stringBehavior; - WhitespaceBehavior whitespaceBehavior; + WhitespaceBehavior whitespaceBehavior = WhitespaceBehavior.skip; CommentBehavior commentBehavior = CommentBehavior.intern; } @@ -409,7 +409,6 @@ const(Token)[] getTokensForParser(ubyte[] sourceCode, LexerConfig config, return CommentType.notDoc; } - config.whitespaceBehavior = WhitespaceBehavior.skip; config.commentBehavior = CommentBehavior.noIntern; auto leadingCommentAppender = appender!(char[])(); @@ -425,6 +424,9 @@ const(Token)[] getTokensForParser(ubyte[] sourceCode, LexerConfig config, { case tok!"specialTokenSequence": case tok!"whitespace": + if (config.whitespaceBehavior == WhitespaceBehavior.include) + output.put(lexer.front); + lexer.popFront(); break; case tok!"comment": From 90c37c1aa5ca4fef5d3278f6c699ffb005114182 Mon Sep 17 00:00:00 2001 From: Sebastian Wilzbach Date: Mon, 10 Jul 2017 23:48:24 +0200 Subject: [PATCH 2/2] Update getTokensForParser header --- src/dparse/lexer.d | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/dparse/lexer.d b/src/dparse/lexer.d index 01d9aae8..471b9880 100644 --- a/src/dparse/lexer.d +++ b/src/dparse/lexer.d @@ -384,9 +384,10 @@ public bool isLiteral(IdType type) pure nothrow @safe @nogc } /** - * Returns: an array of tokens lexed from the given source code to the output range. All - * whitespace tokens are skipped and comments are attached to the token nearest - * to them. + * Returns: an array of tokens lexed from the given source code to the output range. + * Whitespace token are skipped by default, except `WhitespaceBehavior.include` + * has been set explicitly. + * comments are attached to the token nearest to them. */ const(Token)[] getTokensForParser(ubyte[] sourceCode, LexerConfig config, StringCache* cache)