From 1acd2ad8fbaf5104f2e9996e5d165af33a63e816 Mon Sep 17 00:00:00 2001 From: Dmytro Melnychenko Date: Wed, 16 May 2018 21:50:45 +0200 Subject: [PATCH 01/17] Add UntypedTreeTraverser --- compiler/src/dotty/tools/dotc/ast/untpd.scala | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/compiler/src/dotty/tools/dotc/ast/untpd.scala b/compiler/src/dotty/tools/dotc/ast/untpd.scala index 80e4365a943c..4bbcf4fa6519 100644 --- a/compiler/src/dotty/tools/dotc/ast/untpd.scala +++ b/compiler/src/dotty/tools/dotc/ast/untpd.scala @@ -596,6 +596,12 @@ object untpd extends Trees.Instance[Untyped] with UntypedTreeInfo { } } + abstract class UntypedTreeTraverser extends UntypedTreeAccumulator[Unit] { + def traverse(tree: Tree)(implicit ctx: Context): Unit + def apply(x: Unit, tree: Tree)(implicit ctx: Context) = traverse(tree) + protected def traverseChildren(tree: Tree)(implicit ctx: Context) = foldOver((), tree) + } + /** Fold `f` over all tree nodes, in depth-first, prefix order */ class UntypedDeepFolder[X](f: (X, Tree) => X) extends UntypedTreeAccumulator[X] { def apply(x: X, tree: Tree)(implicit ctx: Context): X = foldOver(f(x, tree), tree) From 0350fa6aefa225b8c8654be93c177fd54e1e2174 Mon Sep 17 00:00:00 2001 From: Dmytro Melnychenko Date: Wed, 16 May 2018 21:51:03 +0200 Subject: [PATCH 02/17] Introduce Parser and Scanner based highlighter --- .../dotc/printing/SyntaxHighlighting.scala | 95 ++++++++++++++++++- 1 file changed, 93 insertions(+), 2 deletions(-) diff --git a/compiler/src/dotty/tools/dotc/printing/SyntaxHighlighting.scala b/compiler/src/dotty/tools/dotc/printing/SyntaxHighlighting.scala index 264a2d5d0500..ebe3079bdefe 100644 --- a/compiler/src/dotty/tools/dotc/printing/SyntaxHighlighting.scala +++ b/compiler/src/dotty/tools/dotc/printing/SyntaxHighlighting.scala @@ -2,10 +2,20 @@ package dotty.tools package dotc package printing -import parsing.Tokens._ +import dotty.tools.dotc.core.Contexts.Context +import dotty.tools.dotc.core.StdNames._ +import dotty.tools.dotc.parsing.Parsers.Parser +import dotty.tools.dotc.parsing.Scanners.Scanner +import dotty.tools.dotc.parsing.Tokens._ +import dotty.tools.dotc.reporting.Reporter +import dotty.tools.dotc.reporting.diagnostic.MessageContainer +import dotty.tools.dotc.util.Positions.Position + import scala.annotation.switch import scala.collection.mutable.StringBuilder -import util.Chars +import util.{Chars, SourceFile} + +import scala.collection.mutable /** This object provides functions for syntax highlighting in the REPL */ object SyntaxHighlighting { @@ -354,4 +364,85 @@ object SyntaxHighlighting { newBuf.toIterable } + + private class NoReporter extends Reporter { + override def doReport(m: MessageContainer)(implicit ctx: Context): Unit = () + } + + def highlight(in: String)(ctx0: Context): String = { + import dotty.tools.dotc.ast.untpd._ + + implicit val ctx: Context = ctx0.fresh.setReporter(new NoReporter) + + val sf = new SourceFile("", in.toCharArray) + val p = new Parser(sf) + val s = new Scanner(sf) + val trees = p.blockStatSeq() + + val outputH = Array.fill(in.length)(NoColor) + + def highlightRange(p: Position, color: String): Unit = { + if(p.exists) { + for { + i <- p.start until math.min(p.end, outputH.length) + } outputH(i) = color + } + } + + val treeTraverser = new UntypedTreeTraverser { + def traverse(tree: Tree)(implicit ctx: Context): Unit = { + tree match { + case tpe : TypeDef => + highlightRange(tpe.namePos, TypeColor) + case _ : TypTree => + highlightRange(tree.pos, TypeColor) + case mod: ModuleDef => + highlightRange(mod.namePos, TypeColor) + case v : ValOrDefDef => + highlightRange(v.namePos, ValDefColor) + highlightRange(v.tpt.pos, TypeColor) + case _ : Literal => + highlightRange(tree.pos, LiteralColor) + case _ => + } + traverseChildren(tree) + } + } + + for { + t <- trees + } { + treeTraverser.traverse(t) + } + + val sb = new mutable.StringBuilder() + + while(s.token != EOF) { + val isKwd = isKeyword(s.token) + val offsetStart = s.offset + + + if(s.token == IDENTIFIER && s.name == nme.???) { + highlightRange(Position(s.offset, s.offset + s.name.length), Console.RED_B) + } + s.nextToken() + + if(isKwd) { + val offsetEnd = s.lastOffset + highlightRange(Position(offsetStart, offsetEnd), KeywordColor) + } + } + + for { + idx <- outputH.indices + } { + if(idx == 0 || outputH(idx-1) != outputH(idx)){ + sb.append(outputH(idx)) + } + sb.append(in(idx)) + } + sb.append(NoColor) + + sb.mkString + } } From 2999dd23f7dbb6839eb6a8c5e71e1ce094e9bdc1 Mon Sep 17 00:00:00 2001 From: Dmytro Melnychenko Date: Wed, 16 May 2018 21:51:20 +0200 Subject: [PATCH 03/17] Small improvements in highlighter --- .../dotty/tools/dotc/printing/SyntaxHighlighting.scala | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/compiler/src/dotty/tools/dotc/printing/SyntaxHighlighting.scala b/compiler/src/dotty/tools/dotc/printing/SyntaxHighlighting.scala index ebe3079bdefe..8836545e5253 100644 --- a/compiler/src/dotty/tools/dotc/printing/SyntaxHighlighting.scala +++ b/compiler/src/dotty/tools/dotc/printing/SyntaxHighlighting.scala @@ -369,6 +369,8 @@ object SyntaxHighlighting { override def doReport(m: MessageContainer)(implicit ctx: Context): Unit = () } + private val ignoredKwds = Seq(nme.ARROWkw, nme.EQ, nme.EQL, nme.COLONkw) + def highlight(in: String)(ctx0: Context): String = { import dotty.tools.dotc.ast.untpd._ @@ -418,10 +420,9 @@ object SyntaxHighlighting { val sb = new mutable.StringBuilder() while(s.token != EOF) { - val isKwd = isKeyword(s.token) + val isKwd = isKeyword(s.token) && !ignoredKwds.contains(s.name) val offsetStart = s.offset - if(s.token == IDENTIFIER && s.name == nme.???) { highlightRange(Position(s.offset, s.offset + s.name.length), Console.RED_B) } @@ -441,7 +442,9 @@ object SyntaxHighlighting { } sb.append(in(idx)) } - sb.append(NoColor) + if(outputH.last != NoColor) { + sb.append(NoColor) + } sb.mkString } From ec275f909d648d483191d076937b20a367afcb15 Mon Sep 17 00:00:00 2001 From: Dmytro Melnychenko Date: Wed, 16 May 2018 21:51:43 +0200 Subject: [PATCH 04/17] Switch highlighting tests to new highlighter implementation --- .../dotty/tools/dotc/printing/SyntaxHighlightingTests.scala | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/compiler/test/dotty/tools/dotc/printing/SyntaxHighlightingTests.scala b/compiler/test/dotty/tools/dotc/printing/SyntaxHighlightingTests.scala index 2b8d66d84424..d71da17c9314 100644 --- a/compiler/test/dotty/tools/dotc/printing/SyntaxHighlightingTests.scala +++ b/compiler/test/dotty/tools/dotc/printing/SyntaxHighlightingTests.scala @@ -1,16 +1,16 @@ package dotty.tools.dotc.printing +import dotty.tools.DottyTest import org.junit.Assert._ import org.junit.Test /** Adapted from Ammonite HighlightTests */ -class SyntaxHighlightingTests { +class SyntaxHighlightingTests extends DottyTest { import SyntaxHighlighting._ private def test(source: String, expected: String): Unit = { - val highlighted = SyntaxHighlighting.apply(source) - .mkString + val highlighted = SyntaxHighlighting.highlight(source)(ctx) .replace(NoColor, ">") .replace(CommentColor, " Date: Fri, 18 May 2018 14:17:13 +0200 Subject: [PATCH 05/17] Address review comments --- .../dotc/printing/SyntaxHighlighting.scala | 81 ++++++++++--------- 1 file changed, 41 insertions(+), 40 deletions(-) diff --git a/compiler/src/dotty/tools/dotc/printing/SyntaxHighlighting.scala b/compiler/src/dotty/tools/dotc/printing/SyntaxHighlighting.scala index 8836545e5253..20d8db4bed0f 100644 --- a/compiler/src/dotty/tools/dotc/printing/SyntaxHighlighting.scala +++ b/compiler/src/dotty/tools/dotc/printing/SyntaxHighlighting.scala @@ -369,83 +369,84 @@ object SyntaxHighlighting { override def doReport(m: MessageContainer)(implicit ctx: Context): Unit = () } - private val ignoredKwds = Seq(nme.ARROWkw, nme.EQ, nme.EQL, nme.COLONkw) + private val ignoredKwds = Set(nme.ARROWkw, nme.EQ, nme.EQL, nme.COLONkw) def highlight(in: String)(ctx0: Context): String = { import dotty.tools.dotc.ast.untpd._ implicit val ctx: Context = ctx0.fresh.setReporter(new NoReporter) - val sf = new SourceFile("", in.toCharArray) - val p = new Parser(sf) - val s = new Scanner(sf) - val trees = p.blockStatSeq() + val source = new SourceFile("", in.toCharArray) + val parser = new Parser(source) + val trees = parser.blockStatSeq() - val outputH = Array.fill(in.length)(NoColor) + val colorAt = Array.fill(in.length)(NoColor) - def highlightRange(p: Position, color: String): Unit = { - if(p.exists) { - for { - i <- p.start until math.min(p.end, outputH.length) - } outputH(i) = color + def highlightRange(from: Int, to: Int, color: String) = { + try { + for (i <- from until to) + colorAt(i) = color + } catch { + case _: IndexOutOfBoundsException => + ctx.error("Encountered tree with invalid position, please open an issue with the code snippet that caused the error") } } + def highlightPosition(pos: Position, color: String) = + if (pos.exists) highlightRange(pos.start, pos.end, color) - val treeTraverser = new UntypedTreeTraverser { + val treeHighlighter = new UntypedTreeTraverser { def traverse(tree: Tree)(implicit ctx: Context): Unit = { tree match { case tpe : TypeDef => - highlightRange(tpe.namePos, TypeColor) + highlightPosition(tpe.namePos, TypeColor) case _ : TypTree => - highlightRange(tree.pos, TypeColor) + highlightPosition(tree.pos, TypeColor) case mod: ModuleDef => - highlightRange(mod.namePos, TypeColor) + highlightPosition(mod.namePos, TypeColor) case v : ValOrDefDef => - highlightRange(v.namePos, ValDefColor) - highlightRange(v.tpt.pos, TypeColor) + highlightPosition(v.namePos, ValDefColor) + highlightPosition(v.tpt.pos, TypeColor) case _ : Literal => - highlightRange(tree.pos, LiteralColor) + highlightPosition(tree.pos, LiteralColor) case _ => } traverseChildren(tree) } } - for { - t <- trees - } { - treeTraverser.traverse(t) - } + for (tree <- trees) + treeHighlighter.traverse(tree) - val sb = new mutable.StringBuilder() + val scanner = new Scanner(source) - while(s.token != EOF) { - val isKwd = isKeyword(s.token) && !ignoredKwds.contains(s.name) - val offsetStart = s.offset + while (scanner.token != EOF) { + val isKwd = isKeyword(scanner.token) && !ignoredKwds.contains(scanner.name) + val offsetStart = scanner.offset - if(s.token == IDENTIFIER && s.name == nme.???) { - highlightRange(Position(s.offset, s.offset + s.name.length), Console.RED_B) + if (scanner.token == IDENTIFIER && scanner.name == nme.???) { + highlightRange(scanner.offset, scanner.offset + scanner.name.length, Console.RED_B) } - s.nextToken() + scanner.nextToken() - if(isKwd) { - val offsetEnd = s.lastOffset - highlightRange(Position(offsetStart, offsetEnd), KeywordColor) + if (isKwd) { + val offsetEnd = scanner.lastOffset + highlightPosition(Position(offsetStart, offsetEnd), KeywordColor) } } - for { - idx <- outputH.indices - } { - if(idx == 0 || outputH(idx-1) != outputH(idx)){ - sb.append(outputH(idx)) + val sb = new mutable.StringBuilder() + + for (idx <- colorAt.indices) { + if ( (idx == 0 && colorAt(idx) != NoColor) + || (idx > 0 && colorAt(idx-1) != colorAt(idx))) { + sb.append(colorAt(idx)) } sb.append(in(idx)) } - if(outputH.last != NoColor) { + if (colorAt.nonEmpty && colorAt.last != NoColor) { sb.append(NoColor) } - sb.mkString + sb.toString } } From 1f7acd747ef3ef7d632f064969c8c9872a7940ca Mon Sep 17 00:00:00 2001 From: Dmytro Melnychenko Date: Fri, 18 May 2018 17:50:01 +0200 Subject: [PATCH 06/17] Highlight Ident as type when it is type --- compiler/src/dotty/tools/dotc/printing/SyntaxHighlighting.scala | 2 ++ 1 file changed, 2 insertions(+) diff --git a/compiler/src/dotty/tools/dotc/printing/SyntaxHighlighting.scala b/compiler/src/dotty/tools/dotc/printing/SyntaxHighlighting.scala index 20d8db4bed0f..8ac4723c98d4 100644 --- a/compiler/src/dotty/tools/dotc/printing/SyntaxHighlighting.scala +++ b/compiler/src/dotty/tools/dotc/printing/SyntaxHighlighting.scala @@ -397,6 +397,8 @@ object SyntaxHighlighting { val treeHighlighter = new UntypedTreeTraverser { def traverse(tree: Tree)(implicit ctx: Context): Unit = { tree match { + case id: Ident if id.isType => + highlightPosition(id.pos, TypeColor) case tpe : TypeDef => highlightPosition(tpe.namePos, TypeColor) case _ : TypTree => From 5074d86549022339063bba41ddf615078113e6e3 Mon Sep 17 00:00:00 2001 From: Dmytro Melnychenko Date: Fri, 18 May 2018 17:56:28 +0200 Subject: [PATCH 07/17] Update expected highlight for union types --- .../dotc/printing/SyntaxHighlightingTests.scala | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/compiler/test/dotty/tools/dotc/printing/SyntaxHighlightingTests.scala b/compiler/test/dotty/tools/dotc/printing/SyntaxHighlightingTests.scala index d71da17c9314..1e774eaef25e 100644 --- a/compiler/test/dotty/tools/dotc/printing/SyntaxHighlightingTests.scala +++ b/compiler/test/dotty/tools/dotc/printing/SyntaxHighlightingTests.scala @@ -93,12 +93,12 @@ class SyntaxHighlightingTests extends DottyTest { @Test def unionTypes = { - test("type A = String|Int| Long", " = || ") - test("type B = String |Int| Long", " = || ") - test("type C = String | Int | Long", " = | | ") - test("type D = String&Int& Long", " = && ") - test("type E = String &Int& Long", " = && ") - test("type F = String & Int & Long", " = & & ") - test("fn[String|Char](input)", "fn[|](input)") + test("type A = String|Int| Long", " = ") + test("type B = String |Int| Long", " = ") + test("type C = String | Int | Long", " = ") + test("type D = String&Int& Long", " = ") + test("type E = String &Int& Long", " = ") + test("type F = String & Int & Long", " = ") + test("fn[String|Char](input)", "fn[](input)") } } From 067be4a08c051fe6676d38c19163b76251c38464 Mon Sep 17 00:00:00 2001 From: Dmytro Melnychenko Date: Wed, 23 May 2018 11:32:15 +0200 Subject: [PATCH 08/17] Change ctx.error to println --- compiler/src/dotty/tools/dotc/printing/SyntaxHighlighting.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compiler/src/dotty/tools/dotc/printing/SyntaxHighlighting.scala b/compiler/src/dotty/tools/dotc/printing/SyntaxHighlighting.scala index 8ac4723c98d4..aa87708f894b 100644 --- a/compiler/src/dotty/tools/dotc/printing/SyntaxHighlighting.scala +++ b/compiler/src/dotty/tools/dotc/printing/SyntaxHighlighting.scala @@ -388,7 +388,7 @@ object SyntaxHighlighting { colorAt(i) = color } catch { case _: IndexOutOfBoundsException => - ctx.error("Encountered tree with invalid position, please open an issue with the code snippet that caused the error") + println("Encountered tree with invalid position, please open an issue with the code snippet that caused the error") } } def highlightPosition(pos: Position, color: String) = From 253bf0c5fb31c4b1fb1ef6bbfd4a08932e21cc20 Mon Sep 17 00:00:00 2001 From: Dmytro Melnychenko Date: Wed, 23 May 2018 11:37:56 +0200 Subject: [PATCH 09/17] Add annotations highlight - Highlight is based on MemberDef with annotations present - move scanner highlighter before parser so that annotation @inline wholdn't be highlighted as keyword - annotations are highlighted when full member definition is available --- .../dotc/printing/SyntaxHighlighting.scala | 48 ++++++++++--------- .../printing/SyntaxHighlightingTests.scala | 20 +++++++- 2 files changed, 44 insertions(+), 24 deletions(-) diff --git a/compiler/src/dotty/tools/dotc/printing/SyntaxHighlighting.scala b/compiler/src/dotty/tools/dotc/printing/SyntaxHighlighting.scala index aa87708f894b..31ccbef255df 100644 --- a/compiler/src/dotty/tools/dotc/printing/SyntaxHighlighting.scala +++ b/compiler/src/dotty/tools/dotc/printing/SyntaxHighlighting.scala @@ -369,17 +369,12 @@ object SyntaxHighlighting { override def doReport(m: MessageContainer)(implicit ctx: Context): Unit = () } - private val ignoredKwds = Set(nme.ARROWkw, nme.EQ, nme.EQL, nme.COLONkw) - def highlight(in: String)(ctx0: Context): String = { import dotty.tools.dotc.ast.untpd._ implicit val ctx: Context = ctx0.fresh.setReporter(new NoReporter) val source = new SourceFile("", in.toCharArray) - val parser = new Parser(source) - val trees = parser.blockStatSeq() - val colorAt = Array.fill(in.length)(NoColor) def highlightRange(from: Int, to: Int, color: String) = { @@ -394,18 +389,39 @@ object SyntaxHighlighting { def highlightPosition(pos: Position, color: String) = if (pos.exists) highlightRange(pos.start, pos.end, color) + val scanner = new Scanner(source) + + while (scanner.token != EOF) { + val isKwd = alphaKeywords.contains(scanner.token) + val offsetStart = scanner.offset + + if (scanner.token == IDENTIFIER && scanner.name == nme.???) { + highlightRange(scanner.offset, scanner.offset + scanner.name.length, Console.RED_B) + } + scanner.nextToken() + + if (isKwd) { + val offsetEnd = scanner.lastOffset + highlightPosition(Position(offsetStart, offsetEnd), KeywordColor) + } + } + val treeHighlighter = new UntypedTreeTraverser { def traverse(tree: Tree)(implicit ctx: Context): Unit = { tree match { - case id: Ident if id.isType => + case id : Ident if id.isType => highlightPosition(id.pos, TypeColor) case tpe : TypeDef => + for (annotation <- tpe.rawMods.annotations) + highlightPosition(annotation.pos, AnnotationColor) highlightPosition(tpe.namePos, TypeColor) case _ : TypTree => highlightPosition(tree.pos, TypeColor) case mod: ModuleDef => highlightPosition(mod.namePos, TypeColor) case v : ValOrDefDef => + for (annotation <- v.rawMods.annotations) + highlightPosition(annotation.pos, AnnotationColor) highlightPosition(v.namePos, ValDefColor) highlightPosition(v.tpt.pos, TypeColor) case _ : Literal => @@ -416,26 +432,12 @@ object SyntaxHighlighting { } } + val parser = new Parser(source) + val trees = parser.blockStatSeq() + for (tree <- trees) treeHighlighter.traverse(tree) - val scanner = new Scanner(source) - - while (scanner.token != EOF) { - val isKwd = isKeyword(scanner.token) && !ignoredKwds.contains(scanner.name) - val offsetStart = scanner.offset - - if (scanner.token == IDENTIFIER && scanner.name == nme.???) { - highlightRange(scanner.offset, scanner.offset + scanner.name.length, Console.RED_B) - } - scanner.nextToken() - - if (isKwd) { - val offsetEnd = scanner.lastOffset - highlightPosition(Position(offsetStart, offsetEnd), KeywordColor) - } - } - val sb = new mutable.StringBuilder() for (idx <- colorAt.indices) { diff --git a/compiler/test/dotty/tools/dotc/printing/SyntaxHighlightingTests.scala b/compiler/test/dotty/tools/dotc/printing/SyntaxHighlightingTests.scala index 1e774eaef25e..f26ddcd66384 100644 --- a/compiler/test/dotty/tools/dotc/printing/SyntaxHighlightingTests.scala +++ b/compiler/test/dotty/tools/dotc/printing/SyntaxHighlightingTests.scala @@ -58,7 +58,25 @@ class SyntaxHighlightingTests extends DottyTest { @Test def annotations = { - test("@tailrec", "") + val source = + """ + |@deprecated + |class Foo { + | @inline val bar = 42 + |} + """.stripMargin + + val expected = + """ + | + | { + | = + |} + """.stripMargin + + test(source, expected) + + test("@deprecated class Foo", " ") } @Test From 1f3f36cea968701f3f12182b843386d50b855dbd Mon Sep 17 00:00:00 2001 From: Dmytro Melnychenko Date: Mon, 28 May 2018 10:39:27 +0200 Subject: [PATCH 10/17] Remove old highlighter code and ignore tests with unsupported features --- .../tools/dotc/printing/Formatting.scala | 2 +- .../dotc/printing/SyntaxHighlighting.scala | 339 +----------------- .../dotc/reporting/MessageRendering.scala | 2 +- .../src/dotty/tools/repl/ReplDriver.scala | 9 +- .../printing/SyntaxHighlightingTests.scala | 6 +- 5 files changed, 13 insertions(+), 345 deletions(-) diff --git a/compiler/src/dotty/tools/dotc/printing/Formatting.scala b/compiler/src/dotty/tools/dotc/printing/Formatting.scala index f0ce23408f9d..1fb9392780dc 100644 --- a/compiler/src/dotty/tools/dotc/printing/Formatting.scala +++ b/compiler/src/dotty/tools/dotc/printing/Formatting.scala @@ -84,7 +84,7 @@ object Formatting { case hb: HighlightBuffer => hb.toString case str: String if ctx.settings.color.value != "never" => - new String(SyntaxHighlighting(str).toArray) + SyntaxHighlighting.highlight(str)(ctx) case _ => super.showArg(arg) } } diff --git a/compiler/src/dotty/tools/dotc/printing/SyntaxHighlighting.scala b/compiler/src/dotty/tools/dotc/printing/SyntaxHighlighting.scala index 31ccbef255df..feda84be7cfc 100644 --- a/compiler/src/dotty/tools/dotc/printing/SyntaxHighlighting.scala +++ b/compiler/src/dotty/tools/dotc/printing/SyntaxHighlighting.scala @@ -11,9 +11,7 @@ import dotty.tools.dotc.reporting.Reporter import dotty.tools.dotc.reporting.diagnostic.MessageContainer import dotty.tools.dotc.util.Positions.Position -import scala.annotation.switch -import scala.collection.mutable.StringBuilder -import util.{Chars, SourceFile} +import util.SourceFile import scala.collection.mutable @@ -30,341 +28,6 @@ object SyntaxHighlighting { val TypeColor = Console.MAGENTA val AnnotationColor = Console.MAGENTA - private def none(str: String) = str - private def keyword(str: String) = KeywordColor + str + NoColor - private def typeDef(str: String) = TypeColor + str + NoColor - private def literal(str: String) = LiteralColor + str + NoColor - private def valDef(str: String) = ValDefColor + str + NoColor - private def operator(str: String) = TypeColor + str + NoColor - private def annotation(str: String) = - if (str.trim == "@") str else { AnnotationColor + str + NoColor } - private val tripleQs = Console.RED_B + "???" + NoColor - - private val keywords: Seq[String] = for { - index <- IF to ERASED // All alpha keywords - } yield tokenString(index) - - private val interpolationPrefixes = - 'A' :: 'B' :: 'C' :: 'D' :: 'E' :: 'F' :: 'G' :: 'H' :: 'I' :: 'J' :: 'K' :: - 'L' :: 'M' :: 'N' :: 'O' :: 'P' :: 'Q' :: 'R' :: 'S' :: 'T' :: 'U' :: 'V' :: - 'W' :: 'X' :: 'Y' :: 'Z' :: '$' :: '_' :: 'a' :: 'b' :: 'c' :: 'd' :: 'e' :: - 'f' :: 'g' :: 'h' :: 'i' :: 'j' :: 'k' :: 'l' :: 'm' :: 'n' :: 'o' :: 'p' :: - 'q' :: 'r' :: 's' :: 't' :: 'u' :: 'v' :: 'w' :: 'x' :: 'y' :: 'z' :: Nil - - private val typeEnders = - '{' :: '}' :: ')' :: '(' :: '[' :: ']' :: '=' :: ' ' :: ',' :: '.' :: '|' :: - '&' :: '\n' :: Nil - - def apply(chars: Iterable[Char]): Iterable[Char] = { - var prev: Char = 0 - var remaining = chars.toStream - val newBuf = new StringBuilder - var lastValDefToken = "" - - @inline def keywordStart = - prev == 0 || prev == ' ' || prev == '{' || prev == '(' || - prev == '\n' || prev == '[' || prev == ',' || prev == ':' || - prev == '|' || prev == '&' - - @inline def numberStart(c: Char) = - c.isDigit && (!prev.isLetter || prev == '.' || prev == ' ' || prev == '(' || prev == '\u0000') - - def takeChar(): Char = takeChars(1).head - def takeChars(x: Int): Seq[Char] = { - val taken = remaining.take(x) - remaining = remaining.drop(x) - taken - } - - while (remaining.nonEmpty) { - val n = takeChar() - if (interpolationPrefixes.contains(n)) { - // Interpolation prefixes are a superset of the keyword start chars - val (prefix, after) = remaining.span(interpolationPrefixes.contains) - if (after.startsWith("\"")) { - newBuf += n ++= prefix - prev = prefix.lastOption.getOrElse(n) - if (remaining.nonEmpty) takeChars(prefix.length + 1) // drop 1 for appendLiteral - appendString('"', after.startsWith("\"\"\""), true) - } else { - if (n.isUpper && (keywordStart || prev == '.')) { - appendWhile(n, !typeEnders.contains(_), typeDef) - } else if (keywordStart) { - append(n, keywords.contains(_), { kw => - if (kw == "new") typeDef(kw) else keyword(kw) - }) - } else { - newBuf += n - prev = n - } - } - } else { - (n: @switch) match { - case '/' => - if (remaining.nonEmpty) { - remaining.head match { - case '/' => - takeChar() - eolComment() - case '*' => - takeChar() - blockComment() - case x => - newBuf += '/' - } - } else newBuf += '/' - case '=' => - append('=', _ == "=>", operator) - case '<' => - append('<', { x => x == "<-" || x == "<:" || x == "<%" }, operator) - case '>' => - append('>', { x => x == ">:" }, operator) - case '#' => - if (prev != ' ' && prev != '.') newBuf append operator("#") - else newBuf += n - prev = '#' - case '@' => - appendWhile('@', !typeEnders.contains(_), annotation) - case '\"' => - appendString('\"', multiline = remaining.take(2).mkString == "\"\"", false) - case '\'' => - appendSingleQuote('\'') - case '`' => - appendTo('`', _ == '`', none) - case _ => { - if (n == '?' && remaining.take(2).mkString == "??") { - takeChars(2) - newBuf append tripleQs - prev = '?' - } - else if (n.isUpper && keywordStart) - appendWhile(n, !typeEnders.contains(_), typeDef) - else if (numberStart(n)) { - def isNumber(c: Char): Boolean = - c.isDigit || c == '\u0000' || (c == '.' && remaining.nonEmpty && remaining.head.isDigit) - appendWhile(n, isNumber, literal) - } else - newBuf += n; prev = n - } - } - } - } - - def eolComment() = { - newBuf append (CommentColor + "//") - var curr = '/' - while (curr != '\n' && remaining.nonEmpty) { - curr = takeChar() - newBuf += curr - } - prev = curr - newBuf append NoColor - } - - def blockComment() = { - newBuf append (CommentColor + "/*") - var curr = '*' - var open = 1 - while (open > 0 && remaining.nonEmpty) { - curr = takeChar() - if (curr == '@') { - appendWhile('@', !typeEnders.contains(_), annotation) - newBuf append CommentColor - } - else newBuf += curr - - if (curr == '*' && remaining.nonEmpty) { - curr = takeChar() - newBuf += curr - if (curr == '/') open -= 1 - } else if (curr == '/' && remaining.nonEmpty) { - curr = takeChar() - newBuf += curr - if (curr == '*') open += 1 - } - - if (Chars.isLineBreakChar(curr)) { - newBuf append CommentColor - } - } - prev = curr - newBuf append NoColor - } - - def appendString(delim: Char, multiline: Boolean = false, inInterpolation: Boolean) = { - var curr: Char = 0 - var continue = true - var closing = 0 - newBuf append (LiteralColor + delim) - - def shouldInterpolate = - inInterpolation && curr == '$' && prev != '$' && remaining.nonEmpty - - def interpolate() = { - val next = takeChar() - if (next == '$') { - newBuf += curr - newBuf += next - prev = '$' - } else if (next == '{') { - var open = 1 // keep track of open blocks - newBuf append (ValDefColor + curr) - newBuf += next - while (remaining.nonEmpty && open > 0) { - var c = takeChar() - newBuf += c - if (c == '}') open -= 1 - else if (c == '{') open += 1 - } - newBuf append LiteralColor - } else { - newBuf append (ValDefColor + curr) - newBuf += next - var c: Char = 'a' - while (c.isLetterOrDigit && remaining.nonEmpty) { - c = takeChar() - if (c != '"') newBuf += c - } - newBuf append LiteralColor - if (c == '"') { - newBuf += c - continue = false - } - } - closing = 0 - } - - while (continue && remaining.nonEmpty) { - curr = takeChar() - if (curr == '\\' && remaining.nonEmpty) { - val next = takeChar() - newBuf append (KeywordColor + curr) - if (next == 'u') { - val code = "u" + takeChars(4).mkString - newBuf append code - } else newBuf += next - newBuf append LiteralColor - closing = 0 - } else if (shouldInterpolate) { - interpolate() - } else if (curr == delim && multiline) { - closing += 1 - if (closing == 3) continue = false - newBuf += curr - } else if (curr == delim) { - continue = false - newBuf += curr - } else { - newBuf += curr - closing = 0 - } - - if (Chars.isLineBreakChar(curr)) { - newBuf append LiteralColor - } - } - newBuf append NoColor - prev = curr - } - - def appendSingleQuote(delim: Char) = remaining.take(3) match { - case chr #:: '\'' #:: _ => // single character - newBuf append LiteralColor - newBuf appendAll s"'$chr'" - newBuf append NoColor - takeChars(2) - prev = '\'' - case '\\' #:: chr #:: '\'' #:: _ => // escaped character - newBuf append LiteralColor - newBuf appendAll s"'\\$chr'" - newBuf append NoColor - takeChars(3) - prev = '\'' - case _ => appendWhile(delim, !typeEnders.contains(_), literal) - } - - def append(c: Char, shouldHL: String => Boolean, highlight: String => String) = { - var curr: Char = 0 - val sb = new StringBuilder(s"$c") - - def delim(c: Char) = (c: @switch) match { - case ' ' => true - case '\n' => true - case '(' => true - case ')' => true - case '[' => true - case ']' => true - case ':' => true - case '@' => true - case ',' => true - case '.' => true - case _ => false - } - - val valDefStarterTokens = "var" :: "val" :: "def" :: "case" :: Nil - - /** lastValDefToken is used to check whether we want to show something - * in valDef color or not. There are only a few cases when lastValDefToken - * should be updated, that way we can avoid stopping coloring too early. - * eg.: case A(x, y, z) => ??? - * Without this function only x would be colored. - */ - def updateLastToken(currentToken: String): String = - (lastValDefToken, currentToken) match { - case _ if valDefStarterTokens.contains(currentToken) => currentToken - case (("val" | "var"), "=") => currentToken - case ("case", ("=>" | "class" | "object")) => currentToken - case ("def", _) => currentToken - case _ => lastValDefToken - } - - while (remaining.nonEmpty && !delim(curr)) { - curr = takeChar() - if (!delim(curr)) sb += curr - } - - val str = sb.toString - val toAdd = - if (shouldHL(str)) - highlight(str) - else if (valDefStarterTokens.contains(lastValDefToken) && !List("=", "=>").contains(str)) - valDef(str) - else str - val suffix = if (delim(curr)) s"$curr" else "" - newBuf append (toAdd + suffix) - lastValDefToken = updateLastToken(str) - prev = curr - } - - def appendWhile(c: Char, pred: Char => Boolean, highlight: String => String) = { - var curr: Char = 0 - val sb = new StringBuilder(s"$c") - while (remaining.nonEmpty && pred(curr)) { - curr = takeChar() - if (pred(curr)) sb += curr - } - - val str = sb.toString - val suffix = if (!pred(curr)) s"$curr" else "" - newBuf append (highlight(str) + suffix) - prev = curr - } - - def appendTo(c: Char, pred: Char => Boolean, highlight: String => String) = { - var curr: Char = 0 - val sb = new StringBuilder(s"$c") - while (remaining.nonEmpty && !pred(curr)) { - curr = takeChar() - sb += curr - } - - newBuf append highlight(sb.toString) - prev = curr - } - - newBuf.toIterable - } - private class NoReporter extends Reporter { override def doReport(m: MessageContainer)(implicit ctx: Context): Unit = () } diff --git a/compiler/src/dotty/tools/dotc/reporting/MessageRendering.scala b/compiler/src/dotty/tools/dotc/reporting/MessageRendering.scala index 65443a2377e3..60bb5762416f 100644 --- a/compiler/src/dotty/tools/dotc/reporting/MessageRendering.scala +++ b/compiler/src/dotty/tools/dotc/reporting/MessageRendering.scala @@ -62,7 +62,7 @@ trait MessageRendering { val syntax = if (ctx.settings.color.value != "never") - SyntaxHighlighting(pos.linesSlice).toArray + SyntaxHighlighting.highlight(pos.linesSlice.mkString)(ctx).toArray else pos.linesSlice val lines = linesFrom(syntax) val (before, after) = pos.beforeAndAfterPoint diff --git a/compiler/src/dotty/tools/repl/ReplDriver.scala b/compiler/src/dotty/tools/repl/ReplDriver.scala index 278a1203d5bb..d745428204ba 100644 --- a/compiler/src/dotty/tools/repl/ReplDriver.scala +++ b/compiler/src/dotty/tools/repl/ReplDriver.scala @@ -267,7 +267,7 @@ class ReplDriver(settings: Array[String], typeAliases.map("// defined alias " + _.symbol.showUser) ++ defs.map(rendering.renderMethod) ++ vals.map(rendering.renderVal).flatten - ).foreach(str => out.println(SyntaxHighlighting(str))) + ).foreach(str => out.println(SyntaxHighlighting.highlight(str)(ctx))) state.copy(valIndex = state.valIndex - vals.count(resAndUnit)) } @@ -282,7 +282,7 @@ class ReplDriver(settings: Array[String], x.symbol } .foreach { sym => - out.println(SyntaxHighlighting("// defined " + sym.showUser)) + out.println(SyntaxHighlighting.highlight("// defined " + sym.showUser)(ctx)) } @@ -317,7 +317,8 @@ class ReplDriver(settings: Array[String], initState case Imports => - state.imports.foreach(i => out.println(SyntaxHighlighting(i.show(state.context)))) + implicit val ctx = state.context + state.imports.foreach(i => out.println(SyntaxHighlighting.highlight(i.show)) state case Load(path) => @@ -334,7 +335,7 @@ class ReplDriver(settings: Array[String], case TypeOf(expr) => compiler.typeOf(expr)(newRun(state)).fold( displayErrors, - res => out.println(SyntaxHighlighting(res)) + res => out.println(SyntaxHighlighting.highlight(res)(state.run.runContext)) ) state diff --git a/compiler/test/dotty/tools/dotc/printing/SyntaxHighlightingTests.scala b/compiler/test/dotty/tools/dotc/printing/SyntaxHighlightingTests.scala index f26ddcd66384..7e5f124b07cc 100644 --- a/compiler/test/dotty/tools/dotc/printing/SyntaxHighlightingTests.scala +++ b/compiler/test/dotty/tools/dotc/printing/SyntaxHighlightingTests.scala @@ -2,7 +2,7 @@ package dotty.tools.dotc.printing import dotty.tools.DottyTest import org.junit.Assert._ -import org.junit.Test +import org.junit.{Ignore, Test} /** Adapted from Ammonite HighlightTests */ @@ -26,6 +26,7 @@ class SyntaxHighlightingTests extends DottyTest { } } + @Ignore("comments are not properly supported yet") @Test def comments = { test("//a", "") @@ -56,6 +57,7 @@ class SyntaxHighlightingTests extends DottyTest { test("raw\"\"\"Hello\"\"\"", "raw") } + @Ignore("annotations handling has to be improved") @Test def annotations = { val source = @@ -84,6 +86,7 @@ class SyntaxHighlightingTests extends DottyTest { test("val x = 1 + 2 + 3", " = + + ") } + @Ignore("comments are not properly supported yet") @Test def valDef = { test("val a = 123", " = ") @@ -95,6 +98,7 @@ class SyntaxHighlightingTests extends DottyTest { test("def f2[T](x: T) = { 123 }", " [](x: ) = { }") } + @Ignore("not properly supported yet") @Test def patternMatching = { test("""val aFruit: Fruit = Apple("red", 123)""", From 05aa49d945ebb2a71293a968a30adac43361fd0f Mon Sep 17 00:00:00 2001 From: Allan Renucci Date: Fri, 1 Jun 2018 11:32:11 +0200 Subject: [PATCH 11/17] Polishing Fix #4500 --- .../tools/dotc/printing/Formatting.scala | 4 +- .../dotc/printing/SyntaxHighlighting.scala | 151 +++++++++--------- .../dotc/reporting/MessageRendering.scala | 2 +- .../src/dotty/tools/repl/ReplDriver.scala | 6 +- .../printing/SyntaxHighlightingTests.scala | 76 +++------ 5 files changed, 105 insertions(+), 134 deletions(-) diff --git a/compiler/src/dotty/tools/dotc/printing/Formatting.scala b/compiler/src/dotty/tools/dotc/printing/Formatting.scala index 1fb9392780dc..7eea45fd2c63 100644 --- a/compiler/src/dotty/tools/dotc/printing/Formatting.scala +++ b/compiler/src/dotty/tools/dotc/printing/Formatting.scala @@ -83,8 +83,8 @@ object Formatting { hl.show case hb: HighlightBuffer => hb.toString - case str: String if ctx.settings.color.value != "never" => - SyntaxHighlighting.highlight(str)(ctx) + case str: String => + SyntaxHighlighting.highlight(str) case _ => super.showArg(arg) } } diff --git a/compiler/src/dotty/tools/dotc/printing/SyntaxHighlighting.scala b/compiler/src/dotty/tools/dotc/printing/SyntaxHighlighting.scala index feda84be7cfc..f3f9d49012a9 100644 --- a/compiler/src/dotty/tools/dotc/printing/SyntaxHighlighting.scala +++ b/compiler/src/dotty/tools/dotc/printing/SyntaxHighlighting.scala @@ -1,7 +1,6 @@ -package dotty.tools -package dotc -package printing +package dotty.tools.dotc.printing +import dotty.tools.dotc.ast.untpd import dotty.tools.dotc.core.Contexts.Context import dotty.tools.dotc.core.StdNames._ import dotty.tools.dotc.parsing.Parsers.Parser @@ -10,14 +9,14 @@ import dotty.tools.dotc.parsing.Tokens._ import dotty.tools.dotc.reporting.Reporter import dotty.tools.dotc.reporting.diagnostic.MessageContainer import dotty.tools.dotc.util.Positions.Position - -import util.SourceFile - -import scala.collection.mutable +import dotty.tools.dotc.util.SourceFile /** This object provides functions for syntax highlighting in the REPL */ object SyntaxHighlighting { + /** if true, log erroneous positions being highlighted */ + private final val debug = false + // Keep in sync with SyntaxHighlightingTests val NoColor = Console.RESET val CommentColor = Console.BLUE @@ -32,88 +31,90 @@ object SyntaxHighlighting { override def doReport(m: MessageContainer)(implicit ctx: Context): Unit = () } - def highlight(in: String)(ctx0: Context): String = { - import dotty.tools.dotc.ast.untpd._ - - implicit val ctx: Context = ctx0.fresh.setReporter(new NoReporter) - - val source = new SourceFile("", in.toCharArray) - val colorAt = Array.fill(in.length)(NoColor) + def highlight(in: String)(implicit ctx: Context): String = { + def freshCtx = ctx.fresh.setReporter(new NoReporter) + if (in.isEmpty || ctx.settings.color.value == "never") in + else { + implicit val ctx = freshCtx + val source = new SourceFile("", in.toCharArray) + val colorAt = Array.fill(in.length)(NoColor) - def highlightRange(from: Int, to: Int, color: String) = { - try { + def highlightRange(from: Int, to: Int, color: String) = for (i <- from until to) colorAt(i) = color - } catch { - case _: IndexOutOfBoundsException => - println("Encountered tree with invalid position, please open an issue with the code snippet that caused the error") - } - } - def highlightPosition(pos: Position, color: String) = - if (pos.exists) highlightRange(pos.start, pos.end, color) - - val scanner = new Scanner(source) - while (scanner.token != EOF) { - val isKwd = alphaKeywords.contains(scanner.token) - val offsetStart = scanner.offset - - if (scanner.token == IDENTIFIER && scanner.name == nme.???) { - highlightRange(scanner.offset, scanner.offset + scanner.name.length, Console.RED_B) + def highlightPosition(pos: Position, color: String) = if (pos.exists) { + if (pos.start < 0 || pos.end > in.length) { + if (debug) + println(s"Trying to highlight erroneous position $pos. Input size: ${in.length}") + } + else + highlightRange(pos.start, pos.end, color) } - scanner.nextToken() - if (isKwd) { - val offsetEnd = scanner.lastOffset - highlightPosition(Position(offsetStart, offsetEnd), KeywordColor) + val scanner = new Scanner(source) + while (scanner.token != EOF) { + val start = scanner.offset + val token = scanner.token + val name = scanner.name + scanner.nextToken() + val end = scanner.lastOffset + + if (alphaKeywords.contains(token)) + highlightRange(start, end, KeywordColor) + else if (token == IDENTIFIER && name == nme.???) + highlightRange(start, end, Console.RED_B) } - } - val treeHighlighter = new UntypedTreeTraverser { - def traverse(tree: Tree)(implicit ctx: Context): Unit = { - tree match { - case id : Ident if id.isType => - highlightPosition(id.pos, TypeColor) - case tpe : TypeDef => - for (annotation <- tpe.rawMods.annotations) - highlightPosition(annotation.pos, AnnotationColor) - highlightPosition(tpe.namePos, TypeColor) - case _ : TypTree => - highlightPosition(tree.pos, TypeColor) - case mod: ModuleDef => - highlightPosition(mod.namePos, TypeColor) - case v : ValOrDefDef => - for (annotation <- v.rawMods.annotations) - highlightPosition(annotation.pos, AnnotationColor) - highlightPosition(v.namePos, ValDefColor) - highlightPosition(v.tpt.pos, TypeColor) - case _ : Literal => - highlightPosition(tree.pos, LiteralColor) - case _ => + val treeHighlighter = new untpd.UntypedTreeTraverser { + import untpd._ + + def ignored(tree: NameTree) = { + val name = tree.name.toTermName + // trees named and have weird positions + name == nme.ERROR || name == nme.CONSTRUCTOR } - traverseChildren(tree) - } - } - val parser = new Parser(source) - val trees = parser.blockStatSeq() + def traverse(tree: Tree)(implicit ctx: Context): Unit = { + tree match { + case tree: NameTree if ignored(tree) => + () + case tree: MemberDef /* ValOrDefDef | ModuleDef | TypeDef */ => + for (annotation <- tree.rawMods.annotations) + highlightPosition(annotation.pos, AnnotationColor) + val color = if (tree.isInstanceOf[ValOrDefDef]) ValDefColor else TypeColor + highlightPosition(tree.namePos, color) + case tree : Ident if tree.isType => + highlightPosition(tree.pos, TypeColor) + case _ : TypTree => + highlightPosition(tree.pos, TypeColor) + case _ : Literal => + highlightPosition(tree.pos, LiteralColor) + case _ => + } + traverseChildren(tree) + } + } - for (tree <- trees) - treeHighlighter.traverse(tree) + val parser = new Parser(source) + val trees = parser.blockStatSeq() + for (tree <- trees) + treeHighlighter.traverse(tree) - val sb = new mutable.StringBuilder() + val highlighted = new StringBuilder() - for (idx <- colorAt.indices) { - if ( (idx == 0 && colorAt(idx) != NoColor) - || (idx > 0 && colorAt(idx-1) != colorAt(idx))) { - sb.append(colorAt(idx)) + for (idx <- colorAt.indices) { + val prev = if (idx == 0) NoColor else colorAt(idx - 1) + val curr = colorAt(idx) + if (curr != prev) + highlighted.append(curr) + highlighted.append(in(idx)) } - sb.append(in(idx)) - } - if (colorAt.nonEmpty && colorAt.last != NoColor) { - sb.append(NoColor) - } - sb.toString + if (colorAt.last != NoColor) + highlighted.append(NoColor) + + highlighted.toString + } } } diff --git a/compiler/src/dotty/tools/dotc/reporting/MessageRendering.scala b/compiler/src/dotty/tools/dotc/reporting/MessageRendering.scala index 60bb5762416f..64df413340b6 100644 --- a/compiler/src/dotty/tools/dotc/reporting/MessageRendering.scala +++ b/compiler/src/dotty/tools/dotc/reporting/MessageRendering.scala @@ -62,7 +62,7 @@ trait MessageRendering { val syntax = if (ctx.settings.color.value != "never") - SyntaxHighlighting.highlight(pos.linesSlice.mkString)(ctx).toArray + SyntaxHighlighting.highlight(new String(pos.linesSlice)).toArray else pos.linesSlice val lines = linesFrom(syntax) val (before, after) = pos.beforeAndAfterPoint diff --git a/compiler/src/dotty/tools/repl/ReplDriver.scala b/compiler/src/dotty/tools/repl/ReplDriver.scala index d745428204ba..ee029dfdcbfa 100644 --- a/compiler/src/dotty/tools/repl/ReplDriver.scala +++ b/compiler/src/dotty/tools/repl/ReplDriver.scala @@ -267,7 +267,7 @@ class ReplDriver(settings: Array[String], typeAliases.map("// defined alias " + _.symbol.showUser) ++ defs.map(rendering.renderMethod) ++ vals.map(rendering.renderVal).flatten - ).foreach(str => out.println(SyntaxHighlighting.highlight(str)(ctx))) + ).foreach(str => out.println(SyntaxHighlighting.highlight(str))) state.copy(valIndex = state.valIndex - vals.count(resAndUnit)) } @@ -282,7 +282,9 @@ class ReplDriver(settings: Array[String], x.symbol } .foreach { sym => - out.println(SyntaxHighlighting.highlight("// defined " + sym.showUser)(ctx)) + // FIXME syntax highlighting on comment is currently not working + // out.println(SyntaxHighlighting.highlight("// defined " + sym.showUser)) + out.println(SyntaxHighlighting.CommentColor + "// defined " + sym.showUser + SyntaxHighlighting.NoColor) } diff --git a/compiler/test/dotty/tools/dotc/printing/SyntaxHighlightingTests.scala b/compiler/test/dotty/tools/dotc/printing/SyntaxHighlightingTests.scala index 7e5f124b07cc..7abd5e032b5d 100644 --- a/compiler/test/dotty/tools/dotc/printing/SyntaxHighlightingTests.scala +++ b/compiler/test/dotty/tools/dotc/printing/SyntaxHighlightingTests.scala @@ -26,8 +26,8 @@ class SyntaxHighlightingTests extends DottyTest { } } - @Ignore("comments are not properly supported yet") @Test + @Ignore("Comments are currently not supported") def comments = { test("//a", "") test("/** a */", "") @@ -37,6 +37,8 @@ class SyntaxHighlightingTests extends DottyTest { @Test def types = { test("type Foo = Int", " = ") + test("type A = String | Int", " = ") + test("type B = String & Int", " = ") } @Test @@ -51,76 +53,42 @@ class SyntaxHighlightingTests extends DottyTest { def strings = { // For some reason we currently use literal color for string test("\"Hello\"", "") - test("s\"Hello\"", "s") - test("s\"Hello $name\"", "s") - test("raw\"Hello\"", "raw") - test("raw\"\"\"Hello\"\"\"", "raw") + test("\"\"\"Hello\"\"\"", "") + + // FIXME: '$' should not be colored (literal position is off by one) + // test("s\"Hello\"", "s") + // test("s\"Hello $name\"", "s") + // test("raw\"Hello\"", "raw") + // test("raw\"\"\"Hello\"\"\"", "raw") } - @Ignore("annotations handling has to be improved") @Test def annotations = { - val source = - """ - |@deprecated - |class Foo { - | @inline val bar = 42 - |} - """.stripMargin - - val expected = - """ - | - | { - | = - |} - """.stripMargin - - test(source, expected) - test("@deprecated class Foo", " ") + // test("@Test(\"Hello\") class Foo", " ") // FIXME + test("@annotation.tailrec def foo = 1", " = ") } @Test def expressions = { + test("if (true) 1 else 2", " () ") test("val x = 1 + 2 + 3", " = + + ") } - @Ignore("comments are not properly supported yet") @Test - def valDef = { + def valOrDefDef = { test("val a = 123", " = ") - test("var b = 123 /*Int*/", " = ") - test("""var c = "123" // String""", """ = """) - test("var e:Int = 123;e", " : = ;e") + test("var e: Int = 123", " : = ") test("def f = 123", " = ") - test("def f1(x: Int) = 123", " (x: ) = ") - test("def f2[T](x: T) = { 123 }", " [](x: ) = { }") - } - - @Ignore("not properly supported yet") - @Test - def patternMatching = { - test("""val aFruit: Fruit = Apple("red", 123)""", - """ : = (, )""") - test("""val Apple(color, weight) = aFruit""", - """ (, ) = aFruit""") - test("""case Apple(_, weight) => println(s"apple: $weight kgs")""", - """ (, ) > println(s)""") - test("""case o: Orange => println(s"orange ${o.weight} kgs")""", - """ : > println(s)""") - test("""case m @ Melon(weight) => println(s"melon: ${m.weight} kgs")""", - """ @ () > println(s)""") + test("def f1(x: Int) = 123", " (: ) = ") + test("def f2[T](x: T) = { 123 }", " [](: ) = { }") } @Test - def unionTypes = { - test("type A = String|Int| Long", " = ") - test("type B = String |Int| Long", " = ") - test("type C = String | Int | Long", " = ") - test("type D = String&Int& Long", " = ") - test("type E = String &Int& Long", " = ") - test("type F = String & Int & Long", " = ") - test("fn[String|Char](input)", "fn[](input)") + @Ignore("TODO: Not implemented") + def patterns = { + test("val Foo(x) = foo", ???) + test("val foo @ Foo(x) = bar", ???) + test("x match { case Foo | Bar => 1 }", ???) } } From 20f431b993e1c639313486185347eb252e794fd4 Mon Sep 17 00:00:00 2001 From: Allan Renucci Date: Fri, 1 Jun 2018 18:41:00 +0200 Subject: [PATCH 12/17] Let's turn on debug mode by default If we ignore trees named and , it is not so fragile after all... --- .../dotty/tools/dotc/printing/SyntaxHighlighting.scala | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/compiler/src/dotty/tools/dotc/printing/SyntaxHighlighting.scala b/compiler/src/dotty/tools/dotc/printing/SyntaxHighlighting.scala index f3f9d49012a9..55880f3171ff 100644 --- a/compiler/src/dotty/tools/dotc/printing/SyntaxHighlighting.scala +++ b/compiler/src/dotty/tools/dotc/printing/SyntaxHighlighting.scala @@ -15,7 +15,7 @@ import dotty.tools.dotc.util.SourceFile object SyntaxHighlighting { /** if true, log erroneous positions being highlighted */ - private final val debug = false + private final val debug = true // Keep in sync with SyntaxHighlightingTests val NoColor = Console.RESET @@ -84,11 +84,11 @@ object SyntaxHighlighting { highlightPosition(annotation.pos, AnnotationColor) val color = if (tree.isInstanceOf[ValOrDefDef]) ValDefColor else TypeColor highlightPosition(tree.namePos, color) - case tree : Ident if tree.isType => + case tree: Ident if tree.isType => highlightPosition(tree.pos, TypeColor) - case _ : TypTree => + case _: TypTree => highlightPosition(tree.pos, TypeColor) - case _ : Literal => + case _: Literal => highlightPosition(tree.pos, LiteralColor) case _ => } From d31f32ec6325c702cf6d265a82d4933ab677393d Mon Sep 17 00:00:00 2001 From: Daniel Li Date: Tue, 19 Jun 2018 15:02:06 -0400 Subject: [PATCH 13/17] Fix position error in annotation parameter list parsing Prior to this commit, an annotation with one or more parameter lists would have incorrect positions attached to each parameter list. For example, the annotation @Foo("bar") would have the erroneous position `<4..10>` attached to its `Apply` node instead of the correct position `<4..11>`. This was caused by forget- ting to take into account the closing parenthesis. --- compiler/src/dotty/tools/dotc/parsing/Parsers.scala | 4 +++- .../dotty/tools/dotc/printing/SyntaxHighlightingTests.scala | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/compiler/src/dotty/tools/dotc/parsing/Parsers.scala b/compiler/src/dotty/tools/dotc/parsing/Parsers.scala index 9ffa8cf9c52e..67e5d5b0d954 100644 --- a/compiler/src/dotty/tools/dotc/parsing/Parsers.scala +++ b/compiler/src/dotty/tools/dotc/parsing/Parsers.scala @@ -1447,7 +1447,9 @@ object Parsers { } } if (in.token == LPAREN && (!inClassConstrAnnots || isLegalAnnotArg)) - parArgumentExprss(Apply(fn, parArgumentExprs())) + parArgumentExprss( + atPos(in.offset) { Apply(fn, parArgumentExprs()) } + ) else fn } diff --git a/compiler/test/dotty/tools/dotc/printing/SyntaxHighlightingTests.scala b/compiler/test/dotty/tools/dotc/printing/SyntaxHighlightingTests.scala index 7abd5e032b5d..3a8d8f441dfb 100644 --- a/compiler/test/dotty/tools/dotc/printing/SyntaxHighlightingTests.scala +++ b/compiler/test/dotty/tools/dotc/printing/SyntaxHighlightingTests.scala @@ -65,7 +65,9 @@ class SyntaxHighlightingTests extends DottyTest { @Test def annotations = { test("@deprecated class Foo", " ") - // test("@Test(\"Hello\") class Foo", " ") // FIXME + test("@Test() class Foo", " ") + test("@Test(\"Hello\") class Foo", " ") + test("@Test(\"Hello\")(\"World\") class Foo", " ") test("@annotation.tailrec def foo = 1", " = ") } From a41d4f70e46ca47981104c363141dd64458b62ec Mon Sep 17 00:00:00 2001 From: Daniel Li Date: Wed, 20 Jun 2018 17:32:13 -0400 Subject: [PATCH 14/17] Add tests for mid-expression user inputs These tests currently fail because of a bug in the syntax highlighter. --- .../printing/SyntaxHighlightingTests.scala | 31 ++++++++++++++++--- 1 file changed, 27 insertions(+), 4 deletions(-) diff --git a/compiler/test/dotty/tools/dotc/printing/SyntaxHighlightingTests.scala b/compiler/test/dotty/tools/dotc/printing/SyntaxHighlightingTests.scala index 3a8d8f441dfb..a96b2ec05253 100644 --- a/compiler/test/dotty/tools/dotc/printing/SyntaxHighlightingTests.scala +++ b/compiler/test/dotty/tools/dotc/printing/SyntaxHighlightingTests.scala @@ -36,6 +36,8 @@ class SyntaxHighlightingTests extends DottyTest { @Test def types = { + test("type Foo", " ") + test("type Foo =", " =") test("type Foo = Int", " = ") test("type A = String | Int", " = ") test("type B = String & Int", " = ") @@ -74,14 +76,35 @@ class SyntaxHighlightingTests extends DottyTest { @Test def expressions = { test("if (true) 1 else 2", " () ") - test("val x = 1 + 2 + 3", " = + + ") + test("1 + 2 + 3", " + + ") } @Test def valOrDefDef = { - test("val a = 123", " = ") - test("var e: Int = 123", " : = ") - test("def f = 123", " = ") + test("val", "") + test("val foo", " ") + test("val foo =", " =") + test("val foo = 123", " = ") + + test("var", "") + test("var foo", " ") + test("var foo:", " :") + test("var foo: Int", " : ") + test("var foo: Int =", " : =") + test("var foo: Int = 123", " : = ") + + test("def", "") + test("def foo", " ") + test("def foo(", " (") + test("def foo(bar", " (") + test("def foo(bar:", " (:") + test("def foo(bar: Int", " (: ") + test("def foo(bar: Int)", " (: )") + test("def foo(bar: Int):", " (: ):") + test("def foo(bar: Int): Int", " (: ): ") + test("def foo(bar: Int): Int =", " (: ): =") + test("def foo(bar: Int): Int = 123", " (: ): = ") + test("def f1(x: Int) = 123", " (: ) = ") test("def f2[T](x: T) = { 123 }", " [](: ) = { }") } From 1380ad53d50542e1a0de3c21dee8de11827f43fc Mon Sep 17 00:00:00 2001 From: Daniel Li Date: Wed, 20 Jun 2018 17:53:25 -0400 Subject: [PATCH 15/17] Add tests for long and float literals Test for `1Lx` currently fails; tests for float literals pass. --- .../dotty/tools/dotc/printing/SyntaxHighlightingTests.scala | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/compiler/test/dotty/tools/dotc/printing/SyntaxHighlightingTests.scala b/compiler/test/dotty/tools/dotc/printing/SyntaxHighlightingTests.scala index a96b2ec05253..6d3fd63563fe 100644 --- a/compiler/test/dotty/tools/dotc/printing/SyntaxHighlightingTests.scala +++ b/compiler/test/dotty/tools/dotc/printing/SyntaxHighlightingTests.scala @@ -48,7 +48,11 @@ class SyntaxHighlightingTests extends DottyTest { test("1", "") test("1.1", "") test("1.1.toString", ".toString") - // test("1L", "") + test("1L", "") + test("1Lx", "1Lx") + test("1f", "") + test("1.1f", "") + test("1.1fx", "1.1fx") } @Test From 9ea233baecf4910480f34b1af4f56352e6e10811 Mon Sep 17 00:00:00 2001 From: Allan Renucci Date: Fri, 29 Jun 2018 14:59:40 +0200 Subject: [PATCH 16/17] Fix rebase breakages --- .../dotty/tools/dotc/printing/SyntaxHighlighting.scala | 8 ++------ .../src/dotty/tools/dotc/reporting/MessageRendering.scala | 2 +- compiler/src/dotty/tools/repl/JLineTerminal.scala | 4 ++-- compiler/src/dotty/tools/repl/ReplDriver.scala | 4 ++-- 4 files changed, 7 insertions(+), 11 deletions(-) diff --git a/compiler/src/dotty/tools/dotc/printing/SyntaxHighlighting.scala b/compiler/src/dotty/tools/dotc/printing/SyntaxHighlighting.scala index 55880f3171ff..7a9f6acf3deb 100644 --- a/compiler/src/dotty/tools/dotc/printing/SyntaxHighlighting.scala +++ b/compiler/src/dotty/tools/dotc/printing/SyntaxHighlighting.scala @@ -27,16 +27,12 @@ object SyntaxHighlighting { val TypeColor = Console.MAGENTA val AnnotationColor = Console.MAGENTA - private class NoReporter extends Reporter { - override def doReport(m: MessageContainer)(implicit ctx: Context): Unit = () - } - def highlight(in: String)(implicit ctx: Context): String = { - def freshCtx = ctx.fresh.setReporter(new NoReporter) + def freshCtx = ctx.fresh.setReporter(Reporter.NoReporter) if (in.isEmpty || ctx.settings.color.value == "never") in else { implicit val ctx = freshCtx - val source = new SourceFile("", in.toCharArray) + val source = new SourceFile("", in) val colorAt = Array.fill(in.length)(NoColor) def highlightRange(from: Int, to: Int, color: String) = diff --git a/compiler/src/dotty/tools/dotc/reporting/MessageRendering.scala b/compiler/src/dotty/tools/dotc/reporting/MessageRendering.scala index 64df413340b6..45f1601bdc98 100644 --- a/compiler/src/dotty/tools/dotc/reporting/MessageRendering.scala +++ b/compiler/src/dotty/tools/dotc/reporting/MessageRendering.scala @@ -62,7 +62,7 @@ trait MessageRendering { val syntax = if (ctx.settings.color.value != "never") - SyntaxHighlighting.highlight(new String(pos.linesSlice)).toArray + SyntaxHighlighting.highlight(new String(pos.linesSlice)).toCharArray else pos.linesSlice val lines = linesFrom(syntax) val (before, after) = pos.beforeAndAfterPoint diff --git a/compiler/src/dotty/tools/repl/JLineTerminal.scala b/compiler/src/dotty/tools/repl/JLineTerminal.scala index d73bc295fe78..b7486b17f343 100644 --- a/compiler/src/dotty/tools/repl/JLineTerminal.scala +++ b/compiler/src/dotty/tools/repl/JLineTerminal.scala @@ -63,9 +63,9 @@ final class JLineTerminal extends java.io.Closeable { def close() = terminal.close() /** Provide syntax highlighting */ - private class Highlighter extends reader.Highlighter { + private class Highlighter(implicit ctx: Context) extends reader.Highlighter { def highlight(reader: LineReader, buffer: String): AttributedString = { - val highlighted = SyntaxHighlighting(buffer).mkString + val highlighted = SyntaxHighlighting.highlight(buffer) AttributedString.fromAnsi(highlighted) } } diff --git a/compiler/src/dotty/tools/repl/ReplDriver.scala b/compiler/src/dotty/tools/repl/ReplDriver.scala index ee029dfdcbfa..3507a63ffe8a 100644 --- a/compiler/src/dotty/tools/repl/ReplDriver.scala +++ b/compiler/src/dotty/tools/repl/ReplDriver.scala @@ -320,7 +320,7 @@ class ReplDriver(settings: Array[String], case Imports => implicit val ctx = state.context - state.imports.foreach(i => out.println(SyntaxHighlighting.highlight(i.show)) + state.imports.foreach(i => out.println(SyntaxHighlighting.highlight(i.show))) state case Load(path) => @@ -337,7 +337,7 @@ class ReplDriver(settings: Array[String], case TypeOf(expr) => compiler.typeOf(expr)(newRun(state)).fold( displayErrors, - res => out.println(SyntaxHighlighting.highlight(res)(state.run.runContext)) + res => out.println(SyntaxHighlighting.highlight(res)(state.context)) ) state From 47fb71079b6117944340d35a1152b69b3f8e5a4b Mon Sep 17 00:00:00 2001 From: Guillaume Martres Date: Fri, 29 Jun 2018 18:44:06 +0200 Subject: [PATCH 17/17] Fix position of parsed applications The end position was correctly set but the start position was wrong: the start position of `fn(args)` should be the same as the start position of `fn` itself. --- compiler/src/dotty/tools/dotc/parsing/Parsers.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compiler/src/dotty/tools/dotc/parsing/Parsers.scala b/compiler/src/dotty/tools/dotc/parsing/Parsers.scala index 67e5d5b0d954..a5615053767d 100644 --- a/compiler/src/dotty/tools/dotc/parsing/Parsers.scala +++ b/compiler/src/dotty/tools/dotc/parsing/Parsers.scala @@ -1448,7 +1448,7 @@ object Parsers { } if (in.token == LPAREN && (!inClassConstrAnnots || isLegalAnnotArg)) parArgumentExprss( - atPos(in.offset) { Apply(fn, parArgumentExprs()) } + atPos(startOffset(fn)) { Apply(fn, parArgumentExprs()) } ) else fn }