Add Stable Presentation Compiler (#17528)

This PR adds a stable presentation compiler implementation to dotty.
This will ensure that each future version of Scala 3 is shipped with
presentation compiler compiled against it, guaranteeing that IDE will
support it. It will also improve support for projects relying on
nonbootstrapped compiler such as scaladoc or dotty-language-server, as
it is now possible to easily publish presentation compiler for those
versions from dotty repository.

It also adds vast of tests suits ported from metals, which will also
help to detect unintended changes before they are merged. More
information about this initiative can be found here:
https://contributors.scala-lang.org/t/stable-presentation-compiler-api/6139

Author: rochala

NOTICE.md

@@ -89,15 +89,19 @@ major authors were omitted by oversight.
     details.
 
   * dotty.tools.dotc.coverage: Coverage instrumentation utilities have been
-    adapted from the scoverage plugin for scala 2 [5], which is under the
+    adapted from the scoverage plugin for scala 2 [4], which is under the
     Apache 2.0 license.
 
+  * dooty.tools.pc: Presentation compiler implementation adapted from
+    scalameta/metals [5] mtags module, which is under the Apache 2.0 license.
+
   * The Dotty codebase contains parts which are derived from
-    the ScalaPB protobuf library [4], which is under the Apache 2.0 license.
+    the ScalaPB protobuf library [6], which is under the Apache 2.0 license.
 
 
 [1] https://github.com/scala/scala
 [2] https://github.com/adriaanm/scala/tree/sbt-api-consolidate/src/compiler/scala/tools/sbt
 [3] https://github.com/sbt/sbt/tree/0.13/compile/interface/src/main/scala/xsbt
-[4] https://github.com/lampepfl/dotty/pull/5783/files
-[5] https://github.com/scoverage/scalac-scoverage-plugin
+[4] https://github.com/scoverage/scalac-scoverage-plugin
+[5] https://github.com/scalameta/metals
+[6] https://github.com/lampepfl/dotty/pull/5783/files

build.sbt

@@ -29,6 +29,8 @@ val `scala3-bench-run` = Build.`scala3-bench-run`
 val dist = Build.dist
 val `community-build` = Build.`community-build`
 val `sbt-community-build` = Build.`sbt-community-build`
+val `scala3-presentation-compiler` = Build.`scala3-presentation-compiler`
+val `scala3-presentation-compiler-bootstrapped` = Build.`scala3-presentation-compiler-bootstrapped`
 
 val sjsSandbox = Build.sjsSandbox
 val sjsJUnitTests = Build.sjsJUnitTests

compiler/src/dotty/tools/dotc/CompilationUnit.scala

@@ -5,6 +5,7 @@ import core._
 import Contexts._
 import SymDenotations.ClassDenotation
 import Symbols._
+import Comments.Comment
 import util.{FreshNameCreator, SourceFile, NoSource}
 import util.Spans.Span
 import ast.{tpd, untpd}
@@ -69,6 +70,9 @@ class CompilationUnit protected (val source: SourceFile) {
   /** Can this compilation unit be suspended */
   def isSuspendable: Boolean = true
 
+  /** List of all comments present in this compilation unit */
+  var comments: List[Comment] = Nil
+
   /** Suspends the compilation unit by thowing a SuspendException
    *  and recording the suspended compilation unit
    */

compiler/src/dotty/tools/dotc/interactive/InteractiveDriver.scala

@@ -145,7 +145,7 @@ class InteractiveDriver(val settings: List[String]) extends Driver {
     (fromSource ++ fromClassPath).distinct
   }
 
-  def run(uri: URI, sourceCode: String): List[Diagnostic] = run(uri, toSource(uri, sourceCode))
+  def run(uri: URI, sourceCode: String): List[Diagnostic] = run(uri, SourceFile.virtual(uri, sourceCode))
 
   def run(uri: URI, source: SourceFile): List[Diagnostic] = {
     import typer.ImportInfo._
@@ -297,9 +297,6 @@ class InteractiveDriver(val settings: List[String]) extends Driver {
     cleanupTree(tree)
   }
 
-  private def toSource(uri: URI, sourceCode: String): SourceFile =
-    SourceFile.virtual(Paths.get(uri).toString, sourceCode)
-
   /**
    * Initialize this driver and compiler.
    *
@@ -323,7 +320,7 @@ object InteractiveDriver {
     else
       try
         // We don't use file.file here since it'll be null
-        // for the VirtualFiles created by InteractiveDriver#toSource
+        // for the VirtualFiles created by SourceFile#virtual
         // TODO: To avoid these round trip conversions, we could add an
         // AbstractFile#toUri method and implement it by returning a constant
         // passed as a parameter to a constructor of VirtualFile

compiler/src/dotty/tools/dotc/parsing/ParserPhase.scala

@@ -30,6 +30,7 @@ class Parser extends Phase {
         val p = new Parsers.Parser(unit.source)
         //  p.in.debugTokenStream = true
         val tree = p.parse()
+        ctx.compilationUnit.comments = p.in.comments
         if (p.firstXmlPos.exists && !firstXmlPos.exists)
           firstXmlPos = p.firstXmlPos
         tree

compiler/src/dotty/tools/dotc/parsing/Scanners.scala

@@ -227,11 +227,11 @@ object Scanners {
       */
     private var docstringMap: SortedMap[Int, Comment] = SortedMap.empty
 
-    /* A Buffer for comment positions */
-    private val commentPosBuf = new mutable.ListBuffer[Span]
+    /* A Buffer for comments */
+    private val commentBuf = new mutable.ListBuffer[Comment]
 
-    /** Return a list of all the comment positions */
-    def commentSpans: List[Span] = commentPosBuf.toList
+    /** Return a list of all the comments */
+    def comments: List[Comment] = commentBuf.toList
 
     private def addComment(comment: Comment): Unit = {
       val lookahead = lookaheadReader()
@@ -246,7 +246,7 @@ object Scanners {
     def getDocComment(pos: Int): Option[Comment] = docstringMap.get(pos)
 
     /** A buffer for comments */
-    private val commentBuf = CharBuffer(initialCharBufferSize)
+    private val currentCommentBuf = CharBuffer(initialCharBufferSize)
 
     def toToken(identifier: SimpleName): Token =
       def handleMigration(keyword: Token): Token =
@@ -523,7 +523,7 @@ object Scanners {
      *
      *      The following tokens can start an indentation region:
      *
-     *         :  =  =>  <-  if  then  else  while  do  try  catch  
+     *         :  =  =>  <-  if  then  else  while  do  try  catch
      *         finally  for  yield  match  throw  return  with
      *
      *      Inserting an INDENT starts a new indentation region with the indentation of the current
@@ -1019,7 +1019,7 @@ object Scanners {
 
     private def skipComment(): Boolean = {
       def appendToComment(ch: Char) =
-        if (keepComments) commentBuf.append(ch)
+        if (keepComments) currentCommentBuf.append(ch)
       def nextChar() = {
         appendToComment(ch)
         Scanner.this.nextChar()
@@ -1047,9 +1047,9 @@ object Scanners {
       def finishComment(): Boolean = {
         if (keepComments) {
           val pos = Span(start, charOffset - 1, start)
-          val comment = Comment(pos, commentBuf.toString)
-          commentBuf.clear()
-          commentPosBuf += pos
+          val comment = Comment(pos, currentCommentBuf.toString)
+          currentCommentBuf.clear()
+          commentBuf += comment
 
           if (comment.isDocComment)
             addComment(comment)
@@ -1065,7 +1065,7 @@ object Scanners {
       else if (ch == '*') { nextChar(); skipComment(); finishComment() }
       else {
         // This was not a comment, remove the `/` from the buffer
-        commentBuf.clear()
+        currentCommentBuf.clear()
         false
       }
     }

compiler/src/dotty/tools/dotc/printing/PlainPrinter.scala

@@ -179,7 +179,7 @@ class PlainPrinter(_ctx: Context) extends Printer {
         if (printWithoutPrefix.contains(tp.symbol))
           toText(tp.name)
         else
-          toTextPrefix(tp.prefix) ~ selectionString(tp)
+          toTextPrefixOf(tp) ~ selectionString(tp)
       case tp: TermParamRef =>
         ParamRefNameString(tp) ~ lambdaHash(tp.binder) ~ ".type"
       case tp: TypeParamRef =>
@@ -353,7 +353,7 @@ class PlainPrinter(_ctx: Context) extends Printer {
   def toTextRef(tp: SingletonType): Text = controlled {
     tp match {
       case tp: TermRef =>
-        toTextPrefix(tp.prefix) ~ selectionString(tp)
+        toTextPrefixOf(tp) ~ selectionString(tp)
       case tp: ThisType =>
         nameString(tp.cls) + ".this"
       case SuperType(thistpe: SingletonType, _) =>
@@ -375,15 +375,6 @@ class PlainPrinter(_ctx: Context) extends Printer {
     }
   }
 
-  /** The string representation of this type used as a prefix, including separator */
-  def toTextPrefix(tp: Type): Text = controlled {
-    homogenize(tp) match {
-      case NoPrefix => ""
-      case tp: SingletonType => toTextRef(tp) ~ "."
-      case tp => trimPrefix(toTextLocal(tp)) ~ "#"
-    }
-  }
-
   def toTextCaptureRef(tp: Type): Text =
     homogenize(tp) match
       case tp: TermRef if tp.symbol == defn.captureRoot => Str("cap")
@@ -393,6 +384,15 @@ class PlainPrinter(_ctx: Context) extends Printer {
   protected def isOmittablePrefix(sym: Symbol): Boolean =
     defn.unqualifiedOwnerTypes.exists(_.symbol == sym) || isEmptyPrefix(sym)
 
+  /** The string representation of type prefix, including separator */
+  def toTextPrefixOf(tp: NamedType): Text = controlled {
+      homogenize(tp.prefix) match {
+        case NoPrefix => ""
+        case tp: SingletonType => toTextRef(tp) ~ "."
+        case tp => trimPrefix(toTextLocal(tp)) ~ "#"
+      }
+  }
+
   protected def isEmptyPrefix(sym: Symbol): Boolean =
     sym.isEffectiveRoot || sym.isAnonymousClass || sym.name.isReplWrapperName
 

compiler/src/dotty/tools/dotc/printing/Printer.scala

@@ -4,7 +4,7 @@ package printing
 
 import core._
 import Texts._, ast.Trees._
-import Types.{Type, SingletonType, LambdaParam},
+import Types.{Type, SingletonType, LambdaParam, NamedType},
        Symbols.Symbol, Scopes.Scope, Constants.Constant,
        Names.Name, Denotations._, Annotations.Annotation, Contexts.Context
 import typer.Implicits.SearchResult
@@ -101,7 +101,7 @@ abstract class Printer {
   def toTextRef(tp: SingletonType): Text
 
   /** Textual representation of a prefix of some reference, ending in `.` or `#` */
-  def toTextPrefix(tp: Type): Text
+  def toTextPrefixOf(tp: NamedType): Text
 
   /** Textual representation of a reference in a capture set */
   def toTextCaptureRef(tp: Type): Text

compiler/src/dotty/tools/dotc/printing/RefinedPrinter.scala

@@ -117,21 +117,22 @@ class RefinedPrinter(_ctx: Context) extends PlainPrinter(_ctx) {
     }
   }
 
-  override def toTextPrefix(tp: Type): Text = controlled {
+  override def toTextPrefixOf(tp: NamedType): Text = controlled {
     def isOmittable(sym: Symbol) =
       if printDebug then false
       else if homogenizedView then isEmptyPrefix(sym) // drop <root> and anonymous classes, but not scala, Predef.
       else if sym.isPackageObject then isOmittablePrefix(sym.owner)
       else isOmittablePrefix(sym)
-    tp match {
-      case tp: ThisType if isOmittable(tp.cls) =>
+
+    tp.prefix match {
+      case thisType: ThisType if isOmittable(thisType.cls) =>
         ""
-      case tp @ TermRef(pre, _) =>
-        val sym = tp.symbol
-        if sym.isPackageObject && !homogenizedView && !printDebug then toTextPrefix(pre)
+      case termRef @ TermRef(pre, _) =>
+        val sym = termRef.symbol
+        if sym.isPackageObject && !homogenizedView && !printDebug then toTextPrefixOf(termRef)
         else if (isOmittable(sym)) ""
-        else super.toTextPrefix(tp)
-      case _ => super.toTextPrefix(tp)
+        else super.toTextPrefixOf(tp)
+      case _ => super.toTextPrefixOf(tp)
     }
   }
 
@@ -427,8 +428,7 @@ class RefinedPrinter(_ctx: Context) extends PlainPrinter(_ctx) {
       case id @ Ident(name) =>
         val txt = tree.typeOpt match {
           case tp: NamedType if name != nme.WILDCARD =>
-            val pre = if (tp.symbol.is(JavaStatic)) tp.prefix.widen else tp.prefix
-            toTextPrefix(pre) ~ withPos(selectionString(tp), tree.sourcePos)
+            toTextPrefixOf(tp) ~ withPos(selectionString(tp), tree.sourcePos)
           case _ =>
             toText(name)
         }

compiler/src/dotty/tools/dotc/printing/SyntaxHighlighting.scala

@@ -83,8 +83,8 @@ object SyntaxHighlighting {
         }
       }
 
-      for (span <- scanner.commentSpans)
-        highlightPosition(span, CommentColor)
+      for (comment <- scanner.comments)
+        highlightPosition(comment.span, CommentColor)
 
       object TreeHighlighter extends untpd.UntypedTreeTraverser {
         import untpd._