wip: 2 pass parallel compile rebootstrap post -Yjava-tasty

Author: bishabosha

compiler/src/dotty/tools/dotc/CompilationUnit.scala

@@ -18,6 +18,8 @@ import StdNames.nme
 import scala.annotation.internal.sharable
 import scala.util.control.NoStackTrace
 import transform.MacroAnnotations
+import dotty.tools.dotc.interfaces.AbstractFile
+import dotty.tools.io.NoAbstractFile
 
 class CompilationUnit protected (val source: SourceFile, val info: CompilationUnitInfo | Null) {
 
@@ -45,6 +47,7 @@ class CompilationUnit protected (val source: SourceFile, val info: CompilationUn
 
   /** Pickled TASTY binaries, indexed by class. */
   var pickled: Map[ClassSymbol, () => Array[Byte]] = Map()
+  var outlinePickled: Map[ClassSymbol, () => Array[Byte]] = Map()
 
   /** The fresh name creator for the current unit.
    *  FIXME(#7661): This is not fine-grained enough to enable reproducible builds,
@@ -106,6 +109,8 @@ class CompilationUnit protected (val source: SourceFile, val info: CompilationUn
         ctx.run.nn.suspendedUnits += this
         if ctx.phase == Phases.inliningPhase then
           suspendedAtInliningPhase = true
+        else if ctx.settings.YearlyTastyOutput.value != NoAbstractFile then
+          report.error(i"Compilation units may not be suspended before inlining with -Ypickle-write")
     throw CompilationUnit.SuspendException()
 
   private var myAssignmentSpans: Map[Int, List[Span]] | Null = null

compiler/src/dotty/tools/dotc/Compiler.scala

@@ -40,20 +40,21 @@ class Compiler {
     List(new sbt.ExtractDependencies) :: // Sends information on classes' dependencies to sbt via callbacks
     List(new semanticdb.ExtractSemanticDB.ExtractSemanticInfo) :: // Extract info into .semanticdb files
     List(new PostTyper) ::          // Additional checks and cleanups after type checking
+    // List(new sbt.ExtractAPI.Outline) ::   // [runs in outline] Sends a representation of the API of classes to sbt via callbacks
     List(new sjs.PrepJSInterop) ::  // Additional checks and transformations for Scala.js (Scala.js only)
     List(new SetRootTree) ::        // Set the `rootTreeOrProvider` on class symbols
     Nil
 
   /** Phases dealing with TASTY tree pickling and unpickling */
   protected def picklerPhases: List[List[Phase]] =
-    List(new Pickler) ::            // Generate TASTY info
-    List(new sbt.ExtractAPI) ::     // Sends a representation of the API of classes to sbt via callbacks
-    List(new Inlining) ::           // Inline and execute macros
-    List(new PostInlining) ::       // Add mirror support for inlined code
-    List(new CheckUnused.PostInlining) ::  // Check for unused elements
-    List(new Staging) ::            // Check staging levels and heal staged types
-    List(new Splicing) ::           // Replace level 1 splices with holes
-    List(new PickleQuotes) ::       // Turn quoted trees into explicit run-time data structures
+    List(new Pickler) ::                  // Generate TASTY info
+    List(new sbt.ExtractAPI) ::           // [runs when not outline] Sends a representation of the API of classes to sbt via callbacks
+    List(new Inlining) ::                 // Inline and execute macros
+    List(new PostInlining) ::             // Add mirror support for inlined code
+    List(new CheckUnused.PostInlining) :: // Check for unused elements
+    List(new Staging) ::                  // Check staging levels and heal staged types
+    List(new Splicing) ::                 // Replace level 1 splices with holes
+    List(new PickleQuotes) ::             // Turn quoted trees into explicit run-time data structures
     Nil
 
   /** Phases dealing with the transformation from pickled trees to backend trees */

compiler/src/dotty/tools/dotc/Driver.scala

@@ -13,6 +13,28 @@ import config.Feature
 
 import scala.util.control.NonFatal
 import fromtasty.{TASTYCompiler, TastyFileUtil}
+import dotty.tools.io.NoAbstractFile
+import dotty.tools.io.{VirtualFile, VirtualDirectory}
+
+import java.nio.file.Path as JPath
+import scala.concurrent.*
+import scala.annotation.internal.sharable
+import scala.concurrent.duration.*
+import scala.util.{Success, Failure}
+import scala.annotation.targetName
+import dotty.tools.dotc.classpath.FileUtils.hasScalaExtension
+import dotty.tools.dotc.core.Symbols
+
+object Driver {
+  @sharable lazy val executor =
+    // TODO: systemParallelism may change over time - is it possible to update the pool size?
+    val pool = java.util.concurrent.Executors.newFixedThreadPool(systemParallelism()).nn
+    sys.addShutdownHook(pool.shutdown())
+    ExecutionContext.fromExecutor(pool)
+
+  /** 1 less than the system's own processor count (minimum 1) */
+  def systemParallelism() = math.max(1, Runtime.getRuntime().nn.availableProcessors() - 1)
+}
 
 /** Run the Dotty compiler.
  *
@@ -28,6 +50,183 @@ class Driver {
 
   protected def emptyReporter: Reporter = new StoreReporter(null)
 
+  protected def doCompile(files: List[AbstractFile])(using ictx: Context): Reporter =
+    val isOutline = ictx.settings.Youtline.value(using ictx)
+
+    if !isOutline then inContext(ictx):
+      report.echo(s"basic compilation enabled on files ${files.headOption.map(f => s"$f...").getOrElse("[]")}")(using ictx)
+      doCompile(newCompiler, files) // standard compilation
+    else
+      report.echo(s"Outline compilation enabled on files ${files.headOption.map(f => s"$f...").getOrElse("[]")}")(using ictx)
+      val maxParallelism = ictx.settings.YmaxParallelism.valueIn(ictx.settingsState)
+      val absParallelism = math.abs(maxParallelism)
+      val isParallel = maxParallelism >= 0
+      val parallelism =
+        val ceiling = Driver.systemParallelism()
+        if absParallelism > 0 then math.min(absParallelism, ceiling)
+        else ceiling
+
+      // NOTE: sbt will delete this potentially as soon as you call `apiPhaseCompleted`
+      val pickleWriteOutput = ictx.settings.YearlyTastyOutput.valueIn(ictx.settingsState)
+      val profileDestination = ictx.settings.YprofileDestination.valueIn(ictx.settingsState)
+
+      if pickleWriteOutput == NoAbstractFile then
+        report.error("Requested outline compilation with `-Yexperimental-outline` " +
+          "but did not provide output directory for TASTY files (missing `-Yearly-tasty-output` flag).")(using ictx)
+        return ictx.reporter
+
+      val pickleWriteSource =
+        pickleWriteOutput.underlyingSource match
+          case Some(source) =>
+            source.file.asInstanceOf[java.io.File | Null] match
+              case f: java.io.File => Some(source)
+              case null =>
+                report.warning(s"Could not resolve file of ${source} (of class ${source.getClass.getName})")
+                None
+          case None =>
+            if pickleWriteOutput.isInstanceOf[dotty.tools.io.JarArchive] then
+              report.warning(s"Could not resolve underlying source of jar ${pickleWriteOutput} (of class ${pickleWriteOutput.getClass.getName})")
+              None
+            else
+              report.warning(s"Could not resolve underlying source of ${pickleWriteOutput} (of class ${pickleWriteOutput.getClass.getName})")
+              Some(pickleWriteOutput)
+
+      val outlineOutput = new VirtualDirectory("<outline-classpath>") {
+        override def underlyingSource: Option[AbstractFile] = pickleWriteSource
+      }
+
+      val firstPassCtx = ictx.fresh
+        .setSetting(ictx.settings.YoutlineClasspath, outlineOutput)
+      inContext(firstPassCtx):
+        doCompile(newCompiler, files)
+
+      def secondPassCtx(id: Int, group: List[AbstractFile], promise: scala.concurrent.Promise[Unit]): Context =
+        val profileDestination0 =
+          if profileDestination.nonEmpty then
+            val ext = dotty.tools.io.Path.fileExtension(profileDestination)
+            val filename = dotty.tools.io.Path.fileName(profileDestination)
+            s"$filename-worker-$id${if ext.isEmpty then "" else s".$ext"}"
+          else profileDestination
+
+        val baseCtx = initCtx.fresh
+          .setSettings(ictx.settingsState) // copy over the classpath arguments also
+          .setSetting(ictx.settings.YsecondPass, true)
+          .setSetting(ictx.settings.YoutlineClasspath, outlineOutput)
+          .setCallbacks(ictx.store)
+          .setDepsFinishPromise(promise)
+          .setReporter(if isParallel then new StoreReporter(ictx.reporter) else ictx.reporter)
+
+        if profileDestination0.nonEmpty then
+          baseCtx.setSetting(ictx.settings.YprofileDestination, profileDestination0)
+
+        // if ictx.settings.YoutlineClasspath.valueIn(ictx.settingsState).isEmpty then
+        //   baseCtx.setSetting(baseCtx.settings.YoutlineClasspath, pickleWriteAsClasspath)
+        val fileNames: Array[String] =
+          if sourcesRequired then group.map(_.toString).toArray else Array.empty
+        setup(fileNames, baseCtx) match
+          case Some((_, ctx)) =>
+            assert(ctx.incCallback != null, s"cannot run outline without incremental callback")
+            assert(ctx.depsFinishPromiseOpt.isDefined, s"cannot run outline without dependencies promise")
+            ctx
+          case None => baseCtx
+      end secondPassCtx
+
+      val scalaFiles = files.filter(_.hasScalaExtension)
+
+      // 516 units, 8 cores => maxGroupSize = 65, unitGroups = 8, compilers = 8
+      if !firstPassCtx.reporter.hasErrors && scalaFiles.nonEmpty then
+        val maxGroupSize = Math.ceil(scalaFiles.length.toDouble / parallelism).toInt
+        val fileGroups = scalaFiles.grouped(maxGroupSize).toList
+        val compilers = fileGroups.length
+
+
+
+        def userRequestedSingleGroup = maxParallelism == 1
+
+        // TODO: probably not good to warn here because maybe compile is incremental
+        // if compilers == 1 && !userRequestedSingleGroup then
+        //   val knownParallelism = maxParallelism > 0
+        //   val requestedParallelism = s"Requested parallelism with `-Ymax-parallelism` was ${maxParallelism}"
+        //   val computedAddedum =
+        //     if knownParallelism then "."
+        //     else s""",
+        //       |  therefore operating with computed parallelism of ${parallelism}.""".stripMargin
+        //   val message =
+        //     s"""Outline compilation second pass will run with a single compile group.
+        //       |  ${requestedParallelism}$computedAddedum
+        //       |  With ${scalaUnits.length} units to compile I can only batch them into a single group.
+        //       |  This will increase build times.
+        //       |  Perhaps consider turning off -Youtline for this project.""".stripMargin
+        //   report.warning(message)(using firstPassCtx)
+
+        val promises = fileGroups.map(_ => scala.concurrent.Promise[Unit]())
+
+        locally:
+          import scala.concurrent.ExecutionContext.Implicits.global
+          Future.sequence(promises.map(_.future)).andThen {
+            case Success(_) =>
+              ictx.withIncCallback(_.dependencyPhaseCompleted())
+            case Failure(ex) =>
+              ex.printStackTrace()
+              report.error(s"Exception during parallel compilation: ${ex.getMessage}")(using firstPassCtx)
+          }
+
+        report.echo(s"Compiling $compilers groups of files ${if isParallel then "in parallel" else "sequentially"}")(using firstPassCtx)
+
+        def compileEager(
+            id: Int,
+            promise: Promise[Unit],
+            fileGroup: List[AbstractFile]
+        ): Reporter = {
+          if ctx.settings.verbose.value then
+            report.echo("#Compiling: " + fileGroup.take(3).mkString("", ", ", "..."))
+          val secondCtx = secondPassCtx(id, fileGroup, promise)
+          val reporter = inContext(secondCtx):
+            doCompile(newCompiler, fileGroup) // second pass
+          if !secondCtx.reporter.hasErrors then
+            assert(promise.isCompleted, s"promise was not completed")
+          if ctx.settings.verbose.value then
+            report.echo("#Done: " + fileGroup.mkString(" "))
+          reporter
+        }
+
+        def compileFuture(
+            id: Int,
+            promise: Promise[Unit],
+            fileGroup: List[AbstractFile]
+        )(using ExecutionContext): Future[Reporter] =
+          Future {
+            // println("#Compiling: " + fileGroup.mkString(" "))
+            val secondCtx = secondPassCtx(id, fileGroup, promise)
+            val reporter = inContext(secondCtx):
+              doCompile(newCompiler, fileGroup) // second pass
+            // println("#Done: " + fileGroup.mkString(" "))
+            reporter
+          }
+
+        def fileGroupIds = LazyList.iterate(0)(_ + 1).take(compilers)
+        def taggedGroups = fileGroupIds.lazyZip(promises).lazyZip(fileGroups)
+
+        if isParallel then
+          // val executor = java.util.concurrent.Executors.newFixedThreadPool(compilers).nn
+          given ec: ExecutionContext = Driver.executor // ExecutionContext.fromExecutor(executor)
+          val futureReporters = Future.sequence(taggedGroups.map(compileFuture)).andThen {
+            case Success(reporters) =>
+              reporters.foreach(_.flush()(using firstPassCtx))
+            case Failure(ex) =>
+              ex.printStackTrace
+              report.error(s"Exception during parallel compilation: ${ex.getMessage}")(using firstPassCtx)
+          }
+          Await.ready(futureReporters, Duration.Inf)
+          // executor.shutdown()
+        else
+          taggedGroups.map(compileEager)
+        firstPassCtx.reporter
+      else
+        ictx.withIncCallback(_.dependencyPhaseCompleted()) // may be just java files compiled
+        firstPassCtx.reporter
+  end doCompile
+
   protected def doCompile(compiler: Compiler, files: List[AbstractFile])(using Context): Reporter =
     if files.nonEmpty then
       var runOrNull = ctx.run
@@ -193,7 +392,7 @@ class Driver {
   def process(args: Array[String], rootCtx: Context): Reporter = {
     setup(args, rootCtx) match
       case Some((files, compileCtx)) =>
-        doCompile(newCompiler(using compileCtx), files)(using compileCtx)
+        doCompile(files)(using compileCtx)
       case None =>
         rootCtx.reporter
   }

compiler/src/dotty/tools/dotc/Run.scala

@@ -287,9 +287,10 @@ class Run(comp: Compiler, ictx: Context) extends ImplicitRunInfo with Constraint
       then ActiveProfile(ctx.settings.VprofileDetails.value.max(0).min(1000))
       else NoProfile
 
-    // If testing pickler, make sure to stop after pickling phase:
+    // If testing pickler, of outline first pass, make sure to stop after pickling phase:
     val stopAfter =
       if (ctx.settings.YtestPickler.value) List("pickler")
+      else if (ctx.isOutlineFirstPass) List("sbt-api")
       else ctx.settings.YstopAfter.value
 
     val pluginPlan = ctx.base.addPluginPhases(ctx.base.phasePlan)

compiler/src/dotty/tools/dotc/ast/tpd.scala

@@ -1381,6 +1381,17 @@ object tpd extends Trees.Instance[Type] with TypedTreeInfo {
   def runtimeCall(name: TermName, args: List[Tree])(using Context): Tree =
     Ident(defn.ScalaRuntimeModule.requiredMethod(name).termRef).appliedToTermArgs(args)
 
+  object ElidedTree:
+    def from(tree: Tree)(using Context): Tree =
+      Typed(tree, TypeTree(AnnotatedType(tree.tpe, Annotation(defn.ElidedTreeAnnot, tree.span))))
+
+    def isElided(tree: Tree)(using Context): Boolean = unapply(tree)
+
+    def unapply(tree: Tree)(using Context): Boolean = tree match
+      case Typed(_, tpt) => tpt.tpe.hasAnnotation(defn.ElidedTreeAnnot)
+      case _ => false
+
+
   /** An extractor that pulls out type arguments */
   object MaybePoly:
     def unapply(tree: Tree): Option[(Tree, List[Tree])] = tree match

compiler/src/dotty/tools/dotc/classpath/FileUtils.scala

@@ -23,6 +23,8 @@ object FileUtils {
 
     def hasTastyExtension: Boolean = file.ext.isTasty
 
+    def hasScalaExtension: Boolean = file.ext.isScala
+
     def isTasty: Boolean = !file.isDirectory && hasTastyExtension
 
     def isScalaBinary: Boolean = file.isClass || file.isTasty

compiler/src/dotty/tools/dotc/config/PathResolver.scala

@@ -12,6 +12,8 @@ import PartialFunction.condOpt
 import core.Contexts.*
 import Settings.*
 import dotty.tools.io.File
+import dotty.tools.io.NoAbstractFile
+import dotty.tools.dotc.classpath.FileUtils.isJarOrZip
 
 object PathResolver {
 
@@ -208,21 +210,28 @@ class PathResolver(using c: Context) {
       if (!settings.classpath.isDefault) settings.classpath.value
       else sys.env.getOrElse("CLASSPATH", ".")
 
+    def outlineClasspath: List[dotty.tools.io.AbstractFile] =
+      if c.isOutlineFirstPass then Nil
+      else c.settings.YoutlineClasspath.value match
+        case NoAbstractFile => Nil
+        case file => file :: Nil
+
     import classPathFactory.*
 
     // Assemble the elements!
     def basis: List[Iterable[ClassPath]] =
       val release = Option(ctx.settings.javaOutputVersion.value).filter(_.nonEmpty)
 
       List(
-        JrtClassPath(release),                        // 1. The Java 9+ classpath (backed by the jrt:/ virtual system, if available)
-        classesInPath(javaBootClassPath),             // 2. The Java bootstrap class path.
-        contentsOfDirsInPath(javaExtDirs),            // 3. The Java extension class path.
-        classesInExpandedPath(javaUserClassPath),     // 4. The Java application class path.
-        classesInPath(scalaBootClassPath),            // 5. The Scala boot class path.
-        contentsOfDirsInPath(scalaExtDirs),           // 6. The Scala extension class path.
-        classesInExpandedPath(userClassPath),         // 7. The Scala application class path.
-        sourcesInPath(sourcePath)                     // 8. The Scala source path.
+        outlineClasspath.map(newClassPath),           // 1. The outline classpath (backed by the -Youtline-classpath option)
+        JrtClassPath(release),                        // 2. The Java 9+ classpath (backed by the jrt:/ virtual system, if available)
+        classesInPath(javaBootClassPath),             // 3. The Java bootstrap class path.
+        contentsOfDirsInPath(javaExtDirs),            // 4. The Java extension class path.
+        classesInExpandedPath(javaUserClassPath),     // 5. The Java application class path.
+        classesInPath(scalaBootClassPath),            // 6. The Scala boot class path.
+        contentsOfDirsInPath(scalaExtDirs),           // 7. The Scala extension class path.
+        classesInExpandedPath(userClassPath),         // 8. The Scala application class path.
+        sourcesInPath(sourcePath)                     // 9. The Scala source path.
       )
 
     lazy val containers: List[ClassPath] = basis.flatten.distinct
@@ -238,12 +247,13 @@ class PathResolver(using c: Context) {
       |  scalaExtDirs         = %s
       |  userClassPath        = %s
       |  sourcePath           = %s
+      |  outlineClasspath     = %s
       |}""".trim.stripMargin.format(
         scalaHome,
         ppcp(javaBootClassPath), ppcp(javaExtDirs), ppcp(javaUserClassPath),
         useJavaClassPath,
         ppcp(scalaBootClassPath), ppcp(scalaExtDirs), ppcp(userClassPath),
-        ppcp(sourcePath)
+        ppcp(sourcePath), outlineClasspath.map("\n" + _).mkString
       )
   }