[MRESOLVER-7] Download dependency POMs in parallel

maven-resolver-api/src/main/java/org/eclipse/aether/graph/DefaultDependencyNode.java

@@ -70,7 +70,7 @@ public DefaultDependencyNode( Dependency dependency )
     {
         this.dependency = dependency;
         artifact = ( dependency != null ) ? dependency.getArtifact() : null;
-        children = new ArrayList<>( 0 );
+        children = Collections.synchronizedList( new ArrayList<DependencyNode>( 0 ) );
         aliases = Collections.emptyList();
         relocations = Collections.emptyList();
         repositories = Collections.emptyList();
@@ -88,7 +88,7 @@ public DefaultDependencyNode( Dependency dependency )
     public DefaultDependencyNode( Artifact artifact )
     {
         this.artifact = artifact;
-        children = new ArrayList<>( 0 );
+        children = Collections.synchronizedList( new ArrayList<DependencyNode>( 0 ) );
         aliases = Collections.emptyList();
         relocations = Collections.emptyList();
         repositories = Collections.emptyList();
@@ -106,7 +106,7 @@ public DefaultDependencyNode( DependencyNode node )
     {
         dependency = node.getDependency();
         artifact = node.getArtifact();
-        children = new ArrayList<>( 0 );
+        children = Collections.synchronizedList( new ArrayList<DependencyNode>( 0 ) );
         setAliases( node.getAliases() );
         setRequestContext( node.getRequestContext() );
         setManagedBits( node.getManagedBits() );
@@ -127,7 +127,7 @@ public void setChildren( List<DependencyNode> children )
     {
         if ( children == null )
         {
-            this.children = new ArrayList<>( 0 );
+            this.children = Collections.synchronizedList( new ArrayList<DependencyNode>( 0 ) );
         }
         else
         {

maven-resolver-impl/src/main/java/org/eclipse/aether/internal/impl/collect/DataPool.java

@@ -21,12 +21,12 @@
 
 import java.util.Collection;
 import java.util.Collections;
-import java.util.HashMap;
 import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
 import java.util.WeakHashMap;
+import java.util.concurrent.ConcurrentHashMap;
 
 import org.eclipse.aether.RepositoryCache;
 import org.eclipse.aether.RepositorySystemSession;
@@ -67,9 +67,9 @@ final class DataPool
 
     private Map<Object, Descriptor> descriptors;
 
-    private final Map<Object, Constraint> constraints = new HashMap<>();
+    private final Map<Object, Constraint> constraints = new ConcurrentHashMap<>();
 
-    private final Map<Object, List<DependencyNode>> nodes = new HashMap<>( 256 );
+    private final Map<Object, List<DependencyNode>> nodes = new ConcurrentHashMap<>( 256 );
 
     @SuppressWarnings( "unchecked" )
     DataPool( RepositorySystemSession session )

maven-resolver-impl/src/main/java/org/eclipse/aether/internal/impl/collect/DefaultDependencyCollectionContext.java

@@ -19,6 +19,7 @@
  * under the License.
  */
 
+import java.util.Collections;
 import java.util.List;
 
 import org.eclipse.aether.RepositorySystemSession;
@@ -35,19 +36,19 @@ final class DefaultDependencyCollectionContext
 
     private final RepositorySystemSession session;
 
-    private Artifact artifact;
+    private final Artifact artifact;
 
-    private Dependency dependency;
+    private final Dependency dependency;
 
-    private List<Dependency> managedDependencies;
+    private final List<Dependency> managedDependencies;
 
     DefaultDependencyCollectionContext( RepositorySystemSession session, Artifact artifact,
                                                Dependency dependency, List<Dependency> managedDependencies )
     {
         this.session = session;
         this.artifact = ( dependency != null ) ? dependency.getArtifact() : artifact;
         this.dependency = dependency;
-        this.managedDependencies = managedDependencies;
+        this.managedDependencies = Collections.unmodifiableList( managedDependencies );
     }
 
     public RepositorySystemSession getSession()
@@ -70,13 +71,6 @@ public List<Dependency> getManagedDependencies()
         return managedDependencies;
     }
 
-    public void set( Dependency dependency, List<Dependency> managedDependencies )
-    {
-        artifact = dependency.getArtifact();
-        this.dependency = dependency;
-        this.managedDependencies = managedDependencies;
-    }
-
     @Override
     public String toString()
     {

maven-resolver-impl/src/main/java/org/eclipse/aether/internal/impl/collect/DefaultDependencyCollector.java

@@ -22,11 +22,19 @@
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
+import java.util.Comparator;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.Queue;
+import java.util.concurrent.ConcurrentLinkedQueue;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.locks.LockSupport;
+
 import static java.util.Objects.requireNonNull;
 
 import javax.inject.Inject;
@@ -67,6 +75,7 @@
 import org.eclipse.aether.spi.locator.Service;
 import org.eclipse.aether.spi.locator.ServiceLocator;
 import org.eclipse.aether.util.ConfigUtils;
+import org.eclipse.aether.util.artifact.ArtifactIdUtils;
 import org.eclipse.aether.util.graph.manager.DependencyManagerUtils;
 import org.eclipse.aether.util.graph.transformer.TransformationContextKeys;
 import org.eclipse.aether.version.Version;
@@ -176,7 +185,7 @@ public CollectResult collectDependencies( RepositorySystemSession session, Colle
                                              request.getRequestContext() );
                 rangeRequest.setTrace( trace );
                 rangeResult = versionRangeResolver.resolveVersionRange( session, rangeRequest );
-                versions = filterVersions( root, rangeResult, verFilter, new DefaultVersionFilterContext( session ) );
+                versions = filterVersions( root, rangeResult, verFilter, session );
             }
             catch ( VersionRangeResolutionException e )
             {
@@ -244,22 +253,24 @@ public CollectResult collectDependencies( RepositorySystemSession session, Colle
         {
             DataPool pool = new DataPool( session );
 
-            NodeStack nodes = new NodeStack();
-            nodes.push( node );
-
             DefaultDependencyCollectionContext context =
                 new DefaultDependencyCollectionContext( session, request.getRootArtifact(), root, managedDependencies );
 
-            DefaultVersionFilterContext versionContext = new DefaultVersionFilterContext( session );
-
-            Args args = new Args( session, trace, pool, nodes, context, versionContext, request );
+            Args args = new Args( session, trace, pool, new NodeStack( node ), request, getExecutorService( session ) );
             Results results = new Results( result, session );
-
-            process( args, results, dependencies, repositories,
-                     depSelector != null ? depSelector.deriveChildSelector( context ) : null,
-                     depManager != null ? depManager.deriveChildManager( context ) : null,
-                     depTraverser != null ? depTraverser.deriveChildTraverser( context ) : null,
-                     verFilter != null ? verFilter.deriveChildFilter( context ) : null );
+            try
+            {
+                process( args, results, dependencies, repositories,
+                        depSelector != null ? depSelector.deriveChildSelector( context ) : null,
+                        depManager != null ? depManager.deriveChildManager( context ) : null,
+                        depTraverser != null ? depTraverser.deriveChildTraverser( context ) : null,
+                        verFilter != null ? verFilter.deriveChildFilter( context ) : null );
+                waitUntilComplete( results.collectionTasks );
+            }
+            finally
+            {
+                args.executorService.shutdown();
+            }
 
             errorPath = results.errorPath;
         }
@@ -343,27 +354,98 @@ private static String getId( Artifact a )
         return a.getGroupId() + ':' + a.getArtifactId() + ':' + a.getClassifier() + ':' + a.getExtension();
     }
 
-    @SuppressWarnings( "checkstyle:parameternumber" )
-    private void process( final Args args, Results results, List<Dependency> dependencies,
-                          List<RemoteRepository> repositories, DependencySelector depSelector,
-                          DependencyManager depManager, DependencyTraverser depTraverser, VersionFilter verFilter )
+    private ExecutorService getExecutorService( RepositorySystemSession session )
     {
-        for ( Dependency dependency : dependencies )
+        int nThreads = ConfigUtils.getInteger( session, 5, "maven.descriptor.threads", "maven.artifact.threads" );
+        return Executors.newFixedThreadPool( nThreads );
+    }
+
+    private void waitUntilComplete( Iterable<Future<?>> tasks )
+    {
+        while ( anyIncomplete( tasks ) )
         {
-            processDependency( args, results, repositories, depSelector, depManager, depTraverser, verFilter,
-                               dependency );
+            // CHECKSTYLE_OFF: MagicNumber
+            LockSupport.parkNanos( 50 );
+            // CHECKSTYLE_ON: MagicNumber
         }
     }
 
+    private boolean anyIncomplete( Iterable<Future<?>> tasks )
+    {
+        for ( Future<?> task : tasks )
+        {
+            if ( !task.isDone() )
+            {
+                return true;
+            }
+        }
+        return false;
+    }
+
     @SuppressWarnings( "checkstyle:parameternumber" )
-    private void processDependency( Args args, Results results, List<RemoteRepository> repositories,
-                                    DependencySelector depSelector, DependencyManager depManager,
-                                    DependencyTraverser depTraverser, VersionFilter verFilter, Dependency dependency )
+    private void process( final Args args, final Results results, final List<Dependency> dependencies,
+                          final List<RemoteRepository> repositories, final DependencySelector depSelector,
+                          final DependencyManager depManager, final DependencyTraverser depTraverser,
+                          final VersionFilter verFilter )
     {
+        final List<Future<?>> childrenTasks = new ArrayList<>();
+        for ( final Dependency dependency : dependencies )
+        {
+            Future<?> task = args.executorService.submit( new Runnable()
+            {
+                @Override
+                public void run()
+                {
+                    processDependency( args, results, repositories, depSelector, depManager, depTraverser, verFilter,
+                            dependency, Collections.emptyList(), false );
+                }
+            } );
+            results.collectionTasks.add( task );
+            childrenTasks.add( task );
+        }
+        Future<?> sortingTask = args.executorService.submit( new Runnable()
+        {
+            @Override
+            public void run()
+            {
+                waitUntilComplete( childrenTasks );
+                Collections.sort( args.nodes.top().getChildren(), new ChildrenComparator( dependencies ) );
+            }
+        } );
+        results.collectionTasks.add( sortingTask );
+    }
+
+    private static class ChildrenComparator implements Comparator<DependencyNode>
+    {
+
+        private final Map<String, Integer> order;
+
+        private ChildrenComparator( List<Dependency> dependencies )
+        {
+            this.order = new HashMap<>();
+            for ( int i = 0; i < dependencies.size(); i++ )
+            {
+                order.put( ArtifactIdUtils.toVersionlessId( dependencies.get( i ).getArtifact() ), i );
+            }
+        }
+
+        @Override
+        public int compare( DependencyNode o1, DependencyNode o2 )
+        {
+            int result = getOrder( o1 ) - getOrder( o2 );
+            if ( result != 0 )
+            {
+                return result < 0 ? -1 : 1;
+            }
+            return 0;
+        }
 
-        List<Artifact> relocations = Collections.emptyList();
-        processDependency( args, results, repositories, depSelector, depManager, depTraverser, verFilter, dependency,
-                           relocations, false );
+        private Integer getOrder( DependencyNode node )
+        {
+            String id = ArtifactIdUtils.toVersionlessId( node.getArtifact() );
+            Integer result = order.get( id );
+            return result == null ? 0 : result;
+        }
     }
 
     @SuppressWarnings( "checkstyle:parameternumber" )
@@ -394,7 +476,7 @@ private void processDependency( Args args, Results results, List<RemoteRepositor
 
             rangeResult = cachedResolveRangeResult( rangeRequest, args.pool, args.session );
 
-            versions = filterVersions( dependency, rangeResult, verFilter, args.versionContext );
+            versions = filterVersions( dependency, rangeResult, verFilter, args.session );
         }
         catch ( VersionRangeResolutionException e )
         {
@@ -482,8 +564,9 @@ private void doRecurse( Args args, Results results, List<RemoteRepository> repos
                             DependencyTraverser depTraverser, VersionFilter verFilter, Dependency d,
                             ArtifactDescriptorResult descriptorResult, DefaultDependencyNode child )
     {
-        DefaultDependencyCollectionContext context = args.collectionContext;
-        context.set( d, descriptorResult.getManagedDependencies() );
+        DefaultDependencyCollectionContext context =
+                new DefaultDependencyCollectionContext( args.session, d.getArtifact(), d,
+                        descriptorResult.getManagedDependencies() );
 
         DependencySelector childSelector = depSelector != null ? depSelector.deriveChildSelector( context ) : null;
         DependencyManager childManager = depManager != null ? depManager.deriveChildManager( context ) : null;
@@ -503,13 +586,10 @@ private void doRecurse( Args args, Results results, List<RemoteRepository> repos
         if ( children == null )
         {
             args.pool.putChildren( key, child.getChildren() );
-
-            args.nodes.push( child );
-
-            process( args, results, descriptorResult.getDependencies(), childRepos, childSelector, childManager,
+            Args childArgs = new Args( args.session, args.trace, args.pool, args.nodes.push( child ), args.request,
+                    args.executorService );
+            process( childArgs, results, descriptorResult.getDependencies(), childRepos, childSelector, childManager,
                      childTraverser, childFilter );
-
-            args.nodes.pop();
         }
         else
         {
@@ -645,8 +725,7 @@ private static List<RemoteRepository> getRemoteRepositories( ArtifactRepository
     }
 
     private static List<? extends Version> filterVersions( Dependency dependency, VersionRangeResult rangeResult,
-                                                           VersionFilter verFilter,
-                                                           DefaultVersionFilterContext verContext )
+                                                           VersionFilter verFilter, RepositorySystemSession session )
         throws VersionRangeResolutionException
     {
         if ( rangeResult.getVersions().isEmpty() )
@@ -659,7 +738,8 @@ private static List<? extends Version> filterVersions( Dependency dependency, Ve
         List<? extends Version> versions;
         if ( verFilter != null && rangeResult.getVersionConstraint().getRange() != null )
         {
-            verContext.set( dependency, rangeResult );
+            DefaultVersionFilterContext verContext =
+                    new DefaultVersionFilterContext( session, dependency, rangeResult );
             try
             {
                 verFilter.filterVersions( verContext );
@@ -700,15 +780,12 @@ static class Args
 
         final NodeStack nodes;
 
-        final DefaultDependencyCollectionContext collectionContext;
-
-        final DefaultVersionFilterContext versionContext;
-
         final CollectRequest request;
 
+        final ExecutorService executorService;
+
         Args( RepositorySystemSession session, RequestTrace trace, DataPool pool, NodeStack nodes,
-                     DefaultDependencyCollectionContext collectionContext, DefaultVersionFilterContext versionContext,
-                     CollectRequest request )
+              CollectRequest request, ExecutorService executorService )
         {
             this.session = session;
             this.request = request;
@@ -717,8 +794,7 @@ static class Args
             this.trace = trace;
             this.pool = pool;
             this.nodes = nodes;
-            this.collectionContext = collectionContext;
-            this.versionContext = versionContext;
+            this.executorService = executorService;
         }
 
     }
@@ -734,6 +810,8 @@ static class Results
 
         String errorPath;
 
+        final Queue<Future<?>> collectionTasks = new ConcurrentLinkedQueue<>();
+
         Results( CollectResult result, RepositorySystemSession session )
         {
             this.result = result;