From 1bc8c3fc04c26745f70cd29d8ae67a3c70b96c74 Mon Sep 17 00:00:00 2001 From: Guillaume Nodet Date: Wed, 21 Dec 2022 11:29:22 +0100 Subject: [PATCH] [MNG-7629] Change reactor reader to copy packaged artifacts and reuse them across builds if needed --- .../java/org/apache/maven/DefaultMaven.java | 14 +- .../java/org/apache/maven/ReactorReader.java | 333 ++++++------------ 2 files changed, 126 insertions(+), 221 deletions(-) diff --git a/maven-core/src/main/java/org/apache/maven/DefaultMaven.java b/maven-core/src/main/java/org/apache/maven/DefaultMaven.java index c6658d2f3909..e1880ecae4b0 100644 --- a/maven-core/src/main/java/org/apache/maven/DefaultMaven.java +++ b/maven-core/src/main/java/org/apache/maven/DefaultMaven.java @@ -242,6 +242,13 @@ private MavenExecutionResult doExecute( return addExceptionToResult(result, e); } + try { + WorkspaceReader reactorReader = container.lookup(WorkspaceReader.class, ReactorReader.HINT); + repoSession.setWorkspaceReader(reactorReader); + } catch (ComponentLookupException e) { + return addExceptionToResult(result, e); + } + eventCatapult.fire(ExecutionEvent.Type.ProjectDiscoveryStarted, session, null); Result graphResult = buildGraph(session); @@ -332,12 +339,13 @@ private MavenExecutionResult doExecute( private void setupWorkspaceReader(MavenSession session, DefaultRepositorySystemSession repoSession) throws ComponentLookupException { // Desired order of precedence for workspace readers before querying the local artifact repositories - List workspaceReaders = new ArrayList(); + List workspaceReaders = new ArrayList<>(); // 1) Reactor workspace reader - workspaceReaders.add(container.lookup(WorkspaceReader.class, ReactorReader.HINT)); + WorkspaceReader reactorReader = container.lookup(WorkspaceReader.class, ReactorReader.HINT); + workspaceReaders.add(reactorReader); // 2) Repository system session-scoped workspace reader WorkspaceReader repoWorkspaceReader = repoSession.getWorkspaceReader(); - if (repoWorkspaceReader != null) { + if (repoWorkspaceReader != null && repoWorkspaceReader != reactorReader) { workspaceReaders.add(repoWorkspaceReader); } // 3) .. n) Project-scoped workspace readers diff --git a/maven-core/src/main/java/org/apache/maven/ReactorReader.java b/maven-core/src/main/java/org/apache/maven/ReactorReader.java index 7e6551f4031e..29a5b43b6683 100644 --- a/maven-core/src/main/java/org/apache/maven/ReactorReader.java +++ b/maven-core/src/main/java/org/apache/maven/ReactorReader.java @@ -20,41 +20,34 @@ import javax.inject.Inject; import javax.inject.Named; +import javax.inject.Singleton; import java.io.File; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; -import java.nio.file.Paths; -import java.util.Arrays; -import java.util.Collection; +import java.nio.file.StandardCopyOption; +import java.util.ArrayList; import java.util.Collections; -import java.util.HashSet; -import java.util.Iterator; import java.util.List; -import java.util.Map; import java.util.Objects; -import java.util.Optional; -import java.util.function.Function; -import java.util.function.Predicate; -import java.util.stream.Collectors; +import java.util.regex.Matcher; +import java.util.regex.Pattern; import java.util.stream.Stream; -import org.apache.maven.artifact.ArtifactUtils; +import org.apache.maven.eventspy.EventSpy; +import org.apache.maven.execution.ExecutionEvent; import org.apache.maven.execution.MavenSession; import org.apache.maven.model.Model; import org.apache.maven.project.MavenProject; +import org.apache.maven.project.artifact.ProjectArtifact; import org.apache.maven.repository.internal.MavenWorkspaceReader; +import org.codehaus.plexus.PlexusContainer; import org.eclipse.aether.artifact.Artifact; import org.eclipse.aether.repository.WorkspaceRepository; -import org.eclipse.aether.util.artifact.ArtifactIdUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import static java.util.function.Function.identity; -import static java.util.stream.Collectors.groupingBy; -import static java.util.stream.Collectors.toMap; - /** * An implementation of a workspace reader that knows how to search the Maven reactor for artifacts, either as packaged * jar if it has been built, or only compile output directory if packaging hasn't happened yet. @@ -66,30 +59,15 @@ class ReactorReader implements MavenWorkspaceReader { public static final String HINT = "reactor"; - private static final Collection COMPILE_PHASE_TYPES = - Arrays.asList("jar", "ejb-client", "war", "rar", "ejb3", "par", "sar", "wsr", "har", "app-client"); - private static final Logger LOGGER = LoggerFactory.getLogger(ReactorReader.class); private final MavenSession session; - private final Map projectsByGAV; - private final Map> projectsByGA; private final WorkspaceRepository repository; - private Function projectIntoKey = - s -> ArtifactUtils.key(s.getGroupId(), s.getArtifactId(), s.getVersion()); - - private Function projectIntoVersionlessKey = - s -> ArtifactUtils.versionlessKey(s.getGroupId(), s.getArtifactId()); - @Inject ReactorReader(MavenSession session) { this.session = session; - this.projectsByGAV = session.getAllProjects().stream().collect(toMap(projectIntoKey, identity())); - - this.projectsByGA = projectsByGAV.values().stream().collect(groupingBy(projectIntoVersionlessKey)); - - repository = new WorkspaceRepository("reactor", new HashSet<>(projectsByGAV.keySet())); + this.repository = new WorkspaceRepository("reactor", null); } // @@ -101,227 +79,146 @@ public WorkspaceRepository getRepository() { } public File findArtifact(Artifact artifact) { - String projectKey = ArtifactUtils.key(artifact.getGroupId(), artifact.getArtifactId(), artifact.getVersion()); - - MavenProject project = projectsByGAV.get(projectKey); - - if (project != null) { - File file = find(project, artifact); - if (file == null && project != project.getExecutionProject()) { - file = find(project.getExecutionProject(), artifact); + List projects = session.getAllProjects(); + if (projects != null) { + for (MavenProject p : projects) { + if (Objects.equals(artifact.getGroupId(), p.getGroupId()) + && Objects.equals(artifact.getArtifactId(), p.getArtifactId()) + && Objects.equals(artifact.getBaseVersion(), p.getVersion()) + && Objects.equals(artifact.getExtension(), "pom") + && p.getFile() != null) { + return p.getFile(); + } } - return file; } - - return null; + File file = find(artifact); + return file; } public List findVersions(Artifact artifact) { - String key = ArtifactUtils.versionlessKey(artifact.getGroupId(), artifact.getArtifactId()); - - return Optional.ofNullable(projectsByGA.get(key)).orElse(Collections.emptyList()).stream() - .filter(s -> Objects.nonNull(find(s, artifact))) - .map(MavenProject::getVersion) - .collect(Collectors.collectingAndThen(Collectors.toList(), Collections::unmodifiableList)); + List versions = new ArrayList<>(); + String artifactId = artifact.getArtifactId(); + String groupId = artifact.getGroupId(); + Path repo = getProjectLocalRepo().resolve(groupId).resolve(artifactId); + String classifier = artifact.getClassifier(); + String extension = artifact.getExtension(); + Pattern pattern = Pattern.compile("\\Q" + artifactId + "\\E-(.*)" + + (classifier != null ? "-\\Q" + classifier + "\\E" : "") + + (extension != null ? "." + extension : "")); + try (Stream paths = Files.list(repo)) { + paths.forEach(p -> { + String filename = p.getFileName().toString(); + Matcher matcher = pattern.matcher(filename); + if (matcher.matches()) { + versions.add(matcher.group(1)); + } + }); + } catch (IOException e) { + // ignore + } + return Collections.unmodifiableList(versions); } @Override public Model findModel(Artifact artifact) { - String projectKey = ArtifactUtils.key(artifact.getGroupId(), artifact.getArtifactId(), artifact.getVersion()); - MavenProject project = projectsByGAV.get(projectKey); - return project == null ? null : project.getModel(); + List projects = session.getAllProjects(); + if (projects != null) { + for (MavenProject p : projects) { + if (Objects.equals(artifact.getGroupId(), p.getGroupId()) + && Objects.equals(artifact.getArtifactId(), p.getArtifactId()) + && Objects.equals(artifact.getBaseVersion(), p.getVersion()) + && Objects.equals(artifact.getExtension(), "pom") + && p.getFile() != null) { + return p.getModel(); + } + } + } + return null; } // // Implementation // - private File find(MavenProject project, Artifact artifact) { - if ("pom".equals(artifact.getExtension())) { - return project.getFile(); - } + private File find(Artifact artifact) { + Path target = getArtifactPath(artifact); + return Files.isRegularFile(target) ? target.toFile() : null; + } - Artifact projectArtifact = findMatchingArtifact(project, artifact); - File packagedArtifactFile = determinePreviouslyPackagedArtifactFile(project, projectArtifact); + public void processProject(MavenProject project) { + List artifacts = new ArrayList<>(); - if (hasArtifactFileFromPackagePhase(projectArtifact)) { - return projectArtifact.getFile(); + artifacts.add(RepositoryUtils.toArtifact(new ProjectArtifact(project))); + if (!"pom".equals(project.getPackaging())) { + org.apache.maven.artifact.Artifact mavenMainArtifact = project.getArtifact(); + artifacts.add(RepositoryUtils.toArtifact(mavenMainArtifact)); } - // Check whether an earlier Maven run might have produced an artifact that is still on disk. - else if (packagedArtifactFile != null - && packagedArtifactFile.exists() - && isPackagedArtifactUpToDate(project, packagedArtifactFile, artifact)) { - return packagedArtifactFile; - } else if (!hasBeenPackagedDuringThisSession(project)) { - // fallback to loose class files only if artifacts haven't been packaged yet - // and only for plain old jars. Not war files, not ear files, not anything else. - return determineBuildOutputDirectoryForArtifact(project, artifact); + for (org.apache.maven.artifact.Artifact attached : project.getAttachedArtifacts()) { + artifacts.add(RepositoryUtils.toArtifact(attached)); } - // The fall-through indicates that the artifact cannot be found; - // for instance if package produced nothing or classifier problems. - return null; - } - - private File determineBuildOutputDirectoryForArtifact(final MavenProject project, final Artifact artifact) { - if (isTestArtifact(artifact)) { - if (project.hasLifecyclePhase("test-compile")) { - return new File(project.getBuild().getTestOutputDirectory()); - } - } else { - String type = artifact.getProperty("type", ""); - File outputDirectory = new File(project.getBuild().getOutputDirectory()); - - // Check if the project is being built during this session, and if we can expect any output. - // There is no need to check if the build has created any outputs, see MNG-2222. - boolean projectCompiledDuringThisSession = - project.hasLifecyclePhase("compile") && COMPILE_PHASE_TYPES.contains(type); - - // Check if the project is part of the session (not filtered by -pl, -rf, etc). If so, we check - // if a possible earlier Maven invocation produced some output for that project which we can use. - boolean projectHasOutputFromPreviousSession = - !session.getProjects().contains(project) && outputDirectory.exists(); - - if (projectHasOutputFromPreviousSession || projectCompiledDuringThisSession) { - return outputDirectory; + for (Artifact artifact : artifacts) { + if (artifact.getFile() != null && artifact.getFile().isFile()) { + Path target = getArtifactPath(artifact); + try { + LOGGER.debug("Copying {} to project local repository", artifact); + Files.createDirectories(target.getParent()); + Files.copy(artifact.getFile().toPath(), target, StandardCopyOption.REPLACE_EXISTING); + } catch (IOException e) { + LOGGER.warn("Error while copying artifact to project local repository", e); + } } } - - // The fall-through indicates that the artifact cannot be found; - // for instance if package produced nothing or classifier problems. - return null; } - private File determinePreviouslyPackagedArtifactFile(MavenProject project, Artifact artifact) { - if (artifact == null) { - return null; - } - - String fileName = String.format("%s.%s", project.getBuild().getFinalName(), artifact.getExtension()); - return new File(project.getBuild().getDirectory(), fileName); + private Path getArtifactPath(Artifact artifact) { + String groupId = artifact.getGroupId(); + String artifactId = artifact.getArtifactId(); + String version = artifact.getBaseVersion(); + String classifier = artifact.getClassifier(); + String extension = artifact.getExtension(); + Path repo = getProjectLocalRepo(); + return repo.resolve(groupId) + .resolve(artifactId) + .resolve(artifactId + + "-" + version + + (classifier != null && !classifier.isEmpty() ? "-" + classifier : "") + + (extension != null && !extension.isEmpty() ? "." + extension : "")); } - private boolean hasArtifactFileFromPackagePhase(Artifact projectArtifact) { - return projectArtifact != null - && projectArtifact.getFile() != null - && projectArtifact.getFile().exists(); + private Path getProjectLocalRepo() { + Path root = session.getRequest().getMultiModuleProjectDirectory().toPath(); + Path repo = root.resolve("target").resolve("project-local-repo"); + return repo; } - private boolean isPackagedArtifactUpToDate(MavenProject project, File packagedArtifactFile, Artifact artifact) { - Path outputDirectory = Paths.get(project.getBuild().getOutputDirectory()); - if (!outputDirectory.toFile().exists()) { - return true; - } + @Named + @Singleton + @SuppressWarnings("unused") + static class ReactorReaderSpy implements EventSpy { - try (Stream outputFiles = Files.walk(outputDirectory)) { - // Not using File#lastModified() to avoid a Linux JDK8 milliseconds precision bug: JDK-8177809. - long artifactLastModified = - Files.getLastModifiedTime(packagedArtifactFile.toPath()).toMillis(); + final PlexusContainer container; - if (session.getProjectBuildingRequest().getBuildStartTime() != null) { - long buildStartTime = - session.getProjectBuildingRequest().getBuildStartTime().getTime(); - if (artifactLastModified > buildStartTime) { - return true; - } - } - - Iterator iterator = outputFiles.iterator(); - while (iterator.hasNext()) { - Path outputFile = iterator.next(); + @Inject + ReactorReaderSpy(PlexusContainer container) { + this.container = container; + } - if (Files.isDirectory(outputFile)) { - continue; - } + @Override + public void init(Context context) throws Exception {} - long outputFileLastModified = - Files.getLastModifiedTime(outputFile).toMillis(); - if (outputFileLastModified > artifactLastModified) { - File alternative = determineBuildOutputDirectoryForArtifact(project, artifact); - if (alternative != null) { - LOGGER.warn( - "File '{}' is more recent than the packaged artifact for '{}'; using '{}' instead", - relativizeOutputFile(outputFile), - project.getArtifactId(), - relativizeOutputFile(alternative.toPath())); - } else { - LOGGER.warn( - "File '{}' is more recent than the packaged artifact for '{}'; " - + "cannot use the build output directory for this type of artifact", - relativizeOutputFile(outputFile), - project.getArtifactId()); - } - return false; + @Override + public void onEvent(Object event) throws Exception { + if (event instanceof ExecutionEvent) { + ExecutionEvent ee = (ExecutionEvent) event; + if (ee.getType() == ExecutionEvent.Type.ForkedProjectSucceeded + || ee.getType() == ExecutionEvent.Type.ProjectSucceeded) { + container.lookup(ReactorReader.class).processProject(ee.getProject()); } } - - return true; - } catch (IOException e) { - LOGGER.warn( - "An I/O error occurred while checking if the packaged artifact is up-to-date " - + "against the build output directory. " - + "Continuing with the assumption that it is up-to-date.", - e); - return true; } - } - - private boolean hasBeenPackagedDuringThisSession(MavenProject project) { - return project.hasLifecyclePhase("package") - || project.hasLifecyclePhase("install") - || project.hasLifecyclePhase("deploy"); - } - - private Path relativizeOutputFile(final Path outputFile) { - Path projectBaseDirectory = - Paths.get(session.getRequest().getMultiModuleProjectDirectory().toURI()); - return projectBaseDirectory.relativize(outputFile); - } - - /** - * Tries to resolve the specified artifact from the artifacts of the given project. - * - * @param project The project to try to resolve the artifact from, must not be null. - * @param requestedArtifact The artifact to resolve, must not be null. - * @return The matching artifact from the project or null if not found. Note that this - */ - private Artifact findMatchingArtifact(MavenProject project, Artifact requestedArtifact) { - String requestedRepositoryConflictId = ArtifactIdUtils.toVersionlessId(requestedArtifact); - - Artifact mainArtifact = RepositoryUtils.toArtifact(project.getArtifact()); - if (requestedRepositoryConflictId.equals(ArtifactIdUtils.toVersionlessId(mainArtifact))) { - return mainArtifact; - } - - return RepositoryUtils.toArtifacts(project.getAttachedArtifacts()).stream() - .filter(isRequestedArtifact(requestedArtifact)) - .findFirst() - .orElse(null); - } - - /** - * We are taking as much as we can from the DefaultArtifact.equals(). The requested artifact has no file, so we want - * to remove that from the comparison. - * - * @param requestArtifact checked against the given artifact. - * @return true if equals, false otherwise. - */ - private Predicate isRequestedArtifact(Artifact requestArtifact) { - return s -> s.getArtifactId().equals(requestArtifact.getArtifactId()) - && s.getGroupId().equals(requestArtifact.getGroupId()) - && s.getVersion().equals(requestArtifact.getVersion()) - && s.getExtension().equals(requestArtifact.getExtension()) - && s.getClassifier().equals(requestArtifact.getClassifier()); - } - /** - * Determines whether the specified artifact refers to test classes. - * - * @param artifact The artifact to check, must not be {@code null}. - * @return {@code true} if the artifact refers to test classes, {@code false} otherwise. - */ - private static boolean isTestArtifact(Artifact artifact) { - return ("test-jar".equals(artifact.getProperty("type", ""))) - || ("jar".equals(artifact.getExtension()) && "tests".equals(artifact.getClassifier())); + @Override + public void close() throws Exception {} } }