From 6231067e102fcaf5a5acfb4af349f3a8c39b8166 Mon Sep 17 00:00:00 2001 From: Tamas Cservenak Date: Sat, 3 Dec 2022 22:04:23 +0100 Subject: [PATCH 1/4] [MNG-7619] Reverse Dependency Tree Adds Maven feature that is able to explain why an artifact is present in local repository. Usable for diagnosing resolution issues. --- https://issues.apache.org/jira/browse/MNG-7619 --- ...DefaultRepositorySystemSessionFactory.java | 17 +++ .../aether/ReverseTreeRepositoryListener.java | 114 ++++++++++++++++++ 2 files changed, 131 insertions(+) create mode 100644 maven-core/src/main/java/org/apache/maven/internal/aether/ReverseTreeRepositoryListener.java diff --git a/maven-core/src/main/java/org/apache/maven/internal/aether/DefaultRepositorySystemSessionFactory.java b/maven-core/src/main/java/org/apache/maven/internal/aether/DefaultRepositorySystemSessionFactory.java index 8d1bc07e4f63..78f34cbd7eed 100644 --- a/maven-core/src/main/java/org/apache/maven/internal/aether/DefaultRepositorySystemSessionFactory.java +++ b/maven-core/src/main/java/org/apache/maven/internal/aether/DefaultRepositorySystemSessionFactory.java @@ -59,6 +59,7 @@ import org.eclipse.aether.resolution.ResolutionErrorPolicy; import org.eclipse.aether.spi.localrepo.LocalRepositoryManagerFactory; import org.eclipse.aether.util.ConfigUtils; +import org.eclipse.aether.util.listener.ChainedRepositoryListener; import org.eclipse.aether.util.repository.AuthenticationBuilder; import org.eclipse.aether.util.repository.ChainedLocalRepositoryManager; import org.eclipse.aether.util.repository.DefaultAuthenticationSelector; @@ -89,6 +90,16 @@ public class DefaultRepositorySystemSessionFactory { */ private static final String MAVEN_REPO_LOCAL_TAIL_IGNORE_AVAILABILITY = "maven.repo.local.tail.ignoreAvailability"; + /** + * User property for reverse dependency tree. If enabled, Maven will record ".tracking" directory into local + * repository with "reverse dependency tree", essentially explaining WHY given artifact is present in local + * repository. + * Default: {@code false}, will not record anything. + * + * @since 3.9.0 + */ + private static final String MAVEN_REPO_LOCAL_RECORD_REVERSE_TREE = "maven.repo.local.recordReverseTree"; + private static final String MAVEN_RESOLVER_TRANSPORT_KEY = "maven.resolver.transport"; private static final String MAVEN_RESOLVER_TRANSPORT_DEFAULT = "default"; @@ -351,6 +362,12 @@ public DefaultRepositorySystemSession newRepositorySession(MavenExecutionRequest session.setRepositoryListener(eventSpyDispatcher.chainListener(new LoggingRepositoryListener(logger))); + boolean recordReverseTree = ConfigUtils.getBoolean(session, false, MAVEN_REPO_LOCAL_RECORD_REVERSE_TREE); + if (recordReverseTree) { + session.setRepositoryListener(new ChainedRepositoryListener( + session.getRepositoryListener(), new ReverseTreeRepositoryListener())); + } + mavenRepositorySystem.injectMirror(request.getRemoteRepositories(), request.getMirrors()); mavenRepositorySystem.injectProxy(session, request.getRemoteRepositories()); mavenRepositorySystem.injectAuthentication(session, request.getRemoteRepositories()); diff --git a/maven-core/src/main/java/org/apache/maven/internal/aether/ReverseTreeRepositoryListener.java b/maven-core/src/main/java/org/apache/maven/internal/aether/ReverseTreeRepositoryListener.java new file mode 100644 index 000000000000..2b5ad94ef93a --- /dev/null +++ b/maven-core/src/main/java/org/apache/maven/internal/aether/ReverseTreeRepositoryListener.java @@ -0,0 +1,114 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.maven.internal.aether; + +import static java.util.Objects.requireNonNull; + +import java.io.IOException; +import java.io.UncheckedIOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ListIterator; +import java.util.Objects; +import org.eclipse.aether.AbstractRepositoryListener; +import org.eclipse.aether.RepositoryEvent; +import org.eclipse.aether.RequestTrace; +import org.eclipse.aether.artifact.Artifact; +import org.eclipse.aether.collection.CollectStepData; +import org.eclipse.aether.graph.Dependency; +import org.eclipse.aether.graph.DependencyNode; + +/** + * A class building reverse tree using {@link CollectStepData} trace data provided in {@link RepositoryEvent} + * events fired during collection. + * + * @since 3.9.0 + */ +class ReverseTreeRepositoryListener extends AbstractRepositoryListener { + private static final String EOL = System.lineSeparator(); + + @Override + public void artifactResolved(RepositoryEvent event) { + requireNonNull(event, "event cannot be null"); + + if (!event.getArtifact() + .getFile() + .getPath() + .startsWith(event.getSession().getLocalRepository().getBasedir().getPath())) { + return; // reactor artifact + } + RequestTrace trace = event.getTrace(); + CollectStepData collectStepTrace = null; + while (trace != null) { + if (trace.getData() instanceof CollectStepData) { + collectStepTrace = (CollectStepData) trace.getData(); + break; + } + trace = trace.getParent(); + } + + if (collectStepTrace == null) { + return; + } + + Artifact resolvedArtifact = event.getArtifact(); + Artifact nodeArtifact = collectStepTrace.getNode().getArtifact(); + + if (isInScope(resolvedArtifact, nodeArtifact)) { + Dependency node = collectStepTrace.getNode(); + String trackingData = node + " (" + collectStepTrace.getContext() + ")" + EOL; + String indent = ""; + ListIterator iter = collectStepTrace + .getPath() + .listIterator(collectStepTrace.getPath().size()); + while (iter.hasPrevious()) { + DependencyNode curr = iter.previous(); + indent += " "; + trackingData += indent + curr + " (" + collectStepTrace.getContext() + ")" + EOL; + } + try { + Path trackingDir = + resolvedArtifact.getFile().getParentFile().toPath().resolve(".tracking"); + Files.createDirectories(trackingDir); + Path trackingFile = trackingDir.resolve(collectStepTrace + .getPath() + .get(0) + .getArtifact() + .toString() + .replace(":", "_")); + Files.write(trackingFile, trackingData.getBytes(StandardCharsets.UTF_8)); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + } + + /** + * The event "artifact resolved" if fired WHENEVER an artifact is resolved, BUT it happens also when an artifact + * descriptor (model, the POM) is being built, and parent (and parent of parent...) is being asked for. Hence, this + * method "filters" out in WHICH artifact are we interested in, but it intentionally neglects extension as + * ArtifactDescriptorReader modifies extension to "pom" during collect. So all we have to rely on is GAV only. + */ + private boolean isInScope(Artifact artifact, Artifact nodeArtifact) { + return Objects.equals(artifact.getGroupId(), nodeArtifact.getGroupId()) + && Objects.equals(artifact.getArtifactId(), nodeArtifact.getArtifactId()) + && Objects.equals(artifact.getVersion(), nodeArtifact.getVersion()); + } +} From a8691045d55ab48d7f3170022c2eba870b58c705 Mon Sep 17 00:00:00 2001 From: Tamas Cservenak Date: Sat, 3 Dec 2022 22:52:17 +0100 Subject: [PATCH 2/4] Rework --- .../internal/aether/ReverseTreeRepositoryListener.java | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/maven-core/src/main/java/org/apache/maven/internal/aether/ReverseTreeRepositoryListener.java b/maven-core/src/main/java/org/apache/maven/internal/aether/ReverseTreeRepositoryListener.java index 2b5ad94ef93a..54a19e2bbc80 100644 --- a/maven-core/src/main/java/org/apache/maven/internal/aether/ReverseTreeRepositoryListener.java +++ b/maven-core/src/main/java/org/apache/maven/internal/aether/ReverseTreeRepositoryListener.java @@ -25,6 +25,7 @@ import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; +import java.util.ArrayList; import java.util.ListIterator; import java.util.Objects; import org.eclipse.aether.AbstractRepositoryListener; @@ -42,8 +43,6 @@ * @since 3.9.0 */ class ReverseTreeRepositoryListener extends AbstractRepositoryListener { - private static final String EOL = System.lineSeparator(); - @Override public void artifactResolved(RepositoryEvent event) { requireNonNull(event, "event cannot be null"); @@ -73,7 +72,8 @@ public void artifactResolved(RepositoryEvent event) { if (isInScope(resolvedArtifact, nodeArtifact)) { Dependency node = collectStepTrace.getNode(); - String trackingData = node + " (" + collectStepTrace.getContext() + ")" + EOL; + ArrayList trackingData = new ArrayList<>(); + trackingData.add(node + " (" + collectStepTrace.getContext() + ")"); String indent = ""; ListIterator iter = collectStepTrace .getPath() @@ -81,7 +81,7 @@ public void artifactResolved(RepositoryEvent event) { while (iter.hasPrevious()) { DependencyNode curr = iter.previous(); indent += " "; - trackingData += indent + curr + " (" + collectStepTrace.getContext() + ")" + EOL; + trackingData.add(indent + curr + " (" + collectStepTrace.getContext() + ")"); } try { Path trackingDir = @@ -93,7 +93,7 @@ public void artifactResolved(RepositoryEvent event) { .getArtifact() .toString() .replace(":", "_")); - Files.write(trackingFile, trackingData.getBytes(StandardCharsets.UTF_8)); + Files.write(trackingFile, trackingData, StandardCharsets.UTF_8); } catch (IOException e) { throw new UncheckedIOException(e); } From 115f485918d5a0135f3396c0fdba5badd2c8bf1e Mon Sep 17 00:00:00 2001 From: Tamas Cservenak Date: Sun, 4 Dec 2022 13:29:51 +0100 Subject: [PATCH 3/4] PR comments. --- .../aether/ReverseTreeRepositoryListener.java | 52 ++++++++---- .../ReverseTreeRepositoryListenerTest.java | 85 +++++++++++++++++++ 2 files changed, 122 insertions(+), 15 deletions(-) create mode 100644 maven-core/src/test/java/org/apache/maven/internal/aether/ReverseTreeRepositoryListenerTest.java diff --git a/maven-core/src/main/java/org/apache/maven/internal/aether/ReverseTreeRepositoryListener.java b/maven-core/src/main/java/org/apache/maven/internal/aether/ReverseTreeRepositoryListener.java index 54a19e2bbc80..f9176b68c548 100644 --- a/maven-core/src/main/java/org/apache/maven/internal/aether/ReverseTreeRepositoryListener.java +++ b/maven-core/src/main/java/org/apache/maven/internal/aether/ReverseTreeRepositoryListener.java @@ -30,6 +30,7 @@ import java.util.Objects; import org.eclipse.aether.AbstractRepositoryListener; import org.eclipse.aether.RepositoryEvent; +import org.eclipse.aether.RepositorySystemSession; import org.eclipse.aether.RequestTrace; import org.eclipse.aether.artifact.Artifact; import org.eclipse.aether.collection.CollectStepData; @@ -47,22 +48,11 @@ class ReverseTreeRepositoryListener extends AbstractRepositoryListener { public void artifactResolved(RepositoryEvent event) { requireNonNull(event, "event cannot be null"); - if (!event.getArtifact() - .getFile() - .getPath() - .startsWith(event.getSession().getLocalRepository().getBasedir().getPath())) { - return; // reactor artifact - } - RequestTrace trace = event.getTrace(); - CollectStepData collectStepTrace = null; - while (trace != null) { - if (trace.getData() instanceof CollectStepData) { - collectStepTrace = (CollectStepData) trace.getData(); - break; - } - trace = trace.getParent(); + if (!isLocalRepositoryArtifact(event.getSession(), event.getArtifact())) { + return; } + CollectStepData collectStepTrace = lookupCollectStepData(event.getTrace()); if (collectStepTrace == null) { return; } @@ -100,13 +90,45 @@ public void artifactResolved(RepositoryEvent event) { } } + /** + * Returns {@code true} if passed in artifact is originating from local repository. In other words, we want + * to process and store tracking information ONLY into local repository, not to any other place. This method + * filters out currently built artifacts, as events are fired for them as well, but their resolved artifact + * file would point to checked out source-tree, not the local repository. + *

+ * Visible for testing. + */ + boolean isLocalRepositoryArtifact(RepositorySystemSession session, Artifact artifact) { + return artifact.getFile() + .getPath() + .startsWith(session.getLocalRepository().getBasedir().getPath()); + } + + /** + * Unravels trace tree (going upwards from current node), looking for {@link CollectStepData} trace data. + * This method may return {@code null} if no collect step data found in passed trace data or it's parents. + *

+ * Visible for testing. + */ + CollectStepData lookupCollectStepData(RequestTrace trace) { + CollectStepData collectStepTrace = null; + while (trace != null) { + if (trace.getData() instanceof CollectStepData) { + collectStepTrace = (CollectStepData) trace.getData(); + break; + } + trace = trace.getParent(); + } + return collectStepTrace; + } + /** * The event "artifact resolved" if fired WHENEVER an artifact is resolved, BUT it happens also when an artifact * descriptor (model, the POM) is being built, and parent (and parent of parent...) is being asked for. Hence, this * method "filters" out in WHICH artifact are we interested in, but it intentionally neglects extension as * ArtifactDescriptorReader modifies extension to "pom" during collect. So all we have to rely on is GAV only. */ - private boolean isInScope(Artifact artifact, Artifact nodeArtifact) { + boolean isInScope(Artifact artifact, Artifact nodeArtifact) { return Objects.equals(artifact.getGroupId(), nodeArtifact.getGroupId()) && Objects.equals(artifact.getArtifactId(), nodeArtifact.getArtifactId()) && Objects.equals(artifact.getVersion(), nodeArtifact.getVersion()); diff --git a/maven-core/src/test/java/org/apache/maven/internal/aether/ReverseTreeRepositoryListenerTest.java b/maven-core/src/test/java/org/apache/maven/internal/aether/ReverseTreeRepositoryListenerTest.java new file mode 100644 index 000000000000..414a96e37c86 --- /dev/null +++ b/maven-core/src/test/java/org/apache/maven/internal/aether/ReverseTreeRepositoryListenerTest.java @@ -0,0 +1,85 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.maven.internal.aether; + +import static org.hamcrest.CoreMatchers.equalTo; +import static org.hamcrest.CoreMatchers.nullValue; +import static org.hamcrest.CoreMatchers.sameInstance; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import java.io.File; +import org.eclipse.aether.RepositorySystemSession; +import org.eclipse.aether.RequestTrace; +import org.eclipse.aether.artifact.Artifact; +import org.eclipse.aether.collection.CollectStepData; +import org.eclipse.aether.repository.LocalRepository; +import org.junit.Test; + +/** + * UT for {@link ReverseTreeRepositoryListener}. + */ +public class ReverseTreeRepositoryListenerTest { + private final ReverseTreeRepositoryListener subject = new ReverseTreeRepositoryListener(); + + @Test + public void isLocalRepositoryArtifactTest() { + File baseDir = new File("local/repository"); + LocalRepository localRepository = new LocalRepository(baseDir); + RepositorySystemSession session = mock(RepositorySystemSession.class); + when(session.getLocalRepository()).thenReturn(localRepository); + + Artifact localRepositoryArtifact = mock(Artifact.class); + when(localRepositoryArtifact.getFile()).thenReturn(new File(baseDir, "some/path/within")); + + Artifact nonLocalReposioryArtifact = mock(Artifact.class); + when(nonLocalReposioryArtifact.getFile()).thenReturn(new File("something/completely/different")); + + assertThat(subject.isLocalRepositoryArtifact(session, localRepositoryArtifact), equalTo(true)); + assertThat(subject.isLocalRepositoryArtifact(session, nonLocalReposioryArtifact), equalTo(false)); + } + + @Test + public void lookupCollectStepDataTest() { + RequestTrace doesNotHaveIt = + RequestTrace.newChild(null, "foo").newChild("bar").newChild("baz"); + assertThat(subject.lookupCollectStepData(doesNotHaveIt), nullValue()); + + final CollectStepData data = mock(CollectStepData.class); + + RequestTrace haveItFirst = RequestTrace.newChild(null, data) + .newChild("foo") + .newChild("bar") + .newChild("baz"); + assertThat(subject.lookupCollectStepData(haveItFirst), sameInstance(data)); + + RequestTrace haveItLast = RequestTrace.newChild(null, "foo") + .newChild("bar") + .newChild("baz") + .newChild(data); + assertThat(subject.lookupCollectStepData(haveItLast), sameInstance(data)); + + RequestTrace haveIt = RequestTrace.newChild(null, "foo") + .newChild("bar") + .newChild(data) + .newChild("baz"); + assertThat(subject.lookupCollectStepData(haveIt), sameInstance(data)); + } +} From d973e5f82c4f20371eb9fa5bcef95201a6fc91bf Mon Sep 17 00:00:00 2001 From: Tamas Cservenak Date: Sun, 4 Dec 2022 14:00:19 +0100 Subject: [PATCH 4/4] Make all helper methods static --- .../aether/ReverseTreeRepositoryListener.java | 6 +++--- .../ReverseTreeRepositoryListenerTest.java | 18 ++++++++++-------- 2 files changed, 13 insertions(+), 11 deletions(-) diff --git a/maven-core/src/main/java/org/apache/maven/internal/aether/ReverseTreeRepositoryListener.java b/maven-core/src/main/java/org/apache/maven/internal/aether/ReverseTreeRepositoryListener.java index f9176b68c548..773d2e16fde6 100644 --- a/maven-core/src/main/java/org/apache/maven/internal/aether/ReverseTreeRepositoryListener.java +++ b/maven-core/src/main/java/org/apache/maven/internal/aether/ReverseTreeRepositoryListener.java @@ -98,7 +98,7 @@ public void artifactResolved(RepositoryEvent event) { *

* Visible for testing. */ - boolean isLocalRepositoryArtifact(RepositorySystemSession session, Artifact artifact) { + static boolean isLocalRepositoryArtifact(RepositorySystemSession session, Artifact artifact) { return artifact.getFile() .getPath() .startsWith(session.getLocalRepository().getBasedir().getPath()); @@ -110,7 +110,7 @@ boolean isLocalRepositoryArtifact(RepositorySystemSession session, Artifact arti *

* Visible for testing. */ - CollectStepData lookupCollectStepData(RequestTrace trace) { + static CollectStepData lookupCollectStepData(RequestTrace trace) { CollectStepData collectStepTrace = null; while (trace != null) { if (trace.getData() instanceof CollectStepData) { @@ -128,7 +128,7 @@ CollectStepData lookupCollectStepData(RequestTrace trace) { * method "filters" out in WHICH artifact are we interested in, but it intentionally neglects extension as * ArtifactDescriptorReader modifies extension to "pom" during collect. So all we have to rely on is GAV only. */ - boolean isInScope(Artifact artifact, Artifact nodeArtifact) { + static boolean isInScope(Artifact artifact, Artifact nodeArtifact) { return Objects.equals(artifact.getGroupId(), nodeArtifact.getGroupId()) && Objects.equals(artifact.getArtifactId(), nodeArtifact.getArtifactId()) && Objects.equals(artifact.getVersion(), nodeArtifact.getVersion()); diff --git a/maven-core/src/test/java/org/apache/maven/internal/aether/ReverseTreeRepositoryListenerTest.java b/maven-core/src/test/java/org/apache/maven/internal/aether/ReverseTreeRepositoryListenerTest.java index 414a96e37c86..82f69dbef398 100644 --- a/maven-core/src/test/java/org/apache/maven/internal/aether/ReverseTreeRepositoryListenerTest.java +++ b/maven-core/src/test/java/org/apache/maven/internal/aether/ReverseTreeRepositoryListenerTest.java @@ -37,8 +37,6 @@ * UT for {@link ReverseTreeRepositoryListener}. */ public class ReverseTreeRepositoryListenerTest { - private final ReverseTreeRepositoryListener subject = new ReverseTreeRepositoryListener(); - @Test public void isLocalRepositoryArtifactTest() { File baseDir = new File("local/repository"); @@ -52,15 +50,19 @@ public void isLocalRepositoryArtifactTest() { Artifact nonLocalReposioryArtifact = mock(Artifact.class); when(nonLocalReposioryArtifact.getFile()).thenReturn(new File("something/completely/different")); - assertThat(subject.isLocalRepositoryArtifact(session, localRepositoryArtifact), equalTo(true)); - assertThat(subject.isLocalRepositoryArtifact(session, nonLocalReposioryArtifact), equalTo(false)); + assertThat( + ReverseTreeRepositoryListener.isLocalRepositoryArtifact(session, localRepositoryArtifact), + equalTo(true)); + assertThat( + ReverseTreeRepositoryListener.isLocalRepositoryArtifact(session, nonLocalReposioryArtifact), + equalTo(false)); } @Test public void lookupCollectStepDataTest() { RequestTrace doesNotHaveIt = RequestTrace.newChild(null, "foo").newChild("bar").newChild("baz"); - assertThat(subject.lookupCollectStepData(doesNotHaveIt), nullValue()); + assertThat(ReverseTreeRepositoryListener.lookupCollectStepData(doesNotHaveIt), nullValue()); final CollectStepData data = mock(CollectStepData.class); @@ -68,18 +70,18 @@ public void lookupCollectStepDataTest() { .newChild("foo") .newChild("bar") .newChild("baz"); - assertThat(subject.lookupCollectStepData(haveItFirst), sameInstance(data)); + assertThat(ReverseTreeRepositoryListener.lookupCollectStepData(haveItFirst), sameInstance(data)); RequestTrace haveItLast = RequestTrace.newChild(null, "foo") .newChild("bar") .newChild("baz") .newChild(data); - assertThat(subject.lookupCollectStepData(haveItLast), sameInstance(data)); + assertThat(ReverseTreeRepositoryListener.lookupCollectStepData(haveItLast), sameInstance(data)); RequestTrace haveIt = RequestTrace.newChild(null, "foo") .newChild("bar") .newChild(data) .newChild("baz"); - assertThat(subject.lookupCollectStepData(haveIt), sameInstance(data)); + assertThat(ReverseTreeRepositoryListener.lookupCollectStepData(haveIt), sameInstance(data)); } }