From 1169a569c46c5ccaa7d1cb31fe87d3fb263746b1 Mon Sep 17 00:00:00 2001 From: andyrinne12 Date: Wed, 21 Jun 2023 12:09:56 -0700 Subject: [PATCH 01/13] ModqueryExecutor output logic - `show` implementation added (mostly uses `Query`'s `TargetOutputter` which is now publicly exposed instead of package-private). - `text`, `json` and `graphviz dot` outputters added - Unit testing for `ModqueryExecutor`output logic https://github.com/bazelbuild/bazel/issues/15365 PiperOrigin-RevId: 542325901 Change-Id: I155a326465355432fbb4436b28aecc0697c3ffab --- .../devtools/build/lib/bazel/bzlmod/BUILD | 4 +- .../bzlmod/BazelModuleInspectorValue.java | 24 +- .../build/lib/bazel/bzlmod/modquery/BUILD | 29 + .../modquery/GraphvizOutputFormatter.java | 111 ++++ .../bzlmod/modquery/JsonOutputFormatter.java | 102 ++++ .../modquery}/ModqueryExecutor.java | 210 ++++--- .../modquery}/ModqueryOptions.java | 26 +- .../bzlmod/modquery/OutputFormatters.java | 164 ++++++ .../bzlmod/modquery/TextOutputFormatter.java | 174 ++++++ .../devtools/build/lib/bazel/commands/BUILD | 3 +- .../lib/bazel/commands/ModqueryCommand.java | 14 +- .../query/output/BuildOutputFormatter.java | 7 +- .../query/output/PossibleAttributeValues.java | 8 +- .../devtools/build/lib/bazel/bzlmod/BUILD | 4 +- .../build/lib/bazel/bzlmod/modquery/BUILD | 39 ++ .../bzlmod/modquery/ModqueryExecutorTest.java | 528 ++++++++++++++++++ .../devtools/build/lib/bazel/commands/BUILD | 3 +- .../bazel/commands/ModqueryCommandTest.java | 2 +- .../bazel/commands/ModqueryExecutorTest.java | 238 -------- 19 files changed, 1352 insertions(+), 338 deletions(-) create mode 100644 src/main/java/com/google/devtools/build/lib/bazel/bzlmod/modquery/BUILD create mode 100644 src/main/java/com/google/devtools/build/lib/bazel/bzlmod/modquery/GraphvizOutputFormatter.java create mode 100644 src/main/java/com/google/devtools/build/lib/bazel/bzlmod/modquery/JsonOutputFormatter.java rename src/main/java/com/google/devtools/build/lib/bazel/{commands => bzlmod/modquery}/ModqueryExecutor.java (66%) rename src/main/java/com/google/devtools/build/lib/bazel/{commands => bzlmod/modquery}/ModqueryOptions.java (93%) create mode 100644 src/main/java/com/google/devtools/build/lib/bazel/bzlmod/modquery/OutputFormatters.java create mode 100644 src/main/java/com/google/devtools/build/lib/bazel/bzlmod/modquery/TextOutputFormatter.java create mode 100644 src/test/java/com/google/devtools/build/lib/bazel/bzlmod/modquery/BUILD create mode 100644 src/test/java/com/google/devtools/build/lib/bazel/bzlmod/modquery/ModqueryExecutorTest.java delete mode 100644 src/test/java/com/google/devtools/build/lib/bazel/commands/ModqueryExecutorTest.java diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BUILD b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BUILD index 1ffb2157cacacd..c6afd8832083cf 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BUILD +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BUILD @@ -6,7 +6,9 @@ package( filegroup( name = "srcs", - srcs = glob(["*"]), + srcs = glob(["*"]) + [ + "//src/main/java/com/google/devtools/build/lib/bazel/bzlmod/modquery:srcs", + ], visibility = ["//src:__subpackages__"], ) diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelModuleInspectorValue.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelModuleInspectorValue.java index 667f94e296dbdf..ca4632d32ad8bb 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelModuleInspectorValue.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelModuleInspectorValue.java @@ -196,17 +196,27 @@ public AugmentedModule.Builder addDepReason(String repoName, ResolutionReason re /** The reason why a final dependency of a module was resolved the way it was. */ public enum ResolutionReason { /** The dependency is the original dependency defined in the MODULE.bazel file. */ - ORIGINAL, + ORIGINAL(""), /** The dependency was replaced by the Minimal-Version Selection algorithm. */ - MINIMAL_VERSION_SELECTION, + MINIMAL_VERSION_SELECTION("MVS"), /** The dependency was replaced by a {@code single_version_override} rule. */ - SINGLE_VERSION_OVERRIDE, + SINGLE_VERSION_OVERRIDE("SVO"), /** The dependency was replaced by a {@code multiple_version_override} rule. */ - MULTIPLE_VERSION_OVERRIDE, + MULTIPLE_VERSION_OVERRIDE("MVO"), /** The dependency was replaced by one of the {@link NonRegistryOverride} rules. */ - ARCHIVE_OVERRIDE, - GIT_OVERRIDE, - LOCAL_PATH_OVERRIDE, + ARCHIVE_OVERRIDE("archive"), + GIT_OVERRIDE("git"), + LOCAL_PATH_OVERRIDE("local"); + + private final String label; + + ResolutionReason(String label) { + this.label = label; + } + + public String getLabel() { + return label; + } } } } diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/modquery/BUILD b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/modquery/BUILD new file mode 100644 index 00000000000000..6c0e7657b54973 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/modquery/BUILD @@ -0,0 +1,29 @@ +load("@rules_java//java:defs.bzl", "java_library") + +package( + default_applicable_licenses = ["//:license"], + default_visibility = ["//src:__subpackages__"], +) + +filegroup( + name = "srcs", + srcs = glob(["*"]), + visibility = ["//src:__subpackages__"], +) + +java_library( + name = "modquery", + srcs = glob(["*.java"]), + deps = [ + "//src/main/java/com/google/devtools/build/lib/bazel/bzlmod:common", + "//src/main/java/com/google/devtools/build/lib/bazel/bzlmod:inspection", + "//src/main/java/com/google/devtools/build/lib/bazel/bzlmod:repo_rule_value", + "//src/main/java/com/google/devtools/build/lib/packages", + "//src/main/java/com/google/devtools/build/lib/query2/query/output", + "//src/main/java/com/google/devtools/common/options", + "//third_party:auto_value", + "//third_party:gson", + "//third_party:guava", + "//third_party:jsr305", + ], +) diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/modquery/GraphvizOutputFormatter.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/modquery/GraphvizOutputFormatter.java new file mode 100644 index 00000000000000..91395832d013aa --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/modquery/GraphvizOutputFormatter.java @@ -0,0 +1,111 @@ +// Copyright 2022 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.bazel.bzlmod.modquery; + +import com.google.common.base.Preconditions; +import com.google.devtools.build.lib.bazel.bzlmod.BazelModuleInspectorValue.AugmentedModule; +import com.google.devtools.build.lib.bazel.bzlmod.ModuleKey; +import com.google.devtools.build.lib.bazel.bzlmod.Version; +import com.google.devtools.build.lib.bazel.bzlmod.modquery.ModqueryExecutor.ResultNode; +import com.google.devtools.build.lib.bazel.bzlmod.modquery.ModqueryExecutor.ResultNode.IsIndirect; +import com.google.devtools.build.lib.bazel.bzlmod.modquery.ModqueryExecutor.ResultNode.NodeMetadata; +import com.google.devtools.build.lib.bazel.bzlmod.modquery.OutputFormatters.OutputFormatter; +import java.util.ArrayDeque; +import java.util.Deque; +import java.util.HashSet; +import java.util.Map.Entry; +import java.util.Set; + +/** + * Outputs graph-based results of {@link ModqueryExecutor} in the Graphviz dot format which + * can be further pipelined to create an image graph visualization. + */ +public class GraphvizOutputFormatter extends OutputFormatter { + + @Override + public void output() { + StringBuilder str = new StringBuilder(); + str.append("digraph mygraph {\n") + .append(" ") + .append("node [ shape=box ]\n") + .append(" ") + .append("edge [ fontsize=8 ]\n"); + Set seen = new HashSet<>(); + Deque toVisit = new ArrayDeque<>(); + seen.add(ModuleKey.ROOT); + toVisit.add(ModuleKey.ROOT); + + while (!toVisit.isEmpty()) { + ModuleKey key = toVisit.pop(); + AugmentedModule module = depGraph.get(key); + ResultNode node = result.get(key); + Preconditions.checkNotNull(module); + Preconditions.checkNotNull(node); + String sourceId = toId(key); + + if (key.equals(ModuleKey.ROOT)) { + String rootLabel = String.format("root (%s@%s)", module.getName(), module.getVersion()); + str.append(String.format(" root [ label=\"%s\" ]\n", rootLabel)); + } else if (node.isTarget() || !module.isUsed()) { + String shapeString = node.isTarget() ? "diamond" : "box"; + String styleString = module.isUsed() ? "solid" : "dotted"; + str.append( + String.format(" %s [ shape=%s style=%s ]\n", toId(key), shapeString, styleString)); + } + + for (Entry e : node.getChildrenSortedByKey()) { + ModuleKey childKey = e.getKey(); + IsIndirect childIndirect = e.getValue().isIndirect(); + String childId = toId(childKey); + if (childIndirect == IsIndirect.FALSE) { + String reasonLabel = getReasonLabel(childKey, key); + str.append(String.format(" %s -> %s [ %s ]\n", sourceId, childId, reasonLabel)); + } else { + str.append(String.format(" %s -> %s [ style=dashed ]\n", sourceId, childId)); + } + if (seen.add(childKey)) { + toVisit.add(childKey); + } + } + } + str.append("}"); + printer.println(str); + printer.flush(); + } + + private String toId(ModuleKey key) { + if (key.equals(ModuleKey.ROOT)) { + return "root"; + } + return String.format( + "\"%s@%s\"", + key.getName(), key.getVersion().equals(Version.EMPTY) ? "_" : key.getVersion()); + } + + private String getReasonLabel(ModuleKey key, ModuleKey parent) { + if (!options.extra) { + return ""; + } + Explanation explanation = getExtraResolutionExplanation(key, parent); + if (explanation == null) { + return ""; + } + String label = explanation.getResolutionReason().getLabel(); + if (!label.isEmpty()) { + return String.format("label=%s", label); + } + return ""; + } +} diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/modquery/JsonOutputFormatter.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/modquery/JsonOutputFormatter.java new file mode 100644 index 00000000000000..3a7091b394a1a3 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/modquery/JsonOutputFormatter.java @@ -0,0 +1,102 @@ +// Copyright 2022 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.bazel.bzlmod.modquery; + +import com.google.devtools.build.lib.bazel.bzlmod.BazelModuleInspectorValue.AugmentedModule; +import com.google.devtools.build.lib.bazel.bzlmod.ModuleKey; +import com.google.devtools.build.lib.bazel.bzlmod.modquery.ModqueryExecutor.ResultNode; +import com.google.devtools.build.lib.bazel.bzlmod.modquery.ModqueryExecutor.ResultNode.IsCycle; +import com.google.devtools.build.lib.bazel.bzlmod.modquery.ModqueryExecutor.ResultNode.IsExpanded; +import com.google.devtools.build.lib.bazel.bzlmod.modquery.ModqueryExecutor.ResultNode.IsIndirect; +import com.google.devtools.build.lib.bazel.bzlmod.modquery.ModqueryExecutor.ResultNode.NodeMetadata; +import com.google.devtools.build.lib.bazel.bzlmod.modquery.OutputFormatters.OutputFormatter; +import com.google.gson.GsonBuilder; +import com.google.gson.JsonArray; +import com.google.gson.JsonObject; +import java.util.Map.Entry; + +/** Outputs graph-based results of {@link ModqueryExecutor} in JSON format. */ +public class JsonOutputFormatter extends OutputFormatter { + @Override + public void output() { + JsonObject root = printTree(ModuleKey.ROOT, null, IsExpanded.TRUE, IsIndirect.FALSE); + root.addProperty("root", true); + printer.println(new GsonBuilder().setPrettyPrinting().create().toJson(root)); + } + + public String printKey(ModuleKey key) { + if (key.equals(ModuleKey.ROOT)) { + return "root"; + } + return key.toString(); + } + + JsonObject printTree(ModuleKey key, ModuleKey parent, IsExpanded expanded, IsIndirect indirect) { + ResultNode node = result.get(key); + AugmentedModule module = depGraph.get(key); + JsonObject json = new JsonObject(); + json.addProperty("key", printKey(key)); + if (!key.getName().equals(module.getName())) { + json.addProperty("name", module.getName()); + } + if (!key.getVersion().equals(module.getVersion())) { + json.addProperty("version", module.getVersion().toString()); + } + + if (indirect == IsIndirect.FALSE && options.extra && parent != null) { + Explanation explanation = getExtraResolutionExplanation(key, parent); + if (explanation != null) { + if (!module.isUsed()) { + json.addProperty("unused", true); + json.addProperty("resolvedVersion", explanation.getChangedVersion().toString()); + } else { + json.addProperty("originalVersion", explanation.getChangedVersion().toString()); + } + json.addProperty("resolutionReason", explanation.getChangedVersion().toString()); + if (explanation.getRequestedByModules() != null) { + JsonArray requestedBy = new JsonArray(); + explanation.getRequestedByModules().forEach(k -> requestedBy.add(printKey(k))); + json.add("resolvedRequestedBy", requestedBy); + } + } + } + + if (expanded == IsExpanded.FALSE) { + json.addProperty("unexpanded", true); + return json; + } + + JsonArray deps = new JsonArray(); + JsonArray indirectDeps = new JsonArray(); + JsonArray cycles = new JsonArray(); + for (Entry e : node.getChildrenSortedByEdgeType()) { + ModuleKey childKey = e.getKey(); + IsExpanded childExpanded = e.getValue().isExpanded(); + IsIndirect childIndirect = e.getValue().isIndirect(); + IsCycle childCycles = e.getValue().isCycle(); + if (childCycles == IsCycle.TRUE) { + cycles.add(printTree(childKey, key, IsExpanded.FALSE, IsIndirect.FALSE)); + } else if (childIndirect == IsIndirect.TRUE) { + indirectDeps.add(printTree(childKey, key, childExpanded, IsIndirect.TRUE)); + } else { + deps.add(printTree(childKey, key, childExpanded, IsIndirect.FALSE)); + } + } + json.add("dependencies", deps); + json.add("indirectDependencies", indirectDeps); + json.add("cycles", cycles); + return json; + } +} diff --git a/src/main/java/com/google/devtools/build/lib/bazel/commands/ModqueryExecutor.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/modquery/ModqueryExecutor.java similarity index 66% rename from src/main/java/com/google/devtools/build/lib/bazel/commands/ModqueryExecutor.java rename to src/main/java/com/google/devtools/build/lib/bazel/bzlmod/modquery/ModqueryExecutor.java index 2fcb9fa4ffc239..c68c6b86594844 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/commands/ModqueryExecutor.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/modquery/ModqueryExecutor.java @@ -12,9 +12,10 @@ // See the License for the specific language governing permissions and // limitations under the License. -package com.google.devtools.build.lib.bazel.commands; +package com.google.devtools.build.lib.bazel.bzlmod.modquery; -import static com.google.common.collect.ImmutableMap.toImmutableMap; +import static com.google.common.collect.ImmutableSortedMap.toImmutableSortedMap; +import static java.util.Comparator.reverseOrder; import static java.util.stream.Collectors.toCollection; import com.google.auto.value.AutoValue; @@ -22,12 +23,21 @@ import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; -import com.google.common.collect.ImmutableSortedMap; +import com.google.common.collect.ImmutableSetMultimap; +import com.google.common.collect.ImmutableSortedSet; import com.google.devtools.build.lib.bazel.bzlmod.BazelModuleInspectorValue.AugmentedModule; import com.google.devtools.build.lib.bazel.bzlmod.BzlmodRepoRuleValue; import com.google.devtools.build.lib.bazel.bzlmod.ModuleKey; -import com.google.devtools.build.lib.bazel.commands.ModqueryExecutor.ResultNode.IsExpanded; +import com.google.devtools.build.lib.bazel.bzlmod.modquery.ModqueryExecutor.ResultNode.IsExpanded; +import com.google.devtools.build.lib.bazel.bzlmod.modquery.ModqueryExecutor.ResultNode.IsIndirect; +import com.google.devtools.build.lib.bazel.bzlmod.modquery.ModqueryExecutor.ResultNode.NodeMetadata; +import com.google.devtools.build.lib.packages.RawAttributeMapper; +import com.google.devtools.build.lib.packages.Rule; +import com.google.devtools.build.lib.query2.query.output.BuildOutputFormatter.AttributeReader; +import com.google.devtools.build.lib.query2.query.output.BuildOutputFormatter.TargetOutputter; +import com.google.devtools.build.lib.query2.query.output.PossibleAttributeValues; import com.google.errorprone.annotations.CanIgnoreReturnValue; +import java.io.IOException; import java.io.PrintWriter; import java.io.Writer; import java.util.ArrayDeque; @@ -37,12 +47,14 @@ import java.util.HashSet; import java.util.Map; import java.util.Map.Entry; +import java.util.Objects; import java.util.Set; import javax.annotation.Nullable; /** * Executes inspection queries for {@link - * com.google.devtools.build.lib.bazel.commands.ModqueryCommand}. + * com.google.devtools.build.lib.bazel.commands.ModqueryCommand} and prints the resulted output to + * the reporter's output stream using the different defined {@link OutputFormatters}. */ public class ModqueryExecutor { @@ -59,24 +71,26 @@ public ModqueryExecutor( public void tree(ImmutableSet targets) { ImmutableMap result = expandAndPrune(targets, ImmutableSet.of(), false); - printer.println(result); - printer.println("OUTPUT NOT IMPLEMENTED YET"); + OutputFormatters.getFormatter(options.outputFormat).output(result, depGraph, printer, options); } public void path(ImmutableSet from, ImmutableSet to) { ImmutableMap result = expandAndPrune(from, to, true); - printer.println(result); - printer.println("OUTPUT NOT IMPLEMENTED YET"); + OutputFormatters.getFormatter(options.outputFormat).output(result, depGraph, printer, options); } public void allPaths(ImmutableSet from, ImmutableSet to) { ImmutableMap result = expandAndPrune(from, to, false); - printer.println(result); - printer.println("OUTPUT NOT IMPLEMENTED YET"); + OutputFormatters.getFormatter(options.outputFormat).output(result, depGraph, printer, options); } public void show(ImmutableMap repoRuleValues) { - printer.println("OUTPUT NOT IMPLEMENTED YET"); + RuleDisplayOutputter outputter = new RuleDisplayOutputter(printer); + for (Entry e : repoRuleValues.entrySet()) { + printer.printf("## %s:", e.getKey()); + outputter.outputRule(e.getValue().getRule()); + } + printer.flush(); } /** @@ -104,13 +118,11 @@ ImmutableMap expandAndPrune( rootPinnedChildren.stream() .filter(coloredPaths::contains) .forEach( - moduleKey -> { - if (rootDirectChildren.contains(moduleKey)) { - rootBuilder.addChild(moduleKey, IsExpanded.TRUE); - } else { - rootBuilder.addIndirectChild(moduleKey, IsExpanded.TRUE); - } - }); + moduleKey -> + rootBuilder.addChild( + moduleKey, + IsExpanded.TRUE, + rootDirectChildren.contains(moduleKey) ? IsIndirect.FALSE : IsIndirect.TRUE)); resultBuilder.put(ModuleKey.ROOT, rootBuilder.build()); Set seen = new HashSet<>(rootPinnedChildren); @@ -137,11 +149,11 @@ ImmutableMap expandAndPrune( // wrong answer in cycle edge-case A -> B -> C -> B with target D will not find ABD // \__ D if (!singlePath) { - nodeBuilder.addChild(childKey, IsExpanded.FALSE); + nodeBuilder.addChild(childKey, IsExpanded.FALSE, IsIndirect.FALSE); } continue; } - nodeBuilder.addChild(childKey, IsExpanded.TRUE); + nodeBuilder.addChild(childKey, IsExpanded.TRUE, IsIndirect.FALSE); seen.add(childKey); toVisit.add(childKey); if (singlePath) { @@ -184,19 +196,18 @@ private ImmutableMap pruneByDepth() { parentStack.add(ModuleKey.ROOT); - for (ModuleKey childKey : oldResult.get(ModuleKey.ROOT).getChildren().keySet()) { - rootBuilder.addChild(childKey, IsExpanded.TRUE); - visitVisible(childKey, 1, ModuleKey.ROOT, IsExpanded.TRUE); - } - for (ModuleKey childKey : oldResult.get(ModuleKey.ROOT).getIndirectChildren().keySet()) { - rootBuilder.addIndirectChild(childKey, IsExpanded.TRUE); - visitVisible(childKey, 1, ModuleKey.ROOT, IsExpanded.TRUE); + for (Entry e : + oldResult.get(ModuleKey.ROOT).getChildrenSortedByKey()) { + rootBuilder.addChild(e.getKey(), IsExpanded.TRUE, e.getValue().isIndirect()); + visitVisible(e.getKey(), 1, ModuleKey.ROOT, IsExpanded.TRUE); } // Build everything at the end to allow children to add themselves to their parent's // adjacency list. return resultBuilder.entrySet().stream() - .collect(toImmutableMap(Entry::getKey, e -> e.getValue().build())); + .collect( + toImmutableSortedMap( + ModuleKey.LEXICOGRAPHIC_COMPARATOR, Entry::getKey, e -> e.getValue().build())); } // Handles graph traversal within the specified depth. @@ -209,16 +220,16 @@ private void visitVisible( resultBuilder.put(moduleKey, nodeBuilder); nodeBuilder.setTarget(oldNode.isTarget()); if (depth > 1) { - resultBuilder.get(parentKey).addChild(moduleKey, expanded); + resultBuilder.get(parentKey).addChild(moduleKey, expanded, IsIndirect.FALSE); } if (expanded == IsExpanded.FALSE) { parentStack.remove(moduleKey); return; } - for (Entry e : oldNode.getChildren().entrySet()) { + for (Entry e : oldNode.getChildrenSortedByKey()) { ModuleKey childKey = e.getKey(); - IsExpanded childExpanded = e.getValue(); + IsExpanded childExpanded = e.getValue().isExpanded(); if (notCycle(childKey)) { if (depth < options.depth) { visitVisible(childKey, depth + 1, moduleKey, childExpanded); @@ -226,7 +237,7 @@ private void visitVisible( visitDetached(childKey, moduleKey, moduleKey, childExpanded); } } else if (options.cycles) { - nodeBuilder.addChild(childKey, IsExpanded.FALSE); + nodeBuilder.addCycle(childKey); } } parentStack.remove(moduleKey); @@ -246,11 +257,9 @@ private void visitDetached( if (oldNode.isTarget() || oldNode.isTargetParent()) { ResultNode.Builder parentBuilder = resultBuilder.get(lastVisibleParentKey); - if (lastVisibleParentKey.equals(parentKey)) { - parentBuilder.addChild(moduleKey, expanded); - } else { - parentBuilder.addIndirectChild(moduleKey, expanded); - } + IsIndirect childIndirect = + lastVisibleParentKey.equals(parentKey) ? IsIndirect.FALSE : IsIndirect.TRUE; + parentBuilder.addChild(moduleKey, expanded, childIndirect); resultBuilder.put(moduleKey, nodeBuilder); lastVisibleParentKey = moduleKey; } @@ -259,13 +268,13 @@ private void visitDetached( parentStack.remove(moduleKey); return; } - for (Entry e : oldNode.getChildren().entrySet()) { + for (Entry e : oldNode.getChildrenSortedByKey()) { ModuleKey childKey = e.getKey(); - IsExpanded childExpanded = e.getValue(); + IsExpanded childExpanded = e.getValue().isExpanded(); if (notCycle(childKey)) { visitDetached(childKey, moduleKey, lastVisibleParentKey, childExpanded); } else if (options.cycles) { - nodeBuilder.addChild(childKey, IsExpanded.FALSE); + nodeBuilder.addCycle(childKey); } } parentStack.remove(moduleKey); @@ -323,14 +332,16 @@ private MaybeCompleteSet colorReversePathsToRoot(ImmutableSet dependenciesGraph) { - AugmentedModule module = dependenciesGraph.get(key); + AugmentedModule module = Objects.requireNonNull(dependenciesGraph.get(key)); if (key.equals(ModuleKey.ROOT)) { return false; } @@ -343,8 +354,9 @@ static boolean filterUnused( return true; } + /** A node representing a module that forms the result graph. */ @AutoValue - abstract static class ResultNode { + public abstract static class ResultNode { /** Whether the module is one of the targets in a paths query. */ abstract boolean isTarget(); @@ -355,15 +367,59 @@ abstract static class ResultNode { abstract boolean isTargetParent(); enum IsExpanded { - TRUE, - FALSE + FALSE, + TRUE } - /** List of direct children. True if the children will be expanded in this branch. */ - abstract ImmutableSortedMap getChildren(); + enum IsIndirect { + FALSE, + TRUE + } - /** List of indirect children. True if the children will be expanded in this branch. */ - abstract ImmutableSortedMap getIndirectChildren(); + enum IsCycle { + FALSE, + TRUE + } + + /** Detailed edge type for the {@link ResultNode} graph. */ + @AutoValue + public abstract static class NodeMetadata { + /** + * Whether the node should be expanded from this edge (the same node can appear in multiple + * places in a flattened graph). + */ + public abstract IsExpanded isExpanded(); + + /** Whether the edge is a direct edge or an indirect (transitive) one. */ + public abstract IsIndirect isIndirect(); + + /** Whether the edge is cycling back inside the flattened graph. */ + public abstract IsCycle isCycle(); + + private static NodeMetadata create( + IsExpanded isExpanded, IsIndirect isIndirect, IsCycle isCycle) { + return new AutoValue_ModqueryExecutor_ResultNode_NodeMetadata( + isExpanded, isIndirect, isCycle); + } + } + + /** List of children mapped to detailed edge types. */ + protected abstract ImmutableSetMultimap getChildren(); + + public ImmutableSortedSet> getChildrenSortedByKey() { + return ImmutableSortedSet.copyOf( + Entry.comparingByKey(ModuleKey.LEXICOGRAPHIC_COMPARATOR), getChildren().entries()); + } + + public ImmutableSortedSet> getChildrenSortedByEdgeType() { + return ImmutableSortedSet.copyOf( + Comparator., IsCycle>comparing( + e -> e.getValue().isCycle(), reverseOrder()) + .thenComparing(e -> e.getValue().isExpanded()) + .thenComparing(e -> e.getValue().isIndirect()) + .thenComparing(Entry::getKey, ModuleKey.LEXICOGRAPHIC_COMPARATOR), + getChildren().entries()); + } static ResultNode.Builder builder() { return new AutoValue_ModqueryExecutor_ResultNode.Builder() @@ -376,28 +432,20 @@ abstract static class Builder { abstract ResultNode.Builder setTargetParent(boolean value); - private final ImmutableSortedMap.Builder childrenBuilder = - childrenBuilder(ModuleKey.LEXICOGRAPHIC_COMPARATOR); - private final ImmutableSortedMap.Builder indirectChildrenBuilder = - indirectChildrenBuilder(ModuleKey.LEXICOGRAPHIC_COMPARATOR); - abstract ResultNode.Builder setTarget(boolean value); - abstract ImmutableSortedMap.Builder childrenBuilder( - Comparator comparator); - - abstract ImmutableSortedMap.Builder indirectChildrenBuilder( - Comparator comparator); + abstract ImmutableSetMultimap.Builder childrenBuilder(); @CanIgnoreReturnValue - final Builder addChild(ModuleKey value, IsExpanded expanded) { - childrenBuilder.put(value, expanded); + final Builder addChild(ModuleKey value, IsExpanded expanded, IsIndirect indirect) { + childrenBuilder().put(value, NodeMetadata.create(expanded, indirect, IsCycle.FALSE)); return this; } @CanIgnoreReturnValue - final Builder addIndirectChild(ModuleKey value, IsExpanded expanded) { - indirectChildrenBuilder.put(value, expanded); + final Builder addCycle(ModuleKey value) { + childrenBuilder() + .put(value, NodeMetadata.create(IsExpanded.FALSE, IsIndirect.FALSE, IsCycle.TRUE)); return this; } @@ -432,4 +480,34 @@ static MaybeCompleteSet completeSet() { return new AutoValue_ModqueryExecutor_MaybeCompleteSet<>(null); } } + + /** + * Uses Query's {@link TargetOutputter} to display the generating repo rule and other information. + */ + static class RuleDisplayOutputter { + private static final AttributeReader attrReader = + (rule, attr) -> + // Query's implementation copied + PossibleAttributeValues.forRuleAndAttribute( + rule, attr, /* mayTreatMultipleAsNone= */ true); + private final TargetOutputter targetOutputter; + private final PrintWriter printer; + + RuleDisplayOutputter(PrintWriter printer) { + this.printer = printer; + this.targetOutputter = + new TargetOutputter( + this.printer, + (rule, attr) -> RawAttributeMapper.of(rule).isConfigurable(attr.getName()), + "\n"); + } + + private void outputRule(Rule rule) { + try { + targetOutputter.outputRule(rule, attrReader, this.printer); + } catch (IOException e) { + throw new IllegalStateException(e); + } + } + } } diff --git a/src/main/java/com/google/devtools/build/lib/bazel/commands/ModqueryOptions.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/modquery/ModqueryOptions.java similarity index 93% rename from src/main/java/com/google/devtools/build/lib/bazel/commands/ModqueryOptions.java rename to src/main/java/com/google/devtools/build/lib/bazel/bzlmod/modquery/ModqueryOptions.java index c12675bfd03c4f..08e02f9ab12576 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/commands/ModqueryOptions.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/modquery/ModqueryOptions.java @@ -11,7 +11,7 @@ // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. -package com.google.devtools.build.lib.bazel.commands; +package com.google.devtools.build.lib.bazel.bzlmod.modquery; import static java.util.Arrays.stream; import static java.util.stream.Collectors.joining; @@ -115,7 +115,11 @@ public class ModqueryOptions extends OptionsBase { + "text, json, graph") public OutputFormat outputFormat; - enum QueryType { + /** + * Possible subcommands that can be specified for {@link + * com.google.devtools.build.lib.bazel.commands.ModqueryCommand} + */ + public enum QueryType { DEPS(1), TREE(0), ALL_PATHS(1), @@ -151,7 +155,11 @@ public QueryTypeConverter() { } } - enum Charset { + /** + * Charset to be used in outputting the {@link + * com.google.devtools.build.lib.bazel.commands.ModqueryCommand} result. + */ + public enum Charset { UTF8, ASCII } @@ -163,7 +171,11 @@ public CharsetConverter() { } } - enum OutputFormat { + /** + * Possible formats of the {@link com.google.devtools.build.lib.bazel.commands.ModqueryCommand} + * result. + */ + public enum OutputFormat { TEXT, JSON, GRAPH @@ -178,19 +190,19 @@ public OutputFormatConverter() { /** Argument of a modquery converted from the form name@version or name. */ @AutoValue - abstract static class TargetModule { + public abstract static class TargetModule { static TargetModule create(String name, Version version) { return new AutoValue_ModqueryOptions_TargetModule(name, version); } - abstract String getName(); + public abstract String getName(); /** * If it is null, it represents any (one or multiple) present versions of the module in the dep * graph, which is different from the empty version */ @Nullable - abstract Version getVersion(); + public abstract Version getVersion(); } /** Converts a module target argument string to a properly typed {@link TargetModule} */ diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/modquery/OutputFormatters.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/modquery/OutputFormatters.java new file mode 100644 index 00000000000000..78d40b0596f38d --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/modquery/OutputFormatters.java @@ -0,0 +1,164 @@ +// Copyright 2022 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.bazel.bzlmod.modquery; + +import static java.util.stream.Collectors.joining; + +import com.google.auto.value.AutoValue; +import com.google.common.base.Ascii; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; +import com.google.devtools.build.lib.bazel.bzlmod.BazelModuleInspectorValue.AugmentedModule; +import com.google.devtools.build.lib.bazel.bzlmod.BazelModuleInspectorValue.AugmentedModule.ResolutionReason; +import com.google.devtools.build.lib.bazel.bzlmod.ModuleKey; +import com.google.devtools.build.lib.bazel.bzlmod.Version; +import com.google.devtools.build.lib.bazel.bzlmod.modquery.ModqueryExecutor.ResultNode; +import com.google.devtools.build.lib.bazel.bzlmod.modquery.ModqueryOptions.OutputFormat; +import java.io.PrintWriter; +import javax.annotation.Nullable; + +/** + * Contains the output formatters for the graph-based results of {@link ModqueryExecutor} that can + * be specified using {@link ModqueryOptions#outputFormat}. + */ +public final class OutputFormatters { + + private static final OutputFormatter textFormatter = new TextOutputFormatter(); + private static final OutputFormatter jsonFormatter = new JsonOutputFormatter(); + private static final OutputFormatter graphvizFormatter = new GraphvizOutputFormatter(); + + private OutputFormatters() {} + + static OutputFormatter getFormatter(OutputFormat format) { + switch (format) { + case TEXT: + return textFormatter; + case JSON: + return jsonFormatter; + case GRAPH: + return graphvizFormatter; + } + throw new IllegalArgumentException("Output format cannot be null."); + } + + abstract static class OutputFormatter { + + protected ImmutableMap result; + protected ImmutableMap depGraph; + protected PrintWriter printer; + protected ModqueryOptions options; + + /** Compact representation of the data provided by the {@link ModqueryOptions#extra} flag. */ + @AutoValue + abstract static class Explanation { + + /** The version from/to which the module was changed after resolution. */ + abstract Version getChangedVersion(); + + abstract ResolutionReason getResolutionReason(); + + /** + * The list of modules who originally requested the selected version in the case of + * Minimal-Version-Selection. + */ + @Nullable + abstract ImmutableSet getRequestedByModules(); + + static Explanation create( + Version version, ResolutionReason reason, ImmutableSet requestedByModules) { + return new AutoValue_OutputFormatters_OutputFormatter_Explanation( + version, reason, requestedByModules); + } + + /** + * Gets the exact label that is printed next to the module if the {@link + * ModqueryOptions#extra} flag is enabled. + */ + String toExplanationString(boolean unused) { + String changedVersionLabel = + getChangedVersion().equals(Version.EMPTY) ? "_" : getChangedVersion().toString(); + String toOrWasString = unused ? "to" : "was"; + String reasonString = + getRequestedByModules() != null + ? getRequestedByModules().stream().map(ModuleKey::toString).collect(joining(", ")) + : Ascii.toLowerCase(getResolutionReason().toString()); + return String.format("(%s %s, cause %s)", toOrWasString, changedVersionLabel, reasonString); + } + } + + /** Exposed API of the formatter during which the necessary objects are injected. */ + void output( + ImmutableMap result, + ImmutableMap depGraph, + PrintWriter printer, + ModqueryOptions options) { + this.result = result; + this.depGraph = depGraph; + this.printer = printer; + this.options = options; + output(); + printer.flush(); + } + + /** Internal implementation of the formatter output function. */ + protected abstract void output(); + + /** + * Exists only for testing, because normally the depGraph and options are injected inside the + * public API call. + */ + protected Explanation getExtraResolutionExplanation( + ModuleKey key, + ModuleKey parent, + ImmutableMap depGraph, + ModqueryOptions options) { + this.depGraph = depGraph; + this.options = options; + return getExtraResolutionExplanation(key, parent); + } + + /** + * Returns {@code null} if the module version has not changed during resolution or if the module + * is <root>. + */ + @Nullable + protected Explanation getExtraResolutionExplanation(ModuleKey key, ModuleKey parent) { + if (key.equals(ModuleKey.ROOT)) { + return null; + } + AugmentedModule module = depGraph.get(key); + AugmentedModule parentModule = depGraph.get(parent); + String repoName = parentModule.getAllDeps(options.includeUnused).get(key); + Version changedVersion; + ImmutableSet changedByModules = null; + ResolutionReason reason = parentModule.getDepReasons().get(repoName); + AugmentedModule replacement = + module.isUsed() ? module : depGraph.get(parentModule.getDeps().get(repoName)); + if (reason != ResolutionReason.ORIGINAL) { + if (!module.isUsed()) { + changedVersion = replacement.getVersion(); + } else { + AugmentedModule old = depGraph.get(parentModule.getUnusedDeps().get(repoName)); + changedVersion = old.getVersion(); + } + if (reason == ResolutionReason.MINIMAL_VERSION_SELECTION) { + changedByModules = replacement.getOriginalDependants(); + } + return Explanation.create(changedVersion, reason, changedByModules); + } + return null; + } + } +} diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/modquery/TextOutputFormatter.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/modquery/TextOutputFormatter.java new file mode 100644 index 00000000000000..52b995d78e1dca --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/modquery/TextOutputFormatter.java @@ -0,0 +1,174 @@ +// Copyright 2022 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.bazel.bzlmod.modquery; + +import com.google.common.base.Preconditions; +import com.google.devtools.build.lib.bazel.bzlmod.BazelModuleInspectorValue.AugmentedModule; +import com.google.devtools.build.lib.bazel.bzlmod.ModuleKey; +import com.google.devtools.build.lib.bazel.bzlmod.Version; +import com.google.devtools.build.lib.bazel.bzlmod.modquery.ModqueryExecutor.ResultNode; +import com.google.devtools.build.lib.bazel.bzlmod.modquery.ModqueryExecutor.ResultNode.IsCycle; +import com.google.devtools.build.lib.bazel.bzlmod.modquery.ModqueryExecutor.ResultNode.IsExpanded; +import com.google.devtools.build.lib.bazel.bzlmod.modquery.ModqueryExecutor.ResultNode.IsIndirect; +import com.google.devtools.build.lib.bazel.bzlmod.modquery.ModqueryExecutor.ResultNode.NodeMetadata; +import com.google.devtools.build.lib.bazel.bzlmod.modquery.ModqueryOptions.Charset; +import com.google.devtools.build.lib.bazel.bzlmod.modquery.OutputFormatters.OutputFormatter; +import java.util.ArrayDeque; +import java.util.Deque; +import java.util.Iterator; +import java.util.Map.Entry; +import java.util.Objects; + +/** + * Outputs graph-based results of {@link + * com.google.devtools.build.lib.bazel.bzlmod.modquery.ModqueryExecutor} in a human-readable text + * format. + */ +public class TextOutputFormatter extends OutputFormatter { + + private Deque isLastChildStack; + private DrawCharset drawCharset; + + @Override + public void output() { + if (options.charset == Charset.ASCII) { + drawCharset = DrawCharset.ASCII; + } else { + drawCharset = DrawCharset.UTF8; + } + isLastChildStack = new ArrayDeque<>(); + printTree(ModuleKey.ROOT, null, IsExpanded.TRUE, IsIndirect.FALSE, IsCycle.FALSE, 0); + } + + // Depth-first traversal to print the actual output + void printTree( + ModuleKey key, + ModuleKey parent, + IsExpanded expanded, + IsIndirect indirect, + IsCycle cycle, + int depth) { + ResultNode node = Objects.requireNonNull(result.get(key)); + StringBuilder str = new StringBuilder(); + + if (depth > 0) { + int indents = isLastChildStack.size() - 1; + Iterator value = isLastChildStack.descendingIterator(); + for (int i = 0; i < indents; i++) { + boolean isLastChild = value.next(); + if (isLastChild) { + str.append(drawCharset.emptyIndent); + } else { + str.append(drawCharset.prevChildIndent); + } + } + if (indirect == IsIndirect.TRUE) { + if (isLastChildStack.getFirst()) { + str.append(drawCharset.lastIndirectChildIndent); + } else { + str.append(drawCharset.indirectChildIndent); + } + } else { + if (isLastChildStack.getFirst()) { + str.append(drawCharset.lastChildIndent); + + } else { + str.append(drawCharset.childIndent); + } + } + } + + int totalChildrenNum = node.getChildren().size(); + + if (key.equals(ModuleKey.ROOT)) { + AugmentedModule rootModule = depGraph.get(ModuleKey.ROOT); + Preconditions.checkNotNull(rootModule); + str.append( + String.format( + "root (%s@%s)", + rootModule.getName(), + rootModule.getVersion().equals(Version.EMPTY) ? "_" : rootModule.getVersion())); + } else { + str.append(key).append(" "); + } + + if (cycle == IsCycle.TRUE) { + str.append("(cycle) "); + } else if (expanded == IsExpanded.FALSE) { + str.append("(*) "); + } else { + if (totalChildrenNum != 0 && node.isTarget()) { + str.append("# "); + } + } + AugmentedModule module = Objects.requireNonNull(depGraph.get(key)); + + if (!options.extra && !module.isUsed()) { + str.append("(unused) "); + } + // If the edge is indirect, the parent is not only unknown, but the node could have come + // from + // multiple paths merged in the process, so we skip the resolution explanation. + if (indirect == IsIndirect.FALSE && options.extra && parent != null) { + Explanation explanation = getExtraResolutionExplanation(key, parent); + if (explanation != null) { + str.append(explanation.toExplanationString(!module.isUsed())); + } + } + + this.printer.println(str); + + if (expanded == IsExpanded.FALSE) { + return; + } + + int currChild = 1; + for (Entry e : node.getChildrenSortedByEdgeType()) { + ModuleKey childKey = e.getKey(); + IsExpanded childExpanded = e.getValue().isExpanded(); + IsIndirect childIndirect = e.getValue().isIndirect(); + IsCycle childCycles = e.getValue().isCycle(); + isLastChildStack.push(currChild++ == totalChildrenNum); + printTree(childKey, key, childExpanded, childIndirect, childCycles, depth + 1); + isLastChildStack.pop(); + } + } + + enum DrawCharset { + ASCII(" ", "| ", "|___", "|...", "|___", "|..."), + UTF8(" ", "│ ", "├───", "├╌╌╌", "└───", "└╌╌╌"); + final String emptyIndent; + final String prevChildIndent; + final String childIndent; + final String indirectChildIndent; + final String lastChildIndent; + final String lastIndirectChildIndent; + + DrawCharset( + String emptyIndent, + String prevChildIndent, + String childIndent, + String indirectChildIndent, + String lastChildIndent, + String lastIndirectChildIndent) { + this.emptyIndent = emptyIndent; + this.prevChildIndent = prevChildIndent; + this.childIndent = childIndent; + this.indirectChildIndent = indirectChildIndent; + this.lastChildIndent = lastChildIndent; + this.lastIndirectChildIndent = lastIndirectChildIndent; + } + } +} diff --git a/src/main/java/com/google/devtools/build/lib/bazel/commands/BUILD b/src/main/java/com/google/devtools/build/lib/bazel/commands/BUILD index e6d63e1b2a042e..ee951d3264ae3d 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/commands/BUILD +++ b/src/main/java/com/google/devtools/build/lib/bazel/commands/BUILD @@ -31,6 +31,7 @@ java_library( "//src/main/java/com/google/devtools/build/lib/bazel/bzlmod:common", "//src/main/java/com/google/devtools/build/lib/bazel/bzlmod:inspection", "//src/main/java/com/google/devtools/build/lib/bazel/bzlmod:repo_rule_value", + "//src/main/java/com/google/devtools/build/lib/bazel/bzlmod/modquery", "//src/main/java/com/google/devtools/build/lib/bazel/repository", "//src/main/java/com/google/devtools/build/lib/bazel/repository/starlark", "//src/main/java/com/google/devtools/build/lib/cmdline", @@ -57,8 +58,6 @@ java_library( "//src/main/java/com/google/devtools/common/options", "//src/main/java/net/starlark/java/eval", "//src/main/protobuf:failure_details_java_proto", - "//third_party:auto_value", "//third_party:guava", - "//third_party:jsr305", ], ) diff --git a/src/main/java/com/google/devtools/build/lib/bazel/commands/ModqueryCommand.java b/src/main/java/com/google/devtools/build/lib/bazel/commands/ModqueryCommand.java index 9495652ec440ea..f4a73220fd2564 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/commands/ModqueryCommand.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/commands/ModqueryCommand.java @@ -15,6 +15,7 @@ import static com.google.common.collect.ImmutableMap.toImmutableMap; import static com.google.common.collect.ImmutableSet.toImmutableSet; +import static com.google.devtools.build.lib.bazel.bzlmod.modquery.ModqueryOptions.Charset.UTF8; import static java.nio.charset.StandardCharsets.US_ASCII; import static java.nio.charset.StandardCharsets.UTF_8; @@ -29,11 +30,12 @@ import com.google.devtools.build.lib.bazel.bzlmod.BzlmodRepoRuleValue; import com.google.devtools.build.lib.bazel.bzlmod.ModuleKey; import com.google.devtools.build.lib.bazel.bzlmod.Version; -import com.google.devtools.build.lib.bazel.commands.ModqueryOptions.Charset; -import com.google.devtools.build.lib.bazel.commands.ModqueryOptions.QueryType; -import com.google.devtools.build.lib.bazel.commands.ModqueryOptions.QueryTypeConverter; -import com.google.devtools.build.lib.bazel.commands.ModqueryOptions.TargetModule; -import com.google.devtools.build.lib.bazel.commands.ModqueryOptions.TargetModuleListConverter; +import com.google.devtools.build.lib.bazel.bzlmod.modquery.ModqueryExecutor; +import com.google.devtools.build.lib.bazel.bzlmod.modquery.ModqueryOptions; +import com.google.devtools.build.lib.bazel.bzlmod.modquery.ModqueryOptions.QueryType; +import com.google.devtools.build.lib.bazel.bzlmod.modquery.ModqueryOptions.QueryTypeConverter; +import com.google.devtools.build.lib.bazel.bzlmod.modquery.ModqueryOptions.TargetModule; +import com.google.devtools.build.lib.bazel.bzlmod.modquery.ModqueryOptions.TargetModuleListConverter; import com.google.devtools.build.lib.events.Event; import com.google.devtools.build.lib.pkgcache.PackageOptions; import com.google.devtools.build.lib.runtime.BlazeCommand; @@ -226,7 +228,7 @@ public BlazeCommandResult exec(CommandEnvironment env, OptionsParsingResult opti modqueryOptions, new OutputStreamWriter( env.getReporter().getOutErr().getOutputStream(), - modqueryOptions.charset == Charset.UTF8 ? UTF_8 : US_ASCII)); + modqueryOptions.charset == UTF8 ? UTF_8 : US_ASCII)); switch (query) { case TREE: diff --git a/src/main/java/com/google/devtools/build/lib/query2/query/output/BuildOutputFormatter.java b/src/main/java/com/google/devtools/build/lib/query2/query/output/BuildOutputFormatter.java index 596cebd4c54361..46fed49d97eb0b 100644 --- a/src/main/java/com/google/devtools/build/lib/query2/query/output/BuildOutputFormatter.java +++ b/src/main/java/com/google/devtools/build/lib/query2/query/output/BuildOutputFormatter.java @@ -91,8 +91,8 @@ public void output(Target target, AttributeReader attrReader) throws IOException printed.add(rule.getLabel()); } - /** Outputs a given rule in BUILD-style syntax. */ - private void outputRule(Rule rule, AttributeReader attrReader, Writer writer) + /** Outputs a given rule in BUILD-style syntax. Made visible for Modquery command. */ + public void outputRule(Rule rule, AttributeReader attrReader, Writer writer) throws IOException { // TODO(b/151151653): display the filenames in root-relative form. // This is an incompatible change, but Blaze users (and their editors) @@ -233,7 +233,8 @@ public ThreadSafeOutputFormatterCallback createStreamCallback( createPostFactoStreamCallback(out, options, env.getMainRepoMapping())); } - private static class BuildOutputFormatterCallback extends TextOutputFormatterCallback { + /** BuildOutputFormatter callback for Query. Made visible for ModQuery. */ + public static class BuildOutputFormatterCallback extends TextOutputFormatterCallback { private final TargetOutputter targetOutputter; BuildOutputFormatterCallback(OutputStream out, String lineTerm) { diff --git a/src/main/java/com/google/devtools/build/lib/query2/query/output/PossibleAttributeValues.java b/src/main/java/com/google/devtools/build/lib/query2/query/output/PossibleAttributeValues.java index 391ce0add3556a..ebe499ee132f36 100644 --- a/src/main/java/com/google/devtools/build/lib/query2/query/output/PossibleAttributeValues.java +++ b/src/main/java/com/google/devtools/build/lib/query2/query/output/PossibleAttributeValues.java @@ -58,7 +58,7 @@ private PossibleAttributeValues() {} * respected on a best-effort basis - multiple values may still be returned if an unoptimized * code path is visited. */ - static Iterable forRuleAndAttribute( + public static Iterable forRuleAndAttribute( Rule rule, Attribute attr, boolean mayTreatMultipleAsNone) { AggregatingAttributeMapper attributeMap = AggregatingAttributeMapper.of(rule); if (attr.getType().equals(BuildType.LABEL_LIST) @@ -67,8 +67,8 @@ static Iterable forRuleAndAttribute( // there's currently no syntax for expressing multiple scalar values). This unfortunately // isn't trivial because Bazel's label visitation logic includes special methods built // directly into Type. - return ImmutableList.of( - attributeMap.getReachableLabels(attr.getName(), /*includeSelectKeys=*/ false)); + return ImmutableList.of( + attributeMap.getReachableLabels(attr.getName(), /* includeSelectKeys= */ false)); } Iterable concatenatedSelectsValue = @@ -84,4 +84,4 @@ static Iterable forRuleAndAttribute( attributeMap.visitAttribute(attr.getName(), attr.getType(), mayTreatMultipleAsNone); return possibleValues; } -} \ No newline at end of file +} diff --git a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BUILD b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BUILD index 50de8c6f8cb7ef..6c6c0411d917f3 100644 --- a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BUILD +++ b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BUILD @@ -8,7 +8,9 @@ package( filegroup( name = "srcs", testonly = 0, - srcs = glob(["*"]), + srcs = glob(["*"]) + [ + "//src/test/java/com/google/devtools/build/lib/bazel/bzlmod/modquery:srcs", + ], visibility = ["//src:__subpackages__"], ) diff --git a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/modquery/BUILD b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/modquery/BUILD new file mode 100644 index 00000000000000..a95a84b92cc54c --- /dev/null +++ b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/modquery/BUILD @@ -0,0 +1,39 @@ +load("@rules_java//java:defs.bzl", "java_test") + +package( + default_applicable_licenses = ["//:license"], + default_testonly = 1, + default_visibility = ["//src:__subpackages__"], +) + +filegroup( + name = "srcs", + testonly = 0, + srcs = glob(["*"]), + visibility = ["//src:__subpackages__"], +) + +java_library( + name = "ModqueryTests_lib", + srcs = glob( + ["*.java"], + ), + deps = [ + "//src/main/java/com/google/devtools/build/lib/bazel/bzlmod:common", + "//src/main/java/com/google/devtools/build/lib/bazel/bzlmod:inspection", + "//src/main/java/com/google/devtools/build/lib/bazel/bzlmod/modquery", + "//src/test/java/com/google/devtools/build/lib/bazel/bzlmod:util", + "//third_party:guava", + "//third_party:junit4", + "//third_party:truth", + ], +) + +java_test( + name = "ModqueryTests", + test_class = "com.google.devtools.build.lib.AllTests", + runtime_deps = [ + ":ModqueryTests_lib", + "//src/test/java/com/google/devtools/build/lib:test_runner", + ], +) diff --git a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/modquery/ModqueryExecutorTest.java b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/modquery/ModqueryExecutorTest.java new file mode 100644 index 00000000000000..7a2867eeb4f36e --- /dev/null +++ b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/modquery/ModqueryExecutorTest.java @@ -0,0 +1,528 @@ +// Copyright 2022 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.bazel.bzlmod.modquery; + +import static com.google.common.truth.Truth.assertThat; +import static com.google.devtools.build.lib.bazel.bzlmod.BzlmodTestUtil.AugmentedModuleBuilder.buildAugmentedModule; +import static com.google.devtools.build.lib.bazel.bzlmod.BzlmodTestUtil.createModuleKey; +import static java.nio.charset.StandardCharsets.UTF_8; + +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; +import com.google.devtools.build.lib.bazel.bzlmod.BazelModuleInspectorValue.AugmentedModule; +import com.google.devtools.build.lib.bazel.bzlmod.BazelModuleInspectorValue.AugmentedModule.ResolutionReason; +import com.google.devtools.build.lib.bazel.bzlmod.ModuleKey; +import com.google.devtools.build.lib.bazel.bzlmod.Version; +import com.google.devtools.build.lib.bazel.bzlmod.Version.ParseException; +import com.google.devtools.build.lib.bazel.bzlmod.modquery.ModqueryExecutor.ResultNode; +import com.google.devtools.build.lib.bazel.bzlmod.modquery.ModqueryExecutor.ResultNode.IsExpanded; +import com.google.devtools.build.lib.bazel.bzlmod.modquery.ModqueryExecutor.ResultNode.IsIndirect; +import com.google.devtools.build.lib.bazel.bzlmod.modquery.ModqueryOptions.OutputFormat; +import com.google.devtools.build.lib.bazel.bzlmod.modquery.OutputFormatters.OutputFormatter; +import com.google.devtools.build.lib.bazel.bzlmod.modquery.OutputFormatters.OutputFormatter.Explanation; +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.OutputStreamWriter; +import java.io.StringWriter; +import java.io.Writer; +import java.nio.file.Files; +import java.util.List; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +/** Tests for {@link ModqueryExecutor}. */ +@RunWith(JUnit4.class) +public class ModqueryExecutorTest { + // TODO(andreisolo): Add a Json output test + // TODO(andreisolo): Add a PATH query test + + private final Writer writer = new StringWriter(); + + // Tests for the ModqueryExecutor::expandAndPrune core function. + // + // (* In the ASCII graph hints "__>" or "-->" mean a direct edge, while "..>" means an indirect + // edge. "aaa ..." means module "aaa" is unexpanded.) + + @Test + public void testExpandFromTargetsFirst() throws ParseException { + // aaa -> bbb -> ccc -> ddd + ImmutableMap depGraph = + new ImmutableMap.Builder() + .put( + buildAugmentedModule(ModuleKey.ROOT, "aaa", Version.parse("1.0"), true) + .addDep("bbb", "1.0") + .buildEntry()) + .put( + buildAugmentedModule("bbb", "1.0") + .addStillDependant(ModuleKey.ROOT) + .addDep("ccc", "1.0") + .buildEntry()) + .put( + buildAugmentedModule("ccc", "1.0") + .addStillDependant("bbb", "1.0") + .addDep("ddd", "1.0") + .buildEntry()) + .put(buildAugmentedModule("ddd", "1.0").addStillDependant("ccc", "1.0").buildEntry()) + .buildOrThrow(); + + ModqueryOptions options = ModqueryOptions.getDefaultOptions(); + ModqueryExecutor executor = new ModqueryExecutor(depGraph, options, writer); + + // RESULT: + // ...> ccc -> ddd + // \___> bbb -> ccc ... + assertThat( + executor.expandAndPrune( + ImmutableSet.of(ModuleKey.ROOT, createModuleKey("ccc", "1.0")), + ImmutableSet.of(), + false)) + .containsExactly( + ModuleKey.ROOT, + ResultNode.builder() + .addChild(createModuleKey("bbb", "1.0"), IsExpanded.TRUE, IsIndirect.FALSE) + .addChild(createModuleKey("ccc", "1.0"), IsExpanded.TRUE, IsIndirect.TRUE) + .build(), + createModuleKey("bbb", "1.0"), + ResultNode.builder() + .addChild(createModuleKey("ccc", "1.0"), IsExpanded.FALSE, IsIndirect.FALSE) + .build(), + createModuleKey("ccc", "1.0"), + ResultNode.builder() + .addChild(createModuleKey("ddd", "1.0"), IsExpanded.TRUE, IsIndirect.FALSE) + .build(), + createModuleKey("ddd", "1.0"), + ResultNode.builder().build()) + .inOrder(); + } + + @Test + public void testPathsDepth1_containsAllTargetsWithNestedIndirect() throws ParseException { + // -> bbb -> ccc -> ddd -> eee -> fff -> ggg -> hhh + // ^ / + // \___/ + ImmutableMap depGraph = + new ImmutableMap.Builder() + .put( + buildAugmentedModule(ModuleKey.ROOT, "aaa", Version.parse("1.0"), true) + .addDep("bbb", "1.0") + .buildEntry()) + .put( + buildAugmentedModule("bbb", "1.0") + .addStillDependant(ModuleKey.ROOT) + .addDep("ccc", "1.0") + .buildEntry()) + .put( + buildAugmentedModule("ccc", "1.0") + .addStillDependant("bbb", "1.0") + .addDep("ddd", "1.0") + .buildEntry()) + .put( + buildAugmentedModule("ddd", "1.0") + .addStillDependant("ccc", "1.0") + .addStillDependant("eee", "1.0") + .addDep("eee", "1.0") + .buildEntry()) + .put( + buildAugmentedModule("eee", "1.0") + .addStillDependant("ddd", "1.0") + .addDep("fff", "1.0") + .addDep("ddd", "1.0") + .buildEntry()) + .put( + buildAugmentedModule("fff", "1.0") + .addStillDependant("eee", "1.0") + .addDep("ggg", "1.0") + .buildEntry()) + .put( + buildAugmentedModule("ggg", "1.0") + .addStillDependant("fff", "1.0") + .addDep("hhh", "1.0") + .buildEntry()) + .put(buildAugmentedModule("hhh", "1.0").addStillDependant("ggg", "1.0").buildEntry()) + .buildOrThrow(); + + ModqueryOptions options = ModqueryOptions.getDefaultOptions(); + options.cycles = true; + options.depth = 1; + ModqueryExecutor executor = new ModqueryExecutor(depGraph, options, writer); + ImmutableSet targets = + ImmutableSet.of(createModuleKey("eee", "1.0"), createModuleKey("hhh", "1.0")); + + // RESULT: + // --> bbb ..> ddd --> eee --> ddd (cycle) + // \..> ggg --> hhh + assertThat(executor.expandAndPrune(ImmutableSet.of(ModuleKey.ROOT), targets, false)) + .containsExactly( + ModuleKey.ROOT, + ResultNode.builder() + .addChild(createModuleKey("bbb", "1.0"), IsExpanded.TRUE, IsIndirect.FALSE) + .build(), + createModuleKey("bbb", "1.0"), + ResultNode.builder() + .addChild(createModuleKey("ddd", "1.0"), IsExpanded.TRUE, IsIndirect.TRUE) + .build(), + createModuleKey("ddd", "1.0"), + ResultNode.builder() + .addChild(createModuleKey("eee", "1.0"), IsExpanded.TRUE, IsIndirect.FALSE) + .build(), + createModuleKey("eee", "1.0"), + ResultNode.builder() + .setTarget(true) + .addChild(createModuleKey("ggg", "1.0"), IsExpanded.TRUE, IsIndirect.TRUE) + .addCycle(createModuleKey("ddd", "1.0")) + .build(), + createModuleKey("ggg", "1.0"), + ResultNode.builder() + .addChild(createModuleKey("hhh", "1.0"), IsExpanded.TRUE, IsIndirect.FALSE) + .build(), + createModuleKey("hhh", "1.0"), + ResultNode.builder().setTarget(true).build()) + .inOrder(); + } + + @Test + public void testPathsDepth1_targetParentIsDirectAndIndirectChild() throws ParseException { + // --> bbb --> ccc + // \ |________ + // \ V | + // \__> ddd --> eee + ImmutableMap depGraph = + new ImmutableMap.Builder() + .put( + buildAugmentedModule(ModuleKey.ROOT, "aaa", Version.parse("1.0"), true) + .addDep("bbb", "1.0") + .buildEntry()) + .put( + buildAugmentedModule("bbb", "1.0") + .addStillDependant(ModuleKey.ROOT) + .addDep("ccc", "1.0") + .addDep("ddd", "1.0") + .buildEntry()) + .put( + buildAugmentedModule("ccc", "1.0") + .addStillDependant("bbb", "1.0") + .addDep("ddd", "1.0") + .buildEntry()) + .put( + buildAugmentedModule("ddd", "1.0") + .addStillDependant("bbb", "1.0") + .addStillDependant("ccc", "1.0") + .addStillDependant("eee", "1.0") + .addDep("eee", "1.0") + .buildEntry()) + .put( + buildAugmentedModule("eee", "1.0") + .addStillDependant("ddd", "1.0") + .addStillDependant("eee", "1.0") + .addDep("ddd", "1.0") + .buildEntry()) + .buildOrThrow(); + + ModqueryOptions options = ModqueryOptions.getDefaultOptions(); + options.cycles = true; + options.depth = 1; + ModqueryExecutor executor = new ModqueryExecutor(depGraph, options, writer); + ImmutableSet targets = ImmutableSet.of(createModuleKey("eee", "1.0")); + + // RESULT: + // --> bbb --- ddd --> eee --> ddd (c) + // \ + // \..> ddd ... + assertThat(executor.expandAndPrune(ImmutableSet.of(ModuleKey.ROOT), targets, false)) + .containsExactly( + ModuleKey.ROOT, + ResultNode.builder() + .addChild(createModuleKey("bbb", "1.0"), IsExpanded.TRUE, IsIndirect.FALSE) + .build(), + createModuleKey("bbb", "1.0"), + ResultNode.builder() + .addChild(createModuleKey("ddd", "1.0"), IsExpanded.TRUE, IsIndirect.FALSE) + .addChild(createModuleKey("ddd", "1.0"), IsExpanded.FALSE, IsIndirect.TRUE) + .build(), + createModuleKey("ddd", "1.0"), + ResultNode.builder() + .addChild(createModuleKey("eee", "1.0"), IsExpanded.TRUE, IsIndirect.FALSE) + .build(), + createModuleKey("eee", "1.0"), + ResultNode.builder().setTarget(true).addCycle(createModuleKey("ddd", "1.0")).build()) + .inOrder(); + } + + // TODO(andreisolo): Add more eventual edge-case tests for the #expandAndPrune core method + + //// Tests for the ModqueryExecutor OutputFormatters + // + + @Test + public void testResolutionExplanation_mostCases() throws ParseException { + ImmutableMap depGraph = + new ImmutableMap.Builder() + .put( + buildAugmentedModule(ModuleKey.ROOT, "A", Version.parse("1.0"), true) + .addDep("B", "1.0") + .addDep("C", "1.0") + .buildEntry()) + .put( + buildAugmentedModule("B", "1.0") + .addStillDependant(ModuleKey.ROOT) + .addChangedDep("C", "1.0", "0.1", ResolutionReason.MINIMAL_VERSION_SELECTION) + .addChangedDep("E", "", "1.0", ResolutionReason.LOCAL_PATH_OVERRIDE) + .buildEntry()) + .put( + buildAugmentedModule("C", "1.0") + .addStillDependant(ModuleKey.ROOT) + .addDependant("B", "1.0") + .addChangedDep("D", "1.5", "1.0", ResolutionReason.SINGLE_VERSION_OVERRIDE) + .buildEntry()) + .put(buildAugmentedModule("C", "0.1").addOriginalDependant("B", "1.0").buildEntry()) + .put(buildAugmentedModule("D", "1.0").addOriginalDependant("C", "1.0").buildEntry()) + .put(buildAugmentedModule("D", "1.5").addDependant("C", "1.0").buildEntry()) + .put(buildAugmentedModule("E", "1.0").addOriginalDependant("B", "1.0").buildEntry()) + .put(buildAugmentedModule("E", "").addDependant("B", "1.0").buildEntry()) + .buildOrThrow(); + + ModqueryOptions options = ModqueryOptions.getDefaultOptions(); + options.extra = true; + options.includeUnused = true; + + OutputFormatter formatter = OutputFormatters.getFormatter(OutputFormat.TEXT); + assertThat(formatter.getExtraResolutionExplanation(ModuleKey.ROOT, null, depGraph, options)) + .isNull(); + assertThat( + formatter.getExtraResolutionExplanation( + createModuleKey("B", "1.0"), ModuleKey.ROOT, depGraph, options)) + .isNull(); + + assertThat( + formatter.getExtraResolutionExplanation( + createModuleKey("C", "1.0"), createModuleKey("B", "1.0"), depGraph, options)) + .isEqualTo( + Explanation.create( + Version.parse("0.1"), + ResolutionReason.MINIMAL_VERSION_SELECTION, + ImmutableSet.of(ModuleKey.ROOT))); + + assertThat( + formatter.getExtraResolutionExplanation( + createModuleKey("C", "0.1"), createModuleKey("B", "1.0"), depGraph, options)) + .isEqualTo( + Explanation.create( + Version.parse("1.0"), + ResolutionReason.MINIMAL_VERSION_SELECTION, + ImmutableSet.of(ModuleKey.ROOT))); + + assertThat( + formatter.getExtraResolutionExplanation( + createModuleKey("D", "1.0"), createModuleKey("C", "1.0"), depGraph, options)) + .isEqualTo( + Explanation.create( + Version.parse("1.5"), ResolutionReason.SINGLE_VERSION_OVERRIDE, null)); + + assertThat( + formatter.getExtraResolutionExplanation( + createModuleKey("D", "1.5"), createModuleKey("C", "1.0"), depGraph, options)) + .isEqualTo( + Explanation.create( + Version.parse("1.0"), ResolutionReason.SINGLE_VERSION_OVERRIDE, null)); + + assertThat( + formatter.getExtraResolutionExplanation( + createModuleKey("E", "1.0"), createModuleKey("B", "1.0"), depGraph, options)) + .isEqualTo(Explanation.create(Version.EMPTY, ResolutionReason.LOCAL_PATH_OVERRIDE, null)); + + assertThat( + formatter.getExtraResolutionExplanation( + createModuleKey("E", ""), createModuleKey("B", "1.0"), depGraph, options)) + .isEqualTo( + Explanation.create(Version.parse("1.0"), ResolutionReason.LOCAL_PATH_OVERRIDE, null)); + } + + @Test + public void testTextAndGraphOutput_indirectAndNestedTargetPathsWithUnused() + throws ParseException, IOException { + ImmutableMap depGraph = + new ImmutableMap.Builder() + .put( + buildAugmentedModule(ModuleKey.ROOT, "A", Version.parse("1.0"), true) + .addDep("B", "1.0") + .buildEntry()) + .put( + buildAugmentedModule("B", "1.0") + .addStillDependant(ModuleKey.ROOT) + .addChangedDep("C", "1.0", "0.1", ResolutionReason.SINGLE_VERSION_OVERRIDE) + .addChangedDep("Y", "2.0", "1.0", ResolutionReason.MINIMAL_VERSION_SELECTION) + .buildEntry()) + .put(buildAugmentedModule("C", "0.1").addOriginalDependant("B", "1.0").buildEntry()) + .put( + buildAugmentedModule("C", "1.0") + .addDependant("B", "1.0") + .addDep("D", "1.0") + .buildEntry()) + .put( + buildAugmentedModule("D", "1.0") + .addStillDependant("C", "1.0") + .addStillDependant("E", "1.0") + .addDep("E", "1.0") + .buildEntry()) + .put( + buildAugmentedModule("E", "1.0") + .addStillDependant("D", "1.0") + .addDep("F", "1.0") + .addDep("D", "1.0") + .buildEntry()) + .put( + buildAugmentedModule("F", "1.0") + .addStillDependant("E", "1.0") + .addDep("G", "1.0") + .buildEntry()) + .put( + buildAugmentedModule("G", "1.0") + .addStillDependant("F", "1.0") + .addDep("H", "1.0") + .addDep("Y", "2.0") + .buildEntry()) + .put(buildAugmentedModule("H", "1.0").addStillDependant("G", "1.0").buildEntry()) + .put(buildAugmentedModule("Y", "1.0").addOriginalDependant("B", "1.0").buildEntry()) + .put( + buildAugmentedModule("Y", "2.0") + .addDependant("B", "1.0") + .addStillDependant("G", "1.0") + .buildEntry()) + .buildOrThrow(); + + ImmutableMap result = + ImmutableMap.of( + ModuleKey.ROOT, + ResultNode.builder() + .addChild(createModuleKey("B", "1.0"), IsExpanded.TRUE, IsIndirect.FALSE) + .build(), + createModuleKey("B", "1.0"), + ResultNode.builder() + .addChild(createModuleKey("Y", "1.0"), IsExpanded.TRUE, IsIndirect.FALSE) + .addChild(createModuleKey("Y", "2.0"), IsExpanded.TRUE, IsIndirect.FALSE) + .addChild(createModuleKey("C", "1.0"), IsExpanded.TRUE, IsIndirect.FALSE) + .addChild(createModuleKey("C", "0.1"), IsExpanded.TRUE, IsIndirect.FALSE) + .build(), + createModuleKey("C", "0.1"), + ResultNode.builder().setTarget(true).build(), + createModuleKey("C", "1.0"), + ResultNode.builder() + .setTarget(true) + .addChild(createModuleKey("D", "1.0"), IsExpanded.TRUE, IsIndirect.FALSE) + .build(), + createModuleKey("D", "1.0"), + ResultNode.builder() + .addChild(createModuleKey("E", "1.0"), IsExpanded.TRUE, IsIndirect.FALSE) + .build(), + createModuleKey("E", "1.0"), + ResultNode.builder() + .setTarget(true) + .addChild(createModuleKey("G", "1.0"), IsExpanded.TRUE, IsIndirect.TRUE) + .addCycle(createModuleKey("D", "1.0")) + .build(), + createModuleKey("G", "1.0"), + ResultNode.builder() + .addChild(createModuleKey("H", "1.0"), IsExpanded.TRUE, IsIndirect.FALSE) + .addChild(createModuleKey("Y", "2.0"), IsExpanded.FALSE, IsIndirect.FALSE) + .build(), + createModuleKey("H", "1.0"), + ResultNode.builder().setTarget(true).build(), + createModuleKey("Y", "1.0"), + ResultNode.builder().setTarget(true).build(), + createModuleKey("Y", "2.0"), + ResultNode.builder().setTarget(true).build()); + + ModqueryOptions options = ModqueryOptions.getDefaultOptions(); + options.cycles = true; + options.includeUnused = true; + options.extra = true; + options.depth = 1; + options.outputFormat = OutputFormat.TEXT; + + File file = File.createTempFile("output_text", "txt"); + file.deleteOnExit(); + Writer writer = new OutputStreamWriter(new FileOutputStream(file), UTF_8); + + ModqueryExecutor executor = new ModqueryExecutor(depGraph, options, writer); + ImmutableSet targets = + ImmutableSet.of( + createModuleKey("C", "0.1"), + createModuleKey("C", "1.0"), + createModuleKey("Y", "1.0"), + createModuleKey("Y", "2.0"), + createModuleKey("E", "1.0"), + createModuleKey("H", "1.0")); + + // Double check for human error + assertThat(executor.expandAndPrune(ImmutableSet.of(ModuleKey.ROOT), targets, false)) + .isEqualTo(result); + + executor.allPaths(ImmutableSet.of(ModuleKey.ROOT), targets); + List textOutput = Files.readAllLines(file.toPath()); + + assertThat(textOutput) + .containsExactly( + "root (A@1.0)", + "└───B@1.0 ", + " ├───C@0.1 (to 1.0, cause single_version_override)", + " ├───C@1.0 # (was 0.1, cause single_version_override)", + " │ └───D@1.0 ", + " │ └───E@1.0 # ", + " │ ├───D@1.0 (cycle) ", + " │ └╌╌╌G@1.0 ", + " │ ├───Y@2.0 (*) ", + " │ └───H@1.0 ", + " ├───Y@1.0 (to 2.0, cause G@1.0)", + " └───Y@2.0 (was 1.0, cause G@1.0)") + .inOrder(); + + options.outputFormat = OutputFormat.GRAPH; + File fileGraph = File.createTempFile("output_graph", "txt"); + fileGraph.deleteOnExit(); + writer = new OutputStreamWriter(new FileOutputStream(fileGraph), UTF_8); + executor = new ModqueryExecutor(depGraph, options, writer); + + executor.allPaths(ImmutableSet.of(ModuleKey.ROOT), targets); + List graphOutput = Files.readAllLines(fileGraph.toPath()); + + assertThat(graphOutput) + .containsExactly( + "digraph mygraph {", + " node [ shape=box ]", + " edge [ fontsize=8 ]", + " root [ label=\"root (A@1.0)\" ]", + " root -> \"B@1.0\" [ ]", + " \"B@1.0\" -> \"C@0.1\" [ label=SVO ]", + " \"B@1.0\" -> \"C@1.0\" [ label=SVO ]", + " \"B@1.0\" -> \"Y@1.0\" [ label=MVS ]", + " \"B@1.0\" -> \"Y@2.0\" [ label=MVS ]", + " \"C@0.1\" [ shape=diamond style=dotted ]", + " \"C@1.0\" [ shape=diamond style=solid ]", + " \"C@1.0\" -> \"D@1.0\" [ ]", + " \"Y@1.0\" [ shape=diamond style=dotted ]", + " \"Y@2.0\" [ shape=diamond style=solid ]", + " \"D@1.0\" -> \"E@1.0\" [ ]", + " \"E@1.0\" [ shape=diamond style=solid ]", + " \"E@1.0\" -> \"D@1.0\" [ ]", + " \"E@1.0\" -> \"G@1.0\" [ style=dashed ]", + " \"G@1.0\" -> \"H@1.0\" [ ]", + " \"G@1.0\" -> \"Y@2.0\" [ ]", + " \"H@1.0\" [ shape=diamond style=solid ]", + "}") + .inOrder(); + } +} diff --git a/src/test/java/com/google/devtools/build/lib/bazel/commands/BUILD b/src/test/java/com/google/devtools/build/lib/bazel/commands/BUILD index 80a860ae33f8aa..3addf4c57bb916 100644 --- a/src/test/java/com/google/devtools/build/lib/bazel/commands/BUILD +++ b/src/test/java/com/google/devtools/build/lib/bazel/commands/BUILD @@ -19,11 +19,10 @@ java_library( ), deps = [ "//src/main/java/com/google/devtools/build/lib/bazel/bzlmod:common", - "//src/main/java/com/google/devtools/build/lib/bazel/bzlmod:inspection", + "//src/main/java/com/google/devtools/build/lib/bazel/bzlmod/modquery", "//src/main/java/com/google/devtools/build/lib/bazel/commands", "//src/main/java/com/google/devtools/common/options", "//src/main/protobuf:failure_details_java_proto", - "//src/test/java/com/google/devtools/build/lib/bazel/bzlmod:util", "//third_party:guava", "//third_party:junit4", "//third_party:truth", diff --git a/src/test/java/com/google/devtools/build/lib/bazel/commands/ModqueryCommandTest.java b/src/test/java/com/google/devtools/build/lib/bazel/commands/ModqueryCommandTest.java index 0bb6d882c327c5..63db6881dc071d 100644 --- a/src/test/java/com/google/devtools/build/lib/bazel/commands/ModqueryCommandTest.java +++ b/src/test/java/com/google/devtools/build/lib/bazel/commands/ModqueryCommandTest.java @@ -24,8 +24,8 @@ import com.google.devtools.build.lib.bazel.bzlmod.ModuleKey; import com.google.devtools.build.lib.bazel.bzlmod.Version; import com.google.devtools.build.lib.bazel.bzlmod.Version.ParseException; +import com.google.devtools.build.lib.bazel.bzlmod.modquery.ModqueryOptions.QueryType; import com.google.devtools.build.lib.bazel.commands.ModqueryCommand.InvalidArgumentException; -import com.google.devtools.build.lib.bazel.commands.ModqueryOptions.QueryType; import com.google.devtools.build.lib.server.FailureDetails.ModqueryCommand.Code; import com.google.devtools.common.options.OptionsParsingException; import org.junit.Test; diff --git a/src/test/java/com/google/devtools/build/lib/bazel/commands/ModqueryExecutorTest.java b/src/test/java/com/google/devtools/build/lib/bazel/commands/ModqueryExecutorTest.java deleted file mode 100644 index a0927161980420..00000000000000 --- a/src/test/java/com/google/devtools/build/lib/bazel/commands/ModqueryExecutorTest.java +++ /dev/null @@ -1,238 +0,0 @@ -// Copyright 2022 The Bazel Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package com.google.devtools.build.lib.bazel.commands; - -import static com.google.common.truth.Truth.assertThat; -import static com.google.devtools.build.lib.bazel.bzlmod.BzlmodTestUtil.AugmentedModuleBuilder.buildAugmentedModule; -import static com.google.devtools.build.lib.bazel.bzlmod.BzlmodTestUtil.createModuleKey; - -import com.google.common.collect.ImmutableMap; -import com.google.common.collect.ImmutableSet; -import com.google.devtools.build.lib.bazel.bzlmod.BazelModuleInspectorValue.AugmentedModule; -import com.google.devtools.build.lib.bazel.bzlmod.ModuleKey; -import com.google.devtools.build.lib.bazel.bzlmod.Version; -import com.google.devtools.build.lib.bazel.bzlmod.Version.ParseException; -import com.google.devtools.build.lib.bazel.commands.ModqueryExecutor.ResultNode; -import com.google.devtools.build.lib.bazel.commands.ModqueryExecutor.ResultNode.IsExpanded; -import java.io.StringWriter; -import java.io.Writer; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.junit.runners.JUnit4; - -/** Tests for {@link ModqueryExecutor}. */ -@RunWith(JUnit4.class) -public class ModqueryExecutorTest { - - private final Writer writer = new StringWriter(); - - // Tests for the ModqueryExecutor::expandAndPrune core function. - // - // (* In the ASCII graph hints "__>" or "-->" mean a direct edge, while "..>" means an indirect - // edge. "aaa ..." means module "aaa" is unexpanded.) - - @Test - public void testExpandFromTargetsFirst() throws ParseException { - // aaa -> bbb -> ccc -> ddd - ImmutableMap depGraph = - new ImmutableMap.Builder() - .put( - buildAugmentedModule(ModuleKey.ROOT, "aaa", Version.parse("1.0"), true) - .addDep("bbb", "1.0") - .buildEntry()) - .put( - buildAugmentedModule("bbb", "1.0") - .addStillDependant(ModuleKey.ROOT) - .addDep("ccc", "1.0") - .buildEntry()) - .put( - buildAugmentedModule("ccc", "1.0") - .addStillDependant("bbb", "1.0") - .addDep("ddd", "1.0") - .buildEntry()) - .put(buildAugmentedModule("ddd", "1.0").addStillDependant("ccc", "1.0").buildEntry()) - .buildOrThrow(); - - ModqueryOptions options = ModqueryOptions.getDefaultOptions(); - ModqueryExecutor executor = new ModqueryExecutor(depGraph, options, writer); - - // RESULT: - // ...> ccc -> ddd - // \___> bbb -> ccc ... - assertThat( - executor.expandAndPrune( - ImmutableSet.of(ModuleKey.ROOT, createModuleKey("ccc", "1.0")), - ImmutableSet.of(), - false)) - .containsExactly( - ModuleKey.ROOT, - ResultNode.builder() - .addChild(createModuleKey("bbb", "1.0"), IsExpanded.TRUE) - .addIndirectChild(createModuleKey("ccc", "1.0"), IsExpanded.TRUE) - .build(), - createModuleKey("bbb", "1.0"), - ResultNode.builder().addChild(createModuleKey("ccc", "1.0"), IsExpanded.FALSE).build(), - createModuleKey("ccc", "1.0"), - ResultNode.builder().addChild(createModuleKey("ddd", "1.0"), IsExpanded.TRUE).build(), - createModuleKey("ddd", "1.0"), - ResultNode.builder().build()); - } - - @Test - public void testPathsDepth1_containsAllTargetsWithNestedIndirect() throws ParseException { - // -> bbb -> ccc -> ddd -> eee -> fff -> ggg -> hhh - // ^ / - // \___/ - ImmutableMap depGraph = - new ImmutableMap.Builder() - .put( - buildAugmentedModule(ModuleKey.ROOT, "aaa", Version.parse("1.0"), true) - .addDep("bbb", "1.0") - .buildEntry()) - .put( - buildAugmentedModule("bbb", "1.0") - .addStillDependant(ModuleKey.ROOT) - .addDep("ccc", "1.0") - .buildEntry()) - .put( - buildAugmentedModule("ccc", "1.0") - .addStillDependant("bbb", "1.0") - .addDep("ddd", "1.0") - .buildEntry()) - .put( - buildAugmentedModule("ddd", "1.0") - .addStillDependant("ccc", "1.0") - .addStillDependant("eee", "1.0") - .addDep("eee", "1.0") - .buildEntry()) - .put( - buildAugmentedModule("eee", "1.0") - .addStillDependant("ddd", "1.0") - .addDep("fff", "1.0") - .addDep("ddd", "1.0") - .buildEntry()) - .put( - buildAugmentedModule("fff", "1.0") - .addStillDependant("eee", "1.0") - .addDep("ggg", "1.0") - .buildEntry()) - .put( - buildAugmentedModule("ggg", "1.0") - .addStillDependant("fff", "1.0") - .addDep("hhh", "1.0") - .buildEntry()) - .put(buildAugmentedModule("hhh", "1.0").addStillDependant("ggg", "1.0").buildEntry()) - .buildOrThrow(); - - ModqueryOptions options = ModqueryOptions.getDefaultOptions(); - options.cycles = true; - options.depth = 1; - ModqueryExecutor executor = new ModqueryExecutor(depGraph, options, writer); - ImmutableSet targets = - ImmutableSet.of(createModuleKey("eee", "1.0"), createModuleKey("hhh", "1.0")); - - // RESULT: - // --> bbb ..> ddd --> eee --> ddd (cycle) - // \..> ggg --> hhh - assertThat(executor.expandAndPrune(ImmutableSet.of(ModuleKey.ROOT), targets, false)) - .containsExactly( - ModuleKey.ROOT, - ResultNode.builder().addChild(createModuleKey("bbb", "1.0"), IsExpanded.TRUE).build(), - createModuleKey("bbb", "1.0"), - ResultNode.builder() - .addIndirectChild(createModuleKey("ddd", "1.0"), IsExpanded.TRUE) - .build(), - createModuleKey("ddd", "1.0"), - ResultNode.builder().addChild(createModuleKey("eee", "1.0"), IsExpanded.TRUE).build(), - createModuleKey("eee", "1.0"), - ResultNode.builder() - .setTarget(true) - .addIndirectChild(createModuleKey("ggg", "1.0"), IsExpanded.TRUE) - .addChild(createModuleKey("ddd", "1.0"), IsExpanded.FALSE) - .build(), - createModuleKey("ggg", "1.0"), - ResultNode.builder().addChild(createModuleKey("hhh", "1.0"), IsExpanded.TRUE).build(), - createModuleKey("hhh", "1.0"), - ResultNode.builder().setTarget(true).build()); - } - - @Test - public void testPathsDepth1_targetParentIsDirectAndIndirectChild() throws ParseException { - // --> bbb --> ccc - // \ |________ - // \ V | - // \__> ddd --> eee - ImmutableMap depGraph = - new ImmutableMap.Builder() - .put( - buildAugmentedModule(ModuleKey.ROOT, "aaa", Version.parse("1.0"), true) - .addDep("bbb", "1.0") - .buildEntry()) - .put( - buildAugmentedModule("bbb", "1.0") - .addStillDependant(ModuleKey.ROOT) - .addDep("ccc", "1.0") - .addDep("ddd", "1.0") - .buildEntry()) - .put( - buildAugmentedModule("ccc", "1.0") - .addStillDependant("bbb", "1.0") - .addDep("ddd", "1.0") - .buildEntry()) - .put( - buildAugmentedModule("ddd", "1.0") - .addStillDependant("bbb", "1.0") - .addStillDependant("ccc", "1.0") - .addStillDependant("eee", "1.0") - .addDep("eee", "1.0") - .buildEntry()) - .put( - buildAugmentedModule("eee", "1.0") - .addStillDependant("ddd", "1.0") - .addStillDependant("eee", "1.0") - .addDep("ddd", "1.0") - .buildEntry()) - .buildOrThrow(); - - ModqueryOptions options = ModqueryOptions.getDefaultOptions(); - options.cycles = true; - options.depth = 1; - ModqueryExecutor executor = new ModqueryExecutor(depGraph, options, writer); - ImmutableSet targets = ImmutableSet.of(createModuleKey("eee", "1.0")); - - // RESULT: - // --> bbb --- ddd --> eee --> ddd (c) - // \ - // \..> ddd ... - assertThat(executor.expandAndPrune(ImmutableSet.of(ModuleKey.ROOT), targets, false)) - .containsExactly( - ModuleKey.ROOT, - ResultNode.builder().addChild(createModuleKey("bbb", "1.0"), IsExpanded.TRUE).build(), - createModuleKey("bbb", "1.0"), - ResultNode.builder() - .addChild(createModuleKey("ddd", "1.0"), IsExpanded.TRUE) - .addIndirectChild(createModuleKey("ddd", "1.0"), IsExpanded.FALSE) - .build(), - createModuleKey("ddd", "1.0"), - ResultNode.builder().addChild(createModuleKey("eee", "1.0"), IsExpanded.TRUE).build(), - createModuleKey("eee", "1.0"), - ResultNode.builder() - .setTarget(true) - .addChild(createModuleKey("ddd", "1.0"), IsExpanded.FALSE) - .build()); - } - - // TODO(andreisolo): Add more eventual edge-case tests for the #expandAndPrune core method -} From e8a30972bf34c03a79dc42275e79533544bf4e07 Mon Sep 17 00:00:00 2001 From: andyrinne12 Date: Wed, 12 Jul 2023 09:52:45 -0700 Subject: [PATCH 02/13] Add module extensions to modquery MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Now includes extension usages and repositories inside query graphs along related options - `show` now supports extension-generated repos - Added new subcommand `show_extension` which displays the list of repos generated by that extension and its usages by each module - Since this CL introduces a new argument type to modquery (``), refactored modquery argument parsing logic (see `ModuleArg` and `ExtensionArg`). For a user-friendly description, see the `modquery.txt` file. - Added some basic Python integration tests for modquery (more to come). https://github.com/bazelbuild/bazel/issues/15365 Co-authored-by: Xùdōng Yáng PiperOrigin-RevId: 547524086 Change-Id: If1364f01c3be871343edcd5cee94b1180b4b930f --- .../devtools/build/lib/bazel/bzlmod/BUILD | 2 + .../bzlmod/BazelModuleInspectorFunction.java | 39 +- .../bzlmod/BazelModuleInspectorValue.java | 28 +- .../lib/bazel/bzlmod/ModuleExtensionId.java | 21 + .../bzlmod/SingleExtensionEvalValue.java | 6 +- .../build/lib/bazel/bzlmod/modquery/BUILD | 5 + .../bazel/bzlmod/modquery/ExtensionArg.java | 147 ++++++ .../modquery/GraphvizOutputFormatter.java | 76 ++- .../modquery/InvalidArgumentException.java | 40 ++ .../bzlmod/modquery/JsonOutputFormatter.java | 74 ++- .../bzlmod/modquery/ModqueryExecutor.java | 353 ++++++++----- .../bzlmod/modquery/ModqueryOptions.java | 219 ++++---- .../lib/bazel/bzlmod/modquery/ModuleArg.java | 418 ++++++++++++++++ .../bzlmod/modquery/OutputFormatters.java | 16 +- .../bzlmod/modquery/TextOutputFormatter.java | 118 ++++- .../devtools/build/lib/bazel/commands/BUILD | 3 + .../lib/bazel/commands/ModqueryCommand.java | 441 +++++++++------- .../build/lib/bazel/commands/modquery.txt | 32 +- .../com/google/devtools/build/lib/util/BUILD | 9 + .../build/lib/util/MaybeCompleteSet.java | 87 ++++ .../com/google/devtools/build/lib/bazel/BUILD | 1 - .../lib/bazel/bzlmod/BzlmodTestUtil.java | 8 +- .../build/lib/bazel/bzlmod/modquery/BUILD | 6 + .../bzlmod/modquery/ExtensionArgTest.java | 162 ++++++ .../bzlmod/modquery/ModqueryExecutorTest.java | 317 +++++++++++- .../bazel/bzlmod/modquery/ModuleArgTest.java | 320 ++++++++++++ .../devtools/build/lib/bazel/commands/BUILD | 39 -- .../bazel/commands/ModqueryCommandTest.java | 197 -------- src/test/py/bazel/BUILD | 13 + src/test/py/bazel/bzlmod/modquery_test.py | 469 ++++++++++++++++++ src/test/py/bazel/bzlmod/test_utils.py | 48 +- src/test/py/bazel/test_base.py | 62 ++- 32 files changed, 3014 insertions(+), 762 deletions(-) create mode 100644 src/main/java/com/google/devtools/build/lib/bazel/bzlmod/modquery/ExtensionArg.java create mode 100644 src/main/java/com/google/devtools/build/lib/bazel/bzlmod/modquery/InvalidArgumentException.java create mode 100644 src/main/java/com/google/devtools/build/lib/bazel/bzlmod/modquery/ModuleArg.java create mode 100644 src/main/java/com/google/devtools/build/lib/util/MaybeCompleteSet.java create mode 100644 src/test/java/com/google/devtools/build/lib/bazel/bzlmod/modquery/ExtensionArgTest.java create mode 100644 src/test/java/com/google/devtools/build/lib/bazel/bzlmod/modquery/ModuleArgTest.java delete mode 100644 src/test/java/com/google/devtools/build/lib/bazel/commands/BUILD delete mode 100644 src/test/java/com/google/devtools/build/lib/bazel/commands/ModqueryCommandTest.java create mode 100644 src/test/py/bazel/bzlmod/modquery_test.py diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BUILD b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BUILD index c6afd8832083cf..0f0d34113dbaea 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BUILD +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BUILD @@ -258,6 +258,7 @@ java_library( ], deps = [ ":common", + ":module_extension", "//src/main/java/com/google/devtools/build/lib/skyframe:sky_functions", "//src/main/java/com/google/devtools/build/lib/skyframe/serialization/autocodec:serialization-constant", "//src/main/java/com/google/devtools/build/skyframe:skyframe-objects", @@ -274,6 +275,7 @@ java_library( deps = [ ":common", ":inspection", + ":module_extension", ":resolution", "//src/main/java/com/google/devtools/build/skyframe", "//src/main/java/com/google/devtools/build/skyframe:skyframe-objects", diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelModuleInspectorFunction.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelModuleInspectorFunction.java index c2c3ffe57a5751..cd6bee7fd142f9 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelModuleInspectorFunction.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelModuleInspectorFunction.java @@ -15,12 +15,15 @@ package com.google.devtools.build.lib.bazel.bzlmod; +import static com.google.common.collect.ImmutableList.toImmutableList; import static com.google.common.collect.ImmutableMap.toImmutableMap; import static com.google.common.collect.ImmutableSet.toImmutableSet; import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; +import com.google.common.collect.ImmutableSetMultimap; import com.google.devtools.build.lib.bazel.bzlmod.BazelModuleInspectorValue.AugmentedModule; import com.google.devtools.build.lib.bazel.bzlmod.BazelModuleInspectorValue.AugmentedModule.ResolutionReason; import com.google.devtools.build.lib.bazel.bzlmod.ModuleFileValue.RootModuleFileValue; @@ -28,6 +31,7 @@ import com.google.devtools.build.skyframe.SkyFunctionException; import com.google.devtools.build.skyframe.SkyKey; import com.google.devtools.build.skyframe.SkyValue; +import com.google.devtools.build.skyframe.SkyframeLookupResult; import java.util.HashMap; import java.util.Map; import java.util.Map.Entry; @@ -49,6 +53,10 @@ public SkyValue compute(SkyKey skyKey, Environment env) if (root == null) { return null; } + BazelDepGraphValue depGraphValue = (BazelDepGraphValue) env.getValue(BazelDepGraphValue.KEY); + if (depGraphValue == null) { + return null; + } BazelModuleResolutionValue resolutionValue = (BazelModuleResolutionValue) env.getValue(BazelModuleResolutionValue.KEY); if (resolutionValue == null) { @@ -61,6 +69,12 @@ public SkyValue compute(SkyKey skyKey, Environment env) ImmutableMap depGraph = computeAugmentedGraph(unprunedDepGraph, resolvedDepGraph.keySet(), overrides); + ImmutableSetMultimap extensionToRepoInternalNames = + computeExtensionToRepoInternalNames(depGraphValue, env); + if (extensionToRepoInternalNames == null) { + return null; + } + // Group all ModuleKeys seen by their module name for easy lookup ImmutableMap> modulesIndex = ImmutableMap.copyOf( @@ -70,7 +84,7 @@ public SkyValue compute(SkyKey skyKey, Environment env) AugmentedModule::getName, Collectors.mapping(AugmentedModule::getKey, toImmutableSet())))); - return BazelModuleInspectorValue.create(depGraph, modulesIndex); + return BazelModuleInspectorValue.create(depGraph, modulesIndex, extensionToRepoInternalNames); } public static ImmutableMap computeAugmentedGraph( @@ -157,4 +171,27 @@ public static ImmutableMap computeAugmentedGraph( return depGraphAugmentBuilder.entrySet().stream() .collect(toImmutableMap(Entry::getKey, e -> e.getValue().build())); } + + @Nullable + private ImmutableSetMultimap computeExtensionToRepoInternalNames( + BazelDepGraphValue depGraphValue, Environment env) throws InterruptedException { + ImmutableSet extensionEvalKeys = + depGraphValue.getExtensionUsagesTable().rowKeySet(); + ImmutableList singleEvalKeys = + extensionEvalKeys.stream().map(SingleExtensionEvalValue::key).collect(toImmutableList()); + SkyframeLookupResult singleEvalValues = env.getValuesAndExceptions(singleEvalKeys); + + ImmutableSetMultimap.Builder extensionToRepoInternalNames = + ImmutableSetMultimap.builder(); + for (SingleExtensionEvalValue.Key singleEvalKey : singleEvalKeys) { + SingleExtensionEvalValue singleEvalValue = + (SingleExtensionEvalValue) singleEvalValues.get(singleEvalKey); + if (singleEvalValue == null) { + return null; + } + extensionToRepoInternalNames.putAll( + singleEvalKey.argument(), singleEvalValue.getGeneratedRepoSpecs().keySet()); + } + return extensionToRepoInternalNames.build(); + } } diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelModuleInspectorValue.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelModuleInspectorValue.java index ca4632d32ad8bb..ef7242dc9acd13 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelModuleInspectorValue.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelModuleInspectorValue.java @@ -19,6 +19,8 @@ import com.google.common.collect.ImmutableBiMap; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; +import com.google.common.collect.ImmutableSetMultimap; +import com.google.common.collect.ImmutableSortedMap; import com.google.devtools.build.lib.skyframe.SkyFunctions; import com.google.devtools.build.lib.skyframe.serialization.autocodec.SerializationConstant; import com.google.devtools.build.skyframe.SkyKey; @@ -39,8 +41,10 @@ public abstract class BazelModuleInspectorValue implements SkyValue { public static BazelModuleInspectorValue create( ImmutableMap depGraph, - ImmutableMap> modulesIndex) { - return new AutoValue_BazelModuleInspectorValue(depGraph, modulesIndex); + ImmutableMap> modulesIndex, + ImmutableSetMultimap extensionToRepoInternalNames) { + return new AutoValue_BazelModuleInspectorValue( + depGraph, modulesIndex, extensionToRepoInternalNames); } /** @@ -58,6 +62,13 @@ public static BazelModuleInspectorValue create( */ public abstract ImmutableMap> getModulesIndex(); + /** + * A mapping from a module extension ID, to the list of "internal" names of the repos generated by + * that extension. The "internal" name is the name directly used by the extension when + * instantiating a repo rule. + */ + public abstract ImmutableSetMultimap getExtensionToRepoInternalNames(); + /** * A wrapper for {@link Module}, augmented with references to dependants (and also those who are * not used in the final dep graph). @@ -106,15 +117,18 @@ public abstract static class AugmentedModule { */ public abstract ImmutableMap getDepReasons(); - /** Shortcut for retrieving the union of both used and unused deps based on the unused flag. */ - public ImmutableMap getAllDeps(boolean unused) { + /** + * Shortcut for retrieving the sorted union of both used and unused deps based on the unused + * flag. + */ + public ImmutableSortedMap getAllDeps(boolean unused) { if (!unused) { - return getDeps().inverse(); + return ImmutableSortedMap.copyOf(getDeps().inverse(), ModuleKey.LEXICOGRAPHIC_COMPARATOR); } else { Map map = new HashMap<>(); map.putAll(getDeps().inverse()); map.putAll(getUnusedDeps().inverse()); - return ImmutableMap.copyOf(map); + return ImmutableSortedMap.copyOf(map, ModuleKey.LEXICOGRAPHIC_COMPARATOR); } } @@ -127,7 +141,7 @@ public ImmutableMap getAllDeps(boolean unused) { /** Flag for checking whether the module is present in the resolved dep graph. */ public boolean isUsed() { - return !getDependants().isEmpty(); + return getKey().equals(ModuleKey.ROOT) || !getDependants().isEmpty(); } /** Returns a new {@link AugmentedModule.Builder} with {@code key} set. */ diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/ModuleExtensionId.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/ModuleExtensionId.java index 8d667614832f8c..7448a0ced179ff 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/ModuleExtensionId.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/ModuleExtensionId.java @@ -15,17 +15,33 @@ package com.google.devtools.build.lib.bazel.bzlmod; +import static com.google.common.collect.Comparators.emptiesFirst; +import static com.google.common.primitives.Booleans.falseFirst; +import static java.util.Comparator.comparing; + import com.google.auto.value.AutoValue; import com.google.devtools.build.lib.cmdline.Label; +import java.util.Comparator; import java.util.Optional; /** A unique identifier for a {@link ModuleExtension}. */ @AutoValue public abstract class ModuleExtensionId { + public static final Comparator LEXICOGRAPHIC_COMPARATOR = + comparing(ModuleExtensionId::getBzlFileLabel) + .thenComparing(ModuleExtensionId::getExtensionName) + .thenComparing( + ModuleExtensionId::getIsolationKey, + emptiesFirst(IsolationKey.LEXICOGRAPHIC_COMPARATOR)); /** A unique identifier for a single isolated usage of a fixed module extension. */ @AutoValue abstract static class IsolationKey { + static final Comparator LEXICOGRAPHIC_COMPARATOR = + comparing(IsolationKey::getModule, ModuleKey.LEXICOGRAPHIC_COMPARATOR) + .thenComparing(IsolationKey::isDevUsage, falseFirst()) + .thenComparing(IsolationKey::getIsolatedUsageIndex); + /** The module which contains this isolated usage of a module extension. */ public abstract ModuleKey getModule(); @@ -54,4 +70,9 @@ public static ModuleExtensionId create( Label bzlFileLabel, String extensionName, Optional isolationKey) { return new AutoValue_ModuleExtensionId(bzlFileLabel, extensionName, isolationKey); } + + public String asTargetString() { + return String.format( + "%s%%%s", getBzlFileLabel().getUnambiguousCanonicalForm(), getExtensionName()); + } } diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/SingleExtensionEvalValue.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/SingleExtensionEvalValue.java index d8f6f631420f70..e0f68a43b3243f 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/SingleExtensionEvalValue.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/SingleExtensionEvalValue.java @@ -56,8 +56,12 @@ public static Key key(ModuleExtensionId id) { return Key.create(id); } + /** + * The {@link com.google.devtools.build.skyframe.SkyKey} of a {@link + * com.google.devtools.build.lib.bazel.bzlmod.SingleExtensionEvalValue} + */ @AutoCodec - static class Key extends AbstractSkyKey { + public static class Key extends AbstractSkyKey { private static final Interner interner = BlazeInterners.newWeakInterner(); protected Key(ModuleExtensionId arg) { diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/modquery/BUILD b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/modquery/BUILD index 6c0e7657b54973..411b56e276ce0d 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/modquery/BUILD +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/modquery/BUILD @@ -17,10 +17,15 @@ java_library( deps = [ "//src/main/java/com/google/devtools/build/lib/bazel/bzlmod:common", "//src/main/java/com/google/devtools/build/lib/bazel/bzlmod:inspection", + "//src/main/java/com/google/devtools/build/lib/bazel/bzlmod:module_extension", "//src/main/java/com/google/devtools/build/lib/bazel/bzlmod:repo_rule_value", + "//src/main/java/com/google/devtools/build/lib/cmdline", "//src/main/java/com/google/devtools/build/lib/packages", "//src/main/java/com/google/devtools/build/lib/query2/query/output", + "//src/main/java/com/google/devtools/build/lib/util:maybe_complete_set", "//src/main/java/com/google/devtools/common/options", + "//src/main/java/net/starlark/java/eval", + "//src/main/protobuf:failure_details_java_proto", "//third_party:auto_value", "//third_party:gson", "//third_party:guava", diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/modquery/ExtensionArg.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/modquery/ExtensionArg.java new file mode 100644 index 00000000000000..ed79a2f29fcf10 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/modquery/ExtensionArg.java @@ -0,0 +1,147 @@ +// Copyright 2023 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.bazel.bzlmod.modquery; + +import com.google.auto.value.AutoValue; +import com.google.common.collect.ImmutableBiMap; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Iterables; +import com.google.devtools.build.lib.bazel.bzlmod.BazelModuleInspectorValue.AugmentedModule; +import com.google.devtools.build.lib.bazel.bzlmod.ModuleExtensionId; +import com.google.devtools.build.lib.bazel.bzlmod.ModuleKey; +import com.google.devtools.build.lib.bazel.bzlmod.modquery.ModuleArg.ModuleArgConverter; +import com.google.devtools.build.lib.cmdline.Label; +import com.google.devtools.build.lib.cmdline.Label.RepoContext; +import com.google.devtools.build.lib.cmdline.LabelSyntaxException; +import com.google.devtools.build.lib.cmdline.RepositoryMapping; +import com.google.devtools.build.lib.server.FailureDetails.ModqueryCommand.Code; +import com.google.devtools.common.options.Converter; +import com.google.devtools.common.options.Converters.CommaSeparatedNonEmptyOptionListConverter; +import com.google.devtools.common.options.OptionsParsingException; +import java.util.Optional; + +/** + * Represents a reference to a module extension, parsed from a command-line argument in the form of + * {@code %}. The {@code } part is parsed as a + * {@link ModuleArg}. Valid examples include {@code @rules_java//java:extensions.bzl%toolchains}, + * {@code rules_java@6.1.1//java:extensions.bzl%toolchains}, etc. + */ +@AutoValue +public abstract class ExtensionArg { + public static ExtensionArg create( + ModuleArg moduleArg, String repoRelativeBzlLabel, String extensionName) { + return new AutoValue_ExtensionArg(moduleArg, repoRelativeBzlLabel, extensionName); + } + + public abstract ModuleArg moduleArg(); + + public abstract String repoRelativeBzlLabel(); + + public abstract String extensionName(); + + /** Resolves this {@link ExtensionArg} to a {@link ModuleExtensionId}. */ + public final ModuleExtensionId resolveToExtensionId( + ImmutableMap> modulesIndex, + ImmutableMap depGraph, + ImmutableBiMap baseModuleDeps, + ImmutableBiMap baseModuleUnusedDeps) + throws InvalidArgumentException { + ImmutableSet refModules = + moduleArg() + .resolveToModuleKeys( + modulesIndex, + depGraph, + baseModuleDeps, + baseModuleUnusedDeps, + /* includeUnused= */ false, + /* warnUnused= */ false); + if (refModules.size() != 1) { + throw new InvalidArgumentException( + String.format( + "Module %s, as part of the extension specifier, should represent exactly one module" + + " version. Choose one of: %s.", + moduleArg(), refModules), + Code.INVALID_ARGUMENTS); + } + ModuleKey key = Iterables.getOnlyElement(refModules); + try { + Label label = + Label.parseWithRepoContext( + repoRelativeBzlLabel(), + RepoContext.of( + key.getCanonicalRepoName(), + // Intentionally allow no repo mapping here: it's a repo-relative label! + RepositoryMapping.create(ImmutableMap.of(), key.getCanonicalRepoName()))); + // TODO(wyv): support isolated extension usages? + return ModuleExtensionId.create(label, extensionName(), Optional.empty()); + } catch (LabelSyntaxException e) { + throw new InvalidArgumentException( + String.format("bad label format in %s: %s", repoRelativeBzlLabel(), e.getMessage()), + Code.INVALID_ARGUMENTS, + e); + } + } + + @Override + public final String toString() { + return moduleArg() + repoRelativeBzlLabel() + "%" + extensionName(); + } + + /** Converter for {@link ExtensionArg}. */ + public static class ExtensionArgConverter extends Converter.Contextless { + public static final ExtensionArgConverter INSTANCE = new ExtensionArgConverter(); + + @Override + public ExtensionArg convert(String input) throws OptionsParsingException { + int slashIdx = input.indexOf('/'); + if (slashIdx < 0) { + throw new OptionsParsingException("Invalid argument " + input + ": missing .bzl label"); + } + int percentIdx = input.indexOf('%'); + if (percentIdx < slashIdx) { + throw new OptionsParsingException("Invalid argument " + input + ": missing extension name"); + } + ModuleArg moduleArg = ModuleArgConverter.INSTANCE.convert(input.substring(0, slashIdx)); + return ExtensionArg.create( + moduleArg, input.substring(slashIdx, percentIdx), input.substring(percentIdx + 1)); + } + + @Override + public String getTypeDescription() { + return "an identifier in the format of %"; + } + } + + /** Converter for a comma-separated list of {@link ExtensionArg}s. */ + public static class CommaSeparatedExtensionArgListConverter + extends Converter.Contextless> { + + @Override + public ImmutableList convert(String input) throws OptionsParsingException { + ImmutableList args = new CommaSeparatedNonEmptyOptionListConverter().convert(input); + ImmutableList.Builder extensionArgs = new ImmutableList.Builder<>(); + for (String arg : args) { + extensionArgs.add(ExtensionArgConverter.INSTANCE.convert(arg)); + } + return extensionArgs.build(); + } + + @Override + public String getTypeDescription() { + return "a comma-separated list of s"; + } + } +} diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/modquery/GraphvizOutputFormatter.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/modquery/GraphvizOutputFormatter.java index 91395832d013aa..35c36ed1666240 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/modquery/GraphvizOutputFormatter.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/modquery/GraphvizOutputFormatter.java @@ -14,18 +14,24 @@ package com.google.devtools.build.lib.bazel.bzlmod.modquery; -import com.google.common.base.Preconditions; +import static com.google.common.collect.ImmutableSortedSet.toImmutableSortedSet; + +import com.google.common.collect.ImmutableSortedSet; +import com.google.common.collect.Sets; import com.google.devtools.build.lib.bazel.bzlmod.BazelModuleInspectorValue.AugmentedModule; +import com.google.devtools.build.lib.bazel.bzlmod.ModuleExtensionId; import com.google.devtools.build.lib.bazel.bzlmod.ModuleKey; import com.google.devtools.build.lib.bazel.bzlmod.Version; import com.google.devtools.build.lib.bazel.bzlmod.modquery.ModqueryExecutor.ResultNode; import com.google.devtools.build.lib.bazel.bzlmod.modquery.ModqueryExecutor.ResultNode.IsIndirect; import com.google.devtools.build.lib.bazel.bzlmod.modquery.ModqueryExecutor.ResultNode.NodeMetadata; +import com.google.devtools.build.lib.bazel.bzlmod.modquery.ModqueryOptions.ExtensionShow; import com.google.devtools.build.lib.bazel.bzlmod.modquery.OutputFormatters.OutputFormatter; import java.util.ArrayDeque; import java.util.Deque; import java.util.HashSet; import java.util.Map.Entry; +import java.util.Objects; import java.util.Set; /** @@ -33,31 +39,31 @@ * can be further pipelined to create an image graph visualization. */ public class GraphvizOutputFormatter extends OutputFormatter { + private StringBuilder str; @Override public void output() { - StringBuilder str = new StringBuilder(); + str = new StringBuilder(); str.append("digraph mygraph {\n") .append(" ") .append("node [ shape=box ]\n") .append(" ") .append("edge [ fontsize=8 ]\n"); Set seen = new HashSet<>(); + Set seenExtensions = new HashSet<>(); Deque toVisit = new ArrayDeque<>(); seen.add(ModuleKey.ROOT); toVisit.add(ModuleKey.ROOT); while (!toVisit.isEmpty()) { ModuleKey key = toVisit.pop(); - AugmentedModule module = depGraph.get(key); - ResultNode node = result.get(key); - Preconditions.checkNotNull(module); - Preconditions.checkNotNull(node); + AugmentedModule module = Objects.requireNonNull(depGraph.get(key)); + ResultNode node = Objects.requireNonNull(result.get(key)); String sourceId = toId(key); if (key.equals(ModuleKey.ROOT)) { - String rootLabel = String.format("root (%s@%s)", module.getName(), module.getVersion()); - str.append(String.format(" root [ label=\"%s\" ]\n", rootLabel)); + String rootLabel = String.format(" (%s@%s)", module.getName(), module.getVersion()); + str.append(String.format(" \"\" [ label=\"%s\" ]\n", rootLabel)); } else if (node.isTarget() || !module.isUsed()) { String shapeString = node.isTarget() ? "diamond" : "box"; String styleString = module.isUsed() ? "solid" : "dotted"; @@ -65,6 +71,26 @@ public void output() { String.format(" %s [ shape=%s style=%s ]\n", toId(key), shapeString, styleString)); } + if (options.extensionInfo != ExtensionShow.HIDDEN) { + ImmutableSortedSet extensionsUsed = + extensionRepoImports.keySet().stream() + .filter(e -> extensionRepoImports.get(e).inverse().containsKey(key)) + .collect(toImmutableSortedSet(ModuleExtensionId.LEXICOGRAPHIC_COMPARATOR)); + for (ModuleExtensionId extensionId : extensionsUsed) { + if (options.extensionInfo == ExtensionShow.USAGES) { + str.append(String.format(" %s -> \"%s\"\n", toId(key), toId(extensionId))); + continue; + } + if (seenExtensions.add(extensionId)) { + printExtension(extensionId); + } + ImmutableSortedSet repoImports = + ImmutableSortedSet.copyOf(extensionRepoImports.get(extensionId).inverse().get(key)); + for (String repo : repoImports) { + str.append(String.format(" %s -> %s\n", toId(key), toId(extensionId, repo))); + } + } + } for (Entry e : node.getChildrenSortedByKey()) { ModuleKey childKey = e.getKey(); IsIndirect childIndirect = e.getValue().isIndirect(); @@ -87,15 +113,45 @@ public void output() { private String toId(ModuleKey key) { if (key.equals(ModuleKey.ROOT)) { - return "root"; + return "\"\""; } return String.format( "\"%s@%s\"", key.getName(), key.getVersion().equals(Version.EMPTY) ? "_" : key.getVersion()); } + private String toId(ModuleExtensionId id) { + return id.asTargetString(); + } + + private String toId(ModuleExtensionId id, String repo) { + return String.format("\"%s%%%s\"", toId(id), repo); + } + + private void printExtension(ModuleExtensionId id) { + str.append(String.format(" subgraph \"cluster_%s\" {\n", toId(id))); + str.append(String.format(" label=\"%s\"\n", toId(id))); + if (options.extensionInfo == ExtensionShow.USAGES) { + return; + } + ImmutableSortedSet usedRepos = + ImmutableSortedSet.copyOf(extensionRepoImports.get(id).keySet()); + for (String repo : usedRepos) { + str.append(String.format(" %s [ label=\"%s\" ]\n", toId(id, repo), repo)); + } + if (options.extensionInfo == ExtensionShow.REPOS) { + return; + } + ImmutableSortedSet unusedRepos = + ImmutableSortedSet.copyOf(Sets.difference(extensionRepos.get(id), usedRepos)); + for (String repo : unusedRepos) { + str.append(String.format(" %s [ label=\"%s\" style=dotted ]\n", toId(id, repo), repo)); + } + str.append(" }\n"); + } + private String getReasonLabel(ModuleKey key, ModuleKey parent) { - if (!options.extra) { + if (!options.verbose) { return ""; } Explanation explanation = getExtraResolutionExplanation(key, parent); diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/modquery/InvalidArgumentException.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/modquery/InvalidArgumentException.java new file mode 100644 index 00000000000000..c457ffec8bc45e --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/modquery/InvalidArgumentException.java @@ -0,0 +1,40 @@ +// Copyright 2023 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +package com.google.devtools.build.lib.bazel.bzlmod.modquery; + +import com.google.devtools.build.lib.server.FailureDetails.ModqueryCommand.Code; + +/** + * Exception thrown when a user-input argument is invalid (wrong number of arguments or the + * specified modules do not exist). + */ +public class InvalidArgumentException extends Exception { + private final Code code; + + public InvalidArgumentException(String message, Code code, Exception cause) { + super(message, cause); + this.code = code; + } + + public InvalidArgumentException(String message, Code code) { + super(message); + this.code = code; + } + + public Code getCode() { + return code; + } +} diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/modquery/JsonOutputFormatter.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/modquery/JsonOutputFormatter.java index 3a7091b394a1a3..8d129368e16af3 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/modquery/JsonOutputFormatter.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/modquery/JsonOutputFormatter.java @@ -14,36 +14,81 @@ package com.google.devtools.build.lib.bazel.bzlmod.modquery; +import static com.google.common.collect.ImmutableSortedSet.toImmutableSortedSet; + +import com.google.common.collect.ImmutableSortedSet; +import com.google.common.collect.Sets; import com.google.devtools.build.lib.bazel.bzlmod.BazelModuleInspectorValue.AugmentedModule; +import com.google.devtools.build.lib.bazel.bzlmod.ModuleExtensionId; import com.google.devtools.build.lib.bazel.bzlmod.ModuleKey; import com.google.devtools.build.lib.bazel.bzlmod.modquery.ModqueryExecutor.ResultNode; import com.google.devtools.build.lib.bazel.bzlmod.modquery.ModqueryExecutor.ResultNode.IsCycle; import com.google.devtools.build.lib.bazel.bzlmod.modquery.ModqueryExecutor.ResultNode.IsExpanded; import com.google.devtools.build.lib.bazel.bzlmod.modquery.ModqueryExecutor.ResultNode.IsIndirect; import com.google.devtools.build.lib.bazel.bzlmod.modquery.ModqueryExecutor.ResultNode.NodeMetadata; +import com.google.devtools.build.lib.bazel.bzlmod.modquery.ModqueryOptions.ExtensionShow; import com.google.devtools.build.lib.bazel.bzlmod.modquery.OutputFormatters.OutputFormatter; import com.google.gson.GsonBuilder; import com.google.gson.JsonArray; import com.google.gson.JsonObject; +import java.util.HashSet; import java.util.Map.Entry; +import java.util.Set; /** Outputs graph-based results of {@link ModqueryExecutor} in JSON format. */ public class JsonOutputFormatter extends OutputFormatter { + private Set seenExtensions; + @Override public void output() { - JsonObject root = printTree(ModuleKey.ROOT, null, IsExpanded.TRUE, IsIndirect.FALSE); + seenExtensions = new HashSet<>(); + JsonObject root = printModule(ModuleKey.ROOT, null, IsExpanded.TRUE, IsIndirect.FALSE); root.addProperty("root", true); printer.println(new GsonBuilder().setPrettyPrinting().create().toJson(root)); } public String printKey(ModuleKey key) { if (key.equals(ModuleKey.ROOT)) { - return "root"; + return ""; } return key.toString(); } - JsonObject printTree(ModuleKey key, ModuleKey parent, IsExpanded expanded, IsIndirect indirect) { + // Helper to print module extensions similarly to printModule + private JsonObject printExtension( + ModuleKey key, ModuleExtensionId extensionId, boolean unexpanded) { + JsonObject json = new JsonObject(); + json.addProperty("key", extensionId.asTargetString()); + json.addProperty("unexpanded", unexpanded); + if (options.extensionInfo == ExtensionShow.USAGES) { + return json; + } + ImmutableSortedSet repoImports = + ImmutableSortedSet.copyOf(extensionRepoImports.get(extensionId).inverse().get(key)); + JsonArray usedRepos = new JsonArray(); + for (String usedRepo : repoImports) { + usedRepos.add(usedRepo); + } + json.add("used_repos", usedRepos); + + if (unexpanded || options.extensionInfo == ExtensionShow.REPOS) { + return json; + } + ImmutableSortedSet unusedRepos = + ImmutableSortedSet.copyOf( + Sets.difference( + extensionRepos.get(extensionId), extensionRepoImports.get(extensionId).keySet())); + JsonArray unusedReposJson = new JsonArray(); + for (String unusedRepo : unusedRepos) { + unusedReposJson.add(unusedRepo); + } + json.add("unused_repos", unusedReposJson); + return json; + } + + // Depth-first traversal to display modules (while explicitly detecting cycles) + JsonObject printModule( + ModuleKey key, ModuleKey parent, IsExpanded expanded, IsIndirect indirect) { ResultNode node = result.get(key); AugmentedModule module = depGraph.get(key); JsonObject json = new JsonObject(); @@ -55,7 +100,7 @@ JsonObject printTree(ModuleKey key, ModuleKey parent, IsExpanded expanded, IsInd json.addProperty("version", module.getVersion().toString()); } - if (indirect == IsIndirect.FALSE && options.extra && parent != null) { + if (indirect == IsIndirect.FALSE && options.verbose && parent != null) { Explanation explanation = getExtraResolutionExplanation(key, parent); if (explanation != null) { if (!module.isUsed()) { @@ -87,16 +132,31 @@ JsonObject printTree(ModuleKey key, ModuleKey parent, IsExpanded expanded, IsInd IsIndirect childIndirect = e.getValue().isIndirect(); IsCycle childCycles = e.getValue().isCycle(); if (childCycles == IsCycle.TRUE) { - cycles.add(printTree(childKey, key, IsExpanded.FALSE, IsIndirect.FALSE)); + cycles.add(printModule(childKey, key, IsExpanded.FALSE, IsIndirect.FALSE)); } else if (childIndirect == IsIndirect.TRUE) { - indirectDeps.add(printTree(childKey, key, childExpanded, IsIndirect.TRUE)); + indirectDeps.add(printModule(childKey, key, childExpanded, IsIndirect.TRUE)); } else { - deps.add(printTree(childKey, key, childExpanded, IsIndirect.FALSE)); + deps.add(printModule(childKey, key, childExpanded, IsIndirect.FALSE)); } } json.add("dependencies", deps); json.add("indirectDependencies", indirectDeps); json.add("cycles", cycles); + + if (options.extensionInfo == ExtensionShow.HIDDEN) { + return json; + } + ImmutableSortedSet extensionsUsed = + extensionRepoImports.keySet().stream() + .filter(e -> extensionRepoImports.get(e).inverse().containsKey(key)) + .collect(toImmutableSortedSet(ModuleExtensionId.LEXICOGRAPHIC_COMPARATOR)); + JsonArray extensionUsages = new JsonArray(); + for (ModuleExtensionId extensionId : extensionsUsed) { + boolean unexpandedExtension = !seenExtensions.add(extensionId); + extensionUsages.add(printExtension(key, extensionId, unexpandedExtension)); + } + json.add("extensionUsages", extensionUsages); + return json; } } diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/modquery/ModqueryExecutor.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/modquery/ModqueryExecutor.java index c68c6b86594844..b9cd6ffcef94bf 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/modquery/ModqueryExecutor.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/modquery/ModqueryExecutor.java @@ -14,20 +14,27 @@ package com.google.devtools.build.lib.bazel.bzlmod.modquery; +import static com.google.common.collect.ImmutableSet.toImmutableSet; import static com.google.common.collect.ImmutableSortedMap.toImmutableSortedMap; +import static com.google.common.collect.ImmutableSortedSet.toImmutableSortedSet; import static java.util.Comparator.reverseOrder; -import static java.util.stream.Collectors.toCollection; +import static java.util.stream.Collectors.joining; import com.google.auto.value.AutoValue; import com.google.common.annotations.VisibleForTesting; -import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; import com.google.common.collect.ImmutableSetMultimap; import com.google.common.collect.ImmutableSortedSet; +import com.google.common.collect.ImmutableTable; +import com.google.common.collect.Sets; import com.google.devtools.build.lib.bazel.bzlmod.BazelModuleInspectorValue.AugmentedModule; import com.google.devtools.build.lib.bazel.bzlmod.BzlmodRepoRuleValue; +import com.google.devtools.build.lib.bazel.bzlmod.ModuleExtensionId; +import com.google.devtools.build.lib.bazel.bzlmod.ModuleExtensionUsage; import com.google.devtools.build.lib.bazel.bzlmod.ModuleKey; +import com.google.devtools.build.lib.bazel.bzlmod.Tag; +import com.google.devtools.build.lib.bazel.bzlmod.Version; import com.google.devtools.build.lib.bazel.bzlmod.modquery.ModqueryExecutor.ResultNode.IsExpanded; import com.google.devtools.build.lib.bazel.bzlmod.modquery.ModqueryExecutor.ResultNode.IsIndirect; import com.google.devtools.build.lib.bazel.bzlmod.modquery.ModqueryExecutor.ResultNode.NodeMetadata; @@ -36,20 +43,23 @@ import com.google.devtools.build.lib.query2.query.output.BuildOutputFormatter.AttributeReader; import com.google.devtools.build.lib.query2.query.output.BuildOutputFormatter.TargetOutputter; import com.google.devtools.build.lib.query2.query.output.PossibleAttributeValues; +import com.google.devtools.build.lib.util.MaybeCompleteSet; import com.google.errorprone.annotations.CanIgnoreReturnValue; import java.io.IOException; import java.io.PrintWriter; import java.io.Writer; import java.util.ArrayDeque; +import java.util.Collections; import java.util.Comparator; import java.util.Deque; import java.util.HashMap; import java.util.HashSet; import java.util.Map; import java.util.Map.Entry; -import java.util.Objects; +import java.util.Optional; import java.util.Set; -import javax.annotation.Nullable; +import java.util.function.Predicate; +import net.starlark.java.eval.Starlark; /** * Executes inspection queries for {@link @@ -59,40 +69,83 @@ public class ModqueryExecutor { private final ImmutableMap depGraph; + private final ImmutableTable extensionUsages; + private final ImmutableSetMultimap extensionRepos; + private final Optional> extensionFilter; private final ModqueryOptions options; private final PrintWriter printer; + private ImmutableMap> + extensionRepoImports; public ModqueryExecutor( ImmutableMap depGraph, ModqueryOptions options, Writer writer) { + this( + depGraph, + ImmutableTable.of(), + ImmutableSetMultimap.of(), + Optional.of(MaybeCompleteSet.completeSet()), + options, + writer); + } + + public ModqueryExecutor( + ImmutableMap depGraph, + ImmutableTable extensionUsages, + ImmutableSetMultimap extensionRepos, + Optional> extensionFilter, + ModqueryOptions options, + Writer writer) { this.depGraph = depGraph; + this.extensionUsages = extensionUsages; + this.extensionRepos = extensionRepos; + this.extensionFilter = extensionFilter; this.options = options; this.printer = new PrintWriter(writer); + // Easier lookup table for repo imports by module. + // It is updated after pruneByDepthAndLink to filter out pruned modules. + this.extensionRepoImports = computeRepoImportsTable(depGraph.keySet()); } - public void tree(ImmutableSet targets) { - ImmutableMap result = expandAndPrune(targets, ImmutableSet.of(), false); - OutputFormatters.getFormatter(options.outputFormat).output(result, depGraph, printer, options); + public void tree(ImmutableSet from) { + ImmutableMap result = + expandAndPrune(from, computeExtensionFilterTargets(), false); + OutputFormatters.getFormatter(options.outputFormat) + .output(result, depGraph, extensionRepos, extensionRepoImports, printer, options); } public void path(ImmutableSet from, ImmutableSet to) { - ImmutableMap result = expandAndPrune(from, to, true); - OutputFormatters.getFormatter(options.outputFormat).output(result, depGraph, printer, options); + MaybeCompleteSet targets = + MaybeCompleteSet.unionElements(computeExtensionFilterTargets(), to); + ImmutableMap result = expandAndPrune(from, targets, true); + OutputFormatters.getFormatter(options.outputFormat) + .output(result, depGraph, extensionRepos, extensionRepoImports, printer, options); } public void allPaths(ImmutableSet from, ImmutableSet to) { - ImmutableMap result = expandAndPrune(from, to, false); - OutputFormatters.getFormatter(options.outputFormat).output(result, depGraph, printer, options); + MaybeCompleteSet targets = + MaybeCompleteSet.unionElements(computeExtensionFilterTargets(), to); + ImmutableMap result = expandAndPrune(from, targets, false); + OutputFormatters.getFormatter(options.outputFormat) + .output(result, depGraph, extensionRepos, extensionRepoImports, printer, options); } - public void show(ImmutableMap repoRuleValues) { + public void show(ImmutableMap targetRepoRuleValues) { RuleDisplayOutputter outputter = new RuleDisplayOutputter(printer); - for (Entry e : repoRuleValues.entrySet()) { - printer.printf("## %s:", e.getKey()); + for (Entry e : targetRepoRuleValues.entrySet()) { + printer.printf("## %s:\n", e.getKey()); outputter.outputRule(e.getValue().getRule()); } printer.flush(); } + public void showExtension( + ImmutableSet extensions, ImmutableSet fromUsages) { + for (ModuleExtensionId extension : extensions) { + displayExtension(extension, fromUsages); + } + printer.flush(); + } + /** * The core function which produces the {@link ResultNode} graph for all the graph-generating * queries above. First, it expands the result graph starting from the {@code from} modules, up @@ -106,23 +159,24 @@ public void show(ImmutableMap repoRuleValues) { */ @VisibleForTesting ImmutableMap expandAndPrune( - ImmutableSet from, ImmutableSet to, boolean singlePath) { - final MaybeCompleteSet coloredPaths = colorReversePathsToRoot(to); + ImmutableSet from, MaybeCompleteSet targets, boolean singlePath) { + final MaybeCompleteSet coloredPaths = colorReversePathsToRoot(targets); ImmutableMap.Builder resultBuilder = new ImmutableMap.Builder<>(); + ResultNode.Builder rootBuilder = ResultNode.builder(); ImmutableSet rootDirectChildren = depGraph.get(ModuleKey.ROOT).getAllDeps(options.includeUnused).keySet(); ImmutableSet rootPinnedChildren = - getPinnedChildrenOfRootInTheResultGraph(rootDirectChildren, from); - ResultNode.Builder rootBuilder = ResultNode.builder(); - rootPinnedChildren.stream() - .filter(coloredPaths::contains) - .forEach( - moduleKey -> - rootBuilder.addChild( - moduleKey, - IsExpanded.TRUE, - rootDirectChildren.contains(moduleKey) ? IsIndirect.FALSE : IsIndirect.TRUE)); + getPinnedChildrenOfRootInTheResultGraph(rootDirectChildren, from).stream() + .filter(coloredPaths::contains) + .filter(this::filterBuiltin) + .collect(toImmutableSortedSet(ModuleKey.LEXICOGRAPHIC_COMPARATOR)); + rootPinnedChildren.forEach( + moduleKey -> + rootBuilder.addChild( + moduleKey, + IsExpanded.TRUE, + rootDirectChildren.contains(moduleKey) ? IsIndirect.FALSE : IsIndirect.TRUE)); resultBuilder.put(ModuleKey.ROOT, rootBuilder.build()); Set seen = new HashSet<>(rootPinnedChildren); @@ -133,15 +187,15 @@ ImmutableMap expandAndPrune( ModuleKey key = toVisit.pop(); AugmentedModule module = depGraph.get(key); ResultNode.Builder nodeBuilder = ResultNode.builder(); - nodeBuilder.setTarget(to.contains(key)); + nodeBuilder.setTarget(!targets.isComplete() && targets.contains(key)); - ImmutableSet moduleDeps = module.getAllDeps(options.includeUnused).keySet(); + ImmutableSortedSet moduleDeps = module.getAllDeps(options.includeUnused).keySet(); for (ModuleKey childKey : moduleDeps) { if (!coloredPaths.contains(childKey)) { continue; } - if (to.contains(childKey)) { - nodeBuilder.setTargetParent(true); + if (isBuiltin(childKey) && !options.includeBuiltin) { + continue; } if (seen.contains(childKey)) { // Single paths should not contain cycles or unexpanded (duplicate) children @@ -163,7 +217,7 @@ ImmutableMap expandAndPrune( resultBuilder.put(key, nodeBuilder.build()); } - return new ResultGraphPruner(!to.isEmpty(), resultBuilder.buildOrThrow()).pruneByDepth(); + return new ResultGraphPruner(targets, resultBuilder.buildOrThrow()).pruneByDepth(); } private class ResultGraphPruner { @@ -171,25 +225,28 @@ private class ResultGraphPruner { private final Map oldResult; private final Map resultBuilder; private final Set parentStack; - private final boolean withTargets; + private final MaybeCompleteSet targets; /** - * Prunes the result tree after the specified depth using DFS (because some nodes may still - * appear after the max depth).
+ * Constructs a ResultGraphPruner to prune the result graph after the specified depth. * - * @param withTargets If set, it means that the result tree contains paths to some specific + * @param targets If not complete, it means that the result tree contains paths to some specific * targets. This will cause some branches to contain, after the specified depths, some * targets or target parents. As any other nodes omitted, transitive edges (embedding * multiple edges) will be stored as indirect. * @param oldResult The unpruned result graph. */ - ResultGraphPruner(boolean withTargets, Map oldResult) { + ResultGraphPruner(MaybeCompleteSet targets, Map oldResult) { this.oldResult = oldResult; this.resultBuilder = new HashMap<>(); this.parentStack = new HashSet<>(); - this.withTargets = withTargets; + this.targets = targets; } + /** + * Prunes the result tree after the specified depth using DFS (because some nodes may still + * appear after the max depth). + */ private ImmutableMap pruneByDepth() { ResultNode.Builder rootBuilder = ResultNode.builder(); resultBuilder.put(ModuleKey.ROOT, rootBuilder); @@ -204,10 +261,16 @@ private ImmutableMap pruneByDepth() { // Build everything at the end to allow children to add themselves to their parent's // adjacency list. - return resultBuilder.entrySet().stream() - .collect( - toImmutableSortedMap( - ModuleKey.LEXICOGRAPHIC_COMPARATOR, Entry::getKey, e -> e.getValue().build())); + ImmutableMap result = + resultBuilder.entrySet().stream() + .collect( + toImmutableSortedMap( + ModuleKey.LEXICOGRAPHIC_COMPARATOR, + Entry::getKey, + e -> e.getValue().build())); + // Filter imports for nodes that were pruned during this process. + extensionRepoImports = computeRepoImportsTable(result.keySet()); + return result; } // Handles graph traversal within the specified depth. @@ -215,9 +278,9 @@ private void visitVisible( ModuleKey moduleKey, int depth, ModuleKey parentKey, IsExpanded expanded) { parentStack.add(moduleKey); ResultNode oldNode = oldResult.get(moduleKey); - ResultNode.Builder nodeBuilder = ResultNode.builder(); + ResultNode.Builder nodeBuilder = + resultBuilder.computeIfAbsent(moduleKey, k -> ResultNode.builder()); - resultBuilder.put(moduleKey, nodeBuilder); nodeBuilder.setTarget(oldNode.isTarget()); if (depth > 1) { resultBuilder.get(parentKey).addChild(moduleKey, expanded, IsIndirect.FALSE); @@ -233,7 +296,7 @@ private void visitVisible( if (notCycle(childKey)) { if (depth < options.depth) { visitVisible(childKey, depth + 1, moduleKey, childExpanded); - } else if (withTargets) { + } else if (!targets.isComplete()) { visitDetached(childKey, moduleKey, moduleKey, childExpanded); } } else if (options.cycles) { @@ -255,7 +318,7 @@ private void visitDetached( ResultNode.Builder nodeBuilder = ResultNode.builder(); nodeBuilder.setTarget(oldNode.isTarget()); - if (oldNode.isTarget() || oldNode.isTargetParent()) { + if (oldNode.isTarget() || isTargetParent(oldNode)) { ResultNode.Builder parentBuilder = resultBuilder.get(lastVisibleParentKey); IsIndirect childIndirect = lastVisibleParentKey.equals(parentKey) ? IsIndirect.FALSE : IsIndirect.TRUE; @@ -283,35 +346,64 @@ private void visitDetached( private boolean notCycle(ModuleKey key) { return !parentStack.contains(key); } + + private boolean isTargetParent(ResultNode node) { + return node.getChildren().keys().stream() + .filter(Predicate.not(parentStack::contains)) + .anyMatch(targets::contains); + } } /** - * Return a list of modules that will be the direct children of the root in the result graph - * (original root's direct dependencies along with the specified targets). + * Return a sorted list of modules that will be the direct children of the root in the result + * graph (original root's direct dependencies along with the specified targets). */ - private ImmutableSet getPinnedChildrenOfRootInTheResultGraph( + private ImmutableSortedSet getPinnedChildrenOfRootInTheResultGraph( ImmutableSet rootDirectDeps, ImmutableSet fromTargets) { - Set targetKeys = - fromTargets.stream() - .filter(k -> filterUnused(k, options.includeUnused, true, depGraph)) - .collect(toCollection(HashSet::new)); + Set targetKeys = new HashSet<>(fromTargets); if (fromTargets.contains(ModuleKey.ROOT)) { + targetKeys.remove(ModuleKey.ROOT); targetKeys.addAll(rootDirectDeps); } - return ImmutableSet.copyOf(targetKeys); + return ImmutableSortedSet.copyOf(ModuleKey.LEXICOGRAPHIC_COMPARATOR, targetKeys); + } + + private static boolean intersect( + MaybeCompleteSet a, Set b) { + if (a.isComplete()) { + return !b.isEmpty(); + } + return !Collections.disjoint(a.getElementsIfNotComplete(), b); + } + + /** + * If the extensionFilter option is set, computes the set of target modules that use the specified + * extension(s) and adds them to the list of specified targets if the query is a path(s) query. + */ + private MaybeCompleteSet computeExtensionFilterTargets() { + if (extensionFilter.isEmpty()) { + // If no --extension_filter is set, don't do anything here. + return MaybeCompleteSet.completeSet(); + } + return MaybeCompleteSet.copyOf( + depGraph.keySet().stream() + .filter(this::filterUnused) + .filter(this::filterBuiltin) + .filter(k -> intersect(extensionFilter.get(), extensionUsages.column(k).keySet())) + .collect(toImmutableSet())); } /** * Color all reverse paths from the target modules to the root so only modules which are part of * these paths will be included in the output graph during the breadth-first traversal. */ - private MaybeCompleteSet colorReversePathsToRoot(ImmutableSet targets) { - if (targets.isEmpty()) { + private MaybeCompleteSet colorReversePathsToRoot(MaybeCompleteSet to) { + if (to.isComplete()) { return MaybeCompleteSet.completeSet(); } - Set seen = new HashSet<>(targets); - Deque toVisit = new ArrayDeque<>(targets); + Set seen = new HashSet<>(to.getElementsIfNotComplete()); + Deque toVisit = new ArrayDeque<>(to.getElementsIfNotComplete()); while (!toVisit.isEmpty()) { ModuleKey key = toVisit.pop(); @@ -321,6 +413,9 @@ private MaybeCompleteSet colorReversePathsToRoot(ImmutableSet colorReversePathsToRoot(ImmutableSet dependenciesGraph) { - AugmentedModule module = Objects.requireNonNull(dependenciesGraph.get(key)); - if (key.equals(ModuleKey.ROOT)) { - return false; + /** Compute the multimap of repo imports to modules for each extension. */ + private ImmutableMap> + computeRepoImportsTable(ImmutableSet presentModules) { + ImmutableMap.Builder> resultBuilder = + new ImmutableMap.Builder<>(); + for (ModuleExtensionId extension : extensionUsages.rowKeySet()) { + if (extensionFilter.isPresent() && !extensionFilter.get().contains(extension)) { + continue; + } + ImmutableSetMultimap.Builder modulesToImportsBuilder = + new ImmutableSetMultimap.Builder<>(); + for (Entry usage : + extensionUsages.rowMap().get(extension).entrySet()) { + if (!presentModules.contains(usage.getKey())) { + continue; + } + modulesToImportsBuilder.putAll(usage.getKey(), usage.getValue().getImports().values()); + } + resultBuilder.put(extension, modulesToImportsBuilder.build().inverse()); + } + return resultBuilder.buildOrThrow(); + } + + private boolean filterUnused(ModuleKey key) { + AugmentedModule module = depGraph.get(key); + return options.includeUnused || module.isUsed(); + } + + private boolean filterBuiltin(ModuleKey key) { + return options.includeBuiltin || !isBuiltin(key); + } + + /** Helper to display show_extension info. */ + private void displayExtension(ModuleExtensionId extension, ImmutableSet fromUsages) { + printer.printf("## %s:\n", extension.asTargetString()); + printer.println(); + printer.println("Fetched repositories:"); + // TODO(wyv): if `extension` doesn't exist, we crash. We should report a good error instead! + ImmutableSortedSet usedRepos = + ImmutableSortedSet.copyOf(extensionRepoImports.get(extension).keySet()); + ImmutableSortedSet unusedRepos = + ImmutableSortedSet.copyOf(Sets.difference(extensionRepos.get(extension), usedRepos)); + for (String repo : usedRepos) { + printer.printf( + " - %s (imported by %s)\n", + repo, + extensionRepoImports.get(extension).get(repo).stream() + .sorted(ModuleKey.LEXICOGRAPHIC_COMPARATOR) + .map(ModuleKey::toString) + .collect(joining(", "))); + } + for (String repo : unusedRepos) { + printer.printf(" - %s\n", repo); } - if (!module.isUsed() && !includeUnused) { - return false; + printer.println(); + if (fromUsages.isEmpty()) { + fromUsages = ImmutableSet.copyOf(extensionUsages.rowMap().get(extension).keySet()); } - if (!module.isLoaded()) { - return allowNotLoaded; + for (ModuleKey module : fromUsages) { + if (!extensionUsages.contains(extension, module)) { + continue; + } + ModuleExtensionUsage usage = extensionUsages.get(extension, module); + printer.printf( + "## Usage in %s from %s:%s\n", + module, usage.getLocation().file(), usage.getLocation().line()); + for (Tag tag : usage.getTags()) { + printer.printf( + "%s.%s(%s)\n", + extension.getExtensionName(), + tag.getTagName(), + tag.getAttributeValues().attributes().entrySet().stream() + .map(e -> String.format("%s=%s", e.getKey(), Starlark.repr(e.getValue()))) + .collect(joining(", "))); + } + printer.printf("use_repo(\n"); + printer.printf(" %s,\n", extension.getExtensionName()); + for (Entry repo : usage.getImports().entrySet()) { + printer.printf( + " %s,\n", + repo.getKey().equals(repo.getValue()) + ? String.format("\"%s\"", repo.getKey()) + : String.format("%s=\"%s\"", repo.getKey(), repo.getValue())); + } + printer.printf(")\n\n"); } - return true; + } + + private boolean isBuiltin(ModuleKey key) { + return key.equals(ModuleKey.create("bazel_tools", Version.EMPTY)) + || key.equals(ModuleKey.create("local_config_platform", Version.EMPTY)); } /** A node representing a module that forms the result graph. */ @@ -361,11 +527,6 @@ public abstract static class ResultNode { /** Whether the module is one of the targets in a paths query. */ abstract boolean isTarget(); - /** - * Whether the module is a parent of one of the targets in a paths query (can also be a target). - */ - abstract boolean isTargetParent(); - enum IsExpanded { FALSE, TRUE @@ -422,16 +583,12 @@ public ImmutableSortedSet> getChildrenSortedByEdg } static ResultNode.Builder builder() { - return new AutoValue_ModqueryExecutor_ResultNode.Builder() - .setTarget(false) - .setTargetParent(false); + return new AutoValue_ModqueryExecutor_ResultNode.Builder().setTarget(false); } @AutoValue.Builder abstract static class Builder { - abstract ResultNode.Builder setTargetParent(boolean value); - abstract ResultNode.Builder setTarget(boolean value); abstract ImmutableSetMultimap.Builder childrenBuilder(); @@ -453,34 +610,6 @@ final Builder addCycle(ModuleKey value) { } } - /** - * A set that either contains some elements or is the complete set (has a special - * constructor).
- * A complete set is stored internally as {@code null}. However, passing null to {@link - * #copyOf(Set)} is not allowed. Use {@link #completeSet()} instead. - */ - @AutoValue - abstract static class MaybeCompleteSet { - @Nullable - protected abstract ImmutableSet internalSet(); - - boolean contains(T value) { - if (internalSet() == null) { - return true; - } - return internalSet().contains(value); - } - - static MaybeCompleteSet copyOf(Set nullableSet) { - Preconditions.checkArgument(nullableSet != null); - return new AutoValue_ModqueryExecutor_MaybeCompleteSet(ImmutableSet.copyOf(nullableSet)); - } - - static MaybeCompleteSet completeSet() { - return new AutoValue_ModqueryExecutor_MaybeCompleteSet<>(null); - } - } - /** * Uses Query's {@link TargetOutputter} to display the generating repo rule and other information. */ diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/modquery/ModqueryOptions.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/modquery/ModqueryOptions.java index 08e02f9ab12576..6c5074a9a7c730 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/modquery/ModqueryOptions.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/modquery/ModqueryOptions.java @@ -16,47 +16,41 @@ import static java.util.Arrays.stream; import static java.util.stream.Collectors.joining; -import com.google.auto.value.AutoValue; import com.google.common.base.Ascii; -import com.google.common.base.Preconditions; -import com.google.common.base.Splitter; import com.google.common.collect.ImmutableList; -import com.google.devtools.build.lib.bazel.bzlmod.Version; -import com.google.devtools.build.lib.bazel.bzlmod.Version.ParseException; -import com.google.devtools.common.options.Converter; -import com.google.devtools.common.options.Converters.CommaSeparatedNonEmptyOptionListConverter; +import com.google.devtools.build.lib.bazel.bzlmod.ModuleKey; +import com.google.devtools.build.lib.bazel.bzlmod.modquery.ExtensionArg.CommaSeparatedExtensionArgListConverter; +import com.google.devtools.build.lib.bazel.bzlmod.modquery.ModuleArg.CommaSeparatedModuleArgListConverter; +import com.google.devtools.build.lib.bazel.bzlmod.modquery.ModuleArg.ModuleArgConverter; import com.google.devtools.common.options.EnumConverter; import com.google.devtools.common.options.Option; import com.google.devtools.common.options.OptionDocumentationCategory; import com.google.devtools.common.options.OptionEffectTag; import com.google.devtools.common.options.OptionsBase; -import com.google.devtools.common.options.OptionsParsingException; -import java.util.List; -import javax.annotation.Nullable; /** Options for ModqueryCommand */ public class ModqueryOptions extends OptionsBase { @Option( name = "from", - defaultValue = "root", - converter = TargetModuleListConverter.class, + defaultValue = "", + converter = CommaSeparatedModuleArgListConverter.class, documentationCategory = OptionDocumentationCategory.MODQUERY, effectTags = {OptionEffectTag.EXECUTION}, help = "The module(s) starting from which the dependency graph query will be displayed. Check" - + " each query’s description for the exact semantic. Defaults to root.\n") - public ImmutableList modulesFrom; + + " each query’s description for the exact semantics. Defaults to .\n") + public ImmutableList modulesFrom; @Option( - name = "extra", + name = "verbose", defaultValue = "false", documentationCategory = OptionDocumentationCategory.MODQUERY, effectTags = {OptionEffectTag.EXECUTION}, help = "The queries will also display the reason why modules were resolved to their current" + " version (if changed). Defaults to true only for the explain query.") - public boolean extra; + public boolean verbose; @Option( name = "include_unused", @@ -68,9 +62,54 @@ public class ModqueryOptions extends OptionsBase { + " present in the module resolution graph after selection (due to the" + " Minimal-Version Selection or override rules). This can have different effects for" + " each of the query types i.e. include new paths in the all_paths command, or extra" - + " dependants in the explain command.\n") + + " dependants in the explain command.") public boolean includeUnused; + @Option( + name = "extension_filter", + defaultValue = "null", + converter = CommaSeparatedExtensionArgListConverter.class, + documentationCategory = OptionDocumentationCategory.MODQUERY, + effectTags = {OptionEffectTag.EXECUTION}, + help = + "Only display the usages of these module extensions and the repos generated by them if" + + " their respective flags are set. If set, the result graph will only include paths" + + " that contain modules using the specified extensions. An empty list disables the" + + " filter, effectively specifying all possible extensions.") + public ImmutableList extensionFilter; + + @Option( + name = "extension_info", + defaultValue = "hidden", + converter = ExtensionShowConverter.class, + documentationCategory = OptionDocumentationCategory.MODQUERY, + effectTags = {OptionEffectTag.EXECUTION}, + help = + "Specify how much detail about extension usages to include in the query result." + + " \"Usages\" will only show the extensions names, \"repos\" will also include repos" + + " imported with use_repo, and \"all\" will also show the other repositories" + + " generated by extensions.\n") + public ExtensionShow extensionInfo; + + @Option( + name = "base_module", + defaultValue = "", + converter = ModuleArgConverter.class, + documentationCategory = OptionDocumentationCategory.MODQUERY, + effectTags = {OptionEffectTag.EXECUTION}, + help = "Specify a module relative to which the specified target repos will be interpreted.") + public ModuleArg baseModule; + + @Option( + name = "extension_usages", + defaultValue = "", + converter = CommaSeparatedModuleArgListConverter.class, + documentationCategory = OptionDocumentationCategory.MODQUERY, + effectTags = {OptionEffectTag.EXECUTION}, + help = + "Specify modules whose extension usages will be displayed in the show_extension query.") + public ImmutableList extensionUsages; + @Option( name = "depth", defaultValue = "-1", @@ -93,6 +132,16 @@ public class ModqueryOptions extends OptionsBase { + " default.") public boolean cycles; + @Option( + name = "include_builtin", + defaultValue = "false", + documentationCategory = OptionDocumentationCategory.MODQUERY, + effectTags = {OptionEffectTag.EXECUTION}, + help = + "Include built-in modules in the dependency graph. Disabled by default because it is" + + " quite noisy.") + public boolean includeBuiltin; + @Option( name = "charset", defaultValue = "utf8", @@ -120,18 +169,19 @@ public class ModqueryOptions extends OptionsBase { * com.google.devtools.build.lib.bazel.commands.ModqueryCommand} */ public enum QueryType { - DEPS(1), - TREE(0), - ALL_PATHS(1), - PATH(1), - EXPLAIN(1), - SHOW(1); - - /* Store the number of arguments that it accepts for easy pre-check */ - private final int argNumber; - - QueryType(int argNumber) { - this.argNumber = argNumber; + DEPS(true), + TREE(true), + ALL_PATHS(true), + PATH(true), + EXPLAIN(true), + SHOW(false), + SHOW_EXTENSION(false); + + /** Whether this query type produces a graph output. */ + private final boolean isGraph; + + QueryType(boolean isGraph) { + this.isGraph = isGraph; } @Override @@ -139,8 +189,8 @@ public String toString() { return Ascii.toLowerCase(this.name()); } - public int getArgNumber() { - return argNumber; + public boolean isGraph() { + return isGraph; } public static String printValues() { @@ -155,6 +205,20 @@ public QueryTypeConverter() { } } + enum ExtensionShow { + HIDDEN, + USAGES, + REPOS, + ALL + } + + /** Converts a query type option string to a properly typed {@link ExtensionShow} */ + public static class ExtensionShowConverter extends EnumConverter { + public ExtensionShowConverter() { + super(ExtensionShow.class, "query type"); + } + } + /** * Charset to be used in outputting the {@link * com.google.devtools.build.lib.bazel.commands.ModqueryCommand} result. @@ -188,100 +252,19 @@ public OutputFormatConverter() { } } - /** Argument of a modquery converted from the form name@version or name. */ - @AutoValue - public abstract static class TargetModule { - static TargetModule create(String name, Version version) { - return new AutoValue_ModqueryOptions_TargetModule(name, version); - } - - public abstract String getName(); - - /** - * If it is null, it represents any (one or multiple) present versions of the module in the dep - * graph, which is different from the empty version - */ - @Nullable - public abstract Version getVersion(); - } - - /** Converts a module target argument string to a properly typed {@link TargetModule} */ - static class TargetModuleConverter extends Converter.Contextless { - - @Override - public TargetModule convert(String input) throws OptionsParsingException { - String errorMessage = String.format("Cannot parse the given module argument: %s.", input); - Preconditions.checkArgument(input != null); - // The keyword root takes priority if any module is named the same it can only be referenced - // using the full key. - if (Ascii.equalsIgnoreCase(input, "root")) { - return TargetModule.create("", Version.EMPTY); - } else { - List splits = Splitter.on('@').splitToList(input); - if (splits.isEmpty() || splits.get(0).isEmpty()) { - throw new OptionsParsingException(errorMessage); - } - - if (splits.size() == 2) { - if (splits.get(1).equals("_")) { - return TargetModule.create(splits.get(0), Version.EMPTY); - } - if (splits.get(1).isEmpty()) { - throw new OptionsParsingException(errorMessage); - } - try { - return TargetModule.create(splits.get(0), Version.parse(splits.get(1))); - } catch (ParseException e) { - throw new OptionsParsingException(errorMessage, e); - } - - } else if (splits.size() == 1) { - return TargetModule.create(splits.get(0), null); - } else { - throw new OptionsParsingException(errorMessage); - } - } - } - - @Override - public String getTypeDescription() { - return "root, @ or "; - } - } - - /** Converts a comma-separated module list argument (i.e. A@1.0,B@2.0) */ - public static class TargetModuleListConverter - extends Converter.Contextless> { - - @Override - public ImmutableList convert(String input) throws OptionsParsingException { - CommaSeparatedNonEmptyOptionListConverter listConverter = - new CommaSeparatedNonEmptyOptionListConverter(); - TargetModuleConverter targetModuleConverter = new TargetModuleConverter(); - ImmutableList targetStrings = - listConverter.convert(input, /*conversionContext=*/ null); - ImmutableList.Builder targetModules = new ImmutableList.Builder<>(); - for (String targetInput : targetStrings) { - targetModules.add(targetModuleConverter.convert(targetInput, /*conversionContext=*/ null)); - } - return targetModules.build(); - } - - @Override - public String getTypeDescription() { - return "a list of s separated by comma"; - } - } - + /** Returns a {@link ModqueryOptions} filled with default values for testing. */ static ModqueryOptions getDefaultOptions() { ModqueryOptions options = new ModqueryOptions(); options.depth = Integer.MAX_VALUE; options.cycles = false; options.includeUnused = false; - options.extra = false; - options.modulesFrom = ImmutableList.of(TargetModule.create("", Version.EMPTY)); + options.verbose = false; + options.modulesFrom = + ImmutableList.of(ModuleArg.SpecificVersionOfModule.create(ModuleKey.ROOT)); options.charset = Charset.UTF8; options.outputFormat = OutputFormat.TEXT; + options.extensionFilter = null; + options.extensionInfo = ExtensionShow.HIDDEN; return options; } } diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/modquery/ModuleArg.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/modquery/ModuleArg.java new file mode 100644 index 00000000000000..3b68b1569d8bba --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/modquery/ModuleArg.java @@ -0,0 +1,418 @@ +// Copyright 2023 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.bazel.bzlmod.modquery; + +import static com.google.common.collect.ImmutableMap.toImmutableMap; +import static com.google.common.collect.ImmutableSet.toImmutableSet; + +import com.google.auto.value.AutoValue; +import com.google.common.base.Ascii; +import com.google.common.collect.ImmutableBiMap; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; +import com.google.devtools.build.lib.bazel.bzlmod.BazelModuleInspectorValue.AugmentedModule; +import com.google.devtools.build.lib.bazel.bzlmod.ModuleKey; +import com.google.devtools.build.lib.bazel.bzlmod.Version; +import com.google.devtools.build.lib.bazel.bzlmod.Version.ParseException; +import com.google.devtools.build.lib.cmdline.LabelSyntaxException; +import com.google.devtools.build.lib.cmdline.RepositoryMapping; +import com.google.devtools.build.lib.cmdline.RepositoryName; +import com.google.devtools.build.lib.server.FailureDetails.ModqueryCommand.Code; +import com.google.devtools.common.options.Converter; +import com.google.devtools.common.options.Converters.CommaSeparatedNonEmptyOptionListConverter; +import com.google.devtools.common.options.OptionsParsingException; +import java.util.Optional; +import net.starlark.java.eval.EvalException; + +/** + * Represents a reference to one or more modules in the external dependency graph, used for + * modquery. This is parsed from a command-line argument (either as the value of a flag, or just as + * a bare argument), and can take one of various forms (see implementations). + */ +public interface ModuleArg { + + /** Resolves this module argument to a set of module keys. */ + ImmutableSet resolveToModuleKeys( + ImmutableMap> modulesIndex, + ImmutableMap depGraph, + ImmutableBiMap baseModuleDeps, + ImmutableBiMap baseModuleUnusedDeps, + boolean includeUnused, + boolean warnUnused) + throws InvalidArgumentException; + + /** Resolves this module argument to a set of repo names. */ + ImmutableMap resolveToRepoNames( + ImmutableMap> modulesIndex, + ImmutableMap depGraph, + RepositoryMapping mapping) + throws InvalidArgumentException; + + /** + * Refers to a specific version of a module. Parsed from {@code @}. {@code + * } can be the special string {@code _} to signify the empty version (for non-registry + * overrides). + */ + @AutoValue + abstract class SpecificVersionOfModule implements ModuleArg { + static SpecificVersionOfModule create(ModuleKey key) { + return new AutoValue_ModuleArg_SpecificVersionOfModule(key); + } + + public abstract ModuleKey moduleKey(); + + private void throwIfNonexistent( + ImmutableMap> modulesIndex, + ImmutableMap depGraph, + boolean includeUnused, + boolean warnUnused) + throws InvalidArgumentException { + AugmentedModule mod = depGraph.get(moduleKey()); + if (mod != null && !includeUnused && warnUnused && !mod.isUsed()) { + // Warn the user when unused modules are allowed and the specified version exists, but the + // --include_unused flag was not set. + throw new InvalidArgumentException( + String.format( + "Module version %s is unused as a result of module resolution. Use the" + + " --include_unused flag to include it.", + moduleKey()), + Code.INVALID_ARGUMENTS); + } + if (mod == null || (!includeUnused && !mod.isUsed())) { + ImmutableSet existingKeys = modulesIndex.get(moduleKey().getName()); + if (existingKeys == null) { + throw new InvalidArgumentException( + String.format( + "Module %s does not exist in the dependency graph.", moduleKey().getName()), + Code.INVALID_ARGUMENTS); + } + // If --include_unused is not true, unused modules will be considered non-existent and an + // error will be thrown. + ImmutableSet filteredKeys = + existingKeys.stream() + .filter(k -> includeUnused || depGraph.get(k).isUsed()) + .collect(toImmutableSet()); + throw new InvalidArgumentException( + String.format( + "Module version %s does not exist, available versions: %s.", + moduleKey(), filteredKeys), + Code.INVALID_ARGUMENTS); + } + } + + @Override + public final ImmutableSet resolveToModuleKeys( + ImmutableMap> modulesIndex, + ImmutableMap depGraph, + ImmutableBiMap baseModuleDeps, + ImmutableBiMap baseModuleUnusedDeps, + boolean includeUnused, + boolean warnUnused) + throws InvalidArgumentException { + throwIfNonexistent(modulesIndex, depGraph, includeUnused, warnUnused); + return ImmutableSet.of(moduleKey()); + } + + @Override + public ImmutableMap resolveToRepoNames( + ImmutableMap> modulesIndex, + ImmutableMap depGraph, + RepositoryMapping mapping) + throws InvalidArgumentException { + throwIfNonexistent( + modulesIndex, depGraph, /* includeUnused= */ false, /* warnUnused= */ false); + return ImmutableMap.of(moduleKey().toString(), moduleKey().getCanonicalRepoName()); + } + + @Override + public final String toString() { + return moduleKey().toString(); + } + } + + /** Refers to all versions of a module. Parsed from {@code }. */ + @AutoValue + abstract class AllVersionsOfModule implements ModuleArg { + static AllVersionsOfModule create(String moduleName) { + return new AutoValue_ModuleArg_AllVersionsOfModule(moduleName); + } + + public abstract String moduleName(); + + private ImmutableSet resolveInternal( + ImmutableMap> modulesIndex, + ImmutableMap depGraph, + boolean includeUnused, + boolean warnUnused) + throws InvalidArgumentException { + ImmutableSet existingKeys = modulesIndex.get(moduleName()); + if (existingKeys == null) { + throw new InvalidArgumentException( + String.format("Module %s does not exist in the dependency graph.", moduleName()), + Code.INVALID_ARGUMENTS); + } + ImmutableSet filteredKeys = + existingKeys.stream() + .filter(k -> includeUnused || depGraph.get(k).isUsed()) + .collect(toImmutableSet()); + if (filteredKeys.isEmpty()) { + if (warnUnused) { + throw new InvalidArgumentException( + String.format( + "Module %s is unused as a result of module resolution. Use the --include_unused" + + " flag to include it.", + moduleName()), + Code.INVALID_ARGUMENTS); + } + throw new InvalidArgumentException( + String.format("Module %s does not exist in the dependency graph.", moduleName()), + Code.INVALID_ARGUMENTS); + } + return filteredKeys; + } + + @Override + public ImmutableSet resolveToModuleKeys( + ImmutableMap> modulesIndex, + ImmutableMap depGraph, + ImmutableBiMap baseModuleDeps, + ImmutableBiMap baseModuleUnusedDeps, + boolean includeUnused, + boolean warnUnused) + throws InvalidArgumentException { + return resolveInternal(modulesIndex, depGraph, includeUnused, warnUnused); + } + + @Override + public ImmutableMap resolveToRepoNames( + ImmutableMap> modulesIndex, + ImmutableMap depGraph, + RepositoryMapping mapping) + throws InvalidArgumentException { + return resolveInternal( + modulesIndex, depGraph, /* includeUnused= */ false, /* warnUnused= */ false) + .stream() + .collect(toImmutableMap(ModuleKey::toString, ModuleKey::getCanonicalRepoName)); + } + + @Override + public final String toString() { + return moduleName(); + } + } + + /** + * Refers to a module with the given apparent repo name, in the context of {@code --base_module} + * (or when parsing that flag itself, in the context of the root module). Parsed from + * {@code @}. + */ + @AutoValue + abstract class ApparentRepoName implements ModuleArg { + static ApparentRepoName create(String name) { + return new AutoValue_ModuleArg_ApparentRepoName(name); + } + + public abstract String name(); + + @Override + public ImmutableSet resolveToModuleKeys( + ImmutableMap> modulesIndex, + ImmutableMap depGraph, + ImmutableBiMap baseModuleDeps, + ImmutableBiMap baseModuleUnusedDeps, + boolean includeUnused, + boolean warnUnused) + throws InvalidArgumentException { + ImmutableSet.Builder builder = new ImmutableSet.Builder<>(); + ModuleKey dep = baseModuleDeps.get(name()); + if (dep != null) { + builder.add(dep); + } + ModuleKey unusedDep = baseModuleUnusedDeps.get(name()); + if (includeUnused && unusedDep != null) { + builder.add(unusedDep); + } + var result = builder.build(); + if (result.isEmpty()) { + throw new InvalidArgumentException( + String.format( + "No module with the apparent repo name @%s exists in the dependency graph", name()), + Code.INVALID_ARGUMENTS); + } + return result; + } + + @Override + public ImmutableMap resolveToRepoNames( + ImmutableMap> modulesIndex, + ImmutableMap depGraph, + RepositoryMapping mapping) + throws InvalidArgumentException { + RepositoryName repoName = mapping.get(name()); + if (!repoName.isVisible()) { + throw new InvalidArgumentException( + String.format( + "No repo visible as %s from @%s", name(), repoName.getOwnerRepoDisplayString()), + Code.INVALID_ARGUMENTS); + } + return ImmutableMap.of(toString(), repoName); + } + + @Override + public final String toString() { + return "@" + name(); + } + } + + /** Refers to a module with the given canonical repo name. Parsed from {@code @@}. */ + @AutoValue + abstract class CanonicalRepoName implements ModuleArg { + static CanonicalRepoName create(RepositoryName repoName) { + return new AutoValue_ModuleArg_CanonicalRepoName(repoName); + } + + public abstract RepositoryName repoName(); + + @Override + public ImmutableSet resolveToModuleKeys( + ImmutableMap> modulesIndex, + ImmutableMap depGraph, + ImmutableBiMap baseModuleDeps, + ImmutableBiMap baseModuleUnusedDeps, + boolean includeUnused, + boolean warnUnused) + throws InvalidArgumentException { + Optional mod = + depGraph.values().stream() + .filter(m -> m.getKey().getCanonicalRepoName().equals(repoName())) + .findAny(); + if (mod.isPresent() && !includeUnused && warnUnused && !mod.get().isUsed()) { + // Warn the user when unused modules are allowed and the specified version exists, but the + // --include_unused flag was not set. + throw new InvalidArgumentException( + String.format( + "Module version %s is unused as a result of module resolution. Use the" + + " --include_unused flag to include it.", + mod.get().getKey()), + Code.INVALID_ARGUMENTS); + } + if (mod.isEmpty() || (!includeUnused && !mod.get().isUsed())) { + // If --include_unused is not true, unused modules will be considered non-existent and an + // error will be thrown. + throw new InvalidArgumentException( + String.format( + "No module with the canonical repo name @@%s exists in the dependency graph", + repoName().getName()), + Code.INVALID_ARGUMENTS); + } + return ImmutableSet.of(mod.get().getKey()); + } + + @Override + public ImmutableMap resolveToRepoNames( + ImmutableMap> modulesIndex, + ImmutableMap depGraph, + RepositoryMapping mapping) + throws InvalidArgumentException { + if (depGraph.values().stream() + .filter(m -> m.getKey().getCanonicalRepoName().equals(repoName()) && m.isUsed()) + .findAny() + .isEmpty()) { + throw new InvalidArgumentException( + String.format( + "No module with the canonical repo name @@%s exists in the dependency graph", + repoName().getName()), + Code.INVALID_ARGUMENTS); + } + return ImmutableMap.of(toString(), repoName()); + } + + @Override + public final String toString() { + return "@@" + repoName().getName(); + } + } + + /** Converter for {@link ModuleArg}. */ + final class ModuleArgConverter extends Converter.Contextless { + public static final ModuleArgConverter INSTANCE = new ModuleArgConverter(); + + @Override + public ModuleArg convert(String input) throws OptionsParsingException { + if (Ascii.equalsIgnoreCase(input, "")) { + return SpecificVersionOfModule.create(ModuleKey.ROOT); + } + if (input.startsWith("@@")) { + try { + return CanonicalRepoName.create(RepositoryName.create(input.substring(2))); + } catch (LabelSyntaxException e) { + throw new OptionsParsingException("invalid argument '" + input + "': " + e.getMessage()); + } + } + if (input.startsWith("@")) { + String apparentRepoName = input.substring(1); + try { + RepositoryName.validateUserProvidedRepoName(apparentRepoName); + } catch (EvalException e) { + throw new OptionsParsingException("invalid argument '" + input + "': " + e.getMessage()); + } + return ApparentRepoName.create(apparentRepoName); + } + int atIdx = input.indexOf('@'); + if (atIdx >= 0) { + String moduleName = input.substring(0, atIdx); + String versionStr = input.substring(atIdx + 1); + if (versionStr.isEmpty()) { + throw new OptionsParsingException( + "invalid argument '" + input + "': use _ for the empty version"); + } + try { + Version version = versionStr.equals("_") ? Version.EMPTY : Version.parse(versionStr); + return SpecificVersionOfModule.create(ModuleKey.create(moduleName, version)); + } catch (ParseException e) { + throw new OptionsParsingException("invalid argument '" + input + "': " + e.getMessage()); + } + } + return AllVersionsOfModule.create(input); + } + + @Override + public String getTypeDescription() { + return "\"\" for the root module; @ for a specific version of a" + + " module; for all versions of a module; @ for a repo with the" + + " given apparent name; or @@ for a repo with the given canonical name"; + } + } + + /** Converter for a comma-separated list of {@link ModuleArg}s. */ + class CommaSeparatedModuleArgListConverter + extends Converter.Contextless> { + + @Override + public ImmutableList convert(String input) throws OptionsParsingException { + ImmutableList args = new CommaSeparatedNonEmptyOptionListConverter().convert(input); + ImmutableList.Builder moduleArgs = new ImmutableList.Builder<>(); + for (String arg : args) { + moduleArgs.add(ModuleArgConverter.INSTANCE.convert(arg)); + } + return moduleArgs.build(); + } + + @Override + public String getTypeDescription() { + return "a comma-separated list of s"; + } + } +} diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/modquery/OutputFormatters.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/modquery/OutputFormatters.java index 78d40b0596f38d..0e4bfd7fe62492 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/modquery/OutputFormatters.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/modquery/OutputFormatters.java @@ -20,8 +20,10 @@ import com.google.common.base.Ascii; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; +import com.google.common.collect.ImmutableSetMultimap; import com.google.devtools.build.lib.bazel.bzlmod.BazelModuleInspectorValue.AugmentedModule; import com.google.devtools.build.lib.bazel.bzlmod.BazelModuleInspectorValue.AugmentedModule.ResolutionReason; +import com.google.devtools.build.lib.bazel.bzlmod.ModuleExtensionId; import com.google.devtools.build.lib.bazel.bzlmod.ModuleKey; import com.google.devtools.build.lib.bazel.bzlmod.Version; import com.google.devtools.build.lib.bazel.bzlmod.modquery.ModqueryExecutor.ResultNode; @@ -57,10 +59,13 @@ abstract static class OutputFormatter { protected ImmutableMap result; protected ImmutableMap depGraph; + protected ImmutableSetMultimap extensionRepos; + protected ImmutableMap> + extensionRepoImports; protected PrintWriter printer; protected ModqueryOptions options; - /** Compact representation of the data provided by the {@link ModqueryOptions#extra} flag. */ + /** Compact representation of the data provided by the {@code --verbose} flag. */ @AutoValue abstract static class Explanation { @@ -83,8 +88,8 @@ static Explanation create( } /** - * Gets the exact label that is printed next to the module if the {@link - * ModqueryOptions#extra} flag is enabled. + * Gets the exact label that is printed next to the module if the {@code --verbose} flag is + * enabled. */ String toExplanationString(boolean unused) { String changedVersionLabel = @@ -102,10 +107,15 @@ String toExplanationString(boolean unused) { void output( ImmutableMap result, ImmutableMap depGraph, + ImmutableSetMultimap extensionRepos, + ImmutableMap> + extensionRepoImports, PrintWriter printer, ModqueryOptions options) { this.result = result; this.depGraph = depGraph; + this.extensionRepos = extensionRepos; + this.extensionRepoImports = extensionRepoImports; this.printer = printer; this.options = options; output(); diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/modquery/TextOutputFormatter.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/modquery/TextOutputFormatter.java index 52b995d78e1dca..065916e41c02d8 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/modquery/TextOutputFormatter.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/modquery/TextOutputFormatter.java @@ -14,8 +14,13 @@ package com.google.devtools.build.lib.bazel.bzlmod.modquery; +import static com.google.common.collect.ImmutableSortedSet.toImmutableSortedSet; + import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableSortedSet; +import com.google.common.collect.Sets; import com.google.devtools.build.lib.bazel.bzlmod.BazelModuleInspectorValue.AugmentedModule; +import com.google.devtools.build.lib.bazel.bzlmod.ModuleExtensionId; import com.google.devtools.build.lib.bazel.bzlmod.ModuleKey; import com.google.devtools.build.lib.bazel.bzlmod.Version; import com.google.devtools.build.lib.bazel.bzlmod.modquery.ModqueryExecutor.ResultNode; @@ -24,12 +29,15 @@ import com.google.devtools.build.lib.bazel.bzlmod.modquery.ModqueryExecutor.ResultNode.IsIndirect; import com.google.devtools.build.lib.bazel.bzlmod.modquery.ModqueryExecutor.ResultNode.NodeMetadata; import com.google.devtools.build.lib.bazel.bzlmod.modquery.ModqueryOptions.Charset; +import com.google.devtools.build.lib.bazel.bzlmod.modquery.ModqueryOptions.ExtensionShow; import com.google.devtools.build.lib.bazel.bzlmod.modquery.OutputFormatters.OutputFormatter; import java.util.ArrayDeque; import java.util.Deque; +import java.util.HashSet; import java.util.Iterator; import java.util.Map.Entry; import java.util.Objects; +import java.util.Set; /** * Outputs graph-based results of {@link @@ -40,6 +48,8 @@ public class TextOutputFormatter extends OutputFormatter { private Deque isLastChildStack; private DrawCharset drawCharset; + private Set seenExtensions; + private StringBuilder str; @Override public void output() { @@ -49,20 +59,14 @@ public void output() { drawCharset = DrawCharset.UTF8; } isLastChildStack = new ArrayDeque<>(); - printTree(ModuleKey.ROOT, null, IsExpanded.TRUE, IsIndirect.FALSE, IsCycle.FALSE, 0); + seenExtensions = new HashSet<>(); + str = new StringBuilder(); + printModule(ModuleKey.ROOT, null, IsExpanded.TRUE, IsIndirect.FALSE, IsCycle.FALSE, 0); + this.printer.println(str); } - // Depth-first traversal to print the actual output - void printTree( - ModuleKey key, - ModuleKey parent, - IsExpanded expanded, - IsIndirect indirect, - IsCycle cycle, - int depth) { - ResultNode node = Objects.requireNonNull(result.get(key)); - StringBuilder str = new StringBuilder(); - + // Prints the indents and the tree drawing characters. + private void printTreeDrawing(IsIndirect indirect, int depth) { if (depth > 0) { int indents = isLastChildStack.size() - 1; Iterator value = isLastChildStack.descendingIterator(); @@ -89,21 +93,87 @@ void printTree( } } } + } - int totalChildrenNum = node.getChildren().size(); + // Helper to print module extensions similarly to printModule. + private void printExtension( + ModuleKey key, ModuleExtensionId extensionId, boolean unexpanded, int depth) { + printTreeDrawing(IsIndirect.FALSE, depth); + str.append('$'); + str.append(extensionId.asTargetString()); + str.append(' '); + if (unexpanded && options.extensionInfo == ExtensionShow.ALL) { + str.append("... "); + } + str.append("\n"); + if (options.extensionInfo == ExtensionShow.USAGES) { + return; + } + ImmutableSortedSet repoImports = + ImmutableSortedSet.copyOf(extensionRepoImports.get(extensionId).inverse().get(key)); + ImmutableSortedSet unusedRepos = ImmutableSortedSet.of(); + if (!unexpanded && options.extensionInfo == ExtensionShow.ALL) { + unusedRepos = + ImmutableSortedSet.copyOf( + Sets.difference( + extensionRepos.get(extensionId), extensionRepoImports.get(extensionId).keySet())); + } + int totalChildrenNum = repoImports.size() + unusedRepos.size(); + int currChild = 1; + for (String usedRepo : repoImports) { + isLastChildStack.push(currChild++ == totalChildrenNum); + printExtensionRepo(usedRepo, IsIndirect.FALSE, depth + 1); + isLastChildStack.pop(); + } + if (unexpanded || options.extensionInfo == ExtensionShow.REPOS) { + return; + } + for (String unusedPackage : unusedRepos) { + isLastChildStack.push(currChild++ == totalChildrenNum); + printExtensionRepo(unusedPackage, IsIndirect.TRUE, depth + 1); + isLastChildStack.pop(); + } + } + + // Prints an extension repo line. + private void printExtensionRepo(String repoName, IsIndirect indirectLink, int depth) { + printTreeDrawing(indirectLink, depth); + str.append(repoName).append("\n"); + } + // Depth-first traversal to print the actual output + private void printModule( + ModuleKey key, + ModuleKey parent, + IsExpanded expanded, + IsIndirect indirect, + IsCycle cycle, + int depth) { + printTreeDrawing(indirect, depth); + + ResultNode node = Objects.requireNonNull(result.get(key)); if (key.equals(ModuleKey.ROOT)) { AugmentedModule rootModule = depGraph.get(ModuleKey.ROOT); Preconditions.checkNotNull(rootModule); str.append( String.format( - "root (%s@%s)", + " (%s@%s)", rootModule.getName(), rootModule.getVersion().equals(Version.EMPTY) ? "_" : rootModule.getVersion())); } else { str.append(key).append(" "); } + int totalChildrenNum = node.getChildren().size(); + + ImmutableSortedSet extensionsUsed = + extensionRepoImports.keySet().stream() + .filter(e -> extensionRepoImports.get(e).inverse().containsKey(key)) + .collect(toImmutableSortedSet(ModuleExtensionId.LEXICOGRAPHIC_COMPARATOR)); + if (options.extensionInfo != ExtensionShow.HIDDEN) { + totalChildrenNum += extensionsUsed.size(); + } + if (cycle == IsCycle.TRUE) { str.append("(cycle) "); } else if (expanded == IsExpanded.FALSE) { @@ -114,34 +184,40 @@ void printTree( } } AugmentedModule module = Objects.requireNonNull(depGraph.get(key)); - - if (!options.extra && !module.isUsed()) { + if (!options.verbose && !module.isUsed()) { str.append("(unused) "); } // If the edge is indirect, the parent is not only unknown, but the node could have come - // from - // multiple paths merged in the process, so we skip the resolution explanation. - if (indirect == IsIndirect.FALSE && options.extra && parent != null) { + // from multiple paths merged in the process, so we skip the resolution explanation. + if (indirect == IsIndirect.FALSE && options.verbose && parent != null) { Explanation explanation = getExtraResolutionExplanation(key, parent); if (explanation != null) { str.append(explanation.toExplanationString(!module.isUsed())); } } - this.printer.println(str); + str.append("\n"); if (expanded == IsExpanded.FALSE) { return; } int currChild = 1; + if (options.extensionInfo != ExtensionShow.HIDDEN) { + for (ModuleExtensionId extensionId : extensionsUsed) { + boolean unexpandedExtension = !seenExtensions.add(extensionId); + isLastChildStack.push(currChild++ == totalChildrenNum); + printExtension(key, extensionId, unexpandedExtension, depth + 1); + isLastChildStack.pop(); + } + } for (Entry e : node.getChildrenSortedByEdgeType()) { ModuleKey childKey = e.getKey(); IsExpanded childExpanded = e.getValue().isExpanded(); IsIndirect childIndirect = e.getValue().isIndirect(); IsCycle childCycles = e.getValue().isCycle(); isLastChildStack.push(currChild++ == totalChildrenNum); - printTree(childKey, key, childExpanded, childIndirect, childCycles, depth + 1); + printModule(childKey, key, childExpanded, childIndirect, childCycles, depth + 1); isLastChildStack.pop(); } } diff --git a/src/main/java/com/google/devtools/build/lib/bazel/commands/BUILD b/src/main/java/com/google/devtools/build/lib/bazel/commands/BUILD index ee951d3264ae3d..e16cb80eddb310 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/commands/BUILD +++ b/src/main/java/com/google/devtools/build/lib/bazel/commands/BUILD @@ -30,7 +30,9 @@ java_library( "//src/main/java/com/google/devtools/build/lib/bazel:resolved_event", "//src/main/java/com/google/devtools/build/lib/bazel/bzlmod:common", "//src/main/java/com/google/devtools/build/lib/bazel/bzlmod:inspection", + "//src/main/java/com/google/devtools/build/lib/bazel/bzlmod:module_extension", "//src/main/java/com/google/devtools/build/lib/bazel/bzlmod:repo_rule_value", + "//src/main/java/com/google/devtools/build/lib/bazel/bzlmod:resolution", "//src/main/java/com/google/devtools/build/lib/bazel/bzlmod/modquery", "//src/main/java/com/google/devtools/build/lib/bazel/repository", "//src/main/java/com/google/devtools/build/lib/bazel/repository/starlark", @@ -52,6 +54,7 @@ java_library( "//src/main/java/com/google/devtools/build/lib/util:detailed_exit_code", "//src/main/java/com/google/devtools/build/lib/util:exit_code", "//src/main/java/com/google/devtools/build/lib/util:interrupted_failure_details", + "//src/main/java/com/google/devtools/build/lib/util:maybe_complete_set", "//src/main/java/com/google/devtools/build/lib/vfs", "//src/main/java/com/google/devtools/build/skyframe", "//src/main/java/com/google/devtools/build/skyframe:skyframe-objects", diff --git a/src/main/java/com/google/devtools/build/lib/bazel/commands/ModqueryCommand.java b/src/main/java/com/google/devtools/build/lib/bazel/commands/ModqueryCommand.java index f4a73220fd2564..0945627efea2bc 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/commands/ModqueryCommand.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/commands/ModqueryCommand.java @@ -18,24 +18,32 @@ import static com.google.devtools.build.lib.bazel.bzlmod.modquery.ModqueryOptions.Charset.UTF8; import static java.nio.charset.StandardCharsets.US_ASCII; import static java.nio.charset.StandardCharsets.UTF_8; +import static java.util.stream.Collectors.joining; -import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableBiMap; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; +import com.google.common.collect.ImmutableSortedSet; +import com.google.common.collect.Iterables; +import com.google.devtools.build.lib.bazel.bzlmod.BazelDepGraphValue; import com.google.devtools.build.lib.bazel.bzlmod.BazelModuleInspectorValue; import com.google.devtools.build.lib.bazel.bzlmod.BazelModuleInspectorValue.AugmentedModule; import com.google.devtools.build.lib.bazel.bzlmod.BzlmodRepoRuleValue; +import com.google.devtools.build.lib.bazel.bzlmod.ModuleExtensionId; import com.google.devtools.build.lib.bazel.bzlmod.ModuleKey; -import com.google.devtools.build.lib.bazel.bzlmod.Version; +import com.google.devtools.build.lib.bazel.bzlmod.modquery.ExtensionArg; +import com.google.devtools.build.lib.bazel.bzlmod.modquery.ExtensionArg.ExtensionArgConverter; +import com.google.devtools.build.lib.bazel.bzlmod.modquery.InvalidArgumentException; import com.google.devtools.build.lib.bazel.bzlmod.modquery.ModqueryExecutor; import com.google.devtools.build.lib.bazel.bzlmod.modquery.ModqueryOptions; import com.google.devtools.build.lib.bazel.bzlmod.modquery.ModqueryOptions.QueryType; import com.google.devtools.build.lib.bazel.bzlmod.modquery.ModqueryOptions.QueryTypeConverter; -import com.google.devtools.build.lib.bazel.bzlmod.modquery.ModqueryOptions.TargetModule; -import com.google.devtools.build.lib.bazel.bzlmod.modquery.ModqueryOptions.TargetModuleListConverter; +import com.google.devtools.build.lib.bazel.bzlmod.modquery.ModuleArg; +import com.google.devtools.build.lib.bazel.bzlmod.modquery.ModuleArg.ModuleArgConverter; +import com.google.devtools.build.lib.cmdline.RepositoryMapping; +import com.google.devtools.build.lib.cmdline.RepositoryName; import com.google.devtools.build.lib.events.Event; import com.google.devtools.build.lib.pkgcache.PackageOptions; import com.google.devtools.build.lib.runtime.BlazeCommand; @@ -51,15 +59,20 @@ import com.google.devtools.build.lib.util.AbruptExitException; import com.google.devtools.build.lib.util.DetailedExitCode; import com.google.devtools.build.lib.util.InterruptedFailureDetails; +import com.google.devtools.build.lib.util.MaybeCompleteSet; import com.google.devtools.build.skyframe.EvaluationContext; import com.google.devtools.build.skyframe.EvaluationResult; import com.google.devtools.build.skyframe.SkyKey; import com.google.devtools.build.skyframe.SkyValue; +import com.google.devtools.common.options.OptionPriority.PriorityCategory; +import com.google.devtools.common.options.OptionsParser; import com.google.devtools.common.options.OptionsParsingException; import com.google.devtools.common.options.OptionsParsingResult; import java.io.OutputStreamWriter; import java.util.List; +import java.util.Map.Entry; import java.util.Objects; +import java.util.Optional; /** Queries the Bzlmod external dependency graph. */ @Command( @@ -67,7 +80,6 @@ // TODO(andreisolo): figure out which extra options are really needed options = { ModqueryOptions.class, - // Don't know what these do but were used in fetch PackageOptions.class, KeepGoingOption.class, LoadingPhaseThreadsOption.class @@ -79,8 +91,22 @@ public final class ModqueryCommand implements BlazeCommand { public static final String NAME = "modquery"; + @Override + public void editOptions(OptionsParser optionsParser) { + try { + optionsParser.parse( + PriorityCategory.SOFTWARE_REQUIREMENT, + "Option required by the modquery command", + ImmutableList.of("--enable_bzlmod")); + } catch (OptionsParsingException e) { + // Should never happen. + throw new IllegalStateException("Unexpected exception", e); + } + } + @Override public BlazeCommandResult exec(CommandEnvironment env, OptionsParsingResult options) { + BazelDepGraphValue depGraphValue; BazelModuleInspectorValue moduleInspector; SkyframeExecutor skyframeExecutor = env.getSkyframeExecutor(); @@ -97,7 +123,8 @@ public BlazeCommandResult exec(CommandEnvironment env, OptionsParsingResult opti EvaluationResult evaluationResult = skyframeExecutor.prepareAndGet( - ImmutableSet.of(BazelModuleInspectorValue.KEY), evaluationContext); + ImmutableSet.of(BazelDepGraphValue.KEY, BazelModuleInspectorValue.KEY), + evaluationContext); if (evaluationResult.hasError()) { Exception e = evaluationResult.getError().getException(); @@ -108,6 +135,8 @@ public BlazeCommandResult exec(CommandEnvironment env, OptionsParsingResult opti return reportAndCreateFailureResult(env, message, Code.INVALID_ARGUMENTS); } + depGraphValue = (BazelDepGraphValue) evaluationResult.get(BazelDepGraphValue.KEY); + moduleInspector = (BazelModuleInspectorValue) evaluationResult.get(BazelModuleInspectorValue.KEY); @@ -126,7 +155,7 @@ public BlazeCommandResult exec(CommandEnvironment env, OptionsParsingResult opti if (options.getResidue().isEmpty()) { String errorMessage = - String.format("No query type specified, choose one from : %s.", QueryType.printValues()); + String.format("No query type specified, choose one of : %s.", QueryType.printValues()); return reportAndCreateFailureResult(env, errorMessage, Code.MODQUERY_COMMAND_UNKNOWN); } @@ -142,89 +171,243 @@ public BlazeCommandResult exec(CommandEnvironment env, OptionsParsingResult opti } List args = options.getResidue().subList(1, options.getResidue().size()); - // Arguments are structured as a list of comma-separated target lists for generality, - // even though there can only be 0 or 1 args so far. - ImmutableList> argsKeysList; + // Extract and check the --base_module argument first to use it when parsing the other args. + // Can only be a TargetModule or a repoName relative to the ROOT. + ModuleKey baseModuleKey; AugmentedModule rootModule = moduleInspector.getDepGraph().get(ModuleKey.ROOT); try { - argsKeysList = - parseTargetArgs( - query.getArgNumber(), + ImmutableSet keys = + modqueryOptions.baseModule.resolveToModuleKeys( moduleInspector.getModulesIndex(), - args, + moduleInspector.getDepGraph(), rootModule.getDeps(), rootModule.getUnusedDeps(), - modqueryOptions.includeUnused); + false, + false); + if (keys.size() > 1) { + throw new InvalidArgumentException( + String.format( + "The --base_module option can only specify exactly one module version, choose one" + + " of: %s.", + keys.stream().map(ModuleKey::toString).collect(joining(", "))), + Code.INVALID_ARGUMENTS); + } + baseModuleKey = Iterables.getOnlyElement(keys); + } catch (InvalidArgumentException e) { + return reportAndCreateFailureResult( + env, + String.format( + "In --base_module %s option: %s (Note that unused modules cannot be used here)", + modqueryOptions.baseModule, e.getMessage()), + Code.INVALID_ARGUMENTS); + } + + // The args can have different types depending on the query type, so create multiple + // containers which can be filled accordingly. + ImmutableSet argsAsModules = null; + ImmutableSortedSet argsAsExtensions = null; + ImmutableMap argsAsRepos = null; + + AugmentedModule baseModule = + Objects.requireNonNull(moduleInspector.getDepGraph().get(baseModuleKey)); + RepositoryMapping baseModuleMapping = depGraphValue.getFullRepoMapping(baseModuleKey); + try { + switch (query) { + case TREE: + // TREE doesn't take extra arguments. + if (!args.isEmpty()) { + throw new InvalidArgumentException( + "the 'tree' command doesn't take extra arguments", Code.TOO_MANY_ARGUMENTS); + } + break; + case SHOW: + ImmutableMap.Builder targetToRepoName = + new ImmutableMap.Builder<>(); + for (String arg : args) { + try { + targetToRepoName.putAll( + ModuleArgConverter.INSTANCE + .convert(arg) + .resolveToRepoNames( + moduleInspector.getModulesIndex(), + moduleInspector.getDepGraph(), + baseModuleMapping)); + } catch (InvalidArgumentException | OptionsParsingException e) { + throw new InvalidArgumentException( + String.format( + "In repo argument %s: %s (Note that unused modules cannot be used here)", + arg, e.getMessage()), + Code.INVALID_ARGUMENTS, + e); + } + } + argsAsRepos = targetToRepoName.buildKeepingLast(); + break; + case SHOW_EXTENSION: + ImmutableSortedSet.Builder extensionsBuilder = + new ImmutableSortedSet.Builder<>(ModuleExtensionId.LEXICOGRAPHIC_COMPARATOR); + for (String arg : args) { + try { + extensionsBuilder.add( + ExtensionArgConverter.INSTANCE + .convert(arg) + .resolveToExtensionId( + moduleInspector.getModulesIndex(), + moduleInspector.getDepGraph(), + baseModule.getDeps(), + baseModule.getUnusedDeps())); + } catch (InvalidArgumentException | OptionsParsingException e) { + throw new InvalidArgumentException( + String.format("In extension argument: %s %s", arg, e.getMessage()), + Code.INVALID_ARGUMENTS, + e); + } + } + argsAsExtensions = extensionsBuilder.build(); + break; + default: + ImmutableSet.Builder keysBuilder = new ImmutableSet.Builder<>(); + for (String arg : args) { + try { + keysBuilder.addAll( + ModuleArgConverter.INSTANCE + .convert(arg) + .resolveToModuleKeys( + moduleInspector.getModulesIndex(), + moduleInspector.getDepGraph(), + baseModule.getDeps(), + baseModule.getUnusedDeps(), + modqueryOptions.includeUnused, + /* warnUnused= */ true)); + } catch (InvalidArgumentException | OptionsParsingException e) { + throw new InvalidArgumentException( + String.format("In module argument %s: %s", arg, e.getMessage()), + Code.INVALID_ARGUMENTS); + } + } + argsAsModules = keysBuilder.build(); + } } catch (InvalidArgumentException e) { return reportAndCreateFailureResult(env, e.getMessage(), e.getCode()); - } catch (OptionsParsingException e) { - return reportAndCreateFailureResult(env, e.getMessage(), Code.INVALID_ARGUMENTS); } - /* Extract and check the --from argument */ + /* Extract and check the --from and --extension_usages argument */ ImmutableSet fromKeys; - if (modqueryOptions.modulesFrom == null) { - fromKeys = ImmutableSet.of(ModuleKey.ROOT); - } else { - try { - fromKeys = - targetListToModuleKeySet( - modqueryOptions.modulesFrom, - moduleInspector.getModulesIndex(), - rootModule.getDeps(), - rootModule.getUnusedDeps(), - modqueryOptions.includeUnused); - } catch (InvalidArgumentException e) { - return reportAndCreateFailureResult(env, e.getMessage(), e.getCode()); + ImmutableSet usageKeys; + try { + fromKeys = + moduleArgListToKeys( + modqueryOptions.modulesFrom, + moduleInspector.getModulesIndex(), + moduleInspector.getDepGraph(), + baseModule.getDeps(), + baseModule.getUnusedDeps(), + modqueryOptions.includeUnused); + } catch (InvalidArgumentException e) { + return reportAndCreateFailureResult( + env, + String.format("In --from %s option: %s", modqueryOptions.modulesFrom, e.getMessage()), + Code.INVALID_ARGUMENTS); + } + + try { + usageKeys = + moduleArgListToKeys( + modqueryOptions.extensionUsages, + moduleInspector.getModulesIndex(), + moduleInspector.getDepGraph(), + baseModule.getDeps(), + baseModule.getUnusedDeps(), + modqueryOptions.includeUnused); + } catch (InvalidArgumentException e) { + return reportAndCreateFailureResult( + env, + String.format( + "In --extension_usages %s option: %s (Note that unused modules cannot be used" + + " here)", + modqueryOptions.extensionUsages, e.getMessage()), + Code.INVALID_ARGUMENTS); + } + + /* Extract and check the --extension_filter argument */ + Optional> filterExtensions = Optional.empty(); + if (query.isGraph() && modqueryOptions.extensionFilter != null) { + if (modqueryOptions.extensionFilter.isEmpty()) { + filterExtensions = Optional.of(MaybeCompleteSet.completeSet()); + } else { + try { + filterExtensions = + Optional.of( + MaybeCompleteSet.copyOf( + extensionArgListToIds( + modqueryOptions.extensionFilter, + moduleInspector.getModulesIndex(), + moduleInspector.getDepGraph(), + baseModule.getDeps(), + baseModule.getUnusedDeps()))); + } catch (InvalidArgumentException e) { + return reportAndCreateFailureResult( + env, + String.format( + "In --extension_filter %s option: %s", + modqueryOptions.extensionFilter, e.getMessage()), + Code.INVALID_ARGUMENTS); + } } } - ImmutableMap repoRuleValues = null; - // If the query is a SHOW, also request the BzlmodRepoRuleValues from SkyFrame. - // Unused modules do not have a BzlmodRepoRuleValue, so they are filtered out. - if (query == QueryType.SHOW) { - try { - ImmutableSet keys = - argsKeysList.get(0).stream() - .filter( - k -> - ModqueryExecutor.filterUnused( - k, modqueryOptions.includeUnused, false, moduleInspector.getDepGraph())) - .collect(toImmutableSet()); + ImmutableMap targetRepoRuleValues = null; + try { + // If the query is a SHOW, also request the BzlmodRepoRuleValues from SkyFactory. + if (query == QueryType.SHOW) { ImmutableSet skyKeys = - keys.stream() - .map(k -> BzlmodRepoRuleValue.key(k.getCanonicalRepoName())) - .collect(toImmutableSet()); + argsAsRepos.values().stream().map(BzlmodRepoRuleValue::key).collect(toImmutableSet()); EvaluationResult result = env.getSkyframeExecutor().prepareAndGet(skyKeys, evaluationContext); - repoRuleValues = - keys.stream() + if (result.hasError()) { + Exception e = result.getError().getException(); + String message = "Unexpected error during repository rule evaluation."; + if (e != null) { + message = e.getMessage(); + } + return reportAndCreateFailureResult(env, message, Code.INVALID_ARGUMENTS); + } + targetRepoRuleValues = + argsAsRepos.entrySet().stream() .collect( toImmutableMap( - k -> k, - k -> + Entry::getKey, + e -> (BzlmodRepoRuleValue) - result.get(BzlmodRepoRuleValue.key(k.getCanonicalRepoName())))); - } catch (InterruptedException e) { - String errorMessage = "Modquery interrupted: " + e.getMessage(); - env.getReporter().handle(Event.error(errorMessage)); - return BlazeCommandResult.detailedExitCode( - InterruptedFailureDetails.detailedExitCode(errorMessage)); + result.get(BzlmodRepoRuleValue.key(e.getValue())))); } + } catch (InterruptedException e) { + String errorMessage = "Modquery interrupted: " + e.getMessage(); + env.getReporter().handle(Event.error(errorMessage)); + return BlazeCommandResult.detailedExitCode( + InterruptedFailureDetails.detailedExitCode(errorMessage)); } // Workaround to allow different default value for DEPS and EXPLAIN, and also use // Integer.MAX_VALUE instead of the exact number string. if (modqueryOptions.depth < 1) { - if (query == QueryType.EXPLAIN || query == QueryType.DEPS) { - modqueryOptions.depth = 1; - } else { - modqueryOptions.depth = Integer.MAX_VALUE; + switch (query) { + case EXPLAIN: + modqueryOptions.depth = 1; + break; + case DEPS: + modqueryOptions.depth = 2; + break; + default: + modqueryOptions.depth = Integer.MAX_VALUE; } } ModqueryExecutor modqueryExecutor = new ModqueryExecutor( moduleInspector.getDepGraph(), + depGraphValue.getExtensionUsagesTable(), + moduleInspector.getExtensionToRepoInternalNames(), + filterExtensions, modqueryOptions, new OutputStreamWriter( env.getReporter().getOutErr().getOutputStream(), @@ -235,124 +418,58 @@ public BlazeCommandResult exec(CommandEnvironment env, OptionsParsingResult opti modqueryExecutor.tree(fromKeys); break; case DEPS: - modqueryExecutor.tree(argsKeysList.get(0)); + modqueryExecutor.tree(argsAsModules); break; case PATH: - modqueryExecutor.path(fromKeys, argsKeysList.get(0)); + modqueryExecutor.path(fromKeys, argsAsModules); break; case ALL_PATHS: - modqueryExecutor.allPaths(fromKeys, argsKeysList.get(0)); - break; case EXPLAIN: - modqueryExecutor.allPaths(fromKeys, argsKeysList.get(0)); + modqueryExecutor.allPaths(fromKeys, argsAsModules); break; case SHOW: - Preconditions.checkArgument(repoRuleValues != null); - modqueryExecutor.show(repoRuleValues); + modqueryExecutor.show(targetRepoRuleValues); + break; + case SHOW_EXTENSION: + modqueryExecutor.showExtension(argsAsExtensions, usageKeys); break; } return BlazeCommandResult.success(); } - /** - * A general parser for an undefined number of arguments. The arguments are comma-separated lists - * of {@link TargetModule}s. Each target {@link TargetModule} can represent a {@code repo_name}, - * as defined in the {@code MODULE.bazel} file of the root project, a specific version of a - * module, or all present versions of a module. The root module can only be specified by the - * {@code root} keyword, which takes precedence over the other above (in case of modules named - * root). - */ - @VisibleForTesting - public static ImmutableList> parseTargetArgs( - int requiredArgNum, - ImmutableMap> modulesIndex, - List args, - ImmutableBiMap rootDeps, - ImmutableBiMap rootUnusedDeps, - boolean includeUnused) - throws OptionsParsingException, InvalidArgumentException { - if (requiredArgNum != args.size()) { - throw new InvalidArgumentException( - String.format( - "Invalid number of arguments (provided %d, required %d).", - args.size(), requiredArgNum), - requiredArgNum > args.size() ? Code.MISSING_ARGUMENTS : Code.TOO_MANY_ARGUMENTS); - } - - TargetModuleListConverter converter = new TargetModuleListConverter(); - ImmutableList.Builder> argsKeysListBuilder = - new ImmutableList.Builder<>(); - - for (String arg : args) { - ImmutableList targetList = converter.convert(arg); - ImmutableSet argModuleKeys = - targetListToModuleKeySet( - targetList, modulesIndex, rootDeps, rootUnusedDeps, includeUnused); - argsKeysListBuilder.add(argModuleKeys); - } - return argsKeysListBuilder.build(); - } - - /** Collects a list of {@link TargetModule} into a set of {@link ModuleKey}s. */ - private static ImmutableSet targetListToModuleKeySet( - ImmutableList targetList, + /** Collects a list of {@link ModuleArg} into a set of {@link ModuleKey}s. */ + private static ImmutableSet moduleArgListToKeys( + ImmutableList argList, ImmutableMap> modulesIndex, - ImmutableBiMap rootDeps, - ImmutableBiMap rootUnusedDeps, + ImmutableMap depGraph, + ImmutableBiMap baseModuleDeps, + ImmutableBiMap baseModuleUnusedDeps, boolean includeUnused) throws InvalidArgumentException { ImmutableSet.Builder allTargetKeys = new ImmutableSet.Builder<>(); - for (TargetModule targetModule : targetList) { + for (ModuleArg moduleArg : argList) { allTargetKeys.addAll( - targetToModuleKeySet( - targetModule, modulesIndex, rootDeps, rootUnusedDeps, includeUnused)); + moduleArg.resolveToModuleKeys( + modulesIndex, depGraph, baseModuleDeps, baseModuleUnusedDeps, includeUnused, true)); } return allTargetKeys.build(); } - /** - * Helper to check the module (and version) of the given {@link TargetModule} exists and retrieve - * it, (or retrieve all present versions if it's semantic specifies it, i.e. when - * Version == null). - */ - private static ImmutableSet targetToModuleKeySet( - TargetModule target, + private static ImmutableSortedSet extensionArgListToIds( + ImmutableList args, ImmutableMap> modulesIndex, - ImmutableBiMap rootDeps, - ImmutableBiMap rootUnusedDeps, - boolean includeUnused) + ImmutableMap depGraph, + ImmutableBiMap baseModuleDeps, + ImmutableBiMap baseModuleUnusedDeps) throws InvalidArgumentException { - if (target.getName().isEmpty() && Objects.equals(target.getVersion(), Version.EMPTY)) { - return ImmutableSet.of(ModuleKey.ROOT); - } - if (rootDeps.containsKey(target.getName())) { - if (includeUnused && rootUnusedDeps.containsKey(target.getName())) { - return ImmutableSet.of( - rootDeps.get(target.getName()), rootUnusedDeps.get(target.getName())); - } - return ImmutableSet.of(rootDeps.get(target.getName())); + ImmutableSortedSet.Builder extensionsBuilder = + new ImmutableSortedSet.Builder<>(ModuleExtensionId.LEXICOGRAPHIC_COMPARATOR); + for (ExtensionArg arg : args) { + extensionsBuilder.add( + arg.resolveToExtensionId(modulesIndex, depGraph, baseModuleDeps, baseModuleUnusedDeps)); } - ImmutableSet existingKeys = modulesIndex.get(target.getName()); - - if (existingKeys == null) { - throw new InvalidArgumentException( - String.format("Module %s does not exist in the dependency graph.", target.getName()), - Code.INVALID_ARGUMENTS); - } - - if (target.getVersion() == null) { - return existingKeys; - } - ModuleKey key = ModuleKey.create(target.getName(), target.getVersion()); - if (!existingKeys.contains(key)) { - throw new InvalidArgumentException( - String.format( - "Module version %s@%s does not exist, available versions: %s.", - target.getName(), key, existingKeys), - Code.INVALID_ARGUMENTS); - } - return ImmutableSet.of(key); + return extensionsBuilder.build(); } private static BlazeCommandResult reportAndCreateFailureResult( @@ -377,22 +494,4 @@ private static BlazeCommandResult createFailureResult(String message, Code detai .setMessage(message) .build())); } - - /** - * Exception thrown when a user-input argument is invalid (wrong number of arguments or the - * specified modules do not exist). - */ - @VisibleForTesting - public static class InvalidArgumentException extends Exception { - private final Code code; - - private InvalidArgumentException(String message, Code code) { - super(message); - this.code = code; - } - - public Code getCode() { - return code; - } - } } diff --git a/src/main/java/com/google/devtools/build/lib/bazel/commands/modquery.txt b/src/main/java/com/google/devtools/build/lib/bazel/commands/modquery.txt index 099b2222cd29d3..8a038dde18b023 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/commands/modquery.txt +++ b/src/main/java/com/google/devtools/build/lib/bazel/commands/modquery.txt @@ -1,32 +1,28 @@ Usage: %{product} %{command} [