Implement run-time dependencies for binary targets

rules/builtins.build_defs

@@ -3,9 +3,9 @@
 
 # Do not change the order of arguments to this function without updating the iota in targets.go to match it.
 def build_rule(name:str, cmd:str|dict='', test_cmd:str|dict='', debug_cmd:str='', srcs:list|dict=None, data:list|dict=None,
-               debug_data:list|dict=None, outs:list|dict=None, deps:list=None, exported_deps:list=None, secrets:list|dict=None,
-               tools:str|list|dict=None, test_tools:str|list|dict=None, debug_tools:str|list|dict=None, labels:list=None,
-               visibility:list=CONFIG.DEFAULT_VISIBILITY, hashes:list=None, binary:bool=False, test:bool=False,
+               debug_data:list|dict=None, outs:list|dict=None, deps:list=None, exported_deps:list=None, runtime_deps:list=None,
+               secrets:list|dict=None, tools:str|list|dict=None, test_tools:str|list|dict=None, debug_tools:str|list|dict=None,
+               labels:list=None, visibility:list=CONFIG.DEFAULT_VISIBILITY, hashes:list=None, binary:bool=False, test:bool=False,
                test_only:bool=CONFIG.DEFAULT_TESTONLY, building_description:str=None, needs_transitive_deps:bool=False,
                output_is_complete:bool=False, sandbox:bool=CONFIG.BUILD_SANDBOX, test_sandbox:bool=CONFIG.TEST_SANDBOX,
                no_test_output:bool=False, flaky:bool|int=0, build_timeout:int|str=0, test_timeout:int|str=0, pre_build:function=None,
@@ -249,10 +249,12 @@ def has_label(name:str, prefix:str, all:bool=False) -> bool:
     return len(get_labels(name, prefix, all)) > 0
 def add_label(name:str, label:str):
     pass
-def add_dep(target:str, dep:str, exported:bool=False):
+def add_dep(target:str, dep:str, exported:bool=False, runtime:bool=False):
     pass
 def add_exported_dep(target:str, dep:str):
-    add_dep(target, dep, True)
+    add_dep(target, dep, exported=True)
+def add_runtime_dep(target:str, dep:str):
+    add_dep(target, dep, runtime=True)
 def add_data(target:str, datum:str|list|dict):
     pass
 def add_out(target:str, name:str, out:str=''):

rules/misc_rules.build_defs

@@ -2,7 +2,7 @@
 
 
 def genrule(name:str, cmd:str|list|dict, srcs:list|dict=None, out:str=None, outs:list|dict=None, deps:list=None,
-            exported_deps:list=None, labels:list&features&tags=None, visibility:list=None,
+            exported_deps:list=None, runtime_deps:list=None, labels:list&features&tags=None, visibility:list=None,
             building_description:str='Building...', data:list|dict=None, hashes:list=None, timeout:int=0, binary:bool=False,
             sandbox:bool=None, needs_transitive_deps:bool=False, output_is_complete:bool=True,
             test_only:bool&testonly=False, secrets:list|dict=None, requires:list=None, provides:dict=None,
@@ -34,6 +34,13 @@ def genrule(name:str, cmd:str|list|dict, srcs:list|dict=None, out:str=None, outs
                           names to lists, with similar semantics to those of srcs.
       deps (list): Dependencies of this rule.
       exported_deps (list): Dependencies that will become visible to any rules that depend on this rule.
+      runtime_deps (list): Run-time dependencies of this rule. If this rule is run (i.e. with 'plz run'),
+                           rules in this list, as well as those rules' transitive run-time dependencies,
+                           are guaranteed to be built before this rule runs. If this rule is declared as
+                           a dependency of another rule, the outputs of rules in this list, as well as
+                           the outputs of those rules' transitive run-time dependencies, will exist in
+                           the dependent rule's build environment. Requires the rule to produce a runnable
+                           output (i.e. binary = True).
       tools (str | list | dict): Tools used to build this rule; similar to srcs but are not copied to the
                            temporary build directory. Should be accessed via $(exe //path/to:tool)
                            or similar.
@@ -119,6 +126,7 @@ def genrule(name:str, cmd:str|list|dict, srcs:list|dict=None, out:str=None, outs
         cmd = ' && '.join(cmd) if isinstance(cmd, list) else cmd,
         deps = deps,
         exported_deps = exported_deps,
+        runtime_deps = runtime_deps,
         data = data,
         tools = tools,
         secrets = secrets,
@@ -147,11 +155,12 @@ def genrule(name:str, cmd:str|list|dict, srcs:list|dict=None, out:str=None, outs
 
 
 def gentest(name:str, test_cmd:str|list|dict, labels:list&features&tags=None, cmd:str|list|dict=None, srcs:list|dict=None,
-            outs:list=None, deps:list=None, exported_deps:list=None, tools:str|list|dict=None, test_tools:str|list|dict=None,
-            data:list|dict=None, visibility:list=None, timeout:int=0, needs_transitive_deps:bool=False,
-            flaky:bool|int=0, secrets:list|dict=None, no_test_output:bool=False, test_outputs:list=None,
-            output_is_complete:bool=True, requires:list=None, sandbox:bool=None, size:str=None, local:bool=False,
-            pass_env:list=None, env:dict=None, exit_on_error:bool=CONFIG.EXIT_ON_ERROR, no_test_coverage:bool=False):
+            outs:list=None, deps:list=None, exported_deps:list=None, runtime_deps:list=None, tools:str|list|dict=None,
+            test_tools:str|list|dict=None, data:list|dict=None, visibility:list=None, timeout:int=0,
+            needs_transitive_deps:bool=False, flaky:bool|int=0, secrets:list|dict=None, no_test_output:bool=False,
+            test_outputs:list=None, output_is_complete:bool=True, requires:list=None, sandbox:bool=None, size:str=None,
+            local:bool=False, pass_env:list=None, env:dict=None, exit_on_error:bool=CONFIG.EXIT_ON_ERROR,
+            no_test_coverage:bool=False):
     """A rule which creates a test with an arbitrary command.
 
     The command must return zero on success and nonzero on failure. Test results are written
@@ -172,6 +181,10 @@ def gentest(name:str, test_cmd:str|list|dict, labels:list&features&tags=None, cm
       outs (list): Output files of this rule.
       deps (list): Dependencies of this rule.
       exported_deps (list): Dependencies that will become visible to any rules that depend on this rule.
+      runtime_deps (list): Run-time dependencies of this rule. When the test command runs, rules in this
+                           list, as well as those rules' transitive run-time dependencies, will exist in
+                           the test environment. Requires the rule to produce a runnable output (i.e.
+                           binary = True).
       tools (str | list | dict): Tools used to build this rule; similar to srcs but are not copied to the temporary
                     build directory.
       test_tools (str | list | dict): Like tools but available to test_cmd instead.
@@ -210,6 +223,7 @@ def gentest(name:str, test_cmd:str|list|dict, labels:list&features&tags=None, cm
         outs = outs,
         deps = deps,
         exported_deps = exported_deps,
+        runtime_deps = runtime_deps,
         data = data,
         tools = tools,
         test_tools = test_tools,
@@ -256,7 +270,7 @@ def export_file(name:str, src:str, visibility:list=None, binary:bool=False, test
     )
 
 
-def filegroup(name:str, tag:str='', srcs:list=None, deps:list=None, exported_deps:list=None,
+def filegroup(name:str, tag:str='', srcs:list=None, deps:list=None, exported_deps:list=None, runtime_deps:list=None,
               visibility:list=None, labels:list&features&tags=None, binary:bool=False, output_is_complete:bool=True,
               requires:list=None, provides:dict=None, hashes:list=None, test_only:bool&testonly=False):
     """Defines a collection of files which other rules can depend on.
@@ -271,6 +285,8 @@ def filegroup(name:str, tag:str='', srcs:list=None, deps:list=None, exported_dep
       srcs (list): Source files for the rule.
       deps (list): Dependencies of the rule.
       exported_deps (list): Dependencies that will become visible to any rules that depend on this rule.
+      runtime_deps (list): Run-time dependencies of this rule. Requires the rule to produce a runnable
+                           output (i.e. binary = True).
       visibility (list): Visibility declaration
       labels (list): Labels to apply to this rule
       binary (bool): True to mark the rule outputs as binary
@@ -289,6 +305,7 @@ def filegroup(name:str, tag:str='', srcs:list=None, deps:list=None, exported_dep
         srcs=srcs,
         deps=deps,
         exported_deps=exported_deps,
+        runtime_deps=runtime_deps,
         visibility=visibility,
         building_description='Copying...',
         output_is_complete=output_is_complete,
@@ -373,7 +390,7 @@ def system_library(name:str, srcs:list, deps:list=None, hashes:list=None,
 
 def remote_file(name:str, url:str|list, hashes:list=None, out:str=None, binary:bool=False,
                 visibility:list=None, licences:list=None, test_only:bool&testonly=False,
-                labels:list=[], deps:list=None, exported_deps:list=None,
+                labels:list=[], deps:list=None, exported_deps:list=None, runtime_deps:list=None,
                 extract:bool=False, strip_prefix:str='', _tag:str='',exported_files=[],
                 entry_points:dict={}, username:str=None, password_file:str=None,
                 headers:dict={}, secret_headers:dict={}, pass_env:list=[]):
@@ -392,6 +409,8 @@ def remote_file(name:str, url:str|list, hashes:list=None, out:str=None, binary:b
       labels (list): Labels to apply to this rule.
       deps (list): List of extra dependencies for this rule.
       exported_deps (list): Dependencies that will become visible to any rules that depend on this rule.
+      runtime_deps (list): Run-time dependencies of this rule. Requires the rule to produce a runnable
+                           output (i.e. binary = True).
       extract (bool): Extracts the contents of the downloaded file. It must be either zip or
                       tar format.
       strip_prefix (str): When extracting, strip this prefix from the extracted files.
@@ -454,6 +473,7 @@ def remote_file(name:str, url:str|list, hashes:list=None, out:str=None, binary:b
             building_description = 'Extracting...',
             deps = deps,
             exported_deps = exported_deps,
+            runtime_deps = runtime_deps,
             entry_points = entry_points,
         )
 
@@ -484,6 +504,7 @@ def remote_file(name:str, url:str|list, hashes:list=None, out:str=None, binary:b
         building_description = 'Fetching...',
         deps = deps,
         exported_deps = exported_deps,
+        runtime_deps = runtime_deps,
         test_only = test_only,
         labels = labels,
         sandbox = False,

src/build/incrementality_test.go

@@ -81,9 +81,10 @@ var KnownFields = map[string]bool{
 	"Debug.namedTools": true,
 
 	// These only contribute to the runtime hash, not at build time.
-	"Data":              true,
-	"NamedData":         true,
-	"ContainerSettings": true,
+	"runtimeDependencies": true,
+	"Data":                true,
+	"NamedData":           true,
+	"ContainerSettings":   true,
 
 	// These would ideally not contribute to the hash, but we need that at present
 	// because we don't have a good way to force a recheck of its reverse dependencies.

src/core/build_target.go

@@ -115,6 +115,8 @@ type BuildTarget struct {
 	// Maps the original declaration to whatever dependencies actually got attached,
 	// which may be more than one in some cases. Also contains info about exporting etc.
 	dependencies []depInfo `name:"deps"`
+	// The run-time dependencies of this target.
+	runtimeDependencies []BuildLabel `name:"runtime_deps"`
 	// List of build target patterns that can use this build target.
 	Visibility []BuildLabel
 	// Source files of this rule. Can refer to build rules themselves.
@@ -307,6 +309,7 @@ type depInfo struct {
 	resolved bool           // has the graph resolved it
 	exported bool           // is it an exported dependency
 	internal bool           // is it an internal dependency (that is not picked up implicitly by transitive searches)
+	runtime  bool           // is it a run-time (and therefore implicitly transitive) dependency
 	source   bool           // is it implicit because it's a source (not true if it's a dependency too)
 	data     bool           // is it a data item for a test
 }
@@ -632,7 +635,7 @@ func (target *BuildTarget) DeclaredDependenciesStrict() []BuildLabel {
 	defer target.mutex.RUnlock()
 	ret := make(BuildLabels, 0, len(target.dependencies))
 	for _, dep := range target.dependencies {
-		if !dep.exported && !dep.source && !target.IsTool(*dep.declared) {
+		if !dep.runtime && !dep.exported && !dep.source && !target.IsTool(*dep.declared) {
 			ret = append(ret, *dep.declared)
 		}
 	}
@@ -672,13 +675,13 @@ func (target *BuildTarget) ExternalDependencies() []*BuildTarget {
 	return ret
 }
 
-// BuildDependencies returns the build-time dependencies of this target (i.e. not data, internal nor source).
+// BuildDependencies returns the build-time dependencies of this target (i.e. not run-time dependencies, data, internal nor source).
 func (target *BuildTarget) BuildDependencies() []*BuildTarget {
 	target.mutex.RLock()
 	defer target.mutex.RUnlock()
 	ret := make(BuildTargets, 0, len(target.dependencies))
 	for _, deps := range target.dependencies {
-		if !deps.data && !deps.internal && !deps.source {
+		if !deps.runtime && !deps.data && !deps.internal && !deps.source {
 			for _, dep := range deps.deps {
 				ret = append(ret, dep)
 			}
@@ -701,6 +704,52 @@ func (target *BuildTarget) ExportedDependencies() []BuildLabel {
 	return ret
 }
 
+// RuntimeDependencies returns any run-time dependencies of this target.
+//
+// Although run-time dependencies are transitive, RuntimeDependencies only returns this target's direct run-time
+// dependencies. Use IterAllRuntimeDependencies to iterate over the target's run-time dependencies transitively.
+func (target *BuildTarget) RuntimeDependencies() []BuildLabel {
+	target.mutex.RLock()
+	defer target.mutex.RUnlock()
+	ret := make(BuildLabels, 0, len(target.dependencies))
+	for _, deps := range target.dependencies {
+		if deps.runtime {
+			ret = append(ret, *deps.declared)
+		}
+	}
+	return ret
+}
+
+// IterAllRuntimeDependencies returns an iterator over the transitive run-time dependencies of this target.
+// Require/provide relationships between pairs of targets are resolved as they are with build-time dependencies.
+func (target *BuildTarget) IterAllRuntimeDependencies(graph *BuildGraph) iter.Seq[BuildLabel] {
+	var (
+		push func(*BuildTarget, func(BuildLabel) bool) bool
+		done = make(map[string]bool)
+	)
+	push = func(t *BuildTarget, yield func(BuildLabel) bool) bool {
+		if done[t.String()] {
+			return true
+		}
+		done[t.String()] = true
+		for _, runDep := range t.runtimeDependencies {
+			runDepLabel, _ := runDep.Label()
+			for _, providedDep := range graph.TargetOrDie(runDepLabel).ProvideFor(t) {
+				if !yield(providedDep) {
+					return false
+				}
+				if !push(graph.TargetOrDie(providedDep), yield) {
+					return false
+				}
+			}
+		}
+		return true
+	}
+	return func(yield func(BuildLabel) bool) {
+		push(target, yield)
+	}
+}
+
 // DependenciesFor returns the dependencies that relate to a given label.
 func (target *BuildTarget) DependenciesFor(label BuildLabel) []*BuildTarget {
 	target.mutex.RLock()
@@ -1287,7 +1336,7 @@ func (target *BuildTarget) addSource(sources []BuildInput, source BuildInput) []
 	}
 	// Add a dependency if this is not just a file.
 	if label, ok := source.Label(); ok {
-		target.AddMaybeExportedDependency(label, false, true, false)
+		target.AddMaybeExportedDependency(label, false, true, false, false)
 	}
 	return append(sources, source)
 }
@@ -1653,7 +1702,7 @@ func (target *BuildTarget) AllNamedTools() map[string][]BuildInput {
 
 // AddDependency adds a dependency to this target. It deduplicates against any existing deps.
 func (target *BuildTarget) AddDependency(dep BuildLabel) {
-	target.AddMaybeExportedDependency(dep, false, false, false)
+	target.AddMaybeExportedDependency(dep, false, false, false, false)
 }
 
 // HintDependencies allocates space for at least the given number of dependencies without reallocating.
@@ -1662,17 +1711,30 @@ func (target *BuildTarget) HintDependencies(n int) {
 }
 
 // AddMaybeExportedDependency adds a dependency to this target which may be exported. It deduplicates against any existing deps.
-func (target *BuildTarget) AddMaybeExportedDependency(dep BuildLabel, exported, source, internal bool) {
+func (target *BuildTarget) AddMaybeExportedDependency(dep BuildLabel, exported, source, internal, runtime bool) {
 	if dep == target.Label {
 		log.Fatalf("Attempted to add %s as a dependency of itself.\n", dep)
 	}
+	if runtime {
+		if !target.IsBinary {
+			log.Fatalf("%s: output must be marked as binary to have run-time dependencies", target.String())
+		}
+		target.runtimeDependencies = append(target.runtimeDependencies, dep)
+	}
 	info := target.dependencyInfo(dep)
 	if info == nil {
-		target.dependencies = append(target.dependencies, depInfo{declared: &dep, exported: exported, source: source, internal: internal})
+		target.dependencies = append(target.dependencies, depInfo{
+			declared: &dep,
+			exported: exported,
+			source:   source,
+			internal: internal,
+			runtime:  runtime,
+		})
 	} else {
 		info.exported = info.exported || exported
 		info.source = info.source && source
 		info.internal = info.internal && internal
+		info.runtime = info.runtime && runtime
 		info.data = false // It's not *only* data any more.
 	}
 }