feat(gazelle): Add directives for label format & normalisation

CHANGELOG.md

@@ -42,6 +42,11 @@ A brief description of the categories of changes:
 * (bzlmod) Correctly pass `isolated`, `quiet` and `timeout` values to `whl_library`
   and drop the defaults from the lock file.
 
+### Added
+* (gazelle) Added new `python_label_convention` and `python_label_normalization` directives. These directive 
+  allows altering default Gazelle label format to third-party dependencies useful for re-using Gazelle plugin
+  with other rules, including `rules_pycross`. See [#1939](https://github.com/bazelbuild/rules_python/issues/1939).
+
 ### Removed
 * (pip): Removes the `entrypoint` macro that was replaced by `py_console_script_binary` in 0.26.0.
 

gazelle/README.md

@@ -204,7 +204,10 @@ Python-specific directives are as follows:
 | Appends additional visibility labels to each generated target. This directive can be set multiple times. | |
 | [`# gazelle:python_test_file_pattern`](#directive-python_test_file_pattern) | `*_test.py,test_*.py` |
 | Filenames matching these comma-separated `glob`s will be mapped to `py_test` targets. |
-
+| `# gazelle:python_label_convention` | `$distribution_name$` |
+| Defines the format of the distribution name in labels to third-party deps. Useful for using Gazelle plugin with other rules with different repository conventions (e.g. `rules_pycross`). Full label is always prepended with (pip) repository name, e.g. `@pip//numpy`. |
+| `# gazelle:python_label_normalization` | `snake_case` |
+| Controls how distribution names in labels to third-party deps are normalized. Useful for using Gazelle plugin with other rules with different label conventions (e.g. `rules_pycross` uses PEP-503). Can be "snake_case", "none", or "pep503". |
 
 #### Directive: `python_root`:
 

gazelle/python/configure.go

@@ -67,6 +67,8 @@ func (py *Configurer) KnownDirectives() []string {
 		pythonconfig.DefaultVisibilty,
 		pythonconfig.Visibility,
 		pythonconfig.TestFilePattern,
+		pythonconfig.LabelConvention,
+		pythonconfig.LabelNormalization,
 	}
 }
 
@@ -196,6 +198,23 @@ func (py *Configurer) Configure(c *config.Config, rel string, f *rule.File) {
 				}
 			}
 			config.SetTestFilePattern(globStrings)
+		case pythonconfig.LabelConvention:
+			value := strings.TrimSpace(d.Value)
+			if value == "" {
+				log.Fatalf("directive '%s' requires a value", pythonconfig.LabelConvention)
+			}
+			config.SetLabelConvention(value)
+		case pythonconfig.LabelNormalization:
+			switch directiveArg := strings.ToLower(strings.TrimSpace(d.Value)); directiveArg {
+			case "pep503":
+				config.SetLabelNormalization(pythonconfig.Pep503LabelNormalizationType)
+			case "none":
+				config.SetLabelNormalization(pythonconfig.NoLabelNormalizationType)
+			case "snake_case":
+				config.SetLabelNormalization(pythonconfig.SnakeCaseLabelNormalizationType)
+			default:
+				config.SetLabelNormalization(pythonconfig.DefaultLabelNormalizationType)
+			}
 		}
 	}
 

gazelle/python/testdata/annotation_include_dep/__init__.py

@@ -1,5 +1,5 @@
-import module1
 import foo  # third party package
+import module1
 
 # gazelle:include_dep //foo/bar:baz
 # gazelle:include_dep //hello:world,@star_wars//rebel_alliance/luke:skywalker

gazelle/python/testdata/directive_python_label_convention/README.md

@@ -0,0 +1,4 @@
+# Directive: `python_label_convention`
+
+This test case asserts that the `# gazelle:python_label_convention` directive
+works as intended when set.

gazelle/python/testdata/directive_python_label_convention/test1_unset/BUILD.out

@@ -0,0 +1,11 @@
+load("@rules_python//python:defs.bzl", "py_library")
+
+py_library(
+    name = "test1_unset",
+    srcs = ["bar.py"],
+    visibility = ["//:__subpackages__"],
+    deps = [
+        "@gazelle_python_test//google_cloud_aiplatform",
+        "@gazelle_python_test//google_cloud_storage",
+    ],
+)

gazelle/python/testdata/directive_python_label_convention/test1_unset/bar.py

@@ -0,0 +1,6 @@
+from google.cloud import aiplatform, storage
+
+
+def main():
+    a = dir(aiplatform)
+    b = dir(storage)

gazelle/python/testdata/directive_python_label_convention/test1_unset/gazelle_python.yaml

@@ -0,0 +1,6 @@
+manifest:
+  modules_mapping:
+    google.cloud.aiplatform: google_cloud_aiplatform
+    google.cloud.storage: google_cloud_storage
+  pip_repository:
+    name: gazelle_python_test

gazelle/python/testdata/directive_python_label_convention/test2_custom_prefix_colon/BUILD.in

@@ -0,0 +1 @@
+# gazelle:python_label_convention :$distribution_name$

gazelle/python/testdata/directive_python_label_convention/test2_custom_prefix_colon/BUILD.out

@@ -0,0 +1,13 @@
+load("@rules_python//python:defs.bzl", "py_library")
+
+# gazelle:python_label_convention :$distribution_name$
+
+py_library(
+    name = "test2_custom_prefix_colon",
+    srcs = ["bar.py"],
+    visibility = ["//:__subpackages__"],
+    deps = [
+        "@gazelle_python_test//:google_cloud_aiplatform",
+        "@gazelle_python_test//:google_cloud_storage",
+    ],
+)

gazelle/python/testdata/directive_python_label_convention/test2_custom_prefix_colon/bar.py

@@ -0,0 +1,6 @@
+from google.cloud import aiplatform, storage
+
+
+def main():
+    a = dir(aiplatform)
+    b = dir(storage)

gazelle/python/testdata/directive_python_label_convention/test2_custom_prefix_colon/gazelle_python.yaml

@@ -0,0 +1,6 @@
+manifest:
+  modules_mapping:
+    google.cloud.aiplatform: google_cloud_aiplatform
+    google.cloud.storage: google_cloud_storage
+  pip_repository:
+    name: gazelle_python_test

gazelle/python/testdata/directive_python_label_normalization/README.md

@@ -0,0 +1,4 @@
+# Directive: `python_label_normalization`
+
+This test case asserts that the `# gazelle:python_label_normalization` directive
+works as intended when set.

gazelle/python/testdata/directive_python_label_normalization/test1_type_none/BUILD.in

@@ -0,0 +1 @@
+# gazelle:python_label_normalization none

gazelle/python/testdata/directive_python_label_normalization/test1_type_none/BUILD.out

@@ -0,0 +1,10 @@
+load("@rules_python//python:defs.bzl", "py_library")
+
+# gazelle:python_label_normalization none
+
+py_library(
+    name = "test1_type_none",
+    srcs = ["bar.py"],
+    visibility = ["//:__subpackages__"],
+    deps = ["@gazelle_python_test//google.cloud.storage"],
+)

gazelle/python/testdata/directive_python_label_normalization/test1_type_none/bar.py

@@ -0,0 +1,5 @@
+from google.cloud import storage
+
+
+def main():
+    b = dir(storage)

gazelle/python/testdata/directive_python_label_normalization/test1_type_none/gazelle_python.yaml

@@ -0,0 +1,6 @@
+manifest:
+  modules_mapping:
+    # Weird google.cloud.storage here on purpose to make normalization apparent
+    google.cloud.storage: google.cloud.storage
+  pip_repository:
+    name: gazelle_python_test

gazelle/python/testdata/directive_python_label_normalization/test2_type_pep503/BUILD.in

@@ -0,0 +1 @@
+# gazelle:python_label_normalization pep503

gazelle/python/testdata/directive_python_label_normalization/test2_type_pep503/BUILD.out

@@ -0,0 +1,10 @@
+load("@rules_python//python:defs.bzl", "py_library")
+
+# gazelle:python_label_normalization pep503
+
+py_library(
+    name = "test2_type_pep503",
+    srcs = ["bar.py"],
+    visibility = ["//:__subpackages__"],
+    deps = ["@gazelle_python_test//google-cloud-storage"],
+)

gazelle/python/testdata/directive_python_label_normalization/test2_type_pep503/bar.py

@@ -0,0 +1,5 @@
+from google.cloud import storage
+
+
+def main():
+    b = dir(storage)

gazelle/python/testdata/directive_python_label_normalization/test2_type_pep503/gazelle_python.yaml

@@ -0,0 +1,6 @@
+manifest:
+  modules_mapping:
+    # Weird google.cloud.storage here on purpose to make normalization apparent
+    google.cloud.storage: google.cloud.storage
+  pip_repository:
+    name: gazelle_python_test

gazelle/python/testdata/directive_python_label_normalization/test3_type_snake_case/BUILD.in

@@ -0,0 +1 @@
+# gazelle:python_label_normalization snake_case

gazelle/python/testdata/directive_python_label_normalization/test3_type_snake_case/BUILD.out

@@ -0,0 +1,10 @@
+load("@rules_python//python:defs.bzl", "py_library")
+
+# gazelle:python_label_normalization snake_case
+
+py_library(
+    name = "test3_type_snake_case",
+    srcs = ["bar.py"],
+    visibility = ["//:__subpackages__"],
+    deps = ["@gazelle_python_test//google_cloud_storage"],
+)

gazelle/python/testdata/directive_python_label_normalization/test3_type_snake_case/bar.py

@@ -0,0 +1,5 @@
+from google.cloud import storage
+
+
+def main():
+    b = dir(storage)

gazelle/python/testdata/directive_python_label_normalization/test3_type_snake_case/gazelle_python.yaml

@@ -0,0 +1,6 @@
+manifest:
+  modules_mapping:
+    # Weird google.cloud.storage here on purpose to make normalization apparent
+    google.cloud.storage: google.cloud.storage
+  pip_repository:
+    name: gazelle_python_test

gazelle/python/testdata/directive_python_label_normalization/test4_unset_defaults_to_snake_case/BUILD.out

@@ -0,0 +1,8 @@
+load("@rules_python//python:defs.bzl", "py_library")
+
+py_library(
+    name = "test4_unset_defaults_to_snake_case",
+    srcs = ["bar.py"],
+    visibility = ["//:__subpackages__"],
+    deps = ["@gazelle_python_test//google_cloud_storage"],
+)

gazelle/python/testdata/directive_python_label_normalization/test4_unset_defaults_to_snake_case/bar.py

@@ -0,0 +1,5 @@
+from google.cloud import storage
+
+
+def main():
+    b = dir(storage)

gazelle/python/testdata/directive_python_label_normalization/test4_unset_defaults_to_snake_case/gazelle_python.yaml

@@ -0,0 +1,6 @@
+manifest:
+  modules_mapping:
+    # Weird google.cloud.storage here on purpose to make normalization apparent
+    google.cloud.storage: google.cloud.storage
+  pip_repository:
+    name: gazelle_python_test

gazelle/pythonconfig/pythonconfig.go

@@ -17,6 +17,7 @@ package pythonconfig
 import (
 	"fmt"
 	"path"
+	"regexp"
 	"strings"
 
 	"github.com/emirpasic/gods/lists/singlylinkedlist"
@@ -77,6 +78,13 @@ const (
 	// TestFilePattern represents the directive that controls which python
 	// files are mapped to `py_test` targets.
 	TestFilePattern = "python_test_file_pattern"
+	// LabelConvention represents the directive that defines the format of the
+	// labels to third-party dependencies.
+	LabelConvention = "python_label_convention"
+	// LabelNormalization represents the directive that controls how distribution
+	// names of labels to third-party dependencies are normalized. Supported values
+	// are 'none', 'pep503' and 'snake_case' (default). See LabelNormalizationType.
+	LabelNormalization = "python_label_normalization"
 )
 
 // GenerationModeType represents one of the generation modes for the Python
@@ -96,14 +104,19 @@ const (
 )
 
 const (
-	packageNameNamingConventionSubstitution = "$package_name$"
+	packageNameNamingConventionSubstitution     = "$package_name$"
+	distributionNameLabelConventionSubstitution = "$distribution_name$"
 )
 
 const (
 	// The default visibility label, including a format placeholder for `python_root`.
 	DefaultVisibilityFmtString = "//%s:__subpackages__"
 	// The default globs used to determine pt_test targets.
 	DefaultTestFilePatternString = "*_test.py,test_*.py"
+	// The default convention of label of third-party dependencies.
+	DefaultLabelConvention = "$distribution_name$"
+	// The default normalization applied to distribution names of third-party dependency labels.
+	DefaultLabelNormalizationType = SnakeCaseLabelNormalizationType
 )
 
 // defaultIgnoreFiles is the list of default values used in the
@@ -112,14 +125,6 @@ var defaultIgnoreFiles = map[string]struct{}{
 	"setup.py": {},
 }
 
-func SanitizeDistribution(distributionName string) string {
-	sanitizedDistribution := strings.ToLower(distributionName)
-	sanitizedDistribution = strings.ReplaceAll(sanitizedDistribution, "-", "_")
-	sanitizedDistribution = strings.ReplaceAll(sanitizedDistribution, ".", "_")
-
-	return sanitizedDistribution
-}
-
 // Configs is an extension of map[string]*Config. It provides finding methods
 // on top of the mapping.
 type Configs map[string]*Config
@@ -156,8 +161,18 @@ type Config struct {
 	defaultVisibility            []string
 	visibility                   []string
 	testFilePattern              []string
+	labelConvention              string
+	labelNormalization           LabelNormalizationType
 }
 
+type LabelNormalizationType int
+
+const (
+	NoLabelNormalizationType LabelNormalizationType = iota
+	Pep503LabelNormalizationType
+	SnakeCaseLabelNormalizationType
+)
+
 // New creates a new Config.
 func New(
 	repoRoot string,
@@ -180,6 +195,8 @@ func New(
 		defaultVisibility:            []string{fmt.Sprintf(DefaultVisibilityFmtString, "")},
 		visibility:                   []string{},
 		testFilePattern:              strings.Split(DefaultTestFilePatternString, ","),
+		labelConvention:              DefaultLabelConvention,
+		labelNormalization:           DefaultLabelNormalizationType,
 	}
 }
 
@@ -209,6 +226,8 @@ func (c *Config) NewChild() *Config {
 		defaultVisibility:            c.defaultVisibility,
 		visibility:                   c.visibility,
 		testFilePattern:              c.testFilePattern,
+		labelConvention:              c.labelConvention,
+		labelNormalization:           c.labelNormalization,
 	}
 }
 
@@ -263,10 +282,8 @@ func (c *Config) FindThirdPartyDependency(modName string) (string, bool) {
 				} else if gazelleManifest.PipRepository != nil {
 					distributionRepositoryName = gazelleManifest.PipRepository.Name
 				}
-				sanitizedDistribution := SanitizeDistribution(distributionName)
 
-				// @<repository_name>//<distribution_name>
-				lbl := label.New(distributionRepositoryName, sanitizedDistribution, sanitizedDistribution)
+				lbl := currentCfg.FormatThirdPartyDependency(distributionRepositoryName, distributionName)
 				return lbl.String(), true
 			}
 		}
@@ -443,3 +460,48 @@ func (c *Config) SetTestFilePattern(patterns []string) {
 func (c *Config) TestFilePattern() []string {
 	return c.testFilePattern
 }
+
+// SetLabelConvention sets the label convention used for third-party dependencies.
+func (c *Config) SetLabelConvention(convention string) {
+	c.labelConvention = convention
+}
+
+// LabelConvention returns the label convention used for third-party dependencies.
+func (c *Config) LabelConvention() string {
+	return c.labelConvention
+}
+
+// SetLabelConvention sets the label normalization applied to distribution names of third-party dependencies.
+func (c *Config) SetLabelNormalization(normalizationType LabelNormalizationType) {
+	c.labelNormalization = normalizationType
+}
+
+// LabelConvention returns the label normalization applied to distribution names of third-party dependencies.
+func (c *Config) LabelNormalization() LabelNormalizationType {
+	return c.labelNormalization
+}
+
+// FormatThirdPartyDependency returns a label to a third-party dependency performing all formating and normalization.
+func (c *Config) FormatThirdPartyDependency(repositoryName string, distributionName string) label.Label {
+	conventionalDistributionName := strings.ReplaceAll(c.labelConvention, distributionNameLabelConventionSubstitution, distributionName)
+
+	var normConventionalDistributionName string
+	switch norm := c.LabelNormalization(); norm {
+	case SnakeCaseLabelNormalizationType:
+		// See /python/private/normalize_name.bzl
+		normConventionalDistributionName = strings.ToLower(conventionalDistributionName)
+		normConventionalDistributionName = regexp.MustCompile(`[-_.]+`).ReplaceAllString(normConventionalDistributionName, "_")
+		normConventionalDistributionName = strings.Trim(normConventionalDistributionName, "_")
+	case Pep503LabelNormalizationType:
+		// See https://packaging.python.org/en/latest/specifications/name-normalization/#name-format
+		normConventionalDistributionName = strings.ToLower(conventionalDistributionName) // ... "should be lowercased"
+		normConventionalDistributionName = regexp.MustCompile(`[-_.]+`).ReplaceAllString(normConventionalDistributionName, "-") // ... "all runs of the characters ., -, or _ replaced with a single -"
+		normConventionalDistributionName = strings.Trim(normConventionalDistributionName, "-") // ... "must start and end with a letter or number"
+	default:
+		fallthrough
+	case NoLabelNormalizationType:
+		normConventionalDistributionName = conventionalDistributionName
+	}
+
+	return label.New(repositoryName, normConventionalDistributionName, normConventionalDistributionName)
+}