Add PyPI source type and migrate pip-installed Python plugins

internal/fetchclient/fetchclient.go

@@ -27,6 +27,8 @@ const (
 	goProxyURL        = "https://proxy.golang.org"
 	npmRegistryURL    = "https://registry.npmjs.org"
 	mavenURL          = "https://repo1.maven.org/maven2"
+	// docs: https://packaging.python.org/en/latest/specifications/simple-repository-api/
+	pypiURL = "https://pypi.org/simple"
 )
 
 var (
@@ -36,8 +38,9 @@ var (
 
 // Client is a client used to fetch latest package version.
 type Client struct {
-	httpClient *http.Client
-	ghClient   *github.Client
+	httpClient  *http.Client
+	ghClient    *github.Client
+	pypiBaseURL string
 }
 
 // New returns a new client.
@@ -54,8 +57,9 @@ func New(ctx context.Context) *Client {
 		client = retryableClient.StandardClient()
 	}
 	return &Client{
-		httpClient: client,
-		ghClient:   github.NewClient(client),
+		httpClient:  client,
+		ghClient:    github.NewClient(client),
+		pypiBaseURL: pypiURL,
 	}
 }
 
@@ -103,6 +107,8 @@ func (c *Client) fetch(ctx context.Context, config *source.Config) (string, erro
 		return c.fetchMaven(ctx, config.Source.Maven.Group, config.Source.Maven.Name, ignoreVersions, maxVersion)
 	case config.Source.Crates != nil:
 		return c.fetchCrate(ctx, config.Source.Crates.CrateName, ignoreVersions, maxVersion)
+	case config.Source.PyPI != nil:
+		return c.fetchPyPI(ctx, config.Source.PyPI.Name, ignoreVersions, maxVersion)
 	}
 	return "", errors.New("failed to match a source")
 }
@@ -417,6 +423,53 @@ func (c *Client) fetchGithub(
 	return versions[len(versions)-1], nil
 }
 
+func (c *Client) fetchPyPI(ctx context.Context, name string, ignoreVersions map[string]struct{}, maxVersion string) (string, error) {
+	request, err := http.NewRequestWithContext(
+		ctx,
+		http.MethodGet,
+		fmt.Sprintf("%s/%s/", c.pypiBaseURL, strings.TrimPrefix(name, "/")),
+		nil,
+	)
+	if err != nil {
+		return "", err
+	}
+	request.Header.Set("Accept", "application/vnd.pypi.simple.v1+json")
+	response, err := c.httpClient.Do(request)
+	if err != nil {
+		return "", err
+	}
+	defer response.Body.Close()
+	if response.StatusCode != http.StatusOK {
+		return "", fmt.Errorf("received status code %d retrieving %q", response.StatusCode, request.URL.String())
+	}
+
+	var data struct {
+		Versions []string `json:"versions"`
+	}
+	if err := json.NewDecoder(response.Body).Decode(&data); err != nil {
+		return "", err
+	}
+	var versions []string
+	for _, version := range data.Versions {
+		v, ok := ensureSemverPrefix(version)
+		if !ok {
+			continue
+		}
+		if _, ok := ignoreVersions[v]; ok {
+			continue
+		}
+		if maxVersion != "" && semver.Compare(v, maxVersion) >= 0 {
+			continue
+		}
+		versions = append(versions, v)
+	}
+	if len(versions) == 0 {
+		return "", errors.New("no versions found")
+	}
+	semver.Sort(versions)
+	return versions[len(versions)-1], nil
+}
+
 // ensureSemverPrefix checks if the given version is valid semver, optionally
 // prefixing with "v". The output version is not guaranteed to be the same
 // as input. This function returns false if the version is not valid semver or

internal/fetchclient/fetchclient_test.go

@@ -0,0 +1,83 @@
+package fetchclient
+
+import (
+	"encoding/json"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestFetchPyPI(t *testing.T) {
+	t.Parallel()
+
+	tests := []struct {
+		name           string
+		versions       []string
+		ignoreVersions map[string]struct{}
+		maxVersion     string
+		wantVersion    string
+		wantErr        string
+	}{
+		{
+			name:        "returns latest semver version",
+			versions:    []string{"3.5.0", "3.6.0", "5.0.0", "1.0"},
+			wantVersion: "v5.0.0",
+		},
+		{
+			name:        "skips pre-release versions",
+			versions:    []string{"1.2.5", "2.0.0b7"},
+			wantVersion: "v1.2.5",
+		},
+		{
+			name:           "respects ignore_versions",
+			versions:       []string{"3.6.0", "5.0.0"},
+			ignoreVersions: map[string]struct{}{"v5.0.0": {}},
+			wantVersion:    "v3.6.0",
+		},
+		{
+			name:        "respects max_version exclusive upper bound",
+			versions:    []string{"3.6.0", "5.0.0"},
+			maxVersion:  "v5.0.0",
+			wantVersion: "v3.6.0",
+		},
+		{
+			name:     "error when no valid versions remain",
+			versions: []string{"2.0.0b7", "2.0.0rc1"},
+			wantErr:  "no versions found",
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			t.Parallel()
+			srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+				w.Header().Set("Content-Type", "application/vnd.pypi.simple.v1+json")
+				if err := json.NewEncoder(w).Encode(struct {
+					Versions []string `json:"versions"`
+				}{Versions: tt.versions}); err != nil {
+					http.Error(w, err.Error(), http.StatusInternalServerError)
+				}
+			}))
+			t.Cleanup(srv.Close)
+
+			c := &Client{
+				httpClient:  srv.Client(),
+				pypiBaseURL: srv.URL,
+			}
+			ignoreVersions := tt.ignoreVersions
+			if ignoreVersions == nil {
+				ignoreVersions = map[string]struct{}{}
+			}
+			got, err := c.fetchPyPI(t.Context(), "mypy-protobuf", ignoreVersions, tt.maxVersion)
+			if tt.wantErr != "" {
+				require.ErrorContains(t, err, tt.wantErr)
+				return
+			}
+			require.NoError(t, err)
+			assert.Equal(t, tt.wantVersion, got)
+		})
+	}
+}

internal/source/config.go

@@ -45,6 +45,7 @@ type Source struct {
 	NPMRegistry *NPMRegistryConfig `yaml:"npm_registry"`
 	Maven       *MavenConfig       `yaml:"maven"`
 	Crates      *CratesConfig      `yaml:"crates"`
+	PyPI        *PyPIConfig        `yaml:"pypi"`
 	// IgnoreVersions is a list of versions to ignore when fetching.
 	IgnoreVersions []string `yaml:"ignore_versions"`
 	// MaxVersion is an exclusive upper bound for versions. Versions >= this value will be ignored.
@@ -72,6 +73,8 @@ func (s *Source) Name() string {
 		return "maven"
 	case s.Crates != nil:
 		return "crates"
+	case s.PyPI != nil:
+		return "pypi"
 	}
 	return "unknown"
 }
@@ -91,6 +94,8 @@ func (s *Source) CacheKey() string {
 		return name + "-" + s.Maven.CacheKey()
 	case s.Crates != nil:
 		return name + "-" + s.Crates.CacheKey()
+	case s.PyPI != nil:
+		return name + "-" + s.PyPI.CacheKey()
 	}
 	return name
 }
@@ -162,3 +167,14 @@ var _ Cacheable = (*MavenConfig)(nil)
 func (m MavenConfig) CacheKey() string {
 	return m.Group + "-" + m.Name
 }
+
+// PyPIConfig is the PyPI configuration.
+type PyPIConfig struct {
+	Name string `yaml:"name"`
+}
+
+var _ Cacheable = (*PyPIConfig)(nil)
+
+func (p PyPIConfig) CacheKey() string {
+	return p.Name
+}

internal/source/source_test.go

@@ -15,13 +15,13 @@ func TestGatherSourceFilenames(t *testing.T) {
 	// Walk entire directory with a depth of 1
 	filenames, err := gatherSourceFilenames("testdata/success")
 	require.NoError(t, err)
-	assert.Len(t, filenames, 2)
+	assert.Len(t, filenames, 3)
 	filenames, err = gatherSourceFilenames("testdata/success/connect-go")
 	require.NoError(t, err)
 	assert.Len(t, filenames, 1)
 	filenames, err = gatherSourceFilenames("testdata/success")
 	require.NoError(t, err)
-	assert.Len(t, filenames, 2)
+	assert.Len(t, filenames, 3)
 
 	filenames, err = gatherSourceFilenames("testdata/fail")
 	require.NoError(t, err)
@@ -46,7 +46,7 @@ func TestGatherConfigs(t *testing.T) {
 	t.Parallel()
 	configs, err := GatherConfigs("testdata/success")
 	require.NoError(t, err)
-	assert.Len(t, configs, 2)
+	assert.Len(t, configs, 3)
 
 	for _, config := range configs {
 		name := filepath.Base(filepath.Dir(config.Filename))
@@ -63,6 +63,10 @@ func TestGatherConfigs(t *testing.T) {
 			assert.Equal(t, "@bufbuild/protoc-gen-connect-web", source.Name)
 			assert.True(t, config.Source.Disabled)
 			assert.Nil(t, config.Source.DartFlutter)
+		case "mypy-protobuf":
+			source := config.Source.PyPI
+			require.NotNil(t, source)
+			assert.Equal(t, "mypy-protobuf", source.Name)
 		default:
 			assert.FailNow(t, "unknown plugin name", name)
 		}

internal/source/testdata/success/mypy-protobuf/source.yaml

@@ -0,0 +1,3 @@
+source:
+  pypi:
+    name: mypy-protobuf

plugins/community/danielgtaylor-betterproto/source.yaml

@@ -1,4 +1,3 @@
 source:
-  github:
-    owner: danielgtaylor
-    repository: python-betterproto
+  pypi:
+    name: betterproto

plugins/community/nipunn1313-mypy-grpc/source.yaml

@@ -1,4 +1,3 @@
 source:
-  github:
-    owner: nipunn1313
-    repository: mypy-protobuf
+  pypi:
+    name: mypy-protobuf

plugins/community/nipunn1313-mypy/source.yaml

@@ -1,4 +1,3 @@
 source:
-  github:
-    owner: nipunn1313
-    repository: mypy-protobuf
+  pypi:
+    name: mypy-protobuf

plugins/connectrpc/python/source.yaml

@@ -1,4 +1,3 @@
 source:
-  github:
-    owner: connectrpc
-    repository: connect-python
+  pypi:
+    name: protoc-gen-connect-python