diff --git a/internal/fetchclient/fetchclient.go b/internal/fetchclient/fetchclient.go index 0c76a848a..3b1a70095 100644 --- a/internal/fetchclient/fetchclient.go +++ b/internal/fetchclient/fetchclient.go @@ -27,6 +27,8 @@ const ( goProxyURL = "https://proxy.golang.org" npmRegistryURL = "https://registry.npmjs.org" mavenURL = "https://repo1.maven.org/maven2" + // docs: https://packaging.python.org/en/latest/specifications/simple-repository-api/ + pypiURL = "https://pypi.org/simple" ) var ( @@ -36,8 +38,9 @@ var ( // Client is a client used to fetch latest package version. type Client struct { - httpClient *http.Client - ghClient *github.Client + httpClient *http.Client + ghClient *github.Client + pypiBaseURL string } // New returns a new client. @@ -54,8 +57,9 @@ func New(ctx context.Context) *Client { client = retryableClient.StandardClient() } return &Client{ - httpClient: client, - ghClient: github.NewClient(client), + httpClient: client, + ghClient: github.NewClient(client), + pypiBaseURL: pypiURL, } } @@ -103,6 +107,8 @@ func (c *Client) fetch(ctx context.Context, config *source.Config) (string, erro return c.fetchMaven(ctx, config.Source.Maven.Group, config.Source.Maven.Name, ignoreVersions, maxVersion) case config.Source.Crates != nil: return c.fetchCrate(ctx, config.Source.Crates.CrateName, ignoreVersions, maxVersion) + case config.Source.PyPI != nil: + return c.fetchPyPI(ctx, config.Source.PyPI.Name, ignoreVersions, maxVersion) } return "", errors.New("failed to match a source") } @@ -417,6 +423,53 @@ func (c *Client) fetchGithub( return versions[len(versions)-1], nil } +func (c *Client) fetchPyPI(ctx context.Context, name string, ignoreVersions map[string]struct{}, maxVersion string) (string, error) { + request, err := http.NewRequestWithContext( + ctx, + http.MethodGet, + fmt.Sprintf("%s/%s/", c.pypiBaseURL, strings.TrimPrefix(name, "/")), + nil, + ) + if err != nil { + return "", err + } + request.Header.Set("Accept", "application/vnd.pypi.simple.v1+json") + response, err := c.httpClient.Do(request) + if err != nil { + return "", err + } + defer response.Body.Close() + if response.StatusCode != http.StatusOK { + return "", fmt.Errorf("received status code %d retrieving %q", response.StatusCode, request.URL.String()) + } + + var data struct { + Versions []string `json:"versions"` + } + if err := json.NewDecoder(response.Body).Decode(&data); err != nil { + return "", err + } + var versions []string + for _, version := range data.Versions { + v, ok := ensureSemverPrefix(version) + if !ok { + continue + } + if _, ok := ignoreVersions[v]; ok { + continue + } + if maxVersion != "" && semver.Compare(v, maxVersion) >= 0 { + continue + } + versions = append(versions, v) + } + if len(versions) == 0 { + return "", errors.New("no versions found") + } + semver.Sort(versions) + return versions[len(versions)-1], nil +} + // ensureSemverPrefix checks if the given version is valid semver, optionally // prefixing with "v". The output version is not guaranteed to be the same // as input. This function returns false if the version is not valid semver or diff --git a/internal/fetchclient/fetchclient_test.go b/internal/fetchclient/fetchclient_test.go new file mode 100644 index 000000000..8e91d1f62 --- /dev/null +++ b/internal/fetchclient/fetchclient_test.go @@ -0,0 +1,83 @@ +package fetchclient + +import ( + "encoding/json" + "net/http" + "net/http/httptest" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestFetchPyPI(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + versions []string + ignoreVersions map[string]struct{} + maxVersion string + wantVersion string + wantErr string + }{ + { + name: "returns latest semver version", + versions: []string{"3.5.0", "3.6.0", "5.0.0", "1.0"}, + wantVersion: "v5.0.0", + }, + { + name: "skips pre-release versions", + versions: []string{"1.2.5", "2.0.0b7"}, + wantVersion: "v1.2.5", + }, + { + name: "respects ignore_versions", + versions: []string{"3.6.0", "5.0.0"}, + ignoreVersions: map[string]struct{}{"v5.0.0": {}}, + wantVersion: "v3.6.0", + }, + { + name: "respects max_version exclusive upper bound", + versions: []string{"3.6.0", "5.0.0"}, + maxVersion: "v5.0.0", + wantVersion: "v3.6.0", + }, + { + name: "error when no valid versions remain", + versions: []string{"2.0.0b7", "2.0.0rc1"}, + wantErr: "no versions found", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", "application/vnd.pypi.simple.v1+json") + if err := json.NewEncoder(w).Encode(struct { + Versions []string `json:"versions"` + }{Versions: tt.versions}); err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + } + })) + t.Cleanup(srv.Close) + + c := &Client{ + httpClient: srv.Client(), + pypiBaseURL: srv.URL, + } + ignoreVersions := tt.ignoreVersions + if ignoreVersions == nil { + ignoreVersions = map[string]struct{}{} + } + got, err := c.fetchPyPI(t.Context(), "mypy-protobuf", ignoreVersions, tt.maxVersion) + if tt.wantErr != "" { + require.ErrorContains(t, err, tt.wantErr) + return + } + require.NoError(t, err) + assert.Equal(t, tt.wantVersion, got) + }) + } +} diff --git a/internal/source/config.go b/internal/source/config.go index ca90505ef..c08201681 100644 --- a/internal/source/config.go +++ b/internal/source/config.go @@ -45,6 +45,7 @@ type Source struct { NPMRegistry *NPMRegistryConfig `yaml:"npm_registry"` Maven *MavenConfig `yaml:"maven"` Crates *CratesConfig `yaml:"crates"` + PyPI *PyPIConfig `yaml:"pypi"` // IgnoreVersions is a list of versions to ignore when fetching. IgnoreVersions []string `yaml:"ignore_versions"` // MaxVersion is an exclusive upper bound for versions. Versions >= this value will be ignored. @@ -72,6 +73,8 @@ func (s *Source) Name() string { return "maven" case s.Crates != nil: return "crates" + case s.PyPI != nil: + return "pypi" } return "unknown" } @@ -91,6 +94,8 @@ func (s *Source) CacheKey() string { return name + "-" + s.Maven.CacheKey() case s.Crates != nil: return name + "-" + s.Crates.CacheKey() + case s.PyPI != nil: + return name + "-" + s.PyPI.CacheKey() } return name } @@ -162,3 +167,14 @@ var _ Cacheable = (*MavenConfig)(nil) func (m MavenConfig) CacheKey() string { return m.Group + "-" + m.Name } + +// PyPIConfig is the PyPI configuration. +type PyPIConfig struct { + Name string `yaml:"name"` +} + +var _ Cacheable = (*PyPIConfig)(nil) + +func (p PyPIConfig) CacheKey() string { + return p.Name +} diff --git a/internal/source/source_test.go b/internal/source/source_test.go index b97318b45..7f16e0034 100644 --- a/internal/source/source_test.go +++ b/internal/source/source_test.go @@ -15,13 +15,13 @@ func TestGatherSourceFilenames(t *testing.T) { // Walk entire directory with a depth of 1 filenames, err := gatherSourceFilenames("testdata/success") require.NoError(t, err) - assert.Len(t, filenames, 2) + assert.Len(t, filenames, 3) filenames, err = gatherSourceFilenames("testdata/success/connect-go") require.NoError(t, err) assert.Len(t, filenames, 1) filenames, err = gatherSourceFilenames("testdata/success") require.NoError(t, err) - assert.Len(t, filenames, 2) + assert.Len(t, filenames, 3) filenames, err = gatherSourceFilenames("testdata/fail") require.NoError(t, err) @@ -46,7 +46,7 @@ func TestGatherConfigs(t *testing.T) { t.Parallel() configs, err := GatherConfigs("testdata/success") require.NoError(t, err) - assert.Len(t, configs, 2) + assert.Len(t, configs, 3) for _, config := range configs { name := filepath.Base(filepath.Dir(config.Filename)) @@ -63,6 +63,10 @@ func TestGatherConfigs(t *testing.T) { assert.Equal(t, "@bufbuild/protoc-gen-connect-web", source.Name) assert.True(t, config.Source.Disabled) assert.Nil(t, config.Source.DartFlutter) + case "mypy-protobuf": + source := config.Source.PyPI + require.NotNil(t, source) + assert.Equal(t, "mypy-protobuf", source.Name) default: assert.FailNow(t, "unknown plugin name", name) } diff --git a/internal/source/testdata/success/mypy-protobuf/source.yaml b/internal/source/testdata/success/mypy-protobuf/source.yaml new file mode 100644 index 000000000..310def05b --- /dev/null +++ b/internal/source/testdata/success/mypy-protobuf/source.yaml @@ -0,0 +1,3 @@ +source: + pypi: + name: mypy-protobuf diff --git a/plugins/community/danielgtaylor-betterproto/source.yaml b/plugins/community/danielgtaylor-betterproto/source.yaml index f7191eeb3..6a1d9a4a9 100644 --- a/plugins/community/danielgtaylor-betterproto/source.yaml +++ b/plugins/community/danielgtaylor-betterproto/source.yaml @@ -1,4 +1,3 @@ source: - github: - owner: danielgtaylor - repository: python-betterproto + pypi: + name: betterproto diff --git a/plugins/community/nipunn1313-mypy-grpc/source.yaml b/plugins/community/nipunn1313-mypy-grpc/source.yaml index 4b678c641..310def05b 100644 --- a/plugins/community/nipunn1313-mypy-grpc/source.yaml +++ b/plugins/community/nipunn1313-mypy-grpc/source.yaml @@ -1,4 +1,3 @@ source: - github: - owner: nipunn1313 - repository: mypy-protobuf + pypi: + name: mypy-protobuf diff --git a/plugins/community/nipunn1313-mypy/source.yaml b/plugins/community/nipunn1313-mypy/source.yaml index 4b678c641..310def05b 100644 --- a/plugins/community/nipunn1313-mypy/source.yaml +++ b/plugins/community/nipunn1313-mypy/source.yaml @@ -1,4 +1,3 @@ source: - github: - owner: nipunn1313 - repository: mypy-protobuf + pypi: + name: mypy-protobuf diff --git a/plugins/connectrpc/python/source.yaml b/plugins/connectrpc/python/source.yaml index a72834232..0199a43fa 100644 --- a/plugins/connectrpc/python/source.yaml +++ b/plugins/connectrpc/python/source.yaml @@ -1,4 +1,3 @@ source: - github: - owner: connectrpc - repository: connect-python + pypi: + name: protoc-gen-connect-python