From 2947f8411e8f83e429d850a2a696f86cdb98d0b7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Serdar=20=C3=96zer?= Date: Fri, 19 Dec 2025 16:10:50 +0100 Subject: [PATCH 1/9] feat: some commands are implemented - ensure-storage-exists - list - delete-recursive --- gcs/client/client.go | 101 +++++++++++++++++++++++++++++++++++++++++-- gcs/client/sdk.go | 37 ++++++++++++++++ 2 files changed, 134 insertions(+), 4 deletions(-) diff --git a/gcs/client/client.go b/gcs/client/client.go index dea5424..b7e94ba 100644 --- a/gcs/client/client.go +++ b/gcs/client/client.go @@ -24,9 +24,11 @@ import ( "log" "os" "strings" + "sync" "time" "golang.org/x/oauth2/google" + "google.golang.org/api/iterator" "cloud.google.com/go/storage" @@ -42,6 +44,7 @@ type GCSBlobstore struct { authenticatedGCS *storage.Client publicGCS *storage.Client config *config.GCSCli + projectID string } // validateRemoteConfig determines if the configuration of the client matches @@ -68,6 +71,11 @@ func (client *GCSBlobstore) getObjectHandle(gcs *storage.Client, src string) *st return handle } +func (client *GCSBlobstore) getBucketHandle(gcs *storage.Client) *storage.BucketHandle { + handle := gcs.Bucket(client.config.BucketName) + return handle +} + // New returns a GCSBlobstore configured to operate using the given config // // non-nil error is returned on invalid Client or config. If the configuration @@ -82,7 +90,12 @@ func New(ctx context.Context, cfg *config.GCSCli) (*GCSBlobstore, error) { return nil, fmt.Errorf("creating storage client: %v", err) } - return &GCSBlobstore{authenticatedGCS: authenticatedGCS, publicGCS: publicGCS, config: cfg}, nil + projectID, err := extractProjectID(ctx, cfg) + if err != nil { + return nil, fmt.Errorf("extracting project ID: %v", err) + } + + return &GCSBlobstore{authenticatedGCS: authenticatedGCS, publicGCS: publicGCS, config: cfg, projectID: projectID}, nil } // Get fetches a blob from the GCS blobstore. @@ -246,7 +259,30 @@ func (client *GCSBlobstore) Sign(id string, action string, expiry time.Duration) } func (client *GCSBlobstore) List(prefix string) ([]string, error) { - return nil, errors.New("not implemented") + if client.readOnly() { + return nil, ErrInvalidROWriteOperation + } + + bh := client.getBucketHandle(client.authenticatedGCS) + + it := bh.Objects(context.Background(), &storage.Query{Prefix: prefix}) + + var names []string + for { + attr, err := it.Next() + if err == iterator.Done { + break + } + + if err != nil { + return nil, err + } + + names = append(names, attr.Name) + } + + return names, nil + } func (client *GCSBlobstore) Copy(srcBlob string, dstBlob string) error { @@ -258,9 +294,66 @@ func (client *GCSBlobstore) Properties(dest string) error { } func (client *GCSBlobstore) EnsureStorageExists() error { - return errors.New("not implemented") + if client.readOnly() { + return ErrInvalidROWriteOperation + } + ctx := context.Background() + bh := client.getBucketHandle(client.authenticatedGCS) + + _, err := bh.Attrs(ctx) + if err == storage.ErrBucketNotExist { + + battr := &storage.BucketAttrs{Name: client.config.BucketName} + if client.config.StorageClass != "" { + battr.StorageClass = client.config.StorageClass + } + err = bh.Create(ctx, client.projectID, battr) + if err != nil { + return fmt.Errorf("creating bucket: %w", err) + } + } + if err != nil { + return fmt.Errorf("checking bucket: %w", err) + } + + return nil } func (client *GCSBlobstore) DeleteRecursive(prefix string) error { - return errors.New("not implemented") + if client.readOnly() { + return ErrInvalidROWriteOperation + } + + names, err := client.List(prefix) + if err != nil { + return fmt.Errorf("listing objects: %w", err) + } + + errChan := make(chan error, len(names)) + wg := &sync.WaitGroup{} + for _, n := range names { + name := n + wg.Add(1) + go func() { + defer wg.Done() + err := client.getObjectHandle(client.authenticatedGCS, name).Delete(context.Background()) + if err != nil && !errors.Is(err, storage.ErrObjectNotExist) { + errChan <- fmt.Errorf("deleting object %s: %w", name, err) + } + }() + } + + wg.Wait() + close(errChan) + + var errs []error + for err := range errChan { + errs = append(errs, err) + } + + if len(errs) > 0 { + return errors.Join(errs...) + } + + return nil } diff --git a/gcs/client/sdk.go b/gcs/client/sdk.go index 77c2249..28ce260 100644 --- a/gcs/client/sdk.go +++ b/gcs/client/sdk.go @@ -18,7 +18,9 @@ package client import ( "context" + "encoding/json" "errors" + "fmt" "golang.org/x/oauth2/google" @@ -52,3 +54,38 @@ func newStorageClients(ctx context.Context, cfg *config.GCSCli) (*storage.Client } return authenticatedClient, publicClient, err } + +// extractProjectID extracts the GCP project ID from credentials +func extractProjectID(ctx context.Context, cfg *config.GCSCli) (string, error) { + switch cfg.CredentialsSource { + case config.ServiceAccountFileCredentialsSource: + // Parse service account JSON to extract project_id + var serviceAccount struct { + ProjectID string `json:"project_id"` + } + if err := json.Unmarshal([]byte(cfg.ServiceAccountFile), &serviceAccount); err != nil { + return "", fmt.Errorf("parsing service account JSON: %w", err) + } + if serviceAccount.ProjectID == "" { + return "", errors.New("project_id not found in service account JSON") + } + return serviceAccount.ProjectID, nil + + case config.DefaultCredentialsSource: + // Try to get project ID from default credentials + creds, err := google.FindDefaultCredentials(ctx, storage.ScopeFullControl) + if err != nil { + return "", fmt.Errorf("finding default credentials: %w", err) + } + if creds.ProjectID == "" { + return "", errors.New("project_id not found in default credentials") + } + return creds.ProjectID, nil + + case config.NoneCredentialsSource: + return "", errors.New("cannot create bucket with read-only credentials") + + default: + return "", errors.New("unknown credentials_source") + } +} From d9675316174fc01de66a6a748570a76da704fe52 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Serdar=20=C3=96zer?= Date: Fri, 19 Dec 2025 16:28:08 +0100 Subject: [PATCH 2/9] feat: properties command implemented --- gcs/client/client.go | 31 ++++++++++++++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/gcs/client/client.go b/gcs/client/client.go index b7e94ba..0441f9d 100644 --- a/gcs/client/client.go +++ b/gcs/client/client.go @@ -18,6 +18,7 @@ package client import ( "context" + "encoding/json" "errors" "fmt" "io" @@ -39,6 +40,12 @@ import ( // client disallow an attempted write operation. var ErrInvalidROWriteOperation = errors.New("the client operates in read only mode. Change 'credentials_source' parameter value ") +type BlobProperties struct { + ETag string `json:"etag,omitempty"` + LastModified time.Time `json:"last_modified,omitempty"` + ContentLength int64 `json:"content_length,omitempty"` +} + // GCSBlobstore encapsulates interaction with the GCS blobstore type GCSBlobstore struct { authenticatedGCS *storage.Client @@ -290,7 +297,29 @@ func (client *GCSBlobstore) Copy(srcBlob string, dstBlob string) error { } func (client *GCSBlobstore) Properties(dest string) error { - return errors.New("not implemented") + if client.readOnly() { + return ErrInvalidROWriteOperation + } + oh := client.getObjectHandle(client.authenticatedGCS, dest) + attr, err := oh.Attrs(context.Background()) + + if err != nil { + return fmt.Errorf("getting attributes: %w", err) + } + + props := BlobProperties{ + ETag: strings.Trim(attr.Etag, `"`), + LastModified: attr.Updated, + ContentLength: attr.Size, + } + + output, err := json.MarshalIndent(props, "", " ") + if err != nil { + return fmt.Errorf("failed to marshal blob properties: %w", err) + } + + fmt.Println(string(output)) + return nil } func (client *GCSBlobstore) EnsureStorageExists() error { From 3e7767585f4d439fae32e10825a261c965ee89d9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Serdar=20=C3=96zer?= Date: Mon, 29 Dec 2025 13:54:23 +0100 Subject: [PATCH 3/9] feat: copy command implemented --- gcs/client/client.go | 14 +++++++++++++- gcs/client/sdk.go | 7 +++---- 2 files changed, 16 insertions(+), 5 deletions(-) diff --git a/gcs/client/client.go b/gcs/client/client.go index 0441f9d..fc555fa 100644 --- a/gcs/client/client.go +++ b/gcs/client/client.go @@ -293,7 +293,19 @@ func (client *GCSBlobstore) List(prefix string) ([]string, error) { } func (client *GCSBlobstore) Copy(srcBlob string, dstBlob string) error { - return errors.New("not implemented") + log.Printf("copying an object from %s to %s", srcBlob, dstBlob) + if client.readOnly() { + return ErrInvalidROWriteOperation + } + + srcHandle := client.getObjectHandle(client.authenticatedGCS, srcBlob) + dstHandle := client.getObjectHandle(client.authenticatedGCS, dstBlob) + + _, err := dstHandle.CopierFrom(srcHandle).Run(context.Background()) + if err != nil { + return fmt.Errorf("copying object: %w", err) + } + return nil } func (client *GCSBlobstore) Properties(dest string) error { diff --git a/gcs/client/sdk.go b/gcs/client/sdk.go index 28ce260..4eec4fa 100644 --- a/gcs/client/sdk.go +++ b/gcs/client/sdk.go @@ -55,7 +55,6 @@ func newStorageClients(ctx context.Context, cfg *config.GCSCli) (*storage.Client return authenticatedClient, publicClient, err } -// extractProjectID extracts the GCP project ID from credentials func extractProjectID(ctx context.Context, cfg *config.GCSCli) (string, error) { switch cfg.CredentialsSource { case config.ServiceAccountFileCredentialsSource: @@ -70,7 +69,7 @@ func extractProjectID(ctx context.Context, cfg *config.GCSCli) (string, error) { return "", errors.New("project_id not found in service account JSON") } return serviceAccount.ProjectID, nil - + case config.DefaultCredentialsSource: // Try to get project ID from default credentials creds, err := google.FindDefaultCredentials(ctx, storage.ScopeFullControl) @@ -81,10 +80,10 @@ func extractProjectID(ctx context.Context, cfg *config.GCSCli) (string, error) { return "", errors.New("project_id not found in default credentials") } return creds.ProjectID, nil - + case config.NoneCredentialsSource: return "", errors.New("cannot create bucket with read-only credentials") - + default: return "", errors.New("unknown credentials_source") } From f15163493b418490fb4ef623878f4ed3e99608d2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Serdar=20=C3=96zer?= Date: Tue, 30 Dec 2025 14:52:47 +0100 Subject: [PATCH 4/9] test: integration tests are added - listing with prefix - deleting with prefix - copying an object - ensuring a bucket is exist - retriving attributes of object These tests are added. Also logging in new functionality is added. --- gcs/client/client.go | 19 +++- gcs/integration/assertions.go | 140 ++++++++++++++++++++++++++++ gcs/integration/gcs_general_test.go | 48 +++++++++- gcs/integration/utils.go | 12 +++ 4 files changed, 213 insertions(+), 6 deletions(-) diff --git a/gcs/client/client.go b/gcs/client/client.go index fc555fa..22c86e5 100644 --- a/gcs/client/client.go +++ b/gcs/client/client.go @@ -266,6 +266,11 @@ func (client *GCSBlobstore) Sign(id string, action string, expiry time.Duration) } func (client *GCSBlobstore) List(prefix string) ([]string, error) { + if prefix != "" { + log.Printf("Listing objects in bucket %s with prefix '%s'\n", client.config.BucketName, prefix) + } else { + log.Printf("Listing objects in bucket %s\n", client.config.BucketName) + } if client.readOnly() { return nil, ErrInvalidROWriteOperation } @@ -293,7 +298,7 @@ func (client *GCSBlobstore) List(prefix string) ([]string, error) { } func (client *GCSBlobstore) Copy(srcBlob string, dstBlob string) error { - log.Printf("copying an object from %s to %s", srcBlob, dstBlob) + log.Printf("copying an object from %s to %s\n", srcBlob, dstBlob) if client.readOnly() { return ErrInvalidROWriteOperation } @@ -309,6 +314,7 @@ func (client *GCSBlobstore) Copy(srcBlob string, dstBlob string) error { } func (client *GCSBlobstore) Properties(dest string) error { + log.Printf("Getting properties for object %s/%s\n", client.config.BucketName, dest) if client.readOnly() { return ErrInvalidROWriteOperation } @@ -335,6 +341,7 @@ func (client *GCSBlobstore) Properties(dest string) error { } func (client *GCSBlobstore) EnsureStorageExists() error { + log.Printf("Ensuring bucket '%s' exists\n", client.config.BucketName) if client.readOnly() { return ErrInvalidROWriteOperation } @@ -342,8 +349,7 @@ func (client *GCSBlobstore) EnsureStorageExists() error { bh := client.getBucketHandle(client.authenticatedGCS) _, err := bh.Attrs(ctx) - if err == storage.ErrBucketNotExist { - + if errors.Is(err, storage.ErrBucketNotExist) { battr := &storage.BucketAttrs{Name: client.config.BucketName} if client.config.StorageClass != "" { battr.StorageClass = client.config.StorageClass @@ -361,6 +367,13 @@ func (client *GCSBlobstore) EnsureStorageExists() error { } func (client *GCSBlobstore) DeleteRecursive(prefix string) error { + if prefix != "" { + log.Printf("Deleting all objects in bucket %s with prefix '%s'\n", + client.config.BucketName, prefix) + } else { + log.Printf("Deleting all objects in bucket %s\n", client.config.BucketName) + } + if client.readOnly() { return ErrInvalidROWriteOperation } diff --git a/gcs/integration/assertions.go b/gcs/integration/assertions.go index 8410be5..dee74a6 100644 --- a/gcs/integration/assertions.go +++ b/gcs/integration/assertions.go @@ -17,6 +17,7 @@ package integration import ( + "fmt" "os" . "github.com/onsi/gomega" //nolint:staticcheck @@ -46,6 +47,14 @@ func AssertLifecycleWorks(gcsCLIPath string, ctx AssertContext) { Expect(session.ExitCode()).To(BeZero()) Expect(session.Err.Contents()).To(MatchRegexp("File '.*' exists in bucket '.*'")) + session, err = RunGCSCLI(gcsCLIPath, ctx.ConfigPath, storageType, "properties", ctx.GCSFileName) + Expect(err).ToNot(HaveOccurred()) + Expect(session.ExitCode()).To(BeZero()) + output := string(session.Out.Contents()) + Expect(output).To(MatchRegexp(`"etag":\s*".+?"`)) + Expect(output).To(MatchRegexp(`"last_modified":\s*".+?"`)) + Expect(output).To(MatchRegexp(`"content_length":\s*\d+`)) + tmpLocalFileName := "gcscli-download" defer os.Remove(tmpLocalFileName) //nolint:errcheck @@ -66,3 +75,134 @@ func AssertLifecycleWorks(gcsCLIPath string, ctx AssertContext) { Expect(session.ExitCode()).To(Equal(3)) Expect(session.Err.Contents()).To(MatchRegexp("File '.*' does not exist in bucket '.*'")) } + +func AssertDeleteRecursiveWithPrefixLifecycle(gcsCLIPath string, ctx AssertContext) { + storageType := "gcs" + + fileName1 := MakeContentFile(GenerateRandomString()) + fileName2 := MakeContentFile(GenerateRandomString()) + fileName3 := MakeContentFile(GenerateRandomString()) + prefix := fmt.Sprintf("%s-%s/", "test-prefix-delete-recursive", GenerateRandomString(10)) + dstObject1 := fmt.Sprintf("%s%s", prefix, GenerateRandomString()) + dstObject2 := fmt.Sprintf("%s%s", prefix, GenerateRandomString()) + dstObject3 := GenerateRandomString() + defer os.Remove(fileName1) //nolint:errcheck + defer os.Remove(fileName2) //nolint:errcheck + defer os.Remove(fileName3) //nolint:errcheck + + session, err := RunGCSCLI(gcsCLIPath, ctx.ConfigPath, storageType, "put", fileName1, dstObject1) + Expect(err).ToNot(HaveOccurred()) + Expect(session.ExitCode()).To(BeZero()) + + session, err = RunGCSCLI(gcsCLIPath, ctx.ConfigPath, storageType, "put", fileName2, dstObject2) + Expect(err).ToNot(HaveOccurred()) + Expect(session.ExitCode()).To(BeZero()) + + session, err = RunGCSCLI(gcsCLIPath, ctx.ConfigPath, storageType, "put", fileName3, dstObject3) + Expect(err).ToNot(HaveOccurred()) + Expect(session.ExitCode()).To(BeZero()) + + session, err = RunGCSCLI(gcsCLIPath, ctx.ConfigPath, storageType, "delete-recursive", prefix) + Expect(err).ToNot(HaveOccurred()) + Expect(session.ExitCode()).To(BeZero()) + + session, err = RunGCSCLI(gcsCLIPath, ctx.ConfigPath, storageType, "exists", dstObject3) + Expect(err).ToNot(HaveOccurred()) + Expect(session.ExitCode()).To(BeZero()) + + session, err = RunGCSCLI(gcsCLIPath, ctx.ConfigPath, storageType, "exists", dstObject1) + Expect(err).ToNot(HaveOccurred()) + Expect(session.ExitCode()).To(Equal(3)) + Expect(session.Err.Contents()).To(MatchRegexp("File '.*' does not exist in bucket '.*'")) + + session, err = RunGCSCLI(gcsCLIPath, ctx.ConfigPath, storageType, "exists", dstObject1) + Expect(err).ToNot(HaveOccurred()) + Expect(session.ExitCode()).To(Equal(3)) + Expect(session.Err.Contents()).To(MatchRegexp("File '.*' does not exist in bucket '.*'")) + + //cleanup artifact + session, err = RunGCSCLI(gcsCLIPath, ctx.ConfigPath, storageType, "delete", dstObject3) + Expect(err).ToNot(HaveOccurred()) + Expect(session.ExitCode()).To(BeZero()) + +} + +func AssertCopyLifecycle(gcsCLIPath string, ctx AssertContext) { + storageType := "gcs" + + dstNameToPut := GenerateRandomString() + dstNameToCopy := GenerateRandomString() + + session, err := RunGCSCLI(gcsCLIPath, ctx.ConfigPath, storageType, "put", ctx.ContentFile, dstNameToPut) + Expect(err).ToNot(HaveOccurred()) + Expect(session.ExitCode()).To(BeZero()) + + session, err = RunGCSCLI(gcsCLIPath, ctx.ConfigPath, storageType, "copy", dstNameToPut, dstNameToCopy) + Expect(err).ToNot(HaveOccurred()) + Expect(session.ExitCode()).To(BeZero()) + + tmpFileName := "copy-lifecycle" + defer os.Remove(tmpFileName) //nolint:errcheck + session, err = RunGCSCLI(gcsCLIPath, ctx.ConfigPath, storageType, "get", dstNameToCopy, tmpFileName) + Expect(err).ToNot(HaveOccurred()) + Expect(session.ExitCode()).To(BeZero()) + + contentGet, err := os.ReadFile(tmpFileName) + Expect(err).ToNot(HaveOccurred()) + Expect(string(contentGet)).To(Equal(ctx.ExpectedString)) + + session, err = RunGCSCLI(gcsCLIPath, ctx.ConfigPath, storageType, "delete", dstNameToPut) + Expect(err).ToNot(HaveOccurred()) + Expect(session.ExitCode()).To(BeZero()) + + session, err = RunGCSCLI(gcsCLIPath, ctx.ConfigPath, storageType, "delete", dstNameToCopy) + Expect(err).ToNot(HaveOccurred()) + Expect(session.ExitCode()).To(BeZero()) +} + +func AssertListMultipleWithPrefixLifecycle(gcsCLIPath string, ctx AssertContext) { + storageType := "gcs" + fileName1 := MakeContentFile(GenerateRandomString()) + fileName2 := MakeContentFile(GenerateRandomString()) + fileName3 := MakeContentFile(GenerateRandomString()) + prefix := fmt.Sprintf("%s-%s/", "test-prefix-list", GenerateRandomString(10)) + dstObject1 := fmt.Sprintf("%s%s", prefix, GenerateRandomString()) + dstObject2 := fmt.Sprintf("%s%s", prefix, GenerateRandomString()) + dstObject3 := GenerateRandomString() + defer os.Remove(fileName1) //nolint:errcheck + defer os.Remove(fileName2) //nolint:errcheck + defer os.Remove(fileName3) //nolint:errcheck + + session, err := RunGCSCLI(gcsCLIPath, ctx.ConfigPath, storageType, "put", fileName1, dstObject1) + Expect(err).ToNot(HaveOccurred()) + Expect(session.ExitCode()).To(BeZero()) + + session, err = RunGCSCLI(gcsCLIPath, ctx.ConfigPath, storageType, "put", fileName2, dstObject2) + Expect(err).ToNot(HaveOccurred()) + Expect(session.ExitCode()).To(BeZero()) + + session, err = RunGCSCLI(gcsCLIPath, ctx.ConfigPath, storageType, "put", fileName3, dstObject3) + Expect(err).ToNot(HaveOccurred()) + Expect(session.ExitCode()).To(BeZero()) + + session, err = RunGCSCLI(gcsCLIPath, ctx.ConfigPath, storageType, "list", prefix) + Expect(err).ToNot(HaveOccurred()) + Expect(session.ExitCode()).To(BeZero()) + + objs := string(session.Out.Contents()) + Expect(objs).To(ContainSubstring(dstObject1)) + Expect(objs).To(ContainSubstring(dstObject2)) + Expect(objs).ToNot(ContainSubstring(dstObject3)) + + session, err = RunGCSCLI(gcsCLIPath, ctx.ConfigPath, storageType, "delete", dstObject1) + Expect(err).ToNot(HaveOccurred()) + Expect(session.ExitCode()).To(BeZero()) + + session, err = RunGCSCLI(gcsCLIPath, ctx.ConfigPath, storageType, "delete", dstObject2) + Expect(err).ToNot(HaveOccurred()) + Expect(session.ExitCode()).To(BeZero()) + + session, err = RunGCSCLI(gcsCLIPath, ctx.ConfigPath, storageType, "delete", dstObject3) + Expect(err).ToNot(HaveOccurred()) + Expect(session.ExitCode()).To(BeZero()) +} diff --git a/gcs/integration/gcs_general_test.go b/gcs/integration/gcs_general_test.go index 3beabaf..48bc791 100644 --- a/gcs/integration/gcs_general_test.go +++ b/gcs/integration/gcs_general_test.go @@ -17,15 +17,16 @@ package integration import ( + context "context" "fmt" "os" + "strings" "syscall" - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - "github.com/cloudfoundry/storage-cli/gcs/client" "github.com/cloudfoundry/storage-cli/gcs/config" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" ) var _ = Describe("Integration", func() { @@ -125,5 +126,46 @@ var _ = Describe("Integration", func() { Expect(session.Err.Contents()).To(ContainSubstring("object doesn't exist")) }, configurations) + + DescribeTable("copying will create same content with different name", func(config *config.GCSCli) { + env.AddConfig(config) + AssertCopyLifecycle(gcsCLIPath, env) + }, configurations) + + Context("when bucket is not exist", func() { + DescribeTable("ensure storage exist will create a new bucket", func(cfg *config.GCSCli) { + cfg.BucketName = strings.ToLower(GenerateRandomString()) + env.AddConfig(cfg) + + session, err := RunGCSCLI(gcsCLIPath, env.ConfigPath, storageType, "ensure-storage-exists") + Expect(err).ToNot(HaveOccurred()) + Expect(session.ExitCode()).To(BeZero()) + + deleteBucket(context.Background(), cfg.BucketName, env.ConfigPath) + }, configurations) + }) + + Context("when bucket exists", func() { + DescribeTable("ensure storage exist will not create a new bucket", func(cfg *config.GCSCli) { + env.AddConfig(cfg) + session, err := RunGCSCLI(gcsCLIPath, env.ConfigPath, storageType, "ensure-storage-exists") + Expect(err).ToNot(HaveOccurred()) + Expect(session.ExitCode()).To(BeZero()) + + }, configurations) + }) + + Context("when working with multiple objects", func() { + FDescribeTable("recursive deleting will delete only the objects that have same prefix", func(config *config.GCSCli) { + env.AddConfig(config) + AssertDeleteRecursiveWithPrefixLifecycle(gcsCLIPath, env) + }, + configurations) + DescribeTable("list will output only the objects that have same prefix", func(config *config.GCSCli) { + env.AddConfig(config) + AssertListMultipleWithPrefixLifecycle(gcsCLIPath, env) + }, configurations) + + }) }) }) diff --git a/gcs/integration/utils.go b/gcs/integration/utils.go index 2ba0bfd..5c88153 100644 --- a/gcs/integration/utils.go +++ b/gcs/integration/utils.go @@ -27,6 +27,7 @@ import ( "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" //nolint:staticcheck "github.com/onsi/gomega/gexec" + "golang.org/x/net/context" ) const alphanum = "0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ" @@ -69,6 +70,17 @@ func MakeContentFile(content string) string { return tmpFile.Name() } +func deleteBucket(ctx context.Context, bucketName string, configPath string) { + cfgFile, err := os.Open(configPath) + Expect(err).ToNot(HaveOccurred()) + gcsConfig, err := config.NewFromReader(cfgFile) + Expect(err).ToNot(HaveOccurred()) + gcsClient, err := newSDK(ctx, gcsConfig) + Expect(err).ToNot(HaveOccurred()) + err = gcsClient.Bucket(bucketName).Delete(ctx) + Expect(err).ToNot(HaveOccurred()) +} + // RunGCSCLI run the gcscli and outputs the session // after waiting for it to finish func RunGCSCLI(gcsCLIPath, configPath, storageType, subcommand string, args ...string) (*gexec.Session, error) { From 0e38f39151f41b46e0f35be3c2fbb1015d426805 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Serdar=20=C3=96zer?= Date: Fri, 2 Jan 2026 10:51:32 +0100 Subject: [PATCH 5/9] test: public tests are added - ensure-bucket-exist refactored. ProjectId moved inside that function since it was failing while creating a new read only client. --- gcs/client/client.go | 17 ++++++------ gcs/integration/gcs_general_test.go | 12 ++++++--- gcs/integration/gcs_public_test.go | 41 ++++++++++++++++++++++++++--- 3 files changed, 55 insertions(+), 15 deletions(-) diff --git a/gcs/client/client.go b/gcs/client/client.go index 22c86e5..fcdb121 100644 --- a/gcs/client/client.go +++ b/gcs/client/client.go @@ -51,7 +51,6 @@ type GCSBlobstore struct { authenticatedGCS *storage.Client publicGCS *storage.Client config *config.GCSCli - projectID string } // validateRemoteConfig determines if the configuration of the client matches @@ -97,12 +96,7 @@ func New(ctx context.Context, cfg *config.GCSCli) (*GCSBlobstore, error) { return nil, fmt.Errorf("creating storage client: %v", err) } - projectID, err := extractProjectID(ctx, cfg) - if err != nil { - return nil, fmt.Errorf("extracting project ID: %v", err) - } - - return &GCSBlobstore{authenticatedGCS: authenticatedGCS, publicGCS: publicGCS, config: cfg, projectID: projectID}, nil + return &GCSBlobstore{authenticatedGCS: authenticatedGCS, publicGCS: publicGCS, config: cfg}, nil } // Get fetches a blob from the GCS blobstore. @@ -354,10 +348,17 @@ func (client *GCSBlobstore) EnsureStorageExists() error { if client.config.StorageClass != "" { battr.StorageClass = client.config.StorageClass } - err = bh.Create(ctx, client.projectID, battr) + + projectID, err := extractProjectID(ctx, client.config) + if err != nil { + return fmt.Errorf("extracting project ID: %w", err) + } + + err = bh.Create(ctx, projectID, battr) if err != nil { return fmt.Errorf("creating bucket: %w", err) } + return nil } if err != nil { return fmt.Errorf("checking bucket: %w", err) diff --git a/gcs/integration/gcs_general_test.go b/gcs/integration/gcs_general_test.go index 48bc791..4964c3e 100644 --- a/gcs/integration/gcs_general_test.go +++ b/gcs/integration/gcs_general_test.go @@ -134,14 +134,18 @@ var _ = Describe("Integration", func() { Context("when bucket is not exist", func() { DescribeTable("ensure storage exist will create a new bucket", func(cfg *config.GCSCli) { - cfg.BucketName = strings.ToLower(GenerateRandomString()) - env.AddConfig(cfg) + // create new a newCfg instead of modifying shared cfg accross all tests + newCfg := &config.GCSCli{ + BucketName: strings.ToLower(GenerateRandomString()), + } + + env.AddConfig(newCfg) session, err := RunGCSCLI(gcsCLIPath, env.ConfigPath, storageType, "ensure-storage-exists") Expect(err).ToNot(HaveOccurred()) Expect(session.ExitCode()).To(BeZero()) - deleteBucket(context.Background(), cfg.BucketName, env.ConfigPath) + deleteBucket(context.Background(), newCfg.BucketName, env.ConfigPath) }, configurations) }) @@ -156,7 +160,7 @@ var _ = Describe("Integration", func() { }) Context("when working with multiple objects", func() { - FDescribeTable("recursive deleting will delete only the objects that have same prefix", func(config *config.GCSCli) { + DescribeTable("recursive deleting will delete only the objects that have same prefix", func(config *config.GCSCli) { env.AddConfig(config) AssertDeleteRecursiveWithPrefixLifecycle(gcsCLIPath, env) }, diff --git a/gcs/integration/gcs_public_test.go b/gcs/integration/gcs_public_test.go index b64ca36..5efbc5f 100644 --- a/gcs/integration/gcs_public_test.go +++ b/gcs/integration/gcs_public_test.go @@ -93,21 +93,56 @@ var _ = Describe("GCS Public Bucket", func() { session, err := RunGCSCLI(gcsCLIPath, publicEnv.ConfigPath, storageType, "get", setupEnv.GCSFileName, "/dev/null") Expect(err).ToNot(HaveOccurred()) Expect(session.ExitCode()).ToNot(BeZero()) - Expect(session.Err.Contents()).To(ContainSubstring("object doesn't exist")) + Expect(string(session.Err.Contents())).To(ContainSubstring("object doesn't exist")) }) It("fails to put", func() { session, err := RunGCSCLI(gcsCLIPath, publicEnv.ConfigPath, storageType, "put", publicEnv.ContentFile, publicEnv.GCSFileName) Expect(err).ToNot(HaveOccurred()) Expect(session.ExitCode()).ToNot(BeZero()) - Expect(session.Err.Contents()).To(ContainSubstring(client.ErrInvalidROWriteOperation.Error())) + Expect(string(session.Err.Contents())).To(ContainSubstring(client.ErrInvalidROWriteOperation.Error())) }) It("fails to delete", func() { session, err := RunGCSCLI(gcsCLIPath, publicEnv.ConfigPath, storageType, "delete", publicEnv.GCSFileName) Expect(err).ToNot(HaveOccurred()) Expect(session.ExitCode()).ToNot(BeZero()) - Expect(session.Err.Contents()).To(ContainSubstring(client.ErrInvalidROWriteOperation.Error())) + Expect(string(session.Err.Contents())).To(ContainSubstring(client.ErrInvalidROWriteOperation.Error())) + }) + + It("fails to list", func() { + session, err := RunGCSCLI(gcsCLIPath, publicEnv.ConfigPath, storageType, "list", "prefix") + Expect(err).ToNot(HaveOccurred()) + Expect(session.ExitCode()).ToNot(BeZero()) + Expect(string(session.Err.Contents())).To(ContainSubstring(client.ErrInvalidROWriteOperation.Error())) + }) + + It("fails to get properties", func() { + session, err := RunGCSCLI(gcsCLIPath, publicEnv.ConfigPath, storageType, "properties", publicEnv.GCSFileName) + Expect(err).ToNot(HaveOccurred()) + Expect(session.ExitCode()).ToNot(BeZero()) + Expect(string(session.Err.Contents())).To(ContainSubstring(client.ErrInvalidROWriteOperation.Error())) + }) + + It("fails to delete-recursive", func() { + session, err := RunGCSCLI(gcsCLIPath, publicEnv.ConfigPath, storageType, "delete-recursive", "prefix") + Expect(err).ToNot(HaveOccurred()) + Expect(session.ExitCode()).ToNot(BeZero()) + Expect(string(session.Err.Contents())).To(ContainSubstring(client.ErrInvalidROWriteOperation.Error())) + }) + + It("fails to copy", func() { + session, err := RunGCSCLI(gcsCLIPath, publicEnv.ConfigPath, storageType, "copy", publicEnv.GCSFileName, "destination-object") + Expect(err).ToNot(HaveOccurred()) + Expect(session.ExitCode()).ToNot(BeZero()) + Expect(string(session.Err.Contents())).To(ContainSubstring(client.ErrInvalidROWriteOperation.Error())) + }) + + It("fails to create bucket", func() { + session, err := RunGCSCLI(gcsCLIPath, publicEnv.ConfigPath, storageType, "ensure-storage-exists") + Expect(err).ToNot(HaveOccurred()) + Expect(session.ExitCode()).ToNot(BeZero()) + Expect(string(session.Err.Contents())).To(ContainSubstring(client.ErrInvalidROWriteOperation.Error())) }) }) }) From 5b14bf780829729e7e1500ec7e956132f4592d2b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Serdar=20=C3=96zer?= Date: Fri, 2 Jan 2026 15:56:51 +0100 Subject: [PATCH 6/9] test: invalid copy and delete-recursive idempotent test are added --- gcs/integration/gcs_general_test.go | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/gcs/integration/gcs_general_test.go b/gcs/integration/gcs_general_test.go index 4964c3e..8fa6569 100644 --- a/gcs/integration/gcs_general_test.go +++ b/gcs/integration/gcs_general_test.go @@ -132,6 +132,16 @@ var _ = Describe("Integration", func() { AssertCopyLifecycle(gcsCLIPath, env) }, configurations) + DescribeTable("invalid copy should fail", func(config *config.GCSCli) { + env.AddConfig(config) + + session, err := RunGCSCLI(gcsCLIPath, env.ConfigPath, storageType, "copy", "source-object", "dest-object") + Expect(err).ToNot(HaveOccurred()) + Expect(session.ExitCode()).ToNot(BeZero()) + Expect(string(session.Err.Contents())).To(ContainSubstring("object doesn't exist")) + + }, configurations) + Context("when bucket is not exist", func() { DescribeTable("ensure storage exist will create a new bucket", func(cfg *config.GCSCli) { // create new a newCfg instead of modifying shared cfg accross all tests @@ -171,5 +181,18 @@ var _ = Describe("Integration", func() { }, configurations) }) + + DescribeTable("delete-recursive is idempotent", func(config *config.GCSCli) { + env.AddConfig(config) + + session, err := RunGCSCLI(gcsCLIPath, env.ConfigPath, storageType, "delete-recursive") + Expect(err).ToNot(HaveOccurred()) + Expect(session.ExitCode()).To(BeZero()) + + session, err = RunGCSCLI(gcsCLIPath, env.ConfigPath, storageType, "delete-recursive") + Expect(err).ToNot(HaveOccurred()) + Expect(session.ExitCode()).To(BeZero()) + + }, configurations) }) }) From 4de7ed8e4b9d01c5ae9a970f35db8f2f04186558 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Serdar=20=C3=96zer?= Date: Fri, 2 Jan 2026 16:30:45 +0100 Subject: [PATCH 7/9] fix: url signing integration test's artifacts are deleted --- gcs/integration/gcs_static_test.go | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/gcs/integration/gcs_static_test.go b/gcs/integration/gcs_static_test.go index c23a008..37dc9e0 100644 --- a/gcs/integration/gcs_static_test.go +++ b/gcs/integration/gcs_static_test.go @@ -69,6 +69,12 @@ var _ = Describe("Integration", func() { Expect(err).ToNot(HaveOccurred()) Expect(resp.StatusCode).To(Equal(200)) defer resp.Body.Close() //nolint:errcheck + + //delete test artifact + session, err = RunGCSCLI(gcsCLIPath, ctx.ConfigPath, storageType, "delete", ctx.GCSFileName) + Expect(err).ToNot(HaveOccurred()) + Expect(session.ExitCode()).To(BeZero()) + }) Context("encryption key is set", func() { @@ -123,6 +129,11 @@ var _ = Describe("Integration", func() { Expect(err).ToNot(HaveOccurred()) Expect(resp.StatusCode).To(Equal(200)) resp.Body.Close() //nolint:errcheck + + //delete test artifact + session, err = RunGCSCLI(gcsCLIPath, ctx.ConfigPath, storageType, "delete", ctx.GCSFileName) + Expect(err).ToNot(HaveOccurred()) + Expect(session.ExitCode()).To(BeZero()) }) }) }) From 665417c1fe91e0ef91a315e54dbf31d9e3c7f731 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Serdar=20=C3=96zer?= Date: Mon, 5 Jan 2026 13:06:11 +0100 Subject: [PATCH 8/9] feat: maxConcurency added to DeleteRecursive - semaphore based concurency limit added. Currently we set number of concurent request to 10 --- gcs/client/client.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/gcs/client/client.go b/gcs/client/client.go index fcdb121..590992a 100644 --- a/gcs/client/client.go +++ b/gcs/client/client.go @@ -40,6 +40,9 @@ import ( // client disallow an attempted write operation. var ErrInvalidROWriteOperation = errors.New("the client operates in read only mode. Change 'credentials_source' parameter value ") +// To enforce concurent go routine numbers during delete-recursive operation +const maxConcurrency = 10 + type BlobProperties struct { ETag string `json:"etag,omitempty"` LastModified time.Time `json:"last_modified,omitempty"` @@ -385,12 +388,17 @@ func (client *GCSBlobstore) DeleteRecursive(prefix string) error { } errChan := make(chan error, len(names)) + semaphore := make(chan struct{}, maxConcurrency) wg := &sync.WaitGroup{} for _, n := range names { name := n wg.Add(1) go func() { defer wg.Done() + + semaphore <- struct{}{} + defer func() { <-semaphore }() + err := client.getObjectHandle(client.authenticatedGCS, name).Delete(context.Background()) if err != nil && !errors.Is(err, storage.ErrObjectNotExist) { errChan <- fmt.Errorf("deleting object %s: %w", name, err) From c9d10c0a2611bbf21c69198780d80625d6e89fc4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Serdar=20=C3=96zer?= Date: Tue, 13 Jan 2026 10:49:54 +0100 Subject: [PATCH 9/9] fix: properties print empty json when object not exist --- gcs/client/client.go | 5 ++++- gcs/integration/assertions.go | 33 ++++++++++++++++++++++------- gcs/integration/gcs_general_test.go | 5 +++++ 3 files changed, 34 insertions(+), 9 deletions(-) diff --git a/gcs/client/client.go b/gcs/client/client.go index 590992a..7707d00 100644 --- a/gcs/client/client.go +++ b/gcs/client/client.go @@ -137,7 +137,6 @@ func (client *GCSBlobstore) getReader(gcs *storage.Client, src string) (*storage const retryAttempts = 3 func (client *GCSBlobstore) Put(sourceFilePath string, dest string) error { - src, err := os.Open(sourceFilePath) if err != nil { return err @@ -319,6 +318,10 @@ func (client *GCSBlobstore) Properties(dest string) error { attr, err := oh.Attrs(context.Background()) if err != nil { + if errors.Is(err, storage.ErrObjectNotExist) { + fmt.Println(`{}`) + return nil + } return fmt.Errorf("getting attributes: %w", err) } diff --git a/gcs/integration/assertions.go b/gcs/integration/assertions.go index dee74a6..8f476c4 100644 --- a/gcs/integration/assertions.go +++ b/gcs/integration/assertions.go @@ -47,14 +47,6 @@ func AssertLifecycleWorks(gcsCLIPath string, ctx AssertContext) { Expect(session.ExitCode()).To(BeZero()) Expect(session.Err.Contents()).To(MatchRegexp("File '.*' exists in bucket '.*'")) - session, err = RunGCSCLI(gcsCLIPath, ctx.ConfigPath, storageType, "properties", ctx.GCSFileName) - Expect(err).ToNot(HaveOccurred()) - Expect(session.ExitCode()).To(BeZero()) - output := string(session.Out.Contents()) - Expect(output).To(MatchRegexp(`"etag":\s*".+?"`)) - Expect(output).To(MatchRegexp(`"last_modified":\s*".+?"`)) - Expect(output).To(MatchRegexp(`"content_length":\s*\d+`)) - tmpLocalFileName := "gcscli-download" defer os.Remove(tmpLocalFileName) //nolint:errcheck @@ -206,3 +198,28 @@ func AssertListMultipleWithPrefixLifecycle(gcsCLIPath string, ctx AssertContext) Expect(err).ToNot(HaveOccurred()) Expect(session.ExitCode()).To(BeZero()) } + +func AssertPropertiesLifecycle(gcsCLIPath string, ctx AssertContext) { + storageType := "gcs" + session, err := RunGCSCLI(gcsCLIPath, ctx.ConfigPath, storageType, "put", ctx.ContentFile, ctx.GCSFileName) + Expect(err).ToNot(HaveOccurred()) + Expect(session.ExitCode()).To(BeZero()) + + session, err = RunGCSCLI(gcsCLIPath, ctx.ConfigPath, storageType, "properties", ctx.GCSFileName) + Expect(err).ToNot(HaveOccurred()) + Expect(session.ExitCode()).To(BeZero()) + output := string(session.Out.Contents()) + Expect(output).To(MatchRegexp(`"etag":\s*".+?"`)) + Expect(output).To(MatchRegexp(`"last_modified":\s*".+?"`)) + Expect(output).To(MatchRegexp(`"content_length":\s*\d+`)) + + session, err = RunGCSCLI(gcsCLIPath, ctx.ConfigPath, storageType, "delete", ctx.GCSFileName) + Expect(err).ToNot(HaveOccurred()) + Expect(session.ExitCode()).To(BeZero()) + + session, err = RunGCSCLI(gcsCLIPath, ctx.ConfigPath, storageType, "properties", ctx.GCSFileName) + Expect(err).ToNot(HaveOccurred()) + Expect(session.ExitCode()).To(BeZero()) + Expect(string(session.Out.Contents())).To(MatchRegexp("{}")) + +} diff --git a/gcs/integration/gcs_general_test.go b/gcs/integration/gcs_general_test.go index 8fa6569..49c4fa8 100644 --- a/gcs/integration/gcs_general_test.go +++ b/gcs/integration/gcs_general_test.go @@ -142,6 +142,11 @@ var _ = Describe("Integration", func() { }, configurations) + DescribeTable("properties should print json", func(config *config.GCSCli) { + env.AddConfig(config) + AssertPropertiesLifecycle(gcsCLIPath, env) + }, configurations) + Context("when bucket is not exist", func() { DescribeTable("ensure storage exist will create a new bucket", func(cfg *config.GCSCli) { // create new a newCfg instead of modifying shared cfg accross all tests