feat(storage): add MoveObject functionality to JSON and gRPC

ci/cloudbuild/builds/lib/integration.sh

@@ -31,7 +31,7 @@ source module ci/lib/io.sh
 export PATH="${HOME}/.local/bin:${PATH}"
 python3 -m pip uninstall -y --quiet googleapis-storage-testbench
 python3 -m pip install --upgrade --user --quiet --disable-pip-version-check \
-  "git+https://github.com/googleapis/storage-testbench@v0.46.0"
+  "git+https://github.com/googleapis/storage-testbench@v0.52.0"
 
 # Some of the tests will need a valid roots.pem file.
 rm -f /dev/shm/roots.pem

ci/lib/run_gcs_httpbin_emulator_utils.sh

@@ -127,4 +127,8 @@ create_testbench_resources() {
     curl -s -o /dev/null -X POST --data-binary @- \
       -H "Content-Type: application/json" \
       "${CLOUD_STORAGE_EMULATOR_ENDPOINT}/storage/v1/b?project=${GOOGLE_CLOUD_PROJECT}"
+  printf '{"name": "%s"}' "${GOOGLE_CLOUD_CPP_STORAGE_TEST_FOLDER_BUCKET_NAME}" |
+    curl -s -o /dev/null -X POST --data-binary @- \
+      -H "Content-Type: application/json" \
+      "${CLOUD_STORAGE_EMULATOR_ENDPOINT}/storage/v1/b?project=${GOOGLE_CLOUD_PROJECT}"
 }

google/cloud/storage/client.h

@@ -1372,6 +1372,39 @@ class Client {
     return connection_->UpdateObject(request);
   }
 
+  /**
+   * Moves an existing object to a new or existing object within a HNS enabled
+   * bucket.
+   *
+   * @param bucket_name the name of the bucket in which to move the object. The
+   * bucket must be HNS enabled.
+   * @param source_object_name the name of the source object to move.
+   * @param destination_object_name the destination name of the object after the
+   * move is completed.
+   * @param options a list of optional query parameters and/or request headers.
+   *     Valid types for this operation include
+   *      `IfSourceGenerationMatch`, `IfSourceGenerationNotMatch`,
+   * `IfSourceMetagenerationMatch`, `IfSourceMetagenerationNotMatch`,
+   * `IfGenerationMatch`, `IfGenerationNotMatch`, `IfMetagenerationMatch`
+   *      `IfMetagenerationNotMatch`, `projection`.
+   *
+   * @par Idempotency
+   * This operation is only idempotent if restricted by pre-conditions.
+   */
+  template <typename... Options>
+  StatusOr<ObjectMetadata> MoveObject(std::string bucket_name,
+                                      std::string source_object_name,
+                                      std::string destination_object_name,
+                                      Options&&... options) {
+    google::cloud::internal::OptionsSpan const span(
+        SpanOptions(std::forward<Options>(options)...));
+    internal::MoveObjectRequest request(std::move(bucket_name),
+                                        std::move(source_object_name),
+                                        std::move(destination_object_name));
+    request.set_multiple_options(std::forward<Options>(options)...);
+    return connection_->MoveObject(request);
+  }
+
   /**
    * Patches the metadata in a Google Cloud Storage Object.
    *

google/cloud/storage/idempotency_policy.cc

@@ -92,6 +92,10 @@ bool AlwaysRetryIdempotencyPolicy::IsIdempotent(
     internal::UpdateObjectRequest const&) const {
   return true;
 }
+bool AlwaysRetryIdempotencyPolicy::IsIdempotent(
+    internal::MoveObjectRequest const&) const {
+  return true;
+}
 bool AlwaysRetryIdempotencyPolicy::IsIdempotent(
     internal::PatchObjectRequest const&) const {
   return true;
@@ -341,6 +345,11 @@ bool StrictIdempotencyPolicy::IsIdempotent(
          request.HasOption<IfMetagenerationMatch>();
 }
 
+bool StrictIdempotencyPolicy::IsIdempotent(
+    internal::MoveObjectRequest const& request) const {
+  return request.HasOption<IfGenerationMatch>();
+}
+
 bool StrictIdempotencyPolicy::IsIdempotent(
     internal::PatchObjectRequest const& request) const {
   return request.HasOption<IfMatchEtag>() ||

google/cloud/storage/idempotency_policy.h

@@ -102,6 +102,9 @@ class IdempotencyPolicy {
       internal::DeleteObjectRequest const& request) const = 0;
   virtual bool IsIdempotent(
       internal::UpdateObjectRequest const& request) const = 0;
+  virtual bool IsIdempotent(internal::MoveObjectRequest const&) const {
+    return false;
+  };
   virtual bool IsIdempotent(
       internal::PatchObjectRequest const& request) const = 0;
   virtual bool IsIdempotent(
@@ -238,6 +241,7 @@ class AlwaysRetryIdempotencyPolicy : public IdempotencyPolicy {
       internal::DeleteObjectRequest const& request) const override;
   bool IsIdempotent(
       internal::UpdateObjectRequest const& request) const override;
+  bool IsIdempotent(internal::MoveObjectRequest const& request) const override;
   bool IsIdempotent(internal::PatchObjectRequest const& request) const override;
   bool IsIdempotent(
       internal::ComposeObjectRequest const& request) const override;
@@ -370,6 +374,7 @@ class StrictIdempotencyPolicy : public IdempotencyPolicy {
       internal::DeleteObjectRequest const& request) const override;
   bool IsIdempotent(
       internal::UpdateObjectRequest const& request) const override;
+  bool IsIdempotent(internal::MoveObjectRequest const& request) const override;
   bool IsIdempotent(internal::PatchObjectRequest const& request) const override;
   bool IsIdempotent(
       internal::ComposeObjectRequest const& request) const override;

google/cloud/storage/idempotency_policy_test.cc

@@ -231,6 +231,21 @@ TEST(StrictIdempotencyPolicyTest, UpdateObjectIfMetagenerationMatch) {
   EXPECT_TRUE(policy.IsIdempotent(request));
 }
 
+TEST(StrictIdempotencyPolicyTest, MoveObject) {
+  StrictIdempotencyPolicy policy;
+  internal::MoveObjectRequest request(
+      "test-bucket-name", "test-src-object-name", "test-dst-object-name");
+  EXPECT_FALSE(policy.IsIdempotent(request));
+}
+
+TEST(StrictIdempotencyPolicyTest, MoveObjectIfGenerationMatch) {
+  StrictIdempotencyPolicy policy;
+  internal::MoveObjectRequest request(
+      "test-bucket-name", "test-src-object-name", "test-dst-object-name");
+  request.set_option(IfGenerationMatch(7));
+  EXPECT_TRUE(policy.IsIdempotent(request));
+}
+
 TEST(StrictIdempotencyPolicyTest, PatchObject) {
   StrictIdempotencyPolicy policy;
   internal::PatchObjectRequest request("test-bucket-name", "test-object-name",

google/cloud/storage/internal/connection_impl.cc

@@ -451,6 +451,22 @@ StatusOr<ObjectMetadata> StorageConnectionImpl::UpdateObject(
       google::cloud::internal::CurrentOptions(), request, __func__);
 }
 
+StatusOr<ObjectMetadata> StorageConnectionImpl::MoveObject(
+    MoveObjectRequest const& request) {
+  auto const idempotency = current_idempotency_policy().IsIdempotent(request)
+                               ? Idempotency::kIdempotent
+                               : Idempotency::kNonIdempotent;
+  return RestRetryLoop(
+      current_retry_policy(), current_backoff_policy(), idempotency,
+      [token = MakeIdempotencyToken(), this](
+          rest_internal::RestContext& context, Options const& options,
+          auto const& request) {
+        context.AddHeader(kIdempotencyTokenHeader, token);
+        return stub_->MoveObject(context, options, request);
+      },
+      google::cloud::internal::CurrentOptions(), request, __func__);
+}
+
 StatusOr<ObjectMetadata> StorageConnectionImpl::PatchObject(
     PatchObjectRequest const& request) {
   auto const idempotency = current_idempotency_policy().IsIdempotent(request)

google/cloud/storage/internal/connection_impl.h

@@ -80,6 +80,8 @@ class StorageConnectionImpl
   StatusOr<EmptyResponse> DeleteObject(DeleteObjectRequest const&) override;
   StatusOr<ObjectMetadata> UpdateObject(
       UpdateObjectRequest const& request) override;
+  StatusOr<ObjectMetadata> MoveObject(
+      MoveObjectRequest const& request) override;
   StatusOr<ObjectMetadata> PatchObject(
       PatchObjectRequest const& request) override;
   StatusOr<ObjectMetadata> ComposeObject(

google/cloud/storage/internal/connection_impl_object_copy_test.cc

@@ -122,6 +122,34 @@ TEST(StorageConnectionImpl, RewriteObjectPermanentFailure) {
   EXPECT_THAT(permanent.captured_authority_options(), RetryLoopUsesOptions());
 }
 
+TEST(StorageConnectionImpl, MoveObjectTooManyFailures) {
+  auto transient = MockRetryClientFunction(TransientError());
+  auto mock = std::make_unique<MockGenericStub>();
+  EXPECT_CALL(*mock, options);
+  EXPECT_CALL(*mock, MoveObject).Times(3).WillRepeatedly(transient);
+  auto client =
+      StorageConnectionImpl::Create(std::move(mock), RetryTestOptions());
+  google::cloud::internal::OptionsSpan span(client->options());
+  auto response = client->MoveObject(MoveObjectRequest()).status();
+  EXPECT_THAT(response, StoppedOnTooManyTransients("MoveObject"));
+  EXPECT_THAT(transient.captured_tokens(), RetryLoopUsesSingleToken());
+  EXPECT_THAT(transient.captured_authority_options(), RetryLoopUsesOptions());
+}
+
+TEST(StorageConnectionImpl, MoveObjectPermanentFailure) {
+  auto permanent = MockRetryClientFunction(PermanentError());
+  auto mock = std::make_unique<MockGenericStub>();
+  EXPECT_CALL(*mock, options);
+  EXPECT_CALL(*mock, MoveObject).WillOnce(permanent);
+  auto client =
+      StorageConnectionImpl::Create(std::move(mock), RetryTestOptions());
+  google::cloud::internal::OptionsSpan span(client->options());
+  auto response = client->MoveObject(MoveObjectRequest()).status();
+  EXPECT_THAT(response, StoppedOnPermanentError("MoveObject"));
+  EXPECT_THAT(permanent.captured_tokens(), RetryLoopUsesSingleToken());
+  EXPECT_THAT(permanent.captured_authority_options(), RetryLoopUsesOptions());
+}
+
 }  // namespace
 }  // namespace internal
 GOOGLE_CLOUD_CPP_INLINE_NAMESPACE_END

google/cloud/storage/internal/generic_stub.h

@@ -121,6 +121,9 @@ class GenericStub {
   virtual StatusOr<storage::ObjectMetadata> UpdateObject(
       rest_internal::RestContext&, Options const&,
       storage::internal::UpdateObjectRequest const&) = 0;
+  virtual StatusOr<storage::ObjectMetadata> MoveObject(
+      rest_internal::RestContext&, Options const&,
+      storage::internal::MoveObjectRequest const&) = 0;
   virtual StatusOr<storage::ObjectMetadata> PatchObject(
       rest_internal::RestContext&, Options const&,
       storage::internal::PatchObjectRequest const&) = 0;

google/cloud/storage/internal/generic_stub_adapter.cc

@@ -126,6 +126,11 @@ class GenericStubAdapter : public GenericStub {
       storage::internal::UpdateObjectRequest const& request) override {
     return impl_->UpdateObject(request);
   }
+  StatusOr<storage::ObjectMetadata> MoveObject(
+      rest_internal::RestContext&, Options const&,
+      storage::internal::MoveObjectRequest const& request) override {
+    return impl_->MoveObject(request);
+  }
   StatusOr<storage::ObjectMetadata> PatchObject(
       rest_internal::RestContext&, Options const&,
       storage::internal::PatchObjectRequest const& request) override {

google/cloud/storage/internal/grpc/object_request_parser.cc

@@ -666,10 +666,39 @@ StatusOr<google::storage::v2::RewriteObjectRequest> ToProto(
   return result;
 }
 
+StatusOr<google::storage::v2::MoveObjectRequest> ToProto(
+    storage::internal::MoveObjectRequest const& request) {
+  google::storage::v2::MoveObjectRequest result;
+  SetGenerationConditions(result, request);
+  SetMetagenerationConditions(result, request);
+  if (request.HasOption<storage::IfSourceGenerationMatch>()) {
+    result.set_if_source_generation_match(
+        request.GetOption<storage::IfSourceGenerationMatch>().value());
+  }
+  if (request.HasOption<storage::IfSourceGenerationNotMatch>()) {
+    result.set_if_source_generation_not_match(
+        request.GetOption<storage::IfSourceGenerationNotMatch>().value());
+  }
+  if (request.HasOption<storage::IfSourceMetagenerationMatch>()) {
+    result.set_if_source_metageneration_match(
+        request.GetOption<storage::IfSourceMetagenerationMatch>().value());
+  }
+  if (request.HasOption<storage::IfSourceMetagenerationNotMatch>()) {
+    result.set_if_source_metageneration_not_match(
+        request.GetOption<storage::IfSourceMetagenerationNotMatch>().value());
+  }
+  result.set_bucket(GrpcBucketIdToName(request.bucket_name()));
+  result.set_source_object(request.source_object_name());
+  result.set_destination_object(request.destination_object_name());
+
+  return result;
+}
+
 StatusOr<google::storage::v2::RestoreObjectRequest> ToProto(
     storage::internal::RestoreObjectRequest const& request) {
   google::storage::v2::RestoreObjectRequest result;
   auto status = SetCommonObjectParameters(result, request);
+  if (!status.ok()) return status;
 
   result.set_bucket(GrpcBucketIdToName(request.bucket_name()));
   result.set_object(request.object_name());

google/cloud/storage/internal/grpc/object_request_parser.h

@@ -67,6 +67,9 @@ StatusOr<google::storage::v2::RewriteObjectRequest> ToProto(
 StatusOr<google::storage::v2::RestoreObjectRequest> ToProto(
     storage::internal::RestoreObjectRequest const& request);
 
+StatusOr<google::storage::v2::MoveObjectRequest> ToProto(
+    storage::internal::MoveObjectRequest const& request);
+
 StatusOr<google::storage::v2::StartResumableWriteRequest> ToProto(
     storage::internal::ResumableUploadRequest const& request);
 

google/cloud/storage/internal/grpc/object_request_parser_test.cc

@@ -406,6 +406,51 @@ TEST(GrpcObjectRequestParser, ReadOffsetEmptyRange) {
   EXPECT_THAT(actual, StatusIs(StatusCode::kInvalidArgument));
 }
 
+TEST(GrpcObjectRequestParser, MoveObjectSimpleRequest) {
+  auto constexpr kTextProto = R"pb(
+    bucket: "projects/_/buckets/test-bucket"
+    source_object: "source-object"
+    destination_object: "destination-object"
+  )pb";
+  google::storage::v2::MoveObjectRequest expected;
+  ASSERT_TRUE(TextFormat::ParseFromString(kTextProto, &expected));
+  storage::internal::MoveObjectRequest req("test-bucket", "source-object",
+                                           "destination-object");
+  auto actual = ToProto(req);
+  ASSERT_STATUS_OK(actual);
+  EXPECT_THAT(*actual, IsProtoEqual(expected));
+}
+
+TEST(GrpcObjectRequestParser, MoveObjectRequestAllOptions) {
+  auto constexpr kTextProto = R"pb(
+    bucket: "projects/_/buckets/test-bucket"
+    source_object: "source-object"
+    destination_object: "destination-object"
+    if_source_generation_match: 1
+    if_source_generation_not_match: 2
+    if_source_metageneration_match: 3
+    if_source_metageneration_not_match: 4
+    if_generation_match: 5
+    if_generation_not_match: 6
+    if_metageneration_match: 7
+    if_metageneration_not_match: 8
+  )pb";
+  google::storage::v2::MoveObjectRequest expected;
+  ASSERT_TRUE(TextFormat::ParseFromString(kTextProto, &expected));
+  storage::internal::MoveObjectRequest req("test-bucket", "source-object",
+                                           "destination-object");
+  req.set_multiple_options(
+      storage::IfSourceGenerationMatch(1),
+      storage::IfSourceGenerationNotMatch(2),
+      storage::IfSourceMetagenerationMatch(3),
+      storage::IfSourceMetagenerationNotMatch(4), storage::IfGenerationMatch(5),
+      storage::IfGenerationNotMatch(6), storage::IfMetagenerationMatch(7),
+      storage::IfMetagenerationNotMatch(8));
+  auto actual = ToProto(req);
+  ASSERT_STATUS_OK(actual);
+  EXPECT_THAT(*actual, IsProtoEqual(expected));
+}
+
 TEST(GrpcObjectRequestParser, PatchObjectRequestAllOptions) {
   auto constexpr kTextProto = R"pb(
     predefined_acl: "projectPrivate"

google/cloud/storage/internal/grpc/stub.cc

@@ -483,6 +483,19 @@ StatusOr<storage::ObjectMetadata> GrpcStub::UpdateObject(
   return FromProto(*response, options);
 }
 
+StatusOr<storage::ObjectMetadata> GrpcStub::MoveObject(
+    rest_internal::RestContext& context, Options const& options,
+    storage::internal::MoveObjectRequest const& request) {
+  auto proto = ToProto(request);
+  if (!proto) return std::move(proto).status();
+  grpc::ClientContext ctx;
+  ApplyQueryParameters(ctx, options, request);
+  AddIdempotencyToken(ctx, context);
+  auto response = stub_->MoveObject(ctx, options, *proto);
+  if (!response) return std::move(response).status();
+  return FromProto(*response, options);
+}
+
 StatusOr<storage::ObjectMetadata> GrpcStub::PatchObject(
     rest_internal::RestContext& context, Options const& options,
     storage::internal::PatchObjectRequest const& request) {

google/cloud/storage/internal/grpc/stub.h

@@ -107,6 +107,9 @@ class GrpcStub : public GenericStub {
   StatusOr<storage::ObjectMetadata> UpdateObject(
       rest_internal::RestContext& context, Options const& options,
       storage::internal::UpdateObjectRequest const& request) override;
+  StatusOr<storage::ObjectMetadata> MoveObject(
+      rest_internal::RestContext& context, Options const& options,
+      storage::internal::MoveObjectRequest const& request) override;
   StatusOr<storage::ObjectMetadata> PatchObject(
       rest_internal::RestContext& context, Options const& options,
       storage::internal::PatchObjectRequest const& request) override;

google/cloud/storage/internal/grpc/stub_test.cc

@@ -658,6 +658,28 @@ TEST_F(GrpcClientTest, UpdateObject) {
   EXPECT_EQ(response.status(), PermanentError());
 }
 
+TEST_F(GrpcClientTest, MoveObject) {
+  auto mock = std::make_shared<MockStorageStub>();
+  EXPECT_CALL(*mock, MoveObject)
+      .WillOnce([this](grpc::ClientContext& context, Options const&,
+                       v2::MoveObjectRequest const& request) {
+        auto metadata = GetMetadata(context);
+        EXPECT_THAT(metadata, UnorderedElementsAre(Pair(kIdempotencyTokenHeader,
+                                                        "test-token-1234")));
+        EXPECT_THAT(request.bucket(), "projects/_/buckets/test-bucket");
+        EXPECT_THAT(request.source_object(), "test-source-object");
+        EXPECT_THAT(request.destination_object(), "test-destination-object");
+        return PermanentError();
+      });
+  auto client = CreateTestClient(mock);
+  auto context = TestContext();
+  auto response = client->MoveObject(
+      context, TestOptions(),
+      storage::internal::MoveObjectRequest("test-bucket", "test-source-object",
+                                           "test-destination-object"));
+  EXPECT_EQ(response.status(), PermanentError());
+}
+
 TEST_F(GrpcClientTest, PatchObject) {
   auto mock = std::make_shared<MockStorageStub>();
   EXPECT_CALL(*mock, UpdateObject)

google/cloud/storage/internal/logging_stub.cc

@@ -224,6 +224,16 @@ StatusOr<ObjectMetadata> LoggingStub::UpdateObject(
       context, options, request, __func__);
 }
 
+StatusOr<ObjectMetadata> LoggingStub::MoveObject(
+    rest_internal::RestContext& context, Options const& options,
+    MoveObjectRequest const& request) {
+  return LogWrapper(
+      [this](auto& context, auto const& options, auto& request) {
+        return stub_->MoveObject(context, options, request);
+      },
+      context, options, request, __func__);
+}
+
 StatusOr<ObjectMetadata> LoggingStub::PatchObject(
     rest_internal::RestContext& context, Options const& options,
     PatchObjectRequest const& request) {