feat(storage): Add ability to restore soft deleted objects

ci/cloudbuild/builds/lib/integration.sh

@@ -31,7 +31,7 @@ source module ci/lib/io.sh
 export PATH="${HOME}/.local/bin:${PATH}"
 python3 -m pip uninstall -y --quiet googleapis-storage-testbench
 python3 -m pip install --upgrade --user --quiet --disable-pip-version-check \
-  "git+https://github.com/googleapis/storage-testbench@v0.42.0"
+  "git+https://github.com/googleapis/storage-testbench@v0.46.0"
 
 # Some of the tests will need a valid roots.pem file.
 rm -f /dev/shm/roots.pem

google/cloud/storage/client.h

@@ -1646,6 +1646,52 @@ class Client {
                                std::string{}, std::forward<Options>(options)...)
         .Result();
   }
+
+  /**
+   * Restores a soft-deleted object.
+   *
+   * When a soft-deleted object is restored, a new copy of that object is
+   * created in the same bucket and inherits the same metadata as the
+   * soft-deleted object. The inherited metadata is the metadata that existed
+   * when the original object became soft deleted, with the following
+   * exceptions:
+   *
+   * 1. The createTime of the new object is set to the time at which the
+   * soft-deleted object was restored.
+   * 2. The softDeleteTime and hardDeleteTime values are cleared.
+   * 3. A new generation is assigned and the metageneration is reset to 1.
+   * 4. If the soft-deleted object was in a bucket that had Autoclass enabled,
+   *    the new object is restored to Standard storage.
+   *
+   * @param bucket_name name of the bucket in which the new object will be
+   * created. Must be the same bucket that contained the soft-deleted object
+   * being restored.
+   * @param object_name name of the soft-deleted object to restore.
+   * @param generation specifies the version of the soft-deleted object to
+   * restore.
+   * @param options a list of optional query parameters and/or request headers.
+   *     Valid types for this operation include `DestinationKmsKeyName`,
+   *      `IfGenerationMatch`, `IfGenerationNotMatch`, `IfMetagenerationMatch`,
+   *      `IfMetagenerationNotMatch`, `Projection`, `CopySourceAcl`
+   *
+   * @return The metadata of the restored object.
+   *
+   * @par Idempotency
+   * This operation is only idempotent if restricted by pre-conditions, in this
+   * case, `IfGenerationMatch`.
+   */
+  template <typename... Options>
+  StatusOr<ObjectMetadata> RestoreObject(std::string bucket_name,
+                                         std::string object_name,
+                                         std::int64_t generation,
+                                         Options&&... options) {
+    google::cloud::internal::OptionsSpan const span(
+        SpanOptions(std::forward<Options>(options)...));
+    internal::RestoreObjectRequest request(
+        std::move(bucket_name), std::move(object_name), std::move(generation));
+    request.set_multiple_options(std::forward<Options>(options)...);
+    return connection_->RestoreObject(request);
+  }
   ///@}
 
   /**

google/cloud/storage/idempotency_policy.cc

@@ -104,6 +104,10 @@ bool AlwaysRetryIdempotencyPolicy::IsIdempotent(
     internal::RewriteObjectRequest const&) const {
   return true;
 }
+bool AlwaysRetryIdempotencyPolicy::IsIdempotent(
+    internal::RestoreObjectRequest const&) const {
+  return true;
+}
 bool AlwaysRetryIdempotencyPolicy::IsIdempotent(
     internal::ResumableUploadRequest const&) const {
   return true;
@@ -361,6 +365,11 @@ bool StrictIdempotencyPolicy::IsIdempotent(
   return request.HasOption<IfGenerationMatch>();
 }
 
+bool StrictIdempotencyPolicy::IsIdempotent(
+    internal::RestoreObjectRequest const& request) const {
+  return request.HasOption<IfGenerationMatch>();
+}
+
 bool StrictIdempotencyPolicy::IsIdempotent(
     internal::ResumableUploadRequest const& request) const {
   // Only the pre-conditions on the destination matter. If they are not set, it

google/cloud/storage/idempotency_policy.h

@@ -108,6 +108,9 @@ class IdempotencyPolicy {
       internal::ComposeObjectRequest const& request) const = 0;
   virtual bool IsIdempotent(
       internal::RewriteObjectRequest const& request) const = 0;
+  virtual bool IsIdempotent(internal::RestoreObjectRequest const&) const {
+    return false;
+  };
   virtual bool IsIdempotent(
       internal::ResumableUploadRequest const& request) const = 0;
   virtual bool IsIdempotent(
@@ -240,6 +243,8 @@ class AlwaysRetryIdempotencyPolicy : public IdempotencyPolicy {
       internal::ComposeObjectRequest const& request) const override;
   bool IsIdempotent(
       internal::RewriteObjectRequest const& request) const override;
+  bool IsIdempotent(
+      internal::RestoreObjectRequest const& request) const override;
   bool IsIdempotent(
       internal::ResumableUploadRequest const& request) const override;
   bool IsIdempotent(internal::UploadChunkRequest const& request) const override;
@@ -370,6 +375,8 @@ class StrictIdempotencyPolicy : public IdempotencyPolicy {
       internal::ComposeObjectRequest const& request) const override;
   bool IsIdempotent(
       internal::RewriteObjectRequest const& request) const override;
+  bool IsIdempotent(
+      internal::RestoreObjectRequest const& request) const override;
   bool IsIdempotent(
       internal::ResumableUploadRequest const& request) const override;
   bool IsIdempotent(internal::UploadChunkRequest const& request) const override;

google/cloud/storage/idempotency_policy_test.cc

@@ -288,6 +288,19 @@ TEST(StrictIdempotencyPolicyTest, RewriteObjectIfGenerationMatch) {
   EXPECT_TRUE(policy.IsIdempotent(request));
 }
 
+TEST(StrictIdempotencyPolicyTest, RestoreObject) {
+  StrictIdempotencyPolicy policy;
+  internal::RestoreObjectRequest request("test-bucket", "test-object", 1234);
+  EXPECT_FALSE(policy.IsIdempotent(request));
+}
+
+TEST(StrictIdempotencyPolicyTest, RestoreObjectIfGenerationMatch) {
+  StrictIdempotencyPolicy policy;
+  internal::RestoreObjectRequest request("test-bucket", "test-object", 1234);
+  request.set_option(IfGenerationMatch(0));
+  EXPECT_TRUE(policy.IsIdempotent(request));
+}
+
 TEST(StrictIdempotencyPolicyTest, ListBucketAcl) {
   StrictIdempotencyPolicy policy;
   internal::ListBucketAclRequest request("test-bucket-name");

google/cloud/storage/internal/connection_impl.cc

@@ -499,6 +499,23 @@ StatusOr<RewriteObjectResponse> StorageConnectionImpl::RewriteObject(
       google::cloud::internal::CurrentOptions(), request, __func__);
 }
 
+StatusOr<ObjectMetadata> StorageConnectionImpl::RestoreObject(
+    RestoreObjectRequest const& request) {
+  auto const idempotency = current_idempotency_policy().IsIdempotent(request)
+                               ? Idempotency::kIdempotent
+                               : Idempotency::kNonIdempotent;
+
+  return RestRetryLoop(
+      current_retry_policy(), current_backoff_policy(), idempotency,
+      [token = MakeIdempotencyToken(), this](
+          rest_internal::RestContext& context, Options const& options,
+          auto const& request) {
+        context.AddHeader(kIdempotencyTokenHeader, token);
+        return stub_->RestoreObject(context, options, request);
+      },
+      google::cloud::internal::CurrentOptions(), request, __func__);
+}
+
 StatusOr<CreateResumableUploadResponse>
 StorageConnectionImpl::CreateResumableUpload(
     ResumableUploadRequest const& request) {

google/cloud/storage/internal/connection_impl.h

@@ -86,6 +86,8 @@ class StorageConnectionImpl
       ComposeObjectRequest const& request) override;
   StatusOr<RewriteObjectResponse> RewriteObject(
       RewriteObjectRequest const&) override;
+  StatusOr<ObjectMetadata> RestoreObject(
+      RestoreObjectRequest const& request) override;
 
   StatusOr<CreateResumableUploadResponse> CreateResumableUpload(
       ResumableUploadRequest const& request) override;

google/cloud/storage/internal/connection_impl_object_test.cc

@@ -370,6 +370,34 @@ TEST(StorageConnectionImpl, PatchObjectPermanentFailure) {
   EXPECT_THAT(permanent.captured_authority_options(), RetryLoopUsesOptions());
 }
 
+TEST(StorageConnectionImpl, RestoreObjectTooManyFailures) {
+  auto transient = MockRetryClientFunction(TransientError());
+  auto mock = std::make_unique<MockGenericStub>();
+  EXPECT_CALL(*mock, options);
+  EXPECT_CALL(*mock, RestoreObject).Times(3).WillRepeatedly(transient);
+  auto client =
+      StorageConnectionImpl::Create(std::move(mock), RetryTestOptions());
+  google::cloud::internal::OptionsSpan span(client->options());
+  auto response = client->RestoreObject(RestoreObjectRequest()).status();
+  EXPECT_THAT(response, StoppedOnTooManyTransients("RestoreObject"));
+  EXPECT_THAT(transient.captured_tokens(), RetryLoopUsesSingleToken());
+  EXPECT_THAT(transient.captured_authority_options(), RetryLoopUsesOptions());
+}
+
+TEST(StorageConnectionImpl, RestoreObjectPermanentFailure) {
+  auto permanent = MockRetryClientFunction(PermanentError());
+  auto mock = std::make_unique<MockGenericStub>();
+  EXPECT_CALL(*mock, options);
+  EXPECT_CALL(*mock, RestoreObject).WillOnce(permanent);
+  auto client =
+      StorageConnectionImpl::Create(std::move(mock), RetryTestOptions());
+  google::cloud::internal::OptionsSpan span(client->options());
+  auto response = client->RestoreObject(RestoreObjectRequest()).status();
+  EXPECT_THAT(response, StoppedOnPermanentError("RestoreObject"));
+  EXPECT_THAT(permanent.captured_tokens(), RetryLoopUsesSingleToken());
+  EXPECT_THAT(permanent.captured_authority_options(), RetryLoopUsesOptions());
+}
+
 }  // namespace
 }  // namespace internal
 GOOGLE_CLOUD_CPP_INLINE_NAMESPACE_END

google/cloud/storage/internal/generic_stub.h

@@ -130,6 +130,9 @@ class GenericStub {
   virtual StatusOr<storage::internal::RewriteObjectResponse> RewriteObject(
       rest_internal::RestContext&, Options const&,
       storage::internal::RewriteObjectRequest const&) = 0;
+  virtual StatusOr<storage::ObjectMetadata> RestoreObject(
+      rest_internal::RestContext&, Options const&,
+      storage::internal::RestoreObjectRequest const&) = 0;
 
   virtual StatusOr<storage::internal::CreateResumableUploadResponse>
   CreateResumableUpload(rest_internal::RestContext&, Options const&,

google/cloud/storage/internal/generic_stub_adapter.cc

@@ -141,6 +141,11 @@ class GenericStubAdapter : public GenericStub {
       storage::internal::RewriteObjectRequest const& request) override {
     return impl_->RewriteObject(request);
   }
+  StatusOr<storage::ObjectMetadata> RestoreObject(
+      rest_internal::RestContext&, Options const&,
+      storage::internal::RestoreObjectRequest const& request) override {
+    return impl_->RestoreObject(request);
+  }
 
   StatusOr<storage::internal::CreateResumableUploadResponse>
   CreateResumableUpload(

google/cloud/storage/internal/grpc/object_request_parser.cc

@@ -666,6 +666,22 @@ StatusOr<google::storage::v2::RewriteObjectRequest> ToProto(
   return result;
 }
 
+StatusOr<google::storage::v2::RestoreObjectRequest> ToProto(
+    storage::internal::RestoreObjectRequest const& request) {
+  google::storage::v2::RestoreObjectRequest result;
+  auto status = SetCommonObjectParameters(result, request);
+
+  result.set_bucket(GrpcBucketIdToName(request.bucket_name()));
+  result.set_object(request.object_name());
+  result.set_generation(request.generation());
+  SetGenerationConditions(result, request);
+  SetMetagenerationConditions(result, request);
+  result.set_copy_source_acl(
+      request.GetOption<storage::CopySourceAcl>().value_or(false));
+
+  return result;
+}
+
 storage::internal::RewriteObjectResponse FromProto(
     google::storage::v2::RewriteResponse const& response,
     Options const& options) {

google/cloud/storage/internal/grpc/object_request_parser.h

@@ -64,6 +64,9 @@ storage::internal::RewriteObjectResponse FromProto(
 StatusOr<google::storage::v2::RewriteObjectRequest> ToProto(
     storage::internal::CopyObjectRequest const& request);
 
+StatusOr<google::storage::v2::RestoreObjectRequest> ToProto(
+    storage::internal::RestoreObjectRequest const& request);
+
 StatusOr<google::storage::v2::StartResumableWriteRequest> ToProto(
     storage::internal::ResumableUploadRequest const& request);
 

google/cloud/storage/internal/grpc/object_request_parser_test.cc

@@ -1046,6 +1046,50 @@ TEST(GrpcObjectRequestParser, RewriteObjectResponse) {
   EXPECT_EQ(actual.resource.name(), "object-name");
 }
 
+TEST(GrpcObjectRequestParser, RestoreObjectSimpleRequest) {
+  google::storage::v2::RestoreObjectRequest expected;
+  EXPECT_TRUE(
+      TextFormat::ParseFromString(R"pb(
+                                    bucket: "projects/_/buckets/test-bucket"
+                                    object: "test-object",
+                                    generation: 1234678,
+                                    copy_source_acl: false,
+                                  )pb",
+                                  &expected));
+
+  storage::internal::RestoreObjectRequest req("test-bucket", "test-object",
+                                              1234678);
+
+  auto actual = ToProto(req).value();
+  EXPECT_THAT(actual, IsProtoEqual(expected));
+}
+
+TEST(GrpcObjectRequestParser, RestoreObjectRequestAllFields) {
+  google::storage::v2::RestoreObjectRequest expected;
+  ASSERT_TRUE(TextFormat::ParseFromString(
+      R"pb(
+        bucket: "projects/_/buckets/test-bucket"
+        object: "test-object",
+        generation: 1234678,
+        if_generation_match: 1
+        if_generation_not_match: 2
+        if_metageneration_match: 3
+        if_metageneration_not_match: 4
+        copy_source_acl: true
+      )pb",
+      &expected));
+
+  storage::internal::RestoreObjectRequest req("test-bucket", "test-object",
+                                              1234678);
+  req.set_multiple_options(
+      storage::IfGenerationMatch(1), storage::IfGenerationNotMatch(2),
+      storage::IfMetagenerationMatch(3), storage::IfMetagenerationNotMatch(4),
+      storage::CopySourceAcl(true));
+  auto const actual = ToProto(req);
+  ASSERT_STATUS_OK(actual);
+  EXPECT_THAT(*actual, IsProtoEqual(expected));
+}
+
 TEST(GrpcObjectRequestParser, CopyObjectRequestAllOptions) {
   auto constexpr kTextProto = R"pb(
     destination_bucket: "projects/_/buckets/destination-bucket"

google/cloud/storage/internal/grpc/stub.cc

@@ -522,6 +522,19 @@ StatusOr<storage::internal::RewriteObjectResponse> GrpcStub::RewriteObject(
   return FromProto(*response, options);
 }
 
+StatusOr<storage::ObjectMetadata> GrpcStub::RestoreObject(
+    rest_internal::RestContext& context, Options const& options,
+    storage::internal::RestoreObjectRequest const& request) {
+  auto proto = ToProto(request);
+  if (!proto) return std::move(proto).status();
+  grpc::ClientContext ctx;
+  ApplyQueryParameters(ctx, options, request);
+  AddIdempotencyToken(ctx, context);
+  auto response = stub_->RestoreObject(ctx, options, *proto);
+  if (!response) return std::move(response).status();
+  return FromProto(*response, options);
+}
+
 StatusOr<storage::internal::CreateResumableUploadResponse>
 GrpcStub::CreateResumableUpload(
     rest_internal::RestContext& context, Options const& options,

google/cloud/storage/internal/grpc/stub.h

@@ -116,6 +116,9 @@ class GrpcStub : public GenericStub {
   StatusOr<storage::internal::RewriteObjectResponse> RewriteObject(
       rest_internal::RestContext& context, Options const& options,
       storage::internal::RewriteObjectRequest const& request) override;
+  StatusOr<storage::ObjectMetadata> RestoreObject(
+      rest_internal::RestContext& context, Options const& options,
+      storage::internal::RestoreObjectRequest const& request) override;
 
   StatusOr<storage::internal::CreateResumableUploadResponse>
   CreateResumableUpload(

google/cloud/storage/internal/grpc/stub_test.cc

@@ -744,6 +744,34 @@ TEST_F(GrpcClientTest, RewriteObject) {
   EXPECT_EQ(response.status(), PermanentError());
 }
 
+TEST_F(GrpcClientTest, RestoreObject) {
+  auto mock = std::make_shared<MockStorageStub>();
+  EXPECT_CALL(*mock, RestoreObject)
+      .WillOnce([this](grpc::ClientContext& context, Options const&,
+                       v2::RestoreObjectRequest const& request) {
+        auto metadata = GetMetadata(context);
+        EXPECT_THAT(metadata,
+                    UnorderedElementsAre(
+                        Pair(kIdempotencyTokenHeader, "test-token-1234"),
+                        Pair("x-goog-quota-user", "test-quota-user"),
+                        Pair("x-goog-fieldmask", "field1,field2")));
+        EXPECT_THAT(request.bucket(), "projects/_/buckets/test-bucket");
+        EXPECT_THAT(request.object(), "test-object");
+        EXPECT_THAT(request.generation(), 1234);
+        EXPECT_THAT(request.copy_source_acl(), false);
+        return PermanentError();
+      });
+  auto client = CreateTestClient(mock);
+  auto context = TestContext();
+  auto response = client->RestoreObject(
+      context, TestOptions(),
+      storage::internal::RestoreObjectRequest("test-bucket", "test-object",
+                                              1234)
+          .set_multiple_options(Fields("field1,field2"),
+                                QuotaUser("test-quota-user")));
+  EXPECT_EQ(response.status(), PermanentError());
+}
+
 TEST_F(GrpcClientTest, CreateResumableUpload) {
   auto mock = std::make_shared<MockStorageStub>();
   EXPECT_CALL(*mock, StartResumableWrite)

google/cloud/storage/internal/logging_stub.cc

@@ -254,6 +254,16 @@ StatusOr<RewriteObjectResponse> LoggingStub::RewriteObject(
       context, options, request, __func__);
 }
 
+StatusOr<ObjectMetadata> LoggingStub::RestoreObject(
+    rest_internal::RestContext& context, Options const& options,
+    RestoreObjectRequest const& request) {
+  return LogWrapper(
+      [this](auto& context, auto const& options, auto& request) {
+        return stub_->RestoreObject(context, options, request);
+      },
+      context, options, request, __func__);
+}
+
 StatusOr<CreateResumableUploadResponse> LoggingStub::CreateResumableUpload(
     rest_internal::RestContext& context, Options const& options,
     ResumableUploadRequest const& request) {

google/cloud/storage/internal/logging_stub.h

@@ -97,6 +97,9 @@ class LoggingStub : public storage_internal::GenericStub {
   StatusOr<RewriteObjectResponse> RewriteObject(
       rest_internal::RestContext&, Options const&,
       RewriteObjectRequest const&) override;
+  StatusOr<ObjectMetadata> RestoreObject(
+      rest_internal::RestContext&, Options const&,
+      RestoreObjectRequest const& request) override;
 
   StatusOr<CreateResumableUploadResponse> CreateResumableUpload(
       rest_internal::RestContext&, Options const&,