Skip to content
This repository was archived by the owner on Aug 19, 2019. It is now read-only.

Split the metadata API request when it hits a limit by count or byte size #129

Merged
supriyagarg merged 3 commits into from
Apr 4, 2018
Merged

Split the metadata API request when it hits a limit by count or byte size #129

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
fefa239
Add a flag for the limit of the request size by count.
supriyagarg Apr 4, 2018
4f83350
Update SendMetadata to split the request on hitting the count limit
supriyagarg Apr 4, 2018
8d66b62
Address comments
supriyagarg Apr 4, 2018
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions src/configuration.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,7 @@ constexpr const char kMetadataIngestionDefaultEndpointFormat[] =
"/resourceMetadata:batchUpdate";
constexpr const int kMetadataIngestionDefaultRequestSizeLimitBytes =
8*1024*1024;
constexpr const int kMetadataIngestionDefaultRequestSizeLimitCount = 1000;
constexpr const char kMetadataIngestionDefaultRawContentVersion[] = "0.1";
constexpr const int kInstanceUpdaterDefaultIntervalSeconds = 60*60;
constexpr const char kDefaultInstanceResourceType[] =
Expand Down Expand Up @@ -95,6 +96,8 @@ Configuration::Configuration()
kMetadataIngestionDefaultEndpointFormat),
metadata_ingestion_request_size_limit_bytes_(
kMetadataIngestionDefaultRequestSizeLimitBytes),
metadata_ingestion_request_size_limit_count_(
kMetadataIngestionDefaultRequestSizeLimitCount),
metadata_ingestion_raw_content_version_(
kMetadataIngestionDefaultRawContentVersion),
instance_updater_interval_seconds_(
Expand Down Expand Up @@ -233,6 +236,9 @@ void Configuration::ParseConfiguration(std::istream& input) {
metadata_ingestion_request_size_limit_bytes_ =
config["MetadataIngestionRequestSizeLimitBytes"].as<int>(
metadata_ingestion_request_size_limit_bytes_);
metadata_ingestion_request_size_limit_count_ =
config["MetadataIngestionRequestSizeLimitCount"].as<int>(
metadata_ingestion_request_size_limit_count_);
metadata_ingestion_raw_content_version_ =
config["MetadataIngestionRawContentVersion"].as<std::string>(
metadata_ingestion_raw_content_version_);
Expand Down
5 changes: 5 additions & 0 deletions src/configuration.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -77,6 +77,10 @@ class Configuration {
std::lock_guard<std::mutex> lock(mutex_);
return metadata_ingestion_request_size_limit_bytes_;
}
int MetadataIngestionRequestSizeLimitCount() const {
std::lock_guard<std::mutex> lock(mutex_);
return metadata_ingestion_request_size_limit_count_;
}
const std::string& MetadataIngestionRawContentVersion() const {
std::lock_guard<std::mutex> lock(mutex_);
return metadata_ingestion_raw_content_version_;
Expand Down Expand Up @@ -183,6 +187,7 @@ class Configuration {
std::string metadata_reporter_user_agent_;
std::string metadata_ingestion_endpoint_format_;
int metadata_ingestion_request_size_limit_bytes_;
int metadata_ingestion_request_size_limit_count_;
std::string metadata_ingestion_raw_content_version_;
int instance_updater_interval_seconds_;
std::string instance_resource_type_;
Expand Down
3 changes: 2 additions & 1 deletion src/reporter.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -139,6 +139,7 @@ void MetadataReporter::SendMetadata(
const int empty_size = empty_request->ToString().size();

const int limit_bytes = config_.MetadataIngestionRequestSizeLimitBytes();
const int limit_count = config_.MetadataIngestionRequestSizeLimitCount();
int total_size = empty_size;

std::vector<json::value> entries;
Expand All @@ -165,7 +166,7 @@ void MetadataReporter::SendMetadata(
<< "B, dropping; entry " << *metadata_entry;
continue;
}
if (total_size + size > limit_bytes) {
if (entries.size() == limit_count || total_size + size > limit_bytes) {
SendMetadataRequest(std::move(entries), endpoint, auth_header, user_agent,
config_.VerboseLogging());
entries.clear();
Expand Down
3 changes: 3 additions & 0 deletions test/configuration_unittest.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ void VerifyDefaultConfig(const Configuration& config) {
"v1beta2/projects/{{project_id}}/resourceMetadata:batchUpdate",
config.MetadataIngestionEndpointFormat());
EXPECT_EQ(8*1024*1024, config.MetadataIngestionRequestSizeLimitBytes());
EXPECT_EQ(1000, config.MetadataIngestionRequestSizeLimitCount());
Copy link
Contributor

@ACEmilG ACEmilG Apr 4, 2018

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Please also add a check in the PopulatedConfig test for the non-default case.

Copy link
Contributor Author

@supriyagarg supriyagarg Apr 4, 2018

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

done

EXPECT_EQ("0.1", config.MetadataIngestionRawContentVersion());
EXPECT_EQ(60*60, config.InstanceUpdaterIntervalSeconds());
EXPECT_EQ("", config.InstanceResourceType());
Expand Down Expand Up @@ -56,12 +57,14 @@ TEST(ConfigurationTest, PopulatedConfig) {
"MetadataReporterPurgeDeleted: true\n"
"MetadataReporterUserAgent: \"foobar/foobaz\"\n"
"HealthCheckFile: /a/b/c\n"
"MetadataIngestionRequestSizeLimitCount: 500\n"
));
EXPECT_EQ("TestProjectId", config.ProjectId());
EXPECT_EQ(13, config.MetadataApiNumThreads());
EXPECT_EQ(true, config.MetadataReporterPurgeDeleted());
EXPECT_EQ("foobar/foobaz", config.MetadataReporterUserAgent());
EXPECT_EQ("/a/b/c", config.HealthCheckFile());
EXPECT_EQ(500, config.MetadataIngestionRequestSizeLimitCount());
}

TEST(ConfigurationTest, CommentSkipped) {
Expand Down