Skip to content

[QC-670] Add parameter to limit the size of the objects sent to the ccdb #8013

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
ktf merged 6 commits into from
Jan 31, 2022
Merged

[QC-670] Add parameter to limit the size of the objects sent to the ccdb #8013

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
bb7980b
QC-670 Add parameter to limit the size of the objects sent to the ccdb
Barthelemy Jan 26, 2022
9970bd3
Please consider the following formatting changes
alibuild Jan 26, 2022
ca26520
Merge pull request #9 from alibuild/alibot-cleanup-8013
Barthelemy Jan 26, 2022
4d30375
Add a return value to the store* methods.
Barthelemy Jan 27, 2022
cd9f2ed
Please consider the following formatting changes
alibuild Jan 27, 2022
e4f9f2a
Merge pull request #10 from alibuild/alibot-cleanup-8013
Barthelemy Jan 27, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
46 changes: 33 additions & 13 deletions CCDB/include/CCDB/CcdbApi.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -115,9 +115,13 @@ class CcdbApi //: public DatabaseInterface
* @param metadata Key-values representing the metadata for this object.
* @param startValidityTimestamp Start of validity. If omitted, current timestamp is used.
* @param endValidityTimestamp End of validity. If omitted, current timestamp + 1 day is used.
* @return 0 -> ok,
* positive number -> curl error (https://curl.se/libcurl/c/libcurl-errors.html),
* -1 : object bigger than maxSize,
* -2 : curl initialization error
*/
void storeAsTFile(const TObject* rootObject, std::string const& path, std::map<std::string, std::string> const& metadata,
long startValidityTimestamp = -1, long endValidityTimestamp = -1) const;
int storeAsTFile(const TObject* rootObject, std::string const& path, std::map<std::string, std::string> const& metadata,
long startValidityTimestamp = -1, long endValidityTimestamp = -1, std::vector<char>::size_type maxSize = 0 /*bytes*/) const;

/**
* Store into the CCDB a TFile containing an object of type T (which needs to have a ROOT dictionary)
Expand All @@ -127,19 +131,23 @@ class CcdbApi //: public DatabaseInterface
* @param metadata Key-values representing the metadata for this object.
* @param startValidityTimestamp Start of validity. If omitted, current timestamp is used.
* @param endValidityTimestamp End of validity. If omitted, current timestamp + 1 day is used.
* @return 0 -> ok,
* positive number -> curl error (https://curl.se/libcurl/c/libcurl-errors.html),
* -1 : object bigger than maxSize,
* -2 : curl initialization error
*/
template <typename T>
void storeAsTFileAny(const T* obj, std::string const& path, std::map<std::string, std::string> const& metadata,
long startValidityTimestamp = -1, long endValidityTimestamp = -1) const
int storeAsTFileAny(const T* obj, std::string const& path, std::map<std::string, std::string> const& metadata,
long startValidityTimestamp = -1, long endValidityTimestamp = -1, std::vector<char>::size_type maxSize = 0 /*bytes*/) const
{
storeAsTFile_impl(reinterpret_cast<const void*>(obj), typeid(T), path, metadata, startValidityTimestamp, endValidityTimestamp);
return storeAsTFile_impl(reinterpret_cast<const void*>(obj), typeid(T), path, metadata, startValidityTimestamp, endValidityTimestamp, maxSize);
}

// interface for storing TObject via storeAsTFileAny
void storeAsTFileAny(const TObject* rootobj, std::string const& path, std::map<std::string, std::string> const& metadata,
long startValidityTimestamp = -1, long endValidityTimestamp = -1) const
int storeAsTFileAny(const TObject* rootobj, std::string const& path, std::map<std::string, std::string> const& metadata,
long startValidityTimestamp = -1, long endValidityTimestamp = -1, std::vector<char>::size_type maxSize = 0 /*bytes*/) const
{
storeAsTFile(rootobj, path, metadata, startValidityTimestamp, endValidityTimestamp);
return storeAsTFile(rootobj, path, metadata, startValidityTimestamp, endValidityTimestamp, maxSize);
}

/**
Expand Down Expand Up @@ -392,19 +400,31 @@ class CcdbApi //: public DatabaseInterface
public:
/**
* A generic method to store a binary buffer (e.g. an image of the TMemFile)
* @return 0 -> ok,
* positive number -> curl error (https://curl.se/libcurl/c/libcurl-errors.html),
* -1 : object bigger than maxSize,
* -2 : curl initialization error
*/
void storeAsBinaryFile(const char* buffer, size_t size, const std::string& fileName, const std::string& objectType,
const std::string& path, const std::map<std::string, std::string>& metadata,
long startValidityTimestamp, long endValidityTimestamp) const;
int storeAsBinaryFile(const char* buffer, size_t size, const std::string& fileName, const std::string& objectType,
const std::string& path, const std::map<std::string, std::string>& metadata,
long startValidityTimestamp, long endValidityTimestamp, std::vector<char>::size_type maxSize = 0 /*in bytes*/) const;

/**
* A generic helper implementation to store an obj whose type is given by a std::type_info
* @return 0 -> ok,
* positive number -> curl error (https://curl.se/libcurl/c/libcurl-errors.html),
* -1 : object bigger than maxSize,
* -2 : curl initialization error
*/
void storeAsTFile_impl(const void* obj1, std::type_info const& info, std::string const& path, std::map<std::string, std::string> const& metadata,
long startValidityTimestamp = -1, long endValidityTimestamp = -1) const;
int storeAsTFile_impl(const void* obj1, std::type_info const& info, std::string const& path, std::map<std::string, std::string> const& metadata,
long startValidityTimestamp = -1, long endValidityTimestamp = -1, std::vector<char>::size_type maxSize = 0 /*in bytes*/) const;

/**
* A generic helper implementation to query obj whose type is given by a std::type_info
* @return 0 -> ok,
* positive number -> curl error (https://curl.se/libcurl/c/libcurl-errors.html),
* -1 : object bigger than maxSize,
* -2 : curl initialization error
*/
void* retrieveFromTFile(std::type_info const&, std::string const& path, std::map<std::string, std::string> const& metadata,
long timestamp = -1, std::map<std::string, std::string>* headers = nullptr, std::string const& etag = "",
Expand Down
31 changes: 19 additions & 12 deletions CCDB/src/CcdbApi.cxx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -128,22 +128,25 @@ std::unique_ptr<std::vector<char>> CcdbApi::createObjectImage(const TObject* roo
return o2::utils::MemFileHelper::createFileImage(*rootObject, tmpFileName, CCDBOBJECT_ENTRY);
}

void CcdbApi::storeAsTFile_impl(const void* obj, std::type_info const& tinfo, std::string const& path,
std::map<std::string, std::string> const& metadata,
long startValidityTimestamp, long endValidityTimestamp) const
int CcdbApi::storeAsTFile_impl(const void* obj, std::type_info const& tinfo, std::string const& path,
std::map<std::string, std::string> const& metadata,
long startValidityTimestamp, long endValidityTimestamp,
std::vector<char>::size_type maxSize) const
{
// We need the TClass for this type; will verify if dictionary exists
CcdbObjectInfo info;
auto img = createObjectImage(obj, tinfo, &info);
storeAsBinaryFile(img->data(), img->size(), info.getFileName(), info.getObjectType(),
path, metadata, startValidityTimestamp, endValidityTimestamp);
return storeAsBinaryFile(img->data(), img->size(), info.getFileName(), info.getObjectType(),
path, metadata, startValidityTimestamp, endValidityTimestamp, maxSize);
}

void CcdbApi::storeAsBinaryFile(const char* buffer, size_t size, const std::string& filename, const std::string& objectType,
const std::string& path, const std::map<std::string, std::string>& metadata,
long startValidityTimestamp, long endValidityTimestamp) const
int CcdbApi::storeAsBinaryFile(const char* buffer, size_t size, const std::string& filename, const std::string& objectType,
const std::string& path, const std::map<std::string, std::string>& metadata,
long startValidityTimestamp, long endValidityTimestamp, std::vector<char>::size_type maxSize) const
{
// Store a binary file
if (maxSize > 0 && size > maxSize) {
return -1;
}

// Prepare URL
long sanitizedStartValidityTimestamp = startValidityTimestamp;
Expand All @@ -160,6 +163,7 @@ void CcdbApi::storeAsBinaryFile(const char* buffer, size_t size, const std::stri
// Curl preparation
CURL* curl = nullptr;
curl = curl_easy_init();
int returnValue = 0;

if (curl != nullptr) {
struct curl_httppost* formpost = nullptr;
Expand Down Expand Up @@ -196,6 +200,7 @@ void CcdbApi::storeAsBinaryFile(const char* buffer, size_t size, const std::stri
if (res != CURLE_OK) {
fprintf(stderr, "curl_easy_perform() failed: %s\n",
curl_easy_strerror(res));
returnValue = res;
}
}

Expand All @@ -208,16 +213,18 @@ void CcdbApi::storeAsBinaryFile(const char* buffer, size_t size, const std::stri
curl_slist_free_all(headerlist);
} else {
cerr << "curl initialization failure" << endl;
returnValue = -2;
}
return returnValue;
}

void CcdbApi::storeAsTFile(const TObject* rootObject, std::string const& path, std::map<std::string, std::string> const& metadata,
long startValidityTimestamp, long endValidityTimestamp) const
int CcdbApi::storeAsTFile(const TObject* rootObject, std::string const& path, std::map<std::string, std::string> const& metadata,
long startValidityTimestamp, long endValidityTimestamp, std::vector<char>::size_type maxSize) const
{
// Prepare file
CcdbObjectInfo info;
auto img = createObjectImage(rootObject, &info);
storeAsBinaryFile(img->data(), img->size(), info.getFileName(), info.getObjectType(), path, metadata, startValidityTimestamp, endValidityTimestamp);
return storeAsBinaryFile(img->data(), img->size(), info.getFileName(), info.getObjectType(), path, metadata, startValidityTimestamp, endValidityTimestamp, maxSize);
}

string CcdbApi::getFullUrlForStorage(CURL* curl, const string& path, const string& objtype,
Expand Down
15 changes: 15 additions & 0 deletions CCDB/test/testCcdbApi.cxx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -200,6 +200,21 @@ BOOST_AUTO_TEST_CASE(store_retrieve_TMemFile_templated_test, *utf::precondition(
}
}

BOOST_AUTO_TEST_CASE(store_max_size_test, *utf::precondition(if_reachable()))
{
test_fixture f;

// try to store a user defined class
// since we don't depend on anything, we are putting an object known to CCDB
o2::ccdb::IdPath path;
path.setPath("HelloWorld");

int result = f.api.storeAsTFileAny(&path, basePath + "CCDBPath", f.metadata); // ok
BOOST_CHECK_EQUAL(result, 0);
result = f.api.storeAsTFileAny(&path, basePath + "CCDBPath", f.metadata, -1, -1, 1 /* bytes */); // we know this will fail
BOOST_CHECK_EQUAL(result, -1);
}

/// A test verifying that the DB responds the correct result for given timestamps
BOOST_AUTO_TEST_CASE(timestamptest, *utf::precondition(if_reachable()))
{
Expand Down