Skip to content

CCDBPopulator optionally fatals on failed upload (def=on) #8640

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
shahor02 merged 2 commits into from
Apr 25, 2022
Merged

CCDBPopulator optionally fatals on failed upload (def=on) #8640

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
d2513fc
Use LOG(alarm) instead of writing to cerror
shahor02 Apr 24, 2022
c9016a1
CCDBPopulator optionally fatals on failed upload (def=on)
shahor02 Apr 24, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
27 changes: 13 additions & 14 deletions CCDB/src/CcdbApi.cxx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -144,18 +144,19 @@ int CcdbApi::storeAsBinaryFile(const char* buffer, size_t size, const std::strin
long startValidityTimestamp, long endValidityTimestamp, std::vector<char>::size_type maxSize) const
{
if (maxSize > 0 && size > maxSize) {
LOGP(alarm, "Object will not be uploaded to {} since its size {} exceeds max allowed {}", path, size, maxSize);
return -1;
}

// Prepare URL
long sanitizedStartValidityTimestamp = startValidityTimestamp;
if (startValidityTimestamp == -1) {
cout << "Start of Validity not set, current timestamp used." << endl;
LOGP(info, "Start of Validity not set, current timestamp used.");
sanitizedStartValidityTimestamp = getCurrentTimestamp();
}
long sanitizedEndValidityTimestamp = endValidityTimestamp;
if (endValidityTimestamp == -1) {
cout << "End of Validity not set, start of validity plus 1 day used." << endl;
LOGP(info, "End of Validity not set, start of validity plus 1 day used.");
sanitizedEndValidityTimestamp = getFutureTimestamp(60 * 60 * 24 * 1);
}

Expand Down Expand Up @@ -200,8 +201,7 @@ int CcdbApi::storeAsBinaryFile(const char* buffer, size_t size, const std::strin
res = curl_easy_perform(curl);
/* Check for errors */
if (res != CURLE_OK) {
fprintf(stderr, "curl_easy_perform() failed: %s\n",
curl_easy_strerror(res));
LOGP(alarm, "curl_easy_perform() failed: {}", curl_easy_strerror(res));
returnValue = res;
}
}
Expand All @@ -214,7 +214,7 @@ int CcdbApi::storeAsBinaryFile(const char* buffer, size_t size, const std::strin
/* free slist */
curl_slist_free_all(headerlist);
} else {
cerr << "curl initialization failure" << endl;
LOGP(alarm, "curl initialization failure");
returnValue = -2;
}
return returnValue;
Expand Down Expand Up @@ -485,18 +485,17 @@ bool CcdbApi::receiveObject(void* dataHolder, std::string const& path, std::map<
curlResultCode = curl_easy_perform(curlHandle);

if (curlResultCode != CURLE_OK) {
fprintf(stderr, "curl_easy_perform() failed: %s\n",
curl_easy_strerror(curlResultCode));
LOGP(alarm, "curl_easy_perform() failed: {}", curl_easy_strerror(curlResultCode));
} else {
curlResultCode = curl_easy_getinfo(curlHandle, CURLINFO_RESPONSE_CODE, &responseCode);
if ((curlResultCode == CURLE_OK) && (responseCode < 300)) {
curl_easy_cleanup(curlHandle);
return true;
} else {
if (curlResultCode != CURLE_OK) {
cerr << "invalid URL : " << fullUrl << endl;
LOGP(alarm, "invalid URL {}", fullUrl);
} else {
cerr << "not found under link: " << fullUrl << endl;
LOGP(alarm, "not found under link {}", fullUrl);
}
}
}
Expand Down Expand Up @@ -526,7 +525,7 @@ TObject* CcdbApi::retrieve(std::string const& path, std::map<std::string, std::s
mess.Reset();
result = (TObject*)(mess.ReadObjectAny(mess.GetClass()));
if (result == nullptr) {
cerr << "couldn't retrieve the object " << path << endl;
LOGP(info, "couldn't retrieve the object {}", path);
}
}

Expand Down Expand Up @@ -1068,7 +1067,7 @@ std::string CcdbApi::list(std::string const& path, bool latestOnly, std::string

res = curl_easy_perform(curl);
if (res != CURLE_OK) {
fprintf(stderr, "curl_easy_perform() failed: %s\n", curl_easy_strerror(res));
LOGP(alarm, "curl_easy_perform() failed: {}", curl_easy_strerror(res));
}
}
curl_slist_free_all(headers);
Expand Down Expand Up @@ -1104,7 +1103,7 @@ void CcdbApi::deleteObject(std::string const& path, long timestamp) const
// Perform the request, res will get the return code
res = curl_easy_perform(curl);
if (res != CURLE_OK) {
fprintf(stderr, "curl_easy_perform() failed: %s\n", curl_easy_strerror(res));
LOGP(alarm, "curl_easy_perform() failed: {}", curl_easy_strerror(res));
}
curl_easy_cleanup(curl);
}
Expand All @@ -1129,7 +1128,7 @@ void CcdbApi::truncate(std::string const& path) const
// Perform the request, res will get the return code
res = curl_easy_perform(curl);
if (res != CURLE_OK) {
fprintf(stderr, "curl_easy_perform() failed: %s\n", curl_easy_strerror(res));
LOGP(alarm, "curl_easy_perform() failed: {}", curl_easy_strerror(res));
}
curl_easy_cleanup(curl);
}
Expand Down Expand Up @@ -1380,7 +1379,7 @@ void CcdbApi::updateMetadata(std::string const& path, std::map<std::string, std:
// Perform the request, res will get the return code
res = curl_easy_perform(curl);
if (res != CURLE_OK) {
fprintf(stderr, "curl_easy_perform() failed: %s\n", curl_easy_strerror(res));
LOGP(alarm, "curl_easy_perform() failed: {}", curl_easy_strerror(res));
}
curl_easy_cleanup(curl);
}
Expand Down
1 change: 1 addition & 0 deletions Detectors/Calibration/README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -96,6 +96,7 @@ o2-calibration-ccdb-populator-workflow --sspec-min 0 --sspec-max 1 -b
then the `ObjA` will be uploaded only to the default server (`http://alice-ccdb.cern.ch`), `ObjB` will be uploaded to both default and `local` server and
`ObjC` will be uploaded to the `local` server only.

But default the ccdb-populator-workflow will produce `fatal` on failed upload. To avoid this a switch `--no-fatal-on-failure` can be used.
<!-- doxy
* \subpage refDetectorsCalibrationtestMacros
/doxy -->
12 changes: 9 additions & 3 deletions Detectors/Calibration/workflow/CCDBPopulatorSpec.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,7 @@ class CCDBPopulator : public o2::framework::Task
mCCDBpath = ic.options().get<std::string>("ccdb-path");
mSSpecMin = ic.options().get<std::int64_t>("sspec-min");
mSSpecMax = ic.options().get<std::int64_t>("sspec-max");
mFatalOnFailure = ic.options().get<bool>("no-fatal-on-failure");
auto& mgr = CcdbManager::instance();
mgr.setURL(mCCDBpath);
mAPI.init(mgr.getURL());
Expand Down Expand Up @@ -84,13 +85,17 @@ class CCDBPopulator : public o2::framework::Task

LOG(info) << "Storing in ccdb " << wrp->getPath() << "/" << wrp->getFileName() << " of size " << pld.size()
<< " Valid for " << wrp->getStartValidityTimestamp() << " : " << wrp->getEndValidityTimestamp();
mAPI.storeAsBinaryFile(&pld[0], pld.size(), wrp->getFileName(), wrp->getObjectType(), wrp->getPath(),
*md, wrp->getStartValidityTimestamp(), wrp->getEndValidityTimestamp());
int res = mAPI.storeAsBinaryFile(&pld[0], pld.size(), wrp->getFileName(), wrp->getObjectType(), wrp->getPath(),
*md, wrp->getStartValidityTimestamp(), wrp->getEndValidityTimestamp());
if (res && mFatalOnFailure) {
LOGP(fatal, "failed on uploading to {} / {}", mAPI.getURL(), wrp->getPath());
}
}
}

private:
CcdbApi mAPI;
bool mFatalOnFailure = true; // produce fatal on failed upload
std::int64_t mSSpecMin = -1; // min subspec to accept
std::int64_t mSSpecMax = -1; // max subspec to accept
std::string mCCDBpath = "http://ccdb-test.cern.ch:8080"; // CCDB path
Expand All @@ -115,7 +120,8 @@ DataProcessorSpec getCCDBPopulatorDeviceSpec(const std::string& defCCDB, const s
Options{
{"ccdb-path", VariantType::String, defCCDB, {"Path to CCDB"}},
{"sspec-min", VariantType::Int64, -1L, {"min subspec to accept"}},
{"sspec-max", VariantType::Int64, -1L, {"max subspec to accept"}}}};
{"sspec-max", VariantType::Int64, -1L, {"max subspec to accept"}},
{"no-fatal-on-failure", VariantType::Bool, false, {"do not produce fatal on failed upload"}}}};
}

} // namespace framework
Expand Down