Merged Noise Map branch

DataFormats/Detectors/ITSMFT/common/include/DataFormatsITSMFT/NoiseMap.h

@@ -169,7 +169,6 @@ class NoiseMap
   // Methods required by the calibration framework
   void print();
   void fill(const gsl::span<const CompClusterExt> data);
-  void merge(const NoiseMap* prev) {}
 
   const std::map<int, int>* getChipMap(int chip) const { return chip < (int)mNoisyPixels.size() ? &mNoisyPixels[chip] : nullptr; }
 
@@ -203,6 +202,20 @@ class NoiseMap
     return std::ceil((1. + 1. / t) / (relErr * relErr));
   }
 
+  NoiseMap merge(const NoiseMap* prev)
+  {
+    int incre = 0;
+    for (size_t i = 0; i < (int)mNoisyPixels.size(); ++i) {
+      for (const auto& prev_np : prev->mNoisyPixels[i]) { // only enters this for loop if the "i" chip exists.
+        if (mNoisyPixels[i].find(prev_np.first) == mNoisyPixels[i].end()) {
+          mNoisyPixels[i][prev_np.first] = prev_np.second;
+          incre++;
+        }
+      } // end of for loop on elements of previous noise map
+    } // end of for loop on i (chip ID)
+    return (mNoisyPixels);
+  } // end of void merge
+
   size_t size() const { return mNoisyPixels.size(); }
   void setNumOfStrobes(long n) { mNumOfStrobes = n; }
   void addStrobes(long n) { mNumOfStrobes += n; }

Detectors/ITSMFT/MFT/calibration/include/MFTCalibration/NoiseCalibratorSpec.h

@@ -16,6 +16,8 @@
 
 #include <string>
 
+#include "CCDB/CcdbApi.h"
+
 #include "Framework/DataProcessorSpec.h"
 #include "Framework/Task.h"
 
@@ -48,19 +50,23 @@ class NoiseCalibratorSpec : public Task
   void finaliseCCDB(ConcreteDataMatcher& matcher, void* obj) final;
 
  private:
+  o2::ccdb::CcdbApi api;
   void updateTimeDependentParams(ProcessingContext& pc);
   void sendOutputCcdb(DataAllocator& output);
+  void sendOutputCcdbMerge(DataAllocator& output);
   void sendOutputCcdbDcs(DataAllocator& output);
   void sendOutputDcs(DataAllocator& output);
   void setOutputDcs(const o2::itsmft::NoiseMap& payload);
   o2::itsmft::NoiseMap mNoiseMap{936};
   std::unique_ptr<CALIBRATOR> mCalibrator = nullptr;
   std::shared_ptr<o2::base::GRPGeomRequest> mCCDBRequest;
   std::string mPath;
+  std::string mPathMerge;
   std::string mMeta;
 
   std::vector<std::array<int, 3>> mNoiseMapForDcs;
   std::string mPathDcs;
+  std::string mPathDcsMerge;
   std::string mOutputType;
 
   double mThresh;

Detectors/ITSMFT/MFT/calibration/src/NoiseCalibratorSpec.cxx

@@ -48,6 +48,8 @@ void NoiseCalibratorSpec::init(InitContext& ic)
   LOGP(info, "Setting the probability threshold to {} with relative error {}", probT, probTRelErr);
   mStopMeOnly = ic.options().get<bool>("stop-me-only");
   mPath = ic.options().get<std::string>("path-CCDB");
+  mPathMerge = ic.options().get<std::string>("path-CCDB-merge");
+
   mMeta = ic.options().get<std::string>("meta");
   mStart = ic.options().get<int64_t>("tstart");
   mEnd = ic.options().get<int64_t>("tend");
@@ -57,6 +59,7 @@ void NoiseCalibratorSpec::init(InitContext& ic)
   mPathDcs = ic.options().get<std::string>("path-DCS");
   mOutputType = ic.options().get<std::string>("send-to-server");
   mNoiseMapForDcs.clear();
+  api.init("http://alice-ccdb.cern.ch");
 }
 
 void NoiseCalibratorSpec::run(ProcessingContext& pc)
@@ -72,6 +75,8 @@ void NoiseCalibratorSpec::run(ProcessingContext& pc)
       if (mOutputType.compare("CCDB") == 0) {
         LOG(info) << "Sending an object to Production-CCDB";
         sendOutputCcdb(pc.outputs());
+        LOG(info) << "Sending an object to Production-CCDBMerge";
+        sendOutputCcdbMerge(pc.outputs());
       } else if (mOutputType.compare("DCS") == 0) {
         LOG(info) << "Sending an object to DCS-CCDB";
         sendOutputDcs(pc.outputs());
@@ -92,6 +97,8 @@ void NoiseCalibratorSpec::run(ProcessingContext& pc)
       if (mOutputType.compare("CCDB") == 0) {
         LOG(info) << "Sending an object to Production-CCDB";
         sendOutputCcdb(pc.outputs());
+        LOG(info) << "Sending an object to Production-CCDBMerge";
+        sendOutputCcdbMerge(pc.outputs());
       } else if (mOutputType.compare("DCS") == 0) {
         LOG(info) << "Sending an object to DCS-CCDB";
         sendOutputDcs(pc.outputs());
@@ -237,7 +244,73 @@ void NoiseCalibratorSpec::sendOutputCcdb(DataAllocator& output)
   auto flName = o2::ccdb::CcdbApi::generateFileName("noise");
   auto image = o2::ccdb::CcdbApi::createObjectImage(&payload, &info);
   info.setFileName(flName);
-  LOG(info) << "Sending object " << info.getPath() << "/" << info.getFileName()
+  LOG(info) << "Sending object CCDB " << info.getPath() << "/" << info.getFileName()
+            << " of size " << image->size()
+            << " bytes, valid for " << info.getStartValidityTimestamp()
+            << " : " << info.getEndValidityTimestamp();
+
+  using clbUtils = o2::calibration::Utils;
+  output.snapshot(Output{clbUtils::gDataOriginCDBPayload, "MFT_NoiseMap", 0}, *image.get());
+  output.snapshot(Output{clbUtils::gDataOriginCDBWrapper, "MFT_NoiseMap", 0}, info);
+}
+
+void NoiseCalibratorSpec::sendOutputCcdbMerge(DataAllocator& output)
+{
+
+  LOG(info) << "CCDB-Merge mode";
+
+  static bool done = false;
+  if (done) {
+    return;
+  }
+  done = true;
+
+  mCalibrator->finalize();
+
+  long tstart = mStart;
+  if (tstart == -1) {
+    tstart = o2::ccdb::getCurrentTimestamp();
+  }
+  long tend = mEnd;
+  if (tend == -1) {
+    constexpr long SECONDSPERYEAR = 365 * 24 * 60 * 60;
+    tend = o2::ccdb::getFutureTimestamp(SECONDSPERYEAR);
+  }
+
+  std::map<std::string, std::string> meta;
+  auto toKeyValPairs = [&meta](std::vector<std::string> const& tokens) {
+    for (auto& token : tokens) {
+      auto keyval = Str::tokenize(token, '=', false);
+      if (keyval.size() != 2) {
+        LOG(error) << "Illegal command-line key/value string: " << token;
+        continue;
+      }
+      Str::trim(keyval[1]);
+      meta[keyval[0]] = keyval[1];
+    }
+  };
+  toKeyValPairs(Str::tokenize(mMeta, ';', true));
+
+  long startTF, endTF;
+
+  auto payload = mCalibrator->getNoiseMap();
+  //  const auto& payload = mCalibrator->getNoiseMap(starTF, endTF); //For TimeSlot calibration
+  map<string, string> headers;
+  map<std::string, std::string> filter;
+  auto* payloadPrev1 = api.retrieveFromTFileAny<o2::itsmft::NoiseMap>(mPath, filter, -1, &headers);
+  long validtime = std::stol(headers["Valid-From"]);
+  auto mergedPL = payload;
+  if (validtime > 0) {
+    validtime = validtime - 1;
+    auto* payloadPrev2 = api.retrieveFromTFileAny<o2::itsmft::NoiseMap>(mPath, filter, validtime, &headers);
+    auto bufferPL = payloadPrev2->merge(payloadPrev1);
+    mergedPL = payload.merge(&bufferPL);
+  }
+  o2::ccdb::CcdbObjectInfo info(mPathMerge, "NoiseMap", "noise.root", meta, tstart, tend);
+  auto flName = o2::ccdb::CcdbApi::generateFileName("noise");
+  auto image = o2::ccdb::CcdbApi::createObjectImage(&mergedPL, &info);
+  info.setFileName(flName);
+  LOG(info) << "Sending object ccdb-merge " << info.getPath() << "/" << info.getFileName()
             << " of size " << image->size()
             << " bytes, valid for " << info.getStartValidityTimestamp()
             << " : " << info.getEndValidityTimestamp();
@@ -310,12 +383,15 @@ void NoiseCalibratorSpec::endOfStream(o2::framework::EndOfStreamContext& ec)
   if (mOutputType.compare("CCDB") == 0) {
     LOG(info) << "Sending an object to Production-CCDB";
     sendOutputCcdb(ec.outputs());
+    LOG(info) << "Sending an object to Production-CCDB-Merge";
+    sendOutputCcdbMerge(ec.outputs());
   } else if (mOutputType.compare("DCS") == 0) {
     LOG(info) << "Sending an object to DCS-CCDB";
     sendOutputDcs(ec.outputs());
   } else {
     LOG(info) << "Sending an object to Production-CCDB and DCS-CCDB";
     sendOutputCcdbDcs(ec.outputs());
+    sendOutputCcdbMerge(ec.outputs());
   }
 }
 
@@ -374,7 +450,8 @@ DataProcessorSpec getNoiseCalibratorSpec(bool useDigits)
       {"tstart", VariantType::Int64, -1ll, {"Start of validity timestamp"}},
       {"tend", VariantType::Int64, -1ll, {"End of validity timestamp"}},
       {"path-CCDB", VariantType::String, "/MFT/Calib/NoiseMap", {"Path to write to in CCDB"}},
-      {"path-DCS", VariantType::String, "/MFT/Config/NoiseMap", {"Path to write to in CCDB"}},
+      {"path-CCDB-merge", VariantType::String, "/MFT/Calib/NoiseMapMerged", {"Path to write merged file to in CCDB"}},
+      {"path-DCS", VariantType::String, "/MFT/Config/NoiseMap", {"Path to write to in DCS"}},
       {"meta", VariantType::String, "", {"meta data to write in CCDB"}},
       {"send-to-server", VariantType::String, "CCDB-DCS", {"meta data to write in DCS-CCDB"}},
       {"stop-me-only", VariantType::Bool, false, {"At sufficient statistics stop only this device, otherwise whole workflow"}}}};