From dc1e5d336d5f95c3d0b1f2b179088457d6b87a4e Mon Sep 17 00:00:00 2001 From: nvalle Date: Sun, 4 Aug 2024 21:47:15 +0200 Subject: [PATCH 1/2] ITS - (and MFT), better TF sampling in the deadmap builder --- .../include/ITSMFTWorkflow/DeadMapBuilderSpec.h | 2 +- .../common/workflow/src/DeadMapBuilderSpec.cxx | 14 +++++--------- 2 files changed, 6 insertions(+), 10 deletions(-) diff --git a/Detectors/ITSMFT/common/workflow/include/ITSMFTWorkflow/DeadMapBuilderSpec.h b/Detectors/ITSMFT/common/workflow/include/ITSMFTWorkflow/DeadMapBuilderSpec.h index 14f4575ee972c..9a4df9ad39fd2 100644 --- a/Detectors/ITSMFT/common/workflow/include/ITSMFTWorkflow/DeadMapBuilderSpec.h +++ b/Detectors/ITSMFT/common/workflow/include/ITSMFTWorkflow/DeadMapBuilderSpec.h @@ -104,11 +104,11 @@ class ITSMFTDeadMapBuilder : public Task unsigned long mFirstOrbitTF = 0x0; unsigned long mFirstOrbitRun = 0x0; + long mLastSampledOrbit = 0x0; std::string mDataSource = "chipsstatus"; int mTFSampling = 350; - std::string mSamplingMode = "first-orbit-run"; // Use this default to ensure process of first TF. At the moment, use any other option to sample on absolute orbit value. o2::itsmft::TimeDeadMap mMapObject; diff --git a/Detectors/ITSMFT/common/workflow/src/DeadMapBuilderSpec.cxx b/Detectors/ITSMFT/common/workflow/src/DeadMapBuilderSpec.cxx index 8bb86a547fb25..78b79f22c6db7 100644 --- a/Detectors/ITSMFT/common/workflow/src/DeadMapBuilderSpec.cxx +++ b/Detectors/ITSMFT/common/workflow/src/DeadMapBuilderSpec.cxx @@ -47,7 +47,6 @@ void ITSMFTDeadMapBuilder::init(InitContext& ic) LOG(info) << "ITSMFTDeadMapBuilder init... " << mSelfName; mTFSampling = ic.options().get("tf-sampling"); - mSamplingMode = ic.options().get("sampling-mode"); mTFLength = ic.options().get("tf-length"); mDoLocalOutput = ic.options().get("local-output"); mObjectName = ic.options().get("outfile"); @@ -151,16 +150,14 @@ void ITSMFTDeadMapBuilder::run(ProcessingContext& pc) mFirstOrbitRun = mFirstOrbitTF; } - long sampled_orbit = mFirstOrbitTF; - if (mSamplingMode == "first-orbit-run") { - sampled_orbit = sampled_orbit - mFirstOrbitRun; - } + long sampled_orbit = mFirstOrbitTF - mFirstOrbitRun; - if ((sampled_orbit / mTFLength) % mTFSampling != 0) { + if ((sampled_orbit / mTFLength) % mTFSampling != 0 && sampled_orbit <= mLastSampledOrbit + mTFLength * mTFSampling) { return; } mStepCounter++; + mLastSampledOrbit = sampled_orbit; LOG(info) << "Processing step #" << mStepCounter << " out of " << mTFCounter << " TF received. First orbit " << mFirstOrbitTF; mDeadMapTF.clear(); @@ -270,7 +267,7 @@ void ITSMFTDeadMapBuilder::PrepareOutputCcdb(EndOfStreamContext* ec, std::string if (ec != nullptr) { LOG(important) << "Sending object " << info.getPath() << "/" << info.getFileName() - << "to ccdb-populator, of size " << image->size() << " bytes, valid for " + << " to ccdb-populator, of size " << image->size() << " bytes, valid for " << info.getStartValidityTimestamp() << " : " << info.getEndValidityTimestamp(); if (mRunMFT) { @@ -332,7 +329,7 @@ void ITSMFTDeadMapBuilder::stop() finalizeOutput(); if (!mCCDBUrl.empty()) { std::string detname = mRunMFT ? "MFT" : "ITS"; - LOG(warning) << "endOfStream not processed. Sending output to ccdb from the " << detname << "deadmap builder workflow."; + LOG(warning) << "endOfStream not processed. Sending output to ccdb from the " << detname << " deadmap builder workflow."; PrepareOutputCcdb(nullptr, mCCDBUrl); } else { LOG(alarm) << "endOfStream not processed. Nothing forwarded as output."; @@ -379,7 +376,6 @@ DataProcessorSpec getITSMFTDeadMapBuilderSpec(std::string datasource, bool doMFT outputs, AlgorithmSpec{adaptFromTask(datasource, doMFT)}, Options{{"tf-sampling", VariantType::Int, 350, {"Process every Nth TF. Selection according to first TF orbit."}}, - {"sampling-mode", VariantType::String, "first-orbit-run", {"Use absolute orbit value or offset from first processed orbit."}}, {"tf-length", VariantType::Int, 32, {"Orbits per TF."}}, {"skip-static-map", VariantType::Bool, false, {"Do not fill static part of the map."}}, {"ccdb-url", VariantType::String, "", {"CCDB url. Ignored if endOfStream is processed."}}, From e380aa136f7f9d75f2bb9201278797992ebeeb39 Mon Sep 17 00:00:00 2001 From: nvalle Date: Thu, 8 Aug 2024 21:07:22 +0200 Subject: [PATCH 2/2] improved sampling algorithm --- .../ITSMFTWorkflow/DeadMapBuilderSpec.h | 8 +++- .../workflow/src/DeadMapBuilderSpec.cxx | 48 +++++++++++++++++-- 2 files changed, 52 insertions(+), 4 deletions(-) diff --git a/Detectors/ITSMFT/common/workflow/include/ITSMFTWorkflow/DeadMapBuilderSpec.h b/Detectors/ITSMFT/common/workflow/include/ITSMFTWorkflow/DeadMapBuilderSpec.h index 9a4df9ad39fd2..00f9fc931e679 100644 --- a/Detectors/ITSMFT/common/workflow/include/ITSMFTWorkflow/DeadMapBuilderSpec.h +++ b/Detectors/ITSMFT/common/workflow/include/ITSMFTWorkflow/DeadMapBuilderSpec.h @@ -104,12 +104,17 @@ class ITSMFTDeadMapBuilder : public Task unsigned long mFirstOrbitTF = 0x0; unsigned long mFirstOrbitRun = 0x0; - long mLastSampledOrbit = 0x0; std::string mDataSource = "chipsstatus"; int mTFSampling = 350; + // utils for an improved sampling algorithm + std::unordered_set mSampledTFs{}; + std::deque mSampledHistory{}; + int mTFSamplingTolerance = 20; + int mSampledSlidingWindowSize = 1000; // units: mTFSampling + o2::itsmft::TimeDeadMap mMapObject; void finalizeOutput(); @@ -118,6 +123,7 @@ class ITSMFTDeadMapBuilder : public Task // Utils std::vector getChipIDsOnSameCable(uint16_t); + bool acceptTF(long); o2::framework::DataTakingContext mDataTakingContext{}; o2::framework::TimingInfo mTimingInfo{}; diff --git a/Detectors/ITSMFT/common/workflow/src/DeadMapBuilderSpec.cxx b/Detectors/ITSMFT/common/workflow/src/DeadMapBuilderSpec.cxx index 78b79f22c6db7..39d1bb7df7728 100644 --- a/Detectors/ITSMFT/common/workflow/src/DeadMapBuilderSpec.cxx +++ b/Detectors/ITSMFT/common/workflow/src/DeadMapBuilderSpec.cxx @@ -47,6 +47,12 @@ void ITSMFTDeadMapBuilder::init(InitContext& ic) LOG(info) << "ITSMFTDeadMapBuilder init... " << mSelfName; mTFSampling = ic.options().get("tf-sampling"); + mTFSamplingTolerance = ic.options().get("tf-sampling-tolerance"); + if (mTFSamplingTolerance > mTFSampling) { + LOG(warning) << "Invalid request tf-sampling-tolerance larger or equal than tf-sampling. Setting tolerance to " << mTFSampling - 1; + mTFSamplingTolerance = mTFSampling - 1; + } + mSampledSlidingWindowSize = ic.options().get("tf-sampling-history-size"); mTFLength = ic.options().get("tf-length"); mDoLocalOutput = ic.options().get("local-output"); mObjectName = ic.options().get("outfile"); @@ -66,6 +72,8 @@ void ITSMFTDeadMapBuilder::init(InitContext& ic) N_CHIPS = o2::itsmft::ChipMappingITS::getNChips(); } + mSampledTFs.clear(); + mSampledHistory.clear(); mDeadMapTF.clear(); mStaticChipStatus.clear(); mMapObject.clear(); @@ -75,7 +83,7 @@ void ITSMFTDeadMapBuilder::init(InitContext& ic) mStaticChipStatus.resize(N_CHIPS, false); } - LOG(info) << "Sampling one TF every " << mTFSampling; + LOG(info) << "Sampling one TF every " << mTFSampling << " with " << mTFSamplingTolerance << " TF tolerance"; return; } @@ -94,6 +102,39 @@ std::vector ITSMFTDeadMapBuilder::getChipIDsOnSameCable(uint16_t chip) } } +bool ITSMFTDeadMapBuilder::acceptTF(long orbit) +{ + + // Description of the algorithm: + // Return true if the TF index (calculated as orbit/TF_length) falls within any interval [k * tf_sampling, k * tf_sampling + tolerance) for some integer k, provided no other TFs have been found in the same interval. + + if (mTFSamplingTolerance < 1) { + return ((orbit / mTFLength) % mTFSampling == 0); + } + + if ((orbit / mTFLength) % mTFSampling > mTFSamplingTolerance) { + return false; + } + + long sampling_index = orbit / mTFLength / mTFSampling; + + if (mSampledTFs.find(sampling_index) == mSampledTFs.end()) { + + mSampledTFs.insert(sampling_index); + mSampledHistory.push_back(sampling_index); + + if (mSampledHistory.size() > mSampledSlidingWindowSize) { + long oldIndex = mSampledHistory.front(); + mSampledHistory.pop_front(); + mSampledTFs.erase(oldIndex); + } + + return true; + } + + return false; +} + ////////////////////////////////////////////////////////////////////////////// void ITSMFTDeadMapBuilder::finalizeOutput() @@ -152,12 +193,11 @@ void ITSMFTDeadMapBuilder::run(ProcessingContext& pc) long sampled_orbit = mFirstOrbitTF - mFirstOrbitRun; - if ((sampled_orbit / mTFLength) % mTFSampling != 0 && sampled_orbit <= mLastSampledOrbit + mTFLength * mTFSampling) { + if (!acceptTF(sampled_orbit)) { return; } mStepCounter++; - mLastSampledOrbit = sampled_orbit; LOG(info) << "Processing step #" << mStepCounter << " out of " << mTFCounter << " TF received. First orbit " << mFirstOrbitTF; mDeadMapTF.clear(); @@ -376,6 +416,8 @@ DataProcessorSpec getITSMFTDeadMapBuilderSpec(std::string datasource, bool doMFT outputs, AlgorithmSpec{adaptFromTask(datasource, doMFT)}, Options{{"tf-sampling", VariantType::Int, 350, {"Process every Nth TF. Selection according to first TF orbit."}}, + {"tf-sampling-tolerance", VariantType::Int, 20, {"Tolerance on the tf-sampling value (sliding window size)."}}, + {"tf-sampling-history-size", VariantType::Int, 1000, {"Do not check if new TF is contained in a window that is older than N steps."}}, {"tf-length", VariantType::Int, 32, {"Orbits per TF."}}, {"skip-static-map", VariantType::Bool, false, {"Do not fill static part of the map."}}, {"ccdb-url", VariantType::String, "", {"CCDB url. Ignored if endOfStream is processed."}},