From 75e580c395731eb77fd806262a136fe30bc49929 Mon Sep 17 00:00:00 2001 From: lynnt20 Date: Wed, 2 Jul 2025 15:47:09 -0500 Subject: [PATCH 1/8] make comment syntax uniform --- .../genie_corsika/prodgenie_corsika_proton_rockbox_sbnd.fcl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sbndcode/JobConfigurations/standard/gen/genie_corsika/prodgenie_corsika_proton_rockbox_sbnd.fcl b/sbndcode/JobConfigurations/standard/gen/genie_corsika/prodgenie_corsika_proton_rockbox_sbnd.fcl index 52efb6487..1fb5dd807 100644 --- a/sbndcode/JobConfigurations/standard/gen/genie_corsika/prodgenie_corsika_proton_rockbox_sbnd.fcl +++ b/sbndcode/JobConfigurations/standard/gen/genie_corsika/prodgenie_corsika_proton_rockbox_sbnd.fcl @@ -22,13 +22,13 @@ physics.producers.largeant: @local::sbnd_merge_overlay_sim_sources # Tell the dirt filter to use the new largeant for neutrinos only physics.filters.dirtfilter.SimEnergyDepModuleName: "largeantnu:LArG4DetectorServicevolTPCActive" -#Merge dropped MCParticle collections +# Merge dropped MCParticle collections physics.producers.largeantdropped: @local::sbnd_merge_dropped_mcpart_overlay # Change simple merge inputs since we're using an overlay sample physics.producers.simplemerge.InputSourcesLabels: ["largeant", "largeantdropped"] -// Change the name of largeant->largeantnu and add corsika and largeant merging +# Change the name of largeant->largeantnu and add corsika and largeant merging physics.simulatetpc: [ rns, generator, loader, largeantnu, tpcfilter, corsika, largeantcosmic, largeant, largeantdropped, simplemerge ] physics.simulatedirt: [ rns, generator, loader, largeantnu, "!tpcfilter", dirtfilter, corsika, largeantcosmic, largeant, largeantdropped, simplemerge ] From 2c83865e8f660f9b349dcd112db68cef27f3770e Mon Sep 17 00:00:00 2001 From: lynnt20 Date: Wed, 2 Jul 2025 15:47:24 -0500 Subject: [PATCH 2/8] create new rockbox fcl that includes ccnue filter --- .../prodgenie_corsika_proton_rockbox_ccnue_sbnd.fcl | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 sbndcode/JobConfigurations/standard/gen/genie_corsika/prodgenie_corsika_proton_rockbox_ccnue_sbnd.fcl diff --git a/sbndcode/JobConfigurations/standard/gen/genie_corsika/prodgenie_corsika_proton_rockbox_ccnue_sbnd.fcl b/sbndcode/JobConfigurations/standard/gen/genie_corsika/prodgenie_corsika_proton_rockbox_ccnue_sbnd.fcl new file mode 100644 index 000000000..bd8909aa1 --- /dev/null +++ b/sbndcode/JobConfigurations/standard/gen/genie_corsika/prodgenie_corsika_proton_rockbox_ccnue_sbnd.fcl @@ -0,0 +1,8 @@ +#include "filters_sbnd.fcl" +#include "prodgenie_corsika_proton_rockbox_sbnd.fcl" + +physics.filters.ccnuefilter: @local::sbnd_ccnuefilter + +physics.simulatetpc: [ rns, generator, loader, largeantnu, tpcfilter, + ccnuefilter, # adding this filter to the path + corsika, largeantcosmic, largeant, largeantdropped, simplemerge ] \ No newline at end of file From 280e2c6e8178a594581d0d64c262d244e7b95dd3 Mon Sep 17 00:00:00 2001 From: lynnt20 Date: Wed, 2 Jul 2025 15:47:56 -0500 Subject: [PATCH 3/8] add new `ccnuefilter` filter block, remove deprecated fcl block --- sbndcode/Filters/fcls/filters_sbnd.fcl | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/sbndcode/Filters/fcls/filters_sbnd.fcl b/sbndcode/Filters/fcls/filters_sbnd.fcl index 30e3efb8e..b500461f8 100644 --- a/sbndcode/Filters/fcls/filters_sbnd.fcl +++ b/sbndcode/Filters/fcls/filters_sbnd.fcl @@ -7,10 +7,13 @@ sbnd_finalstateparticlefilter: PDG: [13, 2212] } -sbnd_nuefilter: +sbnd_ccnuefilter: { - module_type: NueFilter - PdgCode: 12 # 12=nue, 14=numu + module_type: GenNuFilter + VtxInTPC: true + CC: true + NC: false + LepPDGs: [11,-11] } From d46ac48d48b02c2196b7fb50924b614bd741c654 Mon Sep 17 00:00:00 2001 From: lynnt20 Date: Tue, 8 Jul 2025 10:52:02 -0500 Subject: [PATCH 4/8] fix to ensure that events with only numu dirt do not pass --- .../prodgenie_corsika_proton_rockbox_ccnue_sbnd.fcl | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/sbndcode/JobConfigurations/standard/gen/genie_corsika/prodgenie_corsika_proton_rockbox_ccnue_sbnd.fcl b/sbndcode/JobConfigurations/standard/gen/genie_corsika/prodgenie_corsika_proton_rockbox_ccnue_sbnd.fcl index bd8909aa1..9fc2123c6 100644 --- a/sbndcode/JobConfigurations/standard/gen/genie_corsika/prodgenie_corsika_proton_rockbox_ccnue_sbnd.fcl +++ b/sbndcode/JobConfigurations/standard/gen/genie_corsika/prodgenie_corsika_proton_rockbox_ccnue_sbnd.fcl @@ -3,6 +3,10 @@ physics.filters.ccnuefilter: @local::sbnd_ccnuefilter -physics.simulatetpc: [ rns, generator, loader, largeantnu, tpcfilter, - ccnuefilter, # adding this filter to the path - corsika, largeantcosmic, largeant, largeantdropped, simplemerge ] \ No newline at end of file +physics.simulatetpc: [ rns, generator, loader, largeantnu, + ccnuefilter, + corsika, largeantcosmic, largeant, largeantdropped, simplemerge] + +physics.simulatedirt: [ rns, generator, loader, largeantnu, "!tpcfilter", + dirtfilter, corsika, largeantcosmic, largeant, largeantdropped, simplemerge, + ccnuefilter] From bb0bde15102aba6e8c8a464778bf9fa07c5fee65 Mon Sep 17 00:00:00 2001 From: lynnt20 Date: Tue, 8 Jul 2025 11:58:49 -0500 Subject: [PATCH 5/8] Revert "fix to ensure that events with only numu dirt do not pass" This reverts commit d46ac48d48b02c2196b7fb50924b614bd741c654. --- .../prodgenie_corsika_proton_rockbox_ccnue_sbnd.fcl | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/sbndcode/JobConfigurations/standard/gen/genie_corsika/prodgenie_corsika_proton_rockbox_ccnue_sbnd.fcl b/sbndcode/JobConfigurations/standard/gen/genie_corsika/prodgenie_corsika_proton_rockbox_ccnue_sbnd.fcl index 9fc2123c6..bd8909aa1 100644 --- a/sbndcode/JobConfigurations/standard/gen/genie_corsika/prodgenie_corsika_proton_rockbox_ccnue_sbnd.fcl +++ b/sbndcode/JobConfigurations/standard/gen/genie_corsika/prodgenie_corsika_proton_rockbox_ccnue_sbnd.fcl @@ -3,10 +3,6 @@ physics.filters.ccnuefilter: @local::sbnd_ccnuefilter -physics.simulatetpc: [ rns, generator, loader, largeantnu, - ccnuefilter, - corsika, largeantcosmic, largeant, largeantdropped, simplemerge] - -physics.simulatedirt: [ rns, generator, loader, largeantnu, "!tpcfilter", - dirtfilter, corsika, largeantcosmic, largeant, largeantdropped, simplemerge, - ccnuefilter] +physics.simulatetpc: [ rns, generator, loader, largeantnu, tpcfilter, + ccnuefilter, # adding this filter to the path + corsika, largeantcosmic, largeant, largeantdropped, simplemerge ] \ No newline at end of file From d5e3270686e3fb9a57ef153929ec81562b3cfd4a Mon Sep 17 00:00:00 2001 From: lynnt20 Date: Tue, 8 Jul 2025 12:00:00 -0500 Subject: [PATCH 6/8] Revert "Merge branch 'develop' into feature/lynnt_ccnuefilter" This reverts commit d93ec8ecd3a849dcf8b35d235ab560d66a200348, reversing changes made to d46ac48d48b02c2196b7fb50924b614bd741c654. --- .github/pull_request_template.md | 7 - CMakeLists.txt | 2 +- README.md | 4 - .../base/cafmakerjob_sbnd_data_base.fcl | 5 +- .../standard/caf/cafmakerjob_sbnd.fcl | 2 - .../caf/cafmakerjob_sbnd_data_sce.fcl | 1 - .../standard/caf/cafmakerjob_sbnd_sce.fcl | 1 - .../standard/reco/reco1_data.fcl | 9 - .../standard/standard_detsim_sbnd.fcl | 6 - .../standard/standard_reco1_sbnd.fcl | 3 - .../Alg/OpDeconvolutionAlgWienerData_tool.cc | 89 +++- .../Alg/opdeconvolution_alg_data.fcl | 1 + .../experiment/sbnd/chndb-base.jsonnet | 33 +- .../pgrapher/experiment/sbnd/dnnroi.jsonnet | 112 ----- .../experiment/sbnd/wcls-nf-sp-data.jsonnet | 179 +------ .../experiment/sbnd/wcls-nf-sp.jsonnet | 179 +------ ...m-drift-depoflux-nf-sp-samples_rec.jsonnet | 373 -------------- ...m-drift-depoflux-nf-sp-samples_tru.jsonnet | 474 ------------------ .../wcls-sim-drift-depoflux-nf-sp.jsonnet | 196 +------- sbndcode/WireCell/wcsimsp_sbnd.fcl | 111 +--- sbndcode/WireCell/wcsp_data_sbnd.fcl | 10 +- sbndcode/WireCell/wirecell_sim_rec_sbnd.fcl | 90 ---- sbndcode/WireCell/wirecell_sim_tru_sbnd.fcl | 91 ---- ups/product_deps | 4 +- 24 files changed, 101 insertions(+), 1881 deletions(-) delete mode 100644 README.md delete mode 100644 sbndcode/WireCell/cfg/pgrapher/experiment/sbnd/dnnroi.jsonnet delete mode 100644 sbndcode/WireCell/cfg/pgrapher/experiment/sbnd/wcls-sim-drift-depoflux-nf-sp-samples_rec.jsonnet delete mode 100644 sbndcode/WireCell/cfg/pgrapher/experiment/sbnd/wcls-sim-drift-depoflux-nf-sp-samples_tru.jsonnet delete mode 100644 sbndcode/WireCell/wirecell_sim_rec_sbnd.fcl delete mode 100644 sbndcode/WireCell/wirecell_sim_tru_sbnd.fcl diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index f6aa43877..64be2e089 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -1,12 +1,6 @@ ## Description Please provide a detailed description of the changes this pull request introduces. -$${\color{red}!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!}$$ - -$${\color{red}\bf{\textrm{IMPORTANT UPDATE June 22nd 2025:}}}$$ If you are making a PR which is intended as a patch for the CURRENT production (which started in Spring 2025), you must make two PRs: one for develop and one for the production/v10_06_00 branch. - -$${\color{red}!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!}$$ - ## Checklist - [ ] Added at least 1 label from [available labels](https://github.com/SBNSoftware/sbndcode/issues/labels?sort=name-asc). - [ ] Assigned at least 1 reviewer under `Reviewers`, @@ -14,7 +8,6 @@ $${\color{red}!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - [ ] Linked any relevant issues under `Developement` - [ ] Does this PR affect CAF data format? If so, please assign a CAF maintainer ([PetrilloAtWork](https://github.com/PetrilloAtWork) or [JosiePaton](https://github.com/JosiePaton)) as additional reviewer. - [ ] Does this affect the standard workflow? -- [ ] Is this PR a patch for the ongoing production? If so, separate PR must also be made for production/v10_06_00 branch! ### Relevant PR links (optional) Does this PR require merging another PR in a different repository (such as sbnanobj/sbnobj etc.)? diff --git a/CMakeLists.txt b/CMakeLists.txt index 7a0f6e62c..d470b1cbc 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -15,7 +15,7 @@ cmake_minimum_required(VERSION 3.20 FATAL_ERROR) -set(${PROJECT_NAME}_CMAKE_PROJECT_VERSION_STRING 10.06.01) +set(${PROJECT_NAME}_CMAKE_PROJECT_VERSION_STRING 10.06.00.01) find_package(cetmodules REQUIRED) project(sbndcode LANGUAGES CXX) diff --git a/README.md b/README.md deleted file mode 100644 index 25a00403e..000000000 --- a/README.md +++ /dev/null @@ -1,4 +0,0 @@ -# sbndcode - -## Getting Started -[SBN Young Guide](https://sbnsoftware.github.io/SBNYoung/SBNYoungIndex.html) diff --git a/sbndcode/JobConfigurations/base/cafmakerjob_sbnd_data_base.fcl b/sbndcode/JobConfigurations/base/cafmakerjob_sbnd_data_base.fcl index 16be4bd56..8ce957011 100644 --- a/sbndcode/JobConfigurations/base/cafmakerjob_sbnd_data_base.fcl +++ b/sbndcode/JobConfigurations/base/cafmakerjob_sbnd_data_base.fcl @@ -125,11 +125,8 @@ physics: # change tools in sbn to have 2D TPC sim/sp input physics.producers.pandoraShowerRazzle.SimChannelLabel: "sptpc2d:gauss" physics.producers.pandoraTrackDazzle.SimChannelLabel: "sptpc2d:gauss" + physics.producers.cnnid.WireLabel: "sptpc2d:gauss" -# uncomment below lines to use DNN ROI SP for Razzle/Dazzle/CNNID -#physics.producers.pandoraShowerRazzle.SimChannelLabel: "sptpc2d:dnnsp" -#physics.producers.pandoraTrackDazzle.SimChannelLabel: "sptpc2d:dnnsp" -#physics.producers.cnnid.WireLabel: "sptpc2d:dnnsp" ### Calorimetry for data physics.producers.cnnid.PointIdAlg.CalorimetryAlg: @local::sbnd_calorimetryalgdata diff --git a/sbndcode/JobConfigurations/standard/caf/cafmakerjob_sbnd.fcl b/sbndcode/JobConfigurations/standard/caf/cafmakerjob_sbnd.fcl index 2281046cf..3adef6ba4 100644 --- a/sbndcode/JobConfigurations/standard/caf/cafmakerjob_sbnd.fcl +++ b/sbndcode/JobConfigurations/standard/caf/cafmakerjob_sbnd.fcl @@ -127,8 +127,6 @@ physics: physics.producers.pandoraShowerRazzle.SimChannelLabel: "simtpc2d:simpleSC" physics.producers.pandoraTrackDazzle.SimChannelLabel: "simtpc2d:simpleSC" physics.producers.cnnid.WireLabel: "simtpc2d:gauss" -# uncomment below line for to use DNN ROI SP for CNNID scores -#physics.producers.cnnid.WireLabel: "simtpc2d:dnnsp" physics.producers.vertexCharge.CaloAlg: @local::sbnd_calorimetryalgmc physics.producers.vertexStub.CaloAlg: @local::sbnd_calorimetryalgmc diff --git a/sbndcode/JobConfigurations/standard/caf/cafmakerjob_sbnd_data_sce.fcl b/sbndcode/JobConfigurations/standard/caf/cafmakerjob_sbnd_data_sce.fcl index 4ffb5ba94..16827b43c 100644 --- a/sbndcode/JobConfigurations/standard/caf/cafmakerjob_sbnd_data_sce.fcl +++ b/sbndcode/JobConfigurations/standard/caf/cafmakerjob_sbnd_data_sce.fcl @@ -11,7 +11,6 @@ physics.producers.cafmaker.TrackChi2PidLabel: "pandoraSCEPidData" physics.producers.cafmaker.CRTSpacePointMatchLabel: "crtspacepointmatchingSCE" physics.producers.cafmaker.SBNDCRTTrackMatchLabel: "crttrackmatchingSCE" physics.producers.cafmaker.OpT0Label: "opt0finderSCE" -physics.producers.cafmaker.TPCPMTBarycenterMatchLabel: "tpcpmtbarycentermatchingSCE" physics.producers.cafmaker.CVNLabel: "cvnSCE" physics.producers.cnnid.ClusterModuleLabel: "pandoraSCE" diff --git a/sbndcode/JobConfigurations/standard/caf/cafmakerjob_sbnd_sce.fcl b/sbndcode/JobConfigurations/standard/caf/cafmakerjob_sbnd_sce.fcl index 177326bfe..5e626a694 100644 --- a/sbndcode/JobConfigurations/standard/caf/cafmakerjob_sbnd_sce.fcl +++ b/sbndcode/JobConfigurations/standard/caf/cafmakerjob_sbnd_sce.fcl @@ -13,7 +13,6 @@ physics.producers.cafmaker.SBNDCRTTrackMatchLabel: "crttrackmatchingSCE" physics.producers.cafmaker.CRTHitMatchLabel: "pandoraSCETrackCRTHit" physics.producers.cafmaker.CRTTrackMatchLabel: "pandoraSCETrackCRTTrack" physics.producers.cafmaker.OpT0Label: "opt0finderSCE" -physics.producers.cafmaker.TPCPMTBarycenterMatchLabel: "tpcpmtbarycentermatchingSCE" physics.producers.cafmaker.CVNLabel: "cvnSCE" physics.producers.cnnid.ClusterModuleLabel: "pandoraSCE" diff --git a/sbndcode/JobConfigurations/standard/reco/reco1_data.fcl b/sbndcode/JobConfigurations/standard/reco/reco1_data.fcl index 563a4b64a..896b52c6c 100644 --- a/sbndcode/JobConfigurations/standard/reco/reco1_data.fcl +++ b/sbndcode/JobConfigurations/standard/reco/reco1_data.fcl @@ -35,12 +35,3 @@ physics.ana: [superadata] outputs.out1.SelectEvents: [ "reco1" ] physics.producers.gaushit.CalDataModuleLabel: "sptpc2d:gauss" - -# uncomment below 4 lines to run DNN ROI finding SP -#physics.producers.sptpc2d.wcls_main.outputers: ["wclsFrameSaver:spsaver" , "wclsFrameSaver:dnnsaver"] -#physics.producers.sptpc2d.wcls_main.structs.use_dnnroi: true -#physics.producers.sptpc2d.wcls_main.structs.nchunks: 2 # should match training config -#physics.producers.sptpc2d.wcls_main.structs.tick_per_slice: 4 # should match training config - -# uncomment below line to run DNN ROI finding SP for reco -#physics.producers.gaushit.CalDataModuleLabel: "sptpc2d:dnnsp" diff --git a/sbndcode/JobConfigurations/standard/standard_detsim_sbnd.fcl b/sbndcode/JobConfigurations/standard/standard_detsim_sbnd.fcl index b13b52cb0..1bb03bcc0 100644 --- a/sbndcode/JobConfigurations/standard/standard_detsim_sbnd.fcl +++ b/sbndcode/JobConfigurations/standard/standard_detsim_sbnd.fcl @@ -92,9 +92,3 @@ outputs: ] } } - -# uncomment below 4 lines to run DNN ROI finding SP -# physics.producers.simtpc2d.wcls_main.outputers: ["wclsDepoFluxWriter:postdrift", "wclsFrameSaver:spsaver", "wclsFrameSaver:dnnsaver"] # "wclsFrameSaver:simdigits" <- by default, do not save RawDigits. Uncomment this line to save RawDigits and set -# physics.producers.simtpc2d.wcls_main.structs.use_dnnroi: true -# physics.producers.simtpc2d.wcls_main.structs.nchunks: 2 # should match training config -# physics.producers.simtpc2d.wcls_main.structs.tick_per_slice: 4 # should match training config diff --git a/sbndcode/JobConfigurations/standard/standard_reco1_sbnd.fcl b/sbndcode/JobConfigurations/standard/standard_reco1_sbnd.fcl index 90dd8eb13..b6a95ce30 100644 --- a/sbndcode/JobConfigurations/standard/standard_reco1_sbnd.fcl +++ b/sbndcode/JobConfigurations/standard/standard_reco1_sbnd.fcl @@ -35,6 +35,3 @@ physics.end_paths: [ @sequence::physics.end_paths, ana ] #outputs table overrides outputs.out1.dataTier: "reconstructed" outputs.out1.outputCommands: [@sequence::outputs.out1.outputCommands, @sequence::sbnd_reco1_drops] - -# uncomment below line to run DNN ROI finding SP for reco -- must have run DNN ROI finding in detsim stage -#physics.producers.gaushit.CalDataModuleLabel: "simtpc2d:dnnsp" diff --git a/sbndcode/OpDetReco/OpDeconvolution/Alg/OpDeconvolutionAlgWienerData_tool.cc b/sbndcode/OpDetReco/OpDeconvolution/Alg/OpDeconvolutionAlgWienerData_tool.cc index d33cc57ee..cbd23acef 100644 --- a/sbndcode/OpDetReco/OpDeconvolution/Alg/OpDeconvolutionAlgWienerData_tool.cc +++ b/sbndcode/OpDetReco/OpDeconvolution/Alg/OpDeconvolutionAlgWienerData_tool.cc @@ -27,8 +27,6 @@ #include "sbndcode/OpDetReco/OpDeconvolution/Alg/OpDeconvolutionAlg.hh" -#include "sbndcode/Calibration/PDSDatabaseInterface/PMTCalibrationDatabase.h" -#include "sbndcode/Calibration/PDSDatabaseInterface/IPMTCalibrationDatabaseService.h" namespace opdet { class OpDeconvolutionAlgWiener; @@ -39,7 +37,7 @@ class opdet::OpDeconvolutionAlgWiener : opdet::OpDeconvolutionAlg { public: explicit OpDeconvolutionAlgWiener(fhicl::ParameterSet const& p); - ~OpDeconvolutionAlgWiener() {delete fFilterTF1;} + ~OpDeconvolutionAlgWiener() {} // Required functions. std::vector RunDeconvolution(std::vector const& wfHandle) override; @@ -67,6 +65,7 @@ class opdet::OpDeconvolutionAlgWiener : opdet::OpDeconvolutionAlg { double fPMTChargeToADC; double fDecoWaveformPrecision; short unsigned int fBaselineSample; + std::string fOpDetDataFile; std::string fFilter; std::string fElectronics; bool fScaleHypoSignal; @@ -81,7 +80,6 @@ class opdet::OpDeconvolutionAlgWiener : opdet::OpDeconvolutionAlg { unsigned int NDecoWf; TF1 *fFilterTF1; - std::vector fSignalHypothesis; std::vector fNoiseHypothesis; @@ -90,8 +88,6 @@ class opdet::OpDeconvolutionAlgWiener : opdet::OpDeconvolutionAlg { short unsigned int fBaseSampleBins; double fBaseVarCut; - sbndDB::PMTCalibrationDatabase const* fPMTCalibrationDatabaseService; - // Declare member functions void ApplyExpoAvSmoothing(std::vector& wf); void ApplyUnAvSmoothing(std::vector& wf); @@ -129,6 +125,7 @@ opdet::OpDeconvolutionAlgWiener::OpDeconvolutionAlgWiener(fhicl::ParameterSet co fPMTChargeToADC = p.get< double >("PMTChargeToADC"); fDecoWaveformPrecision = p.get< double >("DecoWaveformPrecision"); fBaselineSample = p.get< short unsigned int >("BaselineSample"); + fOpDetDataFile = p.get< std::string >("OpDetDataFile"); fSkipChannelList = p.get< std::vector>("SkipChannelList"); fFilter = p.get< std::string >("Filter"); fElectronics = p.get< std::string >("Electronics"); @@ -141,8 +138,6 @@ opdet::OpDeconvolutionAlgWiener::OpDeconvolutionAlgWiener(fhicl::ParameterSet co fBaseVarCut = p.get< double >("BaseVarCut"); fFilterParams = p.get< std::vector >("FilterParams"); - fFilterTF1 = new TF1("FilterTemplate", fFilter.c_str()); - fNormUnAvSmooth=1./(2*fUnAvNeighbours+1); NDecoWf=0; MaxBinsFFT=std::pow(2, fMaxFFTSizePow); @@ -152,10 +147,38 @@ opdet::OpDeconvolutionAlgWiener::OpDeconvolutionAlgWiener(fhicl::ParameterSet co if (fElectronics=="Daphne") fSamplingFreq=fDaphne_Freq/1000.;//in GHz auto const* lar_prop = lar::providerFrom(); - //Load PMT Calibration Database - fPMTCalibrationDatabaseService = lar::providerFrom(); - - if (!fUseParamFilter){ + //Load SER + std::string fname; + cet::search_path sp("FW_SEARCH_PATH"); + sp.find_file(fOpDetDataFile, fname); + TFile* file = TFile::Open(fname.c_str(), "READ"); + std::vector>* SinglePEVec_p; + std::vector* fSinglePEChannels_p; + std::vector* fPeakAmplitude_p; + std::vector> * fFilterParamVector_p; + + file->GetObject("SERChannels", fSinglePEChannels_p); + file->GetObject("SinglePEVec", SinglePEVec_p); + file->GetObject("PeakAmplitude", fPeakAmplitude_p); + file->GetObject("FilterParams", fFilterParamVector_p); + + if (fElectronics=="Daphne") file->GetObject("SinglePEVec_40ftCable_Daphne", SinglePEVec_p); + fSinglePEWaveVector = *SinglePEVec_p; + fSinglePEChannels = *fSinglePEChannels_p; + fPeakAmplitude = *fPeakAmplitude_p; + fFilterParamVector = *fFilterParamVector_p; + + mf::LogInfo("OpDeconvolutionAlg")<<"Loaded SER from "<SetParameter(k, fFilterParams[k]); + mf::LogInfo("OpDeconvolutionAlg")<<"Creating parametrized filter... TF1:"< opdet::OpDeconvolutionAlgWiener::RunDeconvolutio { int channelNumber = wf.ChannelNumber(); auto it = std::find(fSkipChannelList.begin(), fSkipChannelList.end(), channelNumber); + bool AnalyseChannel = false; if (it == fSkipChannelList.end()) { - fSinglePEWave = fPMTCalibrationDatabaseService->getSER(channelNumber); - double SPEAmplitude = fPMTCalibrationDatabaseService->getSPEAmplitude(channelNumber); - double SPEPeakValue = *std::max_element(fSinglePEWave.begin(), fSinglePEWave.end(), [](double a, double b) {return std::abs(a) < std::abs(b);}); - double SinglePENormalization = std::abs(SPEAmplitude/SPEPeakValue); - std::transform(fSinglePEWave.begin(), fSinglePEWave.end(), fSinglePEWave.begin(), [SinglePENormalization](double val) {return val * SinglePENormalization;}); - fSinglePEWave.resize(MaxBinsFFT, 0); - // If use channel dependent param filter - if(fUseParamFilter) + //If it's not try to find its SER in the file + for(size_t i=0; igetGaussFilterPower(channelNumber); - double GaussFilterWC = fPMTCalibrationDatabaseService->getGaussFilterWC(channelNumber); - fFilterParamChannel = {GaussFilterWC, GaussFilterPower}; - for(size_t k=0; kSetParameter(k, fFilterParamChannel[k]); + fSinglePEWave = fSinglePEWaveVector[i]; + double SPEPeakValue = *std::max_element(fSinglePEWave.begin(), fSinglePEWave.end(), [](double a, double b) {return std::abs(a) < std::abs(b);}); + double SinglePENormalization = std::abs(fPeakAmplitude[i]/SPEPeakValue); + std::transform(fSinglePEWave.begin(), fSinglePEWave.end(), fSinglePEWave.begin(), [SinglePENormalization](double val) {return val * SinglePENormalization;}); + fSinglePEWave.resize(MaxBinsFFT, 0); + AnalyseChannel = true; + // If use channel dependent param filter + if(fUseParamFilterInidividualChannel) + { + fFilterParamChannel = fFilterParamVector[i]; + fFilterTF1 = new TF1("FilterTemplate", fFilter.c_str()); + for(size_t k=0; kSetParameter(k, fFilterParamChannel[k]); + } + + + mf::LogInfo("OpDeconvolutionAlg")<<"Creating parametrized filter... TF1:"<MaxBinsFFT){ @@ -502,6 +537,7 @@ std::vector opdet::OpDeconvolutionAlgWiener::DeconvolutionKernel(size_ } } + if(fDebug){ std::string name="h_wienerfilter_"+std::to_string(NDecoWf); TH1F * hs_wiener = tfs->make< TH1F > @@ -509,6 +545,7 @@ std::vector opdet::OpDeconvolutionAlgWiener::DeconvolutionKernel(size_ for(size_t k=0; kSetBinContent(k, TComplex::Abs( kernel[k]*serfft[k] ) ); } + if(fUseParamFilter && fUseParamFilterInidividualChannel) delete fFilterTF1; return kernel; } diff --git a/sbndcode/OpDetReco/OpDeconvolution/Alg/opdeconvolution_alg_data.fcl b/sbndcode/OpDetReco/OpDeconvolution/Alg/opdeconvolution_alg_data.fcl index 6d6821780..fb1f68deb 100644 --- a/sbndcode/OpDetReco/OpDeconvolution/Alg/opdeconvolution_alg_data.fcl +++ b/sbndcode/OpDetReco/OpDeconvolution/Alg/opdeconvolution_alg_data.fcl @@ -6,6 +6,7 @@ OpDeconvolutionAlgData: tool_type: "OpDeconvolutionAlgWienerData" Debug: false MaxFFTSizePow: 16 + OpDetDataFile: "./OpDetSim/digi_pmt_sbnd_data_OV6.root" BaseSampleBins: 30 BaseVarCut: 50 SkipChannelList: [] diff --git a/sbndcode/WireCell/cfg/pgrapher/experiment/sbnd/chndb-base.jsonnet b/sbndcode/WireCell/cfg/pgrapher/experiment/sbnd/chndb-base.jsonnet index f08a523b7..0da0eccc1 100644 --- a/sbndcode/WireCell/cfg/pgrapher/experiment/sbnd/chndb-base.jsonnet +++ b/sbndcode/WireCell/cfg/pgrapher/experiment/sbnd/chndb-base.jsonnet @@ -22,31 +22,16 @@ function(params, anode, field, n, rms_cuts=[]) // coherent noise filtering. // 5638 is the number of channels in a single APA, (1984*2 + 1670), // including the 6 channel gap. - // The induction planes have two types of ch grouping (due to FEMB installation) - // one where the grouping is 32 channels wide and one where it is 128 - // (but split into 2 groups of 64) channels wide. - // This grouping is flipped for the same plane in the second APA. - // The collection planes are grouped by 64. - // Due to a group of dead 32 channels in collection plane in APA0, - // we specify separate groups of 32 (dead) and 32 (alive) within that group of 64 - // so that coherent noise filtering is applied properly to the alive channels. + // The induction planes have to types of ch grouping (due to FEMB), + // one where the grouping is 32 channels wide and one where it is 128 channels wide. + // The collection planes are grouped by 64. - groups: [std.range( 0 + g*32 , 0 + (g+1)*32 - 1) for g in std.range(0,25)] + # first section of u0 - [std.range( 832 + g*64 , 832 + (g+1)*64 - 1) for g in std.range(0,17)] + # second section of u0 - [std.range(1984 + g*64 , 1984 + (g+1)*64 - 1) for g in std.range(0,17)] + # first section of v0 - [std.range(3136 + g*32 , 3136 + (g+1)*32 - 1) for g in std.range(0,25)] + # second section of v0 - - [std.range(5638 + g*64 , 5638 + (g+1)*64 - 1) for g in std.range(0,17)] + # first section of u1 - [std.range(6790 + g*32 , 6790 + (g+1)*32 - 1) for g in std.range(0,25)] + # second section of u1 - [std.range(7622 + g*32 , 7622 + (g+1)*32 - 1) for g in std.range(0,25)] + # first section of v1 - [std.range(8454 + g*64 , 8454 + (g+1)*64 - 1) for g in std.range(0,17)] + # second section of v1 - - [std.range(3968 + n * 5638 + g*64 , 3968 + n * 5638 + (g+1)*64 - 1) for g in std.range(0,2)] + # first three groups of w - [std.range(4160 , 4192 - 1)] + # w apa0 4th group, first 32 dead wire (for completeness) - [std.range(4192 , 4224 - 1)] + # w apa0 4th group, remaining 32 alive wire (next to dead wire) - [std.range(9798 , 9862 - 1)] + # w apa1 4th group of 64 (alive) - [std.range(4224 + n * 5638 + g*64 , 4224 + n * 5638 + (g+1)*64 - 1) for g in std.range(0,8)] + # first half of w (excluding first 4 groups) - [std.range(4806 + n * 5638 + g*64 , 4806 + n * 5638 + (g+1)*64 - 1) for g in std.range(0,12)] , # second half of w + groups: [std.range( 0 + n * 5638 + g*32 , 0 + n * 5638 + (g+1)*32 - 1) for g in std.range(0,26)] + # first section of u + [std.range( 832 + n * 5638 + g*128, 832 + n * 5638 + (g+1)*128 - 1) for g in std.range(0,9)] + # second section of u + [std.range(1984 + n * 5638 + g*128, 1984 + n * 5638 + (g+1)*128 - 1) for g in std.range(0,9)] + # first section of v + [std.range(3136 + n * 5638 + g*32 , 3136 + n * 5638 + (g+1)*32 - 1) for g in std.range(0,26)] + # second section of v + [std.range(3974 + n * 5638 + g*64 , 3974 + n * 5638 + (g+1)*64 - 1) for g in std.range(0,13)] + # first half of w + [std.range(4806 + n * 5638 + g*64 , 4806 + n * 5638 + (g+1)*64 - 1) for g in std.range(0,13)] , # second half of w // Externally determined "bad" channels. // diff --git a/sbndcode/WireCell/cfg/pgrapher/experiment/sbnd/dnnroi.jsonnet b/sbndcode/WireCell/cfg/pgrapher/experiment/sbnd/dnnroi.jsonnet deleted file mode 100644 index 909ad4b23..000000000 --- a/sbndcode/WireCell/cfg/pgrapher/experiment/sbnd/dnnroi.jsonnet +++ /dev/null @@ -1,112 +0,0 @@ -// This produces a function to configure DNN-ROI for one APA given -// anode and torch service (ts) objects. -// -// The prefix is prepended to all internal node names if uniqueness -// beyond anode ID is needed. The output_scale allows for an ad-hoc -// scaling of dnnroi output. The U and W planes will go through -// dnnroi while hte W plane will be shunted. What comes out will be a -// unified frame with frame tag "dnnspN" where "N" is the anode ID. - -local wc = import "wirecell.jsonnet"; -local pg = import "pgraph.jsonnet"; - - -function (anode, ts_p0, ts_p1, prefix="dnnroi", output_scale=1.0, nticks=3428, tick_per_slice=4, nchunks=1) - local apaid = anode.data.ident; - local prename = prefix + std.toString(apaid); - local intags = ['loose_lf%d'%apaid, 'mp2_roi%d'%apaid, - 'mp3_roi%d'%apaid]; - - local dnnroi_u = pg.pnode({ - type: "DNNROIFinding", - name: prename+"u", - data: { - anode: wc.tn(anode), - plane: 0, - intags: intags, - decon_charge_tag: "decon_charge%d" %apaid, - outtag: "dnnsp%du"%apaid, - input_scale: 0.00025, // 1/4000 - output_scale: output_scale, - forward: wc.tn(ts_p0), - tick_per_slice: tick_per_slice, - nticks: nticks, - nchunks: nchunks - } - }, nin=1, nout=1, uses=[ts_p0, anode]); - local dnnroi_v = pg.pnode({ - type: "DNNROIFinding", - name: prename+"v", - data: { - anode: wc.tn(anode), - plane: 1, - intags: intags, - decon_charge_tag: "decon_charge%d" %apaid, - outtag: "dnnsp%dv"%apaid, - input_scale: 0.00025, // 1/4000 - output_scale: output_scale, - forward: wc.tn(ts_p1), - tick_per_slice: tick_per_slice, - nticks: nticks, - nchunks: nchunks - } - }, nin=1, nout=1, uses=[ts_p1, anode]); - local dnnroi_w = pg.pnode({ - type: "PlaneSelector", - name: prename+"w", - data: { - anode: wc.tn(anode), - plane: 2, - tags: ["gauss%d"%apaid], - tag_rules: [{ - frame: {".*":"DNNROIFinding"}, - trace: {["gauss%d"%apaid]:"dnnsp%dw"%apaid}, - }], - } - }, nin=1, nout=1, uses=[anode]); - - local dnnpipes = [dnnroi_u, dnnroi_v, dnnroi_w]; - local dnnfanout = pg.pnode({ - type: "FrameFanout", - name: prename, - data: { - multiplicity: 3 - } - }, nin=1, nout=3); - - local dnnfanin = pg.pnode({ - type: "FrameFanin", - name: prename, - data: { - multiplicity: 3, - tag_rules: [{ - frame: {".*": "dnnsp%d%s" % [apaid,plane]}, - trace: {".*": "dnnsp%d%s" % [apaid,plane]}, - } for plane in ["u", "v", "w"]] - }, - }, nin=3, nout=1); - - local retagger = pg.pnode({ - type: "Retagger", - name: 'dnnroi%d' % apaid, - data: { - // Note: retagger keeps tag_rules an array to be like frame fanin/fanout. - tag_rules: [{ - // Retagger also handles "frame" and "trace" like fanin/fanout - // merge separately all traces like gaussN to gauss. - frame: { - ".*": "dnnsp%d" % apaid - }, - merge: { - ".*": "dnnsp%d" % apaid - }, - }], - }, - }, nin=1, nout=1); - - pg.intern(innodes=[dnnfanout], - outnodes=[retagger], - centernodes=dnnpipes+[dnnfanin], - edges=[pg.edge(dnnfanout, dnnpipes[ind], ind, 0) for ind in [0,1,2]] + - [pg.edge(dnnpipes[ind], dnnfanin, 0, ind) for ind in [0,1,2]] + - [pg.edge(dnnfanin, retagger, 0, 0)]) diff --git a/sbndcode/WireCell/cfg/pgrapher/experiment/sbnd/wcls-nf-sp-data.jsonnet b/sbndcode/WireCell/cfg/pgrapher/experiment/sbnd/wcls-nf-sp-data.jsonnet index 875c5bfd8..e64652a28 100644 --- a/sbndcode/WireCell/cfg/pgrapher/experiment/sbnd/wcls-nf-sp-data.jsonnet +++ b/sbndcode/WireCell/cfg/pgrapher/experiment/sbnd/wcls-nf-sp-data.jsonnet @@ -24,11 +24,6 @@ local sigoutform = std.extVar('signal_output_form'); // eg "sparse" or "dense" local raw_input_label = std.extVar('raw_input_label'); // eg "daq" local use_paramresp = std.extVar('use_paramresp'); // eg "true" or "false" -local use_dnnroi = std.extVar('use_dnnroi'); -local nchunks = std.extVar('nchunks'); -local tick_per_slice = std.extVar('tick_per_slice'); -local dnnroi_model_p0 = std.extVar('dnnroi_model_p0'); -local dnnroi_model_p1 = std.extVar('dnnroi_model_p1'); local g = import 'pgraph.jsonnet'; local f = import 'pgrapher/experiment/sbnd/funcs.jsonnet'; @@ -129,22 +124,6 @@ local wcls_output = { chanmaskmaps: ['bad'], }, }, nin=1, nout=1, uses=[mega_anode]), - - dnnsp_signals: g.pnode({ - type: 'wclsFrameSaver', - name: 'dnnsaver', - data: { - anode: wc.tn(mega_anode), - digitize: false, // true means save as RawDigit, else recob::Wire - frame_tags: ['dnnsp'], - - // this may be needed to convert the decon charge [units:e-] to be consistent with the LArSoft default ?unit? e.g. decon charge * 0.005 --> "charge value" to GaussHitFinder - frame_scale: [0.02, 0.02, 0.02], - nticks: params.daq.nticks, - - chanmaskmaps: ['dnnspbad'], - }, - }, nin=1, nout=1, uses=[mega_anode]), }; local base = import 'pgrapher/experiment/sbnd/chndb-base.jsonnet'; @@ -172,85 +151,14 @@ local nf_maker = import 'pgrapher/experiment/sbnd/nf-data.jsonnet'; //added Ewer local nf_pipes = [nf_maker(params, tools.anodes[n], chndb[n], n, name='nf%d' % n) for n in std.range(0, std.length(tools.anodes) - 1)]; local sp_maker = import 'pgrapher/experiment/sbnd/sp.jsonnet'; -local sp_override = if use_dnnroi then { - sparse: true, - use_roi_debug_mode: true, - save_negtive_charge: true, // TODO: no negative charge in gauss, default is false - use_multi_plane_protection: true, - mp_tick_resolution: 4, - tight_lf_tag: "", - // loose_lf_tag: "", - cleanup_roi_tag: "", - break_roi_loop1_tag: "", - break_roi_loop2_tag: "", - shrink_roi_tag: "", - extend_roi_tag: "", - // m_decon_charge_tag: "", -} else { - sparse: true, -}; -//local sp = sp_maker(params, tools, { sparse: sigoutform == 'sparse' }); -local sp = sp_maker(params, tools, sp_override); +local sp = sp_maker(params, tools, { sparse: sigoutform == 'sparse' }); local sp_pipes = [sp.make_sigproc(a) for a in tools.anodes]; -local dnnroi = import 'dnnroi.jsonnet'; -local ts_p0 = { - type: "TorchService", - name: "dnnroi_p0", - tick_per_slice: tick_per_slice, - data: { - model: dnnroi_model_p0, - device: "cpu", - concurrency: 1, - }, -}; - -local ts_p1 = { - type: "TorchService", - name: "dnnroi_p1", - tick_per_slice: tick_per_slice, - data: { - model: dnnroi_model_p1, - device: "cpu", - concurrency: 1, - }, -}; local magoutput = 'sbnd-data-check.root'; local magnify = import 'pgrapher/experiment/sbnd/magnify-sinks.jsonnet'; local sinks = magnify(tools, magoutput); -local fanout = function (name, multiplicity=2) - g.pnode({ - type: 'FrameFanout', - name: name, - data: { - multiplicity: multiplicity - }, - }, nin=1, nout=multiplicity); - -local sp_fans = [fanout("sp_fan_%d" % n) for n in std.range(0, std.length(tools.anodes) - 1)]; -local dnnroi_pipes = [ dnnroi(tools.anodes[n], ts_p0, ts_p1, output_scale=1, nchunks=nchunks) for n in std.range(0, std.length(tools.anodes) - 1) ]; -local nfsp_pipes = if use_dnnroi then -[ - g.intern( - innodes=[chsel_pipes[n]], - outnodes=[dnnroi_pipes[n],sinks.decon_pipe[n]], - centernodes=[nf_pipes[n], sp_pipes[n], sp_fans[n], sinks.decon_pipe[n]], - edges=[ - g.edge(chsel_pipes[n], nf_pipes[n], 0, 0), - g.edge(nf_pipes[n], sp_pipes[n], 0, 0), - g.edge(sp_pipes[n], sp_fans[n], 0, 0), - g.edge(sp_fans[n], dnnroi_pipes[n], 0, 0), - g.edge(sp_fans[n], sinks.decon_pipe[n], 1, 0), - ], - iports=chsel_pipes[n].iports, - oports=dnnroi_pipes[n].oports+sinks.decon_pipe[n].oports, - name='nfsp_pipe_%d' % n, - ) - for n in std.range(0, std.length(tools.anodes) - 1) -] -else -[ +local nfsp_pipes = [ g.pipeline([ chsel_pipes[n], //sinks.orig_pipe[n], @@ -269,9 +177,8 @@ else local fanpipe = f.fanpipe('FrameFanout', nfsp_pipes, 'FrameFanin', 'sn_mag_nf'); -local retagger = function(name) g.pnode({ +local retagger = g.pnode({ type: 'Retagger', - name: name, data: { // Note: retagger keeps tag_rules an array to be like frame fanin/fanout. tag_rules: [{ @@ -283,89 +190,16 @@ local retagger = function(name) g.pnode({ merge: { 'gauss\\d': 'gauss', 'wiener\\d': 'wiener', - 'dnnsp\\d': 'dnnsp', }, }], }, }, nin=1, nout=1); -local retag_dnnroi = retagger("retag_dnnroi"); -local retag_sp = retagger("retag_sp"); +local sink = g.pnode({ type: 'DumpFrames' }, nin=1, nout=0); -local sink = function(name) g.pnode({ type: 'DumpFrames', name: name }, nin=1, nout=0); -local sink_dnnroi = sink("sink_dnnroi"); -local sink_sp = sink("sink_sp"); - -local fanout_apa = g.pnode({ - type: 'FrameFanout', - name: 'fanout_apa', - data: { - multiplicity: std.length(tools.anodes), - "tag_rules": [ - { - "frame": { - ".*": "orig%d" % n - }, - "trace": { } - } - for n in std.range(0, std.length(tools.anodes) - 1) - ] - }}, - nin=1, nout=std.length(tools.anodes)); -local framefanin = function(name) g.pnode({ - type: 'FrameFanin', - name: name, - data: { - multiplicity: std.length(tools.anodes), - - "tag_rules": [ - { - "frame": { - ".*": "framefanin" - }, - trace: { - ['dnnsp%d' % n]: ['dnnsp%d' % n], - ['gauss%d' % n]: ['gauss%d' % n], - ['wiener%d' % n]: ['wiener%d' % n], - ['threshold%d' % n]: ['threshold%d' % n], - }, - } - for n in std.range(0, std.length(tools.anodes) - 1) - ], - "tags": [ ] - }, -}, nin=std.length(tools.anodes), nout=1); -local fanin_apa_dnnroi = framefanin('fanin_apa_dnnroi'); -local fanin_apa_sp = framefanin('fanin_apa_sp'); - -local graph = if use_dnnroi then - g.intern( - innodes=[wcls_input.adc_digits], - outnodes=[], - centernodes=nfsp_pipes+[fanout_apa, retag_dnnroi, retag_sp, fanin_apa_dnnroi, fanin_apa_sp, wcls_output.sp_signals, wcls_output.dnnsp_signals, sink_dnnroi, sink_sp], - edges=[ - g.edge(wcls_input.adc_digits, fanout_apa, 0, 0), - g.edge(fanout_apa, nfsp_pipes[0], 0, 0), - g.edge(fanout_apa, nfsp_pipes[1], 1, 0), - g.edge(nfsp_pipes[0], fanin_apa_dnnroi, 0, 0), - g.edge(nfsp_pipes[1], fanin_apa_dnnroi, 0, 1), - g.edge(fanin_apa_dnnroi, retag_dnnroi, 0, 0), - g.edge(retag_dnnroi, wcls_output.dnnsp_signals, 0, 0), - g.edge(wcls_output.dnnsp_signals, sink_dnnroi, 0, 0), - g.edge(nfsp_pipes[0], fanin_apa_sp, 1, 0), - g.edge(nfsp_pipes[1], fanin_apa_sp, 1, 1), - g.edge(fanin_apa_sp, retag_sp, 0, 0), - g.edge(retag_sp, wcls_output.sp_signals, 0, 0), - g.edge(wcls_output.sp_signals, sink_sp, 0, 0), - ] -) -else -g.pipeline([wcls_input.adc_digits, -fanpipe, -retag_sp, -wcls_output.sp_signals, -sink_sp]); +// local graph = g.pipeline([wcls_input.adc_digits, rootfile_creation_frames, fanpipe, retagger, wcls_output.sp_signals, sink]); +local graph = g.pipeline([wcls_input.adc_digits, fanpipe, retagger, wcls_output.sp_signals, sink]); local app = { type: 'TbbFlow', @@ -376,4 +210,3 @@ local app = { // Finally, the configuration sequence g.uses(graph) + [app] - diff --git a/sbndcode/WireCell/cfg/pgrapher/experiment/sbnd/wcls-nf-sp.jsonnet b/sbndcode/WireCell/cfg/pgrapher/experiment/sbnd/wcls-nf-sp.jsonnet index 90528f446..9eed918fb 100644 --- a/sbndcode/WireCell/cfg/pgrapher/experiment/sbnd/wcls-nf-sp.jsonnet +++ b/sbndcode/WireCell/cfg/pgrapher/experiment/sbnd/wcls-nf-sp.jsonnet @@ -23,11 +23,6 @@ local epoch = std.extVar('epoch'); // eg "dynamic", "after", "before", "perfect" local sigoutform = std.extVar('signal_output_form'); // eg "sparse" or "dense" local raw_input_label = std.extVar('raw_input_label'); // eg "daq" -local use_dnnroi = std.extVar('use_dnnroi'); -local nchunks = std.extVar('nchunks'); -local tick_per_slice = std.extVar('tick_per_slice'); -local dnnroi_model_p0 = std.extVar('dnnroi_model_p0'); -local dnnroi_model_p1 = std.extVar('dnnroi_model_p1'); local g = import 'pgraph.jsonnet'; local f = import 'pgrapher/experiment/sbnd/funcs.jsonnet'; @@ -109,21 +104,6 @@ local wcls_output = { chanmaskmaps: [], }, }, nin=1, nout=1, uses=[mega_anode]), - dnnsp_signals: g.pnode({ - type: 'wclsFrameSaver', - name: 'dnnsaver', - data: { - anode: wc.tn(mega_anode), - digitize: false, // true means save as RawDigit, else recob::Wire - frame_tags: ['dnnsp'], - - // this may be needed to convert the decon charge [units:e-] to be consistent with the LArSoft default ?unit? e.g. decon charge * 0.005 --> "charge value" to GaussHitFinder - frame_scale: [0.02, 0.02, 0.02], - nticks: params.daq.nticks, - chanmaskmaps: [], - }, - }, nin=1, nout=1, uses=[mega_anode]), - }; local perfect = import 'pgrapher/experiment/sbnd/chndb-perfect.jsonnet'; @@ -153,88 +133,14 @@ local nf_maker = import 'pgrapher/experiment/sbnd/nf.jsonnet'; local nf_pipes = [nf_maker(params, tools.anodes[n], chndb[n], n, name='nf%d' % n) for n in std.range(0, std.length(tools.anodes) - 1)]; local sp_maker = import 'pgrapher/experiment/sbnd/sp.jsonnet'; -local sp_override = if use_dnnroi then { - sparse: true, - use_roi_debug_mode: true, - save_negative_charge: false, // TODO: no negative charge in gauss, default is false - use_multi_plane_protection: true, - do_not_mp_protect_traditional: false, // TODO: do_not_mp_protect_traditional to make a clear ref, defualt is false - mp_tick_resolution:4, - tight_lf_tag: "", - // loose_lf_tag: "", - cleanup_roi_tag: "", - break_roi_loop1_tag: "", - break_roi_loop2_tag: "", - shrink_roi_tag: "", - extend_roi_tag: "", -} else { - sparse: true, -}; local sp = sp_maker(params, tools, { sparse: sigoutform == 'sparse' }); local sp_pipes = [sp.make_sigproc(a) for a in tools.anodes]; -local dnnroi = import 'dnnroi.jsonnet'; -local ts_p0 = { - type: "TorchService", - name: "dnnroi_p0", - tick_per_slice: tick_per_slice, - data: { - model: dnnroi_model_p0, - device: "cpu", - concurrency: 1, - }, -}; - -local ts_p1 = { - type: "TorchService", - name: "dnnroi_p1", - tick_per_slice: tick_per_slice, - data: { - model: dnnroi_model_p1, - device: "cpu", - concurrency: 1, - }, -}; - -local fanout = function (name, multiplicity=2) - g.pnode({ - type: 'FrameFanout', - name: name, - data: { - multiplicity: multiplicity - }, - }, nin=1, nout=multiplicity); - -local sp_fans = [fanout("sp_fan_%d" % n) for n in std.range(0, std.length(tools.anodes) - 1)]; -local dnnroi_pipes = [ dnnroi(tools.anodes[n], ts_p0, ts_p1, output_scale=1, nchunks=nchunks) for n in std.range(0, std.length(tools.anodes) - 1) ]; - - - local magoutput = 'sbnd-data-check.root'; local magnify = import 'pgrapher/experiment/sbnd/magnify-sinks.jsonnet'; local sinks = magnify(tools, magoutput); -local nfsp_pipes = if use_dnnroi then -// oports: 0: dnnroi, 1: traditional sp -[ - g.intern( - innodes=[chsel_pipes[n]], - outnodes=[dnnroi_pipes[n],sp_fans[n]], - centernodes=[nf_pipes[n], sp_pipes[n], sp_fans[n]], - edges=[ - g.edge(chsel_pipes[n], nf_pipes[n], 0, 0), - g.edge(nf_pipes[n], sp_pipes[n], 0, 0), - g.edge(sp_pipes[n], sp_fans[n], 0, 0), - g.edge(sp_fans[n], dnnroi_pipes[n], 0, 0), - ], - iports=chsel_pipes[n].iports, - oports=dnnroi_pipes[n].oports+[sp_fans[n].oports[1]], - name='nfsp_pipe_%d' % n, - ) - for n in std.range(0, std.length(tools.anodes) - 1) -] -else -[ +local nfsp_pipes = [ g.pipeline([ chsel_pipes[n], //sinks.orig_pipe[n], @@ -253,9 +159,8 @@ else local fanpipe = f.fanpipe('FrameFanout', nfsp_pipes, 'FrameFanin', 'sn_mag_nf'); -local retagger = function(name) g.pnode({ +local retagger = g.pnode({ type: 'Retagger', - name: name, data: { // Note: retagger keeps tag_rules an array to be like frame fanin/fanout. tag_rules: [{ @@ -267,90 +172,16 @@ local retagger = function(name) g.pnode({ merge: { 'gauss\\d': 'gauss', 'wiener\\d': 'wiener', - 'dnnsp\\d': 'dnnsp', }, }], }, }, nin=1, nout=1); -local retag_dnnroi = retagger("retag_dnnroi"); -local retag_sp = retagger("retag_sp"); -local sink = function(name) g.pnode({ type: 'DumpFrames', name: name }, nin=1, nout=0); -local sink_dnnroi = sink("sink_dnnroi"); -local sink_sp = sink("sink_sp"); - -local fanout_apa = g.pnode({ - type: 'FrameFanout', - name: 'fanout_apa', - data: { - multiplicity: std.length(tools.anodes), - "tag_rules": [ - { - "frame": { - ".*": "orig%d" % n - }, - "trace": { } - } - for n in std.range(0, std.length(tools.anodes) - 1) - ] - }}, - nin=1, nout=std.length(tools.anodes)); -local framefanin = function(name) g.pnode({ - type: 'FrameFanin', - name: name, - data: { - multiplicity: std.length(tools.anodes), +local sink = g.pnode({ type: 'DumpFrames' }, nin=1, nout=0); - "tag_rules": [ - { - "frame": { - ".*": "framefanin" - }, - trace: { - ['dnnsp%d' % n]: ['dnnsp%d' % n], - ['gauss%d' % n]: ['gauss%d' % n], - ['wiener%d' % n]: ['wiener%d' % n], - ['threshold%d' % n]: ['threshold%d' % n], - }, - } - for n in std.range(0, std.length(tools.anodes) - 1) - ], - "tags": [ ] - }, -}, nin=std.length(tools.anodes), nout=1); -local fanin_apa_dnnroi = framefanin('fanin_apa_dnnroi'); -local fanin_apa_sp = framefanin('fanin_apa_sp'); -local fanpipe = f.fanpipe('FrameFanout', nfsp_pipes, 'FrameFanin', 'sn_mag_nf'); // local graph = g.pipeline([wcls_input.adc_digits, rootfile_creation_frames, fanpipe, retagger, wcls_output.sp_signals, sink]); -local graph = if use_dnnroi then -g.intern( - innodes=[wcls_input.adc_digits], - outnodes=[], - centernodes=nfsp_pipes+[fanout_apa, retag_dnnroi, retag_sp, fanin_apa_dnnroi, fanin_apa_sp, wcls_output.sp_signals, wcls_output.dnnsp_signals, sink_dnnroi, sink_sp], - edges=[ - g.edge(wcls_input.adc_digits, fanout_apa, 0, 0), - g.edge(fanout_apa, nfsp_pipes[0], 0, 0), - g.edge(fanout_apa, nfsp_pipes[1], 1, 0), - g.edge(nfsp_pipes[0], fanin_apa_dnnroi, 0, 0), - g.edge(nfsp_pipes[1], fanin_apa_dnnroi, 0, 1), - g.edge(fanin_apa_dnnroi, retag_dnnroi, 0, 0), - g.edge(retag_dnnroi, wcls_output.dnnsp_signals, 0, 0), - g.edge(wcls_output.dnnsp_signals, sink_dnnroi, 0, 0), - g.edge(nfsp_pipes[0], fanin_apa_sp, 1, 0), - g.edge(nfsp_pipes[1], fanin_apa_sp, 1, 1), - g.edge(fanin_apa_sp, retag_sp, 0, 0), - g.edge(retag_sp, wcls_output.sp_signals, 0, 0), - g.edge(wcls_output.sp_signals, sink_sp, 0, 0), - ] -) -else -g.pipeline([wcls_input.adc_digits, -fanpipe, // nfsp_pipes -retag_sp, -wcls_output.sp_signals, -sink_sp]); - +local graph = g.pipeline([wcls_input.adc_digits, fanpipe, retagger, wcls_output.sp_signals, sink]); local app = { type: 'TbbFlow', @@ -360,4 +191,4 @@ local app = { }; // Finally, the configuration sequence -g.uses(graph) + [app] +g.uses(graph) + [app] \ No newline at end of file diff --git a/sbndcode/WireCell/cfg/pgrapher/experiment/sbnd/wcls-sim-drift-depoflux-nf-sp-samples_rec.jsonnet b/sbndcode/WireCell/cfg/pgrapher/experiment/sbnd/wcls-sim-drift-depoflux-nf-sp-samples_rec.jsonnet deleted file mode 100644 index 15895db20..000000000 --- a/sbndcode/WireCell/cfg/pgrapher/experiment/sbnd/wcls-sim-drift-depoflux-nf-sp-samples_rec.jsonnet +++ /dev/null @@ -1,373 +0,0 @@ -// This is a main entry point for configuring a wire-cell CLI job to -// simulate SBND. It is simplest signal-only simulation with -// one set of nominal field response function. - -local reality = std.extVar('reality'); -local sigoutform = std.extVar('signal_output_form'); // eg "sparse" or "dense" -local savetid = std.extVar("save_track_id"); - -local g = import 'pgraph.jsonnet'; -local f = import 'pgrapher/common/funcs.jsonnet'; -local wc = import 'wirecell.jsonnet'; -local io = import 'pgrapher/common/fileio.jsonnet'; -local tools_maker = import 'pgrapher/common/tools.jsonnet'; - -local data_params = import 'params.jsonnet'; -local simu_params = import 'simparams.jsonnet'; -local params = if reality == 'data' then data_params else simu_params; - -local base = import 'pgrapher/experiment/sbnd/simparams.jsonnet'; -local params = base { - lar: super.lar { // <- super.lar overrides default values - // Longitudinal diffusion constant - DL: std.extVar('DL') * wc.cm2 / wc.s, - // Transverse diffusion constant - DT: std.extVar('DT') * wc.cm2 / wc.s, - // Electron lifetime - lifetime: std.extVar('lifetime') * wc.ms, - // Electron drift speed, assumes a certain applied E-field - drift_speed: std.extVar('driftSpeed') * wc.mm / wc.us, - }, -}; - -local tools = tools_maker(params); -local sim_maker = import 'pgrapher/experiment/sbnd/sim.jsonnet'; -local sim = sim_maker(params, tools); -local nanodes = std.length(tools.anodes); -local anode_iota = std.range(0, nanodes - 1); - -local wcls_maker = import "pgrapher/ui/wcls/nodes.jsonnet"; -local wcls = wcls_maker(params, tools); - -// added Ewerton 2023-03-14 -local wcls_input_sim = { - depos: wcls.input.depos(name="", art_tag=std.extVar('inputTag')), - deposet: g.pnode({ - type: 'wclsSimDepoSetSource', - name: "", - data: { - model: "", - scale: -1, //scale is -1 to correct a sign error in the SimDepoSource converter. - art_tag: std.extVar('inputTag'), //name of upstream art producer of depos "label:instance:processName" - id_is_track: if (savetid == 'true') then false else true, - assn_art_tag: "", - }, - }, nin=0, nout=1), -}; -// Collect all the wc/ls output converters for use below. Note the -// "name" MUST match what is used in theh "outputers" parameter in the -// FHiCL that loads this file. -local mega_anode = { - type: 'MegaAnodePlane', - name: 'meganodes', - data: { - anodes_tn: [wc.tn(anode) for anode in tools.anodes], - }, -}; -local wcls_output_sim = { - // ADC output from simulation - // sim_digits: wcls.output.digits(name="simdigits", tags=["orig"]), - sim_digits: g.pnode({ - type: 'wclsFrameSaver', - name: 'simdigits', - data: { - anode: wc.tn(mega_anode), - digitize: true, // true means save as RawDigit, else recob::Wire - frame_tags: ['daq'], - nticks: params.daq.nticks, - pedestal_mean: 'native', - }, - }, nin=1, nout=1, uses=[mega_anode]), - // The noise filtered "ADC" values. These are truncated for - // art::Event but left as floats for the WCT SP. Note, the tag - // "raw" is somewhat historical as the output is not equivalent to - // "raw data". - nf_digits: wcls.output.digits(name="nfdigits", tags=["raw"]), - // The output of signal processing. Note, there are two signal - // sets each created with its own filter. The "gauss" one is best - // for charge reconstruction, the "wiener" is best for S/N - // separation. Both are used in downstream WC code. - sp_signals: wcls.output.signals(name="spsignals", tags=["gauss", "wiener"]), - // save "threshold" from normal decon for each channel noise - // used in imaging - sp_thresholds: wcls.output.thresholds(name="spthresholds", tags=["wiener"]), -}; - - -local drifter = sim.drifter; - -local setdrifter = g.pnode({ - type: 'DepoSetDrifter', - data: { - drifter: "Drifter" - } - }, nin=1, nout=1, - uses=[drifter]); - -// signal plus noise pipelines -local sn_pipes = sim.splusn_pipelines; - -local hio_sp = [g.pnode({ - type: 'HDF5FrameTap', - name: 'hio_sp%d' % n, - data: { - anode: wc.tn(tools.anodes[n]), - trace_tags: ['loose_lf%d' % n - #, 'tight_lf%d' % n - #, 'cleanup_roi%d' % n - #, 'break_roi_1st%d' % n - #, 'break_roi_2nd%d' % n - #, 'shrink_roi%d' % n - #, 'extend_roi%d' % n - , 'mp3_roi%d' % n - , 'mp2_roi%d' % n - , 'decon_charge%d' % n - , 'gauss%d' % n], - filename: "g4-rec-%d.h5" % n, - chunk: [0, 0], // ncol, nrow - gzip: 2, - tick0: 0, - nticks: 3427, - high_throughput: true, - }, - }, nin=1, nout=1), - for n in std.range(0, std.length(tools.anodes) - 1) - ]; - - -local rng = tools.random; -local wcls_depoflux_writer = g.pnode({ - type: 'wclsDepoFluxWriter', - name: 'postdrift', - data: { - anodes: [wc.tn(anode) for anode in tools.anodes], - field_response: wc.tn(tools.field), - tick: 0.5 * wc.us, - window_start: 0.0 * wc.ms, - window_duration: self.tick * params.daq.nticks, - nsigma: 3.0, - - reference_time: -1700 * wc.us, - - //energy: 1, # equivalent to use_energy = true - simchan_label: 'simpleSC', - sed_label: if (savetid == 'true') then 'ionandscint' else '', - sparse: false, - }, -}, nin=1, nout=1, uses=tools.anodes + [tools.field]); - -local sp_maker = import 'pgrapher/experiment/sbnd/sp.jsonnet'; -local sp = sp_maker(params, tools, { sparse: sigoutform == 'sparse' }); -local sp_pipes = [sp.make_sigproc(a) for a in tools.anodes]; - -local magoutput = 'sbnd-data-check.root'; -local magnify = import 'pgrapher/experiment/sbnd/magnify-sinks.jsonnet'; -local sinks = magnify(tools, magoutput); - -local base = import 'pgrapher/experiment/sbnd/chndb-base.jsonnet'; -//local perfect = import 'pgrapher/experiment/sbnd/chndb-perfect.jsonnet'; - -local chndb = [{ - type: 'OmniChannelNoiseDB', - name: 'ocndbperfect%d' % n, - data: base(params, tools.anodes[n], tools.field, n){dft:wc.tn(tools.dft)}, - //data: perfect(params, tools.anodes[n], tools.field, n){dft:wc.tn(tools.dft)}, - uses: [tools.anodes[n], tools.field, tools.dft], -} for n in anode_iota]; - -local nf_maker = import 'pgrapher/experiment/sbnd/nf.jsonnet'; -local nf_pipes = [nf_maker(params, tools.anodes[n], chndb[n], n, name='nf%d' % n) for n in anode_iota]; - -local multipass1 = [ - g.pipeline([ - sn_pipes[n], - ], - 'multipass%d' % n) - for n in anode_iota -]; - -local multipass2 = [ - g.pipeline([ - sn_pipes[n], - //sinks.orig_pipe[n], - - nf_pipes[n], // NEED to include this pipe for channelmaskmaps - //sinks.raw_pipe[n], - - sp_pipes[n], - hio_sp[n], - - //sinks.decon_pipe[n], - //sinks.threshold_pipe[n], - ], - 'multipass%d' % n) - for n in anode_iota -]; - - -local f_sp = import 'pgrapher/experiment/sbnd/funcs.jsonnet'; - -local outtags = ['orig%d' % n for n in anode_iota]; -local bi_manifold1 = f.fanpipe('DepoSetFanout', multipass1, 'FrameFanin', 'sn_mag_nf', outtags); -local bi_manifold2 = f_sp.fanpipe('DepoSetFanout', multipass2, 'FrameFanin', 'sn_mag_nf_mod2', outtags, "true"); - -local retagger_sim = g.pnode({ - type: 'Retagger', - data: { - // Note: retagger keeps tag_rules an array to be like frame fanin/fanout. - tag_rules: [{ - // Retagger also handles "frame" and "trace" like fanin/fanout - // merge separately all traces like gaussN to gauss. - frame: { - '.*': 'orig', - }, - merge: { - 'orig\\d': 'daq', - }, - }], - }, -}, nin=1, nout=1); - -local sink_sim = sim.frame_sink; - -//===============================NF+SP============================================ - -// Collect all the wc/ls output converters for use below. Note the -// "name" MUST match what is used in theh "outputers" parameter in the -// FHiCL that loads this file. - -local wcls_output_sp = { - // The noise filtered "ADC" values. These are truncated for - // art::Event but left as floats for the WCT SP. Note, the tag - // "raw" is somewhat historical as the output is not equivalent to - // "raw data". - nf_digits: g.pnode({ - type: 'wclsFrameSaver', - name: 'nfsaver', - data: { - anode: wc.tn(mega_anode), - digitize: true, // true means save as RawDigit, else recob::Wire - frame_tags: ['raw'], - }, - }, nin=1, nout=1, uses=[mega_anode]), - - - // The output of signal processing. Note, there are two signal - // sets each created with its own filter. The "gauss" one is best - // for charge reconstruction, the "wiener" is best for S/N - // separation. Both are used in downstream WC code. - sp_signals: g.pnode({ - type: 'wclsFrameSaver', - name: 'spsaver', - data: { - anode: wc.tn(mega_anode), - digitize: false, // true means save as RawDigit, else recob::Wire - frame_tags: ['gauss', 'wiener'], - - // this may be needed to convert the decon charge [units:e-] to be consistent with the LArSoft default - // for SBND, this scale is about ~50. Frame scale needed when using LArSoft producers reading in recob::Wire. - frame_scale: [0.02, 0.02], - nticks: params.daq.nticks, - - // uncomment the below configs to save summaries and cmm - summary_tags: ['wiener'], - summary_operator: {wiener: 'set'}, - summary_scale: [0.02], # summary scale should be the same as frame_scale - chanmaskmaps: ['bad'], - }, - }, nin=1, nout=1, uses=[mega_anode]), - -}; - - -local chsel_pipes = [ - g.pnode({ - type: 'ChannelSelector', - name: 'chsel%d' % n, - data: { - channels: std.range(5638 * n, 5638 * (n + 1) - 1), - }, - }, nin=1, nout=1) - for n in anode_iota -]; - - -local nfsp_pipes = [ - g.pipeline([ - chsel_pipes[n], - //sinks.orig_pipe[n], - - nf_pipes[n], // NEED to include this pipe for channelmaskmaps - //sinks.raw_pipe[n], - - sp_pipes[n], - //sinks.decon_pipe[n], - //sinks.threshold_pipe[n], - // sinks.debug_pipe[n], // use_roi_debug_mode=true in sp.jsonnet - ], - 'nfsp_pipe_%d' % n) - for n in anode_iota -]; - -//local fanpipe = f_sp.fanpipe('FrameFanout', nfsp_pipes, 'FrameFanin', 'sn_mag_nf'); // commented Ewerton 2023-05-24 -local fanpipe = f_sp.fanpipe('FrameFanout', nfsp_pipes, 'FrameFanin', 'sn_mag_nf_mod'); //added Ewerton 2023-05-24 - -local retagger_sp = g.pnode({ - type: 'Retagger', - name: 'sp', //added Ewerton 2023-05-24 - data: { - // Note: retagger keeps tag_rules an array to be like frame fanin/fanout. - tag_rules: [{ - // Retagger also handles "frame" and "trace" like fanin/fanout - // merge separately all traces like gaussN to gauss. - frame: { - '.*': 'retagger', - }, - merge: { - 'gauss\\d': 'gauss', - 'wiener\\d': 'wiener', - }, - }], - }, -}, nin=1, nout=1); - -local sink_sp = g.pnode({ type: 'DumpFrames' }, nin=1, nout=0); - -local graph1 = g.pipeline([ -wcls_input_sim.deposet, //sim -setdrifter, //sim -wcls_depoflux_writer, //sim -bi_manifold1, //sim -retagger_sim, //sim -wcls_output_sim.sim_digits, //sim -fanpipe, //sp -retagger_sp, //sp -wcls_output_sp.sp_signals, //sp -sink_sp //sp -]); - - -local graph2 = g.pipeline([ -wcls_input_sim.deposet, //sim -setdrifter, //sim -wcls_depoflux_writer, //sim -bi_manifold2, //sim -retagger_sp, //sp -wcls_output_sp.sp_signals, //sp -sink_sp //sp -]); - -local save_simdigits = std.extVar('save_simdigits'); - -local graph = if save_simdigits == "true" then graph1 else graph2; - -local app = { - type: 'TbbFlow', - data: { - edges: g.edges(graph), - }, -}; - -// Finally, the configuration sequence -g.uses(graph) + [app] - diff --git a/sbndcode/WireCell/cfg/pgrapher/experiment/sbnd/wcls-sim-drift-depoflux-nf-sp-samples_tru.jsonnet b/sbndcode/WireCell/cfg/pgrapher/experiment/sbnd/wcls-sim-drift-depoflux-nf-sp-samples_tru.jsonnet deleted file mode 100644 index cbe6c9ad6..000000000 --- a/sbndcode/WireCell/cfg/pgrapher/experiment/sbnd/wcls-sim-drift-depoflux-nf-sp-samples_tru.jsonnet +++ /dev/null @@ -1,474 +0,0 @@ -// This is a main entry point for configuring a wire-cell CLI job to -// simulate SBND. It is simplest signal-only simulation with -// one set of nominal field response function. - -local epoch = std.extVar('epoch'); // eg "dynamic", "after", "before", "perfect" -local reality = std.extVar('reality'); -// local sigoutform = std.extVar('signal_output_form'); // eg "sparse" or "dense" -local sigoutform = "sparse"; // eg "sparse" or "dense" -local savetid = std.extVar("save_track_id"); - -local g = import 'pgraph.jsonnet'; -local f = import 'pgrapher/common/funcs.jsonnet'; -local wc = import 'wirecell.jsonnet'; -local io = import 'pgrapher/common/fileio.jsonnet'; -local tools_maker = import 'pgrapher/common/tools.jsonnet'; - -local data_params = import 'params.jsonnet'; -local simu_params = import 'simparams.jsonnet'; -local params = if reality == 'data' then data_params else simu_params; - -local base = import 'pgrapher/experiment/sbnd/simparams.jsonnet'; -local params = base { - lar: super.lar { // <- super.lar overrides default values - // Longitudinal diffusion constant - DL: std.extVar('DL') * wc.cm2 / wc.s, - // Transverse diffusion constant - DT: std.extVar('DT') * wc.cm2 / wc.s, - // Electron lifetime - lifetime: std.extVar('lifetime') * wc.ms, - // Electron drift speed, assumes a certain applied E-field - drift_speed: std.extVar('driftSpeed') * wc.mm / wc.us, - }, -}; - -local tools = tools_maker(params); -local sim_maker = import 'pgrapher/experiment/sbnd/sim.jsonnet'; -local sim = sim_maker(params, tools); -local nanodes = std.length(tools.anodes); -local anode_iota = std.range(0, nanodes - 1); - -local wcls_maker = import "pgrapher/ui/wcls/nodes.jsonnet"; -local wcls = wcls_maker(params, tools); - -// added Ewerton 2023-03-14 -local wcls_input_sim = { - depos: wcls.input.depos(name="", art_tag=std.extVar('inputTag')), //commented Ewerton 2023-03-15 - deposet: g.pnode({ - type: 'wclsSimDepoSetSource', - name: "", - data: { - model: "", - scale: -1, //scale is -1 to correct a sign error in the SimDepoSource converter. - // art_tag: std.extVar('inputTag'), //name of upstream art producer of depos "label:instance:processName" - art_tag: std.extVar('inputTag'), //name of upstream art producer of depos "label:instance:processName" - id_is_track: if (savetid == 'true') then false else true, - assn_art_tag: "", - }, - }, nin=0, nout=1), -}; -// Collect all the wc/ls output converters for use below. Note the -// "name" MUST match what is used in theh "outputers" parameter in the -// FHiCL that loads this file. -local mega_anode = { - type: 'MegaAnodePlane', - name: 'meganodes', - data: { - anodes_tn: [wc.tn(anode) for anode in tools.anodes], - }, -}; -local wcls_output_sim = { - // ADC output from simulation - // sim_digits: wcls.output.digits(name="simdigits", tags=["orig"]), - sim_digits: g.pnode({ - type: 'wclsFrameSaver', - name: 'simdigits', - data: { - // anode: wc.tn(tools.anode), - anode: wc.tn(mega_anode), - digitize: true, // true means save as RawDigit, else recob::Wire - frame_tags: ['daq'], - // nticks: params.daq.nticks, - // chanmaskmaps: ['bad'], - pedestal_mean: 'native', - }, - }, nin=1, nout=1, uses=[mega_anode]), - // The noise filtered "ADC" values. These are truncated for - // art::Event but left as floats for the WCT SP. Note, the tag - // "raw" is somewhat historical as the output is not equivalent to - // "raw data". - nf_digits: wcls.output.digits(name="nfdigits", tags=["raw"]), - // The output of signal processing. Note, there are two signal - // sets each created with its own filter. The "gauss" one is best - // for charge reconstruction, the "wiener" is best for S/N - // separation. Both are used in downstream WC code. - sp_signals: wcls.output.signals(name="spsignals", tags=["gauss", "wiener"]), - // save "threshold" from normal decon for each channel noise - // used in imaging - sp_thresholds: wcls.output.thresholds(name="spthresholds", tags=["threshold"]), -}; - - -local drifter = sim.drifter; - -// added Ewerton 2023-03-14 -local setdrifter = g.pnode({ - type: 'DepoDrifter', - data: { - drifter: "Drifter" - } - }, nin=1, nout=1, - uses=[drifter]); - -// signal plus noise pipelines -// local sn_pipes = sim.signal_pipelines; -local sn_pipes = sim.splusn_pipelines; - -local hio_sp = [g.pnode({ - type: 'HDF5FrameTap', - name: 'hio_sp%d' % n, - data: { - anode: wc.tn(tools.anodes[n]), - trace_tags: ['loose_lf%d' % n - , 'tight_lf%d' % n - , 'cleanup_roi%d' % n - , 'break_roi_1st%d' % n - , 'break_roi_2nd%d' % n - , 'shrink_roi%d' % n - , 'extend_roi%d' % n - , 'mp3_roi%d' % n - , 'mp2_roi%d' % n - , 'decon_charge%d' % n - , 'gauss%d' % n], - filename: "g4-rec-%d.h5" % n, - chunk: [0, 0], // ncol, nrow - gzip: 2, - tick0: 0, - nticks: 3427, - high_throughput: true, - }, - }, nin=1, nout=1), - for n in std.range(0, std.length(tools.anodes) - 1) - ]; - - -local rng = tools.random; -local wcls_deposetsimchannel_sink = g.pnode({ - type: 'wclsSimChannelSink', - name: 'postdrift', - data: { - artlabel: 'simpleSC', // where to save in art::Event - anodes_tn: [wc.tn(anode) for anode in tools.anodes], - rng: wc.tn(rng), - tick: 0.5 * wc.us, - start_time: -0.205 * wc.ms, - readout_time: self.tick * 3427, - nsigma: 3.0, - drift_speed: params.lar.drift_speed, - u_to_rp: 100 * wc.mm, // time to collection plane - v_to_rp: 100 * wc.mm, // time to collection plane - y_to_rp: 100 * wc.mm, - u_time_offset: 0.0 * wc.us, - v_time_offset: 0.0 * wc.us, - y_time_offset: 0.0 * wc.us, - g4_ref_time: -1700 * wc.us, - use_energy: true, - }, -}, nin=1, nout=1, uses=tools.anodes); - -local hio_orig = [g.pnode({ - type: 'HDF5FrameTap', - name: 'hio_orig%d' % n, - data: { - anode: wc.tn(tools.anodes[n]), - trace_tags: ['orig%d'%n], - filename: "g4-rec-%d.h5" % n, - chunk: [0, 0], // ncol, nrow - gzip: 2, - high_throughput: true, - }, - }, nin=1, nout=1), - for n in std.range(0, std.length(tools.anodes) - 1) - ]; - -local sp_maker = import 'pgrapher/experiment/sbnd/sp.jsonnet'; -local sp = sp_maker(params, tools, { sparse: sigoutform == 'sparse' }); -local sp_pipes = [sp.make_sigproc(a) for a in tools.anodes]; - -local magoutput = 'sbnd-data-check.root'; -local magnify = import 'pgrapher/experiment/sbnd/magnify-sinks.jsonnet'; -local sinks = magnify(tools, magoutput); - -local perfect = import 'pgrapher/experiment/sbnd/chndb-perfect.jsonnet'; -//local base = import 'chndb-base_sbnd.jsonnet'; - -local chndb = [{ - type: 'OmniChannelNoiseDB', - name: 'ocndbperfect%d' % n, - data: perfect(params, tools.anodes[n], tools.field, n){dft:wc.tn(tools.dft)}, - // data: base(params, tools.anodes[n], tools.field, n){dft:wc.tn(tools.dft)}, - uses: [tools.anodes[n], tools.field, tools.dft], -} for n in anode_iota]; - -local nf_maker = import 'pgrapher/experiment/sbnd/nf.jsonnet'; -local nf_pipes = [nf_maker(params, tools.anodes[n], chndb[n], n, name='nf%d' % n) for n in anode_iota]; - -local depo_fanout = [g.pnode({ - type:'DepoFanout', - name:'depo_fanout-%d'%n, - data:{ - multiplicity:2, - tags: [], - }}, nin=1, nout=2) for n in anode_iota]; - -// local deposplats = [sim.make_ductor('splat%d'%n, tools.anodes[n], tools.pirs[0], 'DepoSplat', 'deposplat%d'%n) for n in anode_iota] ; - -// The approximated sim+sigproc -local splat = function(params, tools, anode, name=null) { - local apaid = anode.data.ident, - local sufix = if std.type(name) == "null" then apaid else name, - local bg = g.pnode({ - type:'DepoBagger', - name: sufix, - data: { - gate: [params.sim.ductor.start_time, - params.sim.ductor.start_time+params.sim.ductor.readout_time], - }, - }, nin=1, nout=1), - local sp = g.pnode({ - type: 'DepoFluxSplat', - name: sufix, - data: { - anode: wc.tn(anode), - field_response: wc.tn(tools.field), // for speed and origin - sparse: true, - tick: 0.5 * wc.us, - window_start: -0.205 * wc.ms, // TODO: there seems to be an offset between sim and rec? - window_duration: self.tick * params.daq.nticks, - reference_time: 0.0, - "smear_long": [ - 3.5, - 3.5, - 3.5 - ], - "smear_tran": [ - 0.4, //can use 0.3 if too strong, otherwise can use 0.4 - 0.4, - 0.14 - ] - }, - }, nin=1, nout=1, uses=[anode, tools.field]), - local rt = g.pnode({ - type: 'Retagger', - name: sufix, - data: { - // Note: retagger keeps tag_rules an array to be like frame fanin/fanout. - tag_rules: [{ - // Retagger also handles "frame" and "trace" like fanin/fanout - // merge separately all traces like gaussN to gauss. - frame: { - ".*": "deposplat%d" % apaid - }, - merge: { - ".*": "deposplat%d" % apaid - }, - }], - }, - }, nin=1, nout=1), - ret: g.pipeline([bg, sp, rt],"%s-%s" % [bg.name, sp.name]), -}.ret; - -local deposplats = [splat(params, tools, tools.anodes[n]) for n in anode_iota] ; - -local hio_truth = [g.pnode({ - type: 'HDF5FrameTap', - name: 'hio_truth%d' % n, - data: { - anode: wc.tn(tools.anodes[n]), - trace_tags: ['deposplat%d'%n], - filename: "g4-tru-%d.h5" % n, - chunk: [0, 0], // ncol, nrow - gzip: 2, - tick0: 0, - nticks: 3427, - high_throughput: true, - }, - }, nin=1, nout=1), - for n in std.range(0, std.length(tools.anodes) - 1) - ]; - - -local multipass1 = [ - g.pipeline([ - sn_pipes[n], - ], - 'multipass%d' % n) - for n in anode_iota -]; - -local multipass2 = [ - g.pipeline([ - deposplats[n], - hio_truth[n], - ], - 'multipass%d' % n) - for n in anode_iota -]; - - -local f_sp = import 'pgrapher/experiment/sbnd/funcs.jsonnet'; -local outtags = ['orig%d' % n for n in anode_iota]; -local bi_manifold1 = f.fanpipe('DepoFanout', multipass1, 'FrameFanin', 'sn_mag_nf', outtags); -// local bi_manifold2 = f_sp.fanpipe('DepoFanout', multipass2, 'FrameFanin', 'sn_mag_nf_mod2', outtags, "true"); -// local bi_manifold2 = f_sp.fanpipe('DepoFanout', multipass2, 'FrameFanin', 'sn_mag_nf_mod2', outtags, "true"); -local bi_manifold2 = f_sp.fanpipe('DepoFanout', multipass2, 'FrameFanin'); - -local retagger_sim = g.pnode({ - type: 'Retagger', - data: { - // Note: retagger keeps tag_rules an array to be like frame fanin/fanout. - tag_rules: [{ - // Retagger also handles "frame" and "trace" like fanin/fanout - // merge separately all traces like gaussN to gauss. - frame: { - '.*': 'orig', - }, - merge: { - 'orig\\d': 'daq', - }, - }], - }, -}, nin=1, nout=1); - -local sink_sim = sim.frame_sink; - -//===============================NF+SP============================================ - -// Collect all the wc/ls output converters for use below. Note the -// "name" MUST match what is used in theh "outputers" parameter in the -// FHiCL that loads this file. - -local wcls_output_sp = { - // The noise filtered "ADC" values. These are truncated for - // art::Event but left as floats for the WCT SP. Note, the tag - // "raw" is somewhat historical as the output is not equivalent to - // "raw data". - nf_digits: g.pnode({ - type: 'wclsFrameSaver', - name: 'nfsaver', - data: { - // anode: wc.tn(tools.anode), - anode: wc.tn(mega_anode), - digitize: true, // true means save as RawDigit, else recob::Wire - frame_tags: ['raw'], - // nticks: params.daq.nticks, - chanmaskmaps: ['bad'], - }, - }, nin=1, nout=1, uses=[mega_anode]), - - - // The output of signal processing. Note, there are two signal - // sets each created with its own filter. The "gauss" one is best - // for charge reconstruction, the "wiener" is best for S/N - // separation. Both are used in downstream WC code. - sp_signals: g.pnode({ - type: 'wclsFrameSaver', - name: 'spsaver', - data: { - // anode: wc.tn(tools.anode), - anode: wc.tn(mega_anode), - digitize: false, // true means save as RawDigit, else recob::Wire - frame_tags: ['gauss', 'wiener'], - - // this may be needed to convert the decon charge [units:e-] to be consistent with the LArSoft default ?unit? e.g. decon charge * 0.005 --> "charge value" to GaussHitFinder - frame_scale: [1.0, 1.0], - nticks: params.daq.nticks, - chanmaskmaps: [], - //nticks: -1, - }, - }, nin=1, nout=1, uses=[mega_anode]), -}; - - -local chsel_pipes = [ - g.pnode({ - type: 'ChannelSelector', - name: 'chsel%d' % n, - data: { - channels: std.range(5632 * n, 5632 * (n + 1) - 1), - //tags: ['orig%d' % n], // traces tag - }, - }, nin=1, nout=1) - for n in anode_iota -]; - - -local nfsp_pipes = [ - g.pipeline([ - chsel_pipes[n], - //sinks.orig_pipe[n], - - //nf_pipes[n], - //sinks.raw_pipe[n], - - sp_pipes[n], - // hio_sp[n], - //sinks.decon_pipe[n], - //sinks.threshold_pipe[n], - // sinks.debug_pipe[n], // use_roi_debug_mode=true in sp.jsonnet - ], - 'nfsp_pipe_%d' % n) - for n in anode_iota -]; - -local f_sp = import 'pgrapher/experiment/sbnd/funcs.jsonnet'; -//local fanpipe = f_sp.fanpipe('FrameFanout', nfsp_pipes, 'FrameFanin', 'sn_mag_nf'); // commented Ewerton 2023-05-24 -local fanpipe = f_sp.fanpipe('FrameFanout', nfsp_pipes, 'FrameFanin', 'sn_mag_nf_mod'); //added Ewerton 2023-05-24 - -local retagger_sp = g.pnode({ - type: 'Retagger', - name: 'sp', //added Ewerton 2023-05-24 - data: { - // Note: retagger keeps tag_rules an array to be like frame fanin/fanout. - tag_rules: [{ - // Retagger also handles "frame" and "trace" like fanin/fanout - // merge separately all traces like gaussN to gauss. - frame: { - '.*': 'retagger', - }, - merge: { - 'gauss\\d': 'gauss', - 'wiener\\d': 'wiener', - }, - }], - }, -}, nin=1, nout=1); - -local sink_sp = g.pnode({ type: 'DumpFrames' }, nin=1, nout=0); - -local graph1 = g.pipeline([ -wcls_input_sim.deposet, //sim -setdrifter, //sim -wcls_deposetsimchannel_sink, //sim -bi_manifold1, //sim // sn_pipes[n] -retagger_sim, //sim -wcls_output_sim.sim_digits, //sim -fanpipe, //sp <- nf_pipes[n], sp_pipes[n] (needs channel selector to do fanout correctly) -retagger_sp, //sp -wcls_output_sp.sp_signals, //sp -sink_sp //sp -]); - - -local graph2 = g.pipeline([ -wcls_input_sim.depos, //sim -drifter, //sim -wcls_deposetsimchannel_sink, //sim -bi_manifold2, //sim // changed Ewerton 2023-05-28 -sink_sp //sp -]); - -// local save_simdigits = std.extVar('save_simdigits'); -local save_simdigits = "false"; - -local graph = if save_simdigits == "true" then graph1 else graph2; - -local app = { - type: 'TbbFlow', - data: { - edges: g.edges(graph), - }, -}; - -// Finally, the configuration sequence -g.uses(graph) + [app] diff --git a/sbndcode/WireCell/cfg/pgrapher/experiment/sbnd/wcls-sim-drift-depoflux-nf-sp.jsonnet b/sbndcode/WireCell/cfg/pgrapher/experiment/sbnd/wcls-sim-drift-depoflux-nf-sp.jsonnet index 88d6e1ce7..3abe24578 100644 --- a/sbndcode/WireCell/cfg/pgrapher/experiment/sbnd/wcls-sim-drift-depoflux-nf-sp.jsonnet +++ b/sbndcode/WireCell/cfg/pgrapher/experiment/sbnd/wcls-sim-drift-depoflux-nf-sp.jsonnet @@ -5,11 +5,6 @@ local reality = std.extVar('reality'); local sigoutform = std.extVar('signal_output_form'); // eg "sparse" or "dense" local savetid = std.extVar("save_track_id"); -local use_dnnroi = std.extVar('use_dnnroi'); -local nchunks = std.extVar('nchunks'); -local tick_per_slice = std.extVar('tick_per_slice'); -local dnnroi_model_p0 = std.extVar('dnnroi_model_p0'); -local dnnroi_model_p1 = std.extVar('dnnroi_model_p1'); local g = import 'pgraph.jsonnet'; local f = import 'pgrapher/common/funcs.jsonnet'; @@ -138,24 +133,7 @@ local wcls_depoflux_writer = g.pnode({ }, nin=1, nout=1, uses=tools.anodes + [tools.field]); local sp_maker = import 'pgrapher/experiment/sbnd/sp.jsonnet'; -local sp_override = if use_dnnroi then { - sparse: true, - use_roi_debug_mode: true, - save_negative_charge: false, // TODO: no negative charge in gauss, default is false - use_multi_plane_protection: true, - do_not_mp_protect_traditional: false, // TODO: do_not_mp_protect_traditional to make a clear ref, defualt is false - mp_tick_resolution:4, - tight_lf_tag: "", - // loose_lf_tag: "", // comment to use as input to dnnsp - cleanup_roi_tag: "", - break_roi_loop1_tag: "", - break_roi_loop2_tag: "", - shrink_roi_tag: "", - extend_roi_tag: "", -} else { - sparse: true, -}; -local sp = sp_maker(params, tools, sp_override); +local sp = sp_maker(params, tools, { sparse: sigoutform == 'sparse' }); local sp_pipes = [sp.make_sigproc(a) for a in tools.anodes]; local magoutput = 'sbnd-data-check.root'; @@ -273,59 +251,8 @@ local wcls_output_sp = { chanmaskmaps: ['bad'], }, }, nin=1, nout=1, uses=[mega_anode]), - dnnsp_signals: g.pnode({ - type: 'wclsFrameSaver', - name: 'dnnsaver', - data: { - anode: wc.tn(mega_anode), - digitize: false, // true means save as RawDigit, else recob::Wire - frame_tags: ['dnnsp'], - - // this may be needed to convert the decon charge [units:e-] to be consistent with the LArSoft default ?unit? e.g. decon charge * 0.005 --> "charge value" to GaussHitFinder - frame_scale: [0.02], - nticks: params.daq.nticks, - chanmaskmaps: [], - }, - }, nin=1, nout=1, uses=[mega_anode]), }; -local dnnroi = import 'dnnroi.jsonnet'; - -local ts_p0 = { - type: "TorchService", - name: "dnnroi_p0", - tick_per_slice: tick_per_slice, - data: { - model: dnnroi_model_p0, - device: "cpu", - concurrency: 1, - }, -}; - -local ts_p1 = { - type: "TorchService", - name: "dnnroi_p1", - tick_per_slice: tick_per_slice, - data: { - model: dnnroi_model_p1, - device: "cpu", - concurrency: 1, - }, -}; - -local dnnroi_pipes = [ dnnroi(tools.anodes[n], ts_p0, ts_p1, output_scale=1, nchunks=nchunks) for n in std.range(0, std.length(tools.anodes) - 1) ]; - -local fanout = function (name, multiplicity=2) - g.pnode({ - type: 'FrameFanout', - name: name, - data: { - multiplicity: multiplicity - }, - }, nin=1, nout=multiplicity); - -local sp_fans = [fanout("sp_fan_%d" % n) for n in std.range(0, std.length(tools.anodes) - 1)]; - local chsel_pipes = [ @@ -340,27 +267,7 @@ local chsel_pipes = [ ]; -local nfsp_pipes = if use_dnnroi then -// oports: 0: dnnroi, 1: traditional sp -[ - g.intern( - innodes=[chsel_pipes[n]], - outnodes=[dnnroi_pipes[n],sp_fans[n]], - centernodes=[nf_pipes[n], sp_pipes[n], sp_fans[n]], - edges=[ - g.edge(chsel_pipes[n], nf_pipes[n], 0, 0), - g.edge(nf_pipes[n], sp_pipes[n], 0, 0), - g.edge(sp_pipes[n], sp_fans[n], 0, 0), - g.edge(sp_fans[n], dnnroi_pipes[n], 0, 0), - ], - iports=chsel_pipes[n].iports, - oports=dnnroi_pipes[n].oports+[sp_fans[n].oports[1]], - name='nfsp_pipe_%d' % n, - ) - for n in std.range(0, std.length(tools.anodes) - 1) -] -else -[ +local nfsp_pipes = [ g.pipeline([ chsel_pipes[n], //sinks.orig_pipe[n], @@ -380,9 +287,9 @@ else //local fanpipe = f_sp.fanpipe('FrameFanout', nfsp_pipes, 'FrameFanin', 'sn_mag_nf'); // commented Ewerton 2023-05-24 local fanpipe = f_sp.fanpipe('FrameFanout', nfsp_pipes, 'FrameFanin', 'sn_mag_nf_mod'); //added Ewerton 2023-05-24 -local retagger = function(name) g.pnode({ +local retagger_sp = g.pnode({ type: 'Retagger', - name: name, + name: 'sp', //added Ewerton 2023-05-24 data: { // Note: retagger keeps tag_rules an array to be like frame fanin/fanout. tag_rules: [{ @@ -394,63 +301,14 @@ local retagger = function(name) g.pnode({ merge: { 'gauss\\d': 'gauss', 'wiener\\d': 'wiener', - 'dnnsp\\d': 'dnnsp', }, }], }, }, nin=1, nout=1); -local retagger_dnnroi = retagger("retagger_dnnroi"); -local retagger_sp = retagger("retagger_sp"); - -local sink = function(name) g.pnode({ type: 'DumpFrames', name: name }, nin=1, nout=0); -local sink_dnnroi = sink("sink_dnnroi"); -local sink_sp = sink("sink_sp"); - -local fanout_apa = g.pnode({ - type: 'FrameFanout', - name: 'fanout_apa', - data: { - multiplicity: std.length(tools.anodes), - "tag_rules": [ - { - "frame": { - ".*": "orig%d" % n - }, - "trace": { } - } - for n in std.range(0, std.length(tools.anodes) - 1) - ] - }}, - nin=1, nout=std.length(tools.anodes)); - -local framefanin = function(name) g.pnode({ - type: 'FrameFanin', - name: name, - data: { - multiplicity: std.length(tools.anodes), - - "tag_rules": [ - { - "frame": { - ".*": "framefanin" - }, - trace: { - ['dnnsp%d' % n]: ['dnnsp%d' % n], - ['gauss%d' % n]: ['gauss%d' % n], - ['wiener%d' % n]: ['wiener%d' % n], - ['threshold%d' % n]: ['threshold%d' % n], - }, - } - for n in std.range(0, std.length(tools.anodes) - 1) - ], - "tags": [ ] - }, -}, nin=std.length(tools.anodes), nout=1); -local fanin_apa_dnnroi = framefanin('fanin_apa_dnnroi'); -local fanin_apa_sp = framefanin('fanin_apa_sp'); +local sink_sp = g.pnode({ type: 'DumpFrames' }, nin=1, nout=0); -local graph1_trad = g.pipeline([ +local graph1 = g.pipeline([ wcls_input_sim.deposet, //sim setdrifter, //sim wcls_depoflux_writer, //sim @@ -464,7 +322,7 @@ sink_sp //sp ]); -local graph2_trad = g.pipeline([ +local graph2 = g.pipeline([ wcls_input_sim.deposet, //sim setdrifter, //sim wcls_depoflux_writer, //sim @@ -474,45 +332,9 @@ wcls_output_sp.sp_signals, //sp sink_sp //sp ]); -local graph_sim_dnn = g.pipeline([ -wcls_input_sim.deposet, //sim -setdrifter, //sim -wcls_depoflux_writer, //sim -bi_manifold1, //sim -]); - - -local graph_sp_dnn = -g.intern( - innodes=[retagger_sim], - outnodes=[], - centernodes=nfsp_pipes+[wcls_output_sim.sim_digits, fanout_apa, retagger_dnnroi, retagger_sp, fanin_apa_dnnroi, fanin_apa_sp, wcls_output_sp.sp_signals, wcls_output_sp.dnnsp_signals, sink_dnnroi, sink_sp], - edges=[ - g.edge(retagger_sim, wcls_output_sim.sim_digits, 0, 0), - g.edge(wcls_output_sim.sim_digits, fanout_apa, 0, 0), - g.edge(fanout_apa, nfsp_pipes[0], 0, 0), - g.edge(fanout_apa, nfsp_pipes[1], 1, 0), - g.edge(nfsp_pipes[0], fanin_apa_dnnroi, 0, 0), - g.edge(nfsp_pipes[1], fanin_apa_dnnroi, 0, 1), - g.edge(fanin_apa_dnnroi, retagger_dnnroi, 0, 0), - g.edge(retagger_dnnroi, wcls_output_sp.dnnsp_signals, 0, 0), - g.edge(wcls_output_sp.dnnsp_signals, sink_dnnroi, 0, 0), - g.edge(nfsp_pipes[0], fanin_apa_sp, 1, 0), - g.edge(nfsp_pipes[1], fanin_apa_sp, 1, 1), - g.edge(fanin_apa_sp, retagger_sp, 0, 0), - g.edge(retagger_sp, wcls_output_sp.sp_signals, 0, 0), - g.edge(wcls_output_sp.sp_signals, sink_sp, 0, 0), - ] -); - -local graph_dnn = g.pipeline([ -graph_sim_dnn, -graph_sp_dnn, -]); - local save_simdigits = std.extVar('save_simdigits'); -local graph = if use_dnnroi then graph_dnn else if save_simdigits == "true" then graph1_trad else graph2_trad; +local graph = if save_simdigits == "true" then graph1 else graph2; local app = { type: 'TbbFlow', @@ -523,4 +345,4 @@ local app = { // Finally, the configuration sequence g.uses(graph) + [app] - + diff --git a/sbndcode/WireCell/wcsimsp_sbnd.fcl b/sbndcode/WireCell/wcsimsp_sbnd.fcl index 03d5f429a..b72f73d44 100644 --- a/sbndcode/WireCell/wcsimsp_sbnd.fcl +++ b/sbndcode/WireCell/wcsimsp_sbnd.fcl @@ -17,7 +17,7 @@ sbnd_wcls: { configs: [] ## Libraries in which to look for WCT components - plugins: ["WireCellGen", "WireCellSigProc", "WireCellRoot", "WireCellPgraph", "WireCellLarsoft", "WireCellTbb", "WireCellPytorch"] + plugins: ["WireCellGen", "WireCellSigProc", "WireCellRoot", "WireCellPgraph", "WireCellLarsoft", "WireCellTbb"] inputers: [] outputers: [] @@ -58,9 +58,6 @@ sbnd_wcls_simsp.wcls_main.params: { save_simdigits: "false" save_track_id: "true" - - dnnroi_model_p0: "DNN_ROI/plane0.ts" - dnnroi_model_p1: "DNN_ROI/plane1.ts" } sbnd_wcls_simsp.wcls_main.structs: { @@ -74,10 +71,6 @@ sbnd_wcls_simsp.wcls_main.structs: { driftSpeed: 1.563 ## simulated front porch size [us] tick0_time: @local::sbnd_detectorclocks.TriggerOffsetTPC - nticks: 3427 - use_dnnroi: false - nchunks: 2 - tick_per_slice: 4 } # ------------------------------------------------------------------------------------ # @@ -90,9 +83,7 @@ sbnd_wcls_sp.wcls_main.inputers: ["wclsRawFrameSource" # and you must have geo::Geometry service in your environment. # ,"wclsMultiChannelNoiseDB" ] -sbnd_wcls_sp.wcls_main.outputers: ["wclsFrameSaver:spsaver" - # "wclsFrameSaver:dnnsaver" - ] +sbnd_wcls_sp.wcls_main.outputers: ["wclsFrameSaver:spsaver"] sbnd_wcls_sp.wcls_main.params : { # This locates the input raw::RawDigit collection in the art::Event raw_input_label: "daq" @@ -107,103 +98,5 @@ sbnd_wcls_sp.wcls_main.params : { # Save output signal waveforms (recob::Wire) in "sparse" or "dense" form signal_output_form: "sparse" - - dnnroi_model_p0: "DNN_ROI/plane0.ts" - dnnroi_model_p1: "DNN_ROI/plane1.ts" - } - -sbnd_wcls_sp.wcls_main.structs: { - nticks: 3427 - use_dnnroi: false - nchunks: 2 - tick_per_slice: 4 - } - - -# ------------------------------------------------------------------------------------ # - -## Configuration for generating training samples -- input images -sbnd_wcls_samples_rec: @local::sbnd_wcls -sbnd_wcls_samples_rec.wcls_main.plugins: ["WireCellGen", "WireCellSigProc", "WireCellRoot", "WireCellPgraph", "WireCellLarsoft", "WireCellTbb", "WireCellPytorch", "WireCellHio"] -sbnd_wcls_samples_rec.wcls_main.configs: ["pgrapher/experiment/sbnd/wcls-sim-drift-depoflux-nf-sp-samples_rec.jsonnet"] -sbnd_wcls_samples_rec.wcls_main.inputers: ["wclsSimDepoSetSource:"] -sbnd_wcls_samples_rec.wcls_main.outputers:[ "wclsDepoFluxWriter:postdrift", "wclsFrameSaver:simdigits"] -sbnd_wcls_samples_rec.wcls_main.params: { - ## This locates the input SimEnergyDeposits in the art::Event - inputTag: "ionandscint:" - - ## Set "data" vs. "sim". The epoch below probably should follow suit. - reality: "sim" - - ## if epoch is "dynamic" you MUST add - ## "wclsMultiChannelNoiseDB" to "inputers" and must NOT - ## add it if not "dynamic" - epoch: "perfect" - - ## Save output signal waveforms (recob::Wire) in "sparse" or "dense" form - signal_output_form: "sparse" - - ## If save_simdigits="false", comment line with "wclsFrameSaver:simdigits" in outputers - ## If save_simdigits="true", uncomment line with "wclsFrameSaver:simdigits" in outputers - ## this is whether or not to save raw digits!!! - save_simdigits: "true" - - save_track_id: "true" - } - -sbnd_wcls_samples_rec.wcls_main.structs: { - ## Longitudinal diffusion constant [cm2/s] - DL: 4.0 - ## Transverse diffusion constant [cm2/s] - DT: 8.8 - ## Electron lifetime [ms] - lifetime: 100.0 - ## Electron drift speed, assumes 0.5 kV/cm and 88.4 K. Units: mm/us - driftSpeed: 1.563 - } - - -# ------------------------------------------------------------------------------------ # - -## Configuration for generating training samples -- target images -sbnd_wcls_samples_tru: @local::sbnd_wcls -sbnd_wcls_samples_tru.wcls_main.plugins: ["WireCellGen", "WireCellSigProc", "WireCellRoot", "WireCellPgraph", "WireCellLarsoft", "WireCellTbb", "WireCellPytorch", "WireCellHio"] -sbnd_wcls_samples_tru.wcls_main.configs: ["pgrapher/experiment/sbnd/wcls-sim-drift-depoflux-nf-sp-samples_tru.jsonnet"] -sbnd_wcls_samples_tru.wcls_main.inputers: ["wclsSimDepoSource:"] -sbnd_wcls_samples_tru.wcls_main.outputers:["wclsDepoFluxWriter:postdrift", "wclsFrameSaver:simdigits"] -sbnd_wcls_samples_tru.wcls_main.params: { - ## This locates the input SimEnergyDeposits in the art::Event - inputTag: "ionandscint:" - - ## Set "data" vs. "sim". The epoch below probably should follow suit. - reality: "sim" - - ## if epoch is "dynamic" you MUST add - ## "wclsMultiChannelNoiseDB" to "inputers" and must NOT - ## add it if not "dynamic" - epoch: "perfect" - - ## Save output signal waveforms (recob::Wire) in "sparse" or "dense" form - signal_output_form: "sparse" - - ## If save_simdigits="false", comment line with "wclsFrameSaver:simdigits" in outputers - ## If save_simdigits="true", uncomment line with "wclsFrameSaver:simdigits" in outputers - ## this is whether or not to save raw digits!!! - save_simdigits: "true" - - save_track_id: "true" - } - -sbnd_wcls_samples_tru.wcls_main.structs: { - ## Longitudinal diffusion constant [cm2/s] - DL: 4.0 - ## Transverse diffusion constant [cm2/s] - DT: 8.8 - ## Electron lifetime [ms] - lifetime: 100.0 - ## Electron drift speed, assumes 0.5 kV/cm and 88.4 K. Units: mm/us - driftSpeed: 1.563 - } - END_PROLOG diff --git a/sbndcode/WireCell/wcsp_data_sbnd.fcl b/sbndcode/WireCell/wcsp_data_sbnd.fcl index cf8b0d6c9..fad2236c7 100644 --- a/sbndcode/WireCell/wcsp_data_sbnd.fcl +++ b/sbndcode/WireCell/wcsp_data_sbnd.fcl @@ -16,7 +16,7 @@ sbnd_wcls: { configs: [] ## Libraries in which to look for WCT components - plugins: ["WireCellGen", "WireCellSigProc", "WireCellRoot", "WireCellPgraph", "WireCellLarsoft", "WireCellTbb", "WireCellPytorch"] + plugins: ["WireCellGen", "WireCellSigProc", "WireCellRoot", "WireCellPgraph", "WireCellLarsoft", "WireCellTbb"] inputers: [] outputers: [] @@ -53,17 +53,11 @@ sbnd_wcls_sp_data.wcls_main.params : { # whether or not to use calibrated, parametrized electronics response # MUST be a string! use_paramresp: "true" - - dnnroi_model_p0: "DNN_ROI/plane0.ts" - dnnroi_model_p1: "DNN_ROI/plane1.ts" - } + } sbnd_wcls_sp_data.wcls_main.structs: { # Set the waveform sample length, eg, 6000, 15000, "auto" nticks: 3427 - use_dnnroi: false - nchunks: 2 - tick_per_slice: 4 } END_PROLOG diff --git a/sbndcode/WireCell/wirecell_sim_rec_sbnd.fcl b/sbndcode/WireCell/wirecell_sim_rec_sbnd.fcl deleted file mode 100644 index b6dcb7423..000000000 --- a/sbndcode/WireCell/wirecell_sim_rec_sbnd.fcl +++ /dev/null @@ -1,90 +0,0 @@ -# -# File: wirecell_sim_rec.fcl -# Purpose: generates training input images for DNN ROI network -# -# This configuration runs Wire-Cell TPC Simulation and Signal Processing -# -# Input: -# - std::vector with label `ionandscint` -# -# Output: -# - h5 file with images of intermediate ROIs - - -# -# services -# - -#include "simulationservices_sbnd.fcl" -#include "messages_sbnd.fcl" - -# -# modules -# - -#include "detsimmodules_sbnd.fcl" -#include "rootoutput_sbnd.fcl" -#include "wcsimsp_sbnd.fcl" - -process_name: DNNsamplerec - -services: -{ - TFileService: { fileName: @local::sbnd_tfileoutput.fileName } - @table::sbnd_g4_services - FileCatalogMetadata: @local::sbnd_file_catalog_mc - message: { debugModules: ["*"] destinations: { debugmsg:{type: "cout" threshold: "INFO"} } } #added Ewerton 2023-06-30 - TimeTracker: { printSummary: true } -} - - -source: -{ - module_type: RootInput -} - -outputs: -{ - out1: - { - @table::sbnd_rootoutput # inherit shared settings - dataTier: "reconstructed" - #compressionLevel: 1 # TODO better to use no compression here and Huffman encoding - } -} - -# Define and configure some modules to do work on each event. -# First modules are defined; they are scheduled later. -# Modules are grouped by type. -physics: -{ - - producers: - { - # 2D TPC Simulation & Signal Processing - simtpc2d : @local::sbnd_wcls_samples_rec - } - - #define the producer and filter modules for this path, order matters, - simulate: [ simtpc2d ] #added wctsp Ewerton 2023-05-16 - - trigger_paths : [ simulate ] - - #define the output stream, there could be more than one if using filters - stream1: [ out1 ] - - #ie analyzers and output streams. these all run simultaneously - end_paths: [ stream1 ] -} - -# block to define where the output goes. if you defined a filter in the physics -# block and put it in the trigger_paths then you need to put a SelectEvents: {SelectEvents: [XXX]} -# entry in the output stream you want those to go to, where XXX is the label of the filter module(s) - -outputs.out1.outputCommands: [ - "keep *_*_*_*" , - # "drop *_ionandscint_*_*" - "keep *_simdigits_*_*" - ] - - diff --git a/sbndcode/WireCell/wirecell_sim_tru_sbnd.fcl b/sbndcode/WireCell/wirecell_sim_tru_sbnd.fcl deleted file mode 100644 index 0cbae0656..000000000 --- a/sbndcode/WireCell/wirecell_sim_tru_sbnd.fcl +++ /dev/null @@ -1,91 +0,0 @@ -# File: wirecell_sim_tru.fcl -# Purpose: generates training target images for DNN ROI network -# -# This configuration runs Wire-Cell TPC Simulation and Signal Processing -# -# Input: -# - std::vector with label `ionandscint` -# -# Output: -# - h5 file with target image from simulation - -# -# services -# - -#include "simulationservices_sbnd.fcl" -#include "messages_sbnd.fcl" - -# -# modules -# - -#include "detsimmodules_sbnd.fcl" -#include "rootoutput_sbnd.fcl" -#include "wcsimsp_sbnd.fcl" - - -process_name: DNNsampletru - -services: -{ - TFileService: { fileName: @local::sbnd_tfileoutput.fileName } - @table::sbnd_g4_services - FileCatalogMetadata: @local::sbnd_file_catalog_mc - message: { debugModules: ["*"] destinations: { debugmsg:{type: "cout" threshold: "INFO"} } } #added Ewerton 2023-06-30 - TimeTracker: { printSummary: true } - -} - - -source: -{ - module_type: RootInput -} - -outputs: -{ - out1: - { - @table::sbnd_rootoutput # inherit shared settings - dataTier: "reconstructed" - #compressionLevel: 1 # TODO better to use no compression here and Huffman encoding - } -} - -# Define and configure some modules to do work on each event. -# First modules are defined; they are scheduled later. -# Modules are grouped by type. -physics: -{ - - producers: - { - # 2D TPC Simulation & Signal Processing - simtpc2d : @local::sbnd_wcls_samples_tru - } - - #define the producer and filter modules for this path, order matters, - simulate: [ simtpc2d ] #added wctsp Ewerton 2023-05-16 - - trigger_paths : [ simulate ] - - #define the output stream, there could be more than one if using filters - stream1: [ out1 ] - - #ie analyzers and output streams. these all run simultaneously - end_paths: [ stream1 ] -} - -# block to define where the output goes. if you defined a filter in the physics -# block and put it in the trigger_paths then you need to put a SelectEvents: {SelectEvents: [XXX]} -# entry in the output stream you want those to go to, where XXX is the label of the filter module(s) - -outputs.out1.outputCommands: [ - "keep *_*_*_*" , - # "drop *_ionandscint_*_*" - "keep *_simdigits_*_*" - ] - - - diff --git a/ups/product_deps b/ups/product_deps index a54174eb3..a48c4e987 100644 --- a/ups/product_deps +++ b/ups/product_deps @@ -255,8 +255,8 @@ wpdir product_dir wire-cell-cfg product version qual flags sbncode v10_06_00_01 - cetmodules v3_24_01 - only_for_build -sbnd_data v01_32_00 - -sbndutil v10_06_01 - optional +sbnd_data v01_31_00 - +sbndutil v10_06_00 - optional fhiclpy v4_03_05 - end_product_list #################################### From 2d47f244c2e21ce41b043697856af6611fc889fe Mon Sep 17 00:00:00 2001 From: lynnt20 Date: Tue, 8 Jul 2025 12:00:40 -0500 Subject: [PATCH 7/8] fix to ensure that events with only numu dirt do not pass accidentally reverted the wrong commit, re-committing --- .../prodgenie_corsika_proton_rockbox_ccnue_sbnd.fcl | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/sbndcode/JobConfigurations/standard/gen/genie_corsika/prodgenie_corsika_proton_rockbox_ccnue_sbnd.fcl b/sbndcode/JobConfigurations/standard/gen/genie_corsika/prodgenie_corsika_proton_rockbox_ccnue_sbnd.fcl index bd8909aa1..9fc2123c6 100644 --- a/sbndcode/JobConfigurations/standard/gen/genie_corsika/prodgenie_corsika_proton_rockbox_ccnue_sbnd.fcl +++ b/sbndcode/JobConfigurations/standard/gen/genie_corsika/prodgenie_corsika_proton_rockbox_ccnue_sbnd.fcl @@ -3,6 +3,10 @@ physics.filters.ccnuefilter: @local::sbnd_ccnuefilter -physics.simulatetpc: [ rns, generator, loader, largeantnu, tpcfilter, - ccnuefilter, # adding this filter to the path - corsika, largeantcosmic, largeant, largeantdropped, simplemerge ] \ No newline at end of file +physics.simulatetpc: [ rns, generator, loader, largeantnu, + ccnuefilter, + corsika, largeantcosmic, largeant, largeantdropped, simplemerge] + +physics.simulatedirt: [ rns, generator, loader, largeantnu, "!tpcfilter", + dirtfilter, corsika, largeantcosmic, largeant, largeantdropped, simplemerge, + ccnuefilter] From d0735d5613c0df47e13f3f1746247504cee5f436 Mon Sep 17 00:00:00 2001 From: lynnt20 Date: Wed, 9 Jul 2025 09:46:01 -0500 Subject: [PATCH 8/8] Reapply "Merge branch 'develop' into feature/lynnt_ccnuefilter" This reverts commit d5e3270686e3fb9a57ef153929ec81562b3cfd4a. --- .github/pull_request_template.md | 7 + CMakeLists.txt | 2 +- README.md | 4 + .../base/cafmakerjob_sbnd_data_base.fcl | 5 +- .../standard/caf/cafmakerjob_sbnd.fcl | 2 + .../caf/cafmakerjob_sbnd_data_sce.fcl | 1 + .../standard/caf/cafmakerjob_sbnd_sce.fcl | 1 + .../standard/reco/reco1_data.fcl | 9 + .../standard/standard_detsim_sbnd.fcl | 6 + .../standard/standard_reco1_sbnd.fcl | 3 + .../Alg/OpDeconvolutionAlgWienerData_tool.cc | 89 +--- .../Alg/opdeconvolution_alg_data.fcl | 1 - .../experiment/sbnd/chndb-base.jsonnet | 33 +- .../pgrapher/experiment/sbnd/dnnroi.jsonnet | 112 +++++ .../experiment/sbnd/wcls-nf-sp-data.jsonnet | 179 ++++++- .../experiment/sbnd/wcls-nf-sp.jsonnet | 179 ++++++- ...m-drift-depoflux-nf-sp-samples_rec.jsonnet | 373 ++++++++++++++ ...m-drift-depoflux-nf-sp-samples_tru.jsonnet | 474 ++++++++++++++++++ .../wcls-sim-drift-depoflux-nf-sp.jsonnet | 196 +++++++- sbndcode/WireCell/wcsimsp_sbnd.fcl | 111 +++- sbndcode/WireCell/wcsp_data_sbnd.fcl | 10 +- sbndcode/WireCell/wirecell_sim_rec_sbnd.fcl | 90 ++++ sbndcode/WireCell/wirecell_sim_tru_sbnd.fcl | 91 ++++ ups/product_deps | 4 +- 24 files changed, 1881 insertions(+), 101 deletions(-) create mode 100644 README.md create mode 100644 sbndcode/WireCell/cfg/pgrapher/experiment/sbnd/dnnroi.jsonnet create mode 100644 sbndcode/WireCell/cfg/pgrapher/experiment/sbnd/wcls-sim-drift-depoflux-nf-sp-samples_rec.jsonnet create mode 100644 sbndcode/WireCell/cfg/pgrapher/experiment/sbnd/wcls-sim-drift-depoflux-nf-sp-samples_tru.jsonnet create mode 100644 sbndcode/WireCell/wirecell_sim_rec_sbnd.fcl create mode 100644 sbndcode/WireCell/wirecell_sim_tru_sbnd.fcl diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 64be2e089..f6aa43877 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -1,6 +1,12 @@ ## Description Please provide a detailed description of the changes this pull request introduces. +$${\color{red}!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!}$$ + +$${\color{red}\bf{\textrm{IMPORTANT UPDATE June 22nd 2025:}}}$$ If you are making a PR which is intended as a patch for the CURRENT production (which started in Spring 2025), you must make two PRs: one for develop and one for the production/v10_06_00 branch. + +$${\color{red}!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!}$$ + ## Checklist - [ ] Added at least 1 label from [available labels](https://github.com/SBNSoftware/sbndcode/issues/labels?sort=name-asc). - [ ] Assigned at least 1 reviewer under `Reviewers`, @@ -8,6 +14,7 @@ Please provide a detailed description of the changes this pull request introduce - [ ] Linked any relevant issues under `Developement` - [ ] Does this PR affect CAF data format? If so, please assign a CAF maintainer ([PetrilloAtWork](https://github.com/PetrilloAtWork) or [JosiePaton](https://github.com/JosiePaton)) as additional reviewer. - [ ] Does this affect the standard workflow? +- [ ] Is this PR a patch for the ongoing production? If so, separate PR must also be made for production/v10_06_00 branch! ### Relevant PR links (optional) Does this PR require merging another PR in a different repository (such as sbnanobj/sbnobj etc.)? diff --git a/CMakeLists.txt b/CMakeLists.txt index d470b1cbc..7a0f6e62c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -15,7 +15,7 @@ cmake_minimum_required(VERSION 3.20 FATAL_ERROR) -set(${PROJECT_NAME}_CMAKE_PROJECT_VERSION_STRING 10.06.00.01) +set(${PROJECT_NAME}_CMAKE_PROJECT_VERSION_STRING 10.06.01) find_package(cetmodules REQUIRED) project(sbndcode LANGUAGES CXX) diff --git a/README.md b/README.md new file mode 100644 index 000000000..25a00403e --- /dev/null +++ b/README.md @@ -0,0 +1,4 @@ +# sbndcode + +## Getting Started +[SBN Young Guide](https://sbnsoftware.github.io/SBNYoung/SBNYoungIndex.html) diff --git a/sbndcode/JobConfigurations/base/cafmakerjob_sbnd_data_base.fcl b/sbndcode/JobConfigurations/base/cafmakerjob_sbnd_data_base.fcl index 8ce957011..16be4bd56 100644 --- a/sbndcode/JobConfigurations/base/cafmakerjob_sbnd_data_base.fcl +++ b/sbndcode/JobConfigurations/base/cafmakerjob_sbnd_data_base.fcl @@ -125,8 +125,11 @@ physics: # change tools in sbn to have 2D TPC sim/sp input physics.producers.pandoraShowerRazzle.SimChannelLabel: "sptpc2d:gauss" physics.producers.pandoraTrackDazzle.SimChannelLabel: "sptpc2d:gauss" - physics.producers.cnnid.WireLabel: "sptpc2d:gauss" +# uncomment below lines to use DNN ROI SP for Razzle/Dazzle/CNNID +#physics.producers.pandoraShowerRazzle.SimChannelLabel: "sptpc2d:dnnsp" +#physics.producers.pandoraTrackDazzle.SimChannelLabel: "sptpc2d:dnnsp" +#physics.producers.cnnid.WireLabel: "sptpc2d:dnnsp" ### Calorimetry for data physics.producers.cnnid.PointIdAlg.CalorimetryAlg: @local::sbnd_calorimetryalgdata diff --git a/sbndcode/JobConfigurations/standard/caf/cafmakerjob_sbnd.fcl b/sbndcode/JobConfigurations/standard/caf/cafmakerjob_sbnd.fcl index 3adef6ba4..2281046cf 100644 --- a/sbndcode/JobConfigurations/standard/caf/cafmakerjob_sbnd.fcl +++ b/sbndcode/JobConfigurations/standard/caf/cafmakerjob_sbnd.fcl @@ -127,6 +127,8 @@ physics: physics.producers.pandoraShowerRazzle.SimChannelLabel: "simtpc2d:simpleSC" physics.producers.pandoraTrackDazzle.SimChannelLabel: "simtpc2d:simpleSC" physics.producers.cnnid.WireLabel: "simtpc2d:gauss" +# uncomment below line for to use DNN ROI SP for CNNID scores +#physics.producers.cnnid.WireLabel: "simtpc2d:dnnsp" physics.producers.vertexCharge.CaloAlg: @local::sbnd_calorimetryalgmc physics.producers.vertexStub.CaloAlg: @local::sbnd_calorimetryalgmc diff --git a/sbndcode/JobConfigurations/standard/caf/cafmakerjob_sbnd_data_sce.fcl b/sbndcode/JobConfigurations/standard/caf/cafmakerjob_sbnd_data_sce.fcl index 16827b43c..4ffb5ba94 100644 --- a/sbndcode/JobConfigurations/standard/caf/cafmakerjob_sbnd_data_sce.fcl +++ b/sbndcode/JobConfigurations/standard/caf/cafmakerjob_sbnd_data_sce.fcl @@ -11,6 +11,7 @@ physics.producers.cafmaker.TrackChi2PidLabel: "pandoraSCEPidData" physics.producers.cafmaker.CRTSpacePointMatchLabel: "crtspacepointmatchingSCE" physics.producers.cafmaker.SBNDCRTTrackMatchLabel: "crttrackmatchingSCE" physics.producers.cafmaker.OpT0Label: "opt0finderSCE" +physics.producers.cafmaker.TPCPMTBarycenterMatchLabel: "tpcpmtbarycentermatchingSCE" physics.producers.cafmaker.CVNLabel: "cvnSCE" physics.producers.cnnid.ClusterModuleLabel: "pandoraSCE" diff --git a/sbndcode/JobConfigurations/standard/caf/cafmakerjob_sbnd_sce.fcl b/sbndcode/JobConfigurations/standard/caf/cafmakerjob_sbnd_sce.fcl index 5e626a694..177326bfe 100644 --- a/sbndcode/JobConfigurations/standard/caf/cafmakerjob_sbnd_sce.fcl +++ b/sbndcode/JobConfigurations/standard/caf/cafmakerjob_sbnd_sce.fcl @@ -13,6 +13,7 @@ physics.producers.cafmaker.SBNDCRTTrackMatchLabel: "crttrackmatchingSCE" physics.producers.cafmaker.CRTHitMatchLabel: "pandoraSCETrackCRTHit" physics.producers.cafmaker.CRTTrackMatchLabel: "pandoraSCETrackCRTTrack" physics.producers.cafmaker.OpT0Label: "opt0finderSCE" +physics.producers.cafmaker.TPCPMTBarycenterMatchLabel: "tpcpmtbarycentermatchingSCE" physics.producers.cafmaker.CVNLabel: "cvnSCE" physics.producers.cnnid.ClusterModuleLabel: "pandoraSCE" diff --git a/sbndcode/JobConfigurations/standard/reco/reco1_data.fcl b/sbndcode/JobConfigurations/standard/reco/reco1_data.fcl index 896b52c6c..563a4b64a 100644 --- a/sbndcode/JobConfigurations/standard/reco/reco1_data.fcl +++ b/sbndcode/JobConfigurations/standard/reco/reco1_data.fcl @@ -35,3 +35,12 @@ physics.ana: [superadata] outputs.out1.SelectEvents: [ "reco1" ] physics.producers.gaushit.CalDataModuleLabel: "sptpc2d:gauss" + +# uncomment below 4 lines to run DNN ROI finding SP +#physics.producers.sptpc2d.wcls_main.outputers: ["wclsFrameSaver:spsaver" , "wclsFrameSaver:dnnsaver"] +#physics.producers.sptpc2d.wcls_main.structs.use_dnnroi: true +#physics.producers.sptpc2d.wcls_main.structs.nchunks: 2 # should match training config +#physics.producers.sptpc2d.wcls_main.structs.tick_per_slice: 4 # should match training config + +# uncomment below line to run DNN ROI finding SP for reco +#physics.producers.gaushit.CalDataModuleLabel: "sptpc2d:dnnsp" diff --git a/sbndcode/JobConfigurations/standard/standard_detsim_sbnd.fcl b/sbndcode/JobConfigurations/standard/standard_detsim_sbnd.fcl index 1bb03bcc0..b13b52cb0 100644 --- a/sbndcode/JobConfigurations/standard/standard_detsim_sbnd.fcl +++ b/sbndcode/JobConfigurations/standard/standard_detsim_sbnd.fcl @@ -92,3 +92,9 @@ outputs: ] } } + +# uncomment below 4 lines to run DNN ROI finding SP +# physics.producers.simtpc2d.wcls_main.outputers: ["wclsDepoFluxWriter:postdrift", "wclsFrameSaver:spsaver", "wclsFrameSaver:dnnsaver"] # "wclsFrameSaver:simdigits" <- by default, do not save RawDigits. Uncomment this line to save RawDigits and set +# physics.producers.simtpc2d.wcls_main.structs.use_dnnroi: true +# physics.producers.simtpc2d.wcls_main.structs.nchunks: 2 # should match training config +# physics.producers.simtpc2d.wcls_main.structs.tick_per_slice: 4 # should match training config diff --git a/sbndcode/JobConfigurations/standard/standard_reco1_sbnd.fcl b/sbndcode/JobConfigurations/standard/standard_reco1_sbnd.fcl index b6a95ce30..90dd8eb13 100644 --- a/sbndcode/JobConfigurations/standard/standard_reco1_sbnd.fcl +++ b/sbndcode/JobConfigurations/standard/standard_reco1_sbnd.fcl @@ -35,3 +35,6 @@ physics.end_paths: [ @sequence::physics.end_paths, ana ] #outputs table overrides outputs.out1.dataTier: "reconstructed" outputs.out1.outputCommands: [@sequence::outputs.out1.outputCommands, @sequence::sbnd_reco1_drops] + +# uncomment below line to run DNN ROI finding SP for reco -- must have run DNN ROI finding in detsim stage +#physics.producers.gaushit.CalDataModuleLabel: "simtpc2d:dnnsp" diff --git a/sbndcode/OpDetReco/OpDeconvolution/Alg/OpDeconvolutionAlgWienerData_tool.cc b/sbndcode/OpDetReco/OpDeconvolution/Alg/OpDeconvolutionAlgWienerData_tool.cc index cbd23acef..d33cc57ee 100644 --- a/sbndcode/OpDetReco/OpDeconvolution/Alg/OpDeconvolutionAlgWienerData_tool.cc +++ b/sbndcode/OpDetReco/OpDeconvolution/Alg/OpDeconvolutionAlgWienerData_tool.cc @@ -27,6 +27,8 @@ #include "sbndcode/OpDetReco/OpDeconvolution/Alg/OpDeconvolutionAlg.hh" +#include "sbndcode/Calibration/PDSDatabaseInterface/PMTCalibrationDatabase.h" +#include "sbndcode/Calibration/PDSDatabaseInterface/IPMTCalibrationDatabaseService.h" namespace opdet { class OpDeconvolutionAlgWiener; @@ -37,7 +39,7 @@ class opdet::OpDeconvolutionAlgWiener : opdet::OpDeconvolutionAlg { public: explicit OpDeconvolutionAlgWiener(fhicl::ParameterSet const& p); - ~OpDeconvolutionAlgWiener() {} + ~OpDeconvolutionAlgWiener() {delete fFilterTF1;} // Required functions. std::vector RunDeconvolution(std::vector const& wfHandle) override; @@ -65,7 +67,6 @@ class opdet::OpDeconvolutionAlgWiener : opdet::OpDeconvolutionAlg { double fPMTChargeToADC; double fDecoWaveformPrecision; short unsigned int fBaselineSample; - std::string fOpDetDataFile; std::string fFilter; std::string fElectronics; bool fScaleHypoSignal; @@ -80,6 +81,7 @@ class opdet::OpDeconvolutionAlgWiener : opdet::OpDeconvolutionAlg { unsigned int NDecoWf; TF1 *fFilterTF1; + std::vector fSignalHypothesis; std::vector fNoiseHypothesis; @@ -88,6 +90,8 @@ class opdet::OpDeconvolutionAlgWiener : opdet::OpDeconvolutionAlg { short unsigned int fBaseSampleBins; double fBaseVarCut; + sbndDB::PMTCalibrationDatabase const* fPMTCalibrationDatabaseService; + // Declare member functions void ApplyExpoAvSmoothing(std::vector& wf); void ApplyUnAvSmoothing(std::vector& wf); @@ -125,7 +129,6 @@ opdet::OpDeconvolutionAlgWiener::OpDeconvolutionAlgWiener(fhicl::ParameterSet co fPMTChargeToADC = p.get< double >("PMTChargeToADC"); fDecoWaveformPrecision = p.get< double >("DecoWaveformPrecision"); fBaselineSample = p.get< short unsigned int >("BaselineSample"); - fOpDetDataFile = p.get< std::string >("OpDetDataFile"); fSkipChannelList = p.get< std::vector>("SkipChannelList"); fFilter = p.get< std::string >("Filter"); fElectronics = p.get< std::string >("Electronics"); @@ -138,6 +141,8 @@ opdet::OpDeconvolutionAlgWiener::OpDeconvolutionAlgWiener(fhicl::ParameterSet co fBaseVarCut = p.get< double >("BaseVarCut"); fFilterParams = p.get< std::vector >("FilterParams"); + fFilterTF1 = new TF1("FilterTemplate", fFilter.c_str()); + fNormUnAvSmooth=1./(2*fUnAvNeighbours+1); NDecoWf=0; MaxBinsFFT=std::pow(2, fMaxFFTSizePow); @@ -147,38 +152,10 @@ opdet::OpDeconvolutionAlgWiener::OpDeconvolutionAlgWiener(fhicl::ParameterSet co if (fElectronics=="Daphne") fSamplingFreq=fDaphne_Freq/1000.;//in GHz auto const* lar_prop = lar::providerFrom(); - //Load SER - std::string fname; - cet::search_path sp("FW_SEARCH_PATH"); - sp.find_file(fOpDetDataFile, fname); - TFile* file = TFile::Open(fname.c_str(), "READ"); - std::vector>* SinglePEVec_p; - std::vector* fSinglePEChannels_p; - std::vector* fPeakAmplitude_p; - std::vector> * fFilterParamVector_p; - - file->GetObject("SERChannels", fSinglePEChannels_p); - file->GetObject("SinglePEVec", SinglePEVec_p); - file->GetObject("PeakAmplitude", fPeakAmplitude_p); - file->GetObject("FilterParams", fFilterParamVector_p); - - if (fElectronics=="Daphne") file->GetObject("SinglePEVec_40ftCable_Daphne", SinglePEVec_p); - fSinglePEWaveVector = *SinglePEVec_p; - fSinglePEChannels = *fSinglePEChannels_p; - fPeakAmplitude = *fPeakAmplitude_p; - fFilterParamVector = *fFilterParamVector_p; - - mf::LogInfo("OpDeconvolutionAlg")<<"Loaded SER from "<SetParameter(k, fFilterParams[k]); - mf::LogInfo("OpDeconvolutionAlg")<<"Creating parametrized filter... TF1:"<(); + + if (!fUseParamFilter){ //Create light signal hypothesis for "on-fly" Wiener filter fSignalHypothesis.resize(MaxBinsFFT, 0); if(fFilter=="Wiener") @@ -200,38 +177,26 @@ std::vector opdet::OpDeconvolutionAlgWiener::RunDeconvolutio { int channelNumber = wf.ChannelNumber(); auto it = std::find(fSkipChannelList.begin(), fSkipChannelList.end(), channelNumber); - bool AnalyseChannel = false; if (it == fSkipChannelList.end()) { - //If it's not try to find its SER in the file - for(size_t i=0; igetSER(channelNumber); + double SPEAmplitude = fPMTCalibrationDatabaseService->getSPEAmplitude(channelNumber); + double SPEPeakValue = *std::max_element(fSinglePEWave.begin(), fSinglePEWave.end(), [](double a, double b) {return std::abs(a) < std::abs(b);}); + double SinglePENormalization = std::abs(SPEAmplitude/SPEPeakValue); + std::transform(fSinglePEWave.begin(), fSinglePEWave.end(), fSinglePEWave.begin(), [SinglePENormalization](double val) {return val * SinglePENormalization;}); + fSinglePEWave.resize(MaxBinsFFT, 0); + // If use channel dependent param filter + if(fUseParamFilter) { - if(fSinglePEChannels[i]==channelNumber) + double GaussFilterPower = fPMTCalibrationDatabaseService->getGaussFilterPower(channelNumber); + double GaussFilterWC = fPMTCalibrationDatabaseService->getGaussFilterWC(channelNumber); + fFilterParamChannel = {GaussFilterWC, GaussFilterPower}; + for(size_t k=0; kSetParameter(k, fFilterParamChannel[k]); - } - - - mf::LogInfo("OpDeconvolutionAlg")<<"Creating parametrized filter... TF1:"<SetParameter(k, fFilterParamChannel[k]); } + mf::LogInfo("OpDeconvolutionAlg")<<"Creating parametrized filter... TF1:"<MaxBinsFFT){ @@ -537,7 +502,6 @@ std::vector opdet::OpDeconvolutionAlgWiener::DeconvolutionKernel(size_ } } - if(fDebug){ std::string name="h_wienerfilter_"+std::to_string(NDecoWf); TH1F * hs_wiener = tfs->make< TH1F > @@ -545,7 +509,6 @@ std::vector opdet::OpDeconvolutionAlgWiener::DeconvolutionKernel(size_ for(size_t k=0; kSetBinContent(k, TComplex::Abs( kernel[k]*serfft[k] ) ); } - if(fUseParamFilter && fUseParamFilterInidividualChannel) delete fFilterTF1; return kernel; } diff --git a/sbndcode/OpDetReco/OpDeconvolution/Alg/opdeconvolution_alg_data.fcl b/sbndcode/OpDetReco/OpDeconvolution/Alg/opdeconvolution_alg_data.fcl index fb1f68deb..6d6821780 100644 --- a/sbndcode/OpDetReco/OpDeconvolution/Alg/opdeconvolution_alg_data.fcl +++ b/sbndcode/OpDetReco/OpDeconvolution/Alg/opdeconvolution_alg_data.fcl @@ -6,7 +6,6 @@ OpDeconvolutionAlgData: tool_type: "OpDeconvolutionAlgWienerData" Debug: false MaxFFTSizePow: 16 - OpDetDataFile: "./OpDetSim/digi_pmt_sbnd_data_OV6.root" BaseSampleBins: 30 BaseVarCut: 50 SkipChannelList: [] diff --git a/sbndcode/WireCell/cfg/pgrapher/experiment/sbnd/chndb-base.jsonnet b/sbndcode/WireCell/cfg/pgrapher/experiment/sbnd/chndb-base.jsonnet index 0da0eccc1..f08a523b7 100644 --- a/sbndcode/WireCell/cfg/pgrapher/experiment/sbnd/chndb-base.jsonnet +++ b/sbndcode/WireCell/cfg/pgrapher/experiment/sbnd/chndb-base.jsonnet @@ -22,16 +22,31 @@ function(params, anode, field, n, rms_cuts=[]) // coherent noise filtering. // 5638 is the number of channels in a single APA, (1984*2 + 1670), // including the 6 channel gap. - // The induction planes have to types of ch grouping (due to FEMB), - // one where the grouping is 32 channels wide and one where it is 128 channels wide. - // The collection planes are grouped by 64. + // The induction planes have two types of ch grouping (due to FEMB installation) + // one where the grouping is 32 channels wide and one where it is 128 + // (but split into 2 groups of 64) channels wide. + // This grouping is flipped for the same plane in the second APA. + // The collection planes are grouped by 64. + // Due to a group of dead 32 channels in collection plane in APA0, + // we specify separate groups of 32 (dead) and 32 (alive) within that group of 64 + // so that coherent noise filtering is applied properly to the alive channels. - groups: [std.range( 0 + n * 5638 + g*32 , 0 + n * 5638 + (g+1)*32 - 1) for g in std.range(0,26)] + # first section of u - [std.range( 832 + n * 5638 + g*128, 832 + n * 5638 + (g+1)*128 - 1) for g in std.range(0,9)] + # second section of u - [std.range(1984 + n * 5638 + g*128, 1984 + n * 5638 + (g+1)*128 - 1) for g in std.range(0,9)] + # first section of v - [std.range(3136 + n * 5638 + g*32 , 3136 + n * 5638 + (g+1)*32 - 1) for g in std.range(0,26)] + # second section of v - [std.range(3974 + n * 5638 + g*64 , 3974 + n * 5638 + (g+1)*64 - 1) for g in std.range(0,13)] + # first half of w - [std.range(4806 + n * 5638 + g*64 , 4806 + n * 5638 + (g+1)*64 - 1) for g in std.range(0,13)] , # second half of w + groups: [std.range( 0 + g*32 , 0 + (g+1)*32 - 1) for g in std.range(0,25)] + # first section of u0 + [std.range( 832 + g*64 , 832 + (g+1)*64 - 1) for g in std.range(0,17)] + # second section of u0 + [std.range(1984 + g*64 , 1984 + (g+1)*64 - 1) for g in std.range(0,17)] + # first section of v0 + [std.range(3136 + g*32 , 3136 + (g+1)*32 - 1) for g in std.range(0,25)] + # second section of v0 + + [std.range(5638 + g*64 , 5638 + (g+1)*64 - 1) for g in std.range(0,17)] + # first section of u1 + [std.range(6790 + g*32 , 6790 + (g+1)*32 - 1) for g in std.range(0,25)] + # second section of u1 + [std.range(7622 + g*32 , 7622 + (g+1)*32 - 1) for g in std.range(0,25)] + # first section of v1 + [std.range(8454 + g*64 , 8454 + (g+1)*64 - 1) for g in std.range(0,17)] + # second section of v1 + + [std.range(3968 + n * 5638 + g*64 , 3968 + n * 5638 + (g+1)*64 - 1) for g in std.range(0,2)] + # first three groups of w + [std.range(4160 , 4192 - 1)] + # w apa0 4th group, first 32 dead wire (for completeness) + [std.range(4192 , 4224 - 1)] + # w apa0 4th group, remaining 32 alive wire (next to dead wire) + [std.range(9798 , 9862 - 1)] + # w apa1 4th group of 64 (alive) + [std.range(4224 + n * 5638 + g*64 , 4224 + n * 5638 + (g+1)*64 - 1) for g in std.range(0,8)] + # first half of w (excluding first 4 groups) + [std.range(4806 + n * 5638 + g*64 , 4806 + n * 5638 + (g+1)*64 - 1) for g in std.range(0,12)] , # second half of w // Externally determined "bad" channels. // diff --git a/sbndcode/WireCell/cfg/pgrapher/experiment/sbnd/dnnroi.jsonnet b/sbndcode/WireCell/cfg/pgrapher/experiment/sbnd/dnnroi.jsonnet new file mode 100644 index 000000000..909ad4b23 --- /dev/null +++ b/sbndcode/WireCell/cfg/pgrapher/experiment/sbnd/dnnroi.jsonnet @@ -0,0 +1,112 @@ +// This produces a function to configure DNN-ROI for one APA given +// anode and torch service (ts) objects. +// +// The prefix is prepended to all internal node names if uniqueness +// beyond anode ID is needed. The output_scale allows for an ad-hoc +// scaling of dnnroi output. The U and W planes will go through +// dnnroi while hte W plane will be shunted. What comes out will be a +// unified frame with frame tag "dnnspN" where "N" is the anode ID. + +local wc = import "wirecell.jsonnet"; +local pg = import "pgraph.jsonnet"; + + +function (anode, ts_p0, ts_p1, prefix="dnnroi", output_scale=1.0, nticks=3428, tick_per_slice=4, nchunks=1) + local apaid = anode.data.ident; + local prename = prefix + std.toString(apaid); + local intags = ['loose_lf%d'%apaid, 'mp2_roi%d'%apaid, + 'mp3_roi%d'%apaid]; + + local dnnroi_u = pg.pnode({ + type: "DNNROIFinding", + name: prename+"u", + data: { + anode: wc.tn(anode), + plane: 0, + intags: intags, + decon_charge_tag: "decon_charge%d" %apaid, + outtag: "dnnsp%du"%apaid, + input_scale: 0.00025, // 1/4000 + output_scale: output_scale, + forward: wc.tn(ts_p0), + tick_per_slice: tick_per_slice, + nticks: nticks, + nchunks: nchunks + } + }, nin=1, nout=1, uses=[ts_p0, anode]); + local dnnroi_v = pg.pnode({ + type: "DNNROIFinding", + name: prename+"v", + data: { + anode: wc.tn(anode), + plane: 1, + intags: intags, + decon_charge_tag: "decon_charge%d" %apaid, + outtag: "dnnsp%dv"%apaid, + input_scale: 0.00025, // 1/4000 + output_scale: output_scale, + forward: wc.tn(ts_p1), + tick_per_slice: tick_per_slice, + nticks: nticks, + nchunks: nchunks + } + }, nin=1, nout=1, uses=[ts_p1, anode]); + local dnnroi_w = pg.pnode({ + type: "PlaneSelector", + name: prename+"w", + data: { + anode: wc.tn(anode), + plane: 2, + tags: ["gauss%d"%apaid], + tag_rules: [{ + frame: {".*":"DNNROIFinding"}, + trace: {["gauss%d"%apaid]:"dnnsp%dw"%apaid}, + }], + } + }, nin=1, nout=1, uses=[anode]); + + local dnnpipes = [dnnroi_u, dnnroi_v, dnnroi_w]; + local dnnfanout = pg.pnode({ + type: "FrameFanout", + name: prename, + data: { + multiplicity: 3 + } + }, nin=1, nout=3); + + local dnnfanin = pg.pnode({ + type: "FrameFanin", + name: prename, + data: { + multiplicity: 3, + tag_rules: [{ + frame: {".*": "dnnsp%d%s" % [apaid,plane]}, + trace: {".*": "dnnsp%d%s" % [apaid,plane]}, + } for plane in ["u", "v", "w"]] + }, + }, nin=3, nout=1); + + local retagger = pg.pnode({ + type: "Retagger", + name: 'dnnroi%d' % apaid, + data: { + // Note: retagger keeps tag_rules an array to be like frame fanin/fanout. + tag_rules: [{ + // Retagger also handles "frame" and "trace" like fanin/fanout + // merge separately all traces like gaussN to gauss. + frame: { + ".*": "dnnsp%d" % apaid + }, + merge: { + ".*": "dnnsp%d" % apaid + }, + }], + }, + }, nin=1, nout=1); + + pg.intern(innodes=[dnnfanout], + outnodes=[retagger], + centernodes=dnnpipes+[dnnfanin], + edges=[pg.edge(dnnfanout, dnnpipes[ind], ind, 0) for ind in [0,1,2]] + + [pg.edge(dnnpipes[ind], dnnfanin, 0, ind) for ind in [0,1,2]] + + [pg.edge(dnnfanin, retagger, 0, 0)]) diff --git a/sbndcode/WireCell/cfg/pgrapher/experiment/sbnd/wcls-nf-sp-data.jsonnet b/sbndcode/WireCell/cfg/pgrapher/experiment/sbnd/wcls-nf-sp-data.jsonnet index e64652a28..875c5bfd8 100644 --- a/sbndcode/WireCell/cfg/pgrapher/experiment/sbnd/wcls-nf-sp-data.jsonnet +++ b/sbndcode/WireCell/cfg/pgrapher/experiment/sbnd/wcls-nf-sp-data.jsonnet @@ -24,6 +24,11 @@ local sigoutform = std.extVar('signal_output_form'); // eg "sparse" or "dense" local raw_input_label = std.extVar('raw_input_label'); // eg "daq" local use_paramresp = std.extVar('use_paramresp'); // eg "true" or "false" +local use_dnnroi = std.extVar('use_dnnroi'); +local nchunks = std.extVar('nchunks'); +local tick_per_slice = std.extVar('tick_per_slice'); +local dnnroi_model_p0 = std.extVar('dnnroi_model_p0'); +local dnnroi_model_p1 = std.extVar('dnnroi_model_p1'); local g = import 'pgraph.jsonnet'; local f = import 'pgrapher/experiment/sbnd/funcs.jsonnet'; @@ -124,6 +129,22 @@ local wcls_output = { chanmaskmaps: ['bad'], }, }, nin=1, nout=1, uses=[mega_anode]), + + dnnsp_signals: g.pnode({ + type: 'wclsFrameSaver', + name: 'dnnsaver', + data: { + anode: wc.tn(mega_anode), + digitize: false, // true means save as RawDigit, else recob::Wire + frame_tags: ['dnnsp'], + + // this may be needed to convert the decon charge [units:e-] to be consistent with the LArSoft default ?unit? e.g. decon charge * 0.005 --> "charge value" to GaussHitFinder + frame_scale: [0.02, 0.02, 0.02], + nticks: params.daq.nticks, + + chanmaskmaps: ['dnnspbad'], + }, + }, nin=1, nout=1, uses=[mega_anode]), }; local base = import 'pgrapher/experiment/sbnd/chndb-base.jsonnet'; @@ -151,14 +172,85 @@ local nf_maker = import 'pgrapher/experiment/sbnd/nf-data.jsonnet'; //added Ewer local nf_pipes = [nf_maker(params, tools.anodes[n], chndb[n], n, name='nf%d' % n) for n in std.range(0, std.length(tools.anodes) - 1)]; local sp_maker = import 'pgrapher/experiment/sbnd/sp.jsonnet'; -local sp = sp_maker(params, tools, { sparse: sigoutform == 'sparse' }); +local sp_override = if use_dnnroi then { + sparse: true, + use_roi_debug_mode: true, + save_negtive_charge: true, // TODO: no negative charge in gauss, default is false + use_multi_plane_protection: true, + mp_tick_resolution: 4, + tight_lf_tag: "", + // loose_lf_tag: "", + cleanup_roi_tag: "", + break_roi_loop1_tag: "", + break_roi_loop2_tag: "", + shrink_roi_tag: "", + extend_roi_tag: "", + // m_decon_charge_tag: "", +} else { + sparse: true, +}; +//local sp = sp_maker(params, tools, { sparse: sigoutform == 'sparse' }); +local sp = sp_maker(params, tools, sp_override); local sp_pipes = [sp.make_sigproc(a) for a in tools.anodes]; +local dnnroi = import 'dnnroi.jsonnet'; +local ts_p0 = { + type: "TorchService", + name: "dnnroi_p0", + tick_per_slice: tick_per_slice, + data: { + model: dnnroi_model_p0, + device: "cpu", + concurrency: 1, + }, +}; + +local ts_p1 = { + type: "TorchService", + name: "dnnroi_p1", + tick_per_slice: tick_per_slice, + data: { + model: dnnroi_model_p1, + device: "cpu", + concurrency: 1, + }, +}; local magoutput = 'sbnd-data-check.root'; local magnify = import 'pgrapher/experiment/sbnd/magnify-sinks.jsonnet'; local sinks = magnify(tools, magoutput); -local nfsp_pipes = [ +local fanout = function (name, multiplicity=2) + g.pnode({ + type: 'FrameFanout', + name: name, + data: { + multiplicity: multiplicity + }, + }, nin=1, nout=multiplicity); + +local sp_fans = [fanout("sp_fan_%d" % n) for n in std.range(0, std.length(tools.anodes) - 1)]; +local dnnroi_pipes = [ dnnroi(tools.anodes[n], ts_p0, ts_p1, output_scale=1, nchunks=nchunks) for n in std.range(0, std.length(tools.anodes) - 1) ]; +local nfsp_pipes = if use_dnnroi then +[ + g.intern( + innodes=[chsel_pipes[n]], + outnodes=[dnnroi_pipes[n],sinks.decon_pipe[n]], + centernodes=[nf_pipes[n], sp_pipes[n], sp_fans[n], sinks.decon_pipe[n]], + edges=[ + g.edge(chsel_pipes[n], nf_pipes[n], 0, 0), + g.edge(nf_pipes[n], sp_pipes[n], 0, 0), + g.edge(sp_pipes[n], sp_fans[n], 0, 0), + g.edge(sp_fans[n], dnnroi_pipes[n], 0, 0), + g.edge(sp_fans[n], sinks.decon_pipe[n], 1, 0), + ], + iports=chsel_pipes[n].iports, + oports=dnnroi_pipes[n].oports+sinks.decon_pipe[n].oports, + name='nfsp_pipe_%d' % n, + ) + for n in std.range(0, std.length(tools.anodes) - 1) +] +else +[ g.pipeline([ chsel_pipes[n], //sinks.orig_pipe[n], @@ -177,8 +269,9 @@ local nfsp_pipes = [ local fanpipe = f.fanpipe('FrameFanout', nfsp_pipes, 'FrameFanin', 'sn_mag_nf'); -local retagger = g.pnode({ +local retagger = function(name) g.pnode({ type: 'Retagger', + name: name, data: { // Note: retagger keeps tag_rules an array to be like frame fanin/fanout. tag_rules: [{ @@ -190,16 +283,89 @@ local retagger = g.pnode({ merge: { 'gauss\\d': 'gauss', 'wiener\\d': 'wiener', + 'dnnsp\\d': 'dnnsp', }, }], }, }, nin=1, nout=1); -local sink = g.pnode({ type: 'DumpFrames' }, nin=1, nout=0); +local retag_dnnroi = retagger("retag_dnnroi"); +local retag_sp = retagger("retag_sp"); -// local graph = g.pipeline([wcls_input.adc_digits, rootfile_creation_frames, fanpipe, retagger, wcls_output.sp_signals, sink]); -local graph = g.pipeline([wcls_input.adc_digits, fanpipe, retagger, wcls_output.sp_signals, sink]); +local sink = function(name) g.pnode({ type: 'DumpFrames', name: name }, nin=1, nout=0); +local sink_dnnroi = sink("sink_dnnroi"); +local sink_sp = sink("sink_sp"); + +local fanout_apa = g.pnode({ + type: 'FrameFanout', + name: 'fanout_apa', + data: { + multiplicity: std.length(tools.anodes), + "tag_rules": [ + { + "frame": { + ".*": "orig%d" % n + }, + "trace": { } + } + for n in std.range(0, std.length(tools.anodes) - 1) + ] + }}, + nin=1, nout=std.length(tools.anodes)); +local framefanin = function(name) g.pnode({ + type: 'FrameFanin', + name: name, + data: { + multiplicity: std.length(tools.anodes), + + "tag_rules": [ + { + "frame": { + ".*": "framefanin" + }, + trace: { + ['dnnsp%d' % n]: ['dnnsp%d' % n], + ['gauss%d' % n]: ['gauss%d' % n], + ['wiener%d' % n]: ['wiener%d' % n], + ['threshold%d' % n]: ['threshold%d' % n], + }, + } + for n in std.range(0, std.length(tools.anodes) - 1) + ], + "tags": [ ] + }, +}, nin=std.length(tools.anodes), nout=1); +local fanin_apa_dnnroi = framefanin('fanin_apa_dnnroi'); +local fanin_apa_sp = framefanin('fanin_apa_sp'); + +local graph = if use_dnnroi then + g.intern( + innodes=[wcls_input.adc_digits], + outnodes=[], + centernodes=nfsp_pipes+[fanout_apa, retag_dnnroi, retag_sp, fanin_apa_dnnroi, fanin_apa_sp, wcls_output.sp_signals, wcls_output.dnnsp_signals, sink_dnnroi, sink_sp], + edges=[ + g.edge(wcls_input.adc_digits, fanout_apa, 0, 0), + g.edge(fanout_apa, nfsp_pipes[0], 0, 0), + g.edge(fanout_apa, nfsp_pipes[1], 1, 0), + g.edge(nfsp_pipes[0], fanin_apa_dnnroi, 0, 0), + g.edge(nfsp_pipes[1], fanin_apa_dnnroi, 0, 1), + g.edge(fanin_apa_dnnroi, retag_dnnroi, 0, 0), + g.edge(retag_dnnroi, wcls_output.dnnsp_signals, 0, 0), + g.edge(wcls_output.dnnsp_signals, sink_dnnroi, 0, 0), + g.edge(nfsp_pipes[0], fanin_apa_sp, 1, 0), + g.edge(nfsp_pipes[1], fanin_apa_sp, 1, 1), + g.edge(fanin_apa_sp, retag_sp, 0, 0), + g.edge(retag_sp, wcls_output.sp_signals, 0, 0), + g.edge(wcls_output.sp_signals, sink_sp, 0, 0), + ] +) +else +g.pipeline([wcls_input.adc_digits, +fanpipe, +retag_sp, +wcls_output.sp_signals, +sink_sp]); local app = { type: 'TbbFlow', @@ -210,3 +376,4 @@ local app = { // Finally, the configuration sequence g.uses(graph) + [app] + diff --git a/sbndcode/WireCell/cfg/pgrapher/experiment/sbnd/wcls-nf-sp.jsonnet b/sbndcode/WireCell/cfg/pgrapher/experiment/sbnd/wcls-nf-sp.jsonnet index 9eed918fb..90528f446 100644 --- a/sbndcode/WireCell/cfg/pgrapher/experiment/sbnd/wcls-nf-sp.jsonnet +++ b/sbndcode/WireCell/cfg/pgrapher/experiment/sbnd/wcls-nf-sp.jsonnet @@ -23,6 +23,11 @@ local epoch = std.extVar('epoch'); // eg "dynamic", "after", "before", "perfect" local sigoutform = std.extVar('signal_output_form'); // eg "sparse" or "dense" local raw_input_label = std.extVar('raw_input_label'); // eg "daq" +local use_dnnroi = std.extVar('use_dnnroi'); +local nchunks = std.extVar('nchunks'); +local tick_per_slice = std.extVar('tick_per_slice'); +local dnnroi_model_p0 = std.extVar('dnnroi_model_p0'); +local dnnroi_model_p1 = std.extVar('dnnroi_model_p1'); local g = import 'pgraph.jsonnet'; local f = import 'pgrapher/experiment/sbnd/funcs.jsonnet'; @@ -104,6 +109,21 @@ local wcls_output = { chanmaskmaps: [], }, }, nin=1, nout=1, uses=[mega_anode]), + dnnsp_signals: g.pnode({ + type: 'wclsFrameSaver', + name: 'dnnsaver', + data: { + anode: wc.tn(mega_anode), + digitize: false, // true means save as RawDigit, else recob::Wire + frame_tags: ['dnnsp'], + + // this may be needed to convert the decon charge [units:e-] to be consistent with the LArSoft default ?unit? e.g. decon charge * 0.005 --> "charge value" to GaussHitFinder + frame_scale: [0.02, 0.02, 0.02], + nticks: params.daq.nticks, + chanmaskmaps: [], + }, + }, nin=1, nout=1, uses=[mega_anode]), + }; local perfect = import 'pgrapher/experiment/sbnd/chndb-perfect.jsonnet'; @@ -133,14 +153,88 @@ local nf_maker = import 'pgrapher/experiment/sbnd/nf.jsonnet'; local nf_pipes = [nf_maker(params, tools.anodes[n], chndb[n], n, name='nf%d' % n) for n in std.range(0, std.length(tools.anodes) - 1)]; local sp_maker = import 'pgrapher/experiment/sbnd/sp.jsonnet'; +local sp_override = if use_dnnroi then { + sparse: true, + use_roi_debug_mode: true, + save_negative_charge: false, // TODO: no negative charge in gauss, default is false + use_multi_plane_protection: true, + do_not_mp_protect_traditional: false, // TODO: do_not_mp_protect_traditional to make a clear ref, defualt is false + mp_tick_resolution:4, + tight_lf_tag: "", + // loose_lf_tag: "", + cleanup_roi_tag: "", + break_roi_loop1_tag: "", + break_roi_loop2_tag: "", + shrink_roi_tag: "", + extend_roi_tag: "", +} else { + sparse: true, +}; local sp = sp_maker(params, tools, { sparse: sigoutform == 'sparse' }); local sp_pipes = [sp.make_sigproc(a) for a in tools.anodes]; +local dnnroi = import 'dnnroi.jsonnet'; +local ts_p0 = { + type: "TorchService", + name: "dnnroi_p0", + tick_per_slice: tick_per_slice, + data: { + model: dnnroi_model_p0, + device: "cpu", + concurrency: 1, + }, +}; + +local ts_p1 = { + type: "TorchService", + name: "dnnroi_p1", + tick_per_slice: tick_per_slice, + data: { + model: dnnroi_model_p1, + device: "cpu", + concurrency: 1, + }, +}; + +local fanout = function (name, multiplicity=2) + g.pnode({ + type: 'FrameFanout', + name: name, + data: { + multiplicity: multiplicity + }, + }, nin=1, nout=multiplicity); + +local sp_fans = [fanout("sp_fan_%d" % n) for n in std.range(0, std.length(tools.anodes) - 1)]; +local dnnroi_pipes = [ dnnroi(tools.anodes[n], ts_p0, ts_p1, output_scale=1, nchunks=nchunks) for n in std.range(0, std.length(tools.anodes) - 1) ]; + + + local magoutput = 'sbnd-data-check.root'; local magnify = import 'pgrapher/experiment/sbnd/magnify-sinks.jsonnet'; local sinks = magnify(tools, magoutput); -local nfsp_pipes = [ +local nfsp_pipes = if use_dnnroi then +// oports: 0: dnnroi, 1: traditional sp +[ + g.intern( + innodes=[chsel_pipes[n]], + outnodes=[dnnroi_pipes[n],sp_fans[n]], + centernodes=[nf_pipes[n], sp_pipes[n], sp_fans[n]], + edges=[ + g.edge(chsel_pipes[n], nf_pipes[n], 0, 0), + g.edge(nf_pipes[n], sp_pipes[n], 0, 0), + g.edge(sp_pipes[n], sp_fans[n], 0, 0), + g.edge(sp_fans[n], dnnroi_pipes[n], 0, 0), + ], + iports=chsel_pipes[n].iports, + oports=dnnroi_pipes[n].oports+[sp_fans[n].oports[1]], + name='nfsp_pipe_%d' % n, + ) + for n in std.range(0, std.length(tools.anodes) - 1) +] +else +[ g.pipeline([ chsel_pipes[n], //sinks.orig_pipe[n], @@ -159,8 +253,9 @@ local nfsp_pipes = [ local fanpipe = f.fanpipe('FrameFanout', nfsp_pipes, 'FrameFanin', 'sn_mag_nf'); -local retagger = g.pnode({ +local retagger = function(name) g.pnode({ type: 'Retagger', + name: name, data: { // Note: retagger keeps tag_rules an array to be like frame fanin/fanout. tag_rules: [{ @@ -172,16 +267,90 @@ local retagger = g.pnode({ merge: { 'gauss\\d': 'gauss', 'wiener\\d': 'wiener', + 'dnnsp\\d': 'dnnsp', }, }], }, }, nin=1, nout=1); +local retag_dnnroi = retagger("retag_dnnroi"); +local retag_sp = retagger("retag_sp"); -local sink = g.pnode({ type: 'DumpFrames' }, nin=1, nout=0); +local sink = function(name) g.pnode({ type: 'DumpFrames', name: name }, nin=1, nout=0); +local sink_dnnroi = sink("sink_dnnroi"); +local sink_sp = sink("sink_sp"); + +local fanout_apa = g.pnode({ + type: 'FrameFanout', + name: 'fanout_apa', + data: { + multiplicity: std.length(tools.anodes), + "tag_rules": [ + { + "frame": { + ".*": "orig%d" % n + }, + "trace": { } + } + for n in std.range(0, std.length(tools.anodes) - 1) + ] + }}, + nin=1, nout=std.length(tools.anodes)); +local framefanin = function(name) g.pnode({ + type: 'FrameFanin', + name: name, + data: { + multiplicity: std.length(tools.anodes), + "tag_rules": [ + { + "frame": { + ".*": "framefanin" + }, + trace: { + ['dnnsp%d' % n]: ['dnnsp%d' % n], + ['gauss%d' % n]: ['gauss%d' % n], + ['wiener%d' % n]: ['wiener%d' % n], + ['threshold%d' % n]: ['threshold%d' % n], + }, + } + for n in std.range(0, std.length(tools.anodes) - 1) + ], + "tags": [ ] + }, +}, nin=std.length(tools.anodes), nout=1); +local fanin_apa_dnnroi = framefanin('fanin_apa_dnnroi'); +local fanin_apa_sp = framefanin('fanin_apa_sp'); +local fanpipe = f.fanpipe('FrameFanout', nfsp_pipes, 'FrameFanin', 'sn_mag_nf'); // local graph = g.pipeline([wcls_input.adc_digits, rootfile_creation_frames, fanpipe, retagger, wcls_output.sp_signals, sink]); -local graph = g.pipeline([wcls_input.adc_digits, fanpipe, retagger, wcls_output.sp_signals, sink]); +local graph = if use_dnnroi then +g.intern( + innodes=[wcls_input.adc_digits], + outnodes=[], + centernodes=nfsp_pipes+[fanout_apa, retag_dnnroi, retag_sp, fanin_apa_dnnroi, fanin_apa_sp, wcls_output.sp_signals, wcls_output.dnnsp_signals, sink_dnnroi, sink_sp], + edges=[ + g.edge(wcls_input.adc_digits, fanout_apa, 0, 0), + g.edge(fanout_apa, nfsp_pipes[0], 0, 0), + g.edge(fanout_apa, nfsp_pipes[1], 1, 0), + g.edge(nfsp_pipes[0], fanin_apa_dnnroi, 0, 0), + g.edge(nfsp_pipes[1], fanin_apa_dnnroi, 0, 1), + g.edge(fanin_apa_dnnroi, retag_dnnroi, 0, 0), + g.edge(retag_dnnroi, wcls_output.dnnsp_signals, 0, 0), + g.edge(wcls_output.dnnsp_signals, sink_dnnroi, 0, 0), + g.edge(nfsp_pipes[0], fanin_apa_sp, 1, 0), + g.edge(nfsp_pipes[1], fanin_apa_sp, 1, 1), + g.edge(fanin_apa_sp, retag_sp, 0, 0), + g.edge(retag_sp, wcls_output.sp_signals, 0, 0), + g.edge(wcls_output.sp_signals, sink_sp, 0, 0), + ] +) +else +g.pipeline([wcls_input.adc_digits, +fanpipe, // nfsp_pipes +retag_sp, +wcls_output.sp_signals, +sink_sp]); + local app = { type: 'TbbFlow', @@ -191,4 +360,4 @@ local app = { }; // Finally, the configuration sequence -g.uses(graph) + [app] \ No newline at end of file +g.uses(graph) + [app] diff --git a/sbndcode/WireCell/cfg/pgrapher/experiment/sbnd/wcls-sim-drift-depoflux-nf-sp-samples_rec.jsonnet b/sbndcode/WireCell/cfg/pgrapher/experiment/sbnd/wcls-sim-drift-depoflux-nf-sp-samples_rec.jsonnet new file mode 100644 index 000000000..15895db20 --- /dev/null +++ b/sbndcode/WireCell/cfg/pgrapher/experiment/sbnd/wcls-sim-drift-depoflux-nf-sp-samples_rec.jsonnet @@ -0,0 +1,373 @@ +// This is a main entry point for configuring a wire-cell CLI job to +// simulate SBND. It is simplest signal-only simulation with +// one set of nominal field response function. + +local reality = std.extVar('reality'); +local sigoutform = std.extVar('signal_output_form'); // eg "sparse" or "dense" +local savetid = std.extVar("save_track_id"); + +local g = import 'pgraph.jsonnet'; +local f = import 'pgrapher/common/funcs.jsonnet'; +local wc = import 'wirecell.jsonnet'; +local io = import 'pgrapher/common/fileio.jsonnet'; +local tools_maker = import 'pgrapher/common/tools.jsonnet'; + +local data_params = import 'params.jsonnet'; +local simu_params = import 'simparams.jsonnet'; +local params = if reality == 'data' then data_params else simu_params; + +local base = import 'pgrapher/experiment/sbnd/simparams.jsonnet'; +local params = base { + lar: super.lar { // <- super.lar overrides default values + // Longitudinal diffusion constant + DL: std.extVar('DL') * wc.cm2 / wc.s, + // Transverse diffusion constant + DT: std.extVar('DT') * wc.cm2 / wc.s, + // Electron lifetime + lifetime: std.extVar('lifetime') * wc.ms, + // Electron drift speed, assumes a certain applied E-field + drift_speed: std.extVar('driftSpeed') * wc.mm / wc.us, + }, +}; + +local tools = tools_maker(params); +local sim_maker = import 'pgrapher/experiment/sbnd/sim.jsonnet'; +local sim = sim_maker(params, tools); +local nanodes = std.length(tools.anodes); +local anode_iota = std.range(0, nanodes - 1); + +local wcls_maker = import "pgrapher/ui/wcls/nodes.jsonnet"; +local wcls = wcls_maker(params, tools); + +// added Ewerton 2023-03-14 +local wcls_input_sim = { + depos: wcls.input.depos(name="", art_tag=std.extVar('inputTag')), + deposet: g.pnode({ + type: 'wclsSimDepoSetSource', + name: "", + data: { + model: "", + scale: -1, //scale is -1 to correct a sign error in the SimDepoSource converter. + art_tag: std.extVar('inputTag'), //name of upstream art producer of depos "label:instance:processName" + id_is_track: if (savetid == 'true') then false else true, + assn_art_tag: "", + }, + }, nin=0, nout=1), +}; +// Collect all the wc/ls output converters for use below. Note the +// "name" MUST match what is used in theh "outputers" parameter in the +// FHiCL that loads this file. +local mega_anode = { + type: 'MegaAnodePlane', + name: 'meganodes', + data: { + anodes_tn: [wc.tn(anode) for anode in tools.anodes], + }, +}; +local wcls_output_sim = { + // ADC output from simulation + // sim_digits: wcls.output.digits(name="simdigits", tags=["orig"]), + sim_digits: g.pnode({ + type: 'wclsFrameSaver', + name: 'simdigits', + data: { + anode: wc.tn(mega_anode), + digitize: true, // true means save as RawDigit, else recob::Wire + frame_tags: ['daq'], + nticks: params.daq.nticks, + pedestal_mean: 'native', + }, + }, nin=1, nout=1, uses=[mega_anode]), + // The noise filtered "ADC" values. These are truncated for + // art::Event but left as floats for the WCT SP. Note, the tag + // "raw" is somewhat historical as the output is not equivalent to + // "raw data". + nf_digits: wcls.output.digits(name="nfdigits", tags=["raw"]), + // The output of signal processing. Note, there are two signal + // sets each created with its own filter. The "gauss" one is best + // for charge reconstruction, the "wiener" is best for S/N + // separation. Both are used in downstream WC code. + sp_signals: wcls.output.signals(name="spsignals", tags=["gauss", "wiener"]), + // save "threshold" from normal decon for each channel noise + // used in imaging + sp_thresholds: wcls.output.thresholds(name="spthresholds", tags=["wiener"]), +}; + + +local drifter = sim.drifter; + +local setdrifter = g.pnode({ + type: 'DepoSetDrifter', + data: { + drifter: "Drifter" + } + }, nin=1, nout=1, + uses=[drifter]); + +// signal plus noise pipelines +local sn_pipes = sim.splusn_pipelines; + +local hio_sp = [g.pnode({ + type: 'HDF5FrameTap', + name: 'hio_sp%d' % n, + data: { + anode: wc.tn(tools.anodes[n]), + trace_tags: ['loose_lf%d' % n + #, 'tight_lf%d' % n + #, 'cleanup_roi%d' % n + #, 'break_roi_1st%d' % n + #, 'break_roi_2nd%d' % n + #, 'shrink_roi%d' % n + #, 'extend_roi%d' % n + , 'mp3_roi%d' % n + , 'mp2_roi%d' % n + , 'decon_charge%d' % n + , 'gauss%d' % n], + filename: "g4-rec-%d.h5" % n, + chunk: [0, 0], // ncol, nrow + gzip: 2, + tick0: 0, + nticks: 3427, + high_throughput: true, + }, + }, nin=1, nout=1), + for n in std.range(0, std.length(tools.anodes) - 1) + ]; + + +local rng = tools.random; +local wcls_depoflux_writer = g.pnode({ + type: 'wclsDepoFluxWriter', + name: 'postdrift', + data: { + anodes: [wc.tn(anode) for anode in tools.anodes], + field_response: wc.tn(tools.field), + tick: 0.5 * wc.us, + window_start: 0.0 * wc.ms, + window_duration: self.tick * params.daq.nticks, + nsigma: 3.0, + + reference_time: -1700 * wc.us, + + //energy: 1, # equivalent to use_energy = true + simchan_label: 'simpleSC', + sed_label: if (savetid == 'true') then 'ionandscint' else '', + sparse: false, + }, +}, nin=1, nout=1, uses=tools.anodes + [tools.field]); + +local sp_maker = import 'pgrapher/experiment/sbnd/sp.jsonnet'; +local sp = sp_maker(params, tools, { sparse: sigoutform == 'sparse' }); +local sp_pipes = [sp.make_sigproc(a) for a in tools.anodes]; + +local magoutput = 'sbnd-data-check.root'; +local magnify = import 'pgrapher/experiment/sbnd/magnify-sinks.jsonnet'; +local sinks = magnify(tools, magoutput); + +local base = import 'pgrapher/experiment/sbnd/chndb-base.jsonnet'; +//local perfect = import 'pgrapher/experiment/sbnd/chndb-perfect.jsonnet'; + +local chndb = [{ + type: 'OmniChannelNoiseDB', + name: 'ocndbperfect%d' % n, + data: base(params, tools.anodes[n], tools.field, n){dft:wc.tn(tools.dft)}, + //data: perfect(params, tools.anodes[n], tools.field, n){dft:wc.tn(tools.dft)}, + uses: [tools.anodes[n], tools.field, tools.dft], +} for n in anode_iota]; + +local nf_maker = import 'pgrapher/experiment/sbnd/nf.jsonnet'; +local nf_pipes = [nf_maker(params, tools.anodes[n], chndb[n], n, name='nf%d' % n) for n in anode_iota]; + +local multipass1 = [ + g.pipeline([ + sn_pipes[n], + ], + 'multipass%d' % n) + for n in anode_iota +]; + +local multipass2 = [ + g.pipeline([ + sn_pipes[n], + //sinks.orig_pipe[n], + + nf_pipes[n], // NEED to include this pipe for channelmaskmaps + //sinks.raw_pipe[n], + + sp_pipes[n], + hio_sp[n], + + //sinks.decon_pipe[n], + //sinks.threshold_pipe[n], + ], + 'multipass%d' % n) + for n in anode_iota +]; + + +local f_sp = import 'pgrapher/experiment/sbnd/funcs.jsonnet'; + +local outtags = ['orig%d' % n for n in anode_iota]; +local bi_manifold1 = f.fanpipe('DepoSetFanout', multipass1, 'FrameFanin', 'sn_mag_nf', outtags); +local bi_manifold2 = f_sp.fanpipe('DepoSetFanout', multipass2, 'FrameFanin', 'sn_mag_nf_mod2', outtags, "true"); + +local retagger_sim = g.pnode({ + type: 'Retagger', + data: { + // Note: retagger keeps tag_rules an array to be like frame fanin/fanout. + tag_rules: [{ + // Retagger also handles "frame" and "trace" like fanin/fanout + // merge separately all traces like gaussN to gauss. + frame: { + '.*': 'orig', + }, + merge: { + 'orig\\d': 'daq', + }, + }], + }, +}, nin=1, nout=1); + +local sink_sim = sim.frame_sink; + +//===============================NF+SP============================================ + +// Collect all the wc/ls output converters for use below. Note the +// "name" MUST match what is used in theh "outputers" parameter in the +// FHiCL that loads this file. + +local wcls_output_sp = { + // The noise filtered "ADC" values. These are truncated for + // art::Event but left as floats for the WCT SP. Note, the tag + // "raw" is somewhat historical as the output is not equivalent to + // "raw data". + nf_digits: g.pnode({ + type: 'wclsFrameSaver', + name: 'nfsaver', + data: { + anode: wc.tn(mega_anode), + digitize: true, // true means save as RawDigit, else recob::Wire + frame_tags: ['raw'], + }, + }, nin=1, nout=1, uses=[mega_anode]), + + + // The output of signal processing. Note, there are two signal + // sets each created with its own filter. The "gauss" one is best + // for charge reconstruction, the "wiener" is best for S/N + // separation. Both are used in downstream WC code. + sp_signals: g.pnode({ + type: 'wclsFrameSaver', + name: 'spsaver', + data: { + anode: wc.tn(mega_anode), + digitize: false, // true means save as RawDigit, else recob::Wire + frame_tags: ['gauss', 'wiener'], + + // this may be needed to convert the decon charge [units:e-] to be consistent with the LArSoft default + // for SBND, this scale is about ~50. Frame scale needed when using LArSoft producers reading in recob::Wire. + frame_scale: [0.02, 0.02], + nticks: params.daq.nticks, + + // uncomment the below configs to save summaries and cmm + summary_tags: ['wiener'], + summary_operator: {wiener: 'set'}, + summary_scale: [0.02], # summary scale should be the same as frame_scale + chanmaskmaps: ['bad'], + }, + }, nin=1, nout=1, uses=[mega_anode]), + +}; + + +local chsel_pipes = [ + g.pnode({ + type: 'ChannelSelector', + name: 'chsel%d' % n, + data: { + channels: std.range(5638 * n, 5638 * (n + 1) - 1), + }, + }, nin=1, nout=1) + for n in anode_iota +]; + + +local nfsp_pipes = [ + g.pipeline([ + chsel_pipes[n], + //sinks.orig_pipe[n], + + nf_pipes[n], // NEED to include this pipe for channelmaskmaps + //sinks.raw_pipe[n], + + sp_pipes[n], + //sinks.decon_pipe[n], + //sinks.threshold_pipe[n], + // sinks.debug_pipe[n], // use_roi_debug_mode=true in sp.jsonnet + ], + 'nfsp_pipe_%d' % n) + for n in anode_iota +]; + +//local fanpipe = f_sp.fanpipe('FrameFanout', nfsp_pipes, 'FrameFanin', 'sn_mag_nf'); // commented Ewerton 2023-05-24 +local fanpipe = f_sp.fanpipe('FrameFanout', nfsp_pipes, 'FrameFanin', 'sn_mag_nf_mod'); //added Ewerton 2023-05-24 + +local retagger_sp = g.pnode({ + type: 'Retagger', + name: 'sp', //added Ewerton 2023-05-24 + data: { + // Note: retagger keeps tag_rules an array to be like frame fanin/fanout. + tag_rules: [{ + // Retagger also handles "frame" and "trace" like fanin/fanout + // merge separately all traces like gaussN to gauss. + frame: { + '.*': 'retagger', + }, + merge: { + 'gauss\\d': 'gauss', + 'wiener\\d': 'wiener', + }, + }], + }, +}, nin=1, nout=1); + +local sink_sp = g.pnode({ type: 'DumpFrames' }, nin=1, nout=0); + +local graph1 = g.pipeline([ +wcls_input_sim.deposet, //sim +setdrifter, //sim +wcls_depoflux_writer, //sim +bi_manifold1, //sim +retagger_sim, //sim +wcls_output_sim.sim_digits, //sim +fanpipe, //sp +retagger_sp, //sp +wcls_output_sp.sp_signals, //sp +sink_sp //sp +]); + + +local graph2 = g.pipeline([ +wcls_input_sim.deposet, //sim +setdrifter, //sim +wcls_depoflux_writer, //sim +bi_manifold2, //sim +retagger_sp, //sp +wcls_output_sp.sp_signals, //sp +sink_sp //sp +]); + +local save_simdigits = std.extVar('save_simdigits'); + +local graph = if save_simdigits == "true" then graph1 else graph2; + +local app = { + type: 'TbbFlow', + data: { + edges: g.edges(graph), + }, +}; + +// Finally, the configuration sequence +g.uses(graph) + [app] + diff --git a/sbndcode/WireCell/cfg/pgrapher/experiment/sbnd/wcls-sim-drift-depoflux-nf-sp-samples_tru.jsonnet b/sbndcode/WireCell/cfg/pgrapher/experiment/sbnd/wcls-sim-drift-depoflux-nf-sp-samples_tru.jsonnet new file mode 100644 index 000000000..cbe6c9ad6 --- /dev/null +++ b/sbndcode/WireCell/cfg/pgrapher/experiment/sbnd/wcls-sim-drift-depoflux-nf-sp-samples_tru.jsonnet @@ -0,0 +1,474 @@ +// This is a main entry point for configuring a wire-cell CLI job to +// simulate SBND. It is simplest signal-only simulation with +// one set of nominal field response function. + +local epoch = std.extVar('epoch'); // eg "dynamic", "after", "before", "perfect" +local reality = std.extVar('reality'); +// local sigoutform = std.extVar('signal_output_form'); // eg "sparse" or "dense" +local sigoutform = "sparse"; // eg "sparse" or "dense" +local savetid = std.extVar("save_track_id"); + +local g = import 'pgraph.jsonnet'; +local f = import 'pgrapher/common/funcs.jsonnet'; +local wc = import 'wirecell.jsonnet'; +local io = import 'pgrapher/common/fileio.jsonnet'; +local tools_maker = import 'pgrapher/common/tools.jsonnet'; + +local data_params = import 'params.jsonnet'; +local simu_params = import 'simparams.jsonnet'; +local params = if reality == 'data' then data_params else simu_params; + +local base = import 'pgrapher/experiment/sbnd/simparams.jsonnet'; +local params = base { + lar: super.lar { // <- super.lar overrides default values + // Longitudinal diffusion constant + DL: std.extVar('DL') * wc.cm2 / wc.s, + // Transverse diffusion constant + DT: std.extVar('DT') * wc.cm2 / wc.s, + // Electron lifetime + lifetime: std.extVar('lifetime') * wc.ms, + // Electron drift speed, assumes a certain applied E-field + drift_speed: std.extVar('driftSpeed') * wc.mm / wc.us, + }, +}; + +local tools = tools_maker(params); +local sim_maker = import 'pgrapher/experiment/sbnd/sim.jsonnet'; +local sim = sim_maker(params, tools); +local nanodes = std.length(tools.anodes); +local anode_iota = std.range(0, nanodes - 1); + +local wcls_maker = import "pgrapher/ui/wcls/nodes.jsonnet"; +local wcls = wcls_maker(params, tools); + +// added Ewerton 2023-03-14 +local wcls_input_sim = { + depos: wcls.input.depos(name="", art_tag=std.extVar('inputTag')), //commented Ewerton 2023-03-15 + deposet: g.pnode({ + type: 'wclsSimDepoSetSource', + name: "", + data: { + model: "", + scale: -1, //scale is -1 to correct a sign error in the SimDepoSource converter. + // art_tag: std.extVar('inputTag'), //name of upstream art producer of depos "label:instance:processName" + art_tag: std.extVar('inputTag'), //name of upstream art producer of depos "label:instance:processName" + id_is_track: if (savetid == 'true') then false else true, + assn_art_tag: "", + }, + }, nin=0, nout=1), +}; +// Collect all the wc/ls output converters for use below. Note the +// "name" MUST match what is used in theh "outputers" parameter in the +// FHiCL that loads this file. +local mega_anode = { + type: 'MegaAnodePlane', + name: 'meganodes', + data: { + anodes_tn: [wc.tn(anode) for anode in tools.anodes], + }, +}; +local wcls_output_sim = { + // ADC output from simulation + // sim_digits: wcls.output.digits(name="simdigits", tags=["orig"]), + sim_digits: g.pnode({ + type: 'wclsFrameSaver', + name: 'simdigits', + data: { + // anode: wc.tn(tools.anode), + anode: wc.tn(mega_anode), + digitize: true, // true means save as RawDigit, else recob::Wire + frame_tags: ['daq'], + // nticks: params.daq.nticks, + // chanmaskmaps: ['bad'], + pedestal_mean: 'native', + }, + }, nin=1, nout=1, uses=[mega_anode]), + // The noise filtered "ADC" values. These are truncated for + // art::Event but left as floats for the WCT SP. Note, the tag + // "raw" is somewhat historical as the output is not equivalent to + // "raw data". + nf_digits: wcls.output.digits(name="nfdigits", tags=["raw"]), + // The output of signal processing. Note, there are two signal + // sets each created with its own filter. The "gauss" one is best + // for charge reconstruction, the "wiener" is best for S/N + // separation. Both are used in downstream WC code. + sp_signals: wcls.output.signals(name="spsignals", tags=["gauss", "wiener"]), + // save "threshold" from normal decon for each channel noise + // used in imaging + sp_thresholds: wcls.output.thresholds(name="spthresholds", tags=["threshold"]), +}; + + +local drifter = sim.drifter; + +// added Ewerton 2023-03-14 +local setdrifter = g.pnode({ + type: 'DepoDrifter', + data: { + drifter: "Drifter" + } + }, nin=1, nout=1, + uses=[drifter]); + +// signal plus noise pipelines +// local sn_pipes = sim.signal_pipelines; +local sn_pipes = sim.splusn_pipelines; + +local hio_sp = [g.pnode({ + type: 'HDF5FrameTap', + name: 'hio_sp%d' % n, + data: { + anode: wc.tn(tools.anodes[n]), + trace_tags: ['loose_lf%d' % n + , 'tight_lf%d' % n + , 'cleanup_roi%d' % n + , 'break_roi_1st%d' % n + , 'break_roi_2nd%d' % n + , 'shrink_roi%d' % n + , 'extend_roi%d' % n + , 'mp3_roi%d' % n + , 'mp2_roi%d' % n + , 'decon_charge%d' % n + , 'gauss%d' % n], + filename: "g4-rec-%d.h5" % n, + chunk: [0, 0], // ncol, nrow + gzip: 2, + tick0: 0, + nticks: 3427, + high_throughput: true, + }, + }, nin=1, nout=1), + for n in std.range(0, std.length(tools.anodes) - 1) + ]; + + +local rng = tools.random; +local wcls_deposetsimchannel_sink = g.pnode({ + type: 'wclsSimChannelSink', + name: 'postdrift', + data: { + artlabel: 'simpleSC', // where to save in art::Event + anodes_tn: [wc.tn(anode) for anode in tools.anodes], + rng: wc.tn(rng), + tick: 0.5 * wc.us, + start_time: -0.205 * wc.ms, + readout_time: self.tick * 3427, + nsigma: 3.0, + drift_speed: params.lar.drift_speed, + u_to_rp: 100 * wc.mm, // time to collection plane + v_to_rp: 100 * wc.mm, // time to collection plane + y_to_rp: 100 * wc.mm, + u_time_offset: 0.0 * wc.us, + v_time_offset: 0.0 * wc.us, + y_time_offset: 0.0 * wc.us, + g4_ref_time: -1700 * wc.us, + use_energy: true, + }, +}, nin=1, nout=1, uses=tools.anodes); + +local hio_orig = [g.pnode({ + type: 'HDF5FrameTap', + name: 'hio_orig%d' % n, + data: { + anode: wc.tn(tools.anodes[n]), + trace_tags: ['orig%d'%n], + filename: "g4-rec-%d.h5" % n, + chunk: [0, 0], // ncol, nrow + gzip: 2, + high_throughput: true, + }, + }, nin=1, nout=1), + for n in std.range(0, std.length(tools.anodes) - 1) + ]; + +local sp_maker = import 'pgrapher/experiment/sbnd/sp.jsonnet'; +local sp = sp_maker(params, tools, { sparse: sigoutform == 'sparse' }); +local sp_pipes = [sp.make_sigproc(a) for a in tools.anodes]; + +local magoutput = 'sbnd-data-check.root'; +local magnify = import 'pgrapher/experiment/sbnd/magnify-sinks.jsonnet'; +local sinks = magnify(tools, magoutput); + +local perfect = import 'pgrapher/experiment/sbnd/chndb-perfect.jsonnet'; +//local base = import 'chndb-base_sbnd.jsonnet'; + +local chndb = [{ + type: 'OmniChannelNoiseDB', + name: 'ocndbperfect%d' % n, + data: perfect(params, tools.anodes[n], tools.field, n){dft:wc.tn(tools.dft)}, + // data: base(params, tools.anodes[n], tools.field, n){dft:wc.tn(tools.dft)}, + uses: [tools.anodes[n], tools.field, tools.dft], +} for n in anode_iota]; + +local nf_maker = import 'pgrapher/experiment/sbnd/nf.jsonnet'; +local nf_pipes = [nf_maker(params, tools.anodes[n], chndb[n], n, name='nf%d' % n) for n in anode_iota]; + +local depo_fanout = [g.pnode({ + type:'DepoFanout', + name:'depo_fanout-%d'%n, + data:{ + multiplicity:2, + tags: [], + }}, nin=1, nout=2) for n in anode_iota]; + +// local deposplats = [sim.make_ductor('splat%d'%n, tools.anodes[n], tools.pirs[0], 'DepoSplat', 'deposplat%d'%n) for n in anode_iota] ; + +// The approximated sim+sigproc +local splat = function(params, tools, anode, name=null) { + local apaid = anode.data.ident, + local sufix = if std.type(name) == "null" then apaid else name, + local bg = g.pnode({ + type:'DepoBagger', + name: sufix, + data: { + gate: [params.sim.ductor.start_time, + params.sim.ductor.start_time+params.sim.ductor.readout_time], + }, + }, nin=1, nout=1), + local sp = g.pnode({ + type: 'DepoFluxSplat', + name: sufix, + data: { + anode: wc.tn(anode), + field_response: wc.tn(tools.field), // for speed and origin + sparse: true, + tick: 0.5 * wc.us, + window_start: -0.205 * wc.ms, // TODO: there seems to be an offset between sim and rec? + window_duration: self.tick * params.daq.nticks, + reference_time: 0.0, + "smear_long": [ + 3.5, + 3.5, + 3.5 + ], + "smear_tran": [ + 0.4, //can use 0.3 if too strong, otherwise can use 0.4 + 0.4, + 0.14 + ] + }, + }, nin=1, nout=1, uses=[anode, tools.field]), + local rt = g.pnode({ + type: 'Retagger', + name: sufix, + data: { + // Note: retagger keeps tag_rules an array to be like frame fanin/fanout. + tag_rules: [{ + // Retagger also handles "frame" and "trace" like fanin/fanout + // merge separately all traces like gaussN to gauss. + frame: { + ".*": "deposplat%d" % apaid + }, + merge: { + ".*": "deposplat%d" % apaid + }, + }], + }, + }, nin=1, nout=1), + ret: g.pipeline([bg, sp, rt],"%s-%s" % [bg.name, sp.name]), +}.ret; + +local deposplats = [splat(params, tools, tools.anodes[n]) for n in anode_iota] ; + +local hio_truth = [g.pnode({ + type: 'HDF5FrameTap', + name: 'hio_truth%d' % n, + data: { + anode: wc.tn(tools.anodes[n]), + trace_tags: ['deposplat%d'%n], + filename: "g4-tru-%d.h5" % n, + chunk: [0, 0], // ncol, nrow + gzip: 2, + tick0: 0, + nticks: 3427, + high_throughput: true, + }, + }, nin=1, nout=1), + for n in std.range(0, std.length(tools.anodes) - 1) + ]; + + +local multipass1 = [ + g.pipeline([ + sn_pipes[n], + ], + 'multipass%d' % n) + for n in anode_iota +]; + +local multipass2 = [ + g.pipeline([ + deposplats[n], + hio_truth[n], + ], + 'multipass%d' % n) + for n in anode_iota +]; + + +local f_sp = import 'pgrapher/experiment/sbnd/funcs.jsonnet'; +local outtags = ['orig%d' % n for n in anode_iota]; +local bi_manifold1 = f.fanpipe('DepoFanout', multipass1, 'FrameFanin', 'sn_mag_nf', outtags); +// local bi_manifold2 = f_sp.fanpipe('DepoFanout', multipass2, 'FrameFanin', 'sn_mag_nf_mod2', outtags, "true"); +// local bi_manifold2 = f_sp.fanpipe('DepoFanout', multipass2, 'FrameFanin', 'sn_mag_nf_mod2', outtags, "true"); +local bi_manifold2 = f_sp.fanpipe('DepoFanout', multipass2, 'FrameFanin'); + +local retagger_sim = g.pnode({ + type: 'Retagger', + data: { + // Note: retagger keeps tag_rules an array to be like frame fanin/fanout. + tag_rules: [{ + // Retagger also handles "frame" and "trace" like fanin/fanout + // merge separately all traces like gaussN to gauss. + frame: { + '.*': 'orig', + }, + merge: { + 'orig\\d': 'daq', + }, + }], + }, +}, nin=1, nout=1); + +local sink_sim = sim.frame_sink; + +//===============================NF+SP============================================ + +// Collect all the wc/ls output converters for use below. Note the +// "name" MUST match what is used in theh "outputers" parameter in the +// FHiCL that loads this file. + +local wcls_output_sp = { + // The noise filtered "ADC" values. These are truncated for + // art::Event but left as floats for the WCT SP. Note, the tag + // "raw" is somewhat historical as the output is not equivalent to + // "raw data". + nf_digits: g.pnode({ + type: 'wclsFrameSaver', + name: 'nfsaver', + data: { + // anode: wc.tn(tools.anode), + anode: wc.tn(mega_anode), + digitize: true, // true means save as RawDigit, else recob::Wire + frame_tags: ['raw'], + // nticks: params.daq.nticks, + chanmaskmaps: ['bad'], + }, + }, nin=1, nout=1, uses=[mega_anode]), + + + // The output of signal processing. Note, there are two signal + // sets each created with its own filter. The "gauss" one is best + // for charge reconstruction, the "wiener" is best for S/N + // separation. Both are used in downstream WC code. + sp_signals: g.pnode({ + type: 'wclsFrameSaver', + name: 'spsaver', + data: { + // anode: wc.tn(tools.anode), + anode: wc.tn(mega_anode), + digitize: false, // true means save as RawDigit, else recob::Wire + frame_tags: ['gauss', 'wiener'], + + // this may be needed to convert the decon charge [units:e-] to be consistent with the LArSoft default ?unit? e.g. decon charge * 0.005 --> "charge value" to GaussHitFinder + frame_scale: [1.0, 1.0], + nticks: params.daq.nticks, + chanmaskmaps: [], + //nticks: -1, + }, + }, nin=1, nout=1, uses=[mega_anode]), +}; + + +local chsel_pipes = [ + g.pnode({ + type: 'ChannelSelector', + name: 'chsel%d' % n, + data: { + channels: std.range(5632 * n, 5632 * (n + 1) - 1), + //tags: ['orig%d' % n], // traces tag + }, + }, nin=1, nout=1) + for n in anode_iota +]; + + +local nfsp_pipes = [ + g.pipeline([ + chsel_pipes[n], + //sinks.orig_pipe[n], + + //nf_pipes[n], + //sinks.raw_pipe[n], + + sp_pipes[n], + // hio_sp[n], + //sinks.decon_pipe[n], + //sinks.threshold_pipe[n], + // sinks.debug_pipe[n], // use_roi_debug_mode=true in sp.jsonnet + ], + 'nfsp_pipe_%d' % n) + for n in anode_iota +]; + +local f_sp = import 'pgrapher/experiment/sbnd/funcs.jsonnet'; +//local fanpipe = f_sp.fanpipe('FrameFanout', nfsp_pipes, 'FrameFanin', 'sn_mag_nf'); // commented Ewerton 2023-05-24 +local fanpipe = f_sp.fanpipe('FrameFanout', nfsp_pipes, 'FrameFanin', 'sn_mag_nf_mod'); //added Ewerton 2023-05-24 + +local retagger_sp = g.pnode({ + type: 'Retagger', + name: 'sp', //added Ewerton 2023-05-24 + data: { + // Note: retagger keeps tag_rules an array to be like frame fanin/fanout. + tag_rules: [{ + // Retagger also handles "frame" and "trace" like fanin/fanout + // merge separately all traces like gaussN to gauss. + frame: { + '.*': 'retagger', + }, + merge: { + 'gauss\\d': 'gauss', + 'wiener\\d': 'wiener', + }, + }], + }, +}, nin=1, nout=1); + +local sink_sp = g.pnode({ type: 'DumpFrames' }, nin=1, nout=0); + +local graph1 = g.pipeline([ +wcls_input_sim.deposet, //sim +setdrifter, //sim +wcls_deposetsimchannel_sink, //sim +bi_manifold1, //sim // sn_pipes[n] +retagger_sim, //sim +wcls_output_sim.sim_digits, //sim +fanpipe, //sp <- nf_pipes[n], sp_pipes[n] (needs channel selector to do fanout correctly) +retagger_sp, //sp +wcls_output_sp.sp_signals, //sp +sink_sp //sp +]); + + +local graph2 = g.pipeline([ +wcls_input_sim.depos, //sim +drifter, //sim +wcls_deposetsimchannel_sink, //sim +bi_manifold2, //sim // changed Ewerton 2023-05-28 +sink_sp //sp +]); + +// local save_simdigits = std.extVar('save_simdigits'); +local save_simdigits = "false"; + +local graph = if save_simdigits == "true" then graph1 else graph2; + +local app = { + type: 'TbbFlow', + data: { + edges: g.edges(graph), + }, +}; + +// Finally, the configuration sequence +g.uses(graph) + [app] diff --git a/sbndcode/WireCell/cfg/pgrapher/experiment/sbnd/wcls-sim-drift-depoflux-nf-sp.jsonnet b/sbndcode/WireCell/cfg/pgrapher/experiment/sbnd/wcls-sim-drift-depoflux-nf-sp.jsonnet index 3abe24578..88d6e1ce7 100644 --- a/sbndcode/WireCell/cfg/pgrapher/experiment/sbnd/wcls-sim-drift-depoflux-nf-sp.jsonnet +++ b/sbndcode/WireCell/cfg/pgrapher/experiment/sbnd/wcls-sim-drift-depoflux-nf-sp.jsonnet @@ -5,6 +5,11 @@ local reality = std.extVar('reality'); local sigoutform = std.extVar('signal_output_form'); // eg "sparse" or "dense" local savetid = std.extVar("save_track_id"); +local use_dnnroi = std.extVar('use_dnnroi'); +local nchunks = std.extVar('nchunks'); +local tick_per_slice = std.extVar('tick_per_slice'); +local dnnroi_model_p0 = std.extVar('dnnroi_model_p0'); +local dnnroi_model_p1 = std.extVar('dnnroi_model_p1'); local g = import 'pgraph.jsonnet'; local f = import 'pgrapher/common/funcs.jsonnet'; @@ -133,7 +138,24 @@ local wcls_depoflux_writer = g.pnode({ }, nin=1, nout=1, uses=tools.anodes + [tools.field]); local sp_maker = import 'pgrapher/experiment/sbnd/sp.jsonnet'; -local sp = sp_maker(params, tools, { sparse: sigoutform == 'sparse' }); +local sp_override = if use_dnnroi then { + sparse: true, + use_roi_debug_mode: true, + save_negative_charge: false, // TODO: no negative charge in gauss, default is false + use_multi_plane_protection: true, + do_not_mp_protect_traditional: false, // TODO: do_not_mp_protect_traditional to make a clear ref, defualt is false + mp_tick_resolution:4, + tight_lf_tag: "", + // loose_lf_tag: "", // comment to use as input to dnnsp + cleanup_roi_tag: "", + break_roi_loop1_tag: "", + break_roi_loop2_tag: "", + shrink_roi_tag: "", + extend_roi_tag: "", +} else { + sparse: true, +}; +local sp = sp_maker(params, tools, sp_override); local sp_pipes = [sp.make_sigproc(a) for a in tools.anodes]; local magoutput = 'sbnd-data-check.root'; @@ -251,8 +273,59 @@ local wcls_output_sp = { chanmaskmaps: ['bad'], }, }, nin=1, nout=1, uses=[mega_anode]), + dnnsp_signals: g.pnode({ + type: 'wclsFrameSaver', + name: 'dnnsaver', + data: { + anode: wc.tn(mega_anode), + digitize: false, // true means save as RawDigit, else recob::Wire + frame_tags: ['dnnsp'], + + // this may be needed to convert the decon charge [units:e-] to be consistent with the LArSoft default ?unit? e.g. decon charge * 0.005 --> "charge value" to GaussHitFinder + frame_scale: [0.02], + nticks: params.daq.nticks, + chanmaskmaps: [], + }, + }, nin=1, nout=1, uses=[mega_anode]), }; +local dnnroi = import 'dnnroi.jsonnet'; + +local ts_p0 = { + type: "TorchService", + name: "dnnroi_p0", + tick_per_slice: tick_per_slice, + data: { + model: dnnroi_model_p0, + device: "cpu", + concurrency: 1, + }, +}; + +local ts_p1 = { + type: "TorchService", + name: "dnnroi_p1", + tick_per_slice: tick_per_slice, + data: { + model: dnnroi_model_p1, + device: "cpu", + concurrency: 1, + }, +}; + +local dnnroi_pipes = [ dnnroi(tools.anodes[n], ts_p0, ts_p1, output_scale=1, nchunks=nchunks) for n in std.range(0, std.length(tools.anodes) - 1) ]; + +local fanout = function (name, multiplicity=2) + g.pnode({ + type: 'FrameFanout', + name: name, + data: { + multiplicity: multiplicity + }, + }, nin=1, nout=multiplicity); + +local sp_fans = [fanout("sp_fan_%d" % n) for n in std.range(0, std.length(tools.anodes) - 1)]; + local chsel_pipes = [ @@ -267,7 +340,27 @@ local chsel_pipes = [ ]; -local nfsp_pipes = [ +local nfsp_pipes = if use_dnnroi then +// oports: 0: dnnroi, 1: traditional sp +[ + g.intern( + innodes=[chsel_pipes[n]], + outnodes=[dnnroi_pipes[n],sp_fans[n]], + centernodes=[nf_pipes[n], sp_pipes[n], sp_fans[n]], + edges=[ + g.edge(chsel_pipes[n], nf_pipes[n], 0, 0), + g.edge(nf_pipes[n], sp_pipes[n], 0, 0), + g.edge(sp_pipes[n], sp_fans[n], 0, 0), + g.edge(sp_fans[n], dnnroi_pipes[n], 0, 0), + ], + iports=chsel_pipes[n].iports, + oports=dnnroi_pipes[n].oports+[sp_fans[n].oports[1]], + name='nfsp_pipe_%d' % n, + ) + for n in std.range(0, std.length(tools.anodes) - 1) +] +else +[ g.pipeline([ chsel_pipes[n], //sinks.orig_pipe[n], @@ -287,9 +380,9 @@ local nfsp_pipes = [ //local fanpipe = f_sp.fanpipe('FrameFanout', nfsp_pipes, 'FrameFanin', 'sn_mag_nf'); // commented Ewerton 2023-05-24 local fanpipe = f_sp.fanpipe('FrameFanout', nfsp_pipes, 'FrameFanin', 'sn_mag_nf_mod'); //added Ewerton 2023-05-24 -local retagger_sp = g.pnode({ +local retagger = function(name) g.pnode({ type: 'Retagger', - name: 'sp', //added Ewerton 2023-05-24 + name: name, data: { // Note: retagger keeps tag_rules an array to be like frame fanin/fanout. tag_rules: [{ @@ -301,14 +394,63 @@ local retagger_sp = g.pnode({ merge: { 'gauss\\d': 'gauss', 'wiener\\d': 'wiener', + 'dnnsp\\d': 'dnnsp', }, }], }, }, nin=1, nout=1); -local sink_sp = g.pnode({ type: 'DumpFrames' }, nin=1, nout=0); +local retagger_dnnroi = retagger("retagger_dnnroi"); +local retagger_sp = retagger("retagger_sp"); + +local sink = function(name) g.pnode({ type: 'DumpFrames', name: name }, nin=1, nout=0); +local sink_dnnroi = sink("sink_dnnroi"); +local sink_sp = sink("sink_sp"); + +local fanout_apa = g.pnode({ + type: 'FrameFanout', + name: 'fanout_apa', + data: { + multiplicity: std.length(tools.anodes), + "tag_rules": [ + { + "frame": { + ".*": "orig%d" % n + }, + "trace": { } + } + for n in std.range(0, std.length(tools.anodes) - 1) + ] + }}, + nin=1, nout=std.length(tools.anodes)); + +local framefanin = function(name) g.pnode({ + type: 'FrameFanin', + name: name, + data: { + multiplicity: std.length(tools.anodes), + + "tag_rules": [ + { + "frame": { + ".*": "framefanin" + }, + trace: { + ['dnnsp%d' % n]: ['dnnsp%d' % n], + ['gauss%d' % n]: ['gauss%d' % n], + ['wiener%d' % n]: ['wiener%d' % n], + ['threshold%d' % n]: ['threshold%d' % n], + }, + } + for n in std.range(0, std.length(tools.anodes) - 1) + ], + "tags": [ ] + }, +}, nin=std.length(tools.anodes), nout=1); +local fanin_apa_dnnroi = framefanin('fanin_apa_dnnroi'); +local fanin_apa_sp = framefanin('fanin_apa_sp'); -local graph1 = g.pipeline([ +local graph1_trad = g.pipeline([ wcls_input_sim.deposet, //sim setdrifter, //sim wcls_depoflux_writer, //sim @@ -322,7 +464,7 @@ sink_sp //sp ]); -local graph2 = g.pipeline([ +local graph2_trad = g.pipeline([ wcls_input_sim.deposet, //sim setdrifter, //sim wcls_depoflux_writer, //sim @@ -332,9 +474,45 @@ wcls_output_sp.sp_signals, //sp sink_sp //sp ]); +local graph_sim_dnn = g.pipeline([ +wcls_input_sim.deposet, //sim +setdrifter, //sim +wcls_depoflux_writer, //sim +bi_manifold1, //sim +]); + + +local graph_sp_dnn = +g.intern( + innodes=[retagger_sim], + outnodes=[], + centernodes=nfsp_pipes+[wcls_output_sim.sim_digits, fanout_apa, retagger_dnnroi, retagger_sp, fanin_apa_dnnroi, fanin_apa_sp, wcls_output_sp.sp_signals, wcls_output_sp.dnnsp_signals, sink_dnnroi, sink_sp], + edges=[ + g.edge(retagger_sim, wcls_output_sim.sim_digits, 0, 0), + g.edge(wcls_output_sim.sim_digits, fanout_apa, 0, 0), + g.edge(fanout_apa, nfsp_pipes[0], 0, 0), + g.edge(fanout_apa, nfsp_pipes[1], 1, 0), + g.edge(nfsp_pipes[0], fanin_apa_dnnroi, 0, 0), + g.edge(nfsp_pipes[1], fanin_apa_dnnroi, 0, 1), + g.edge(fanin_apa_dnnroi, retagger_dnnroi, 0, 0), + g.edge(retagger_dnnroi, wcls_output_sp.dnnsp_signals, 0, 0), + g.edge(wcls_output_sp.dnnsp_signals, sink_dnnroi, 0, 0), + g.edge(nfsp_pipes[0], fanin_apa_sp, 1, 0), + g.edge(nfsp_pipes[1], fanin_apa_sp, 1, 1), + g.edge(fanin_apa_sp, retagger_sp, 0, 0), + g.edge(retagger_sp, wcls_output_sp.sp_signals, 0, 0), + g.edge(wcls_output_sp.sp_signals, sink_sp, 0, 0), + ] +); + +local graph_dnn = g.pipeline([ +graph_sim_dnn, +graph_sp_dnn, +]); + local save_simdigits = std.extVar('save_simdigits'); -local graph = if save_simdigits == "true" then graph1 else graph2; +local graph = if use_dnnroi then graph_dnn else if save_simdigits == "true" then graph1_trad else graph2_trad; local app = { type: 'TbbFlow', @@ -345,4 +523,4 @@ local app = { // Finally, the configuration sequence g.uses(graph) + [app] - + diff --git a/sbndcode/WireCell/wcsimsp_sbnd.fcl b/sbndcode/WireCell/wcsimsp_sbnd.fcl index b72f73d44..03d5f429a 100644 --- a/sbndcode/WireCell/wcsimsp_sbnd.fcl +++ b/sbndcode/WireCell/wcsimsp_sbnd.fcl @@ -17,7 +17,7 @@ sbnd_wcls: { configs: [] ## Libraries in which to look for WCT components - plugins: ["WireCellGen", "WireCellSigProc", "WireCellRoot", "WireCellPgraph", "WireCellLarsoft", "WireCellTbb"] + plugins: ["WireCellGen", "WireCellSigProc", "WireCellRoot", "WireCellPgraph", "WireCellLarsoft", "WireCellTbb", "WireCellPytorch"] inputers: [] outputers: [] @@ -58,6 +58,9 @@ sbnd_wcls_simsp.wcls_main.params: { save_simdigits: "false" save_track_id: "true" + + dnnroi_model_p0: "DNN_ROI/plane0.ts" + dnnroi_model_p1: "DNN_ROI/plane1.ts" } sbnd_wcls_simsp.wcls_main.structs: { @@ -71,6 +74,10 @@ sbnd_wcls_simsp.wcls_main.structs: { driftSpeed: 1.563 ## simulated front porch size [us] tick0_time: @local::sbnd_detectorclocks.TriggerOffsetTPC + nticks: 3427 + use_dnnroi: false + nchunks: 2 + tick_per_slice: 4 } # ------------------------------------------------------------------------------------ # @@ -83,7 +90,9 @@ sbnd_wcls_sp.wcls_main.inputers: ["wclsRawFrameSource" # and you must have geo::Geometry service in your environment. # ,"wclsMultiChannelNoiseDB" ] -sbnd_wcls_sp.wcls_main.outputers: ["wclsFrameSaver:spsaver"] +sbnd_wcls_sp.wcls_main.outputers: ["wclsFrameSaver:spsaver" + # "wclsFrameSaver:dnnsaver" + ] sbnd_wcls_sp.wcls_main.params : { # This locates the input raw::RawDigit collection in the art::Event raw_input_label: "daq" @@ -98,5 +107,103 @@ sbnd_wcls_sp.wcls_main.params : { # Save output signal waveforms (recob::Wire) in "sparse" or "dense" form signal_output_form: "sparse" + + dnnroi_model_p0: "DNN_ROI/plane0.ts" + dnnroi_model_p1: "DNN_ROI/plane1.ts" + } + +sbnd_wcls_sp.wcls_main.structs: { + nticks: 3427 + use_dnnroi: false + nchunks: 2 + tick_per_slice: 4 + } + + +# ------------------------------------------------------------------------------------ # + +## Configuration for generating training samples -- input images +sbnd_wcls_samples_rec: @local::sbnd_wcls +sbnd_wcls_samples_rec.wcls_main.plugins: ["WireCellGen", "WireCellSigProc", "WireCellRoot", "WireCellPgraph", "WireCellLarsoft", "WireCellTbb", "WireCellPytorch", "WireCellHio"] +sbnd_wcls_samples_rec.wcls_main.configs: ["pgrapher/experiment/sbnd/wcls-sim-drift-depoflux-nf-sp-samples_rec.jsonnet"] +sbnd_wcls_samples_rec.wcls_main.inputers: ["wclsSimDepoSetSource:"] +sbnd_wcls_samples_rec.wcls_main.outputers:[ "wclsDepoFluxWriter:postdrift", "wclsFrameSaver:simdigits"] +sbnd_wcls_samples_rec.wcls_main.params: { + ## This locates the input SimEnergyDeposits in the art::Event + inputTag: "ionandscint:" + + ## Set "data" vs. "sim". The epoch below probably should follow suit. + reality: "sim" + + ## if epoch is "dynamic" you MUST add + ## "wclsMultiChannelNoiseDB" to "inputers" and must NOT + ## add it if not "dynamic" + epoch: "perfect" + + ## Save output signal waveforms (recob::Wire) in "sparse" or "dense" form + signal_output_form: "sparse" + + ## If save_simdigits="false", comment line with "wclsFrameSaver:simdigits" in outputers + ## If save_simdigits="true", uncomment line with "wclsFrameSaver:simdigits" in outputers + ## this is whether or not to save raw digits!!! + save_simdigits: "true" + + save_track_id: "true" + } + +sbnd_wcls_samples_rec.wcls_main.structs: { + ## Longitudinal diffusion constant [cm2/s] + DL: 4.0 + ## Transverse diffusion constant [cm2/s] + DT: 8.8 + ## Electron lifetime [ms] + lifetime: 100.0 + ## Electron drift speed, assumes 0.5 kV/cm and 88.4 K. Units: mm/us + driftSpeed: 1.563 + } + + +# ------------------------------------------------------------------------------------ # + +## Configuration for generating training samples -- target images +sbnd_wcls_samples_tru: @local::sbnd_wcls +sbnd_wcls_samples_tru.wcls_main.plugins: ["WireCellGen", "WireCellSigProc", "WireCellRoot", "WireCellPgraph", "WireCellLarsoft", "WireCellTbb", "WireCellPytorch", "WireCellHio"] +sbnd_wcls_samples_tru.wcls_main.configs: ["pgrapher/experiment/sbnd/wcls-sim-drift-depoflux-nf-sp-samples_tru.jsonnet"] +sbnd_wcls_samples_tru.wcls_main.inputers: ["wclsSimDepoSource:"] +sbnd_wcls_samples_tru.wcls_main.outputers:["wclsDepoFluxWriter:postdrift", "wclsFrameSaver:simdigits"] +sbnd_wcls_samples_tru.wcls_main.params: { + ## This locates the input SimEnergyDeposits in the art::Event + inputTag: "ionandscint:" + + ## Set "data" vs. "sim". The epoch below probably should follow suit. + reality: "sim" + + ## if epoch is "dynamic" you MUST add + ## "wclsMultiChannelNoiseDB" to "inputers" and must NOT + ## add it if not "dynamic" + epoch: "perfect" + + ## Save output signal waveforms (recob::Wire) in "sparse" or "dense" form + signal_output_form: "sparse" + + ## If save_simdigits="false", comment line with "wclsFrameSaver:simdigits" in outputers + ## If save_simdigits="true", uncomment line with "wclsFrameSaver:simdigits" in outputers + ## this is whether or not to save raw digits!!! + save_simdigits: "true" + + save_track_id: "true" + } + +sbnd_wcls_samples_tru.wcls_main.structs: { + ## Longitudinal diffusion constant [cm2/s] + DL: 4.0 + ## Transverse diffusion constant [cm2/s] + DT: 8.8 + ## Electron lifetime [ms] + lifetime: 100.0 + ## Electron drift speed, assumes 0.5 kV/cm and 88.4 K. Units: mm/us + driftSpeed: 1.563 + } + END_PROLOG diff --git a/sbndcode/WireCell/wcsp_data_sbnd.fcl b/sbndcode/WireCell/wcsp_data_sbnd.fcl index fad2236c7..cf8b0d6c9 100644 --- a/sbndcode/WireCell/wcsp_data_sbnd.fcl +++ b/sbndcode/WireCell/wcsp_data_sbnd.fcl @@ -16,7 +16,7 @@ sbnd_wcls: { configs: [] ## Libraries in which to look for WCT components - plugins: ["WireCellGen", "WireCellSigProc", "WireCellRoot", "WireCellPgraph", "WireCellLarsoft", "WireCellTbb"] + plugins: ["WireCellGen", "WireCellSigProc", "WireCellRoot", "WireCellPgraph", "WireCellLarsoft", "WireCellTbb", "WireCellPytorch"] inputers: [] outputers: [] @@ -53,11 +53,17 @@ sbnd_wcls_sp_data.wcls_main.params : { # whether or not to use calibrated, parametrized electronics response # MUST be a string! use_paramresp: "true" - } + + dnnroi_model_p0: "DNN_ROI/plane0.ts" + dnnroi_model_p1: "DNN_ROI/plane1.ts" + } sbnd_wcls_sp_data.wcls_main.structs: { # Set the waveform sample length, eg, 6000, 15000, "auto" nticks: 3427 + use_dnnroi: false + nchunks: 2 + tick_per_slice: 4 } END_PROLOG diff --git a/sbndcode/WireCell/wirecell_sim_rec_sbnd.fcl b/sbndcode/WireCell/wirecell_sim_rec_sbnd.fcl new file mode 100644 index 000000000..b6dcb7423 --- /dev/null +++ b/sbndcode/WireCell/wirecell_sim_rec_sbnd.fcl @@ -0,0 +1,90 @@ +# +# File: wirecell_sim_rec.fcl +# Purpose: generates training input images for DNN ROI network +# +# This configuration runs Wire-Cell TPC Simulation and Signal Processing +# +# Input: +# - std::vector with label `ionandscint` +# +# Output: +# - h5 file with images of intermediate ROIs + + +# +# services +# + +#include "simulationservices_sbnd.fcl" +#include "messages_sbnd.fcl" + +# +# modules +# + +#include "detsimmodules_sbnd.fcl" +#include "rootoutput_sbnd.fcl" +#include "wcsimsp_sbnd.fcl" + +process_name: DNNsamplerec + +services: +{ + TFileService: { fileName: @local::sbnd_tfileoutput.fileName } + @table::sbnd_g4_services + FileCatalogMetadata: @local::sbnd_file_catalog_mc + message: { debugModules: ["*"] destinations: { debugmsg:{type: "cout" threshold: "INFO"} } } #added Ewerton 2023-06-30 + TimeTracker: { printSummary: true } +} + + +source: +{ + module_type: RootInput +} + +outputs: +{ + out1: + { + @table::sbnd_rootoutput # inherit shared settings + dataTier: "reconstructed" + #compressionLevel: 1 # TODO better to use no compression here and Huffman encoding + } +} + +# Define and configure some modules to do work on each event. +# First modules are defined; they are scheduled later. +# Modules are grouped by type. +physics: +{ + + producers: + { + # 2D TPC Simulation & Signal Processing + simtpc2d : @local::sbnd_wcls_samples_rec + } + + #define the producer and filter modules for this path, order matters, + simulate: [ simtpc2d ] #added wctsp Ewerton 2023-05-16 + + trigger_paths : [ simulate ] + + #define the output stream, there could be more than one if using filters + stream1: [ out1 ] + + #ie analyzers and output streams. these all run simultaneously + end_paths: [ stream1 ] +} + +# block to define where the output goes. if you defined a filter in the physics +# block and put it in the trigger_paths then you need to put a SelectEvents: {SelectEvents: [XXX]} +# entry in the output stream you want those to go to, where XXX is the label of the filter module(s) + +outputs.out1.outputCommands: [ + "keep *_*_*_*" , + # "drop *_ionandscint_*_*" + "keep *_simdigits_*_*" + ] + + diff --git a/sbndcode/WireCell/wirecell_sim_tru_sbnd.fcl b/sbndcode/WireCell/wirecell_sim_tru_sbnd.fcl new file mode 100644 index 000000000..0cbae0656 --- /dev/null +++ b/sbndcode/WireCell/wirecell_sim_tru_sbnd.fcl @@ -0,0 +1,91 @@ +# File: wirecell_sim_tru.fcl +# Purpose: generates training target images for DNN ROI network +# +# This configuration runs Wire-Cell TPC Simulation and Signal Processing +# +# Input: +# - std::vector with label `ionandscint` +# +# Output: +# - h5 file with target image from simulation + +# +# services +# + +#include "simulationservices_sbnd.fcl" +#include "messages_sbnd.fcl" + +# +# modules +# + +#include "detsimmodules_sbnd.fcl" +#include "rootoutput_sbnd.fcl" +#include "wcsimsp_sbnd.fcl" + + +process_name: DNNsampletru + +services: +{ + TFileService: { fileName: @local::sbnd_tfileoutput.fileName } + @table::sbnd_g4_services + FileCatalogMetadata: @local::sbnd_file_catalog_mc + message: { debugModules: ["*"] destinations: { debugmsg:{type: "cout" threshold: "INFO"} } } #added Ewerton 2023-06-30 + TimeTracker: { printSummary: true } + +} + + +source: +{ + module_type: RootInput +} + +outputs: +{ + out1: + { + @table::sbnd_rootoutput # inherit shared settings + dataTier: "reconstructed" + #compressionLevel: 1 # TODO better to use no compression here and Huffman encoding + } +} + +# Define and configure some modules to do work on each event. +# First modules are defined; they are scheduled later. +# Modules are grouped by type. +physics: +{ + + producers: + { + # 2D TPC Simulation & Signal Processing + simtpc2d : @local::sbnd_wcls_samples_tru + } + + #define the producer and filter modules for this path, order matters, + simulate: [ simtpc2d ] #added wctsp Ewerton 2023-05-16 + + trigger_paths : [ simulate ] + + #define the output stream, there could be more than one if using filters + stream1: [ out1 ] + + #ie analyzers and output streams. these all run simultaneously + end_paths: [ stream1 ] +} + +# block to define where the output goes. if you defined a filter in the physics +# block and put it in the trigger_paths then you need to put a SelectEvents: {SelectEvents: [XXX]} +# entry in the output stream you want those to go to, where XXX is the label of the filter module(s) + +outputs.out1.outputCommands: [ + "keep *_*_*_*" , + # "drop *_ionandscint_*_*" + "keep *_simdigits_*_*" + ] + + + diff --git a/ups/product_deps b/ups/product_deps index a48c4e987..a54174eb3 100644 --- a/ups/product_deps +++ b/ups/product_deps @@ -255,8 +255,8 @@ wpdir product_dir wire-cell-cfg product version qual flags sbncode v10_06_00_01 - cetmodules v3_24_01 - only_for_build -sbnd_data v01_31_00 - -sbndutil v10_06_00 - optional +sbnd_data v01_32_00 - +sbndutil v10_06_01 - optional fhiclpy v4_03_05 - end_product_list ####################################