Copy of PR #590 and PR 591 for production branch

fcl/reco/Definitions/stage0_icarus_defs.fcl

@@ -15,6 +15,7 @@
 #include "crt_decoderdefs_icarus.fcl"
 #include "crthitproducer.fcl"
 #include "wcls-decode-to-sig-base.fcl"
+#include "icarus_FilterDataIntegrity.fcl"
 #include "icarus_FilterCRTPMTMatching.fcl"
 
 BEGIN_PROLOG
@@ -154,6 +155,8 @@ icarus_stage0_filters:
                              TriggerType:        "Unknown"
                           }
 
+   filterdataintegrity: @local::icarus_FilterDataIntegrity
+
    crtpmtmatchingfilter: @local::icarus_FilterCRTPMTMatching
 }
 
@@ -261,20 +264,23 @@ icarus_stage0_CRT:                  [
                                     ]
 
 icarus_stage0_data:                 [
+                                      filterdataintegrity,
                                       @sequence::icarus_stage0_PMT, 
                                       @sequence::icarus_stage0_CRT,
                                       daqTPCROI, 
                                       @sequence::icarus_stage0_multiTPC
                                     ]
 
 icarus_stage0_2d_data:              [
+                                      filterdataintegrity,
                                       @sequence::icarus_stage0_PMT, 
                                       @sequence::icarus_stage0_CRT,
                                       daqTPCROI, 
                                       @sequence::icarus_stage0_2d_multiTPC
                                     ]
 
 icarus_stage0_data_crtpmtfilter:    [
+                                      filterdataintegrity,
                                       @sequence::icarus_stage0_PMT, 
                                       @sequence::icarus_stage0_CRT,
                                       crtpmtmatchingfilter,
@@ -283,6 +289,7 @@ icarus_stage0_data_crtpmtfilter:    [
                                     ]
 
 icarus_stage0_2d_data_crtpmtfilter: [
+                                      filterdataintegrity,
                                       @sequence::icarus_stage0_PMT, 
                                       @sequence::icarus_stage0_CRT,
                                       crtpmtmatchingfilter,

fcl/reco/Definitions/stage0_icarus_defs_run1.fcl

@@ -15,6 +15,7 @@
 #include "crt_decoderdefs_icarus.fcl"
 #include "crthitproducer.fcl"
 #include "wcls-decode-to-sig-base.fcl"
+#include "icarus_FilterDataIntegrity.fcl"
 #include "icarus_FilterCRTPMTMatching.fcl"
 
 BEGIN_PROLOG
@@ -160,6 +161,9 @@ icarus_stage0_filters:
                              TriggerDataLabel:   "daqTrigger"
                              TriggerType:        "Unknown"
                           }
+
+   filterdataintegrity: @local::icarus_FilterDataIntegrity
+
    crtpmtmatchingfilter: @local::icarus_FilterCRTPMTMatching
 }
 
@@ -265,13 +269,15 @@ icarus_stage0_CRT:                 [
                                    ]
 
 icarus_stage0_data:                [
+                                     filterdataintegrity,
                                      @sequence::icarus_stage0_PMT, 
                                      @sequence::icarus_stage0_CRT,
                                      daqTPCROI, 
                                      @sequence::icarus_stage0_multiTPC
                                    ]
 
 icarus_stage0_data_crtpmtfilter:   [
+                                     filterdataintegrity,
                                      @sequence::icarus_stage0_PMT, 
                                      @sequence::icarus_stage0_CRT,
                                      crtpmtmatchingfilter,
@@ -280,13 +286,15 @@ icarus_stage0_data_crtpmtfilter:   [
                                   ]
 
 icarus_stage0_2d_data:             [
+                                     filterdataintegrity,
                                      @sequence::icarus_stage0_PMT, 
                                      @sequence::icarus_stage0_CRT,
                                      daqTPCROI, 
                                      @sequence::icarus_stage0_2d_multiTPC
                                    ]
 
 icarus_stage0_2d_data_crtpmtfilter: [
+                                      filterdataintegrity,
                                       @sequence::icarus_stage0_PMT, 
                                       @sequence::icarus_stage0_CRT,
                                       crtpmtmatchingfilter,

fcl/reco/Definitions/stage0_icarus_driver_common.fcl

@@ -65,6 +65,7 @@ outputs:
    fileName: "%ifb_%tc-%p.root"
    fileProperties: {maxInputFiles: 1}
    checkFileName: false
+   SelectEvents: [path]
  }
 }
 

fcl/reco/Definitions/stage0_icarus_driver_common_run1.fcl

@@ -65,6 +65,7 @@ outputs:
    fileName: "%ifb_%tc-%p.root"
    fileProperties: {maxInputFiles: 1}
    checkFileName: false
+   SelectEvents: [path]
  }
 }
 

fcl/reco/Definitions/stage1_icarus_defs.fcl

@@ -147,18 +147,13 @@ icarus_analysis_larcv_modules:     [  @sequence::icarus_analysis_modules
                                    ]
 
 # Set up filtering of cluster3D hits by cryostat
-icarus_filter_cluster3DCryoW:      [ cluster3DCryoW,
+# Changed slightly to faciliate larcv processing
+icarus_filter_cluster3D:           [ cluster3DCryoE,
+                                     cluster3DCryoW,
+                                     TPCHitFilterCryoE,
                                      TPCHitFilterCryoW
                                    ]
 
-icarus_filter_cluster3DCryoE:      [ cluster3DCryoE,
-                                     TPCHitFilterCryoE
-                                   ]
-
-icarus_filter_cluster3D:           [ @sequence::icarus_filter_cluster3DCryoW,
-                                     @sequence::icarus_filter_cluster3DCryoE
-                                   ]
-
 icarus_reco_cluster3DCryoW:        [ cluster3DCryoW ]
 
 icarus_reco_cluster3DCryoE:        [ cluster3DCryoE ]

fcl/reco/Definitions/stage1_icarus_driver_common.fcl

@@ -47,7 +47,7 @@ physics:
  #reco sequence and trigger_paths to be defined elsewhere
 
  stream1:    [ out1 ]
- end_paths:  [stream1]
+ end_paths:  [ stream1 ]
 
 }
 

fcl/reco/Stage0/Run1/stage0_run1_icarus.fcl

@@ -15,7 +15,7 @@ physics.trigger_paths: [ path ]
 physics.end_paths:     [ outana, streamROOT ]
 
 # Drop the artdaq format files on output
-outputs.outCommon.outputCommands:         ["keep *_*_*_*", "drop *_*_*_DAQ*", "drop *_ophituncorrected_*_*", "drop *_daqTPCROI_*_*", "drop *_decon1droi_*_*", "drop *_decon1DroiTPC*_*_*" ]
+outputs.rootOutput.outputCommands: ["keep *_*_*_*", "drop *_*_*_DAQ*", "drop *_ophituncorrected_*_*", "drop *_daqTPCROI_*_*", "drop *_decon1droi_*_*", "drop *_decon1DroiTPC*_*_*" ]outputs.rootOutput.SelectEvents:   [path]
 
 ## Modify the event selection for the purity analyzers
 physics.analyzers.purityinfoana0.SelectEvents:    [ path ]

fcl/reco/Stage0/Run1/stage0_run1_icarus_lite.fcl

@@ -14,5 +14,5 @@
 #include "stage0_run1_icarus.fcl"
 
 # Drop truth data products.
-outputs.outCommon.outputCommands: [ @sequence::outputs.outCommon.outputCommands,
-                                    @sequence::reco_drops ]
+outputs.rootOutput.outputCommands: [ @sequence::outputs.rootOutput.outputCommands,
+                                     @sequence::reco_drops ]

fcl/reco/Stage0/Run1/stage0_run1_icarus_mc.fcl

@@ -29,8 +29,8 @@ physics.trigger_paths: [ path ]
 physics.end_paths:     [ outana, streamROOT ]
 
 # Drop the artdaq format files on output
-#outputs.outCommon.outputCommands: ["keep *_*_*_*", "drop *_MCDecodeTPCROI_*_*", "drop *_decon1droi_*_*","drop raw::RawDigits_*_*_*" ]
-outputs.outCommon.outputCommands: ["keep *_*_*_*", "drop *_decon1droi_*_*" ]
+#outputs.rootOutput.outputCommands: ["keep *_*_*_*", "drop *_MCDecodeTPCROI_*_*", "drop *_decon1droi_*_*","drop raw::RawDigits_*_*_*" ]
+outputs.rootOutput.outputCommands: ["keep *_*_*_*", "drop *_decon1droi_*_*" ]
 
 # Set the expected input for ophit
 physics.producers.ophit:     @local::icarus_ophit_MC

fcl/reco/Stage0/Run1/stage0_run1_raw_icarus.fcl

@@ -5,4 +5,4 @@
 #include "stage0_run1_icarus.fcl"
 
 # Drop the artdaq format files on output
-outputs.outCommon.outputCommands:  ["keep *_*_*_*", "drop *_*_*_DAQ*", "drop *_decon1droi_*_*", "drop *_decon1DroiTPC*_*_*" ]
+outputs.rootOutput.outputCommands:  ["keep *_*_*_*", "drop *_*_*_DAQ*", "drop *_decon1droi_*_*", "drop *_decon1DroiTPC*_*_*" ]

fcl/reco/Stage0/Run1/stage0_run1_raw_icarus_lite.fcl

@@ -15,5 +15,5 @@
 
 # Drop truth data products.
 
-outputs.outCommon.outputCommands: [ @sequence::outputs.outCommon.outputCommands,
-                                    @sequence::reco_drops ]
+outputs.rootOutput.outputCommands: [ @sequence::outputs.rootOutput.outputCommands,
+                                     @sequence::reco_drops ]

fcl/reco/Stage0/Run1/stage0_run1_rawpmt_icarus.fcl

@@ -18,7 +18,7 @@ physics.trigger_paths: [ path ]
 physics.end_paths:     [ outana, streamROOT ]
 
 # Drop the artdaq format files on output
-outputs.outCommon.outputCommands:         ["keep *_*_*_*", "drop *_*_*_DAQ*", "drop *_daqTPCROI_*_*", "drop *_decon1droi_*_*", "drop *_decon1DroiTPC*_*_*" ]
+outputs.rootOutput.outputCommands:         ["keep *_*_*_*", "drop *_*_*_DAQ*", "drop *_daqTPCROI_*_*", "drop *_decon1droi_*_*", "drop *_decon1DroiTPC*_*_*" ]
 
 ## Modify the event selection for the purity analyzers
 physics.analyzers.purityinfoana0.SelectEvents:    [ path ]

fcl/reco/Stage0/Run1/stage0_run1_wc_icarus.fcl

@@ -16,7 +16,7 @@ physics.trigger_paths: [ path ]
 physics.end_paths:     [ outana, streamCommon ]
 
 # Drop the artdaq format files on output
-outputs.outCommon.outputCommands:         ["keep *_*_*_*", "drop *_*_*_DAQ*", "drop *_daqTPCROI_*_*", "drop *_decon1droi_*_*", "drop *_decon1DroiTPC*_*_*", "drop *_decon2Droi*_*_*", "drop *_roifindr_*_*" ]
+outputs.rootOutput.outputCommands:         ["keep *_*_*_*", "drop *_*_*_DAQ*", "drop *_daqTPCROI_*_*", "drop *_decon1droi_*_*", "drop *_decon1DroiTPC*_*_*", "drop *_decon2Droi*_*_*", "drop *_roifindr_*_*" ]
 
 # Override the hit finder input 
 physics.producers.gaushitTPCWW.CalDataModuleLabel: "roifinder2d:PHYSCRATEDATATPCWW"

fcl/reco/Stage0/Run1/stage0_run1_wc_icarus_lite.fcl

@@ -15,5 +15,5 @@
 
 # Drop truth data products.
 
-outputs.outCommon.outputCommands: [ @sequence::outputs.outCommon.outputCommands,
-                                    @sequence::reco_drops ]
+outputs.rootOutput.outputCommands: [ @sequence::outputs.rootOutput.outputCommands,
+                                     @sequence::reco_drops ]

icaruscode/Analysis/trigger/magic_raw_to_triggeremu_icarus_Run1_data.fcl

@@ -145,15 +145,15 @@ physics.filters: {
 
 physics.pathE: [
     @sequence::physics.pathE
-  , @sequence::icarus_filter_cluster3DCryoE
+  , @sequence::icarus_reco_cluster3DCryoE
   , @sequence::icarus_reco_pandoraGausCryoE
   , fmatchCryoE
   , @sequence::icarus_crttrack
   , caloskimCalorimetryCryoE
   ]
 physics.pathW: [
     @sequence::physics.pathW
-  , @sequence::icarus_filter_cluster3DCryoW
+  , @sequence::icarus_reco_cluster3DCryoW
   , @sequence::icarus_reco_pandoraGausCryoW
   , fmatchCryoW
   , @sequence::icarus_crttrack

icaruscode/Filters/CMakeLists.txt

@@ -2,6 +2,8 @@ art_make_library()
 set(  MODULE_LIBRARIES
     sbnobj::Common_Trigger
     icarusalg::Utilities
+    sbndaq_artdaq_core::sbndaq-artdaq-core_Overlays_ICARUS
+    artdaq_core::artdaq-core_Utilities
     larcorealg::Geometry
     larcore::Geometry_Geometry_service
     larsim::Simulation
@@ -38,6 +40,7 @@ set(  MODULE_LIBRARIES
   )
 cet_build_plugin(FilterNeutrinosActiveVolume art::module LIBRARIES ${MODULE_LIBRARIES})
 cet_build_plugin(FilterNumberTPCHits art::module LIBRARIES ${MODULE_LIBRARIES})
+cet_build_plugin(FilterDataIntegrity art::module LIBRARIES ${MODULE_LIBRARIES})
 cet_build_plugin(FilterOpFlash art::module LIBRARIES ${MODULE_LIBRARIES})
 cet_build_plugin(FilterParticlesActiveVolume art::module LIBRARIES ${MODULE_LIBRARIES})
 cet_build_plugin(TriggerTypeFilter art::module LIBRARIES ${MODULE_LIBRARIES})

icaruscode/Filters/FilterDataIntegrity_module.cc

@@ -0,0 +1,91 @@
+////////////////////////////////////////////////////////////////////////
+//
+// FilterDataIntegrity class
+//
+////////////////////////////////////////////////////////////////////////
+
+/// Framework includes
+#include "art/Framework/Core/ModuleMacros.h"
+#include "art/Framework/Core/EDFilter.h"
+#include "art/Framework/Principal/Event.h"
+#include "messagefacility/MessageLogger/MessageLogger.h"
+
+#include "artdaq-core/Data/Fragment.hh"
+
+#include <set>
+#include <string>
+
+///filters for events, etc
+namespace filter
+{
+
+    class FilterDataIntegrity : public art::EDFilter
+    {
+
+    public:
+        explicit FilterDataIntegrity(fhicl::ParameterSet const &);
+
+        bool filter(art::Event &evt) override;
+
+    private:
+
+    // We will use this to keep track of the expected fragments in an event
+    // but note we will assume the first event is complete to set this list
+    std::set<int> fExpectedFragments;
+
+    }; //class FilterDataIntegrity
+}
+
+///////////////////////////////////////////////////////
+
+filter::FilterDataIntegrity::FilterDataIntegrity(fhicl::ParameterSet const &pset)
+    : EDFilter{pset}
+{
+    return;
+}
+
+bool filter::FilterDataIntegrity::filter(art::Event &event)
+{
+    bool filterPass = true;
+
+	// get all the artdaq fragment collections in the event.
+	std::vector<art::Handle<std::vector<artdaq::Fragment>>> fragmentHandles;
+	fragmentHandles = event.getMany<std::vector<artdaq::Fragment>>();
+
+    std::set<int> missingFragments(fExpectedFragments);
+    int           emptyFragments(0);
+
+    for (const auto& handle : fragmentHandles)
+    {
+        for (const auto& fragment : *handle)
+        {
+            int fragmentID = fragment.fragmentID();
+
+            fExpectedFragments.insert(fragmentID);
+            missingFragments.erase(fragmentID);
+
+            std::string instanceName = handle.provenance()->productInstanceName();
+            std::size_t found        = instanceName.find("Empty");
+
+            if (found != std::string::npos) emptyFragments++;
+        }
+    }
+
+    mf::LogDebug("FilterDataIntegrity") << "Expected fragments: " << fExpectedFragments.size() << " expected, has " << missingFragments.size() << " missing. There are " << emptyFragments << " empty fragments";
+
+    if (!missingFragments.empty() || emptyFragments > 0)
+    {
+        mf::LogInfo("FilterDataIntegrity") << "Bad fragments: " << missingFragments.size() << ", " << emptyFragments;
+
+        filterPass = false;
+    }
+
+    return filterPass;
+}
+
+namespace filter
+{
+
+    DEFINE_ART_MODULE(FilterDataIntegrity)
+
+} //namespace filt

icaruscode/Filters/icarus_FilterDataIntegrity.fcl

@@ -0,0 +1,8 @@
+BEGIN_PROLOG
+
+icarus_FilterDataIntegrity:
+{ 
+  module_type:     "FilterDataIntegrity"
+}
+
+END_PROLOG