From 26ad2267064589a7d3117fb4cbd8ab2052104c04 Mon Sep 17 00:00:00 2001 From: Victor Schwan Date: Thu, 16 May 2024 19:32:02 +0200 Subject: [PATCH 01/33] variables outsourced to config.py; condorJobs_sim runs until actual execution at the end --- .gitignore | 1 + TrackingPerformance/Condor/condorJobs_reco.py | 129 ++++++----- TrackingPerformance/Condor/condorJobs_sim.py | 178 +++++++++------ .../Plotting/analysis_tracking.py | 202 +++++++++++------- TrackingPerformance/Plotting/testing.py | 13 ++ TrackingPerformance/config.py | 26 +++ 6 files changed, 356 insertions(+), 193 deletions(-) create mode 100644 .gitignore create mode 100644 TrackingPerformance/Plotting/testing.py create mode 100644 TrackingPerformance/config.py diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..61f2dc9 --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +**/__pycache__/ diff --git a/TrackingPerformance/Condor/condorJobs_reco.py b/TrackingPerformance/Condor/condorJobs_reco.py index 8d4816f..579e4b5 100644 --- a/TrackingPerformance/Condor/condorJobs_reco.py +++ b/TrackingPerformance/Condor/condorJobs_reco.py @@ -2,26 +2,37 @@ import os import sys -import ROOT -import argparse -import subprocess +import ROOT +import sys +from os.path import dirname, abspath, join + +# Add the project root directory to the sys.path +project_root = dirname(dirname(abspath(__file__))) +# Append the project root to sys.path if not already present +if project_root not in sys.path: + sys.path.append(project_root) + +# import config +import config # ========================== # Parameters Initialisation # ========================== # Define lists of parameters for reconstruction -thetaList_ = ["10", "20", "30", "40", "50", "60", "70", "80", "89"] -#thetaList_ = ["70", "80", "89"] -momentumList_ = ["1", "2", "5", "10", "20", "50", "100", "200"] -#momentumList_ = ["1", "10", "100"] -particleList_ = ["mu"]#,"e" ,"pi"] -#ResVDX_UV_ = ['0.001'] +thetaList_ = config.thetaList_ +momentumList_ = config.momentumList_ +particleList_ = config.particleList_ -DetectorModelList_ = ["CLD_o3_v01"] # FCCee_o1_v04 CLD_o2_v05 CLD_o3_v01 -Nevts_ = "10000" +DetectorModelList_ = config.detectorModel +Nevts_ = "10000" Nevt_per_job = "1000" # Set the desired number of events per job -N_jobs = int(int(Nevts_) / int(Nevt_per_job)) * len(particleList_) * len(thetaList_) * len(momentumList_) +N_jobs = ( + int(int(Nevts_) / int(Nevt_per_job)) + * len(particleList_) + * len(thetaList_) + * len(momentumList_) +) total_events = int(Nevts_) num_jobs = total_events // int(Nevt_per_job) @@ -30,19 +41,23 @@ # =========================== # Define directories for input and output directory_jobs = f"CondorJobs/Rec_{particleList_[0]}_{DetectorModelList_[0]}" -#setup = "/cvmfs/sw-nightlies.hsf.org/key4hep/setup.sh" # nightlies -setup = "/cvmfs/sw.hsf.org/key4hep/setup.sh" # stable -#InputDirectory = f"/eos/user/g/gasadows/Output/TrackingPerformance/{DetectorModelList_[0]}/SIM/3T/" +# setup = "/cvmfs/sw-nightlies.hsf.org/key4hep/setup.sh" # nightlies +setup = "/cvmfs/sw.hsf.org/key4hep/setup.sh" # stable +# InputDirectory = f"/eos/user/g/gasadows/Output/TrackingPerformance/{DetectorModelList_[0]}/SIM/3T/" InputDirectory = f"/eos/experiment/fcc/users/g/gasadows/TrackingPerformance/{DetectorModelList_[0]}/SIM/3T/" -EosDir = f"/eos/user/g/gasadows/Output/TrackingPerformance/{DetectorModelList_[0]}/REC/3T/" +EosDir = ( + f"/eos/user/g/gasadows/Output/TrackingPerformance/{DetectorModelList_[0]}/REC/3T/" +) -#steering_file = "CLDReconstruction.py" -steering_file = "/afs/cern.ch/user/g/gasadows/CLDConfig/CLDConfig/CLDReconstruction_3T.py" +# steering_file = "CLDReconstruction.py" +steering_file = ( + "/afs/cern.ch/user/g/gasadows/CLDConfig/CLDConfig/CLDReconstruction_3T.py" +) # Enable output checks check_output = True # Set to True to enable checks, False to disable - # It will check if the ouputs exist and contain correct number of events - # if not it will send job to rerun reconstruction +# It will check if the ouputs exist and contain correct number of events +# if not it will send job to rerun reconstruction JobFlavour = "tomorrow" # Job flavours: @@ -57,24 +72,29 @@ # Set default value if ResVDX_UV_ is not defined or empty try: if not ResVDX_UV_: - ResVDX_UV_ = ['0.003'] + ResVDX_UV_ = ["0.003"] except NameError: - ResVDX_UV_ = ['0.003'] + ResVDX_UV_ = ["0.003"] # Check if the directory exists and exit if it does if os.path.exists(directory_jobs): - print(f"Error: Directory '{directory_jobs}' already exists and should not be overwritten.") + print( + f"Error: Directory '{directory_jobs}' already exists and should not be overwritten." + ) sys.exit(1) # Create output directories if they don't exist [os.makedirs(directory, exist_ok=True) for directory in [EosDir, directory_jobs]] # ======================= -# Simulation Job Creation +# Simulation Job Creation # ======================= # Create all possible combinations import itertools -list_of_combined_variables = itertools.product(thetaList_, momentumList_, particleList_, DetectorModelList_) + +list_of_combined_variables = itertools.product( + thetaList_, momentumList_, particleList_, DetectorModelList_ +) need_to_create_scripts = False @@ -82,23 +102,27 @@ for task_index in range(num_jobs): outputFileName = f"REC_{dect}" - outputFileName+= f"_{part}" - outputFileName+= f"_{theta}_deg" - outputFileName+= f"_{momentum}_GeV" - outputFileName+= f"_{Nevt_per_job}_evts" - outputFileName+= f"_{task_index}" - - inputFile= os.path.join(InputDirectory + f"/{part}", f"SIM_{dect}_{part}_{theta}_deg_{momentum}_GeV_{Nevt_per_job}_evts_{task_index}_edm4hep.root") - #inputFile= os.path.join(InputDirectory + f"/{part}", f"SIM_{dect}_{part}_{theta}_deg_{momentum}_GeV_{Nevt_per_job}_evts_edm4hep.root") - #input_file= os.path.join(InputDirectory, "SIMTest_" + dect + "_" + part + "_" + theta + "_deg_" + momentum + "_GeV_" + Nevts_ + "_evts.slcio") + outputFileName += f"_{part}" + outputFileName += f"_{theta}_deg" + outputFileName += f"_{momentum}_GeV" + outputFileName += f"_{Nevt_per_job}_evts" + outputFileName += f"_{task_index}" + + inputFile = os.path.join( + InputDirectory + f"/{part}", + f"SIM_{dect}_{part}_{theta}_deg_{momentum}_GeV_{Nevt_per_job}_evts_{task_index}_edm4hep.root", + ) + # inputFile= os.path.join(InputDirectory + f"/{part}", f"SIM_{dect}_{part}_{theta}_deg_{momentum}_GeV_{Nevt_per_job}_evts_edm4hep.root") + # input_file= os.path.join(InputDirectory, "SIMTest_" + dect + "_" + part + "_" + theta + "_deg_" + momentum + "_GeV_" + Nevts_ + "_evts.slcio") # Check if the input file exists if not os.path.exists(inputFile): print(f"Error: Input file {inputFile} does not exist. Skipping job.") - continue + continue # Check if the output file already exists and has correct Nb of events - output_dir = os.path.join(EosDir, part); os.makedirs(output_dir, exist_ok=True) - output_file = output_dir +"/"+ outputFileName + "_edm4hep.root" + output_dir = os.path.join(EosDir, part) + os.makedirs(output_dir, exist_ok=True) + output_file = output_dir + "/" + outputFileName + "_edm4hep.root" if check_output and os.path.exists(output_file): root_file = ROOT.TFile(output_file, "READ") events_tree = root_file.Get("events") @@ -109,15 +133,20 @@ need_to_create_scripts = True # Create aida output Dir - output_dir_aida = os.path.join(output_dir, "aida_outputs"); os.makedirs(output_dir_aida, exist_ok=True) + output_dir_aida = os.path.join(output_dir, "aida_outputs") + os.makedirs(output_dir_aida, exist_ok=True) arguments = ( - #f" --GeoSvc.detectors=/afs/cern.ch/work/g/gasadows/k4geo/FCCee/CLD/compact/{DetectorModelList_[0]}_3T/{DetectorModelList_[0]}.xml"+ - f" --GeoSvc.detectors=$K4GEO/FCCee/CLD/compact/{DetectorModelList_[0]}/{DetectorModelList_[0]}.xml"+ - " --inputFiles " + inputFile + " --outputBasename " + outputFileName+ - f" --VXDDigitiserResUV={ResVDX_UV_[0]}" + - " --trackingOnly" + - " -n " + Nevt_per_job + # f" --GeoSvc.detectors=/afs/cern.ch/work/g/gasadows/k4geo/FCCee/CLD/compact/{DetectorModelList_[0]}_3T/{DetectorModelList_[0]}.xml"+ + f" --GeoSvc.detectors=$K4GEO/FCCee/CLD/compact/{DetectorModelList_[0]}/{DetectorModelList_[0]}.xml" + + " --inputFiles " + + inputFile + + " --outputBasename " + + outputFileName + + f" --VXDDigitiserResUV={ResVDX_UV_[0]}" + + " --trackingOnly" + + " -n " + + Nevt_per_job ) command = f"k4run {steering_file} " + arguments + " > /dev/null" @@ -131,7 +160,10 @@ f"xrdcp {outputFileName}_edm4hep.root root://eosuser.cern.ch/{output_dir} \n" f"xrdcp {outputFileName}_aida.root root://eosuser.cern.ch/{output_dir_aida} \n" ) - bash_file = directory_jobs + f"/bash_script_{dect}_{part}_{momentum}_{theta}_{task_index}.sh" + bash_file = ( + directory_jobs + + f"/bash_script_{dect}_{part}_{momentum}_{theta}_{task_index}.sh" + ) with open(bash_file, "w") as file: file.write(bash_script) file.close() @@ -150,7 +182,7 @@ "output = output.$(ClusterId).$(ProcId).out \n" "error = error.$(ClusterId).$(ProcId).err \n" "log = log.$(ClusterId).log \n" - f"+JobFlavour = \"{JobFlavour}\" \n" + f'+JobFlavour = "{JobFlavour}" \n' "queue filename matching files *.sh \n" ) condor_file = directory_jobs + "/condor_script.sub" @@ -161,7 +193,4 @@ # ==================== # Submit Job to Condor # ==================== -os.system("cd "+ directory_jobs + "; condor_submit condor_script.sub") - - - +os.system("cd " + directory_jobs + "; condor_submit condor_script.sub") diff --git a/TrackingPerformance/Condor/condorJobs_sim.py b/TrackingPerformance/Condor/condorJobs_sim.py index 7ec5faf..732cb88 100644 --- a/TrackingPerformance/Condor/condorJobs_sim.py +++ b/TrackingPerformance/Condor/condorJobs_sim.py @@ -2,43 +2,58 @@ import os import sys -import ROOT -import argparse -import subprocess -import time; +import ROOT +import time +from os.path import dirname, abspath, join -ts = time.time() # Get current timestamp for unique identifiers +verbose = False + + +# Add the project root directory to the sys.path +project_root = dirname(dirname(abspath(__file__))) +# Append the project root to sys.path if not already present +if project_root not in sys.path: + sys.path.append(project_root) + +# import config +import config + +ts = time.time() # Get current timestamp for unique identifiers # ========================== # Parameters Initialisation # ========================== # Define lists of parameters for reconstruction -thetaList_ = ["10", "20", "30", "40", "50", "60", "70", "80", "89"] -momentumList_ = ["1", "2", "5", "10", "20", "50", "100", "200"] -#momentumList_ = ["1", "10", "100"] -particleList_ = ["mu"]#,"e" ,"pi"] +thetaList_ = config.thetaList_ +momentumList_ = config.momentumList_ +particleList_ = config.particleList_ + +DetectorModelList_ = config.detectorModel +Nevts_ = config.Nevts_ +Nevt_per_job = config.Nevt_per_job # Set the desired number of events per job -DetectorModelList_ = ["CLD_o3_v01"] # FCCee_o1_v04 CLD_o2_v05 CLD_o3_v01 -Nevts_ = "10000" -Nevt_per_job = "1000" # Set the desired number of events per job -N_jobs = int(int(Nevts_) / int(Nevt_per_job)) * len(particleList_) * len(thetaList_) * len(momentumList_) total_events = int(Nevts_) -num_jobs = total_events // int(Nevt_per_job) +num_jobs_per_para_set = total_events // int( + Nevt_per_job +) # number of parallel jobs with same parameter combination/set +N_jobs = ( + num_jobs_per_para_set * len(particleList_) * len(thetaList_) * len(momentumList_) +) # total number of jobs # =========================== # Directory Setup and Checks # =========================== # Define directories for input and output -directory_jobs = f"CondorJobs/Sim_mu_{DetectorModelList_[0]}" -#setup = "/cvmfs/sw-nightlies.hsf.org/key4hep/setup.sh" # nightlies -setup = "/cvmfs/sw.hsf.org/key4hep/setup.sh" # stable -EosDir = f"/eos/user/g/gasadows/Output/TrackingPerformance/{DetectorModelList_[0]}/SIM/3T" +directory_jobs = join(config.SIMcondorDir, f"mu_{DetectorModelList_[0]}") +# setup = "/cvmfs/sw-nightlies.hsf.org/key4hep/setup.sh" # nightlies +setup = "/cvmfs/sw.hsf.org/key4hep/setup.sh" # stable +outputEosDir = join(config.dataDir, f"{DetectorModelList_[0]}/SIM") # Enable output checks check_output = True # Set to True to enable checks, False to disable - # It will check if the ouputs exist and contain correct number of events - # if not it will send job to rerun simulation +# It will check if the ouputs exist and contain correct number of events +# if not it will send job to rerun simulation JobFlavour = "testmatch" # Job flavours: @@ -52,36 +67,51 @@ # Check if the directory exists and exit if it does -if os.path.exists(directory_jobs): - print(f"Error: Directory '{directory_jobs}' already exists and should not be overwritten.") +try: + os.makedirs(directory_jobs, exist_ok=False) +except FileExistsError: + print( + f"Error: Directory '{directory_jobs}' already exists and should not be overwritten." + ) sys.exit(1) -# Create output directories if they don't exist -[os.makedirs(directory, exist_ok=True) for directory in [EosDir, directory_jobs]] +os.makedirs( + outputEosDir, exist_ok=True +) # This will create the directory if it doesn't exist, without raising an error if it does + # ======================= -# Simulation Job Creation +# Simulation Job Creation # ======================= # Create all possible combinations import itertools -list_of_combined_variables = itertools.product(thetaList_, momentumList_, particleList_, DetectorModelList_) -need_to_create_scripts = False +iter_of_combined_variables = itertools.product( + thetaList_, momentumList_, particleList_, DetectorModelList_ +) -for theta, momentum, part, dect in list_of_combined_variables: - for task_index in range(num_jobs): +need_to_create_scripts = False - output_file_name = f"SIM_{dect}" - output_file_name+= f"_{part}" - output_file_name+= f"_{theta}_deg" - output_file_name+= f"_{momentum}_GeV" - output_file_name+= f"_{Nevt_per_job}_evts" - output_file_name+= f"_{task_index}" - output_file_name+= f"_edm4hep.root" +if verbose: + print(f"1st of {N_jobs} different parameter combinations starts") +for counter, (theta, momentum, part, dect) in enumerate(iter_of_combined_variables): + for task_index in range(num_jobs_per_para_set): + + output_file_name_parts = [ + f"SIM_{dect}", + f"{part}", + f"{theta}_deg", + f"{momentum}_GeV", + f"{Nevt_per_job}_evts", + f"{task_index}", + "edm4hep.root", + ] + output_file_name = "_".join(output_file_name_parts) # Check if the output file already exists and has correct Nb of events - output_dir = os.path.join(EosDir, part); os.makedirs(output_dir, exist_ok=True) - output_file = os.path.join(output_dir, output_file_name) + output_dir = join(outputEosDir, part) + os.makedirs(output_dir, exist_ok=True) + output_file = join(output_dir, output_file_name) if check_output and os.path.exists(output_file): root_file = ROOT.TFile(output_file, "READ") events_tree = root_file.Get("events") @@ -92,40 +122,64 @@ root_file.Close() else: need_to_create_scripts = True - + time.sleep(1) - seed = str(time.time()%1000) - - arguments = ( - #f" --compactFile /afs/cern.ch/work/g/gasadows/k4geo/FCCee/CLD/compact/{DetectorModelList_[0]}_3T/{DetectorModelList_[0]}.xml " - f" --compactFile ${K4GEO}/FCCee/CLD/compact/{DetectorModelList_[0]}/{DetectorModelList_[0]}.xml " - "--outputFile " + output_file_name + " " - "--steeringFile " + "CLDConfig/CLDConfig/cld_steer.py " - "--random.seed " + seed + " " - "--enableGun " - "--gun.particle " + part + "- " - "--gun.energy " + momentum + "*GeV " - "--gun.distribution uniform " - "--gun.thetaMin " + theta + "*deg " - "--gun.thetaMax " + theta + "*deg " - "--crossingAngleBoost 0 " - "--numberOfEvents " + Nevt_per_job - ) - command = "ddsim " + arguments + " > /dev/null" + seed = str(time.time() % 1000) + + # if len(DetectorModelList_) != 1 or DetectorModelList_[0] != "ILD_l5_v11": + # raise ValueError("so far only ILD_l5_v11 possible") + + # Build ddsim command + arguments = [ + # f" --compactFile /afs/cern.ch/work/g/gasadows/k4geo/FCCee/CLD/compact/{DetectorModelList_[0]}_3T/{DetectorModelList_[0]}.xml " + f" --compactFile $k4geo_DIR/{config.detModPaths['ILD_l5_v11']}", # Note the change to use double quotes for the dictionary key + f"--outputFile {output_file_name}", + f"--steeringFile {config.sim_steering_file}", # "CLDConfig/CLDConfig/cld_steer.py " + # TODO: FIX + # f"--random.seed {seed}", + # "--enableGun", + # f"--gun.particle {part}-", + # f"--gun.energy {momentum}*GeV", + # "--gun.distribution uniform", + # f"--gun.thetaMin {theta}*deg", + # f"--gun.thetaMax {theta}*deg", + # "--crossingAngleBoost 0", + # f"--numberOfEvents {Nevt_per_job}", + ] + command = f"ddsim {' '.join(arguments)} > /dev/null" # Write bash script for job execution bash_script = ( "#!/bin/bash \n" f"source {setup} \n" - "git clone https://github.com/gaswk/CLDConfig.git \n" + "git clone https://github.com/Victor-Schwan/TrackingStudies.git \n" f"{command} \n" f"xrdcp {output_file_name} root://eosuser.cern.ch/{output_dir} \n" ) - bash_file = directory_jobs + f"/bash_script_{dect}_{part}_{momentum}_{theta}_{task_index}.sh" + bash_file = ( + directory_jobs + + f"/bash_script_{dect}_{part}_{momentum}_{theta}_{task_index}.sh" + ) + bash_file_name_parts = [ + "bash_script", + dect, + part, + f"{theta}_deg", + f"{momentum}_GeV", + str(task_index), + ] + bash_file = f"{directory_jobs}/{'_'.join(bash_file_name_parts)}.sh" + with open(bash_file, "w") as file: file.write(bash_script) file.close() + if verbose: + print(f" {task_index+1} of {num_jobs_per_para_set} parallel jobs done") + + if verbose: + print(f"{counter+1} of {N_jobs} different parameter combinations done") + if not need_to_create_scripts: print("All output files are correct.") sys.exit(0) @@ -140,7 +194,7 @@ "output = output.$(ClusterId).$(ProcId).out \n" "error = error.$(ClusterId).$(ProcId).err \n" "log = log.$(ClusterId).log \n" - f"+JobFlavour = \"{JobFlavour}\" \n" + f'+JobFlavour = "{JobFlavour}" \n' "queue filename matching files *.sh \n" ) condor_file = directory_jobs + "/condor_script.sub" @@ -151,6 +205,4 @@ # ==================== # Submit Job to Condor # ==================== -os.system("cd "+ directory_jobs + "; condor_submit condor_script.sub") - - +# os.system("cd " + directory_jobs + "; condor_submit condor_script.sub") diff --git a/TrackingPerformance/Plotting/analysis_tracking.py b/TrackingPerformance/Plotting/analysis_tracking.py index d6d218a..e30e384 100644 --- a/TrackingPerformance/Plotting/analysis_tracking.py +++ b/TrackingPerformance/Plotting/analysis_tracking.py @@ -1,30 +1,50 @@ -import os - -# Lists of parameters -ParticleList = ["mu"]#, "e", "pi"] -ThetaList = ["10", "20", "30", "40", "50", "60", "70", "80", "89"] -#ThetaList = ["70", "80", "89"] -MomentumList = ["1", "2", "5", "10", "20", "50", "100", "200"] -#MomentumList = ["1", "10", "100"] -DetectorModel = ["CLD_o2_v05"] # FCCee_o1_v04 CLD_o2_v05 CLD_o3_v01 -Nevts = "10000" -Nevts_per_job = "1000" +import sys +from os.path import dirname, abspath, join + +# Add the project root directory to the sys.path +project_root = dirname(dirname(abspath(__file__))) +# Append the project root to sys.path if not already present +if project_root not in sys.path: + sys.path.append(project_root) + +# import config +import config + +# ========================== +# Parameters Initialisation +# ========================== +# Define lists of parameters for reconstruction +thetaList_ = config.thetaList_ +momentumList_ = config.momentumList_ +particleList_ = config.particleList_ + +DetectorModel = config.detectorModel +Nevts_ = config.Nevts_ +Nevt_per_job = config.Nevt_per_job # Set the desired number of events per job # Output and input directories -#inputDir = f"/eos/experiment/fcc/users/g/gasadows/TrackingPerformance/{DetectorModel[0]}/REC/mu/VXD_3mic" -outputDir = f"/eos/user/g/gasadows/Output/TrackingPerformance/{DetectorModel[0]}/analysis/3T_2/mu/" -inputDir = f"/eos/user/g/gasadows/Output/TrackingPerformance/{DetectorModel[0]}/REC/3T/mu" +outputDir = join(config.dataDir, f"TrackingPerformance/{DetectorModel[0]}/analysis/") +inputDir = join(config.dataDir, f"TrackingPerformance/{DetectorModel[0]}/REC/") + processList = { - f"REC_{DetectorModel[0]}_{particle}_{theta}_deg_{momentum}_GeV_{Nevts_per_job}_evts_{i}_edm4hep":{"output":f"{particle}_{theta}deg_{momentum}GeV_{Nevts_per_job}evts_{i}"} - for particle in ParticleList for theta in ThetaList for momentum in MomentumList for i in range(int(Nevts)//int(Nevts_per_job))} + f"REC_{DetectorModel[0]}_{particle}_{theta}_deg_{momentum}_GeV_{Nevts_per_job}_evts_{i}_edm4hep": { + "output": f"{particle}_{theta}deg_{momentum}GeV_{Nevts_per_job}evts_{i}" + } + for particle in ParticleList + for theta in ThetaList + for momentum in MomentumList + for i in range(int(Nevts) // int(Nevts_per_job)) +} # Optional: ncpus, default is 4, -1 uses all cores available nCPUS = -1 -#USER DEFINED CODE +# USER DEFINED CODE import ROOT -ROOT.gInterpreter.Declare(""" + +ROOT.gInterpreter.Declare( + """ ROOT::VecOps::RVec MCTruthTrackIndex(ROOT::VecOps::RVec trackIndex, ROOT::VecOps::RVec mcIndex, ROOT::VecOps::RVec mc) @@ -37,81 +57,103 @@ } return res; } -""") +""" +) ROOT.gInterpreter.Declare("#include ") -#END USER DEFINED CODE +# END USER DEFINED CODE # List of variables to analyse varList = ["pt", "d0", "z0", "phi0", "omega", "tanLambda", "p", "phi", "theta"] -class RDFanalysis(): + +class RDFanalysis: def analysers(df): - df2 = (df - - .Alias("MCTrackAssociations0", "_SiTracksMCTruthLink_rec.index") - .Alias("MCTrackAssociations1", "_SiTracksMCTruthLink_sim.index") - .Alias("SiTracks_Refitted_1", "_SiTracks_Refitted_trackStates") - - .Define("GunParticle_index", "MCParticles.generatorStatus == 1") - .Define("GunParticle", "MCParticles[GunParticle_index][0]") - - .Define("trackStates_IP", "SiTracks_Refitted_1[SiTracks_Refitted_1.location == 1]") - .Define("MC2TrackIndex", "MCTruthTrackIndex(MCTrackAssociations0, MCTrackAssociations1, MCParticles)") - .Define("GunParticleTrackIndex", "MC2TrackIndex[GunParticle_index][0]") - .Define("GunParticleTSIP", "trackStates_IP[GunParticleTrackIndex]") - - .Define("MatchedGunParticle_1", "MCParticles[MC2TrackIndex != -1]") - .Define("MatchedGunParticle", "FCCAnalyses::MCParticle::sel_genStatus(1) (MatchedGunParticle_1)") - - .Define("trackData", "SiTracks_Refitted[GunParticleTrackIndex]") - - # Helix calculations and definitions - .Define("GunParticleTSIPHelix", "auto h = HelixClass_double(); h.Initialize_Canonical(GunParticleTSIP.phi, GunParticleTSIP.D0, GunParticleTSIP.Z0, GunParticleTSIP.omega, GunParticleTSIP.tanLambda, 2); return h;") - .Define("reco_pt", "GunParticleTSIPHelix.getPXY()") - .Define("reco_d0", "GunParticleTSIP.D0") - .Define("reco_z0", "GunParticleTSIP.Z0") - .Define("reco_phi0", "GunParticleTSIP.phi") - .Define("reco_omega", "GunParticleTSIP.omega") - .Define("reco_tanLambda", "GunParticleTSIP.tanLambda") - .Define("reco_pvec", "auto p = GunParticleTSIPHelix.getMomentum(); return ROOT::Math::XYZVector(p[0], p[1], p[2]);") - .Define("reco_p", "reco_pvec.R()") - .Define("reco_phi", "reco_pvec.Phi()") - .Define("reco_theta", "reco_pvec.Theta()") - - .Define("GunParticleMCMom", "std::vector v = {GunParticle.momentum.x, GunParticle.momentum.y, GunParticle.momentum.z}; return v;") - .Define("GunParticleMCPos", "std::vector v = {GunParticle.vertex.x, GunParticle.vertex.y, GunParticle.vertex.z}; return v;") - .Define("GunParticleMCHelix", "auto h = HelixClass_double(); h.Initialize_VP(GunParticleMCPos.data(), GunParticleMCMom.data(), -1, 2); return h;") - .Define("true_pt", "GunParticleMCHelix.getPXY()") - .Define("true_d0", "GunParticleMCHelix.getD0()") - .Define("true_z0", "GunParticleMCHelix.getZ0()") - .Define("true_phi0", "GunParticleMCHelix.getPhi0()") - .Define("true_omega", "GunParticleMCHelix.getOmega()") - .Define("true_tanLambda", "GunParticleMCHelix.getTanLambda()") - .Define("true_pvec", "ROOT::Math::XYZVector(GunParticleMCMom[0], GunParticleMCMom[1], GunParticleMCMom[2])") - .Define("true_p", "true_pvec.R()") - .Define("true_phi", "true_pvec.Phi()") - .Define("true_theta", "true_pvec.Theta()") - - .Define("chi2_trk", "trackData.chi2") - .Define("ndf_trk", "trackData.ndf") - .Define("chi2_over_ndf", "chi2_trk / ndf_trk") - .Filter("chi2_over_ndf < 10 ") - - # Remove fake tracks - .Filter("abs( (reco_pt - true_pt) / true_pt) <= 0.20") - .Filter("abs( (reco_phi - true_phi) / true_phi) <= 0.20 ") - .Filter("abs( (reco_theta - true_theta) / true_theta) <= 0.20 ") - - .Define("num_reconstructed_tracks", "trackStates_IP.size() > 0 ? 1 : 0") + df2 = ( + df.Alias("MCTrackAssociations0", "_SiTracksMCTruthLink_rec.index") + .Alias("MCTrackAssociations1", "_SiTracksMCTruthLink_sim.index") + .Alias("SiTracks_Refitted_1", "_SiTracks_Refitted_trackStates") + .Define("GunParticle_index", "MCParticles.generatorStatus == 1") + .Define("GunParticle", "MCParticles[GunParticle_index][0]") + .Define( + "trackStates_IP", + "SiTracks_Refitted_1[SiTracks_Refitted_1.location == 1]", + ) + .Define( + "MC2TrackIndex", + "MCTruthTrackIndex(MCTrackAssociations0, MCTrackAssociations1, MCParticles)", + ) + .Define("GunParticleTrackIndex", "MC2TrackIndex[GunParticle_index][0]") + .Define("GunParticleTSIP", "trackStates_IP[GunParticleTrackIndex]") + .Define("MatchedGunParticle_1", "MCParticles[MC2TrackIndex != -1]") + .Define( + "MatchedGunParticle", + "FCCAnalyses::MCParticle::sel_genStatus(1) (MatchedGunParticle_1)", + ) + .Define("trackData", "SiTracks_Refitted[GunParticleTrackIndex]") + # Helix calculations and definitions + .Define( + "GunParticleTSIPHelix", + "auto h = HelixClass_double(); h.Initialize_Canonical(GunParticleTSIP.phi, GunParticleTSIP.D0, GunParticleTSIP.Z0, GunParticleTSIP.omega, GunParticleTSIP.tanLambda, 2); return h;", + ) + .Define("reco_pt", "GunParticleTSIPHelix.getPXY()") + .Define("reco_d0", "GunParticleTSIP.D0") + .Define("reco_z0", "GunParticleTSIP.Z0") + .Define("reco_phi0", "GunParticleTSIP.phi") + .Define("reco_omega", "GunParticleTSIP.omega") + .Define("reco_tanLambda", "GunParticleTSIP.tanLambda") + .Define( + "reco_pvec", + "auto p = GunParticleTSIPHelix.getMomentum(); return ROOT::Math::XYZVector(p[0], p[1], p[2]);", + ) + .Define("reco_p", "reco_pvec.R()") + .Define("reco_phi", "reco_pvec.Phi()") + .Define("reco_theta", "reco_pvec.Theta()") + .Define( + "GunParticleMCMom", + "std::vector v = {GunParticle.momentum.x, GunParticle.momentum.y, GunParticle.momentum.z}; return v;", + ) + .Define( + "GunParticleMCPos", + "std::vector v = {GunParticle.vertex.x, GunParticle.vertex.y, GunParticle.vertex.z}; return v;", + ) + .Define( + "GunParticleMCHelix", + "auto h = HelixClass_double(); h.Initialize_VP(GunParticleMCPos.data(), GunParticleMCMom.data(), -1, 2); return h;", + ) + .Define("true_pt", "GunParticleMCHelix.getPXY()") + .Define("true_d0", "GunParticleMCHelix.getD0()") + .Define("true_z0", "GunParticleMCHelix.getZ0()") + .Define("true_phi0", "GunParticleMCHelix.getPhi0()") + .Define("true_omega", "GunParticleMCHelix.getOmega()") + .Define("true_tanLambda", "GunParticleMCHelix.getTanLambda()") + .Define( + "true_pvec", + "ROOT::Math::XYZVector(GunParticleMCMom[0], GunParticleMCMom[1], GunParticleMCMom[2])", + ) + .Define("true_p", "true_pvec.R()") + .Define("true_phi", "true_pvec.Phi()") + .Define("true_theta", "true_pvec.Theta()") + .Define("chi2_trk", "trackData.chi2") + .Define("ndf_trk", "trackData.ndf") + .Define("chi2_over_ndf", "chi2_trk / ndf_trk") + .Filter("chi2_over_ndf < 10 ") + # Remove fake tracks + .Filter("abs( (reco_pt - true_pt) / true_pt) <= 0.20") + .Filter("abs( (reco_phi - true_phi) / true_phi) <= 0.20 ") + .Filter("abs( (reco_theta - true_theta) / true_theta) <= 0.20 ") + .Define("num_reconstructed_tracks", "trackStates_IP.size() > 0 ? 1 : 0") ) for v in varList: df2 = df2.Define(f"delta_{v}", f"reco_{v} - true_{v}") df2 = df2.Filter(f"std::isfinite(delta_{v})") if "phi0" in varList: - # Correct phi wrap around - df2 = df2.Redefine("delta_phi0", "delta_phi0 < -ROOT::Math::Pi() ? delta_phi0 + 2 * ROOT::Math::Pi() : delta_phi0") + # Correct phi wrap around + df2 = df2.Redefine( + "delta_phi0", + "delta_phi0 < -ROOT::Math::Pi() ? delta_phi0 + 2 * ROOT::Math::Pi() : delta_phi0", + ) return df2 diff --git a/TrackingPerformance/Plotting/testing.py b/TrackingPerformance/Plotting/testing.py new file mode 100644 index 0000000..1e6a1b4 --- /dev/null +++ b/TrackingPerformance/Plotting/testing.py @@ -0,0 +1,13 @@ +import sys +import os + +# Add the project root directory to the sys.path +project_root = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +sys.path.append(project_root) + +# Now you can import config +import config + +# Access the baseDir from config.py + +print(f"Base directory: {config.baseDir}") diff --git a/TrackingPerformance/config.py b/TrackingPerformance/config.py new file mode 100644 index 0000000..5cda975 --- /dev/null +++ b/TrackingPerformance/config.py @@ -0,0 +1,26 @@ +from os.path import join + +# define base directory +baseDir = "/eos/home-v/vschwan/promotion/" +# define directory to store output +dataDir = join(baseDir, "data/") +SIMcondorDir = join(dataDir, "sim/condor_jobs/") +sim_steering_file = "/eos/home-v/vschwan/promotion/ILDConfig/StandardConfig/production/TPC_debug_muon_steer.py" + +# ========================== +# Job Parameters Initialisation +# ========================== + +Nevts_ = "300" +Nevt_per_job = "100" # Set the desired number of events per job + +# ========================== +# Parameters Initialisation +# ========================== +detectorModel = ["ILD_l5_v11"] +detModPaths = {"ILD_l5_v11": "ILD/compact/ILD_l5_v11/ILD_l5_v11.xml"} +# Define lists of parameters for reconstruction +thetaList_ = ["10", "20"] # , "30", "40", "50", "60", "70", "80", "89" +momentumList_ = ["1", "2"] # , "5", "10", "20", "50", "100", "200" +# momentumList_ = ["1", "10", "100"] +particleList_ = ["mu"] # ,"e" ,"pi"] From 98804488ca6b9c4a5ababbce8b7b21ae36f9b492 Mon Sep 17 00:00:00 2001 From: Victor Schwan Date: Tue, 21 May 2024 11:32:36 +0200 Subject: [PATCH 02/33] switch from str paths to Path objects --- TrackingPerformance/Condor/condorJobs_sim.py | 32 +++++++++----------- TrackingPerformance/config.py | 14 +++++---- 2 files changed, 22 insertions(+), 24 deletions(-) diff --git a/TrackingPerformance/Condor/condorJobs_sim.py b/TrackingPerformance/Condor/condorJobs_sim.py index 732cb88..4304922 100644 --- a/TrackingPerformance/Condor/condorJobs_sim.py +++ b/TrackingPerformance/Condor/condorJobs_sim.py @@ -4,16 +4,16 @@ import sys import ROOT import time -from os.path import dirname, abspath, join +from pathlib import Path verbose = False # Add the project root directory to the sys.path -project_root = dirname(dirname(abspath(__file__))) +project_root = Path(__file__).resolve().parent.parent # Append the project root to sys.path if not already present if project_root not in sys.path: - sys.path.append(project_root) + sys.path.append(str(project_root)) # import config import config @@ -45,10 +45,10 @@ # Directory Setup and Checks # =========================== # Define directories for input and output -directory_jobs = join(config.SIMcondorDir, f"mu_{DetectorModelList_[0]}") +directory_jobs = config.SIMcondorDir / f"mu_{DetectorModelList_[0]}" # setup = "/cvmfs/sw-nightlies.hsf.org/key4hep/setup.sh" # nightlies setup = "/cvmfs/sw.hsf.org/key4hep/setup.sh" # stable -outputEosDir = join(config.dataDir, f"{DetectorModelList_[0]}/SIM") +outputEosDir = config.dataDir / f"{DetectorModelList_[0]}/SIM" # Enable output checks check_output = True # Set to True to enable checks, False to disable @@ -68,15 +68,15 @@ # Check if the directory exists and exit if it does try: - os.makedirs(directory_jobs, exist_ok=False) + directory_jobs.mkdir(parents=True, exist_ok=False) except FileExistsError: print( f"Error: Directory '{directory_jobs}' already exists and should not be overwritten." ) sys.exit(1) -os.makedirs( - outputEosDir, exist_ok=True +outputEosDir.mkdir( + parents=True, exist_ok=True ) # This will create the directory if it doesn't exist, without raising an error if it does @@ -109,10 +109,10 @@ output_file_name = "_".join(output_file_name_parts) # Check if the output file already exists and has correct Nb of events - output_dir = join(outputEosDir, part) - os.makedirs(output_dir, exist_ok=True) - output_file = join(output_dir, output_file_name) - if check_output and os.path.exists(output_file): + output_dir = outputEosDir / part + output_dir.mkdir(parents=True, exist_ok=True) + output_file = output_dir / output_file_name + if check_output and output_file.exists(): root_file = ROOT.TFile(output_file, "READ") events_tree = root_file.Get("events") if events_tree: @@ -156,10 +156,6 @@ f"{command} \n" f"xrdcp {output_file_name} root://eosuser.cern.ch/{output_dir} \n" ) - bash_file = ( - directory_jobs - + f"/bash_script_{dect}_{part}_{momentum}_{theta}_{task_index}.sh" - ) bash_file_name_parts = [ "bash_script", dect, @@ -168,7 +164,7 @@ f"{momentum}_GeV", str(task_index), ] - bash_file = f"{directory_jobs}/{'_'.join(bash_file_name_parts)}.sh" + bash_file = directory_jobs / "_".join(bash_file_name_parts) + ".sh" with open(bash_file, "w") as file: file.write(bash_script) @@ -197,7 +193,7 @@ f'+JobFlavour = "{JobFlavour}" \n' "queue filename matching files *.sh \n" ) -condor_file = directory_jobs + "/condor_script.sub" +condor_file = directory_jobs / "condor_script.sub" with open(condor_file, "w") as file2: file2.write(condor_script) file2.close() diff --git a/TrackingPerformance/config.py b/TrackingPerformance/config.py index 5cda975..9a29d21 100644 --- a/TrackingPerformance/config.py +++ b/TrackingPerformance/config.py @@ -1,11 +1,13 @@ -from os.path import join +from pathlib import Path # define base directory -baseDir = "/eos/home-v/vschwan/promotion/" +baseDir = Path("/eos/home-v/vschwan/promotion/") # define directory to store output -dataDir = join(baseDir, "data/") -SIMcondorDir = join(dataDir, "sim/condor_jobs/") -sim_steering_file = "/eos/home-v/vschwan/promotion/ILDConfig/StandardConfig/production/TPC_debug_muon_steer.py" +dataDir = baseDir / "data" +SIMcondorDir = dataDir / "sim" / "condor_jobs" +sim_steering_file = ( + baseDir / "ILDConfig" / "StandardConfig" / "production" / "TPC_debug_muon_steer.py" +) # ========================== # Job Parameters Initialisation @@ -18,7 +20,7 @@ # Parameters Initialisation # ========================== detectorModel = ["ILD_l5_v11"] -detModPaths = {"ILD_l5_v11": "ILD/compact/ILD_l5_v11/ILD_l5_v11.xml"} +detModPaths = {"ILD_l5_v11": Path("ILD/compact/ILD_l5_v11/ILD_l5_v11.xml")} # Define lists of parameters for reconstruction thetaList_ = ["10", "20"] # , "30", "40", "50", "60", "70", "80", "89" momentumList_ = ["1", "2"] # , "5", "10", "20", "50", "100", "200" From 06033d085439a15bd55071618ba2be96b1778abd Mon Sep 17 00:00:00 2001 From: Victor Schwan Date: Tue, 21 May 2024 11:47:31 +0200 Subject: [PATCH 03/33] minor fix --- TrackingPerformance/Condor/condorJobs_sim.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/TrackingPerformance/Condor/condorJobs_sim.py b/TrackingPerformance/Condor/condorJobs_sim.py index 4304922..cb36341 100644 --- a/TrackingPerformance/Condor/condorJobs_sim.py +++ b/TrackingPerformance/Condor/condorJobs_sim.py @@ -164,7 +164,7 @@ f"{momentum}_GeV", str(task_index), ] - bash_file = directory_jobs / "_".join(bash_file_name_parts) + ".sh" + bash_file = (directory_jobs / "_".join(bash_file_name_parts)).with_suffix(".sh") with open(bash_file, "w") as file: file.write(bash_script) From ecb75d11665724a939775adb3ade346d7c34c5e2 Mon Sep 17 00:00:00 2001 From: Victor Schwan Date: Tue, 21 May 2024 15:49:55 +0200 Subject: [PATCH 04/33] reco: switched to pathlib, more config vars added; conffig: environment moved here --- TrackingPerformance/Condor/condorJobs_reco.py | 103 ++++++++++-------- TrackingPerformance/Condor/condorJobs_sim.py | 31 +++--- TrackingPerformance/config.py | 15 ++- 3 files changed, 87 insertions(+), 62 deletions(-) diff --git a/TrackingPerformance/Condor/condorJobs_reco.py b/TrackingPerformance/Condor/condorJobs_reco.py index 579e4b5..3008a21 100644 --- a/TrackingPerformance/Condor/condorJobs_reco.py +++ b/TrackingPerformance/Condor/condorJobs_reco.py @@ -1,17 +1,18 @@ #!/usr/bin/env python -import os +from os import system # for execution at the end import sys import ROOT -import sys -from os.path import dirname, abspath, join +from pathlib import Path +# ========================== +# Import config +# ========================== # Add the project root directory to the sys.path -project_root = dirname(dirname(abspath(__file__))) +project_root = Path(__file__).resolve().parent.parent # Append the project root to sys.path if not already present if project_root not in sys.path: - sys.path.append(project_root) - + sys.path.append(str(project_root)) # import config import config @@ -24,9 +25,9 @@ particleList_ = config.particleList_ DetectorModelList_ = config.detectorModel -Nevts_ = "10000" +Nevts_ = config.Nevts_ -Nevt_per_job = "1000" # Set the desired number of events per job +Nevt_per_job = config.Nevt_per_job N_jobs = ( int(int(Nevts_) / int(Nevt_per_job)) * len(particleList_) @@ -39,20 +40,18 @@ # =========================== # Directory Setup and Checks # =========================== + +# Define environment setup path +environ_path = config.setup + # Define directories for input and output -directory_jobs = f"CondorJobs/Rec_{particleList_[0]}_{DetectorModelList_[0]}" -# setup = "/cvmfs/sw-nightlies.hsf.org/key4hep/setup.sh" # nightlies -setup = "/cvmfs/sw.hsf.org/key4hep/setup.sh" # stable +directory_jobs = config.RECcondorDir / f"{particleList_[0]}_{DetectorModelList_[0]}" # InputDirectory = f"/eos/user/g/gasadows/Output/TrackingPerformance/{DetectorModelList_[0]}/SIM/3T/" -InputDirectory = f"/eos/experiment/fcc/users/g/gasadows/TrackingPerformance/{DetectorModelList_[0]}/SIM/3T/" -EosDir = ( - f"/eos/user/g/gasadows/Output/TrackingPerformance/{DetectorModelList_[0]}/REC/3T/" -) +SIMEosDir = config.dataDir / f"{DetectorModelList_[0]}" / "SIM" # input +RECEosDir = config.dataDir / f"{DetectorModelList_[0]}" / "REC" # output # steering_file = "CLDReconstruction.py" -steering_file = ( - "/afs/cern.ch/user/g/gasadows/CLDConfig/CLDConfig/CLDReconstruction_3T.py" -) +steering_file = config.rec_steering_file # Enable output checks check_output = True # Set to True to enable checks, False to disable @@ -77,14 +76,15 @@ ResVDX_UV_ = ["0.003"] # Check if the directory exists and exit if it does -if os.path.exists(directory_jobs): +if directory_jobs.exists(): print( f"Error: Directory '{directory_jobs}' already exists and should not be overwritten." ) sys.exit(1) # Create output directories if they don't exist -[os.makedirs(directory, exist_ok=True) for directory in [EosDir, directory_jobs]] +RECEosDir.mkdir(parents=True, exist_ok=True) +directory_jobs.mkdir(parents=True, exist_ok=True) # ======================= # Simulation Job Creation @@ -101,29 +101,40 @@ for theta, momentum, part, dect in list_of_combined_variables: for task_index in range(num_jobs): - outputFileName = f"REC_{dect}" - outputFileName += f"_{part}" - outputFileName += f"_{theta}_deg" - outputFileName += f"_{momentum}_GeV" - outputFileName += f"_{Nevt_per_job}_evts" - outputFileName += f"_{task_index}" - - inputFile = os.path.join( - InputDirectory + f"/{part}", - f"SIM_{dect}_{part}_{theta}_deg_{momentum}_GeV_{Nevt_per_job}_evts_{task_index}_edm4hep.root", - ) - # inputFile= os.path.join(InputDirectory + f"/{part}", f"SIM_{dect}_{part}_{theta}_deg_{momentum}_GeV_{Nevt_per_job}_evts_edm4hep.root") - # input_file= os.path.join(InputDirectory, "SIMTest_" + dect + "_" + part + "_" + theta + "_deg_" + momentum + "_GeV_" + Nevts_ + "_evts.slcio") + output_file_name_parts = [ + f"REC_{dect}", + f"{part}", + f"{theta}_deg", + f"{momentum}_GeV", + f"{Nevt_per_job}_evts", + f"{task_index}", + ] + output_file_name = "_".join(output_file_name_parts) + + input_file_name_parts = [ + f"SIM_{dect}", + f"{part}", + f"{theta}_deg", + f"{momentum}_GeV", + f"{Nevt_per_job}_evts", + f"{task_index}", + ] + input_file_name = "_".join(input_file_name_parts) + input_file_path = Path(input_file_name).with_suffix(".edm4hep.root") + inputFile = ( + SIMEosDir / part / input_file_path + ) # FIXME: reasonable that part is twice in the path? # Check if the input file exists - if not os.path.exists(inputFile): + if not inputFile.exists(): print(f"Error: Input file {inputFile} does not exist. Skipping job.") continue # Check if the output file already exists and has correct Nb of events - output_dir = os.path.join(EosDir, part) - os.makedirs(output_dir, exist_ok=True) - output_file = output_dir + "/" + outputFileName + "_edm4hep.root" - if check_output and os.path.exists(output_file): + output_dir = RECEosDir / part + output_dir.mkdir(parents=True, exist_ok=True) + output_file = (output_dir / output_file_name).with_suffix("_edm4hep.root") + + if check_output and output_file.exists(): root_file = ROOT.TFile(output_file, "READ") events_tree = root_file.Get("events") if events_tree and events_tree.GetEntries() == int(Nevt_per_job): @@ -133,8 +144,8 @@ need_to_create_scripts = True # Create aida output Dir - output_dir_aida = os.path.join(output_dir, "aida_outputs") - os.makedirs(output_dir_aida, exist_ok=True) + output_dir_aida = output_dir / "aida_outputs" + output_dir_aida.mkdir(exist_ok=True) arguments = ( # f" --GeoSvc.detectors=/afs/cern.ch/work/g/gasadows/k4geo/FCCee/CLD/compact/{DetectorModelList_[0]}_3T/{DetectorModelList_[0]}.xml"+ @@ -142,7 +153,7 @@ + " --inputFiles " + inputFile + " --outputBasename " - + outputFileName + + output_file_name + f" --VXDDigitiserResUV={ResVDX_UV_[0]}" + " --trackingOnly" + " -n " @@ -153,12 +164,12 @@ # Write bash script for job execution bash_script = ( "#!/bin/bash \n" - f"source {setup} \n" + f"source {environ_path} \n" "git clone https://github.com/gaswk/CLDConfig.git \n" "cd " + "CLDConfig/CLDConfig" + "\n" f"{command} \n" - f"xrdcp {outputFileName}_edm4hep.root root://eosuser.cern.ch/{output_dir} \n" - f"xrdcp {outputFileName}_aida.root root://eosuser.cern.ch/{output_dir_aida} \n" + f"xrdcp {output_file_name}_edm4hep.root root://eosuser.cern.ch/{output_dir} \n" + f"xrdcp {output_file_name}_aida.root root://eosuser.cern.ch/{output_dir_aida} \n" ) bash_file = ( directory_jobs @@ -193,4 +204,6 @@ # ==================== # Submit Job to Condor # ==================== -os.system("cd " + directory_jobs + "; condor_submit condor_script.sub") +system( + "cd " + str(directory_jobs) + "; condor_submit condor_script.sub" +) # FIXME: use subprocess instead? diff --git a/TrackingPerformance/Condor/condorJobs_sim.py b/TrackingPerformance/Condor/condorJobs_sim.py index cb36341..01c10f2 100644 --- a/TrackingPerformance/Condor/condorJobs_sim.py +++ b/TrackingPerformance/Condor/condorJobs_sim.py @@ -1,6 +1,6 @@ #!/usr/bin/env python -import os +from os import system # for execution at the end import sys import ROOT import time @@ -8,13 +8,14 @@ verbose = False - +# ========================== +# Import config +# ========================== # Add the project root directory to the sys.path project_root = Path(__file__).resolve().parent.parent # Append the project root to sys.path if not already present if project_root not in sys.path: sys.path.append(str(project_root)) - # import config import config @@ -44,11 +45,13 @@ # =========================== # Directory Setup and Checks # =========================== + +# Define environment setup path +environ_path = config.setup + # Define directories for input and output -directory_jobs = config.SIMcondorDir / f"mu_{DetectorModelList_[0]}" -# setup = "/cvmfs/sw-nightlies.hsf.org/key4hep/setup.sh" # nightlies -setup = "/cvmfs/sw.hsf.org/key4hep/setup.sh" # stable -outputEosDir = config.dataDir / f"{DetectorModelList_[0]}/SIM" +directory_jobs = config.SIMcondorDir / f"{particleList_[0]}_{DetectorModelList_[0]}" +SIMEosDir = config.dataDir / f"{DetectorModelList_[0]}" / "SIM" # output # Enable output checks check_output = True # Set to True to enable checks, False to disable @@ -75,7 +78,7 @@ ) sys.exit(1) -outputEosDir.mkdir( +SIMEosDir.mkdir( parents=True, exist_ok=True ) # This will create the directory if it doesn't exist, without raising an error if it does @@ -104,18 +107,18 @@ f"{momentum}_GeV", f"{Nevt_per_job}_evts", f"{task_index}", - "edm4hep.root", ] output_file_name = "_".join(output_file_name_parts) + output_file_path = Path(output_file_name).with_suffix(".edm4hep.root") # Check if the output file already exists and has correct Nb of events - output_dir = outputEosDir / part + output_dir = SIMEosDir / part / output_file_path output_dir.mkdir(parents=True, exist_ok=True) output_file = output_dir / output_file_name if check_output and output_file.exists(): root_file = ROOT.TFile(output_file, "READ") events_tree = root_file.Get("events") - if events_tree: + if events_tree: # FIXME: why no else? if events_tree.GetEntries() == int(Nevt_per_job): root_file.Close() continue @@ -151,7 +154,7 @@ # Write bash script for job execution bash_script = ( "#!/bin/bash \n" - f"source {setup} \n" + f"source {environ_path} \n" "git clone https://github.com/Victor-Schwan/TrackingStudies.git \n" f"{command} \n" f"xrdcp {output_file_name} root://eosuser.cern.ch/{output_dir} \n" @@ -201,4 +204,6 @@ # ==================== # Submit Job to Condor # ==================== -# os.system("cd " + directory_jobs + "; condor_submit condor_script.sub") +system( + "cd " + str(directory_jobs) + "; condor_submit condor_script.sub" +) # FIXME: use subprocess instead? diff --git a/TrackingPerformance/config.py b/TrackingPerformance/config.py index 9a29d21..b647222 100644 --- a/TrackingPerformance/config.py +++ b/TrackingPerformance/config.py @@ -1,13 +1,20 @@ from pathlib import Path +# define environment setup script path +stable = Path("/cvmfs/sw.hsf.org/key4hep/setup.sh") +nightlies = Path("/cvmfs/sw-nightlies.hsf.org/key4hep/setup.sh") +setup = nightlies # choose either stable or nightlies + # define base directory baseDir = Path("/eos/home-v/vschwan/promotion/") -# define directory to store output + +# define directory to store output and subdirs dataDir = baseDir / "data" SIMcondorDir = dataDir / "sim" / "condor_jobs" -sim_steering_file = ( - baseDir / "ILDConfig" / "StandardConfig" / "production" / "TPC_debug_muon_steer.py" -) +RECcondorDir = dataDir / "rec" / "condor_jobs" +ILDDir = baseDir / "ILDConfig" / "StandardConfig" / "production" +sim_steering_file = ILDDir / "TPC_debug_muon_steer.py" +rec_steering_file = ILDDir / "ILDReconstruction.py" # ========================== # Job Parameters Initialisation From 389170dc108354c07bb2061b46e77be647e0fcdc Mon Sep 17 00:00:00 2001 From: Victor Schwan Date: Tue, 21 May 2024 15:56:49 +0200 Subject: [PATCH 05/33] time stuff removed; wait slowed down the execution significantly, maybe perf_counter_ns() is better without wait --- TrackingPerformance/Condor/condorJobs_sim.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/TrackingPerformance/Condor/condorJobs_sim.py b/TrackingPerformance/Condor/condorJobs_sim.py index 01c10f2..121bfa1 100644 --- a/TrackingPerformance/Condor/condorJobs_sim.py +++ b/TrackingPerformance/Condor/condorJobs_sim.py @@ -3,7 +3,6 @@ from os import system # for execution at the end import sys import ROOT -import time from pathlib import Path verbose = False @@ -19,8 +18,6 @@ # import config import config -ts = time.time() # Get current timestamp for unique identifiers - # ========================== # Parameters Initialisation # ========================== @@ -126,9 +123,6 @@ else: need_to_create_scripts = True - time.sleep(1) - seed = str(time.time() % 1000) - # if len(DetectorModelList_) != 1 or DetectorModelList_[0] != "ILD_l5_v11": # raise ValueError("so far only ILD_l5_v11 possible") @@ -139,7 +133,6 @@ f"--outputFile {output_file_name}", f"--steeringFile {config.sim_steering_file}", # "CLDConfig/CLDConfig/cld_steer.py " # TODO: FIX - # f"--random.seed {seed}", # "--enableGun", # f"--gun.particle {part}-", # f"--gun.energy {momentum}*GeV", From fee177125d693c86cd35846016d75927e51b5399 Mon Sep 17 00:00:00 2001 From: Victor Schwan Date: Tue, 21 May 2024 16:23:17 +0200 Subject: [PATCH 06/33] job number calc in verbose mode fixed --- TrackingPerformance/Condor/condorJobs_sim.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/TrackingPerformance/Condor/condorJobs_sim.py b/TrackingPerformance/Condor/condorJobs_sim.py index 121bfa1..76eec3e 100644 --- a/TrackingPerformance/Condor/condorJobs_sim.py +++ b/TrackingPerformance/Condor/condorJobs_sim.py @@ -5,7 +5,7 @@ import ROOT from pathlib import Path -verbose = False +verbose = True # ========================== # Import config @@ -32,12 +32,13 @@ total_events = int(Nevts_) -num_jobs_per_para_set = total_events // int( +N_para_sets = ( + len(DetectorModelList_) * len(particleList_) * len(thetaList_) * len(momentumList_) +) +N_jobs_per_para_set = total_events // int( Nevt_per_job ) # number of parallel jobs with same parameter combination/set -N_jobs = ( - num_jobs_per_para_set * len(particleList_) * len(thetaList_) * len(momentumList_) -) # total number of jobs +N_jobs = N_jobs_per_para_set * N_para_sets # total number of jobs # =========================== # Directory Setup and Checks @@ -95,7 +96,7 @@ if verbose: print(f"1st of {N_jobs} different parameter combinations starts") for counter, (theta, momentum, part, dect) in enumerate(iter_of_combined_variables): - for task_index in range(num_jobs_per_para_set): + for task_index in range(N_jobs_per_para_set): output_file_name_parts = [ f"SIM_{dect}", @@ -167,10 +168,10 @@ file.close() if verbose: - print(f" {task_index+1} of {num_jobs_per_para_set} parallel jobs done") + print(f" {task_index+1} of {N_jobs_per_para_set} parallel jobs done") if verbose: - print(f"{counter+1} of {N_jobs} different parameter combinations done") + print(f"{counter+1} of {N_para_sets} different parameter combinations done") if not need_to_create_scripts: print("All output files are correct.") From ddc294b84ce64077eeb15ad559a5497da50f04ae Mon Sep 17 00:00:00 2001 From: Victor Schwan Date: Tue, 21 May 2024 16:26:03 +0200 Subject: [PATCH 07/33] verbose mode for debugging of slow loop removed --- TrackingPerformance/Condor/condorJobs_sim.py | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/TrackingPerformance/Condor/condorJobs_sim.py b/TrackingPerformance/Condor/condorJobs_sim.py index 76eec3e..07cd7f2 100644 --- a/TrackingPerformance/Condor/condorJobs_sim.py +++ b/TrackingPerformance/Condor/condorJobs_sim.py @@ -5,8 +5,6 @@ import ROOT from pathlib import Path -verbose = True - # ========================== # Import config # ========================== @@ -93,8 +91,7 @@ need_to_create_scripts = False -if verbose: - print(f"1st of {N_jobs} different parameter combinations starts") + for counter, (theta, momentum, part, dect) in enumerate(iter_of_combined_variables): for task_index in range(N_jobs_per_para_set): @@ -167,11 +164,6 @@ file.write(bash_script) file.close() - if verbose: - print(f" {task_index+1} of {N_jobs_per_para_set} parallel jobs done") - - if verbose: - print(f"{counter+1} of {N_para_sets} different parameter combinations done") if not need_to_create_scripts: print("All output files are correct.") From 5d8fa2225b9801471316835d0d1f95a1225a1188 Mon Sep 17 00:00:00 2001 From: Victor Schwan Date: Mon, 27 May 2024 19:08:42 +0200 Subject: [PATCH 08/33] minor refactoring Condor/condorJobs_sim.py --- TrackingPerformance/Condor/condorJobs_sim.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/TrackingPerformance/Condor/condorJobs_sim.py b/TrackingPerformance/Condor/condorJobs_sim.py index 07cd7f2..60bb92e 100644 --- a/TrackingPerformance/Condor/condorJobs_sim.py +++ b/TrackingPerformance/Condor/condorJobs_sim.py @@ -1,10 +1,11 @@ #!/usr/bin/env python -from os import system # for execution at the end import sys -import ROOT +from os import system # for execution at the end from pathlib import Path +import ROOT + # ========================== # Import config # ========================== @@ -50,11 +51,11 @@ SIMEosDir = config.dataDir / f"{DetectorModelList_[0]}" / "SIM" # output # Enable output checks -check_output = True # Set to True to enable checks, False to disable +CHECK_OUTPUT = True # Set to True to enable checks, False to disable # It will check if the ouputs exist and contain correct number of events # if not it will send job to rerun simulation -JobFlavour = "testmatch" +JOB_FLAVOR = "testmatch" # Job flavours: # espresso = 20 minutes # microcentury = 1 hour @@ -110,7 +111,7 @@ output_dir = SIMEosDir / part / output_file_path output_dir.mkdir(parents=True, exist_ok=True) output_file = output_dir / output_file_name - if check_output and output_file.exists(): + if CHECK_OUTPUT and output_file.exists(): root_file = ROOT.TFile(output_file, "READ") events_tree = root_file.Get("events") if events_tree: # FIXME: why no else? @@ -160,7 +161,7 @@ ] bash_file = (directory_jobs / "_".join(bash_file_name_parts)).with_suffix(".sh") - with open(bash_file, "w") as file: + with open(bash_file, "w", encoding="utf-8") as file: file.write(bash_script) file.close() @@ -179,11 +180,11 @@ "output = output.$(ClusterId).$(ProcId).out \n" "error = error.$(ClusterId).$(ProcId).err \n" "log = log.$(ClusterId).log \n" - f'+JobFlavour = "{JobFlavour}" \n' + f'+JobFlavour = "{JOB_FLAVOR}" \n' "queue filename matching files *.sh \n" ) condor_file = directory_jobs / "condor_script.sub" -with open(condor_file, "w") as file2: +with open(condor_file, "w", encoding="utf-8") as file2: file2.write(condor_script) file2.close() From c293833238fe0f31f3d78ae69e9533b63f050c4a Mon Sep 17 00:00:00 2001 From: Victor Schwan Date: Thu, 30 May 2024 13:04:20 +0200 Subject: [PATCH 09/33] minor --- TrackingPerformance/Condor/condorJobs_reco.py | 7 ++++--- TrackingPerformance/Condor/condorJobs_sim.py | 2 +- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/TrackingPerformance/Condor/condorJobs_reco.py b/TrackingPerformance/Condor/condorJobs_reco.py index 3008a21..88d693f 100644 --- a/TrackingPerformance/Condor/condorJobs_reco.py +++ b/TrackingPerformance/Condor/condorJobs_reco.py @@ -1,10 +1,11 @@ #!/usr/bin/env python -from os import system # for execution at the end import sys -import ROOT +from os import system # for execution at the end from pathlib import Path +import ROOT + # ========================== # Import config # ========================== @@ -165,7 +166,7 @@ bash_script = ( "#!/bin/bash \n" f"source {environ_path} \n" - "git clone https://github.com/gaswk/CLDConfig.git \n" + "git clone https://github.com/gaswk/CLDConfig.git \n" # FIXME: new repo location in k4hep "cd " + "CLDConfig/CLDConfig" + "\n" f"{command} \n" f"xrdcp {output_file_name}_edm4hep.root root://eosuser.cern.ch/{output_dir} \n" diff --git a/TrackingPerformance/Condor/condorJobs_sim.py b/TrackingPerformance/Condor/condorJobs_sim.py index 60bb92e..81abd40 100644 --- a/TrackingPerformance/Condor/condorJobs_sim.py +++ b/TrackingPerformance/Condor/condorJobs_sim.py @@ -147,7 +147,7 @@ bash_script = ( "#!/bin/bash \n" f"source {environ_path} \n" - "git clone https://github.com/Victor-Schwan/TrackingStudies.git \n" + "git clone https://github.com/Victor-Schwan/TrackingStudies.git \n" # FIXME: new repo location f"{command} \n" f"xrdcp {output_file_name} root://eosuser.cern.ch/{output_dir} \n" ) From 51dbf5482bf979794fc9b0dee146a56c2bbfbeca Mon Sep 17 00:00:00 2001 From: Victor Schwan Date: Thu, 30 May 2024 13:36:40 +0200 Subject: [PATCH 10/33] added afs paths to config --- TrackingPerformance/config.py | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/TrackingPerformance/config.py b/TrackingPerformance/config.py index b647222..3c4954d 100644 --- a/TrackingPerformance/config.py +++ b/TrackingPerformance/config.py @@ -5,16 +5,23 @@ nightlies = Path("/cvmfs/sw-nightlies.hsf.org/key4hep/setup.sh") setup = nightlies # choose either stable or nightlies -# define base directory -baseDir = Path("/eos/home-v/vschwan/promotion/") +# ========================== +# define base directories +# ========================== + +# those files are available to job +baseAFSDir = Path("/afs/cern.ch/user/") / "v/vschwan/promotion" +# not directly available to job, only for storing purposes +baseEOSDir = Path("/eos/") / "home-v/vschwan/promotion/" # define directory to store output and subdirs -dataDir = baseDir / "data" +dataDir = baseEOSDir / "data" SIMcondorDir = dataDir / "sim" / "condor_jobs" RECcondorDir = dataDir / "rec" / "condor_jobs" -ILDDir = baseDir / "ILDConfig" / "StandardConfig" / "production" -sim_steering_file = ILDDir / "TPC_debug_muon_steer.py" -rec_steering_file = ILDDir / "ILDReconstruction.py" +# detector specific +detectorDIR = baseAFSDir / "ILDConfig" / "StandardConfig" / "production" +sim_steering_file = detectorDIR / "TPC_debug_muon_steer.py" +rec_steering_file = detectorDIR / "ILDReconstruction.py" # ========================== # Job Parameters Initialisation From 7548679ce6fd96c9bb5710cf1447b358f1af66d0 Mon Sep 17 00:00:00 2001 From: Victor Schwan Date: Fri, 31 May 2024 11:07:56 +0200 Subject: [PATCH 11/33] mostly chhanges to paths --- TrackingPerformance/Condor/condorJobs_reco.py | 2 +- TrackingPerformance/Condor/condorJobs_sim.py | 35 ++++++++++++++----- TrackingPerformance/config.py | 17 +++++---- 3 files changed, 38 insertions(+), 16 deletions(-) diff --git a/TrackingPerformance/Condor/condorJobs_reco.py b/TrackingPerformance/Condor/condorJobs_reco.py index 88d693f..bea2037 100644 --- a/TrackingPerformance/Condor/condorJobs_reco.py +++ b/TrackingPerformance/Condor/condorJobs_reco.py @@ -166,7 +166,7 @@ bash_script = ( "#!/bin/bash \n" f"source {environ_path} \n" - "git clone https://github.com/gaswk/CLDConfig.git \n" # FIXME: new repo location in k4hep + "git clone https://github.com/gaswk/CLDConfig.git \n" # FIXME: new repo location to config (to switch easily to local fork) "cd " + "CLDConfig/CLDConfig" + "\n" f"{command} \n" f"xrdcp {output_file_name}_edm4hep.root root://eosuser.cern.ch/{output_dir} \n" diff --git a/TrackingPerformance/Condor/condorJobs_sim.py b/TrackingPerformance/Condor/condorJobs_sim.py index 81abd40..e8389b5 100644 --- a/TrackingPerformance/Condor/condorJobs_sim.py +++ b/TrackingPerformance/Condor/condorJobs_sim.py @@ -9,6 +9,7 @@ # ========================== # Import config # ========================== + # Add the project root directory to the sys.path project_root = Path(__file__).resolve().parent.parent # Append the project root to sys.path if not already present @@ -17,9 +18,23 @@ # import config import config +# ========================== +# Check paths +# ========================== + +assert ( + config.sim_steering_file.exists() +), f"The file {config.sim_steering_file} does not exist" +assert ( + config.rec_steering_file.exists() +), f"The file {config.rec_steering_file} does not exist" +assert config.detectorDIR.exists(), f"The folder {config.detectorDIR} does not exist" + + # ========================== # Parameters Initialisation # ========================== + # Define lists of parameters for reconstruction thetaList_ = config.thetaList_ momentumList_ = config.momentumList_ @@ -48,7 +63,7 @@ # Define directories for input and output directory_jobs = config.SIMcondorDir / f"{particleList_[0]}_{DetectorModelList_[0]}" -SIMEosDir = config.dataDir / f"{DetectorModelList_[0]}" / "SIM" # output +SIMEOSDir = config.dataDir / f"{DetectorModelList_[0]}" / "SIM" # output # Enable output checks CHECK_OUTPUT = True # Set to True to enable checks, False to disable @@ -75,7 +90,7 @@ ) sys.exit(1) -SIMEosDir.mkdir( +SIMEOSDir.mkdir( parents=True, exist_ok=True ) # This will create the directory if it doesn't exist, without raising an error if it does @@ -83,6 +98,7 @@ # ======================= # Simulation Job Creation # ======================= + # Create all possible combinations import itertools @@ -104,15 +120,16 @@ f"{Nevt_per_job}_evts", f"{task_index}", ] - output_file_name = "_".join(output_file_name_parts) - output_file_path = Path(output_file_name).with_suffix(".edm4hep.root") + output_file_name = Path("_".join(output_file_name_parts)).with_suffix( + ".edm4hep.root" + ) # Check if the output file already exists and has correct Nb of events - output_dir = SIMEosDir / part / output_file_path + output_dir = SIMEOSDir / part output_dir.mkdir(parents=True, exist_ok=True) - output_file = output_dir / output_file_name - if CHECK_OUTPUT and output_file.exists(): - root_file = ROOT.TFile(output_file, "READ") + output_file_path = output_dir / output_file_name + if CHECK_OUTPUT and output_file_path.exists(): + root_file = ROOT.TFile(output_file_path, "READ") events_tree = root_file.Get("events") if events_tree: # FIXME: why no else? if events_tree.GetEntries() == int(Nevt_per_job): @@ -147,7 +164,7 @@ bash_script = ( "#!/bin/bash \n" f"source {environ_path} \n" - "git clone https://github.com/Victor-Schwan/TrackingStudies.git \n" # FIXME: new repo location + "git clone https://github.com/Victor-Schwan/TrackingStudies.git \n" # FIXME: new repo location to config (to switch easily to local fork) f"{command} \n" f"xrdcp {output_file_name} root://eosuser.cern.ch/{output_dir} \n" ) diff --git a/TrackingPerformance/config.py b/TrackingPerformance/config.py index 3c4954d..4a12474 100644 --- a/TrackingPerformance/config.py +++ b/TrackingPerformance/config.py @@ -5,6 +5,7 @@ nightlies = Path("/cvmfs/sw-nightlies.hsf.org/key4hep/setup.sh") setup = nightlies # choose either stable or nightlies + # ========================== # define base directories # ========================== @@ -12,23 +13,27 @@ # those files are available to job baseAFSDir = Path("/afs/cern.ch/user/") / "v/vschwan/promotion" # not directly available to job, only for storing purposes -baseEOSDir = Path("/eos/") / "home-v/vschwan/promotion/" +baseEOSDir = Path("/eos/") / "home-v/vschwan/promotion" -# define directory to store output and subdirs +# define directory to store output dataDir = baseEOSDir / "data" -SIMcondorDir = dataDir / "sim" / "condor_jobs" -RECcondorDir = dataDir / "rec" / "condor_jobs" +# define dirs +SIMcondorDir = baseAFSDir / "sim" / "condor_jobs" +RECcondorDir = baseAFSDir / "rec" / "condor_jobs" # detector specific +# FIXME: extract following from dict based on detectorModel var? detectorDIR = baseAFSDir / "ILDConfig" / "StandardConfig" / "production" sim_steering_file = detectorDIR / "TPC_debug_muon_steer.py" rec_steering_file = detectorDIR / "ILDReconstruction.py" + # ========================== # Job Parameters Initialisation # ========================== -Nevts_ = "300" -Nevt_per_job = "100" # Set the desired number of events per job +Nevts_ = "1" +Nevt_per_job = "1" # Set the desired number of events per job + # ========================== # Parameters Initialisation From e41ee39e547dec30c515d31ef225dffb5f615b87 Mon Sep 17 00:00:00 2001 From: Victor Schwan Date: Fri, 31 May 2024 13:40:47 +0200 Subject: [PATCH 12/33] condorJobs_sim: argparse to choose config file, introduced functions; cld_config added for cld specific variables --- TrackingPerformance/Condor/condorJobs_sim.py | 413 ++++++++++--------- TrackingPerformance/cld_config.py | 47 +++ TrackingPerformance/config.py | 2 +- 3 files changed, 265 insertions(+), 197 deletions(-) create mode 100644 TrackingPerformance/cld_config.py diff --git a/TrackingPerformance/Condor/condorJobs_sim.py b/TrackingPerformance/Condor/condorJobs_sim.py index e8389b5..f2d99db 100644 --- a/TrackingPerformance/Condor/condorJobs_sim.py +++ b/TrackingPerformance/Condor/condorJobs_sim.py @@ -1,213 +1,234 @@ #!/usr/bin/env python +import argparse +import importlib.util import sys from os import system # for execution at the end from pathlib import Path import ROOT -# ========================== -# Import config -# ========================== - -# Add the project root directory to the sys.path -project_root = Path(__file__).resolve().parent.parent -# Append the project root to sys.path if not already present -if project_root not in sys.path: - sys.path.append(str(project_root)) -# import config -import config - -# ========================== -# Check paths -# ========================== - -assert ( - config.sim_steering_file.exists() -), f"The file {config.sim_steering_file} does not exist" -assert ( - config.rec_steering_file.exists() -), f"The file {config.rec_steering_file} does not exist" -assert config.detectorDIR.exists(), f"The folder {config.detectorDIR} does not exist" - - -# ========================== -# Parameters Initialisation -# ========================== - -# Define lists of parameters for reconstruction -thetaList_ = config.thetaList_ -momentumList_ = config.momentumList_ -particleList_ = config.particleList_ - -DetectorModelList_ = config.detectorModel -Nevts_ = config.Nevts_ -Nevt_per_job = config.Nevt_per_job # Set the desired number of events per job - - -total_events = int(Nevts_) -N_para_sets = ( - len(DetectorModelList_) * len(particleList_) * len(thetaList_) * len(momentumList_) -) -N_jobs_per_para_set = total_events // int( - Nevt_per_job -) # number of parallel jobs with same parameter combination/set -N_jobs = N_jobs_per_para_set * N_para_sets # total number of jobs - -# =========================== -# Directory Setup and Checks -# =========================== - -# Define environment setup path -environ_path = config.setup - -# Define directories for input and output -directory_jobs = config.SIMcondorDir / f"{particleList_[0]}_{DetectorModelList_[0]}" -SIMEOSDir = config.dataDir / f"{DetectorModelList_[0]}" / "SIM" # output - -# Enable output checks -CHECK_OUTPUT = True # Set to True to enable checks, False to disable -# It will check if the ouputs exist and contain correct number of events -# if not it will send job to rerun simulation - -JOB_FLAVOR = "testmatch" -# Job flavours: -# espresso = 20 minutes -# microcentury = 1 hour -# longlunch = 2 hours -# workday = 8 hours -# tomorrow = 1 day -# testmatch = 3 days -# nextweek = 1 week - - -# Check if the directory exists and exit if it does -try: - directory_jobs.mkdir(parents=True, exist_ok=False) -except FileExistsError: - print( - f"Error: Directory '{directory_jobs}' already exists and should not be overwritten." - ) - sys.exit(1) - -SIMEOSDir.mkdir( - parents=True, exist_ok=True -) # This will create the directory if it doesn't exist, without raising an error if it does +def parse_args(): + parser = argparse.ArgumentParser( + description="Script to run with user configuration." + ) + parser.add_argument( + "--config", + type=Path, + required=True, + help="Path to the configuration file.", + ) + return parser.parse_args() + + +def load_config(config_path): + spec = importlib.util.spec_from_file_location("config", config_path) + config = importlib.util.module_from_spec(spec) + spec.loader.exec_module(config) + return config + + +def main(): + + args = parse_args() + config = load_config(args.config) + + # ========================== + # Check paths + # ========================== + + assert ( + config.sim_steering_file.exists() + ), f"The file {config.sim_steering_file} does not exist" + assert ( + config.rec_steering_file.exists() + ), f"The file {config.rec_steering_file} does not exist" + assert ( + config.detectorDIR.exists() + ), f"The folder {config.detectorDIR} does not exist" + + # ========================== + # Parameters Initialisation + # ========================== + + # Define lists of parameters for reconstruction + thetaList_ = config.thetaList_ + momentumList_ = config.momentumList_ + particleList_ = config.particleList_ + + DetectorModelList_ = config.detectorModel + Nevts_ = config.Nevts_ + Nevt_per_job = config.Nevt_per_job # Set the desired number of events per job + + total_events = int(Nevts_) + N_para_sets = ( + len(DetectorModelList_) + * len(particleList_) + * len(thetaList_) + * len(momentumList_) + ) + N_jobs_per_para_set = total_events // int( + Nevt_per_job + ) # number of parallel jobs with same parameter combination/set + N_jobs = N_jobs_per_para_set * N_para_sets # total number of jobs + + # =========================== + # Directory Setup and Checks + # =========================== + + # Define environment setup path + environ_path = config.setup + + # Define directories for input and output + directory_jobs = config.SIMcondorDir / f"{particleList_[0]}_{DetectorModelList_[0]}" + SIMEOSDir = config.dataDir / f"{DetectorModelList_[0]}" / "SIM" # output + + # Enable output checks + CHECK_OUTPUT = True # Set to True to enable checks, False to disable + # It will check if the ouputs exist and contain correct number of events + # if not it will send job to rerun simulation + + JOB_FLAVOR = "testmatch" + # Job flavours: + # espresso = 20 minutes + # microcentury = 1 hour + # longlunch = 2 hours + # workday = 8 hours + # tomorrow = 1 day + # testmatch = 3 days + # nextweek = 1 week + + # Check if the directory exists and exit if it does + try: + directory_jobs.mkdir(parents=True, exist_ok=False) + except FileExistsError: + print( + f"Error: Directory '{directory_jobs}' already exists and should not be overwritten." + ) + sys.exit(1) -# ======================= -# Simulation Job Creation -# ======================= + SIMEOSDir.mkdir( + parents=True, exist_ok=True + ) # This will create the directory if it doesn't exist, without raising an error if it does -# Create all possible combinations -import itertools + # ======================= + # Simulation Job Creation + # ======================= -iter_of_combined_variables = itertools.product( - thetaList_, momentumList_, particleList_, DetectorModelList_ -) + # Create all possible combinations + import itertools -need_to_create_scripts = False + iter_of_combined_variables = itertools.product( + thetaList_, momentumList_, particleList_, DetectorModelList_ + ) + need_to_create_scripts = False + + for theta, momentum, part, dect in iter_of_combined_variables: + for task_index in range(N_jobs_per_para_set): + + output_file_name_parts = [ + f"SIM_{dect}", + f"{part}", + f"{theta}_deg", + f"{momentum}_GeV", + f"{Nevt_per_job}_evts", + f"{task_index}", + ] + output_file_name = Path("_".join(output_file_name_parts)).with_suffix( + ".edm4hep.root" + ) + + # Check if the output file already exists and has correct Nb of events + output_dir = SIMEOSDir / part + output_dir.mkdir(parents=True, exist_ok=True) + output_file_path = output_dir / output_file_name + if CHECK_OUTPUT and output_file_path.exists(): + root_file = ROOT.TFile(output_file_path, "READ") + events_tree = root_file.Get("events") + if events_tree: # FIXME: why no else? + if events_tree.GetEntries() == int(Nevt_per_job): + root_file.Close() + continue + root_file.Close() + else: + need_to_create_scripts = True + + # if len(DetectorModelList_) != 1 or DetectorModelList_[0] != "ILD_l5_v11": + # raise ValueError("so far only ILD_l5_v11 possible") + + # Build ddsim command + arguments = [ + # f" --compactFile /afs/cern.ch/work/g/gasadows/k4geo/FCCee/CLD/compact/{DetectorModelList_[0]}_3T/{DetectorModelList_[0]}.xml " + f" --compactFile $k4geo_DIR/{config.detModPaths['ILD_l5_v11']}", # Note the change to use double quotes for the dictionary key + f"--outputFile {output_file_name}", + f"--steeringFile {config.sim_steering_file}", # "CLDConfig/CLDConfig/cld_steer.py " + # TODO: FIX + # "--enableGun", + # f"--gun.particle {part}-", + # f"--gun.energy {momentum}*GeV", + # "--gun.distribution uniform", + # f"--gun.thetaMin {theta}*deg", + # f"--gun.thetaMax {theta}*deg", + # "--crossingAngleBoost 0", + # f"--numberOfEvents {Nevt_per_job}", + ] + command = f"ddsim {' '.join(arguments)} > /dev/null" + + # Write bash script for job execution + bash_script = ( + "#!/bin/bash \n" + f"source {environ_path} \n" + "git clone https://github.com/Victor-Schwan/TrackingStudies.git \n" # FIXME: new repo location to config (to switch easily to local fork) + f"{command} \n" + f"xrdcp {output_file_name} root://eosuser.cern.ch/{output_dir} \n" + ) + bash_file_name_parts = [ + "bash_script", + dect, + part, + f"{theta}_deg", + f"{momentum}_GeV", + str(task_index), + ] + bash_file = (directory_jobs / "_".join(bash_file_name_parts)).with_suffix( + ".sh" + ) + + with open(bash_file, "w", encoding="utf-8") as b_file: + b_file.write(bash_script) + b_file.close() + + if not need_to_create_scripts: + print("All output files are correct.") + sys.exit(0) + + # ============================ + # Condor Submission Script + # ============================ + # Write the condor submission script + condor_script = ( + "executable = $(filename) \n" + "arguments = $(ClusterId) $(ProcId) \n" + "output = output.$(ClusterId).$(ProcId).out \n" + "error = error.$(ClusterId).$(ProcId).err \n" + "log = log.$(ClusterId).log \n" + f'+JobFlavour = "{JOB_FLAVOR}" \n' + "queue filename matching files *.sh \n" + ) + condor_file = directory_jobs / "condor_script.sub" + with open(condor_file, "w", encoding="utf-8") as c_file: + c_file.write(condor_script) + c_file.close() -for counter, (theta, momentum, part, dect) in enumerate(iter_of_combined_variables): - for task_index in range(N_jobs_per_para_set): + # ==================== + # Submit Job to Condor + # ==================== + system( + "cd " + str(directory_jobs) + "; condor_submit condor_script.sub" + ) # FIXME: use subprocess instead? - output_file_name_parts = [ - f"SIM_{dect}", - f"{part}", - f"{theta}_deg", - f"{momentum}_GeV", - f"{Nevt_per_job}_evts", - f"{task_index}", - ] - output_file_name = Path("_".join(output_file_name_parts)).with_suffix( - ".edm4hep.root" - ) - # Check if the output file already exists and has correct Nb of events - output_dir = SIMEOSDir / part - output_dir.mkdir(parents=True, exist_ok=True) - output_file_path = output_dir / output_file_name - if CHECK_OUTPUT and output_file_path.exists(): - root_file = ROOT.TFile(output_file_path, "READ") - events_tree = root_file.Get("events") - if events_tree: # FIXME: why no else? - if events_tree.GetEntries() == int(Nevt_per_job): - root_file.Close() - continue - root_file.Close() - else: - need_to_create_scripts = True - - # if len(DetectorModelList_) != 1 or DetectorModelList_[0] != "ILD_l5_v11": - # raise ValueError("so far only ILD_l5_v11 possible") - - # Build ddsim command - arguments = [ - # f" --compactFile /afs/cern.ch/work/g/gasadows/k4geo/FCCee/CLD/compact/{DetectorModelList_[0]}_3T/{DetectorModelList_[0]}.xml " - f" --compactFile $k4geo_DIR/{config.detModPaths['ILD_l5_v11']}", # Note the change to use double quotes for the dictionary key - f"--outputFile {output_file_name}", - f"--steeringFile {config.sim_steering_file}", # "CLDConfig/CLDConfig/cld_steer.py " - # TODO: FIX - # "--enableGun", - # f"--gun.particle {part}-", - # f"--gun.energy {momentum}*GeV", - # "--gun.distribution uniform", - # f"--gun.thetaMin {theta}*deg", - # f"--gun.thetaMax {theta}*deg", - # "--crossingAngleBoost 0", - # f"--numberOfEvents {Nevt_per_job}", - ] - command = f"ddsim {' '.join(arguments)} > /dev/null" - - # Write bash script for job execution - bash_script = ( - "#!/bin/bash \n" - f"source {environ_path} \n" - "git clone https://github.com/Victor-Schwan/TrackingStudies.git \n" # FIXME: new repo location to config (to switch easily to local fork) - f"{command} \n" - f"xrdcp {output_file_name} root://eosuser.cern.ch/{output_dir} \n" - ) - bash_file_name_parts = [ - "bash_script", - dect, - part, - f"{theta}_deg", - f"{momentum}_GeV", - str(task_index), - ] - bash_file = (directory_jobs / "_".join(bash_file_name_parts)).with_suffix(".sh") - - with open(bash_file, "w", encoding="utf-8") as file: - file.write(bash_script) - file.close() - - -if not need_to_create_scripts: - print("All output files are correct.") - sys.exit(0) - -# ============================ -# Condor Submission Script -# ============================ -# Write the condor submission script -condor_script = ( - "executable = $(filename) \n" - "arguments = $(ClusterId) $(ProcId) \n" - "output = output.$(ClusterId).$(ProcId).out \n" - "error = error.$(ClusterId).$(ProcId).err \n" - "log = log.$(ClusterId).log \n" - f'+JobFlavour = "{JOB_FLAVOR}" \n' - "queue filename matching files *.sh \n" -) -condor_file = directory_jobs / "condor_script.sub" -with open(condor_file, "w", encoding="utf-8") as file2: - file2.write(condor_script) - file2.close() - -# ==================== -# Submit Job to Condor -# ==================== -system( - "cd " + str(directory_jobs) + "; condor_submit condor_script.sub" -) # FIXME: use subprocess instead? +if __name__ == "__main__": + main() diff --git a/TrackingPerformance/cld_config.py b/TrackingPerformance/cld_config.py new file mode 100644 index 0000000..aa9dadb --- /dev/null +++ b/TrackingPerformance/cld_config.py @@ -0,0 +1,47 @@ +from pathlib import Path + +# define environment setup script path +stable = Path("/cvmfs/sw.hsf.org/key4hep/setup.sh") +nightlies = Path("/cvmfs/sw-nightlies.hsf.org/key4hep/setup.sh") +setup = nightlies # choose either stable or nightlies + + +# ========================== +# define base directories +# ========================== + +# those files are available to job +baseAFSDir = Path("/afs/cern.ch/user/") / "U/USER/CHANGE/PATH" # FIXME +# not directly available to job, only for storing purposes +baseEOSDir = Path("/eos/") / "HOME-U/USER/CHANGE/PATH" # FIXME + +# define directory to store output +dataDir = baseEOSDir / "data" +# define dirs +SIMcondorDir = baseAFSDir / "sim" / "condor_jobs" +RECcondorDir = baseAFSDir / "rec" / "condor_jobs" +# detector specific +# FIXME: extract following from dict based on detectorModel var? +detectorDIR = baseAFSDir / "CHANGE" / "PATH" # FIXME +sim_steering_file = detectorDIR / "CHANGE" / "PATH" # FIXME +rec_steering_file = detectorDIR / "CHANGE" / "PATH" # FIXME + + +# ========================== +# Job Parameters Initialisation +# ========================== + +Nevts_ = "30" +Nevt_per_job = "10" # Set the desired number of events per job + + +# ========================== +# Parameters Initialisation +# ========================== +detectorModel = ["CLD_model_1"] +detModPaths = {"CLD_model_1": Path("CHANGE/PATH")} # FIXME +# Define lists of parameters for reconstruction +thetaList_ = ["10"] # , "20" , "30", "40", "50", "60", "70", "80", "89" +momentumList_ = ["1", "2"] # , "5", "10", "20", "50", "100", "200" +# momentumList_ = ["1", "10", "100"] +particleList_ = ["mu"] # ,"e" ,"pi"] diff --git a/TrackingPerformance/config.py b/TrackingPerformance/config.py index 4a12474..8c4f803 100644 --- a/TrackingPerformance/config.py +++ b/TrackingPerformance/config.py @@ -41,7 +41,7 @@ detectorModel = ["ILD_l5_v11"] detModPaths = {"ILD_l5_v11": Path("ILD/compact/ILD_l5_v11/ILD_l5_v11.xml")} # Define lists of parameters for reconstruction -thetaList_ = ["10", "20"] # , "30", "40", "50", "60", "70", "80", "89" +thetaList_ = ["10"] # , "20" , "30", "40", "50", "60", "70", "80", "89" momentumList_ = ["1", "2"] # , "5", "10", "20", "50", "100", "200" # momentumList_ = ["1", "10", "100"] particleList_ = ["mu"] # ,"e" ,"pi"] From 18bb9cd236c8ca7f2e3734199253ac3b8991682d Mon Sep 17 00:00:00 2001 From: Victor Schwan Date: Fri, 31 May 2024 14:19:58 +0200 Subject: [PATCH 13/33] condorJobs_sim: improved argparsing: .py suffix can be omitted --- TrackingPerformance/Condor/condorJobs_sim.py | 18 ++++++++++++++++-- TrackingPerformance/{config.py => config_v.py} | 4 ++++ 2 files changed, 20 insertions(+), 2 deletions(-) rename TrackingPerformance/{config.py => config_v.py} (97%) diff --git a/TrackingPerformance/Condor/condorJobs_sim.py b/TrackingPerformance/Condor/condorJobs_sim.py index f2d99db..5cac4e7 100644 --- a/TrackingPerformance/Condor/condorJobs_sim.py +++ b/TrackingPerformance/Condor/condorJobs_sim.py @@ -17,12 +17,26 @@ def parse_args(): "--config", type=Path, required=True, - help="Path to the configuration file.", + help="Path to the configuration file. The suffix '.py' can be omitted.", ) return parser.parse_args() -def load_config(config_path): +def load_config(config_name): + + config_file = Path(config_name) + + # Ensure the config file has the .py extension + if config_file.suffix != ".py": + config_file = config_file.with_suffix(".py") + + # Create an absolute path + config_path = config_file.resolve() + + # Ensure the file exists + if not config_path.exists(): + raise FileNotFoundError(f"The configuration file {config_path} does not exist.") + spec = importlib.util.spec_from_file_location("config", config_path) config = importlib.util.module_from_spec(spec) spec.loader.exec_module(config) diff --git a/TrackingPerformance/config.py b/TrackingPerformance/config_v.py similarity index 97% rename from TrackingPerformance/config.py rename to TrackingPerformance/config_v.py index 8c4f803..1dd7c6f 100644 --- a/TrackingPerformance/config.py +++ b/TrackingPerformance/config_v.py @@ -1,3 +1,7 @@ +""" +Victors config file for ILD FCCee models +""" + from pathlib import Path # define environment setup script path From 46bfe638980e4daa4da929958b7920a3ef75dbee Mon Sep 17 00:00:00 2001 From: Victor Schwan Date: Fri, 31 May 2024 14:54:48 +0200 Subject: [PATCH 14/33] parse_arg and load_config outsourced to utils; easily reusable in e.g. condorJobs_reco --- TrackingPerformance/Condor/condorJobs_sim.py | 43 ++--------- TrackingPerformance/Condor/utils.py | 78 ++++++++++++++++++++ 2 files changed, 85 insertions(+), 36 deletions(-) create mode 100644 TrackingPerformance/Condor/utils.py diff --git a/TrackingPerformance/Condor/condorJobs_sim.py b/TrackingPerformance/Condor/condorJobs_sim.py index 5cac4e7..d432bac 100644 --- a/TrackingPerformance/Condor/condorJobs_sim.py +++ b/TrackingPerformance/Condor/condorJobs_sim.py @@ -1,50 +1,19 @@ #!/usr/bin/env python -import argparse -import importlib.util import sys from os import system # for execution at the end from pathlib import Path import ROOT - - -def parse_args(): - parser = argparse.ArgumentParser( - description="Script to run with user configuration." - ) - parser.add_argument( - "--config", - type=Path, - required=True, - help="Path to the configuration file. The suffix '.py' can be omitted.", - ) - return parser.parse_args() - - -def load_config(config_name): - - config_file = Path(config_name) - - # Ensure the config file has the .py extension - if config_file.suffix != ".py": - config_file = config_file.with_suffix(".py") - - # Create an absolute path - config_path = config_file.resolve() - - # Ensure the file exists - if not config_path.exists(): - raise FileNotFoundError(f"The configuration file {config_path} does not exist.") - - spec = importlib.util.spec_from_file_location("config", config_path) - config = importlib.util.module_from_spec(spec) - spec.loader.exec_module(config) - return config +from utils import load_config, parse_args def main(): + # ========================== + # Load specified config file + # ========================== + args = parse_args() config = load_config(args.config) @@ -221,6 +190,7 @@ def main(): # ============================ # Condor Submission Script # ============================ + # Write the condor submission script condor_script = ( "executable = $(filename) \n" @@ -239,6 +209,7 @@ def main(): # ==================== # Submit Job to Condor # ==================== + system( "cd " + str(directory_jobs) + "; condor_submit condor_script.sub" ) # FIXME: use subprocess instead? diff --git a/TrackingPerformance/Condor/utils.py b/TrackingPerformance/Condor/utils.py new file mode 100644 index 0000000..ecaf87d --- /dev/null +++ b/TrackingPerformance/Condor/utils.py @@ -0,0 +1,78 @@ +""" +utilities used across several scripts +""" + +import argparse +import importlib.util +from pathlib import Path + + +def parse_args(): + """ + Parse command-line arguments. + + This function sets up the argument parser for the script, specifying + the expected arguments and their types. It expects a `--config` argument, + which is required and should be a Path object pointing to the configuration + file. The suffix '.py' can be omitted and will be appended automatically if + missing. + + Returns: + argparse.Namespace: An object containing the parsed command-line arguments. + """ + parser = argparse.ArgumentParser( + description="Script to run with user configuration." + ) + parser.add_argument( + "--config", + type=Path, + required=True, + help="Path to the configuration file. The suffix '.py' can be omitted.", + ) + return parser.parse_args() + + +def load_config(config_name): + """ + Load a configuration module from the given file name. + + This function accepts the name of a configuration file, ensures it has the + '.py' extension (adding it if necessary), resolves it to an absolute path, + checks if the file exists, and loads it as a Python module. + + Args: + config_name (str or Path): The name of the configuration file. Can be passed + without the '.py' extension, which will be added + automatically if not present. + + Returns: + module: The loaded configuration module. + + Raises: + ValueError: If the provided configuration file has an extension other than '.py'. + FileNotFoundError: If the resolved configuration file does not exist. + """ + config_file = Path(config_name) + + # Ensure the config file has the .py extension + if config_file.suffix: + # If a suffix is present and it's not '.py', raise an error + if config_file.suffix != ".py": + raise ValueError( + f"The configuration file {config_file} must have a .py extension." + ) + else: + # If no suffix is present, add .py + config_file = config_file.with_suffix(".py") + + # Create an absolute path + config_path = config_file.resolve() + + # Ensure the file exists + if not config_path.exists(): + raise FileNotFoundError(f"The configuration file {config_path} does not exist.") + + spec = importlib.util.spec_from_file_location("config", config_path) + config = importlib.util.module_from_spec(spec) + spec.loader.exec_module(config) + return config From acebf1c27e3ca31b875df1de74b28e811e57f372 Mon Sep 17 00:00:00 2001 From: Victor Schwan Date: Fri, 31 May 2024 17:43:18 +0200 Subject: [PATCH 15/33] JOB_FLAVOR outsourced to config --- TrackingPerformance/Condor/condorJobs_sim.py | 21 +++++++------------- TrackingPerformance/cld_config.py | 9 +++++++++ TrackingPerformance/config_v.py | 9 +++++++++ 3 files changed, 25 insertions(+), 14 deletions(-) diff --git a/TrackingPerformance/Condor/condorJobs_sim.py b/TrackingPerformance/Condor/condorJobs_sim.py index d432bac..a13235d 100644 --- a/TrackingPerformance/Condor/condorJobs_sim.py +++ b/TrackingPerformance/Condor/condorJobs_sim.py @@ -68,19 +68,12 @@ def main(): SIMEOSDir = config.dataDir / f"{DetectorModelList_[0]}" / "SIM" # output # Enable output checks - CHECK_OUTPUT = True # Set to True to enable checks, False to disable - # It will check if the ouputs exist and contain correct number of events - # if not it will send job to rerun simulation - - JOB_FLAVOR = "testmatch" - # Job flavours: - # espresso = 20 minutes - # microcentury = 1 hour - # longlunch = 2 hours - # workday = 8 hours - # tomorrow = 1 day - # testmatch = 3 days - # nextweek = 1 week + CHECK_OUTPUT = True + """ + Set to True to enable checks, False to disable + It will check if the ouputs exist and contain correct number of events + if not it will send job to rerun simulation + """ # Check if the directory exists and exit if it does try: @@ -198,7 +191,7 @@ def main(): "output = output.$(ClusterId).$(ProcId).out \n" "error = error.$(ClusterId).$(ProcId).err \n" "log = log.$(ClusterId).log \n" - f'+JobFlavour = "{JOB_FLAVOR}" \n' + f'+JobFlavour = "{config.JOB_FLAVOR}" \n' "queue filename matching files *.sh \n" ) condor_file = directory_jobs / "condor_script.sub" diff --git a/TrackingPerformance/cld_config.py b/TrackingPerformance/cld_config.py index aa9dadb..4fde683 100644 --- a/TrackingPerformance/cld_config.py +++ b/TrackingPerformance/cld_config.py @@ -33,6 +33,15 @@ Nevts_ = "30" Nevt_per_job = "10" # Set the desired number of events per job +JOB_FLAVOR = "longlunch" +# Job flavours: +# espresso = 20 minutes +# microcentury = 1 hour +# longlunch = 2 hours +# workday = 8 hours +# tomorrow = 1 day +# testmatch = 3 days +# nextweek = 1 week # ========================== diff --git a/TrackingPerformance/config_v.py b/TrackingPerformance/config_v.py index 1dd7c6f..28e1cba 100644 --- a/TrackingPerformance/config_v.py +++ b/TrackingPerformance/config_v.py @@ -37,6 +37,15 @@ Nevts_ = "1" Nevt_per_job = "1" # Set the desired number of events per job +JOB_FLAVOR = "espresso" +# Job flavours: +# espresso = 20 minutes +# microcentury = 1 hour +# longlunch = 2 hours +# workday = 8 hours +# tomorrow = 1 day +# testmatch = 3 days +# nextweek = 1 week # ========================== From 0600e0d3ab24ff3bfd0167c740db89d6eb60eb8a Mon Sep 17 00:00:00 2001 From: Victor Schwan Date: Mon, 3 Jun 2024 15:41:38 +0200 Subject: [PATCH 16/33] config gun args actually incorporated in ddsim command; clone in `bash_script` in `condorJobs_sim` removed --- TrackingPerformance/Condor/condorJobs_reco.py | 4 ++-- TrackingPerformance/Condor/condorJobs_sim.py | 18 ++++++++---------- 2 files changed, 10 insertions(+), 12 deletions(-) diff --git a/TrackingPerformance/Condor/condorJobs_reco.py b/TrackingPerformance/Condor/condorJobs_reco.py index bea2037..0f18060 100644 --- a/TrackingPerformance/Condor/condorJobs_reco.py +++ b/TrackingPerformance/Condor/condorJobs_reco.py @@ -166,8 +166,8 @@ bash_script = ( "#!/bin/bash \n" f"source {environ_path} \n" - "git clone https://github.com/gaswk/CLDConfig.git \n" # FIXME: new repo location to config (to switch easily to local fork) - "cd " + "CLDConfig/CLDConfig" + "\n" + "git clone https://github.com/gaswk/CLDConfig.git \n" # FIXME: see issues + "cd " + "CLDConfig/CLDConfig" + "\n" # FIXME: CLD should not be hardcoded f"{command} \n" f"xrdcp {output_file_name}_edm4hep.root root://eosuser.cern.ch/{output_dir} \n" f"xrdcp {output_file_name}_aida.root root://eosuser.cern.ch/{output_dir_aida} \n" diff --git a/TrackingPerformance/Condor/condorJobs_sim.py b/TrackingPerformance/Condor/condorJobs_sim.py index a13235d..93fb9f2 100644 --- a/TrackingPerformance/Condor/condorJobs_sim.py +++ b/TrackingPerformance/Condor/condorJobs_sim.py @@ -140,15 +140,14 @@ def main(): f" --compactFile $k4geo_DIR/{config.detModPaths['ILD_l5_v11']}", # Note the change to use double quotes for the dictionary key f"--outputFile {output_file_name}", f"--steeringFile {config.sim_steering_file}", # "CLDConfig/CLDConfig/cld_steer.py " - # TODO: FIX - # "--enableGun", - # f"--gun.particle {part}-", - # f"--gun.energy {momentum}*GeV", - # "--gun.distribution uniform", - # f"--gun.thetaMin {theta}*deg", - # f"--gun.thetaMax {theta}*deg", - # "--crossingAngleBoost 0", - # f"--numberOfEvents {Nevt_per_job}", + "--enableGun", + f"--gun.particle {part}-", + f"--gun.energy {momentum}*GeV", + "--gun.distribution uniform", + f"--gun.thetaMin {theta}*deg", + f"--gun.thetaMax {theta}*deg", + "--crossingAngleBoost 0", + f"--numberOfEvents {Nevt_per_job}", ] command = f"ddsim {' '.join(arguments)} > /dev/null" @@ -156,7 +155,6 @@ def main(): bash_script = ( "#!/bin/bash \n" f"source {environ_path} \n" - "git clone https://github.com/Victor-Schwan/TrackingStudies.git \n" # FIXME: new repo location to config (to switch easily to local fork) f"{command} \n" f"xrdcp {output_file_name} root://eosuser.cern.ch/{output_dir} \n" ) From 7975c8b0ee6dfaf3d1cb8a3a00fd7de7db6fc072 Mon Sep 17 00:00:00 2001 From: Victor Schwan Date: Mon, 3 Jun 2024 15:50:34 +0200 Subject: [PATCH 17/33] reco: adde cfile from --- TrackingPerformance/Condor/condorJobs_reco.py | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/TrackingPerformance/Condor/condorJobs_reco.py b/TrackingPerformance/Condor/condorJobs_reco.py index 0f18060..665fdb6 100644 --- a/TrackingPerformance/Condor/condorJobs_reco.py +++ b/TrackingPerformance/Condor/condorJobs_reco.py @@ -5,17 +5,15 @@ from pathlib import Path import ROOT +from utils import load_config, parse_args # ========================== -# Import config +# Load specified config file # ========================== -# Add the project root directory to the sys.path -project_root = Path(__file__).resolve().parent.parent -# Append the project root to sys.path if not already present -if project_root not in sys.path: - sys.path.append(str(project_root)) -# import config -import config + +args = parse_args() +config = load_config(args.config) + # ========================== # Parameters Initialisation @@ -167,7 +165,7 @@ "#!/bin/bash \n" f"source {environ_path} \n" "git clone https://github.com/gaswk/CLDConfig.git \n" # FIXME: see issues - "cd " + "CLDConfig/CLDConfig" + "\n" # FIXME: CLD should not be hardcoded + "cd " + "CLDConfig/CLDConfig" + "\n" # FIXME: CLD should not be hardcoded f"{command} \n" f"xrdcp {output_file_name}_edm4hep.root root://eosuser.cern.ch/{output_dir} \n" f"xrdcp {output_file_name}_aida.root root://eosuser.cern.ch/{output_dir_aida} \n" From 7c91e9524f12c1b6faa8e120a99d38d80aeef926 Mon Sep 17 00:00:00 2001 From: Victor Schwan Date: Mon, 3 Jun 2024 16:57:04 +0200 Subject: [PATCH 18/33] renamed config template --- TrackingPerformance/{cld_config.py => config_template.py} | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) rename TrackingPerformance/{cld_config.py => config_template.py} (90%) diff --git a/TrackingPerformance/cld_config.py b/TrackingPerformance/config_template.py similarity index 90% rename from TrackingPerformance/cld_config.py rename to TrackingPerformance/config_template.py index 4fde683..30f8856 100644 --- a/TrackingPerformance/cld_config.py +++ b/TrackingPerformance/config_template.py @@ -47,10 +47,10 @@ # ========================== # Parameters Initialisation # ========================== -detectorModel = ["CLD_model_1"] -detModPaths = {"CLD_model_1": Path("CHANGE/PATH")} # FIXME +detectorModel = ["detector_model_1"] +detModPaths = {"detector_model_1": Path("CHANGE/PATH")} # FIXME # Define lists of parameters for reconstruction -thetaList_ = ["10"] # , "20" , "30", "40", "50", "60", "70", "80", "89" +thetaList_ = ["10", "20"] # , "30", "40", "50", "60", "70", "80", "89" momentumList_ = ["1", "2"] # , "5", "10", "20", "50", "100", "200" # momentumList_ = ["1", "10", "100"] particleList_ = ["mu"] # ,"e" ,"pi"] From 6a8a2380dae6793c5a4846547df3f0337aa00fe4 Mon Sep 17 00:00:00 2001 From: Victor Schwan Date: Mon, 3 Jun 2024 17:01:02 +0200 Subject: [PATCH 19/33] minor addendum config template --- TrackingPerformance/config_template.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/TrackingPerformance/config_template.py b/TrackingPerformance/config_template.py index 30f8856..207ea58 100644 --- a/TrackingPerformance/config_template.py +++ b/TrackingPerformance/config_template.py @@ -1,3 +1,7 @@ +""" +copy this template of a config file and adapt it to your needs +""" + from pathlib import Path # define environment setup script path From 915b9b6133d438507c6c755f172b0fb188d5ed06 Mon Sep 17 00:00:00 2001 From: Victor Schwan Date: Wed, 5 Jun 2024 14:56:39 +0200 Subject: [PATCH 20/33] double definitions of vars removed: now only definied in config file and not anymore again in scripts; integer variables defined as `ints` and not `strings`: unnecessary type casting removed --- TrackingPerformance/Condor/condorJobs_reco.py | 53 ++++++++----------- TrackingPerformance/Condor/condorJobs_sim.py | 51 +++++++++--------- TrackingPerformance/config_template.py | 13 ++--- TrackingPerformance/config_v.py | 13 ++--- 4 files changed, 60 insertions(+), 70 deletions(-) diff --git a/TrackingPerformance/Condor/condorJobs_reco.py b/TrackingPerformance/Condor/condorJobs_reco.py index 665fdb6..ef8b27c 100644 --- a/TrackingPerformance/Condor/condorJobs_reco.py +++ b/TrackingPerformance/Condor/condorJobs_reco.py @@ -18,39 +18,25 @@ # ========================== # Parameters Initialisation # ========================== -# Define lists of parameters for reconstruction -thetaList_ = config.thetaList_ -momentumList_ = config.momentumList_ -particleList_ = config.particleList_ - -DetectorModelList_ = config.detectorModel -Nevts_ = config.Nevts_ - -Nevt_per_job = config.Nevt_per_job N_jobs = ( - int(int(Nevts_) / int(Nevt_per_job)) - * len(particleList_) - * len(thetaList_) - * len(momentumList_) + int(config.Nevts_ / config.Nevts_per_job) + * len(config.particleList_) + * len(config.thetaList_) + * len(config.momentumList_) ) -total_events = int(Nevts_) -num_jobs = total_events // int(Nevt_per_job) +num_jobs = config.Nevts_ // config.Nevts_per_job # FIXME # =========================== # Directory Setup and Checks # =========================== -# Define environment setup path -environ_path = config.setup - # Define directories for input and output -directory_jobs = config.RECcondorDir / f"{particleList_[0]}_{DetectorModelList_[0]}" +directory_jobs = ( + config.RECcondorDir / f"{config.particleList_[0]}_{config.detectorModelList[0]}" +) # InputDirectory = f"/eos/user/g/gasadows/Output/TrackingPerformance/{DetectorModelList_[0]}/SIM/3T/" -SIMEosDir = config.dataDir / f"{DetectorModelList_[0]}" / "SIM" # input -RECEosDir = config.dataDir / f"{DetectorModelList_[0]}" / "REC" # output - -# steering_file = "CLDReconstruction.py" -steering_file = config.rec_steering_file +SIMEosDir = config.dataDir / f"{config.detectorModelList[0]}" / "SIM" # input +RECEosDir = config.dataDir / f"{config.detectorModelList[0]}" / "REC" # output # Enable output checks check_output = True # Set to True to enable checks, False to disable @@ -92,7 +78,10 @@ import itertools list_of_combined_variables = itertools.product( - thetaList_, momentumList_, particleList_, DetectorModelList_ + config.thetaList_, + config.momentumList_, + config.particleList_, + config.detectorModelList, ) need_to_create_scripts = False @@ -105,7 +94,7 @@ f"{part}", f"{theta}_deg", f"{momentum}_GeV", - f"{Nevt_per_job}_evts", + f"{config.Nevts_per_job}_evts", f"{task_index}", ] output_file_name = "_".join(output_file_name_parts) @@ -115,7 +104,7 @@ f"{part}", f"{theta}_deg", f"{momentum}_GeV", - f"{Nevt_per_job}_evts", + f"{config.Nevts_per_job}_evts", f"{task_index}", ] input_file_name = "_".join(input_file_name_parts) @@ -136,7 +125,7 @@ if check_output and output_file.exists(): root_file = ROOT.TFile(output_file, "READ") events_tree = root_file.Get("events") - if events_tree and events_tree.GetEntries() == int(Nevt_per_job): + if events_tree and events_tree.GetEntries() == config.Nevts_per_job: root_file.Close() continue root_file.Close() @@ -148,7 +137,7 @@ arguments = ( # f" --GeoSvc.detectors=/afs/cern.ch/work/g/gasadows/k4geo/FCCee/CLD/compact/{DetectorModelList_[0]}_3T/{DetectorModelList_[0]}.xml"+ - f" --GeoSvc.detectors=$K4GEO/FCCee/CLD/compact/{DetectorModelList_[0]}/{DetectorModelList_[0]}.xml" + f" --GeoSvc.detectors=$K4GEO/FCCee/CLD/compact/{config.detectorModelList[0]}/{config.detectorModelList[0]}.xml" + " --inputFiles " + inputFile + " --outputBasename " @@ -156,14 +145,14 @@ + f" --VXDDigitiserResUV={ResVDX_UV_[0]}" + " --trackingOnly" + " -n " - + Nevt_per_job + + config.Nevts_per_job ) - command = f"k4run {steering_file} " + arguments + " > /dev/null" + command = f"k4run {config.rec_steering_file} " + arguments + " > /dev/null" # Write bash script for job execution bash_script = ( "#!/bin/bash \n" - f"source {environ_path} \n" + f"source {config.setup} \n" "git clone https://github.com/gaswk/CLDConfig.git \n" # FIXME: see issues "cd " + "CLDConfig/CLDConfig" + "\n" # FIXME: CLD should not be hardcoded f"{command} \n" diff --git a/TrackingPerformance/Condor/condorJobs_sim.py b/TrackingPerformance/Condor/condorJobs_sim.py index 93fb9f2..6bc47b4 100644 --- a/TrackingPerformance/Condor/condorJobs_sim.py +++ b/TrackingPerformance/Condor/condorJobs_sim.py @@ -35,37 +35,33 @@ def main(): # Parameters Initialisation # ========================== - # Define lists of parameters for reconstruction - thetaList_ = config.thetaList_ - momentumList_ = config.momentumList_ - particleList_ = config.particleList_ + assert isinstance(config.Nevts_, int), "config.Nevts_ must be of type integer" + assert isinstance( + config.Nevts_per_job, int + ), "config.Nevts_per_job must be of type integer" - DetectorModelList_ = config.detectorModel - Nevts_ = config.Nevts_ - Nevt_per_job = config.Nevt_per_job # Set the desired number of events per job - - total_events = int(Nevts_) N_para_sets = ( - len(DetectorModelList_) - * len(particleList_) - * len(thetaList_) - * len(momentumList_) + len(config.detectorModelList) + * len(config.particleList_) + * len(config.thetaList_) + * len(config.momentumList_) ) - N_jobs_per_para_set = total_events // int( - Nevt_per_job + N_jobs_per_para_set = ( # FIXME + config.Nevts_ // config.Nevts_per_job ) # number of parallel jobs with same parameter combination/set - N_jobs = N_jobs_per_para_set * N_para_sets # total number of jobs + N_jobs = ( + N_jobs_per_para_set * N_para_sets + ) # total number of jobs, can be printed for debugging/information # =========================== # Directory Setup and Checks # =========================== - # Define environment setup path - environ_path = config.setup - # Define directories for input and output - directory_jobs = config.SIMcondorDir / f"{particleList_[0]}_{DetectorModelList_[0]}" - SIMEOSDir = config.dataDir / f"{DetectorModelList_[0]}" / "SIM" # output + directory_jobs = ( + config.SIMcondorDir / f"{config.particleList_[0]}_{config.detectorModelList[0]}" + ) + SIMEOSDir = config.dataDir / f"{config.detectorModelList[0]}" / "SIM" # output # Enable output checks CHECK_OUTPUT = True @@ -96,7 +92,10 @@ def main(): import itertools iter_of_combined_variables = itertools.product( - thetaList_, momentumList_, particleList_, DetectorModelList_ + config.thetaList_, + config.momentumList_, + config.particleList_, + config.detectorModelList, ) need_to_create_scripts = False @@ -109,7 +108,7 @@ def main(): f"{part}", f"{theta}_deg", f"{momentum}_GeV", - f"{Nevt_per_job}_evts", + f"{config.Nevts_per_job}_evts", f"{task_index}", ] output_file_name = Path("_".join(output_file_name_parts)).with_suffix( @@ -124,7 +123,7 @@ def main(): root_file = ROOT.TFile(output_file_path, "READ") events_tree = root_file.Get("events") if events_tree: # FIXME: why no else? - if events_tree.GetEntries() == int(Nevt_per_job): + if events_tree.GetEntries() == config.Nevts_per_job: root_file.Close() continue root_file.Close() @@ -147,14 +146,14 @@ def main(): f"--gun.thetaMin {theta}*deg", f"--gun.thetaMax {theta}*deg", "--crossingAngleBoost 0", - f"--numberOfEvents {Nevt_per_job}", + f"--numberOfEvents {config.Nevts_per_job}", ] command = f"ddsim {' '.join(arguments)} > /dev/null" # Write bash script for job execution bash_script = ( "#!/bin/bash \n" - f"source {environ_path} \n" + f"source {config.setup} \n" f"{command} \n" f"xrdcp {output_file_name} root://eosuser.cern.ch/{output_dir} \n" ) diff --git a/TrackingPerformance/config_template.py b/TrackingPerformance/config_template.py index 207ea58..c60d872 100644 --- a/TrackingPerformance/config_template.py +++ b/TrackingPerformance/config_template.py @@ -35,8 +35,8 @@ # Job Parameters Initialisation # ========================== -Nevts_ = "30" -Nevt_per_job = "10" # Set the desired number of events per job +Nevts_ = 30 +Nevts_per_job = 10 # Set the desired number of events per job JOB_FLAVOR = "longlunch" # Job flavours: # espresso = 20 minutes @@ -51,10 +51,11 @@ # ========================== # Parameters Initialisation # ========================== -detectorModel = ["detector_model_1"] +# FIXME: Should this be a list? Often only element 0 accessed +detectorModelList = ["detector_model_1"] detModPaths = {"detector_model_1": Path("CHANGE/PATH")} # FIXME # Define lists of parameters for reconstruction -thetaList_ = ["10", "20"] # , "30", "40", "50", "60", "70", "80", "89" -momentumList_ = ["1", "2"] # , "5", "10", "20", "50", "100", "200" -# momentumList_ = ["1", "10", "100"] +thetaList_ = [10, 20] # , 30, 40, 50, 60, 70, 80, 89 +momentumList_ = [1, 2] # , 5, 10, 20, 50, 100, 200 +# momentumList_ = [1, 10, 100] particleList_ = ["mu"] # ,"e" ,"pi"] diff --git a/TrackingPerformance/config_v.py b/TrackingPerformance/config_v.py index 28e1cba..f7f2a89 100644 --- a/TrackingPerformance/config_v.py +++ b/TrackingPerformance/config_v.py @@ -35,8 +35,8 @@ # Job Parameters Initialisation # ========================== -Nevts_ = "1" -Nevt_per_job = "1" # Set the desired number of events per job +Nevts_ = 1 +Nevts_per_job = 1 # Set the desired number of events per job JOB_FLAVOR = "espresso" # Job flavours: # espresso = 20 minutes @@ -51,10 +51,11 @@ # ========================== # Parameters Initialisation # ========================== -detectorModel = ["ILD_l5_v11"] +# FIXME: Should this be a list? Often only element 0 accessed +detectorModelList = ["ILD_l5_v11"] detModPaths = {"ILD_l5_v11": Path("ILD/compact/ILD_l5_v11/ILD_l5_v11.xml")} # Define lists of parameters for reconstruction -thetaList_ = ["10"] # , "20" , "30", "40", "50", "60", "70", "80", "89" -momentumList_ = ["1", "2"] # , "5", "10", "20", "50", "100", "200" -# momentumList_ = ["1", "10", "100"] +thetaList_ = [10] # , 20 , 30, 40, 50, 60, 70, 80, 89 +momentumList_ = [1, 2] # , 5, 10, 20, 50, 100, 200 +# momentumList_ = [1, 10, 100] particleList_ = ["mu"] # ,"e" ,"pi"] From dab3fa7e9d39fec515dec805f1d7ab11ab266708 Mon Sep 17 00:00:00 2001 From: Victor Schwan Date: Wed, 5 Jun 2024 17:17:42 +0200 Subject: [PATCH 21/33] number of events specified in config is ceiled instead of floored (rounding when dividing Nevts to jobs) --- TrackingPerformance/Condor/condorJobs_reco.py | 3 ++- TrackingPerformance/Condor/condorJobs_sim.py | 13 +++++++------ TrackingPerformance/config_template.py | 2 +- TrackingPerformance/config_v.py | 4 ++-- 4 files changed, 12 insertions(+), 10 deletions(-) diff --git a/TrackingPerformance/Condor/condorJobs_reco.py b/TrackingPerformance/Condor/condorJobs_reco.py index ef8b27c..b9d819a 100644 --- a/TrackingPerformance/Condor/condorJobs_reco.py +++ b/TrackingPerformance/Condor/condorJobs_reco.py @@ -1,6 +1,7 @@ #!/usr/bin/env python import sys +from math import ceil from os import system # for execution at the end from pathlib import Path @@ -24,7 +25,7 @@ * len(config.thetaList_) * len(config.momentumList_) ) -num_jobs = config.Nevts_ // config.Nevts_per_job # FIXME +num_jobs = ceil(config.Nevts_ / config.Nevts_per_job) # Nevts is lower limit # =========================== # Directory Setup and Checks diff --git a/TrackingPerformance/Condor/condorJobs_sim.py b/TrackingPerformance/Condor/condorJobs_sim.py index 6bc47b4..ffa828e 100644 --- a/TrackingPerformance/Condor/condorJobs_sim.py +++ b/TrackingPerformance/Condor/condorJobs_sim.py @@ -1,6 +1,7 @@ #!/usr/bin/env python import sys +from math import ceil from os import system # for execution at the end from pathlib import Path @@ -46,12 +47,12 @@ def main(): * len(config.thetaList_) * len(config.momentumList_) ) - N_jobs_per_para_set = ( # FIXME - config.Nevts_ // config.Nevts_per_job - ) # number of parallel jobs with same parameter combination/set - N_jobs = ( - N_jobs_per_para_set * N_para_sets - ) # total number of jobs, can be printed for debugging/information + # number of parallel jobs with same parameter combination/set + N_jobs_per_para_set = ceil( + config.Nevts_ / config.Nevts_per_job + ) # Nevts is lower limit + # total number of jobs, can be printed for debugging/information + N_jobs = N_jobs_per_para_set * N_para_sets # =========================== # Directory Setup and Checks diff --git a/TrackingPerformance/config_template.py b/TrackingPerformance/config_template.py index c60d872..f244d6f 100644 --- a/TrackingPerformance/config_template.py +++ b/TrackingPerformance/config_template.py @@ -35,7 +35,7 @@ # Job Parameters Initialisation # ========================== -Nevts_ = 30 +Nevts_ = 30 # lower limit (rounding might be necessary) Nevts_per_job = 10 # Set the desired number of events per job JOB_FLAVOR = "longlunch" # Job flavours: diff --git a/TrackingPerformance/config_v.py b/TrackingPerformance/config_v.py index f7f2a89..f18727e 100644 --- a/TrackingPerformance/config_v.py +++ b/TrackingPerformance/config_v.py @@ -35,7 +35,7 @@ # Job Parameters Initialisation # ========================== -Nevts_ = 1 +Nevts_ = 1 # lower limit (rounding might be necessary) Nevts_per_job = 1 # Set the desired number of events per job JOB_FLAVOR = "espresso" # Job flavours: @@ -56,6 +56,6 @@ detModPaths = {"ILD_l5_v11": Path("ILD/compact/ILD_l5_v11/ILD_l5_v11.xml")} # Define lists of parameters for reconstruction thetaList_ = [10] # , 20 , 30, 40, 50, 60, 70, 80, 89 -momentumList_ = [1, 2] # , 5, 10, 20, 50, 100, 200 +momentumList_ = [1] # , 2 , 5, 10, 20, 50, 100, 200 # momentumList_ = [1, 10, 100] particleList_ = ["mu"] # ,"e" ,"pi"] From 2d9db2896ae3cbd382e016dfcc4413f88e06bead Mon Sep 17 00:00:00 2001 From: Victor Schwan Date: Fri, 7 Jun 2024 15:20:13 +0200 Subject: [PATCH 22/33] Alignment of names in and --- TrackingPerformance/Condor/condorJobs_reco.py | 28 +++++++++++-------- 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/TrackingPerformance/Condor/condorJobs_reco.py b/TrackingPerformance/Condor/condorJobs_reco.py index b9d819a..dbde6f3 100644 --- a/TrackingPerformance/Condor/condorJobs_reco.py +++ b/TrackingPerformance/Condor/condorJobs_reco.py @@ -19,13 +19,17 @@ # ========================== # Parameters Initialisation # ========================== -N_jobs = ( - int(config.Nevts_ / config.Nevts_per_job) + +N_para_sets = ( + len(config.detectorModelList) * len(config.particleList_) * len(config.thetaList_) * len(config.momentumList_) ) -num_jobs = ceil(config.Nevts_ / config.Nevts_per_job) # Nevts is lower limit +# number of parallel jobs with same parameter combination/set +N_jobs_per_para_set = ceil(config.Nevts_ / config.Nevts_per_job) # Nevts is lower limit +# total number of jobs, can be printed for debugging/information +N_jobs = N_jobs_per_para_set * N_para_sets # =========================== # Directory Setup and Checks @@ -78,7 +82,7 @@ # Create all possible combinations import itertools -list_of_combined_variables = itertools.product( +iter_of_combined_variables = itertools.product( config.thetaList_, config.momentumList_, config.particleList_, @@ -87,8 +91,8 @@ need_to_create_scripts = False -for theta, momentum, part, dect in list_of_combined_variables: - for task_index in range(num_jobs): +for theta, momentum, part, dect in iter_of_combined_variables: + for task_index in range(N_jobs_per_para_set): output_file_name_parts = [ f"REC_{dect}", @@ -164,9 +168,9 @@ directory_jobs + f"/bash_script_{dect}_{part}_{momentum}_{theta}_{task_index}.sh" ) - with open(bash_file, "w") as file: - file.write(bash_script) - file.close() + with open(bash_file, "w") as b_file: + b_file.write(bash_script) + b_file.close() if not need_to_create_scripts: print("All output files are correct.") @@ -186,9 +190,9 @@ "queue filename matching files *.sh \n" ) condor_file = directory_jobs + "/condor_script.sub" -with open(condor_file, "w") as file2: - file2.write(condor_script) - file2.close() +with open(condor_file, "w") as c_file: + c_file.write(condor_script) + c_file.close() # ==================== # Submit Job to Condor From 7ece7d1c0201910f4920c84a71b61735158c823f Mon Sep 17 00:00:00 2001 From: Victor Schwan Date: Fri, 7 Jun 2024 17:44:02 +0200 Subject: [PATCH 23/33] refac, alignment of naming and structure in sim and reco --- TrackingPerformance/Condor/condorJobs_reco.py | 398 +++++++++--------- TrackingPerformance/Condor/condorJobs_sim.py | 33 +- 2 files changed, 226 insertions(+), 205 deletions(-) diff --git a/TrackingPerformance/Condor/condorJobs_reco.py b/TrackingPerformance/Condor/condorJobs_reco.py index dbde6f3..487e4c5 100644 --- a/TrackingPerformance/Condor/condorJobs_reco.py +++ b/TrackingPerformance/Condor/condorJobs_reco.py @@ -8,195 +8,217 @@ import ROOT from utils import load_config, parse_args -# ========================== -# Load specified config file -# ========================== - -args = parse_args() -config = load_config(args.config) - - -# ========================== -# Parameters Initialisation -# ========================== - -N_para_sets = ( - len(config.detectorModelList) - * len(config.particleList_) - * len(config.thetaList_) - * len(config.momentumList_) -) -# number of parallel jobs with same parameter combination/set -N_jobs_per_para_set = ceil(config.Nevts_ / config.Nevts_per_job) # Nevts is lower limit -# total number of jobs, can be printed for debugging/information -N_jobs = N_jobs_per_para_set * N_para_sets - -# =========================== -# Directory Setup and Checks -# =========================== - -# Define directories for input and output -directory_jobs = ( - config.RECcondorDir / f"{config.particleList_[0]}_{config.detectorModelList[0]}" -) -# InputDirectory = f"/eos/user/g/gasadows/Output/TrackingPerformance/{DetectorModelList_[0]}/SIM/3T/" -SIMEosDir = config.dataDir / f"{config.detectorModelList[0]}" / "SIM" # input -RECEosDir = config.dataDir / f"{config.detectorModelList[0]}" / "REC" # output - -# Enable output checks -check_output = True # Set to True to enable checks, False to disable -# It will check if the ouputs exist and contain correct number of events -# if not it will send job to rerun reconstruction - -JobFlavour = "tomorrow" -# Job flavours: -# espresso = 20 minutes -# microcentury = 1 hour -# longlunch = 2 hours -# workday = 8 hours -# tomorrow = 1 day -# testmatch = 3 days -# nextweek = 1 week - -# Set default value if ResVDX_UV_ is not defined or empty -try: - if not ResVDX_UV_: + +def main(): + + # ========================== + # Load specified config file + # ========================== + + args = parse_args() + config = load_config(args.config) + + # ========================== + # Check paths + # ========================== + + assert ( + config.rec_steering_file.exists() + ), f"The file {config.rec_steering_file} does not exist" + assert ( + config.detectorDIR.exists() + ), f"The folder {config.detectorDIR} does not exist" + + # ========================== + # Parameters Initialisation + # ========================== + + N_para_sets = ( + len(config.detectorModelList) + * len(config.particleList_) + * len(config.thetaList_) + * len(config.momentumList_) + ) + # number of parallel jobs with same parameter combination/set + N_jobs_per_para_set = ceil( + config.Nevts_ / config.Nevts_per_job + ) # Nevts is lower limit + # total number of jobs, can be printed for debugging/information + N_jobs = N_jobs_per_para_set * N_para_sets + + # =========================== + # Directory Setup and Checks + # =========================== + + # Define directories for input and output + directory_jobs = ( + config.RECcondorDir / f"{config.particleList_[0]}_{config.detectorModelList[0]}" + ) + # InputDirectory = f"/eos/user/g/gasadows/Output/TrackingPerformance/{DetectorModelList_[0]}/SIM/3T/" + SIMEOSDir = config.dataDir / f"{config.detectorModelList[0]}" / "SIM" # input + RECEOSDir = config.dataDir / f"{config.detectorModelList[0]}" / "REC" # output + + # Enable output checks + CHECK_OUTPUT = True # Set to True to enable checks, False to disable + # It will check if the ouputs exist and contain correct number of events + # if not it will send job to rerun reconstruction + + # Set default value if ResVDX_UV_ is not defined or empty + try: + if not ResVDX_UV_: + ResVDX_UV_ = ["0.003"] + except NameError: ResVDX_UV_ = ["0.003"] -except NameError: - ResVDX_UV_ = ["0.003"] -# Check if the directory exists and exit if it does -if directory_jobs.exists(): - print( - f"Error: Directory '{directory_jobs}' already exists and should not be overwritten." + # Check if the directory exists and exit if it does + if directory_jobs.exists(): + print( + f"Error: Directory '{directory_jobs}' already exists and should not be overwritten." + ) + sys.exit(1) + + # Create output directories if they don't exist + RECEOSDir.mkdir(parents=True, exist_ok=True) + directory_jobs.mkdir(parents=True, exist_ok=True) + + # ======================= + # Reconstruction Job Creation + # ======================= + + # Create all possible combinations + import itertools + + iter_of_combined_variables = itertools.product( + config.thetaList_, + config.momentumList_, + config.particleList_, + config.detectorModelList, ) - sys.exit(1) - -# Create output directories if they don't exist -RECEosDir.mkdir(parents=True, exist_ok=True) -directory_jobs.mkdir(parents=True, exist_ok=True) - -# ======================= -# Simulation Job Creation -# ======================= -# Create all possible combinations -import itertools - -iter_of_combined_variables = itertools.product( - config.thetaList_, - config.momentumList_, - config.particleList_, - config.detectorModelList, -) - -need_to_create_scripts = False - -for theta, momentum, part, dect in iter_of_combined_variables: - for task_index in range(N_jobs_per_para_set): - - output_file_name_parts = [ - f"REC_{dect}", - f"{part}", - f"{theta}_deg", - f"{momentum}_GeV", - f"{config.Nevts_per_job}_evts", - f"{task_index}", - ] - output_file_name = "_".join(output_file_name_parts) - - input_file_name_parts = [ - f"SIM_{dect}", - f"{part}", - f"{theta}_deg", - f"{momentum}_GeV", - f"{config.Nevts_per_job}_evts", - f"{task_index}", - ] - input_file_name = "_".join(input_file_name_parts) - input_file_path = Path(input_file_name).with_suffix(".edm4hep.root") - inputFile = ( - SIMEosDir / part / input_file_path - ) # FIXME: reasonable that part is twice in the path? - - # Check if the input file exists - if not inputFile.exists(): - print(f"Error: Input file {inputFile} does not exist. Skipping job.") - continue - # Check if the output file already exists and has correct Nb of events - output_dir = RECEosDir / part - output_dir.mkdir(parents=True, exist_ok=True) - output_file = (output_dir / output_file_name).with_suffix("_edm4hep.root") - - if check_output and output_file.exists(): - root_file = ROOT.TFile(output_file, "READ") - events_tree = root_file.Get("events") - if events_tree and events_tree.GetEntries() == config.Nevts_per_job: - root_file.Close() + + NEED_TO_CREATE_SCRIPTS = False + + for theta, momentum, part, dect in iter_of_combined_variables: + for task_index in range(N_jobs_per_para_set): + + output_file_name_parts = [ + f"REC_{dect}", + f"{part}", + f"{theta}_deg", + f"{momentum}_GeV", + f"{config.Nevts_per_job}_evts", + f"{task_index}", + ] + output_file_name = "_".join(output_file_name_parts) + + input_file_name_parts = [ + f"SIM_{dect}", + f"{part}", + f"{theta}_deg", + f"{momentum}_GeV", + f"{config.Nevts_per_job}_evts", + f"{task_index}", + ] + input_file_path = Path("_".join(input_file_name_parts)).with_suffix( + ".edm4hep.root" + ) + inputFile = ( + SIMEOSDir / part / input_file_path + ) # FIXME: reasonable that part is twice in the path? + + # Check if the input file exists + if not inputFile.exists(): + print(f"Error: Input file {inputFile} does not exist. Skipping job.") continue - root_file.Close() - need_to_create_scripts = True - - # Create aida output Dir - output_dir_aida = output_dir / "aida_outputs" - output_dir_aida.mkdir(exist_ok=True) - - arguments = ( - # f" --GeoSvc.detectors=/afs/cern.ch/work/g/gasadows/k4geo/FCCee/CLD/compact/{DetectorModelList_[0]}_3T/{DetectorModelList_[0]}.xml"+ - f" --GeoSvc.detectors=$K4GEO/FCCee/CLD/compact/{config.detectorModelList[0]}/{config.detectorModelList[0]}.xml" - + " --inputFiles " - + inputFile - + " --outputBasename " - + output_file_name - + f" --VXDDigitiserResUV={ResVDX_UV_[0]}" - + " --trackingOnly" - + " -n " - + config.Nevts_per_job - ) - command = f"k4run {config.rec_steering_file} " + arguments + " > /dev/null" - - # Write bash script for job execution - bash_script = ( - "#!/bin/bash \n" - f"source {config.setup} \n" - "git clone https://github.com/gaswk/CLDConfig.git \n" # FIXME: see issues - "cd " + "CLDConfig/CLDConfig" + "\n" # FIXME: CLD should not be hardcoded - f"{command} \n" - f"xrdcp {output_file_name}_edm4hep.root root://eosuser.cern.ch/{output_dir} \n" - f"xrdcp {output_file_name}_aida.root root://eosuser.cern.ch/{output_dir_aida} \n" - ) - bash_file = ( - directory_jobs - + f"/bash_script_{dect}_{part}_{momentum}_{theta}_{task_index}.sh" - ) - with open(bash_file, "w") as b_file: - b_file.write(bash_script) - b_file.close() - -if not need_to_create_scripts: - print("All output files are correct.") - sys.exit(0) - -# ============================ -# Condor Submission Script -# ============================ -# Write the condor submission script -condor_script = ( - "executable = $(filename) \n" - "arguments = $(ClusterId) $(ProcId) \n" - "output = output.$(ClusterId).$(ProcId).out \n" - "error = error.$(ClusterId).$(ProcId).err \n" - "log = log.$(ClusterId).log \n" - f'+JobFlavour = "{JobFlavour}" \n' - "queue filename matching files *.sh \n" -) -condor_file = directory_jobs + "/condor_script.sub" -with open(condor_file, "w") as c_file: - c_file.write(condor_script) - c_file.close() - -# ==================== -# Submit Job to Condor -# ==================== -system( - "cd " + str(directory_jobs) + "; condor_submit condor_script.sub" -) # FIXME: use subprocess instead? + # Check if the output file already exists and has correct Nb of events + output_dir = RECEOSDir / part + output_dir.mkdir(parents=True, exist_ok=True) + output_file = (output_dir / output_file_name).with_suffix("_edm4hep.root") + + # FIXME: Issue #4 + if CHECK_OUTPUT and output_file.exists(): + root_file = ROOT.TFile(output_file, "READ") + events_tree = root_file.Get("events") + if events_tree and events_tree.GetEntries() == config.Nevts_per_job: + root_file.Close() + continue + root_file.Close() + NEED_TO_CREATE_SCRIPTS = True + + # Create aida output Dir + output_dir_aida = output_dir / "aida_outputs" + output_dir_aida.mkdir(exist_ok=True) + + arguments = ( + # f" --GeoSvc.detectors=/afs/cern.ch/work/g/gasadows/k4geo/FCCee/CLD/compact/{DetectorModelList_[0]}_3T/{DetectorModelList_[0]}.xml"+ + f" --GeoSvc.detectors=$K4GEO/FCCee/CLD/compact/{config.detectorModelList[0]}/{config.detectorModelList[0]}.xml" + + " --inputFiles " + + inputFile + + " --outputBasename " + + output_file_name + + f" --VXDDigitiserResUV={ResVDX_UV_[0]}" + + " --trackingOnly" + + " -n " + + config.Nevts_per_job + ) + command = f"k4run {config.rec_steering_file} " + arguments + " > /dev/null" + + # Write bash script for job execution + bash_script = ( + "#!/bin/bash \n" + f"source {config.setup} \n" + "git clone https://github.com/gaswk/CLDConfig.git \n" # FIXME: see issues + "cd " + + "CLDConfig/CLDConfig" + + "\n" # FIXME: CLD should not be hardcoded + f"{command} \n" + f"xrdcp {output_file_name}_edm4hep.root root://eosuser.cern.ch/{output_dir} \n" + f"xrdcp {output_file_name}_aida.root root://eosuser.cern.ch/{output_dir_aida} \n" + ) + bash_file_name_parts = [ + "bash_script", + dect, + part, + f"{theta}_deg", + f"{momentum}_GeV", + str(task_index), + ] + bash_file_path = ( + directory_jobs / "_".join(bash_file_name_parts) + ).with_suffix(".sh") + + with open(bash_file_path, "w", encoding="utf-8") as bash_file: + bash_file.write(bash_script) + bash_file.close() + + if not NEED_TO_CREATE_SCRIPTS: + print("All output files are correct.") + sys.exit(0) + + # ============================ + # Condor Submission Script + # ============================ + # Write the condor submission script + condor_script = ( + "executable = $(filename) \n" + "arguments = $(ClusterId) $(ProcId) \n" + "output = output.$(ClusterId).$(ProcId).out \n" + "error = error.$(ClusterId).$(ProcId).err \n" + "log = log.$(ClusterId).log \n" + f'+JobFlavour = "{config.JOB_FLAVOR}" \n' + "queue filename matching files *.sh \n" + ) + condor_file_path = directory_jobs / "condor_script.sub" + with open(condor_file_path, "w", encoding="utf-8") as condor_file: + condor_file.write(condor_script) + condor_file.close() + + # ==================== + # Submit Job to Condor + # ==================== + system( + "cd " + str(directory_jobs) + "; condor_submit condor_script.sub" + ) # FIXME: use subprocess instead? + + +if __name__ == "__main__": + main() diff --git a/TrackingPerformance/Condor/condorJobs_sim.py b/TrackingPerformance/Condor/condorJobs_sim.py index ffa828e..ec0624d 100644 --- a/TrackingPerformance/Condor/condorJobs_sim.py +++ b/TrackingPerformance/Condor/condorJobs_sim.py @@ -25,9 +25,6 @@ def main(): assert ( config.sim_steering_file.exists() ), f"The file {config.sim_steering_file} does not exist" - assert ( - config.rec_steering_file.exists() - ), f"The file {config.rec_steering_file} does not exist" assert ( config.detectorDIR.exists() ), f"The folder {config.detectorDIR} does not exist" @@ -99,7 +96,7 @@ def main(): config.detectorModelList, ) - need_to_create_scripts = False + NEED_TO_CREATE_SCRIPTS = False for theta, momentum, part, dect in iter_of_combined_variables: for task_index in range(N_jobs_per_para_set): @@ -120,16 +117,18 @@ def main(): output_dir = SIMEOSDir / part output_dir.mkdir(parents=True, exist_ok=True) output_file_path = output_dir / output_file_name + + # FIXME: Issue #4 if CHECK_OUTPUT and output_file_path.exists(): root_file = ROOT.TFile(output_file_path, "READ") events_tree = root_file.Get("events") - if events_tree: # FIXME: why no else? + if events_tree: if events_tree.GetEntries() == config.Nevts_per_job: root_file.Close() continue root_file.Close() else: - need_to_create_scripts = True + NEED_TO_CREATE_SCRIPTS = True # if len(DetectorModelList_) != 1 or DetectorModelList_[0] != "ILD_l5_v11": # raise ValueError("so far only ILD_l5_v11 possible") @@ -166,15 +165,15 @@ def main(): f"{momentum}_GeV", str(task_index), ] - bash_file = (directory_jobs / "_".join(bash_file_name_parts)).with_suffix( - ".sh" - ) + bash_file_path = ( + directory_jobs / "_".join(bash_file_name_parts) + ).with_suffix(".sh") - with open(bash_file, "w", encoding="utf-8") as b_file: - b_file.write(bash_script) - b_file.close() + with open(bash_file_path, "w", encoding="utf-8") as bash_file: + bash_file.write(bash_script) + bash_file.close() - if not need_to_create_scripts: + if not NEED_TO_CREATE_SCRIPTS: print("All output files are correct.") sys.exit(0) @@ -192,10 +191,10 @@ def main(): f'+JobFlavour = "{config.JOB_FLAVOR}" \n' "queue filename matching files *.sh \n" ) - condor_file = directory_jobs / "condor_script.sub" - with open(condor_file, "w", encoding="utf-8") as c_file: - c_file.write(condor_script) - c_file.close() + condor_file_path = directory_jobs / "condor_script.sub" + with open(condor_file_path, "w", encoding="utf-8") as condor_file: + condor_file.write(condor_script) + condor_file.close() # ==================== # Submit Job to Condor From 138a7b5b946bfdd848d23d502bdaa293537ecc1f Mon Sep 17 00:00:00 2001 From: Victor Schwan Date: Tue, 11 Jun 2024 11:35:17 +0200 Subject: [PATCH 24/33] fix path for xrootd --- TrackingPerformance/config_v.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/TrackingPerformance/config_v.py b/TrackingPerformance/config_v.py index f18727e..721666d 100644 --- a/TrackingPerformance/config_v.py +++ b/TrackingPerformance/config_v.py @@ -17,7 +17,7 @@ # those files are available to job baseAFSDir = Path("/afs/cern.ch/user/") / "v/vschwan/promotion" # not directly available to job, only for storing purposes -baseEOSDir = Path("/eos/") / "home-v/vschwan/promotion" +baseEOSDir = Path("/eos/") / "user/v/vschwan/promotion" # define directory to store output dataDir = baseEOSDir / "data" From 298dcfaba3aa91d47f263aa255d19f8c3a943b59 Mon Sep 17 00:00:00 2001 From: Victor Schwan Date: Tue, 11 Jun 2024 13:43:17 +0200 Subject: [PATCH 25/33] minor fix; refacs: mainly renaming vars --- TrackingPerformance/Condor/condorJobs_reco.py | 69 +++++++++---------- TrackingPerformance/Condor/condorJobs_sim.py | 63 +++++++++-------- TrackingPerformance/config_template.py | 30 ++++---- TrackingPerformance/config_v.py | 30 ++++---- 4 files changed, 97 insertions(+), 95 deletions(-) diff --git a/TrackingPerformance/Condor/condorJobs_reco.py b/TrackingPerformance/Condor/condorJobs_reco.py index 487e4c5..cee008c 100644 --- a/TrackingPerformance/Condor/condorJobs_reco.py +++ b/TrackingPerformance/Condor/condorJobs_reco.py @@ -2,7 +2,7 @@ import sys from math import ceil -from os import system # for execution at the end +from os import fspath, system # for execution at the end from pathlib import Path import ROOT @@ -26,25 +26,25 @@ def main(): config.rec_steering_file.exists() ), f"The file {config.rec_steering_file} does not exist" assert ( - config.detectorDIR.exists() - ), f"The folder {config.detectorDIR} does not exist" + config.detector_dir.exists() + ), f"The folder {config.detector_dir} does not exist" # ========================== # Parameters Initialisation # ========================== - N_para_sets = ( - len(config.detectorModelList) - * len(config.particleList_) - * len(config.thetaList_) - * len(config.momentumList_) + n_para_sets = ( + len(config.detector_model_list) + * len(config.particle_list) + * len(config.theta_list) + * len(config.momentum_list) ) # number of parallel jobs with same parameter combination/set - N_jobs_per_para_set = ceil( - config.Nevts_ / config.Nevts_per_job + n_jobs_per_para_set = ceil( + config.N_EVTS / config.N_EVTS_PER_JOB ) # Nevts is lower limit # total number of jobs, can be printed for debugging/information - N_jobs = N_jobs_per_para_set * N_para_sets + n_jobs = n_jobs_per_para_set * n_para_sets # =========================== # Directory Setup and Checks @@ -52,11 +52,11 @@ def main(): # Define directories for input and output directory_jobs = ( - config.RECcondorDir / f"{config.particleList_[0]}_{config.detectorModelList[0]}" + config.rec_condor_dir + / f"{config.particle_list[0]}_{config.detector_model_list[0]}" ) - # InputDirectory = f"/eos/user/g/gasadows/Output/TrackingPerformance/{DetectorModelList_[0]}/SIM/3T/" - SIMEOSDir = config.dataDir / f"{config.detectorModelList[0]}" / "SIM" # input - RECEOSDir = config.dataDir / f"{config.detectorModelList[0]}" / "REC" # output + sim_eos_dir = config.data_dir / f"{config.detector_model_list[0]}" / "SIM" # input + rec_eos_dir = config.data_dir / f"{config.detector_model_list[0]}" / "REC" # output # Enable output checks CHECK_OUTPUT = True # Set to True to enable checks, False to disable @@ -78,7 +78,7 @@ def main(): sys.exit(1) # Create output directories if they don't exist - RECEOSDir.mkdir(parents=True, exist_ok=True) + rec_eos_dir.mkdir(parents=True, exist_ok=True) directory_jobs.mkdir(parents=True, exist_ok=True) # ======================= @@ -89,23 +89,23 @@ def main(): import itertools iter_of_combined_variables = itertools.product( - config.thetaList_, - config.momentumList_, - config.particleList_, - config.detectorModelList, + config.theta_list, + config.momentum_list, + config.particle_list, + config.detector_model_list, ) NEED_TO_CREATE_SCRIPTS = False for theta, momentum, part, dect in iter_of_combined_variables: - for task_index in range(N_jobs_per_para_set): + for task_index in range(n_jobs_per_para_set): output_file_name_parts = [ f"REC_{dect}", f"{part}", f"{theta}_deg", f"{momentum}_GeV", - f"{config.Nevts_per_job}_evts", + f"{config.N_EVTS_PER_JOB}_evts", f"{task_index}", ] output_file_name = "_".join(output_file_name_parts) @@ -115,30 +115,30 @@ def main(): f"{part}", f"{theta}_deg", f"{momentum}_GeV", - f"{config.Nevts_per_job}_evts", + f"{config.N_EVTS_PER_JOB}_evts", f"{task_index}", ] input_file_path = Path("_".join(input_file_name_parts)).with_suffix( ".edm4hep.root" ) - inputFile = ( - SIMEOSDir / part / input_file_path + input_file = ( + sim_eos_dir / part / input_file_path ) # FIXME: reasonable that part is twice in the path? # Check if the input file exists - if not inputFile.exists(): - print(f"Error: Input file {inputFile} does not exist. Skipping job.") + if not input_file.exists(): + print(f"Error: Input file {input_file} does not exist. Skipping job.") continue # Check if the output file already exists and has correct Nb of events - output_dir = RECEOSDir / part + output_dir = rec_eos_dir / part output_dir.mkdir(parents=True, exist_ok=True) output_file = (output_dir / output_file_name).with_suffix("_edm4hep.root") # FIXME: Issue #4 if CHECK_OUTPUT and output_file.exists(): - root_file = ROOT.TFile(output_file, "READ") + root_file = ROOT.TFile(fspath(output_file), "READ") events_tree = root_file.Get("events") - if events_tree and events_tree.GetEntries() == config.Nevts_per_job: + if events_tree and events_tree.GetEntries() == config.N_EVTS_PER_JOB: root_file.Close() continue root_file.Close() @@ -149,16 +149,15 @@ def main(): output_dir_aida.mkdir(exist_ok=True) arguments = ( - # f" --GeoSvc.detectors=/afs/cern.ch/work/g/gasadows/k4geo/FCCee/CLD/compact/{DetectorModelList_[0]}_3T/{DetectorModelList_[0]}.xml"+ - f" --GeoSvc.detectors=$K4GEO/FCCee/CLD/compact/{config.detectorModelList[0]}/{config.detectorModelList[0]}.xml" + f" --GeoSvc.detectors=$K4GEO/FCCee/CLD/compact/{config.detector_model_list[0]}/{config.detector_model_list[0]}.xml" + " --inputFiles " - + inputFile + + input_file + " --outputBasename " + output_file_name + f" --VXDDigitiserResUV={ResVDX_UV_[0]}" + " --trackingOnly" + " -n " - + config.Nevts_per_job + + config.N_EVTS_PER_JOB ) command = f"k4run {config.rec_steering_file} " + arguments + " > /dev/null" @@ -216,7 +215,7 @@ def main(): # Submit Job to Condor # ==================== system( - "cd " + str(directory_jobs) + "; condor_submit condor_script.sub" + "cd " + fspath(directory_jobs) + "; condor_submit condor_script.sub" ) # FIXME: use subprocess instead? diff --git a/TrackingPerformance/Condor/condorJobs_sim.py b/TrackingPerformance/Condor/condorJobs_sim.py index ec0624d..e9f07e9 100644 --- a/TrackingPerformance/Condor/condorJobs_sim.py +++ b/TrackingPerformance/Condor/condorJobs_sim.py @@ -2,7 +2,7 @@ import sys from math import ceil -from os import system # for execution at the end +from os import fspath, system # for execution at the end from pathlib import Path import ROOT @@ -26,30 +26,30 @@ def main(): config.sim_steering_file.exists() ), f"The file {config.sim_steering_file} does not exist" assert ( - config.detectorDIR.exists() - ), f"The folder {config.detectorDIR} does not exist" + config.detector_dir.exists() + ), f"The folder {config.detector_dir} does not exist" # ========================== # Parameters Initialisation # ========================== - assert isinstance(config.Nevts_, int), "config.Nevts_ must be of type integer" + assert isinstance(config.N_EVTS, int), "config.N_EVTS must be of type integer" assert isinstance( - config.Nevts_per_job, int - ), "config.Nevts_per_job must be of type integer" - - N_para_sets = ( - len(config.detectorModelList) - * len(config.particleList_) - * len(config.thetaList_) - * len(config.momentumList_) + config.N_EVTS_PER_JOB, int + ), "config.N_EVTS_PER_JOB must be of type integer" + + n_para_sets = ( + len(config.detector_model_list) + * len(config.particle_list) + * len(config.theta_list) + * len(config.momentum_list) ) # number of parallel jobs with same parameter combination/set - N_jobs_per_para_set = ceil( - config.Nevts_ / config.Nevts_per_job + n_jobs_per_para_set = ceil( + config.N_EVTS / config.N_EVTS_PER_JOB ) # Nevts is lower limit # total number of jobs, can be printed for debugging/information - N_jobs = N_jobs_per_para_set * N_para_sets + n_jobs = n_jobs_per_para_set * n_para_sets # =========================== # Directory Setup and Checks @@ -57,13 +57,15 @@ def main(): # Define directories for input and output directory_jobs = ( - config.SIMcondorDir / f"{config.particleList_[0]}_{config.detectorModelList[0]}" + config.sim_condor_dir + / f"{config.particle_list[0]}_{config.detector_model_list[0]}" ) - SIMEOSDir = config.dataDir / f"{config.detectorModelList[0]}" / "SIM" # output + sim_eos_dir = config.data_dir / f"{config.detector_model_list[0]}" / "SIM" # output # Enable output checks CHECK_OUTPUT = True """ + -does not work- Set to True to enable checks, False to disable It will check if the ouputs exist and contain correct number of events if not it will send job to rerun simulation @@ -78,7 +80,7 @@ def main(): ) sys.exit(1) - SIMEOSDir.mkdir( + sim_eos_dir.mkdir( parents=True, exist_ok=True ) # This will create the directory if it doesn't exist, without raising an error if it does @@ -90,23 +92,23 @@ def main(): import itertools iter_of_combined_variables = itertools.product( - config.thetaList_, - config.momentumList_, - config.particleList_, - config.detectorModelList, + config.theta_list, + config.momentum_list, + config.particle_list, + config.detector_model_list, ) NEED_TO_CREATE_SCRIPTS = False for theta, momentum, part, dect in iter_of_combined_variables: - for task_index in range(N_jobs_per_para_set): + for task_index in range(n_jobs_per_para_set): output_file_name_parts = [ f"SIM_{dect}", f"{part}", f"{theta}_deg", f"{momentum}_GeV", - f"{config.Nevts_per_job}_evts", + f"{config.N_EVTS_PER_JOB}_evts", f"{task_index}", ] output_file_name = Path("_".join(output_file_name_parts)).with_suffix( @@ -114,16 +116,16 @@ def main(): ) # Check if the output file already exists and has correct Nb of events - output_dir = SIMEOSDir / part + output_dir = sim_eos_dir / part output_dir.mkdir(parents=True, exist_ok=True) output_file_path = output_dir / output_file_name # FIXME: Issue #4 if CHECK_OUTPUT and output_file_path.exists(): - root_file = ROOT.TFile(output_file_path, "READ") + root_file = ROOT.TFile(fspath(output_file_path), "READ") events_tree = root_file.Get("events") if events_tree: - if events_tree.GetEntries() == config.Nevts_per_job: + if events_tree.GetEntries() == config.N_EVTS_PER_JOB: root_file.Close() continue root_file.Close() @@ -136,7 +138,7 @@ def main(): # Build ddsim command arguments = [ # f" --compactFile /afs/cern.ch/work/g/gasadows/k4geo/FCCee/CLD/compact/{DetectorModelList_[0]}_3T/{DetectorModelList_[0]}.xml " - f" --compactFile $k4geo_DIR/{config.detModPaths['ILD_l5_v11']}", # Note the change to use double quotes for the dictionary key + f" --compactFile $k4geo_DIR/{config.det_mod_paths['ILD_l5_v11']}", # Note the change to use double quotes for the dictionary key f"--outputFile {output_file_name}", f"--steeringFile {config.sim_steering_file}", # "CLDConfig/CLDConfig/cld_steer.py " "--enableGun", @@ -146,7 +148,7 @@ def main(): f"--gun.thetaMin {theta}*deg", f"--gun.thetaMax {theta}*deg", "--crossingAngleBoost 0", - f"--numberOfEvents {config.Nevts_per_job}", + f"--numberOfEvents {config.N_EVTS_PER_JOB}", ] command = f"ddsim {' '.join(arguments)} > /dev/null" @@ -175,6 +177,7 @@ def main(): if not NEED_TO_CREATE_SCRIPTS: print("All output files are correct.") + print(f"The output file path: {output_file_path}") sys.exit(0) # ============================ @@ -201,7 +204,7 @@ def main(): # ==================== system( - "cd " + str(directory_jobs) + "; condor_submit condor_script.sub" + "cd " + fspath(directory_jobs) + "; condor_submit condor_script.sub" ) # FIXME: use subprocess instead? diff --git a/TrackingPerformance/config_template.py b/TrackingPerformance/config_template.py index f244d6f..adebf79 100644 --- a/TrackingPerformance/config_template.py +++ b/TrackingPerformance/config_template.py @@ -15,28 +15,28 @@ # ========================== # those files are available to job -baseAFSDir = Path("/afs/cern.ch/user/") / "U/USER/CHANGE/PATH" # FIXME +base_afs_dir = Path("/afs/cern.ch/user") / "U/USER/CHANGE/PATH" # FIXME # not directly available to job, only for storing purposes -baseEOSDir = Path("/eos/") / "HOME-U/USER/CHANGE/PATH" # FIXME +base_eos_dir = Path("/eos/user") / "U/USER/CHANGE/PATH" # FIXME # define directory to store output -dataDir = baseEOSDir / "data" +data_dir = base_eos_dir / "data" # define dirs -SIMcondorDir = baseAFSDir / "sim" / "condor_jobs" -RECcondorDir = baseAFSDir / "rec" / "condor_jobs" +sim_condor_dir = base_afs_dir / "sim" / "condor_jobs" +rec_condor_dir = base_afs_dir / "rec" / "condor_jobs" # detector specific # FIXME: extract following from dict based on detectorModel var? -detectorDIR = baseAFSDir / "CHANGE" / "PATH" # FIXME -sim_steering_file = detectorDIR / "CHANGE" / "PATH" # FIXME -rec_steering_file = detectorDIR / "CHANGE" / "PATH" # FIXME +detector_dir = base_afs_dir / "CHANGE" / "PATH" # FIXME +sim_steering_file = detector_dir / "CHANGE" / "PATH" # FIXME +rec_steering_file = detector_dir / "CHANGE" / "PATH" # FIXME # ========================== # Job Parameters Initialisation # ========================== -Nevts_ = 30 # lower limit (rounding might be necessary) -Nevts_per_job = 10 # Set the desired number of events per job +N_EVTS = 30 # lower limit (rounding might be necessary) +N_EVTS_PER_JOB = 10 # Set the desired number of events per job JOB_FLAVOR = "longlunch" # Job flavours: # espresso = 20 minutes @@ -52,10 +52,10 @@ # Parameters Initialisation # ========================== # FIXME: Should this be a list? Often only element 0 accessed -detectorModelList = ["detector_model_1"] -detModPaths = {"detector_model_1": Path("CHANGE/PATH")} # FIXME +detector_model_list = ["detector_model_1"] +det_mod_paths = {"detector_model_1": Path("CHANGE/PATH")} # FIXME # Define lists of parameters for reconstruction -thetaList_ = [10, 20] # , 30, 40, 50, 60, 70, 80, 89 -momentumList_ = [1, 2] # , 5, 10, 20, 50, 100, 200 +theta_list = [10, 20] # , 30, 40, 50, 60, 70, 80, 89 +momentum_list = [1, 2] # , 5, 10, 20, 50, 100, 200 # momentumList_ = [1, 10, 100] -particleList_ = ["mu"] # ,"e" ,"pi"] +particle_list = ["mu"] # ,"e" ,"pi"] diff --git a/TrackingPerformance/config_v.py b/TrackingPerformance/config_v.py index 721666d..a055be9 100644 --- a/TrackingPerformance/config_v.py +++ b/TrackingPerformance/config_v.py @@ -15,28 +15,28 @@ # ========================== # those files are available to job -baseAFSDir = Path("/afs/cern.ch/user/") / "v/vschwan/promotion" +base_afs_dir = Path("/afs/cern.ch/user") / "v/vschwan/promotion" # not directly available to job, only for storing purposes -baseEOSDir = Path("/eos/") / "user/v/vschwan/promotion" +base_eos_dir = Path("/eos/user") / "v/vschwan/promotion" # define directory to store output -dataDir = baseEOSDir / "data" +data_dir = base_eos_dir / "data" # define dirs -SIMcondorDir = baseAFSDir / "sim" / "condor_jobs" -RECcondorDir = baseAFSDir / "rec" / "condor_jobs" +sim_condor_dir = base_afs_dir / "sim" / "condor_jobs" +rec_condor_dir = base_afs_dir / "rec" / "condor_jobs" # detector specific # FIXME: extract following from dict based on detectorModel var? -detectorDIR = baseAFSDir / "ILDConfig" / "StandardConfig" / "production" -sim_steering_file = detectorDIR / "TPC_debug_muon_steer.py" -rec_steering_file = detectorDIR / "ILDReconstruction.py" +detector_dir = base_afs_dir / "ILDConfig" / "StandardConfig" / "production" +sim_steering_file = detector_dir / "TPC_debug_muon_steer.py" +rec_steering_file = detector_dir / "ILDReconstruction.py" # ========================== # Job Parameters Initialisation # ========================== -Nevts_ = 1 # lower limit (rounding might be necessary) -Nevts_per_job = 1 # Set the desired number of events per job +N_EVTS = 1 # lower limit (rounding might be necessary) +N_EVTS_PER_JOB = 1 # Set the desired number of events per job JOB_FLAVOR = "espresso" # Job flavours: # espresso = 20 minutes @@ -52,10 +52,10 @@ # Parameters Initialisation # ========================== # FIXME: Should this be a list? Often only element 0 accessed -detectorModelList = ["ILD_l5_v11"] -detModPaths = {"ILD_l5_v11": Path("ILD/compact/ILD_l5_v11/ILD_l5_v11.xml")} +detector_model_list = ["ILD_l5_v11"] +det_mod_paths = {"ILD_l5_v11": Path("ILD/compact/ILD_l5_v11/ILD_l5_v11.xml")} # Define lists of parameters for reconstruction -thetaList_ = [10] # , 20 , 30, 40, 50, 60, 70, 80, 89 -momentumList_ = [1] # , 2 , 5, 10, 20, 50, 100, 200 +theta_list = [10] # , 20 , 30, 40, 50, 60, 70, 80, 89 +momentum_list = [1] # , 2 , 5, 10, 20, 50, 100, 200 # momentumList_ = [1, 10, 100] -particleList_ = ["mu"] # ,"e" ,"pi"] +particle_list = ["mu"] # ,"e" ,"pi"] From 5e3cef02bf5cd066641722cede0987135d0e0d77 Mon Sep 17 00:00:00 2001 From: Victor Schwan Date: Tue, 11 Jun 2024 17:11:09 +0200 Subject: [PATCH 26/33] sim: output file rm on afs; minor fixes; make Plotting/analysis_tracking compatible with general changes --- TrackingPerformance/Condor/condorJobs_reco.py | 4 +- TrackingPerformance/Condor/condorJobs_sim.py | 9 ++-- .../Plotting/analysis_tracking.py | 46 +++++++++---------- TrackingPerformance/Plotting/testing.py | 13 ------ 4 files changed, 25 insertions(+), 47 deletions(-) delete mode 100644 TrackingPerformance/Plotting/testing.py diff --git a/TrackingPerformance/Condor/condorJobs_reco.py b/TrackingPerformance/Condor/condorJobs_reco.py index cee008c..7959b1b 100644 --- a/TrackingPerformance/Condor/condorJobs_reco.py +++ b/TrackingPerformance/Condor/condorJobs_reco.py @@ -121,9 +121,7 @@ def main(): input_file_path = Path("_".join(input_file_name_parts)).with_suffix( ".edm4hep.root" ) - input_file = ( - sim_eos_dir / part / input_file_path - ) # FIXME: reasonable that part is twice in the path? + input_file = sim_eos_dir / part / input_file_path # Check if the input file exists if not input_file.exists(): diff --git a/TrackingPerformance/Condor/condorJobs_sim.py b/TrackingPerformance/Condor/condorJobs_sim.py index e9f07e9..14af1c7 100644 --- a/TrackingPerformance/Condor/condorJobs_sim.py +++ b/TrackingPerformance/Condor/condorJobs_sim.py @@ -132,15 +132,11 @@ def main(): else: NEED_TO_CREATE_SCRIPTS = True - # if len(DetectorModelList_) != 1 or DetectorModelList_[0] != "ILD_l5_v11": - # raise ValueError("so far only ILD_l5_v11 possible") - # Build ddsim command arguments = [ - # f" --compactFile /afs/cern.ch/work/g/gasadows/k4geo/FCCee/CLD/compact/{DetectorModelList_[0]}_3T/{DetectorModelList_[0]}.xml " - f" --compactFile $k4geo_DIR/{config.det_mod_paths['ILD_l5_v11']}", # Note the change to use double quotes for the dictionary key + f" --compactFile {Path('$k4geo_DIR') / config.det_mod_paths[config.detector_model_list[0]]}", f"--outputFile {output_file_name}", - f"--steeringFile {config.sim_steering_file}", # "CLDConfig/CLDConfig/cld_steer.py " + f"--steeringFile {config.sim_steering_file}", "--enableGun", f"--gun.particle {part}-", f"--gun.energy {momentum}*GeV", @@ -158,6 +154,7 @@ def main(): f"source {config.setup} \n" f"{command} \n" f"xrdcp {output_file_name} root://eosuser.cern.ch/{output_dir} \n" + f"rm {output_file_name}" ) bash_file_name_parts = [ "bash_script", diff --git a/TrackingPerformance/Plotting/analysis_tracking.py b/TrackingPerformance/Plotting/analysis_tracking.py index e30e384..c8d1a7d 100644 --- a/TrackingPerformance/Plotting/analysis_tracking.py +++ b/TrackingPerformance/Plotting/analysis_tracking.py @@ -1,40 +1,36 @@ -import sys -from os.path import dirname, abspath, join +from ..Condor.utils import ( # FIXME: https://pylint.readthedocs.io/en/latest/user_guide/messages/error/relative-beyond-top-level.html + load_config, + parse_args, +) -# Add the project root directory to the sys.path -project_root = dirname(dirname(abspath(__file__))) -# Append the project root to sys.path if not already present -if project_root not in sys.path: - sys.path.append(project_root) +# ========================== +# Load specified config file +# ========================== -# import config -import config +args = parse_args() +config = load_config(args.config) # ========================== # Parameters Initialisation # ========================== -# Define lists of parameters for reconstruction -thetaList_ = config.thetaList_ -momentumList_ = config.momentumList_ -particleList_ = config.particleList_ - -DetectorModel = config.detectorModel -Nevts_ = config.Nevts_ -Nevt_per_job = config.Nevt_per_job # Set the desired number of events per job # Output and input directories -outputDir = join(config.dataDir, f"TrackingPerformance/{DetectorModel[0]}/analysis/") -inputDir = join(config.dataDir, f"TrackingPerformance/{DetectorModel[0]}/REC/") +output_dir = ( + config.data_dir / f"TrackingPerformance/{config.detector_model_list[0]}/analysis" +) +input_dir = config.data_dir / f"TrackingPerformance/{config.detector_model_list[0]}/REC" processList = { - f"REC_{DetectorModel[0]}_{particle}_{theta}_deg_{momentum}_GeV_{Nevts_per_job}_evts_{i}_edm4hep": { - "output": f"{particle}_{theta}deg_{momentum}GeV_{Nevts_per_job}evts_{i}" + f"REC_{config.detector_model_list[0]}_{particle}_{theta}_deg_{momentum}_GeV_{config.N_EVTS_PER_JOB}_evts_{i}_edm4hep": { + "output": f"{particle}_{theta}deg_{momentum}GeV_{config.N_EVTS_PER_JOB}evts_{i}" } - for particle in ParticleList - for theta in ThetaList - for momentum in MomentumList - for i in range(int(Nevts) // int(Nevts_per_job)) + for particle in config.particle_list + for theta in config.theta_list + for momentum in config.momentum_list + for i in range( + config.N_EVTS // config.N_EVTS_PER_JOB + ) # FIXME: use ceil from math instead? } # Optional: ncpus, default is 4, -1 uses all cores available diff --git a/TrackingPerformance/Plotting/testing.py b/TrackingPerformance/Plotting/testing.py deleted file mode 100644 index 1e6a1b4..0000000 --- a/TrackingPerformance/Plotting/testing.py +++ /dev/null @@ -1,13 +0,0 @@ -import sys -import os - -# Add the project root directory to the sys.path -project_root = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) -sys.path.append(project_root) - -# Now you can import config -import config - -# Access the baseDir from config.py - -print(f"Base directory: {config.baseDir}") From 8c2a8e41365e8bbf54704c9d6632c90ffc0ccc73 Mon Sep 17 00:00:00 2001 From: Victor Schwan Date: Tue, 11 Jun 2024 17:46:15 +0200 Subject: [PATCH 27/33] changes to reco requested by Gaelle --- TrackingPerformance/Condor/condorJobs_reco.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/TrackingPerformance/Condor/condorJobs_reco.py b/TrackingPerformance/Condor/condorJobs_reco.py index 7959b1b..7cc0a50 100644 --- a/TrackingPerformance/Condor/condorJobs_reco.py +++ b/TrackingPerformance/Condor/condorJobs_reco.py @@ -130,7 +130,7 @@ def main(): # Check if the output file already exists and has correct Nb of events output_dir = rec_eos_dir / part output_dir.mkdir(parents=True, exist_ok=True) - output_file = (output_dir / output_file_name).with_suffix("_edm4hep.root") + output_file = (output_dir / output_file_name).with_suffix(".edm4hep.root") # FIXME: Issue #4 if CHECK_OUTPUT and output_file.exists(): @@ -149,13 +149,13 @@ def main(): arguments = ( f" --GeoSvc.detectors=$K4GEO/FCCee/CLD/compact/{config.detector_model_list[0]}/{config.detector_model_list[0]}.xml" + " --inputFiles " - + input_file + + fspath(input_file) + " --outputBasename " - + output_file_name + + fspath(output_file_name) + f" --VXDDigitiserResUV={ResVDX_UV_[0]}" + " --trackingOnly" + " -n " - + config.N_EVTS_PER_JOB + + str(config.N_EVTS_PER_JOB) ) command = f"k4run {config.rec_steering_file} " + arguments + " > /dev/null" @@ -163,12 +163,12 @@ def main(): bash_script = ( "#!/bin/bash \n" f"source {config.setup} \n" - "git clone https://github.com/gaswk/CLDConfig.git \n" # FIXME: see issues + "git clone https://github.com/key4hep/CLDConfig.git \n" # FIXME: see issues "cd " - + "CLDConfig/CLDConfig" - + "\n" # FIXME: CLD should not be hardcoded + + "CLDConfig/CLDConfig" # FIXME: CLD should not be hardcoded + + "\n" f"{command} \n" - f"xrdcp {output_file_name}_edm4hep.root root://eosuser.cern.ch/{output_dir} \n" + f"xrdcp {output_file_name}.edm4hep.root root://eosuser.cern.ch/{output_dir} \n" f"xrdcp {output_file_name}_aida.root root://eosuser.cern.ch/{output_dir_aida} \n" ) bash_file_name_parts = [ From bcd52b35d3905ce6cc6d0a683222f240a4990aee Mon Sep 17 00:00:00 2001 From: Victor Schwan Date: Tue, 11 Jun 2024 17:48:25 +0200 Subject: [PATCH 28/33] ResVDX_UV_ removed from reco as requested by Gaelle --- TrackingPerformance/Condor/condorJobs_reco.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/TrackingPerformance/Condor/condorJobs_reco.py b/TrackingPerformance/Condor/condorJobs_reco.py index 7cc0a50..d896ccc 100644 --- a/TrackingPerformance/Condor/condorJobs_reco.py +++ b/TrackingPerformance/Condor/condorJobs_reco.py @@ -63,13 +63,6 @@ def main(): # It will check if the ouputs exist and contain correct number of events # if not it will send job to rerun reconstruction - # Set default value if ResVDX_UV_ is not defined or empty - try: - if not ResVDX_UV_: - ResVDX_UV_ = ["0.003"] - except NameError: - ResVDX_UV_ = ["0.003"] - # Check if the directory exists and exit if it does if directory_jobs.exists(): print( @@ -152,7 +145,6 @@ def main(): + fspath(input_file) + " --outputBasename " + fspath(output_file_name) - + f" --VXDDigitiserResUV={ResVDX_UV_[0]}" + " --trackingOnly" + " -n " + str(config.N_EVTS_PER_JOB) From b5a93fd2c5fcf322fbad6b77a454a7ba629faf30 Mon Sep 17 00:00:00 2001 From: Victor Schwan Date: Wed, 12 Jun 2024 13:31:23 +0200 Subject: [PATCH 29/33] Revert Plotting/analysis_tracking.py to state before Victor's PR --- .../Plotting/analysis_tracking.py | 198 +++++++----------- 1 file changed, 80 insertions(+), 118 deletions(-) diff --git a/TrackingPerformance/Plotting/analysis_tracking.py b/TrackingPerformance/Plotting/analysis_tracking.py index c8d1a7d..d6d218a 100644 --- a/TrackingPerformance/Plotting/analysis_tracking.py +++ b/TrackingPerformance/Plotting/analysis_tracking.py @@ -1,46 +1,30 @@ -from ..Condor.utils import ( # FIXME: https://pylint.readthedocs.io/en/latest/user_guide/messages/error/relative-beyond-top-level.html - load_config, - parse_args, -) - -# ========================== -# Load specified config file -# ========================== - -args = parse_args() -config = load_config(args.config) - -# ========================== -# Parameters Initialisation -# ========================== +import os + +# Lists of parameters +ParticleList = ["mu"]#, "e", "pi"] +ThetaList = ["10", "20", "30", "40", "50", "60", "70", "80", "89"] +#ThetaList = ["70", "80", "89"] +MomentumList = ["1", "2", "5", "10", "20", "50", "100", "200"] +#MomentumList = ["1", "10", "100"] +DetectorModel = ["CLD_o2_v05"] # FCCee_o1_v04 CLD_o2_v05 CLD_o3_v01 +Nevts = "10000" +Nevts_per_job = "1000" # Output and input directories -output_dir = ( - config.data_dir / f"TrackingPerformance/{config.detector_model_list[0]}/analysis" -) -input_dir = config.data_dir / f"TrackingPerformance/{config.detector_model_list[0]}/REC" - +#inputDir = f"/eos/experiment/fcc/users/g/gasadows/TrackingPerformance/{DetectorModel[0]}/REC/mu/VXD_3mic" +outputDir = f"/eos/user/g/gasadows/Output/TrackingPerformance/{DetectorModel[0]}/analysis/3T_2/mu/" +inputDir = f"/eos/user/g/gasadows/Output/TrackingPerformance/{DetectorModel[0]}/REC/3T/mu" processList = { - f"REC_{config.detector_model_list[0]}_{particle}_{theta}_deg_{momentum}_GeV_{config.N_EVTS_PER_JOB}_evts_{i}_edm4hep": { - "output": f"{particle}_{theta}deg_{momentum}GeV_{config.N_EVTS_PER_JOB}evts_{i}" - } - for particle in config.particle_list - for theta in config.theta_list - for momentum in config.momentum_list - for i in range( - config.N_EVTS // config.N_EVTS_PER_JOB - ) # FIXME: use ceil from math instead? -} + f"REC_{DetectorModel[0]}_{particle}_{theta}_deg_{momentum}_GeV_{Nevts_per_job}_evts_{i}_edm4hep":{"output":f"{particle}_{theta}deg_{momentum}GeV_{Nevts_per_job}evts_{i}"} + for particle in ParticleList for theta in ThetaList for momentum in MomentumList for i in range(int(Nevts)//int(Nevts_per_job))} # Optional: ncpus, default is 4, -1 uses all cores available nCPUS = -1 -# USER DEFINED CODE +#USER DEFINED CODE import ROOT - -ROOT.gInterpreter.Declare( - """ +ROOT.gInterpreter.Declare(""" ROOT::VecOps::RVec MCTruthTrackIndex(ROOT::VecOps::RVec trackIndex, ROOT::VecOps::RVec mcIndex, ROOT::VecOps::RVec mc) @@ -53,103 +37,81 @@ } return res; } -""" -) +""") ROOT.gInterpreter.Declare("#include ") -# END USER DEFINED CODE +#END USER DEFINED CODE # List of variables to analyse varList = ["pt", "d0", "z0", "phi0", "omega", "tanLambda", "p", "phi", "theta"] - -class RDFanalysis: +class RDFanalysis(): def analysers(df): - df2 = ( - df.Alias("MCTrackAssociations0", "_SiTracksMCTruthLink_rec.index") - .Alias("MCTrackAssociations1", "_SiTracksMCTruthLink_sim.index") - .Alias("SiTracks_Refitted_1", "_SiTracks_Refitted_trackStates") - .Define("GunParticle_index", "MCParticles.generatorStatus == 1") - .Define("GunParticle", "MCParticles[GunParticle_index][0]") - .Define( - "trackStates_IP", - "SiTracks_Refitted_1[SiTracks_Refitted_1.location == 1]", - ) - .Define( - "MC2TrackIndex", - "MCTruthTrackIndex(MCTrackAssociations0, MCTrackAssociations1, MCParticles)", - ) - .Define("GunParticleTrackIndex", "MC2TrackIndex[GunParticle_index][0]") - .Define("GunParticleTSIP", "trackStates_IP[GunParticleTrackIndex]") - .Define("MatchedGunParticle_1", "MCParticles[MC2TrackIndex != -1]") - .Define( - "MatchedGunParticle", - "FCCAnalyses::MCParticle::sel_genStatus(1) (MatchedGunParticle_1)", - ) - .Define("trackData", "SiTracks_Refitted[GunParticleTrackIndex]") - # Helix calculations and definitions - .Define( - "GunParticleTSIPHelix", - "auto h = HelixClass_double(); h.Initialize_Canonical(GunParticleTSIP.phi, GunParticleTSIP.D0, GunParticleTSIP.Z0, GunParticleTSIP.omega, GunParticleTSIP.tanLambda, 2); return h;", - ) - .Define("reco_pt", "GunParticleTSIPHelix.getPXY()") - .Define("reco_d0", "GunParticleTSIP.D0") - .Define("reco_z0", "GunParticleTSIP.Z0") - .Define("reco_phi0", "GunParticleTSIP.phi") - .Define("reco_omega", "GunParticleTSIP.omega") - .Define("reco_tanLambda", "GunParticleTSIP.tanLambda") - .Define( - "reco_pvec", - "auto p = GunParticleTSIPHelix.getMomentum(); return ROOT::Math::XYZVector(p[0], p[1], p[2]);", - ) - .Define("reco_p", "reco_pvec.R()") - .Define("reco_phi", "reco_pvec.Phi()") - .Define("reco_theta", "reco_pvec.Theta()") - .Define( - "GunParticleMCMom", - "std::vector v = {GunParticle.momentum.x, GunParticle.momentum.y, GunParticle.momentum.z}; return v;", - ) - .Define( - "GunParticleMCPos", - "std::vector v = {GunParticle.vertex.x, GunParticle.vertex.y, GunParticle.vertex.z}; return v;", - ) - .Define( - "GunParticleMCHelix", - "auto h = HelixClass_double(); h.Initialize_VP(GunParticleMCPos.data(), GunParticleMCMom.data(), -1, 2); return h;", - ) - .Define("true_pt", "GunParticleMCHelix.getPXY()") - .Define("true_d0", "GunParticleMCHelix.getD0()") - .Define("true_z0", "GunParticleMCHelix.getZ0()") - .Define("true_phi0", "GunParticleMCHelix.getPhi0()") - .Define("true_omega", "GunParticleMCHelix.getOmega()") - .Define("true_tanLambda", "GunParticleMCHelix.getTanLambda()") - .Define( - "true_pvec", - "ROOT::Math::XYZVector(GunParticleMCMom[0], GunParticleMCMom[1], GunParticleMCMom[2])", - ) - .Define("true_p", "true_pvec.R()") - .Define("true_phi", "true_pvec.Phi()") - .Define("true_theta", "true_pvec.Theta()") - .Define("chi2_trk", "trackData.chi2") - .Define("ndf_trk", "trackData.ndf") - .Define("chi2_over_ndf", "chi2_trk / ndf_trk") - .Filter("chi2_over_ndf < 10 ") - # Remove fake tracks - .Filter("abs( (reco_pt - true_pt) / true_pt) <= 0.20") - .Filter("abs( (reco_phi - true_phi) / true_phi) <= 0.20 ") - .Filter("abs( (reco_theta - true_theta) / true_theta) <= 0.20 ") - .Define("num_reconstructed_tracks", "trackStates_IP.size() > 0 ? 1 : 0") + df2 = (df + + .Alias("MCTrackAssociations0", "_SiTracksMCTruthLink_rec.index") + .Alias("MCTrackAssociations1", "_SiTracksMCTruthLink_sim.index") + .Alias("SiTracks_Refitted_1", "_SiTracks_Refitted_trackStates") + + .Define("GunParticle_index", "MCParticles.generatorStatus == 1") + .Define("GunParticle", "MCParticles[GunParticle_index][0]") + + .Define("trackStates_IP", "SiTracks_Refitted_1[SiTracks_Refitted_1.location == 1]") + .Define("MC2TrackIndex", "MCTruthTrackIndex(MCTrackAssociations0, MCTrackAssociations1, MCParticles)") + .Define("GunParticleTrackIndex", "MC2TrackIndex[GunParticle_index][0]") + .Define("GunParticleTSIP", "trackStates_IP[GunParticleTrackIndex]") + + .Define("MatchedGunParticle_1", "MCParticles[MC2TrackIndex != -1]") + .Define("MatchedGunParticle", "FCCAnalyses::MCParticle::sel_genStatus(1) (MatchedGunParticle_1)") + + .Define("trackData", "SiTracks_Refitted[GunParticleTrackIndex]") + + # Helix calculations and definitions + .Define("GunParticleTSIPHelix", "auto h = HelixClass_double(); h.Initialize_Canonical(GunParticleTSIP.phi, GunParticleTSIP.D0, GunParticleTSIP.Z0, GunParticleTSIP.omega, GunParticleTSIP.tanLambda, 2); return h;") + .Define("reco_pt", "GunParticleTSIPHelix.getPXY()") + .Define("reco_d0", "GunParticleTSIP.D0") + .Define("reco_z0", "GunParticleTSIP.Z0") + .Define("reco_phi0", "GunParticleTSIP.phi") + .Define("reco_omega", "GunParticleTSIP.omega") + .Define("reco_tanLambda", "GunParticleTSIP.tanLambda") + .Define("reco_pvec", "auto p = GunParticleTSIPHelix.getMomentum(); return ROOT::Math::XYZVector(p[0], p[1], p[2]);") + .Define("reco_p", "reco_pvec.R()") + .Define("reco_phi", "reco_pvec.Phi()") + .Define("reco_theta", "reco_pvec.Theta()") + + .Define("GunParticleMCMom", "std::vector v = {GunParticle.momentum.x, GunParticle.momentum.y, GunParticle.momentum.z}; return v;") + .Define("GunParticleMCPos", "std::vector v = {GunParticle.vertex.x, GunParticle.vertex.y, GunParticle.vertex.z}; return v;") + .Define("GunParticleMCHelix", "auto h = HelixClass_double(); h.Initialize_VP(GunParticleMCPos.data(), GunParticleMCMom.data(), -1, 2); return h;") + .Define("true_pt", "GunParticleMCHelix.getPXY()") + .Define("true_d0", "GunParticleMCHelix.getD0()") + .Define("true_z0", "GunParticleMCHelix.getZ0()") + .Define("true_phi0", "GunParticleMCHelix.getPhi0()") + .Define("true_omega", "GunParticleMCHelix.getOmega()") + .Define("true_tanLambda", "GunParticleMCHelix.getTanLambda()") + .Define("true_pvec", "ROOT::Math::XYZVector(GunParticleMCMom[0], GunParticleMCMom[1], GunParticleMCMom[2])") + .Define("true_p", "true_pvec.R()") + .Define("true_phi", "true_pvec.Phi()") + .Define("true_theta", "true_pvec.Theta()") + + .Define("chi2_trk", "trackData.chi2") + .Define("ndf_trk", "trackData.ndf") + .Define("chi2_over_ndf", "chi2_trk / ndf_trk") + .Filter("chi2_over_ndf < 10 ") + + # Remove fake tracks + .Filter("abs( (reco_pt - true_pt) / true_pt) <= 0.20") + .Filter("abs( (reco_phi - true_phi) / true_phi) <= 0.20 ") + .Filter("abs( (reco_theta - true_theta) / true_theta) <= 0.20 ") + + .Define("num_reconstructed_tracks", "trackStates_IP.size() > 0 ? 1 : 0") ) for v in varList: df2 = df2.Define(f"delta_{v}", f"reco_{v} - true_{v}") df2 = df2.Filter(f"std::isfinite(delta_{v})") if "phi0" in varList: - # Correct phi wrap around - df2 = df2.Redefine( - "delta_phi0", - "delta_phi0 < -ROOT::Math::Pi() ? delta_phi0 + 2 * ROOT::Math::Pi() : delta_phi0", - ) + # Correct phi wrap around + df2 = df2.Redefine("delta_phi0", "delta_phi0 < -ROOT::Math::Pi() ? delta_phi0 + 2 * ROOT::Math::Pi() : delta_phi0") return df2 From 6fa04795e3d0e4c81ea7f350a3979dc4d050dbb5 Mon Sep 17 00:00:00 2001 From: Victor Schwan Date: Wed, 12 Jun 2024 13:52:07 +0200 Subject: [PATCH 30/33] reco: change aida file suffix to .aida.root --- TrackingPerformance/Condor/condorJobs_reco.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/TrackingPerformance/Condor/condorJobs_reco.py b/TrackingPerformance/Condor/condorJobs_reco.py index d896ccc..eb5d650 100644 --- a/TrackingPerformance/Condor/condorJobs_reco.py +++ b/TrackingPerformance/Condor/condorJobs_reco.py @@ -161,7 +161,7 @@ def main(): + "\n" f"{command} \n" f"xrdcp {output_file_name}.edm4hep.root root://eosuser.cern.ch/{output_dir} \n" - f"xrdcp {output_file_name}_aida.root root://eosuser.cern.ch/{output_dir_aida} \n" + f"xrdcp {output_file_name}.aida.root root://eosuser.cern.ch/{output_dir_aida} \n" ) bash_file_name_parts = [ "bash_script", From 237bd0635ec80bfa2439402788b3b2c6e6193822 Mon Sep 17 00:00:00 2001 From: Victor Schwan Date: Wed, 19 Jun 2024 13:53:16 +0200 Subject: [PATCH 31/33] minor: typehint main functions --- TrackingPerformance/Condor/condorJobs_reco.py | 2 +- TrackingPerformance/Condor/condorJobs_sim.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/TrackingPerformance/Condor/condorJobs_reco.py b/TrackingPerformance/Condor/condorJobs_reco.py index eb5d650..c3d3566 100644 --- a/TrackingPerformance/Condor/condorJobs_reco.py +++ b/TrackingPerformance/Condor/condorJobs_reco.py @@ -9,7 +9,7 @@ from utils import load_config, parse_args -def main(): +def main() -> None: # ========================== # Load specified config file diff --git a/TrackingPerformance/Condor/condorJobs_sim.py b/TrackingPerformance/Condor/condorJobs_sim.py index 14af1c7..b98b959 100644 --- a/TrackingPerformance/Condor/condorJobs_sim.py +++ b/TrackingPerformance/Condor/condorJobs_sim.py @@ -9,7 +9,7 @@ from utils import load_config, parse_args -def main(): +def main() -> None: # ========================== # Load specified config file From 4163c0b1fe4637e03c10747413794dea27fb61da Mon Sep 17 00:00:00 2001 From: Victor Schwan Date: Wed, 19 Jun 2024 13:55:59 +0200 Subject: [PATCH 32/33] config option EDM4HEP_SUFFIX_WITH_UNDERSCORE added and path building in condorJobs_sim and _reco adapted --- TrackingPerformance/Condor/condorJobs_reco.py | 25 ++++++++++++++----- TrackingPerformance/Condor/condorJobs_sim.py | 13 +++++++--- TrackingPerformance/config_template.py | 2 ++ TrackingPerformance/config_v.py | 2 ++ 4 files changed, 33 insertions(+), 9 deletions(-) diff --git a/TrackingPerformance/Condor/condorJobs_reco.py b/TrackingPerformance/Condor/condorJobs_reco.py index c3d3566..bcf84cb 100644 --- a/TrackingPerformance/Condor/condorJobs_reco.py +++ b/TrackingPerformance/Condor/condorJobs_reco.py @@ -111,9 +111,15 @@ def main() -> None: f"{config.N_EVTS_PER_JOB}_evts", f"{task_index}", ] - input_file_path = Path("_".join(input_file_name_parts)).with_suffix( - ".edm4hep.root" - ) + if config.EDM4HEP_SUFFIX_WITH_UNDERSCORE: + input_file_name_parts.append("edm4hep") + input_file_path = Path("_".join(input_file_name_parts)).with_suffix( + ".root" + ) + else: + input_file_path = Path("_".join(input_file_name_parts)).with_suffix( + ".edm4hep.root" + ) input_file = sim_eos_dir / part / input_file_path # Check if the input file exists @@ -123,7 +129,14 @@ def main() -> None: # Check if the output file already exists and has correct Nb of events output_dir = rec_eos_dir / part output_dir.mkdir(parents=True, exist_ok=True) - output_file = (output_dir / output_file_name).with_suffix(".edm4hep.root") + if config.EDM4HEP_SUFFIX_WITH_UNDERSCORE: + output_file = ( + output_dir / (output_file_name + "_edm4hep") + ).with_suffix(".root") + else: + output_file = (output_dir / output_file_name).with_suffix( + ".edm4hep.root" + ) # FIXME: Issue #4 if CHECK_OUTPUT and output_file.exists(): @@ -160,8 +173,8 @@ def main() -> None: + "CLDConfig/CLDConfig" # FIXME: CLD should not be hardcoded + "\n" f"{command} \n" - f"xrdcp {output_file_name}.edm4hep.root root://eosuser.cern.ch/{output_dir} \n" - f"xrdcp {output_file_name}.aida.root root://eosuser.cern.ch/{output_dir_aida} \n" + f"xrdcp {output_file_name}{'_' if config.EDM4HEP_SUFFIX_WITH_UNDERSCORE else '.'}edm4hep.root root://eosuser.cern.ch/{output_dir} \n" + f"xrdcp {output_file_name}{'_' if config.EDM4HEP_SUFFIX_WITH_UNDERSCORE else '.'}aida.root root://eosuser.cern.ch/{output_dir_aida} \n" ) bash_file_name_parts = [ "bash_script", diff --git a/TrackingPerformance/Condor/condorJobs_sim.py b/TrackingPerformance/Condor/condorJobs_sim.py index b98b959..e66c572 100644 --- a/TrackingPerformance/Condor/condorJobs_sim.py +++ b/TrackingPerformance/Condor/condorJobs_sim.py @@ -111,9 +111,16 @@ def main() -> None: f"{config.N_EVTS_PER_JOB}_evts", f"{task_index}", ] - output_file_name = Path("_".join(output_file_name_parts)).with_suffix( - ".edm4hep.root" - ) + + if config.EDM4HEP_SUFFIX_WITH_UNDERSCORE: + output_file_name_parts.append("edm4hep") + output_file_name = Path("_".join(output_file_name_parts)).with_suffix( + ".root" + ) + else: + output_file_name = Path("_".join(output_file_name_parts)).with_suffix( + ".edm4hep.root" + ) # Check if the output file already exists and has correct Nb of events output_dir = sim_eos_dir / part diff --git a/TrackingPerformance/config_template.py b/TrackingPerformance/config_template.py index adebf79..e1e99e9 100644 --- a/TrackingPerformance/config_template.py +++ b/TrackingPerformance/config_template.py @@ -9,6 +9,8 @@ nightlies = Path("/cvmfs/sw-nightlies.hsf.org/key4hep/setup.sh") setup = nightlies # choose either stable or nightlies +# False: filepath_edm4hep.root; False: filepath.edm4hep.root +EDM4HEP_SUFFIX_WITH_UNDERSCORE = False # ========================== # define base directories diff --git a/TrackingPerformance/config_v.py b/TrackingPerformance/config_v.py index a055be9..9fe471e 100644 --- a/TrackingPerformance/config_v.py +++ b/TrackingPerformance/config_v.py @@ -9,6 +9,8 @@ nightlies = Path("/cvmfs/sw-nightlies.hsf.org/key4hep/setup.sh") setup = nightlies # choose either stable or nightlies +# False: filepath_edm4hep.root; False: filepath.edm4hep.root +EDM4HEP_SUFFIX_WITH_UNDERSCORE = False # ========================== # define base directories From bdd3deb79411abfa8a06657012c3d336b1d677e5 Mon Sep 17 00:00:00 2001 From: Victor Schwan Date: Wed, 19 Jun 2024 14:22:44 +0200 Subject: [PATCH 33/33] minor fixup --- TrackingPerformance/config_template.py | 2 +- TrackingPerformance/config_v.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/TrackingPerformance/config_template.py b/TrackingPerformance/config_template.py index e1e99e9..0f8584a 100644 --- a/TrackingPerformance/config_template.py +++ b/TrackingPerformance/config_template.py @@ -9,7 +9,7 @@ nightlies = Path("/cvmfs/sw-nightlies.hsf.org/key4hep/setup.sh") setup = nightlies # choose either stable or nightlies -# False: filepath_edm4hep.root; False: filepath.edm4hep.root +# True: filepath_edm4hep.root; False: filepath.edm4hep.root EDM4HEP_SUFFIX_WITH_UNDERSCORE = False # ========================== diff --git a/TrackingPerformance/config_v.py b/TrackingPerformance/config_v.py index 9fe471e..56a25c3 100644 --- a/TrackingPerformance/config_v.py +++ b/TrackingPerformance/config_v.py @@ -9,7 +9,7 @@ nightlies = Path("/cvmfs/sw-nightlies.hsf.org/key4hep/setup.sh") setup = nightlies # choose either stable or nightlies -# False: filepath_edm4hep.root; False: filepath.edm4hep.root +# True: filepath_edm4hep.root; False: filepath.edm4hep.root EDM4HEP_SUFFIX_WITH_UNDERSCORE = False # ==========================