From 828dcb72f0f91788981d2d2bf1b86056ad347a97 Mon Sep 17 00:00:00 2001 From: Derek Weitzel Date: Mon, 24 Apr 2017 11:47:09 -0500 Subject: [PATCH 1/8] Use redirector to enable multi origins --- bin/stashcp | 41 ++++++++++++++---------- bin/stashcp2/tests/test_inside_docker.sh | 10 ++++++ 2 files changed, 34 insertions(+), 17 deletions(-) diff --git a/bin/stashcp b/bin/stashcp index 41d1bde..154b3ed 100755 --- a/bin/stashcp +++ b/bin/stashcp @@ -15,7 +15,7 @@ import shutil import logging from urlparse import urlparse - +main_redirector = "root://redirector.osgstorage.org" TIMEOUT = 300 DIFF = TIMEOUT * 10 @@ -23,18 +23,7 @@ DIFF = TIMEOUT * 10 def doStashCpSingle(sourceFile, destination, cache, debug=False): - logging.debug("Checking size of file.") - (xrdfs_stdout, xrdfs_stderr) = subprocess.Popen(["xrdfs", "root://stash.osgconnect.net", "stat", sourceFile], stdout=subprocess.PIPE).communicate() - xrdcp_version = subprocess.Popen(['echo $(xrdcp -V 2>&1)'], stdout=subprocess.PIPE, shell=True).communicate()[0][:-1] - try: - fileSize = int(re.findall(r"Size: \d+", xrdfs_stdout)[0].split(": ")[1]) - except Exception as e: - sys.stderr.write("Unable to find size of file\n") - print str(xrdfs_stdout) - sys.stderr.write(str(xrdfs_stderr)) - sys.stderr.write("\n") - return 1 - logging.debug("Size of the file %s is %s", sourceFile, str(fileSize)) + #cache=get_best_stashcache() logging.debug("Using Cache %s", cache) @@ -44,8 +33,7 @@ def doStashCpSingle(sourceFile, destination, cache, debug=False): filename = destination + '/' + sourceFile.split('/')[-1] payload = {} - payload['xrdcp_version'] = xrdcp_version - payload['filesize'] = fileSize + payload['filename'] = sourceFile payload['sitename'] = sitename payload.update(parse_job_ad()) @@ -66,12 +54,14 @@ def doStashCpSingle(sourceFile, destination, cache, debug=False): logging.debug("Succesfully copied file from CVMFS!") end1 = int(time.time()*1000) dlSz=os.stat(destination).st_size + filesize = os.stat(cvmfs_file).st_size dltime=end1-start1 destSpace=1 status = 'Success' payload['timestamp']=end1 payload['host']="CVMFS" payload['download_size']=dlSz + payload['filesize'] = filesize payload['download_time']=dltime payload['destination_space']=destSpace payload['status']=status @@ -89,6 +79,23 @@ def doStashCpSingle(sourceFile, destination, cache, debug=False): else: logging.debug("CVMFS File does not exist") + # Now check the size of the file with xrootd + logging.debug("Checking size of file.") + (xrdfs_stdout, xrdfs_stderr) = subprocess.Popen(["xrdfs", main_redirector, "stat", sourceFile], stdout=subprocess.PIPE).communicate() + xrdcp_version = subprocess.Popen(['echo $(xrdcp -V 2>&1)'], stdout=subprocess.PIPE, shell=True).communicate()[0][:-1] + try: + fileSize = int(re.findall(r"Size: \d+", xrdfs_stdout)[0].split(": ")[1]) + except Exception as e: + sys.stderr.write("Unable to find size of file\n") + print str(xrdfs_stdout) + sys.stderr.write(str(xrdfs_stderr)) + sys.stderr.write("\n") + return 1 + logging.debug("Size of the file %s is %s", sourceFile, str(fileSize)) + + payload['xrdcp_version'] = xrdcp_version + payload['filesize'] = fileSize + end1=int(time.time()*1000) payload['end1']=end1 payload['start1']=start1 @@ -179,9 +186,9 @@ def parse_job_ad(): return temp_list def dostashcpdirectory(sourceDir, destination, cache, debug=False): - sourceItems = subprocess.Popen(["xrdfs", "root://stash.osgconnect.net", "ls", sourceDir], stdout=subprocess.PIPE).communicate()[0].split() + sourceItems = subprocess.Popen(["xrdfs", main_redirector, "ls", sourceDir], stdout=subprocess.PIPE).communicate()[0].split() for remote_file in sourceItems: - command2 = 'xrdfs root://stash.osgconnect.net stat '+ remote_file + ' | grep "IsDir" | wc -l' + command2 = 'xrdfs ' + main_redirector + ' stat '+ remote_file + ' | grep "IsDir" | wc -l' isdir=subprocess.Popen([command2],stdout=subprocess.PIPE,shell=True).communicate()[0].split()[0] if isdir!='0': result = dostashcpdirectory(remote_file, destination, cache, debug) diff --git a/bin/stashcp2/tests/test_inside_docker.sh b/bin/stashcp2/tests/test_inside_docker.sh index c6315a2..31a1ee6 100755 --- a/bin/stashcp2/tests/test_inside_docker.sh +++ b/bin/stashcp2/tests/test_inside_docker.sh @@ -79,5 +79,15 @@ fi /StashCache/bin/stashcp --cache=$XRD_CACHE -d -r /user/dweitzel/public/blast/queries ./ ls -lah +rm -rf queries + +/StashCache/bin/stashcp --cache=$XRD_CACHE -d /xenon/rucio/x1t_SR001_170419_1605_mv/73/90/XENON1T-0-000008000-000008999-000001000.zip + +result=`md5sum XENON1T-0-000008000-000008999-000001000.zip | awk '{print $1;}'` +if [ "$result" != "dd00dd6a6b1e0de4a3b8ecf1a34b24b3" ]; then + exit 1 +fi + + From 58c885223770f60b52d8e550d626396cdb14ba8a Mon Sep 17 00:00:00 2001 From: Derek Weitzel Date: Mon, 24 Apr 2017 11:53:16 -0500 Subject: [PATCH 2/8] Use stash origin for recursive transfers --- bin/stashcp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/bin/stashcp b/bin/stashcp index d9317fe..ed8be33 100755 --- a/bin/stashcp +++ b/bin/stashcp @@ -16,6 +16,7 @@ import logging from urlparse import urlparse main_redirector = "root://redirector.osgstorage.org" +stash_origin = "root://stash.osgconnect.net" TIMEOUT = 300 DIFF = TIMEOUT * 10 @@ -186,9 +187,9 @@ def parse_job_ad(): return temp_list def dostashcpdirectory(sourceDir, destination, cache, debug=False): - sourceItems = subprocess.Popen(["xrdfs", main_redirector, "ls", sourceDir], stdout=subprocess.PIPE).communicate()[0].split() + sourceItems = subprocess.Popen(["xrdfs", stash_origin, "ls", sourceDir], stdout=subprocess.PIPE).communicate()[0].split() for remote_file in sourceItems: - command2 = 'xrdfs ' + main_redirector + ' stat '+ remote_file + ' | grep "IsDir" | wc -l' + command2 = 'xrdfs ' + stash_origin + ' stat '+ remote_file + ' | grep "IsDir" | wc -l' isdir=subprocess.Popen([command2],stdout=subprocess.PIPE,shell=True).communicate()[0].split()[0] if isdir!='0': result = dostashcpdirectory(remote_file, destination, cache, debug) From a0ca3c2a9e4dfd5cdc23a84d211a531b62571718 Mon Sep 17 00:00:00 2001 From: Derek Weitzel Date: Mon, 24 Apr 2017 11:59:04 -0500 Subject: [PATCH 3/8] Fixing test --- bin/stashcp2/tests/test_inside_docker.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/stashcp2/tests/test_inside_docker.sh b/bin/stashcp2/tests/test_inside_docker.sh index 31a1ee6..44aac02 100755 --- a/bin/stashcp2/tests/test_inside_docker.sh +++ b/bin/stashcp2/tests/test_inside_docker.sh @@ -81,7 +81,7 @@ ls -lah rm -rf queries -/StashCache/bin/stashcp --cache=$XRD_CACHE -d /xenon/rucio/x1t_SR001_170419_1605_mv/73/90/XENON1T-0-000008000-000008999-000001000.zip +/StashCache/bin/stashcp --cache=$XRD_CACHE -d /xenon/rucio/x1t_SR001_170419_1605_mv/73/90/XENON1T-0-000008000-000008999-000001000.zip ./ result=`md5sum XENON1T-0-000008000-000008999-000001000.zip | awk '{print $1;}'` if [ "$result" != "dd00dd6a6b1e0de4a3b8ecf1a34b24b3" ]; then From 9921ee59132f233a056c13d61638599c662c3f53 Mon Sep 17 00:00:00 2001 From: Derek Weitzel Date: Mon, 24 Apr 2017 12:24:01 -0500 Subject: [PATCH 4/8] Adding root to beginning of syr cache --- .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 49fb7a6..35bd644 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,13 +8,13 @@ env: - OS_TYPE=centos OS_VERSION=6 XRD_CACHE="root://mwt2-stashcache.campuscluster.illinois.edu" - OS_TYPE=centos OS_VERSION=6 XRD_CACHE="root://hcc-stash.unl.edu" - OS_TYPE=centos OS_VERSION=6 XRD_CACHE="root://osgxroot.usatlas.bnl.gov" - - OS_TYPE=centos OS_VERSION=6 XRD_CACHE="its-condor-xrootd1.syr.edu" + - OS_TYPE=centos OS_VERSION=6 XRD_CACHE="root://its-condor-xrootd1.syr.edu" - OS_TYPE=centos OS_VERSION=7 XRD_CACHE="root://stash.osgconnect.net" - OS_TYPE=centos OS_VERSION=7 XRD_CACHE="root://xrd-cache-1.t2.ucsd.edu" - OS_TYPE=centos OS_VERSION=7 XRD_CACHE="root://mwt2-stashcache.campuscluster.illinois.edu" - OS_TYPE=centos OS_VERSION=7 XRD_CACHE="root://hcc-stash.unl.edu" - OS_TYPE=centos OS_VERSION=7 XRD_CACHE="root://osgxroot.usatlas.bnl.gov" - - OS_TYPE=centos OS_VERSION=7 XRD_CACHE="its-condor-xrootd1.syr.edu" + - OS_TYPE=centos OS_VERSION=7 XRD_CACHE="root://its-condor-xrootd1.syr.edu" services: From 1274190bb7b075602adafd36151d668cd351fcc7 Mon Sep 17 00:00:00 2001 From: Derek Weitzel Date: Mon, 24 Apr 2017 12:24:46 -0500 Subject: [PATCH 5/8] Removing stash.osgconnect.net as it is not a cache, origin --- .travis.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 35bd644..8a0f457 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,13 +3,11 @@ env: matrix: - OS_TYPE=centos OS_VERSION=6 - OS_TYPE=centos OS_VERSION=7 - - OS_TYPE=centos OS_VERSION=6 XRD_CACHE="root://stash.osgconnect.net" - OS_TYPE=centos OS_VERSION=6 XRD_CACHE="root://xrd-cache-1.t2.ucsd.edu" - OS_TYPE=centos OS_VERSION=6 XRD_CACHE="root://mwt2-stashcache.campuscluster.illinois.edu" - OS_TYPE=centos OS_VERSION=6 XRD_CACHE="root://hcc-stash.unl.edu" - OS_TYPE=centos OS_VERSION=6 XRD_CACHE="root://osgxroot.usatlas.bnl.gov" - OS_TYPE=centos OS_VERSION=6 XRD_CACHE="root://its-condor-xrootd1.syr.edu" - - OS_TYPE=centos OS_VERSION=7 XRD_CACHE="root://stash.osgconnect.net" - OS_TYPE=centos OS_VERSION=7 XRD_CACHE="root://xrd-cache-1.t2.ucsd.edu" - OS_TYPE=centos OS_VERSION=7 XRD_CACHE="root://mwt2-stashcache.campuscluster.illinois.edu" - OS_TYPE=centos OS_VERSION=7 XRD_CACHE="root://hcc-stash.unl.edu" From a823ba0b28baa2bcc80da9675d75799bdf365318 Mon Sep 17 00:00:00 2001 From: Derek Weitzel Date: Mon, 24 Apr 2017 12:46:59 -0500 Subject: [PATCH 6/8] Adding syracuse, and removing stash as caches --- bin/caches.json | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/bin/caches.json b/bin/caches.json index 9c843c0..4af05e2 100755 --- a/bin/caches.json +++ b/bin/caches.json @@ -1,9 +1,7 @@ [ - {"name":"root://student02.ci-connect.net", "status":0, "longitude":-87.603, "latitude":41.78}, - {"name":"root://stash.osgconnect.net", "status":1, "longitude":-87.603, "latitude":41.78}, - {"name":"root://cm3.bu.edu", "status":0, "longitude":-72.6167, "latitude":42.2042}, {"name":"root://xrd-cache-1.t2.ucsd.edu", "status":1, "longitude":-117.2359, "latitude":32.8807}, {"name":"root://mwt2-stashcache.campuscluster.illinois.edu", "status":1, "longitude":-88.23, "latitude":40.107487}, {"name":"root://hcc-stash.unl.edu", "status":1, "longitude":-96.702991, "latitude":40.812957}, - {"name":"root://osgxroot.usatlas.bnl.gov", "status":1, "longitude":-72.9333, "latitude":40.8167} + {"name":"root://osgxroot.usatlas.bnl.gov", "status":1, "longitude":-72.9333, "latitude":40.8167}, + {"name":"root://its-condor-xrootd1.syr.edu", "status":1, "longitude":-72.9333, "latitude":40.8167} ] From b632e702ca32d457a039eeb6bca381994bf5dd0b Mon Sep 17 00:00:00 2001 From: Derek Weitzel Date: Mon, 24 Apr 2017 12:57:28 -0500 Subject: [PATCH 7/8] If fails on cache, use main redirector --- bin/stashcp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bin/stashcp b/bin/stashcp index ed8be33..ffb1030 100755 --- a/bin/stashcp +++ b/bin/stashcp @@ -131,8 +131,8 @@ def doStashCpSingle(sourceFile, destination, cache, debug=False): es_send(payload) else: #pull from origin - logging.warning("XrdCP from cache failed on %s, pulling from origin", cache) - cache="root://stash.osgconnect.net" + logging.warning("XrdCP from cache failed on %s, pulling from main redirector", cache) + cache=main_redirector start3 = int(time.time()*1000) xrd_exit=timed_transfer(filename=sourceFile, debug=debug, cache=cache, destination=destination) end3=int(time.time()*1000) From 363e743f81b6a28490889064d26bed5e3c8593fc Mon Sep 17 00:00:00 2001 From: Derek Weitzel Date: Mon, 24 Apr 2017 14:20:16 -0500 Subject: [PATCH 8/8] Fixing tests --- bin/stashcp2/tests/test_inside_docker.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/stashcp2/tests/test_inside_docker.sh b/bin/stashcp2/tests/test_inside_docker.sh index 44aac02..6d1ee5c 100755 --- a/bin/stashcp2/tests/test_inside_docker.sh +++ b/bin/stashcp2/tests/test_inside_docker.sh @@ -42,7 +42,7 @@ cp /StashCache/bin/stashcp2/tests/job.ad ./.job.ad # Test against a file that is known to not exist set +e -/StashCache/bin/stashcp --cache=$XRD_CACHE /blah/does/not/exist +/StashCache/bin/stashcp --cache=$XRD_CACHE /blah/does/not/exist ./ if [ $? -eq 0 ]; then echo "Failed to exit with non-zero exit status when it should have" exit 1