From 6a11688c768dd854fbb671cf08194dca331cc45d Mon Sep 17 00:00:00 2001 From: Onno Zweers Date: Thu, 19 Jun 2025 22:21:12 +0200 Subject: [PATCH 1/5] Getting WebDAV door from the API This commit adds a function `get_webdav_door` that allows Ada to get the URL of a WebDAV door from the API. Ada does not know about WebDAV doors; it only knows the API's URL. There are two ways you can ask the API for a WebDAV door. Both methods are implemented. If the first method fails, the second will be tried. Testing this function may be a bit difficult, because it is not used yet. I tested it by calling `get_webdav_door "$api"` just before the call to `api_call`. To test the second method, I commented out the first `return` command in `get_webdav_door`. With this function, we can start implementing uploads and downloads into Ada. I'd also like to implement get_api_from_webdav. That would allow users to provide a WebDAV URL if they don 't know the address of the API. But that is less urgent. --- ada/ada | 76 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 76 insertions(+) diff --git a/ada/ada b/ada/ada index 6ea6b70..6f75feb 100755 --- a/ada/ada +++ b/ada/ada @@ -971,6 +971,82 @@ check_authentication() { } +get_webdav_door () { + # The dCache API itself does not allow uploading and downloading + # of data. We need a WebDAV door for that. + # A properly configured API can redirect us to a WebDAV door, + # which we can use for uploading and downloading data. + # This function will return the WebDAV door. + local api="$1" + local api_server="${api%/api/v*}" + local webdav_door + local command + command='$debug && set -x + curl "${curl_authorization[@]}" \ + "${curl_options_common[@]}" \ + -H "Authorization: bearer $BEARER_TOKEN" \ + -H "Accept: application/json" \ + "$api_server/scripts/config.js" \ + | jq -r ".\"dcache-view.endpoints.webdav\"" ' + if $dry_run ; then + echo "$command" + else + $debug && echo "Getting WebDAV door from $api_server/scripts/config.js ..." + webdav_door=$(eval "$command") + fi + # If we got a WebDAV door, we're done! Print value and return. + if [ -n "$webdav_door" ] ; then + echo "$webdav_door" + return 0 + fi + $debug && echo "Method failed. Trying next method." + # If the first method failed, we try a backup method. + # This uses the /doors API call, which lists all doors. + # The list of doors can be quite long. + # From the list, we select the most appropriate. + # The door should have: + # - protocol https (this excludes xrootd and gridftp) + # - all paths: / (root path; we don't want a door that confines us) + # - a lot of tags (which means it's publically advertised) + # - port 443 is preferred (reduces firewall issues) + # - a low load (we don't want a busy door) + command='$debug && set -x + curl "${curl_authorization[@]}" \ + "${curl_options_common[@]}" \ + -H "Accept: application/json" \ + -X GET "$api/doors" \ + | jq -r " + map(select( + .protocol == \"https\" and + .root == \"/\" and + ((.readPaths // []) | index(\"/\")) and + ((.writePaths // []) | index(\"/\")) + )) + | sort_by( + -((.tags // []) | length), # prefer doors with more tags + (.port != 443), # prefer port 443 + .load # prefer lower load + ) + | .[0] as \$best + | \"\(\$best.protocol)://\(\$best.addresses[0]):\(\$best.port)\" + " + ' + if $dry_run ; then + echo "$command" + else + $debug && echo "Getting WebDAV door from $api/doors ..." + webdav_door=$(eval "$command") + fi + # If we got a WebDAV door, we're done! Print value and return. + if [ -n "$webdav_door" ] ; then + echo "$webdav_door" + return 0 + else + return 1 + fi +} + + urlencode () { # We use jq for encoding the URL, because we need jq anyway. $debug && echo "urlencoding '$1' to '$(printf '%s' "$1" | jq -sRr @uri)'" 1>&2 From 66d907fa20907609db62e4f036c5fb5ccb623bfc Mon Sep 17 00:00:00 2001 From: Onno Zweers Date: Wed, 2 Jul 2025 15:40:16 +0200 Subject: [PATCH 2/5] Send debug output to stderr instead of stdout; print error when no WebDAV found * The WebDAV door should be sent to stdout, that means that any other output should be sent to stderr to keep them separate. * It's nice to print an error message when no WebDAV door could be found. This includes a hint to try with --debug, to speed up troubleshooting. --- ada/ada | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/ada/ada b/ada/ada index 6f75feb..dbdfcd1 100755 --- a/ada/ada +++ b/ada/ada @@ -991,7 +991,7 @@ get_webdav_door () { if $dry_run ; then echo "$command" else - $debug && echo "Getting WebDAV door from $api_server/scripts/config.js ..." + $debug && echo "Getting WebDAV door from $api_server/scripts/config.js ..." 1>&2 webdav_door=$(eval "$command") fi # If we got a WebDAV door, we're done! Print value and return. @@ -999,7 +999,7 @@ get_webdav_door () { echo "$webdav_door" return 0 fi - $debug && echo "Method failed. Trying next method." + $debug && echo "Method failed. Trying next method." 1>&2 # If the first method failed, we try a backup method. # This uses the /doors API call, which lists all doors. # The list of doors can be quite long. @@ -1034,7 +1034,7 @@ get_webdav_door () { if $dry_run ; then echo "$command" else - $debug && echo "Getting WebDAV door from $api/doors ..." + $debug && echo "Getting WebDAV door from $api/doors ..." 1>&2 webdav_door=$(eval "$command") fi # If we got a WebDAV door, we're done! Print value and return. @@ -1042,6 +1042,7 @@ get_webdav_door () { echo "$webdav_door" return 0 else + echo "Unable to get WebDAV door. Try again with --debug to get more information." 1>&2 return 1 fi } From f6731dcf17254a3a5a18ca0d8ac6d7148df7406b Mon Sep 17 00:00:00 2001 From: Onno Zweers Date: Wed, 2 Jul 2025 16:12:21 +0200 Subject: [PATCH 3/5] get_webdav_door: add method: read WebDAV URL from Rclone config file is specified --- ada/ada | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/ada/ada b/ada/ada index dbdfcd1..e4070fc 100755 --- a/ada/ada +++ b/ada/ada @@ -974,13 +974,24 @@ check_authentication() { get_webdav_door () { # The dCache API itself does not allow uploading and downloading # of data. We need a WebDAV door for that. - # A properly configured API can redirect us to a WebDAV door, - # which we can use for uploading and downloading data. - # This function will return the WebDAV door. + # This function will return a WebDAV door. + # There can be several methods to find a WebDAV door. + # One is, reading it from an Rclone config file. + # Two other methods ask the API for a WebDAV door. + # If no door can be found, an error code is returned. local api="$1" local api_server="${api%/api/v*}" local webdav_door local command + # Method 1: if there's an Rclone config file, read the WebDAV door from that. + if [ -f "$tokenfile" ] ; then + $debug && echo "Getting WebDAV door from '$tokenfile' ..." 1>&2 + webdav_door=$(awk -F' *= *' '/^url *=/ {print $2}' "$tokenfile") + echo "$webdav_door" + return 0 + fi + $debug && echo "Method failed. Trying next method." 1>&2 + # Method 2: read the /scripts/config.js file from the API server. command='$debug && set -x curl "${curl_authorization[@]}" \ "${curl_options_common[@]}" \ @@ -1000,7 +1011,7 @@ get_webdav_door () { return 0 fi $debug && echo "Method failed. Trying next method." 1>&2 - # If the first method failed, we try a backup method. + # Method 3. # This uses the /doors API call, which lists all doors. # The list of doors can be quite long. # From the list, we select the most appropriate. From 3464f477d3cf66c7d61062872ba164ee7317edf0 Mon Sep 17 00:00:00 2001 From: Onno Zweers Date: Thu, 3 Jul 2025 13:36:59 +0200 Subject: [PATCH 4/5] Be more polite please ;-) --- ada/ada | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ada/ada b/ada/ada index e4070fc..a53d18a 100755 --- a/ada/ada +++ b/ada/ada @@ -1053,7 +1053,7 @@ get_webdav_door () { echo "$webdav_door" return 0 else - echo "Unable to get WebDAV door. Try again with --debug to get more information." 1>&2 + echo "Unable to get WebDAV door. Please try again with --debug to get more information." 1>&2 return 1 fi } From bc2d446ba0d19103b432d163702e1e2ae5a3013a Mon Sep 17 00:00:00 2001 From: Onno Zweers Date: Thu, 3 Jul 2025 13:41:37 +0200 Subject: [PATCH 5/5] Return only when the method has succeeded. --- ada/ada | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/ada/ada b/ada/ada index a53d18a..3a50c5c 100755 --- a/ada/ada +++ b/ada/ada @@ -987,8 +987,10 @@ get_webdav_door () { if [ -f "$tokenfile" ] ; then $debug && echo "Getting WebDAV door from '$tokenfile' ..." 1>&2 webdav_door=$(awk -F' *= *' '/^url *=/ {print $2}' "$tokenfile") - echo "$webdav_door" - return 0 + if [ -n "$webdav_door" ] ; then + echo "$webdav_door" + return 0 + fi fi $debug && echo "Method failed. Trying next method." 1>&2 # Method 2: read the /scripts/config.js file from the API server.