Replace {RCurl} with {curl}

.github/workflows/docker.yml

@@ -9,7 +9,7 @@ name: Docker
 #   latest and develop
 # - when a pull request is created and updated  to make sure the
 #   Dockerfile is still valid.
-# To be able to push to dockerhub, this execpts the following
+# To be able to push to dockerhub, this expects the following
 # secrets to be set in the project:
 # - DOCKERHUB_USERNAME : username that can push to the org
 # - DOCKERHUB_PASSWORD : password asscoaited with the username

CHANGELOG.md

@@ -56,6 +56,8 @@ convert data for a single PFT fixed (#1329, #2974, #2981)
 
 - Using R4.0 and R4.1 tags to build PEcAn. Default is now 4.1
 - Database connections consistently use `DBI::dbConnect` instead of the deprecated `dplyr::src_postgres` (#2881). This change should be invisible to most users, but it involved converting a lot of internal variables from `bety$con` to `con`. If you see errors involving these symbols it means we missed a place, so please report them as bugs.
+- `PEcAn.utils::download.url` argument `retry404` is now renamed to `retry` and
+  now functions as intended (it was being ignored completely before).
 - Update URL for MERRA downloads (#2888)
 - PEcAn.logger is now BSD-3 License
 - Skipped ICOS and MERRA download tests when running in github actions

base/db/DESCRIPTION

@@ -41,6 +41,7 @@ Description: The Predictive Ecosystem Carbon Analyzer (PEcAn) is a scientific
     streamline the interaction between data and models, and to improve the
     efficacy of scientific investigation.
 Imports:
+    curl,
     DBI,
     dbplyr,
     dplyr,
@@ -55,7 +56,6 @@ Imports:
     PEcAn.utils,
     purrr,
     R.utils,
-    RCurl,
     rlang,
     tibble,
     tidyr,

base/db/R/convert_input.R

@@ -605,18 +605,27 @@ convert_input <-
     }
 
     # create curl options
+    curloptions <- list(followlocation = TRUE)
     if (!is.null(browndog$username) && !is.null(browndog$password)) {
-      userpwd <- paste(browndog$username, browndog$password, sep = ":")
-      curloptions <- list(userpwd = userpwd, httpauth = 1L)
+      curloptions$userpwd = paste(
+        browndog$username, browndog$password, sep = ":")
+      curloptions$httpauth = 1L
     }
-    curloptions <- c(curloptions, followlocation = TRUE)
 
     # check if we can do conversion
-    out.html <- RCurl::getURL(paste0("http://dap-dev.ncsa.illinois.edu:8184/inputs/",
-                              browndog$inputtype), .opts = curloptions)
-    if (outputtype %in% unlist(strsplit(out.html, "\n"))) {
-      PEcAn.logger::logger.info(paste("Conversion from", browndog$inputtype, "to", outputtype, 
-                        "through Brown Dog"))
+    h <- curl::new_handle()
+    curl::handle_setopt(h, .list = curloptions)
+    out.html <- readLines(
+      curl::curl(
+        url = paste0(
+          "http://dap-dev.ncsa.illinois.edu:8184/inputs/",
+          browndog$inputtype),
+        handle = h))
+    if (outputtype %in% out.html) {
+      PEcAn.logger::logger.info(
+        "Conversion from", browndog$inputtype,
+        "to", outputtype,
+        "through Brown Dog")
       conversion <- "browndog"
     }
   }
@@ -637,13 +646,16 @@ convert_input <-
     }
 
     # post zipped file to Brown Dog
-    html <- RCurl::postForm(url, fileData = RCurl::fileUpload(zipfile), .opts = curloptions)
+    h <- curl::new_handle()
+    curl::handle_setopt(handle = h, .list = curloptions)
+    curl::handle_setform(handle = h, fileData = curl::form_file(zipfile))
+    html <- readLines(curl::curl(url = url, handle = h))
     link <- XML::getHTMLLinks(html)
     file.remove(zipfile)
 
     # download converted file
     outfile <- file.path(outfolder, unlist(strsplit(basename(link), "_"))[2])
-    PEcAn.utils::download.url(url = link, file = outfile, timeout = 600, .opts = curloptions, retry404 = TRUE)
+    PEcAn.utils::download.url(url = link, file = outfile, timeout = 600, .opts = curloptions, retry = TRUE)
 
     # unzip downloaded file if necessary
     if (file.exists(outfile)) {

base/utils/DESCRIPTION

@@ -33,12 +33,12 @@ Description: The Predictive Ecosystem Carbon Analyzer
     investigation.
 Imports:
     abind (>= 1.4.5),
+    curl,
     dplyr,
     lubridate (>= 1.6.0),
     magrittr,
     ncdf4 (>= 1.15),
     PEcAn.logger,
-    RCurl,
     rlang,
     stringi,
     units

base/utils/R/download.url.R

@@ -1,6 +1,6 @@
 ##' Try and download a file.
 ##' 
-##' This will download a file, if retry404 and 404 is returned it will
+##' This will download a file, if retry is set and 404 is returned it will
 ##' wait until the file is available. If the file is still not available
 ##' after timeout tries, it will return NA. If the file is downloaded
 ##' it will return the name of the file
@@ -13,26 +13,37 @@
 ##' @param timeout number of seconds to wait for file (default 600)
 ##' @param .opts list of options for curl, for example to download from a
 ##'        protected site use list(userpwd=userpass, httpauth = 1L)
-##' @param retry404 retry on a 404, this is used by Brown Dog
+##' @param retry retry if url not found yet, this is used by Brown Dog
 ##' @return returns name of file if successful or NA if not.
 ##' 
 ##' @examples
 ##' \dontrun{
 ##' download.url('http://localhost/', index.html)
 ##' }
-download.url <- function(url, file, timeout = 600, .opts = list(), retry404 = TRUE) {
-  dir.create(basename(file), recursive = TRUE)
+download.url <- function(url, file, timeout = 600, .opts = list(), retry = TRUE) {
   count <- 0
-  while (!RCurl::url.exists(url, .opts = .opts) && count < timeout) {
+  while (retry && !url_found(url) && count < timeout) {
     count <- count + 1
     Sys.sleep(1)
   }
-  if (count >= timeout) {
+  if (count >= timeout || (!retry && !url_found(url))) {
     return(NA)
   }
-  f <- RCurl::CFILE(file, mode = "wb")
-  RCurl::curlPerform(url = url, writedata = f@ref, .opts = .opts)
-  RCurl::close(f)
+  dir.create(dirname(file), recursive = TRUE)
+  res <- curl::curl_download(
+    url = url,
+    destfile = file,
+    handle = curl::new_handle(.list = .opts))
 
-  return(file)
+  res
 } # download.url
+
+
+# An approximate replacement for RCurl::url.exists
+# Treats any 200 status as success (NB does not follow redirects!)
+url_found <- function(url) {
+  h <- curl::new_handle(nobody = 1L) # "nobody" = header-only request
+  res <- curl::curl_fetch_memory(url, handle = h)
+
+  res$status_code %/% 200 == 1
+}

base/utils/R/mail.R

@@ -9,8 +9,6 @@
 
 ##' Sends email. This assumes the program sendmail is installed. 
 ##'
-##' @title Clear EBI-CLUSTER worker node local scratch directories of old PEcAn output
-##' @name sendmail
 ##' @param from the sender of the mail message
 ##' @param to the receipient of the mail message
 ##' @param subject the subject of the mail message

base/visualization/DESCRIPTION

@@ -37,7 +37,6 @@ Imports:
     PEcAn.logger,
     PEcAn.utils,
     plyr (>= 1.8.4),
-    RCurl,
     reshape2,
     rlang,
     stringr(>= 1.1.0)

docker/depends/pecan.depends.R

@@ -29,6 +29,7 @@ wanted <- c(
 'BrownDog',
 'coda',
 'corrplot',
+'curl',
 'data.table',
 'dataone',
 'datapack',

models/biocro/inst/salix_benchmarks.Rmd

@@ -2,10 +2,11 @@ BioCro fitting, parameterization, and testing
 ========================================================
 
 ```{r}
-require(data.table)
-require(lubridate)
-require(ggplot2)
-require(PEcAn.DB)
+library(data.table)
+library(lubridate)
+library(ggplot2)
+library(PEcAn.DB)
+library(curl)
 load(system.file("extdata", "salix.RData", package = "BioCro"))
 settings.xml <- system.file("extdata/pecan.biocro.xml", 
                             package = "PEcAn.BIOCRO")
@@ -34,10 +35,11 @@ salix <- salix.us[stand_age > 2,list(lat = mean(lat), lon = mean(lon), yield = m
 
 ```{r}
 
-require(RCurl)
-file.url <- getURL("https://www.betydb.org/miscanthusyield.csv", 
-                   ssl.verifypeer = FALSE)
-mxg <- read.csv(textConnection(file.url))
+
+mxg_file_con <- curl(
+  url = "https://www.betydb.org/miscanthusyield.csv",
+  handle = new_handle(ssl_verifypeer = FALSE))
+mxg <- read.csv(mxg_file_con)
 
 
 salix$model <- vector(mode="numeric", length = nrow(salix))
@@ -103,4 +105,4 @@ trait.summary <-  salix.traits[sort(n),
 trait.summary2 <- trait.summary[with(trait.summary, rank(n + rank(trait)/1000)),]
 ggplot(data = trait.summary, aes(x = trait, y = n, order = n + rank(trait)/100)) + geom_point() + geom_linerange(aes(ymin = 0, ymax = n)) #+ coord_flip() + theme_bw()
 
-```
+```

models/maat/vignettes/running_maat_in_pecan.Rmd

@@ -190,7 +190,6 @@ In this example we will run ten MAAT model ensembles in serial based on paramete
 ```{r run_maat}
 library(PEcAn.all)
 library(PEcAn.utils)
-library(RCurl)
 
 setwd("~")
 getwd()

modules/data.atmosphere/DESCRIPTION

@@ -25,6 +25,7 @@ Depends:
 Imports:
     abind (>= 1.4.5),
     amerifluxr,
+    curl,
     data.table,
     dplyr,
     geonames (> 0.998),
@@ -45,7 +46,6 @@ Imports:
     PEcAn.utils,
     purrr (>= 0.2.3),
     raster,
-    RCurl,
     REddyProc,
     reshape2,
     rgdal,

modules/data.atmosphere/R/download.GLDAS.R

@@ -128,7 +128,7 @@ download.GLDAS <- function(outfolder, start_date, end_date, site_id, lat.in, lon
         dap_file <- paste0(dap_base, "/", year, "/", doy, "/", dap.log[h, 1], ".ascii?")
 
         # Query lat/lon
-        latlon <- RCurl::getURL(paste0(dap_file, "lat[0:1:599],lon[0:1:1439]"))
+        latlon <- curl::curl_download(paste0(dap_file, "lat[0:1:599],lon[0:1:1439]"))
         lat.ind <- gregexpr("lat", latlon)
         lon.ind <- gregexpr("lon", latlon)
         lats <- as.vector(utils::read.table(
@@ -153,7 +153,7 @@ download.GLDAS <- function(outfolder, start_date, end_date, site_id, lat.in, lon
         }
         dap_query <- substr(dap_query, 2, nchar(dap_query))
 
-        dap.out <- RCurl::getURL(paste0(dap_file, dap_query))
+        dap.out <- curl::curl_download(paste0(dap_file, dap_query))
         for (v in seq_len(nrow(var))) {
           var.now <- var$DAP.name[v]
           ind.1 <- gregexpr(paste(var.now, var.now, sep = "."), dap.out)

modules/data.atmosphere/R/download.NLDAS.R

@@ -123,7 +123,7 @@ download.NLDAS <- function(outfolder, start_date, end_date, site_id, lat.in, lon
                            mo.now, day.mo, ".", hr, ".002.grb.ascii?")
 
         # Query lat/lon
-        latlon <- RCurl::getURL(paste0(dap_file, "lat[0:1:223],lon[0:1:463]"))
+        latlon <- curl::curl_download(paste0(dap_file, "lat[0:1:223],lon[0:1:463]"))
         lat.ind <- gregexpr("lat", latlon)
         lon.ind <- gregexpr("lon", latlon)
         lats <- as.vector(utils::read.table(con <- textConnection(substr(latlon, lat.ind[[1]][3],
@@ -146,7 +146,7 @@ download.NLDAS <- function(outfolder, start_date, end_date, site_id, lat.in, lon
         }
         dap_query <- substr(dap_query, 2, nchar(dap_query))
 
-        dap.out <- RCurl::getURL(paste0(dap_file, dap_query))
+        dap.out <- curl::curl_download(paste0(dap_file, dap_query))
         for (v in seq_len(nrow(var))) {
           var.now <- var$DAP.name[v]
           ind.1   <- gregexpr(paste(var.now, var.now, sep = "."), dap.out)

modules/data.atmosphere/R/met.process.R

@@ -527,8 +527,14 @@ browndog.met <- function(browndog, source, site, start_date, end_date, model, di
 
   userpass <- paste(browndog$username, browndog$password, sep = ":")
   curloptions <- list(userpwd = userpass, httpauth = 1L, followlocation = TRUE)
-  result <- RCurl::postForm(paste0(browndog$url, formatname, "/"),
-                     fileData = RCurl::fileUpload("pecan.xml", xmldata, "text/xml"), .opts = curloptions)
+
+  result <- httr::POST(
+    url = paste0(browndog$url, formatname, "/"),
+    config = do.call(httr::config, curloptions),
+    httr::content_type("text/xml"),
+    body = xmldata)
+  httr::warn_for_status(result)
+  result_txt <- httr::content(result, "text")
   url <- gsub(".*<a.*>(.*)</a>.*", "\\1", result)
   PEcAn.logger::logger.info("browndog download url :", url)
   downloadedfile <- PEcAn.utils::download.url(url, outputfile, 600, curloptions)

modules/data.land/DESCRIPTION

@@ -24,6 +24,7 @@ Description: The Predictive Ecosystem Carbon Analyzer (PEcAn) is a scientific
     efficacy of scientific investigation.
 Imports:
     coda,
+    curl,
     datapack,
     dplyr,
     dplR,
@@ -45,9 +46,9 @@ Imports:
     PEcAn.utils,
     PEcAn.visualization,
     purrr,
+    RCurl,
     rjags,
     rlang,
-    RCurl,
     RPostgreSQL,
     sf,
     sirt,

modules/data.land/R/extract_soil_nc.R

@@ -77,7 +77,7 @@ extract_soil_gssurgo<-function(outdir, lat, lon, size=1, radius=500, depths=c(0.
     #the output is a gml file which need to be downloaded and read as a spatial file but I don't do that.
     #I just read the file as a text and parse it out and try to find the mukey==mapunitkey
     xmll <-
-      RCurl::getURL(mu.Path,
+      curl::curl_download(mu.Path,
                     ssl.verifyhost = FALSE,
                     ssl.verifypeer = FALSE
       )

modules/data.land/R/land.utils.R

@@ -3,7 +3,7 @@ get.elevation <- function(lat, lon) {
 
   url  <- paste("http://www.earthtools.org/height", lat, lon, sep = "/")
 
-  page <- RCurl::getURL(url)
+  page <- paste0(readLines(curl::curl(url)), collapse = "\n")
   ans  <- XML::xmlTreeParse(page, useInternalNodes = TRUE)
   heightNode <- XML::xpathApply(ans, "//meters")[[1]]
   return(as.numeric(XML::xmlValue(heightNode)))

modules/data.land/tests/Rcheck_reference.log

@@ -10,12 +10,12 @@
 * checking package namespace information ... OK
 * checking package dependencies ... OK
 
-Imports includes 32 non-default packages.
+Imports includes 33 non-default packages.
 Importing from so many packages makes the package vulnerable to any of
 them becoming unavailable.  Move as many as possible to Suggests and
 use conditionally.
 * checking package dependencies ... NOTE
-Imports includes 32 non-default packages.
+Imports includes 33 non-default packages.
 Importing from so many packages makes the package vulnerable to any of
 them becoming unavailable.  Move as many as possible to Suggests and
 use conditionally.