[KVM] Direct download agnostic of the storage provider

agent/conf/agent.properties

@@ -115,6 +115,9 @@ domr.scripts.dir=scripts/network/domr/kvm
 # set the hypervisor type, values are: kvm, lxc
 hypervisor.type=kvm
 
+# This parameter specifies a directory on the host local storage for temporary storing direct download templates
+#direct.download.temporary.download.location=/var/lib/libvirt/images
+
 # set the hypervisor URI. Usually there is no need for changing this
 # For KVM: qemu:///system
 # For LXC: lxc:///

agent/src/main/java/com/cloud/agent/direct/download/DirectTemplateDownloader.java

@@ -19,49 +19,15 @@
 
 package com.cloud.agent.direct.download;
 
-public interface DirectTemplateDownloader {
-
-    class DirectTemplateInformation {
-        private String installPath;
-        private Long size;
-        private String checksum;
-
-        public DirectTemplateInformation(String installPath, Long size, String checksum) {
-            this.installPath = installPath;
-            this.size = size;
-            this.checksum = checksum;
-        }
-
-        public String getInstallPath() {
-            return installPath;
-        }
+import com.cloud.utils.Pair;
 
-        public Long getSize() {
-            return size;
-        }
-
-        public String getChecksum() {
-            return checksum;
-        }
-    }
+public interface DirectTemplateDownloader {
 
     /**
      * Perform template download to pool specified on downloader creation
-     * @return true if successful, false if not
-     */
-    boolean downloadTemplate();
-
-    /**
-     * Perform extraction (if necessary) and installation of previously downloaded template
-     * @return true if successful, false if not
-     */
-    boolean extractAndInstallDownloadedTemplate();
-
-    /**
-     * Get template information after it is properly installed on pool
-     * @return template information
+     * @return (true if successful, false if not, download file path)
      */
-    DirectTemplateInformation getTemplateInformation();
+    Pair<Boolean, String> downloadTemplate();
 
     /**
      * Perform checksum validation of previously downloadeed template

agent/src/main/java/com/cloud/agent/direct/download/DirectTemplateDownloaderImpl.java

@@ -19,7 +19,6 @@
 package com.cloud.agent.direct.download;
 
 import com.cloud.utils.exception.CloudRuntimeException;
-import com.cloud.utils.script.Script;
 import org.apache.cloudstack.utils.security.DigestHelper;
 import org.apache.commons.lang.StringUtils;
 import org.apache.log4j.Logger;
@@ -28,35 +27,37 @@
 import java.io.FileInputStream;
 import java.io.IOException;
 import java.security.NoSuchAlgorithmException;
-import java.util.UUID;
 
 public abstract class DirectTemplateDownloaderImpl implements DirectTemplateDownloader {
 
     private String url;
     private String destPoolPath;
     private Long templateId;
     private String downloadedFilePath;
-    private String installPath;
     private String checksum;
     private boolean redownload = false;
+    protected String temporaryDownloadPath;
+
     public static final Logger s_logger = Logger.getLogger(DirectTemplateDownloaderImpl.class.getName());
 
-    protected DirectTemplateDownloaderImpl(final String url, final String destPoolPath, final Long templateId, final String checksum) {
+    protected DirectTemplateDownloaderImpl(final String url, final String destPoolPath, final Long templateId,
+                                           final String checksum, final String temporaryDownloadPath) {
         this.url = url;
         this.destPoolPath = destPoolPath;
         this.templateId = templateId;
         this.checksum = checksum;
+        this.temporaryDownloadPath = temporaryDownloadPath;
     }
 
     private static String directDownloadDir = "template";
 
     /**
      * Return direct download temporary path to download template
      */
-    protected static String getDirectDownloadTempPath(Long templateId) {
+    protected String getDirectDownloadTempPath(Long templateId) {
         String templateIdAsString = String.valueOf(templateId);
-        return directDownloadDir + File.separator + templateIdAsString.substring(0,1) +
-                File.separator + templateIdAsString;
+        return this.temporaryDownloadPath + File.separator + directDownloadDir + File.separator +
+                templateIdAsString.substring(0,1) + File.separator + templateIdAsString;
     }
 
     /**
@@ -113,64 +114,6 @@ public String getFileNameFromUrl() {
         return urlParts[urlParts.length - 1];
     }
 
-    /**
-     * Checks if downloaded template is extractable
-     * @return true if it should be extracted, false if not
-     */
-    private boolean isTemplateExtractable() {
-        String type = Script.runSimpleBashScript("file " + downloadedFilePath + " | awk -F' ' '{print $2}'");
-        return type.equalsIgnoreCase("bzip2") || type.equalsIgnoreCase("gzip") || type.equalsIgnoreCase("zip");
-    }
-
-    @Override
-    public boolean extractAndInstallDownloadedTemplate() {
-        installPath = UUID.randomUUID().toString();
-        if (isTemplateExtractable()) {
-            extractDownloadedTemplate();
-        } else {
-            Script.runSimpleBashScript("mv " + downloadedFilePath + " " + getInstallFullPath());
-        }
-        return true;
-    }
-
-    /**
-     * Return install full path
-     */
-    private String getInstallFullPath() {
-        return destPoolPath + File.separator + installPath;
-    }
-
-    /**
-     * Return extract command to execute given downloaded file
-     */
-    private String getExtractCommandForDownloadedFile() {
-        if (downloadedFilePath.endsWith(".zip")) {
-            return "unzip -p " + downloadedFilePath + " | cat > " + getInstallFullPath();
-        } else if (downloadedFilePath.endsWith(".bz2")) {
-            return "bunzip2 -c " + downloadedFilePath + " > " + getInstallFullPath();
-        } else if (downloadedFilePath.endsWith(".gz")) {
-            return "gunzip -c " + downloadedFilePath + " > " + getInstallFullPath();
-        } else {
-            throw new CloudRuntimeException("Unable to extract template " + templateId + " on " + downloadedFilePath);
-        }
-    }
-
-    /**
-     * Extract downloaded template into installPath, remove compressed file
-     */
-    private void extractDownloadedTemplate() {
-        String extractCommand = getExtractCommandForDownloadedFile();
-        Script.runSimpleBashScript(extractCommand);
-        Script.runSimpleBashScript("rm -f " + downloadedFilePath);
-    }
-
-    @Override
-    public DirectTemplateInformation getTemplateInformation() {
-        String sizeResult = Script.runSimpleBashScript("ls -als " + getInstallFullPath() + " | awk '{print $1}'");
-        long size = Long.parseLong(sizeResult);
-        return new DirectTemplateInformation(installPath, size, checksum);
-    }
-
     @Override
     public boolean validateChecksum() {
         if (StringUtils.isNotBlank(checksum)) {

agent/src/main/java/com/cloud/agent/direct/download/HttpDirectTemplateDownloader.java

@@ -27,6 +27,8 @@
 import java.util.HashMap;
 import java.util.Map;
 
+import com.cloud.utils.Pair;
+import com.cloud.utils.exception.CloudRuntimeException;
 import org.apache.commons.collections.MapUtils;
 import org.apache.commons.httpclient.HttpClient;
 import org.apache.commons.httpclient.HttpStatus;
@@ -35,8 +37,6 @@
 import org.apache.commons.io.IOUtils;
 import org.apache.log4j.Logger;
 
-import com.cloud.utils.exception.CloudRuntimeException;
-
 public class HttpDirectTemplateDownloader extends DirectTemplateDownloaderImpl {
 
     protected HttpClient client;
@@ -45,20 +45,25 @@ public class HttpDirectTemplateDownloader extends DirectTemplateDownloaderImpl {
     protected GetMethod request;
     protected Map<String, String> reqHeaders = new HashMap<>();
 
-    public HttpDirectTemplateDownloader(String url, Long templateId, String destPoolPath, String checksum, Map<String, String> headers, Integer connectTimeout, Integer soTimeout) {
-        super(url, destPoolPath, templateId, checksum);
+    public HttpDirectTemplateDownloader(String url, Long templateId, String destPoolPath, String checksum,
+                                        Map<String, String> headers, Integer connectTimeout, Integer soTimeout, String downloadPath) {
+        super(url, destPoolPath, templateId, checksum, downloadPath);
         s_httpClientManager.getParams().setConnectionTimeout(connectTimeout == null ? 5000 : connectTimeout);
         s_httpClientManager.getParams().setSoTimeout(soTimeout == null ? 5000 : soTimeout);
         client = new HttpClient(s_httpClientManager);
         request = createRequest(url, headers);
         String downloadDir = getDirectDownloadTempPath(templateId);
-        createTemporaryDirectoryAndFile(downloadDir);
+        File tempFile = createTemporaryDirectoryAndFile(downloadDir);
+        setDownloadedFilePath(tempFile.getAbsolutePath());
     }
 
-    protected void createTemporaryDirectoryAndFile(String downloadDir) {
-        createFolder(getDestPoolPath() + File.separator + downloadDir);
-        File f = new File(getDestPoolPath() + File.separator + downloadDir + File.separator + getFileNameFromUrl());
-        setDownloadedFilePath(f.getAbsolutePath());
+    /**
+     * Create download directory (if it does not exist) and set the download file
+     * @return
+     */
+    protected File createTemporaryDirectoryAndFile(String downloadDir) {
+        createFolder(downloadDir);
+        return new File(downloadDir + File.separator + getFileNameFromUrl());
     }
 
     protected GetMethod createRequest(String downloadUrl, Map<String, String> headers) {
@@ -74,12 +79,12 @@ protected GetMethod createRequest(String downloadUrl, Map<String, String> header
     }
 
     @Override
-    public boolean downloadTemplate() {
+    public Pair<Boolean, String> downloadTemplate() {
         try {
             int status = client.executeMethod(request);
             if (status != HttpStatus.SC_OK) {
                 s_logger.warn("Not able to download template, status code: " + status);
-                return false;
+                return new Pair<>(false, null);
             }
             return performDownload();
         } catch (IOException e) {
@@ -89,7 +94,7 @@ public boolean downloadTemplate() {
         }
     }
 
-    protected boolean performDownload() {
+    protected Pair<Boolean, String> performDownload() {
         s_logger.info("Downloading template " + getTemplateId() + " from " + getUrl() + " to: " + getDownloadedFilePath());
         try (
                 InputStream in = request.getResponseBodyAsStream();
@@ -98,8 +103,8 @@ protected boolean performDownload() {
             IOUtils.copy(in, out);
         } catch (IOException e) {
             s_logger.error("Error downloading template " + getTemplateId() + " due to: " + e.getMessage());
-            return false;
+            return new Pair<>(false, null);
         }
-        return true;
+        return new Pair<>(true, getDownloadedFilePath());
     }
 }

agent/src/main/java/com/cloud/agent/direct/download/HttpsDirectTemplateDownloader.java

@@ -19,6 +19,23 @@
 
 package com.cloud.agent.direct.download;
 
+import com.cloud.utils.Pair;
+import com.cloud.utils.exception.CloudRuntimeException;
+import com.cloud.utils.script.Script;
+import org.apache.commons.io.IOUtils;
+import org.apache.http.HttpEntity;
+import org.apache.http.client.methods.CloseableHttpResponse;
+import org.apache.commons.collections.MapUtils;
+import org.apache.http.client.config.RequestConfig;
+import org.apache.http.client.methods.HttpGet;
+import org.apache.http.client.methods.HttpUriRequest;
+import org.apache.http.conn.ssl.SSLConnectionSocketFactory;
+import org.apache.http.conn.ssl.TrustSelfSignedStrategy;
+import org.apache.http.impl.client.CloseableHttpClient;
+import org.apache.http.impl.client.HttpClients;
+import org.apache.http.ssl.SSLContexts;
+
+import javax.net.ssl.SSLContext;
 import java.io.File;
 import java.io.FileInputStream;
 import java.io.FileOutputStream;
@@ -32,31 +49,14 @@
 import java.security.cert.CertificateException;
 import java.util.Map;
 
-import javax.net.ssl.SSLContext;
-
-import org.apache.commons.collections.MapUtils;
-import org.apache.commons.io.IOUtils;
-import org.apache.http.HttpEntity;
-import org.apache.http.client.config.RequestConfig;
-import org.apache.http.client.methods.CloseableHttpResponse;
-import org.apache.http.client.methods.HttpGet;
-import org.apache.http.client.methods.HttpUriRequest;
-import org.apache.http.conn.ssl.SSLConnectionSocketFactory;
-import org.apache.http.conn.ssl.TrustSelfSignedStrategy;
-import org.apache.http.impl.client.CloseableHttpClient;
-import org.apache.http.impl.client.HttpClients;
-import org.apache.http.ssl.SSLContexts;
-
-import com.cloud.utils.exception.CloudRuntimeException;
-import com.cloud.utils.script.Script;
-
 public class HttpsDirectTemplateDownloader extends HttpDirectTemplateDownloader {
 
     private CloseableHttpClient httpsClient;
     private HttpUriRequest req;
 
-    public HttpsDirectTemplateDownloader(String url, Long templateId, String destPoolPath, String checksum, Map<String, String> headers, Integer connectTimeout, Integer soTimeout, Integer connectionRequestTimeout) {
-        super(url, templateId, destPoolPath, checksum, headers, connectTimeout, soTimeout);
+    public HttpsDirectTemplateDownloader(String url, Long templateId, String destPoolPath, String checksum, Map<String, String> headers,
+                                         Integer connectTimeout, Integer soTimeout, Integer connectionRequestTimeout, String temporaryDownloadPath) {
+        super(url, templateId, destPoolPath, checksum, headers, connectTimeout, soTimeout, temporaryDownloadPath);
         SSLContext sslcontext = null;
         try {
             sslcontext = getSSLContext();
@@ -98,7 +98,7 @@ private SSLContext getSSLContext() throws KeyStoreException, NoSuchAlgorithmExce
     }
 
     @Override
-    public boolean downloadTemplate() {
+    public Pair<Boolean, String> downloadTemplate() {
         CloseableHttpResponse response;
         try {
             response = httpsClient.execute(req);
@@ -111,7 +111,7 @@ public boolean downloadTemplate() {
     /**
      * Consume response and persist it on getDownloadedFilePath() file
      */
-    protected boolean consumeResponse(CloseableHttpResponse response) {
+    protected Pair<Boolean, String> consumeResponse(CloseableHttpResponse response) {
         s_logger.info("Downloading template " + getTemplateId() + " from " + getUrl() + " to: " + getDownloadedFilePath());
         if (response.getStatusLine().getStatusCode() != 200) {
             throw new CloudRuntimeException("Error on HTTPS response");
@@ -123,9 +123,9 @@ protected boolean consumeResponse(CloseableHttpResponse response) {
             IOUtils.copy(in, out);
         } catch (Exception e) {
             s_logger.error("Error parsing response for template " + getTemplateId() + " due to: " + e.getMessage());
-            return false;
+            return new Pair<>(false, null);
         }
-        return true;
+        return new Pair<>(true, getDownloadedFilePath());
     }
 
 }