Skip to content

edge-zone,kvm,iso,cks: allow k8s deployment with direct-download iso #8142

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
DaanHoogland merged 11 commits into from
Nov 10, 2023
Merged

edge-zone,kvm,iso,cks: allow k8s deployment with direct-download iso #8142

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
11 commits
Select commit Hold shift + click to select a range
65247f6
wip
shwstppr Mar 3, 2023
3b0ea65
Merge branch '4.18' into cks-version-direct
shwstppr Oct 10, 2023
9241d00
fix
shwstppr Oct 10, 2023
2ec2220
fix for selecting correct pool ref
shwstppr Oct 23, 2023
a32e1e6
fix since
shwstppr Oct 25, 2023
4dd05a9
Merge branch '4.18' into cks-version-direct
shwstppr Oct 26, 2023
a7da524
address testing suggestions
shwstppr Nov 7, 2023
76f8c8f
fix, address comments
shwstppr Nov 7, 2023
49e019d
fix, refactor direct-download version logic
shwstppr Nov 7, 2023
f7c02b9
ui issue
shwstppr Nov 7, 2023
af37049
fix for consecutive zone change
shwstppr Nov 7, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions api/src/main/java/org/apache/cloudstack/api/command/user/iso/RegisterIsoCmd.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -177,6 +177,9 @@ public void setUrl(String url) {
}

public Long getZoneId() {
if (zoneId == null || zoneId == -1) {
return null;
}
return zoneId;
}

Expand Down Expand Up @@ -220,6 +223,10 @@ public boolean isDirectDownload() {
return directDownload == null ? false : directDownload;
}

public void setDirectDownload(Boolean directDownload) {
this.directDownload = directDownload;
}

public boolean isPasswordEnabled() {
return passwordEnabled == null ? false : passwordEnabled;
}
Expand Down
49 changes: 26 additions & 23 deletions ...rage/image/src/main/java/org/apache/cloudstack/storage/image/TemplateDataFactoryImpl.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,11 +20,10 @@

import java.util.ArrayList;
import java.util.List;
import java.util.stream.Collectors;

import javax.inject.Inject;

import com.cloud.hypervisor.Hypervisor;
import com.cloud.utils.exception.CloudRuntimeException;
import org.apache.cloudstack.direct.download.DirectDownloadManager;
import org.apache.cloudstack.engine.subsystem.api.storage.DataObject;
import org.apache.cloudstack.engine.subsystem.api.storage.DataStore;
Expand All @@ -36,18 +35,21 @@
import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreDao;
import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreVO;
import org.apache.cloudstack.storage.image.store.TemplateObject;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.log4j.Logger;
import org.springframework.stereotype.Component;

import com.cloud.host.HostVO;
import com.cloud.host.dao.HostDao;
import com.cloud.hypervisor.Hypervisor;
import com.cloud.storage.DataStoreRole;
import com.cloud.storage.VMTemplateStoragePoolVO;
import com.cloud.storage.VMTemplateStorageResourceAssoc;
import com.cloud.storage.VMTemplateVO;
import com.cloud.storage.dao.VMTemplateDao;
import com.cloud.storage.dao.VMTemplatePoolDao;
import com.cloud.utils.exception.CloudRuntimeException;

@Component
public class TemplateDataFactoryImpl implements TemplateDataFactory {
Expand Down Expand Up @@ -203,12 +205,7 @@ public List<TemplateInfo> listTemplateOnCache(long templateId) {
* Given existing spool refs, return one pool id existing on pools and refs
*/
private Long getOneMatchingPoolIdFromRefs(List<VMTemplateStoragePoolVO> existingRefs, List<StoragePoolVO> pools) {
if (pools.isEmpty()) {
throw new CloudRuntimeException("No storage pools found");
}
if (existingRefs.isEmpty()) {
return pools.get(0).getId();
} else {
if (!existingRefs.isEmpty()) {
for (VMTemplateStoragePoolVO ref : existingRefs) {
for (StoragePoolVO p : pools) {
if (ref.getPoolId() == p.getId()) {
Expand All @@ -217,45 +214,51 @@ private Long getOneMatchingPoolIdFromRefs(List<VMTemplateStoragePoolVO> existing
}
}
}
return null;
return pools.get(0).getId();
}

/**
* Retrieve storage pools with scope = cluster or zone matching clusterId or dataCenterId depending on their scope
* Retrieve storage pools with scope = cluster or zone or local matching clusterId or dataCenterId or hostId depending on their scope
*/
private List<StoragePoolVO> getStoragePoolsFromClusterOrZone(Long clusterId, long dataCenterId, Hypervisor.HypervisorType hypervisorType) {
private List<StoragePoolVO> getStoragePoolsForScope(long dataCenterId, Long clusterId, long hostId, Hypervisor.HypervisorType hypervisorType) {
List<StoragePoolVO> pools = new ArrayList<>();
if (clusterId != null) {
List<StoragePoolVO> clusterPools = primaryDataStoreDao.listPoolsByCluster(clusterId);
clusterPools = clusterPools.stream().filter(p -> !p.isLocal()).collect(Collectors.toList());
pools.addAll(clusterPools);
}
List<StoragePoolVO> zonePools = primaryDataStoreDao.findZoneWideStoragePoolsByHypervisor(dataCenterId, hypervisorType);
pools.addAll(zonePools);
List<StoragePoolVO> localPools = primaryDataStoreDao.findLocalStoragePoolsByHostAndTags(hostId, null);
pools.addAll(localPools);
return pools;
}

protected Long getBypassedTemplateExistingOrNewPoolId(VMTemplateVO templateVO, Long hostId) {
HostVO host = hostDao.findById(hostId);
List<StoragePoolVO> pools = getStoragePoolsForScope(host.getDataCenterId(), host.getClusterId(), hostId, host.getHypervisorType());
if (CollectionUtils.isEmpty(pools)) {
throw new CloudRuntimeException(String.format("No storage pool found to download template: %s", templateVO.getName()));
}
List<VMTemplateStoragePoolVO> existingRefs = templatePoolDao.listByTemplateId(templateVO.getId());
return getOneMatchingPoolIdFromRefs(existingRefs, pools);
}

@Override
public TemplateInfo getReadyBypassedTemplateOnPrimaryStore(long templateId, Long poolId, Long hostId) {
VMTemplateVO templateVO = imageDataDao.findById(templateId);
if (templateVO == null || !templateVO.isDirectDownload()) {
return null;
}
Long pool = poolId;
Long templatePoolId = poolId;
if (poolId == null) {
//Get ISO from existing pool ref
HostVO host = hostDao.findById(hostId);
List<StoragePoolVO> pools = getStoragePoolsFromClusterOrZone(host.getClusterId(), host.getDataCenterId(), host.getHypervisorType());
List<VMTemplateStoragePoolVO> existingRefs = templatePoolDao.listByTemplateId(templateId);
pool = getOneMatchingPoolIdFromRefs(existingRefs, pools);
}
if (pool == null) {
throw new CloudRuntimeException("No storage pool found where to download template: " + templateId);
templatePoolId = getBypassedTemplateExistingOrNewPoolId(templateVO, hostId);
}
VMTemplateStoragePoolVO spoolRef = templatePoolDao.findByPoolTemplate(pool, templateId, null);
VMTemplateStoragePoolVO spoolRef = templatePoolDao.findByPoolTemplate(templatePoolId, templateId, null);
if (spoolRef == null) {
directDownloadManager.downloadTemplate(templateId, pool, hostId);
directDownloadManager.downloadTemplate(templateId, templatePoolId, hostId);
}
DataStore store = storeMgr.getDataStore(pool, DataStoreRole.Primary);
DataStore store = storeMgr.getDataStore(templatePoolId, DataStoreRole.Primary);
return this.getTemplate(templateId, store);
}

Expand Down
6 changes: 4 additions & 2 deletions ...hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStoragePoolManager.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -294,12 +294,14 @@ public KVMStoragePool getStoragePoolByURI(String uri) {
String uuid = null;
String sourceHost = "";
StoragePoolType protocol = null;
if (storageUri.getScheme().equalsIgnoreCase("nfs") || storageUri.getScheme().equalsIgnoreCase("NetworkFilesystem")) {
final String scheme = storageUri.getScheme().toLowerCase();
List<String> acceptedSchemes = List.of("nfs", "networkfilesystem", "filesystem");
if (acceptedSchemes.contains(scheme)) {
sourcePath = storageUri.getPath();
sourcePath = sourcePath.replace("//", "/");
sourceHost = storageUri.getHost();
uuid = UUID.nameUUIDFromBytes(new String(sourceHost + sourcePath).getBytes()).toString();
protocol = StoragePoolType.NetworkFilesystem;
protocol = scheme.equals("filesystem") ? StoragePoolType.Filesystem: StoragePoolType.NetworkFilesystem;
}

// secondary storage registers itself through here
Expand Down
59 changes: 35 additions & 24 deletions ...s/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,23 +25,26 @@
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Paths;
import java.text.DateFormat;
import java.text.SimpleDateFormat;
import java.util.Arrays;
import java.util.Date;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.UUID;
import java.util.stream.Collectors;

import javax.naming.ConfigurationException;

import org.apache.cloudstack.direct.download.DirectDownloadHelper;
import org.apache.cloudstack.direct.download.DirectTemplateDownloader;
import com.cloud.storage.ScopeType;
import com.cloud.storage.Volume;
import org.apache.cloudstack.agent.directdownload.DirectDownloadAnswer;
import org.apache.cloudstack.agent.directdownload.DirectDownloadCommand;
import org.apache.cloudstack.direct.download.DirectDownloadHelper;
import org.apache.cloudstack.direct.download.DirectTemplateDownloader;
import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotInfo;
import org.apache.cloudstack.storage.command.AttachAnswer;
import org.apache.cloudstack.storage.command.AttachCommand;
Expand Down Expand Up @@ -71,7 +74,10 @@
import org.apache.cloudstack.utils.qemu.QemuObject;
import org.apache.commons.collections.MapUtils;
import org.apache.commons.io.FileUtils;

import org.apache.commons.lang3.BooleanUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.builder.ToStringBuilder;
import org.apache.commons.lang3.builder.ToStringStyle;
import org.apache.log4j.Logger;
import org.libvirt.Connect;
import org.libvirt.Domain;
Expand Down Expand Up @@ -110,9 +116,11 @@
import com.cloud.hypervisor.kvm.resource.wrapper.LibvirtUtilitiesHelper;
import com.cloud.storage.JavaStorageLayer;
import com.cloud.storage.MigrationOptions;
import com.cloud.storage.ScopeType;
import com.cloud.storage.Storage.ImageFormat;
import com.cloud.storage.Storage.StoragePoolType;
import com.cloud.storage.StorageLayer;
import com.cloud.storage.Volume;
import com.cloud.storage.resource.StorageProcessor;
import com.cloud.storage.template.Processor;
import com.cloud.storage.template.Processor.FormatInfo;
Expand All @@ -126,16 +134,6 @@
import com.cloud.utils.storage.S3.S3Utils;
import com.cloud.vm.VmDetailConstants;

import java.nio.file.Files;
import java.nio.file.Paths;
import java.util.HashSet;
import java.util.Set;
import java.util.stream.Collectors;
import org.apache.commons.lang3.BooleanUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.builder.ToStringBuilder;
import org.apache.commons.lang3.builder.ToStringStyle;

public class KVMStorageProcessor implements StorageProcessor {
private static final Logger s_logger = Logger.getLogger(KVMStorageProcessor.class);
private final KVMStoragePoolManager storagePoolMgr;
Expand Down Expand Up @@ -1074,16 +1072,23 @@ private void deleteSnapshotOnPrimary(final CopyCommand cmd, final SnapshotObject
s_logger.debug(String.format("This backup is temporary, not deleting snapshot [%s] on primary storage [%s]", snapshotPath, primaryPool.getUuid()));
}
}
protected synchronized void attachOrDetachISO(final Connect conn, final String vmName, String isoPath, final boolean isAttach, Map<String, String> params) throws

protected synchronized void attachOrDetachISO(final Connect conn, final String vmName, String isoPath, final boolean isAttach, Map<String, String> params, DataStoreTO store) throws
LibvirtException, InternalErrorException {
DiskDef iso = new DiskDef();
boolean isUefiEnabled = MapUtils.isNotEmpty(params) && params.containsKey("UEFI");
if (isoPath != null && isAttach) {
final int index = isoPath.lastIndexOf("/");
final String path = isoPath.substring(0, index);
final String name = isoPath.substring(index + 1);
final KVMStoragePool secondaryPool = storagePoolMgr.getStoragePoolByURI(path);
final KVMPhysicalDisk isoVol = secondaryPool.getPhysicalDisk(name);
KVMStoragePool storagePool;
if (store instanceof PrimaryDataStoreTO) {
PrimaryDataStoreTO primaryDataStoreTO = (PrimaryDataStoreTO)store;
storagePool = storagePoolMgr.getStoragePool(primaryDataStoreTO.getPoolType(), store.getUuid());
} else {
storagePool = storagePoolMgr.getStoragePoolByURI(path);
}
final KVMPhysicalDisk isoVol = storagePool.getPhysicalDisk(name);
isoPath = isoVol.getPath();

iso.defISODisk(isoPath, isUefiEnabled);
Expand Down Expand Up @@ -1112,7 +1117,7 @@ public Answer attachIso(final AttachCommand cmd) {
try {
String dataStoreUrl = getDataStoreUrlFromStore(store);
final Connect conn = LibvirtConnection.getConnectionByVmName(cmd.getVmName());
attachOrDetachISO(conn, cmd.getVmName(), dataStoreUrl + File.separator + isoTO.getPath(), true, cmd.getControllerInfo());
attachOrDetachISO(conn, cmd.getVmName(), dataStoreUrl + File.separator + isoTO.getPath(), true, cmd.getControllerInfo(), store);
} catch (final LibvirtException e) {
return new Answer(cmd, false, e.toString());
} catch (final InternalErrorException e) {
Expand All @@ -1133,7 +1138,7 @@ public Answer dettachIso(final DettachCommand cmd) {
try {
String dataStoreUrl = getDataStoreUrlFromStore(store);
final Connect conn = LibvirtConnection.getConnectionByVmName(cmd.getVmName());
attachOrDetachISO(conn, cmd.getVmName(), dataStoreUrl + File.separator + isoTO.getPath(), false, cmd.getParams());
attachOrDetachISO(conn, cmd.getVmName(), dataStoreUrl + File.separator + isoTO.getPath(), false, cmd.getParams(), store);
} catch (final LibvirtException e) {
return new Answer(cmd, false, e.toString());
} catch (final InternalErrorException e) {
Expand All @@ -1149,19 +1154,25 @@ public Answer dettachIso(final DettachCommand cmd) {
* Return data store URL from store
*/
private String getDataStoreUrlFromStore(DataStoreTO store) {
if (!(store instanceof NfsTO) && (!(store instanceof PrimaryDataStoreTO) ||
store instanceof PrimaryDataStoreTO && !((PrimaryDataStoreTO) store).getPoolType().equals(StoragePoolType.NetworkFilesystem))) {
List<StoragePoolType> supportedPoolType = List.of(StoragePoolType.NetworkFilesystem, StoragePoolType.Filesystem);
if (!(store instanceof NfsTO) && (!(store instanceof PrimaryDataStoreTO) || !supportedPoolType.contains(((PrimaryDataStoreTO) store).getPoolType()))) {
s_logger.error(String.format("Unsupported protocol, store: %s", store.getUuid()));
throw new InvalidParameterValueException("unsupported protocol");
}

if (store instanceof NfsTO) {
NfsTO nfsStore = (NfsTO)store;
return nfsStore.getUrl();
} else if (store instanceof PrimaryDataStoreTO && ((PrimaryDataStoreTO) store).getPoolType().equals(StoragePoolType.NetworkFilesystem)) {
} else if (store instanceof PrimaryDataStoreTO) {
//In order to support directly downloaded ISOs
StoragePoolType poolType = ((PrimaryDataStoreTO)store).getPoolType();
String psHost = ((PrimaryDataStoreTO) store).getHost();
String psPath = ((PrimaryDataStoreTO) store).getPath();
return "nfs://" + psHost + File.separator + psPath;
if (StoragePoolType.NetworkFilesystem.equals(poolType)) {
return "nfs://" + psHost + File.separator + psPath;
} else if (StoragePoolType.Filesystem.equals(poolType)) {
return StoragePoolType.Filesystem.toString().toLowerCase() + "://" + psHost + File.separator + psPath;
}
}
return store.getUrl();
}
Expand Down
15 changes: 13 additions & 2 deletions ...etes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -118,6 +118,7 @@
import com.cloud.network.dao.NetworkDao;
import com.cloud.network.dao.NetworkVO;
import com.cloud.network.dao.PhysicalNetworkDao;
import com.cloud.network.router.NetworkHelper;
import com.cloud.network.rules.FirewallRule;
import com.cloud.network.rules.FirewallRuleVO;
import com.cloud.network.security.SecurityGroupManager;
Expand Down Expand Up @@ -243,6 +244,8 @@ public class KubernetesClusterManagerImpl extends ManagerBase implements Kuberne
private SecurityGroupManager securityGroupManager;
@Inject
public SecurityGroupService securityGroupService;
@Inject
public NetworkHelper networkHelper;

private void logMessage(final Level logLevel, final String message, final Exception e) {
if (logLevel == Level.WARN) {
Expand Down Expand Up @@ -347,8 +350,12 @@ private IpAddress getSourceNatIp(Network network) {

public VMTemplateVO getKubernetesServiceTemplate(DataCenter dataCenter, Hypervisor.HypervisorType hypervisorType) {
VMTemplateVO template = templateDao.findSystemVMReadyTemplate(dataCenter.getId(), hypervisorType);
if (DataCenter.Type.Edge.equals(dataCenter.getType()) && template != null && !template.isDirectDownload()) {
LOGGER.debug(String.format("Template %s can not be used for edge zone %s", template, dataCenter));
template = templateDao.findRoutingTemplate(hypervisorType, networkHelper.getHypervisorRouterTemplateConfigMap().get(hypervisorType).valueIn(dataCenter.getId()));
}
if (template == null) {
throw new CloudRuntimeException("Not able to find the System templates or not downloaded in zone " + dataCenter.getId());
throw new CloudRuntimeException("Not able to find the System or Routing template in ready state for the zone " + dataCenter.getUuid());
}
return template;
}
Expand Down Expand Up @@ -650,7 +657,11 @@ private void validateKubernetesClusterCreateParameters(final CreateKubernetesClu
}

if (Grouping.AllocationState.Disabled == zone.getAllocationState()) {
throw new PermissionDeniedException(String.format("Cannot perform this operation, zone ID: %s is currently disabled", zone.getUuid()));
throw new PermissionDeniedException(String.format("Zone ID: %s is currently disabled", zone.getUuid()));
}

if (DataCenter.Type.Edge.equals(zone.getType()) && networkId == null) {
throw new PermissionDeniedException("Kubernetes clusters cannot be created on an edge zone without an existing network");
}

if (!isKubernetesServiceConfigured(zone)) {
Expand Down
Loading