From 6c867956617bba1b390d5c12f83ed3c9121693ca Mon Sep 17 00:00:00 2001 From: Lucas Moura Date: Fri, 13 Nov 2020 14:46:32 -0300 Subject: [PATCH 1/8] Fix LXD image_serial for vms --- examples/lxd.py | 2 ++ pycloudlib/azure/cloud.py | 2 +- pycloudlib/cloud.py | 2 +- pycloudlib/ec2/cloud.py | 2 +- pycloudlib/gce/cloud.py | 2 +- pycloudlib/kvm/cloud.py | 2 +- pycloudlib/lxd/cloud.py | 6 ++++-- pycloudlib/oci/cloud.py | 2 +- 8 files changed, 12 insertions(+), 8 deletions(-) diff --git a/examples/lxd.py b/examples/lxd.py index d4083898..a1a92ba0 100755 --- a/examples/lxd.py +++ b/examples/lxd.py @@ -192,6 +192,8 @@ def launch_virtual_machine(): ) image_id = lxd.released_image(release=RELEASE, is_vm=True) + image_serial = lxd.image_serial(image_id, is_vm=True) + print("Image serial: {}".format(image_serial)) name = 'pycloudlib-vm' inst = lxd.launch( name=name, image_id=image_id, is_vm=True) diff --git a/pycloudlib/azure/cloud.py b/pycloudlib/azure/cloud.py index 647653d2..8656ae7c 100644 --- a/pycloudlib/azure/cloud.py +++ b/pycloudlib/azure/cloud.py @@ -83,7 +83,7 @@ def __init__( self.resource_group = self._create_resource_group() self.base_tag = tag - def image_serial(self, image_id): + def image_serial(self, image_id, **kwargs): """Find the image serial of the latest daily image for a particular release. Args: diff --git a/pycloudlib/cloud.py b/pycloudlib/cloud.py index ef129b3f..c8720744 100644 --- a/pycloudlib/cloud.py +++ b/pycloudlib/cloud.py @@ -71,7 +71,7 @@ def daily_image(self, release, **kwargs): raise NotImplementedError @abstractmethod - def image_serial(self, image_id): + def image_serial(self, image_id, **kwargs): """Find the image serial of the latest daily image for a particular release. Args: diff --git a/pycloudlib/ec2/cloud.py b/pycloudlib/ec2/cloud.py index 4aad3bf5..b3d395a0 100644 --- a/pycloudlib/ec2/cloud.py +++ b/pycloudlib/ec2/cloud.py @@ -103,7 +103,7 @@ def daily_image(self, release, arch='amd64', root_store='ssd'): image = self._find_image(release, arch, root_store) return image['id'] - def image_serial(self, image_id): + def image_serial(self, image_id, **kwargs): """Find the image serial of a given EC2 image ID. Args: diff --git a/pycloudlib/gce/cloud.py b/pycloudlib/gce/cloud.py index 9e144205..7d19dc62 100644 --- a/pycloudlib/gce/cloud.py +++ b/pycloudlib/gce/cloud.py @@ -92,7 +92,7 @@ def daily_image(self, release, arch='amd64'): self._log.debug('finding daily Ubuntu image for %s', release) return self._find_image(release, daily=True, arch=arch) - def image_serial(self, image_id): + def image_serial(self, image_id, **kwargs): """Find the image serial of the latest daily image for a particular release. Args: diff --git a/pycloudlib/kvm/cloud.py b/pycloudlib/kvm/cloud.py index d1fe969b..45b76b32 100644 --- a/pycloudlib/kvm/cloud.py +++ b/pycloudlib/kvm/cloud.py @@ -156,7 +156,7 @@ def daily_image(self, release, arch=LOCAL_UBUNTU_ARCH): image_data['sha256']) return image - def image_serial(self, image_id): + def image_serial(self, image_id, **kwargs): """Find the image serial of a given LXD image. Args: diff --git a/pycloudlib/lxd/cloud.py b/pycloudlib/lxd/cloud.py index c882e011..d6fc7efe 100644 --- a/pycloudlib/lxd/cloud.py +++ b/pycloudlib/lxd/cloud.py @@ -448,11 +448,13 @@ def _image_info(self, image_id, is_vm=False): return image_info - def image_serial(self, image_id): + def image_serial(self, image_id, is_vm=False, **kwargs): """Find the image serial of a given LXD image. Args: image_id: string, LXD image fingerprint + is_vm: boolean, specify if the image_id represents a + virtual machine Returns: string, serial of latest image @@ -461,7 +463,7 @@ def image_serial(self, image_id): self._log.debug( 'finding image serial for LXD Ubuntu image %s', image_id) - image_info = self._image_info(image_id) + image_info = self._image_info(image_id, is_vm=is_vm) return image_info[0]['version_name'] diff --git a/pycloudlib/oci/cloud.py b/pycloudlib/oci/cloud.py index 759245d0..d47bf222 100644 --- a/pycloudlib/oci/cloud.py +++ b/pycloudlib/oci/cloud.py @@ -109,7 +109,7 @@ def daily_image(self, release, operating_system='Canonical Ubuntu'): image_id = image_response.data[0].id return image_id - def image_serial(self, image_id): + def image_serial(self, image_id, **kwargs): """Find the image serial of the latest daily image for a particular release. Args: From 4f0a756e6599f3240ec8f6f66fbf85a470f9c524 Mon Sep 17 00:00:00 2001 From: Lucas Moura Date: Fri, 13 Nov 2020 15:02:11 -0300 Subject: [PATCH 2/8] Add key type to ssh keys generated by paramiko --- pycloudlib/lxd/cloud.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/pycloudlib/lxd/cloud.py b/pycloudlib/lxd/cloud.py index d6fc7efe..c56f7f7e 100644 --- a/pycloudlib/lxd/cloud.py +++ b/pycloudlib/lxd/cloud.py @@ -238,11 +238,19 @@ def init( profile_list = [profile_name] if self.key_pair: + pub_key = self.key_pair.public_key_content + + # When we create keys through paramiko, we end up not + # having the key type on the public key content. Because + # of that, we are manually adding the ssh-rsa type into it + if "ssh-" not in pub_key: + pub_key = "ssh-rsa {}".format(pub_key) + ssh_user_data = textwrap.dedent( """\ ssh_authorized_keys: - {} - """.format(self.key_pair.public_key_content) + """.format(pub_key) ) if user_data: From 4cb448f5c31fe61a0152aa453cdd10b8e34b04ca Mon Sep 17 00:00:00 2001 From: Lucas Moura Date: Fri, 13 Nov 2020 15:26:50 -0300 Subject: [PATCH 3/8] Fix launch of LXD instances with no name --- examples/lxd.py | 3 +++ pycloudlib/lxd/cloud.py | 5 ++++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/examples/lxd.py b/examples/lxd.py index a1a92ba0..cee0b3ed 100755 --- a/examples/lxd.py +++ b/examples/lxd.py @@ -149,6 +149,9 @@ def launch_options(): def basic_lifecycle(): """Demonstrate basic set of lifecycle operations with LXD.""" lxd = pycloudlib.LXD('example-basic') + inst = lxd.launch(image_id=RELEASE) + inst.delete() + name = 'pycloudlib-daily' inst = lxd.launch(name=name, image_id=RELEASE) inst.console_log() diff --git a/pycloudlib/lxd/cloud.py b/pycloudlib/lxd/cloud.py index c56f7f7e..1801a93f 100644 --- a/pycloudlib/lxd/cloud.py +++ b/pycloudlib/lxd/cloud.py @@ -221,7 +221,10 @@ def init( release = self._daily_remote + ':' + release self._log.debug("Full release to launch: '%s'", release) - cmd = ['lxc', 'init', release, name] + cmd = ['lxc', 'init', release] + + if name: + cmd.append(name) if is_vm: cmd.append('--vm') From e4f47bbd04ecdb44d793b3fc4637ef58cc2d672b Mon Sep 17 00:00:00 2001 From: Lucas Moura Date: Fri, 13 Nov 2020 15:32:17 -0300 Subject: [PATCH 4/8] Only use images remote for xenial vm images --- pycloudlib/lxd/cloud.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pycloudlib/lxd/cloud.py b/pycloudlib/lxd/cloud.py index 1801a93f..a7f5c7d8 100644 --- a/pycloudlib/lxd/cloud.py +++ b/pycloudlib/lxd/cloud.py @@ -399,7 +399,7 @@ def _search_for_image( string, LXD fingerprint of latest image """ - if release == "xenial": + if is_vm and release == "xenial": # xenial needs to launch images:ubuntu/16.04/cloud # because it contains the HWE kernel which has vhost-vsock support return self.XENIAL_IMAGE_VSOCK_SUPPORT From 6304e839b45896fd72c8ba66df151d3f7a844976 Mon Sep 17 00:00:00 2001 From: Lucas Moura Date: Mon, 16 Nov 2020 12:45:06 -0300 Subject: [PATCH 5/8] Split LXD containers and vm into distinct classes --- examples/lxd.py | 8 +- pycloudlib/__init__.py | 3 +- pycloudlib/azure/cloud.py | 2 +- pycloudlib/cloud.py | 2 +- pycloudlib/ec2/cloud.py | 2 +- pycloudlib/gce/cloud.py | 2 +- pycloudlib/kvm/cloud.py | 2 +- pycloudlib/lxd/cloud.py | 437 ++++++++++++++++++++--------- pycloudlib/lxd/tests/test_cloud.py | 18 +- pycloudlib/oci/cloud.py | 2 +- 10 files changed, 333 insertions(+), 145 deletions(-) diff --git a/examples/lxd.py b/examples/lxd.py index cee0b3ed..93441847 100755 --- a/examples/lxd.py +++ b/examples/lxd.py @@ -177,7 +177,7 @@ def basic_lifecycle(): def launch_virtual_machine(): """Demonstrate launching virtual machine scenario.""" - lxd = pycloudlib.LXD('example-vm') + lxd = pycloudlib.LXDVirtualMachine('example-vm') pub_key_path = "lxd-pubkey" priv_key_path = "lxd-privkey" @@ -194,12 +194,12 @@ def launch_virtual_machine(): private_key_path=priv_key_path ) - image_id = lxd.released_image(release=RELEASE, is_vm=True) - image_serial = lxd.image_serial(image_id, is_vm=True) + image_id = lxd.released_image(release=RELEASE) + image_serial = lxd.image_serial(image_id) print("Image serial: {}".format(image_serial)) name = 'pycloudlib-vm' inst = lxd.launch( - name=name, image_id=image_id, is_vm=True) + name=name, image_id=image_id) print("Is vm: {}".format(inst.is_vm)) result = inst.execute("lsb_release -a") print(result) diff --git a/pycloudlib/__init__.py b/pycloudlib/__init__.py index dd48e256..498321d2 100644 --- a/pycloudlib/__init__.py +++ b/pycloudlib/__init__.py @@ -6,7 +6,7 @@ from pycloudlib.azure.cloud import Azure from pycloudlib.ec2.cloud import EC2 from pycloudlib.gce.cloud import GCE -from pycloudlib.lxd.cloud import LXD +from pycloudlib.lxd.cloud import LXD, LXDVirtualMachine from pycloudlib.kvm.cloud import KVM from pycloudlib.oci.cloud import OCI @@ -15,6 +15,7 @@ 'EC2', 'GCE', 'LXD', + 'LXDVirtualMachine', 'KVM', 'OCI', ] diff --git a/pycloudlib/azure/cloud.py b/pycloudlib/azure/cloud.py index 8656ae7c..647653d2 100644 --- a/pycloudlib/azure/cloud.py +++ b/pycloudlib/azure/cloud.py @@ -83,7 +83,7 @@ def __init__( self.resource_group = self._create_resource_group() self.base_tag = tag - def image_serial(self, image_id, **kwargs): + def image_serial(self, image_id): """Find the image serial of the latest daily image for a particular release. Args: diff --git a/pycloudlib/cloud.py b/pycloudlib/cloud.py index c8720744..ef129b3f 100644 --- a/pycloudlib/cloud.py +++ b/pycloudlib/cloud.py @@ -71,7 +71,7 @@ def daily_image(self, release, **kwargs): raise NotImplementedError @abstractmethod - def image_serial(self, image_id, **kwargs): + def image_serial(self, image_id): """Find the image serial of the latest daily image for a particular release. Args: diff --git a/pycloudlib/ec2/cloud.py b/pycloudlib/ec2/cloud.py index b3d395a0..4aad3bf5 100644 --- a/pycloudlib/ec2/cloud.py +++ b/pycloudlib/ec2/cloud.py @@ -103,7 +103,7 @@ def daily_image(self, release, arch='amd64', root_store='ssd'): image = self._find_image(release, arch, root_store) return image['id'] - def image_serial(self, image_id, **kwargs): + def image_serial(self, image_id): """Find the image serial of a given EC2 image ID. Args: diff --git a/pycloudlib/gce/cloud.py b/pycloudlib/gce/cloud.py index 7d19dc62..9e144205 100644 --- a/pycloudlib/gce/cloud.py +++ b/pycloudlib/gce/cloud.py @@ -92,7 +92,7 @@ def daily_image(self, release, arch='amd64'): self._log.debug('finding daily Ubuntu image for %s', release) return self._find_image(release, daily=True, arch=arch) - def image_serial(self, image_id, **kwargs): + def image_serial(self, image_id): """Find the image serial of the latest daily image for a particular release. Args: diff --git a/pycloudlib/kvm/cloud.py b/pycloudlib/kvm/cloud.py index 45b76b32..d1fe969b 100644 --- a/pycloudlib/kvm/cloud.py +++ b/pycloudlib/kvm/cloud.py @@ -156,7 +156,7 @@ def daily_image(self, release, arch=LOCAL_UBUNTU_ARCH): image_data['sha256']) return image - def image_serial(self, image_id, **kwargs): + def image_serial(self, image_id): """Find the image serial of a given LXD image. Args: diff --git a/pycloudlib/lxd/cloud.py b/pycloudlib/lxd/cloud.py index a7f5c7d8..49743c96 100644 --- a/pycloudlib/lxd/cloud.py +++ b/pycloudlib/lxd/cloud.py @@ -3,6 +3,7 @@ import io import re import textwrap +from abc import abstractmethod import paramiko from pycloudlib.cloud import BaseCloud @@ -29,18 +30,13 @@ def __init__(self, release, is_vm): ) -class LXD(BaseCloud): - """LXD Cloud Class.""" +class BaseLXD(BaseCloud): + """LXD Base Cloud Class.""" _type = 'lxd' _daily_remote = 'ubuntu-daily' _releases_remote = 'ubuntu' - XENIAL_IMAGE_VSOCK_SUPPORT = "images:ubuntu/16.04/cloud" - VM_HASH_KEY = "combined_disk1-img_sha256" - TRUSTY_CONTAINER_HASH_KEY = "combined_rootxz_sha256" - CONTAINER_HASH_KEY = "combined_squashfs_sha256" - def __init__(self, tag, timestamp_suffix=True): """Initialize LXD cloud class. @@ -156,45 +152,14 @@ def create_key_pair(self): pub_key = key.get_base64() key.write_private_key(priv_str, password=None) - return pub_key, priv_str.getvalue() - - def _extract_release_from_image_id(self, image_id, is_vm=False): - """Extract the base release from the image_id. - - Args: - image_id: string, [:], what release to launch - (default remote: ) - - Returns: - A string contaning the base release from the image_id that is used - to launch the image. - """ - release_regex = ( - "(.*ubuntu.*(?P(" + - "|".join(UBUNTU_RELEASE_VERSION_MAP) + "|" + - "|".join(UBUNTU_RELEASE_VERSION_MAP.values()) + - ")).*)" - ) - ubuntu_match = re.match(release_regex, image_id) - if ubuntu_match: - release = ubuntu_match.groupdict()["release"] - for codename, version in UBUNTU_RELEASE_VERSION_MAP.items(): - if release in (codename, version): - return codename - - # If we have a hash in the image_id we need to query simplestreams to - # identify the release. - return self._image_info(image_id, is_vm)[0]["release"] + return "ssh-rsa {}".format(pub_key), priv_str.getvalue() # pylint: disable=R0914,R0912,R0915 - def init( + def _prepare_command( self, name, release, ephemeral=False, network=None, storage=None, inst_type=None, profile_list=None, user_data=None, - config_dict=None, is_vm=False): - """Init a container. - - This will initialize a container, but not launch or start it. - If no remote is specified pycloudlib default to daily images. + config_dict=None): + """Build a the command to be used to launch the LXD instance. Args: name: string, what to call the instance @@ -207,12 +172,10 @@ def init( profile_list: list, optional, profile(s) to use user_data: used by cloud-init to run custom scripts/configuration config_dict: dict, optional, configuration values to pass - is_vm: boolean, optional, defines if a virtual machine will - be created Returns: - The created LXD instance object - + A list of string representing the command to be run to + launch the LXD instance. """ profile_list = profile_list if profile_list else [] config_dict = config_dict if config_dict else {} @@ -226,34 +189,12 @@ def init( if name: cmd.append(name) - if is_vm: - cmd.append('--vm') - base_release = self._extract_release_from_image_id(release, is_vm) - - if not profile_list: - profile_name = "pycloudlib-vm-{}".format(base_release) - - self.create_profile( - profile_name=profile_name, - profile_config=base_vm_profiles[base_release] - ) - - profile_list = [profile_name] - if self.key_pair: - pub_key = self.key_pair.public_key_content - - # When we create keys through paramiko, we end up not - # having the key type on the public key content. Because - # of that, we are manually adding the ssh-rsa type into it - if "ssh-" not in pub_key: - pub_key = "ssh-rsa {}".format(pub_key) - ssh_user_data = textwrap.dedent( """\ ssh_authorized_keys: - {} - """.format(pub_key) + """.format(self.key_pair.public_key_content) ) if user_data: @@ -297,18 +238,58 @@ def init( cmd.append('--config') cmd.append('user.user-data=%s' % user_data) - self._log.debug('Creating new instance...') + return cmd + + def init( + self, name, release, ephemeral=False, network=None, storage=None, + inst_type=None, profile_list=None, user_data=None, + config_dict=None): + """Init a container. + + This will initialize a container, but not launch or start it. + If no remote is specified pycloudlib default to daily images. + + Args: + name: string, what to call the instance + release: string, [:], what release to launch + (default remote: ) + ephemeral: boolean, ephemeral, otherwise persistent + network: string, optional, network name to use + storage: string, optional, storage name to use + inst_type: string, optional, type to use + profile_list: list, optional, profile(s) to use + user_data: used by cloud-init to run custom scripts/configuration + config_dict: dict, optional, configuration values to pass + + Returns: + The created LXD instance object + + """ + cmd = self._prepare_command( + name=name, + release=release, + ephemeral=ephemeral, + network=network, + storage=storage, + inst_type=inst_type, + profile_list=profile_list, + user_data=user_data, + config_dict=config_dict + ) + print(cmd) result = subp(cmd) + if not name: name = result.split('Instance name is: ')[1] + self._log.debug('Created %s', name) return LXDInstance(name, self.key_pair) def launch(self, image_id, instance_type=None, user_data=None, wait=True, name=None, ephemeral=False, network=None, storage=None, - profile_list=None, config_dict=None, is_vm=False, **kwargs): + profile_list=None, config_dict=None, **kwargs): """Set up and launch a container. This will init and start a container with the provided settings. @@ -325,27 +306,32 @@ def launch(self, image_id, instance_type=None, user_data=None, wait=True, storage: string, storage name to use profile_list: list, profile(s) to use config_dict: dict, configuration values to pass - is_vm: boolean, optional, defines if a virtual machine will - be created Returns: The created LXD instance object """ - instance = self.init(name, image_id, ephemeral, network, - storage, instance_type, profile_list, user_data, - config_dict, is_vm) + instance = self.init( + name=name, + release=image_id, + ephemeral=ephemeral, + network=network, + storage=storage, + inst_type=instance_type, + profile_list=profile_list, + user_data=user_data, + config_dict=config_dict + ) instance.start(wait) + return instance - def released_image(self, release, arch=LOCAL_UBUNTU_ARCH, is_vm=False): + def released_image(self, release, arch=LOCAL_UBUNTU_ARCH): """Find the LXD fingerprint of the latest released image. Args: release: string, Ubuntu release to look for arch: string, architecture to use - is_vm: boolean, specify if the image_id represents a - virtual machine Returns: string, LXD fingerprint of latest image @@ -356,18 +342,15 @@ def released_image(self, release, arch=LOCAL_UBUNTU_ARCH, is_vm=False): remote=self._releases_remote, daily=False, release=release, - arch=arch, - is_vm=is_vm + arch=arch ) - def daily_image(self, release, arch=LOCAL_UBUNTU_ARCH, is_vm=False): + def daily_image(self, release, arch=LOCAL_UBUNTU_ARCH): """Find the LXD fingerprint of the latest daily image. Args: release: string, Ubuntu release to look for arch: string, architecture to use - is_vm: boolean, specify if the image_id represents a - virtual machine Returns: string, LXD fingerprint of latest image @@ -378,12 +361,30 @@ def daily_image(self, release, arch=LOCAL_UBUNTU_ARCH, is_vm=False): remote=self._daily_remote, daily=True, release=release, - arch=arch, - is_vm=is_vm + arch=arch ) + @abstractmethod + def _get_image_hash_key(self, release=None): + """Get the correct hash key to be used to launch LXD instance. + + When query simplestreams for image information, we receive a + dictionary of metadata. In that metadata we have the necessary + information to allows us to launch the required image. However, + we must know which key to use in the metadata dict to allows + to launch the image. + + Args: + release: string, optional, Ubuntu release + + Returns + A string specifying which key of the metadata dictionary + should be used to launch the image. + """ + raise NotImplementedError + def _search_for_image( - self, remote, daily, release, arch=LOCAL_UBUNTU_ARCH, is_vm=False + self, remote, daily, release, arch=LOCAL_UBUNTU_ARCH ): """Find the LXD fingerprint in a given remote. @@ -392,43 +393,22 @@ def _search_for_image( daily: boolean, search on daily remote release: string, Ubuntu release to look for arch: string, architecture to use - is_vm: boolean, specify if the image_id represents a - virtual machine Returns: string, LXD fingerprint of latest image """ - if is_vm and release == "xenial": - # xenial needs to launch images:ubuntu/16.04/cloud - # because it contains the HWE kernel which has vhost-vsock support - return self.XENIAL_IMAGE_VSOCK_SUPPORT - - if is_vm and release == "trusty": - # trusty is not supported on LXD vms - raise UnsupportedReleaseException( - release="trusty", - is_vm=is_vm - ) - image_data = self._find_image(release, arch, daily=daily) - - if is_vm: - image_hash_key = self.VM_HASH_KEY - elif release == "trusty": - image_hash_key = self.TRUSTY_CONTAINER_HASH_KEY - else: - image_hash_key = self.CONTAINER_HASH_KEY + image_hash_key = self._get_image_hash_key(release) return '%s:%s' % (remote, image_data[image_hash_key]) - def _image_info(self, image_id, is_vm=False): + def _image_info(self, image_id, image_hash_key=None): """Find the image serial of a given LXD image. Args: image_id: string, LXD image fingerprint - is_vm: boolean, specify if the image_id represents a - virtual machine + image_hash_key: string, the metadata key used to launch the image Returns: dict, image info available for the image_id @@ -443,29 +423,19 @@ def _image_info(self, image_id, is_vm=False): elif remote != self._daily_remote: raise RuntimeError('Unknown remote: %s' % remote) - if is_vm: - image_hash_key = self.VM_HASH_KEY - else: - image_hash_key = self.CONTAINER_HASH_KEY + if not image_hash_key: + image_hash_key = self._get_image_hash_key() filters = ['%s=%s' % (image_hash_key, image_id)] image_info = self._streams_query(filters, daily=daily) - if not image_info: - # If this is a trusty image, the hash key for it is different. - # We will perform a second query for this situation. - filters = ['%s=%s' % (self.TRUSTY_CONTAINER_HASH_KEY, image_id)] - image_info = self._streams_query(filters, daily=daily) - return image_info - def image_serial(self, image_id, is_vm=False, **kwargs): + def image_serial(self, image_id): """Find the image serial of a given LXD image. Args: image_id: string, LXD image fingerprint - is_vm: boolean, specify if the image_id represents a - virtual machine Returns: string, serial of latest image @@ -474,7 +444,7 @@ def image_serial(self, image_id, is_vm=False, **kwargs): self._log.debug( 'finding image serial for LXD Ubuntu image %s', image_id) - image_info = self._image_info(image_id, is_vm=is_vm) + image_info = self._image_info(image_id) return image_info[0]['version_name'] @@ -524,3 +494,220 @@ def _find_image(self, release, arch=LOCAL_UBUNTU_ARCH, daily=True): ] return self._streams_query(filters, daily)[0] + + +class LXD(BaseLXD): + """LXD Containers Cloud Class.""" + + TRUSTY_CONTAINER_HASH_KEY = "combined_rootxz_sha256" + CONTAINER_HASH_KEY = "combined_squashfs_sha256" + + def _get_image_hash_key(self, release=None): + """Get the correct hash key to be used to launch LXD instance. + + When query simplestreams for image information, we receive a + dictionary of metadata. In that metadata we have the necessary + information to allows us to launch the required image. However, + we must know which key to use in the metadata dict to allows + to launch the image. + + Args: + release: string, optional, Ubuntu release + + Returns + A string specifying which key of the metadata dictionary + should be used to launch the image. + """ + if release == "trusty": + return self.TRUSTY_CONTAINER_HASH_KEY + + return self.CONTAINER_HASH_KEY + + def _image_info(self, image_id, image_hash_key=None): + """Find the image serial of a given LXD image. + + Args: + image_id: string, LXD image fingerprint + image_hash_key: string, the metadata key used to launch the image + + Returns: + dict, image info available for the image_id + + """ + image_info = super()._image_info( + image_id=image_id, + image_hash_key=self.CONTAINER_HASH_KEY + ) + + if not image_info: + # If this is a trusty image, the hash key for it is different. + # We will perform a second query for this situation. + image_info = super()._image_info( + image_id=image_id, + image_hash_key=self.TRUSTY_CONTAINER_HASH_KEY + ) + + return image_info + + +class LXDVirtualMachine(BaseLXD): + """LXD Virtual Machine Cloud Class.""" + + XENIAL_IMAGE_VSOCK_SUPPORT = "images:ubuntu/16.04/cloud" + VM_HASH_KEY = "combined_disk1-img_sha256" + + def _extract_release_from_image_id(self, image_id): + """Extract the base release from the image_id. + + Args: + image_id: string, [:], what release to launch + (default remote: ) + + Returns: + A string containing the base release from the image_id that is used + to launch the image. + """ + release_regex = ( + "(.*ubuntu.*(?P(" + + "|".join(UBUNTU_RELEASE_VERSION_MAP) + "|" + + "|".join(UBUNTU_RELEASE_VERSION_MAP.values()) + + ")).*)" + ) + ubuntu_match = re.match(release_regex, image_id) + if ubuntu_match: + release = ubuntu_match.groupdict()["release"] + for codename, version in UBUNTU_RELEASE_VERSION_MAP.items(): + if release in (codename, version): + return codename + + # If we have a hash in the image_id we need to query simplestreams to + # identify the release. + return self._image_info(image_id)[0]["release"] + + def _build_necessary_profiles(self, release=None): + """Build necessary profiles to launch the LXD instance. + + Args: + release: string, [:], what release to launch + (default remote: ) + + Returns: + A list containing the profiles created + """ + base_release = self._extract_release_from_image_id(release) + profile_name = "pycloudlib-vm-{}".format(base_release) + + self.create_profile( + profile_name=profile_name, + profile_config=base_vm_profiles[base_release] + ) + + return [profile_name] + + def _prepare_command( + self, name, release, ephemeral=False, network=None, storage=None, + inst_type=None, profile_list=None, user_data=None, + config_dict=None): + """Build a the command to be used to launch the LXD instance. + + Args: + name: string, what to call the instance + release: string, [:], what release to launch + (default remote: ) + ephemeral: boolean, ephemeral, otherwise persistent + network: string, optional, network name to use + storage: string, optional, storage name to use + inst_type: string, optional, type to use + profile_list: list, optional, profile(s) to use + user_data: used by cloud-init to run custom scripts/configuration + config_dict: dict, optional, configuration values to pass + + Returns: + A list of string representing the command to be run to + launch the LXD instance. + """ + if not profile_list: + profile_list = self._build_necessary_profiles(release=release) + + cmd = super()._prepare_command( + name=name, + release=release, + ephemeral=ephemeral, + network=network, + storage=storage, + inst_type=inst_type, + profile_list=profile_list, + user_data=user_data, + config_dict=config_dict + ) + + cmd.append("--vm") + + return cmd + + def _get_image_hash_key(self, release=None): + """Get the correct hash key to be used to launch LXD instance. + + When query simplestreams for image information, we receive a + dictionary of metadata. In that metadata we have the necessary + information to allows us to launch the required image. However, + we must know which key to use in the metadata dict to allows + to launch the image. + + Args: + release: string, optional, Ubuntu release + + Returns + A string specifying which key of the metadata dictionary + should be used to launch the image. + """ + return self.VM_HASH_KEY + + def _search_for_image( + self, remote, daily, release, arch=LOCAL_UBUNTU_ARCH + ): + """Find the LXD fingerprint in a given remote. + + Args: + remote: string, remote to prepend to image_id + daily: boolean, search on daily remote + release: string, Ubuntu release to look for + arch: string, architecture to use + + Returns: + string, LXD fingerprint of latest image + + """ + if release == "xenial": + # xenial needs to launch images:ubuntu/16.04/cloud + # because it contains the HWE kernel which has vhost-vsock support + return self.XENIAL_IMAGE_VSOCK_SUPPORT + + if release == "trusty": + # trusty is not supported on LXD vms + raise UnsupportedReleaseException( + release="trusty", + is_vm=True + ) + + return super()._search_for_image( + remote=remote, + daily=daily, + release=release, + arch=arch + ) + + def image_serial(self, image_id): + """Find the image serial of a given LXD image. + + Args: + image_id: string, LXD image fingerprint + + Returns: + string, serial of latest image + + """ + if image_id == self.XENIAL_IMAGE_VSOCK_SUPPORT: + return "" + + return super().image_serial(image_id=image_id) diff --git a/pycloudlib/lxd/tests/test_cloud.py b/pycloudlib/lxd/tests/test_cloud.py index 0db668b1..17504325 100644 --- a/pycloudlib/lxd/tests/test_cloud.py +++ b/pycloudlib/lxd/tests/test_cloud.py @@ -4,7 +4,8 @@ from unittest import mock import pytest -from pycloudlib.lxd.cloud import LXD, UnsupportedReleaseException +from pycloudlib.lxd.cloud import (LXD, LXDVirtualMachine, + UnsupportedReleaseException) class TestProfileCreation: @@ -81,7 +82,7 @@ def test_create_profile_that_does_not_exist( class TestExtractReleaseFromImageId: - """Test pycloudlib.lxd.cloud._extract_release_from_image_id method.""" + """Test LXDVirtualMachine _extract_release_from_image_id method.""" @pytest.mark.parametrize( "image_id, expected_release", @@ -95,16 +96,16 @@ def test_extract_release_from_non_hashed_image_id( self, image_id, expected_release ): # pylint: disable=W0212 """Tests extracting release from non hashed image id.""" - cloud = LXD(tag="test") + cloud = LXDVirtualMachine(tag="test") assert expected_release == cloud._extract_release_from_image_id( image_id) - @mock.patch.object(LXD, "_image_info") + @mock.patch.object(LXDVirtualMachine, "_image_info") def test_extract_release_from_hashed_image_id( self, m_image_info ): # pylint: disable=W0212 """Tests extracting release from a non hashed image id.""" - cloud = LXD(tag="test") + cloud = LXDVirtualMachine(tag="test") m_image_info.return_value = [ { @@ -118,25 +119,24 @@ def test_extract_release_from_hashed_image_id( image_id) assert m_image_info.call_args_list == [ - mock.call(image_id, False) + mock.call(image_id) ] class TestSearchForImage: - """Tests covering pycloudlib.lxd.cloud._search_for_image method.""" + """Tests pycloudlib.lxd.cloud._search_for_image method.""" def test_trusty_image_not_supported_when_launching_vms( self ): # pylint: disable=W0212 """Tests searching for trusty image for launching LXD vms.""" - cloud = LXD(tag="test") + cloud = LXDVirtualMachine(tag="test") with pytest.raises(UnsupportedReleaseException) as excinfo: cloud._search_for_image( remote="remote", daily=False, release="trusty", - is_vm=True ) assert "Release trusty is not supported for LXD vms" == str( diff --git a/pycloudlib/oci/cloud.py b/pycloudlib/oci/cloud.py index d47bf222..759245d0 100644 --- a/pycloudlib/oci/cloud.py +++ b/pycloudlib/oci/cloud.py @@ -109,7 +109,7 @@ def daily_image(self, release, operating_system='Canonical Ubuntu'): image_id = image_response.data[0].id return image_id - def image_serial(self, image_id, **kwargs): + def image_serial(self, image_id): """Find the image serial of the latest daily image for a particular release. Args: From 37155cd835031d9a92844b66ca57b3fb30a88071 Mon Sep 17 00:00:00 2001 From: Lucas Moura Date: Tue, 17 Nov 2020 09:49:00 -0300 Subject: [PATCH 6/8] Add travis file --- .travis.yml | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) create mode 100644 .travis.yml diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 00000000..3a37b425 --- /dev/null +++ b/.travis.yml @@ -0,0 +1,35 @@ +language: python +dist: bionic + + +install: + # Required so `git describe` will definitely find a tag; see + # https://github.com/travis-ci/travis-ci/issues/7422 + - git fetch --unshallow + +matrix: + fast_finish: true + include: + - name: run-lxd-example + install: + - pip install -r requirements.txt + script: + - sudo apt-get remove --yes --purge lxd lxd-client + - sudo rm -Rf /var/lib/lxd + - sudo snap install lxd + - sudo lxd init --auto + - sudo usermod -a -G lxd $USER + - sg lxd -c "python examples/lxd.py" + - name: run-cloudinit-integration-tests + install: + - pip install -r requirements.txt + - pip install pytest + script: + - sudo apt-get remove --yes --purge lxd lxd-client + - sudo rm -Rf /var/lib/lxd + - sudo snap install lxd + - sudo lxd init --auto + - sudo usermod -a -G lxd $USER + - git clone https://github.com/canonical/cloud-init.git + - cd cloud-init + - sg lxd -c "pytest tests/integration_tests/" From 29ee9140cce2d213ddb75b1bd9b528de01f8a697 Mon Sep 17 00:00:00 2001 From: Lucas Moura Date: Tue, 17 Nov 2020 11:52:29 -0300 Subject: [PATCH 7/8] Keep running lxd commands as sudo --- pycloudlib/lxd/instance.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pycloudlib/lxd/instance.py b/pycloudlib/lxd/instance.py index 9a0fa0b4..b8aad3ff 100644 --- a/pycloudlib/lxd/instance.py +++ b/pycloudlib/lxd/instance.py @@ -33,7 +33,7 @@ def _run_command(self, command, stdin): if self.key_pair: return super()._run_command(command, stdin) - base_cmd = ['lxc', 'exec', "--user", "1000", self.name, '--'] + base_cmd = ['lxc', 'exec', self.name, '--'] return subp(base_cmd + list(command), rcs=None) @property From 0afe7eecdc54ad08c1b78967b571a8eeb26d7276 Mon Sep 17 00:00:00 2001 From: Lucas Moura Date: Tue, 17 Nov 2020 18:35:40 -0300 Subject: [PATCH 8/8] Update LXD abstraction --- examples/lxd.py | 10 +++++----- pycloudlib/__init__.py | 3 ++- pycloudlib/lxd/cloud.py | 26 ++++++++++++++++++++------ pycloudlib/lxd/tests/test_cloud.py | 14 +++++++++----- 4 files changed, 36 insertions(+), 17 deletions(-) diff --git a/examples/lxd.py b/examples/lxd.py index 93441847..5b480cfc 100755 --- a/examples/lxd.py +++ b/examples/lxd.py @@ -19,7 +19,7 @@ def snapshot_instance(): instance to the snapshot level. Finally, launch another instance from the snapshot of the instance. """ - lxd = pycloudlib.LXD('example-snapshot') + lxd = pycloudlib.LXDContainer('example-snapshot') inst = lxd.launch(name='pycloudlib-snapshot-base', image_id=RELEASE) snapshot_name = 'snapshot' @@ -43,7 +43,7 @@ def modify_instance(): Once started the instance demonstrates some interactions with the instance. """ - lxd = pycloudlib.LXD('example-modify') + lxd = pycloudlib.LXDContainer('example-modify') inst = lxd.init('pycloudlib-modify-inst', RELEASE) inst.edit('limits.memory', '3GB') @@ -64,7 +64,7 @@ def launch_multiple(): waiting for the instance to start each time. Note that the wait_for_delete method is not used, as LXD does not do any waiting. """ - lxd = pycloudlib.LXD('example-multiple') + lxd = pycloudlib.LXDContainer('example-multiple') instances = [] for num in range(3): @@ -97,7 +97,7 @@ def launch_options(): Finally, an instance with custom configurations options. """ - lxd = pycloudlib.LXD('example-launch') + lxd = pycloudlib.LXDContainer('example-launch') kvm_profile = textwrap.dedent( """\ devices: @@ -148,7 +148,7 @@ def launch_options(): def basic_lifecycle(): """Demonstrate basic set of lifecycle operations with LXD.""" - lxd = pycloudlib.LXD('example-basic') + lxd = pycloudlib.LXDContainer('example-basic') inst = lxd.launch(image_id=RELEASE) inst.delete() diff --git a/pycloudlib/__init__.py b/pycloudlib/__init__.py index 498321d2..ed1839ea 100644 --- a/pycloudlib/__init__.py +++ b/pycloudlib/__init__.py @@ -6,7 +6,7 @@ from pycloudlib.azure.cloud import Azure from pycloudlib.ec2.cloud import EC2 from pycloudlib.gce.cloud import GCE -from pycloudlib.lxd.cloud import LXD, LXDVirtualMachine +from pycloudlib.lxd.cloud import LXD, LXDContainer, LXDVirtualMachine from pycloudlib.kvm.cloud import KVM from pycloudlib.oci.cloud import OCI @@ -15,6 +15,7 @@ 'EC2', 'GCE', 'LXD', + 'LXDContainer', 'LXDVirtualMachine', 'KVM', 'OCI', diff --git a/pycloudlib/lxd/cloud.py b/pycloudlib/lxd/cloud.py index 49743c96..6ea27d3f 100644 --- a/pycloudlib/lxd/cloud.py +++ b/pycloudlib/lxd/cloud.py @@ -4,6 +4,7 @@ import re import textwrap from abc import abstractmethod +import warnings import paramiko from pycloudlib.cloud import BaseCloud @@ -30,7 +31,7 @@ def __init__(self, release, is_vm): ) -class BaseLXD(BaseCloud): +class _BaseLXD(BaseCloud): """LXD Base Cloud Class.""" _type = 'lxd' @@ -149,10 +150,10 @@ def create_key_pair(self): key = paramiko.RSAKey.generate(4096) priv_str = io.StringIO() - pub_key = key.get_base64() + pub_key = "{} {}".format(key.get_name(), key.get_base64()) key.write_private_key(priv_str, password=None) - return "ssh-rsa {}".format(pub_key), priv_str.getvalue() + return pub_key, priv_str.getvalue() # pylint: disable=R0914,R0912,R0915 def _prepare_command( @@ -496,7 +497,7 @@ def _find_image(self, release, arch=LOCAL_UBUNTU_ARCH, daily=True): return self._streams_query(filters, daily)[0] -class LXD(BaseLXD): +class LXDContainer(_BaseLXD): """LXD Containers Cloud Class.""" TRUSTY_CONTAINER_HASH_KEY = "combined_rootxz_sha256" @@ -550,7 +551,16 @@ def _image_info(self, image_id, image_hash_key=None): return image_info -class LXDVirtualMachine(BaseLXD): +class LXD(LXDContainer): + """Old LXD Container Cloud Class (Kept for compatibility issues).""" + + def __init__(self, *args, **kwargs): + """Run LXDContainer constructor.""" + warnings.warn("LXD class is deprecated; use LXDContainer instead.") + super().__init__(*args, **kwargs) + + +class LXDVirtualMachine(_BaseLXD): """LXD Virtual Machine Cloud Class.""" XENIAL_IMAGE_VSOCK_SUPPORT = "images:ubuntu/16.04/cloud" @@ -681,6 +691,10 @@ def _search_for_image( if release == "xenial": # xenial needs to launch images:ubuntu/16.04/cloud # because it contains the HWE kernel which has vhost-vsock support + self._log.debug( + "Xenial needs to use %s image because of lxd-agent support", + self.XENIAL_IMAGE_VSOCK_SUPPORT + ) return self.XENIAL_IMAGE_VSOCK_SUPPORT if release == "trusty": @@ -708,6 +722,6 @@ def image_serial(self, image_id): """ if image_id == self.XENIAL_IMAGE_VSOCK_SUPPORT: - return "" + return None return super().image_serial(image_id=image_id) diff --git a/pycloudlib/lxd/tests/test_cloud.py b/pycloudlib/lxd/tests/test_cloud.py index 17504325..d4bb05ca 100644 --- a/pycloudlib/lxd/tests/test_cloud.py +++ b/pycloudlib/lxd/tests/test_cloud.py @@ -4,7 +4,8 @@ from unittest import mock import pytest -from pycloudlib.lxd.cloud import (LXD, LXDVirtualMachine, +from pycloudlib.lxd.cloud import (LXDContainer, + LXDVirtualMachine, UnsupportedReleaseException) @@ -15,7 +16,7 @@ class TestProfileCreation: def test_create_profile_that_already_exists(self, m_subp): """Tests creating a profile that already exists.""" m_subp.return_value = ["test_profile"] - cloud = LXD(tag="test") + cloud = LXDContainer(tag="test") fake_stdout = io.StringIO() with contextlib.redirect_stdout(fake_stdout): @@ -36,7 +37,7 @@ def test_create_profile_that_already_exists_with_force( ): """Tests creating an existing profile with force parameter.""" m_subp.return_value = ["test_profile"] - cloud = LXD(tag="test") + cloud = LXDContainer(tag="test") profile_name = "test_profile" profile_config = "profile_config" @@ -62,7 +63,7 @@ def test_create_profile_that_does_not_exist( ): """Tests creating a new profile.""" m_subp.return_value = ["other_profile"] - cloud = LXD(tag="test") + cloud = LXDContainer(tag="test") profile_name = "test_profile" profile_config = "profile_config" @@ -82,7 +83,10 @@ def test_create_profile_that_does_not_exist( class TestExtractReleaseFromImageId: - """Test LXDVirtualMachine _extract_release_from_image_id method.""" + """Test LXDVirtualMachine _extract_release_from_image_id method. + + This method should only be executed by LXDVirtualMachine instances. + """ @pytest.mark.parametrize( "image_id, expected_release",