Remove (deprecated) apt-key (SC-89)

cloudinit/config/cc_apt_configure.py

@@ -11,6 +11,7 @@
 import glob
 import os
 import re
+import pathlib
 from textwrap import dedent
 
 from cloudinit.config.schema import (
@@ -27,6 +28,10 @@
 # this will match 'XXX:YYY' (ie, 'cloud-archive:foo' or 'ppa:bar')
 ADD_APT_REPO_MATCH = r"^[\w-]+:\w"
 
+APT_LOCAL_KEYS = '/etc/apt/trusted.gpg'
+APT_TRUSTED_GPG_DIR = '/etc/apt/trusted.gpg.d/'
+CLOUD_INIT_GPG_DIR = '/etc/apt/cloud-init.gpg.d/'
+
 frequency = PER_INSTANCE
 distros = ["ubuntu", "debian"]
 mirror_property = {
@@ -139,7 +144,7 @@
               source1:
                   keyid: 'keyid'
                   keyserver: 'keyserverurl'
-                  source: 'deb http://<url>/ xenial main'
+                  source: 'deb [signed-by=$KEY_FILE] http://<url>/ xenial main'
               source2:
                   source: 'ppa:<ppa-name>'
               source3:
@@ -312,7 +317,8 @@
                             - ``$MIRROR``
                             - ``$RELEASE``
                             - ``$PRIMARY``
-                            - ``$SECURITY``""")
+                            - ``$SECURITY``
+                            - ``$KEY_FILE``""")
                 },
                 'conf': {
                     'type': 'string',
@@ -381,7 +387,8 @@
                             - ``$MIRROR``
                             - ``$PRIMARY``
                             - ``$SECURITY``
-                            - ``$RELEASE``""")
+                            - ``$RELEASE``
+                            - ``$KEY_FILE``""")
                 }
             }
         }
@@ -683,7 +690,7 @@ def add_mirror_keys(cfg, target):
     """Adds any keys included in the primary/security mirror clauses"""
     for key in ('primary', 'security'):
         for mirror in cfg.get(key, []):
-            add_apt_key(mirror, target)
+            add_apt_key(mirror, target, file_name=key)
 
 
 def generate_sources_list(cfg, release, mirrors, cloud):
@@ -714,20 +721,21 @@ def generate_sources_list(cfg, release, mirrors, cloud):
     util.write_file(aptsrc, disabled, mode=0o644)
 
 
-def add_apt_key_raw(key, target=None):
+def add_apt_key_raw(key, file_name, hardened=False, target=None):
     """
     actual adding of a key as defined in key argument
     to the system
     """
     LOG.debug("Adding key:\n'%s'", key)
     try:
-        subp.subp(['apt-key', 'add', '-'], data=key.encode(), target=target)
+        name = pathlib.Path(file_name).stem
+        return apt_key('add', output_file=name, data=key, hardened=hardened)
     except subp.ProcessExecutionError:
         LOG.exception("failed to add apt GPG Key to apt keyring")
         raise
 
 
-def add_apt_key(ent, target=None):
+def add_apt_key(ent, target=None, hardened=False, file_name=None):
     """
     Add key to the system as defined in ent (if any).
     Supports raw keys or keyid's
@@ -741,7 +749,10 @@ def add_apt_key(ent, target=None):
         ent['key'] = gpg.getkeybyid(ent['keyid'], keyserver)
 
     if 'key' in ent:
-        add_apt_key_raw(ent['key'], target)
+        return add_apt_key_raw(
+            ent['key'],
+            file_name or ent['filename'],
+            hardened=hardened)
 
 
 def update_packages(cloud):
@@ -751,9 +762,28 @@ def update_packages(cloud):
 def add_apt_sources(srcdict, cloud, target=None, template_params=None,
                     aa_repo_match=None):
     """
-    add entries in /etc/apt/sources.list.d for each abbreviated
-    sources.list entry in 'srcdict'.  When rendering template, also
-    include the values in dictionary searchList
+    install keys and repo source .list files defined in 'sources'
+
+    for each 'source' entry in the config:
+        1. expand template variables and write source .list file in
+                /etc/apt/sources.list.d/
+        2. install defined keys
+        3. update packages via distro-specific method (i.e. apt-key update)
+
+
+    @param srcdict: a dict containing elements required
+    @param cloud: cloud instance object
+
+    Example srcdict value:
+    {
+    'rio-grande-repo': {
+        'source': 'deb [signed-by=$KEY_FILE] $MIRROR $RELEASE main',
+        'keyid': 'B59D 5F15 97A5 04B7 E230  6DCA 0620 BBCF 0368 3F77',
+        'keyserver': 'pgp.mit.edu'
+        }
+    }
+
+    Note: Deb822 format is not supported
     """
     if template_params is None:
         template_params = {}
@@ -770,7 +800,11 @@ def add_apt_sources(srcdict, cloud, target=None, template_params=None,
         if 'filename' not in ent:
             ent['filename'] = filename
 
-        add_apt_key(ent, target)
+        if 'source' in ent and '$KEY_FILE' in ent['source']:
+            key_file = add_apt_key(ent, target, hardened=True)
+            template_params['KEY_FILE'] = key_file
+        else:
+            key_file = add_apt_key(ent, target)
 
         if 'source' not in ent:
             continue
@@ -1006,7 +1040,7 @@ def get_arch_mirrorconfig(cfg, mirrortype, arch):
     # select the specification matching the target arch
     default = None
     for mirror_cfg_elem in mirror_cfg_list:
-        arches = mirror_cfg_elem.get("arches")
+        arches = mirror_cfg_elem.get("arches", [])
         if arch in arches:
             return mirror_cfg_elem
         if "default" in arches:
@@ -1089,6 +1123,81 @@ def apply_apt_config(cfg, proxy_fname, config_fname):
         LOG.debug("no apt config configured, removed %s", config_fname)
 
 
+def apt_key(command, output_file=None, data=None, hardened=False,
+            human_output=True):
+    """apt-key replacement
+
+    commands implemented: 'add', 'list', 'finger'
+
+    @param output_file: name of output gpg file (without .gpg or .asc)
+    @param data: key contents
+    @param human_output: list keys formatted for human parsing
+    @param hardened: write keys to to /etc/apt/cloud-init.gpg.d/ (referred to
+    with [signed-by] in sources file)
+    """
+
+    def _get_key_files():
+        """return all apt keys
+
+        /etc/apt/trusted.gpg (if it exists) and all keyfiles (and symlinks to
+        keyfiles) in /etc/apt/trusted.gpg.d/ are returned
+
+        based on apt-key implementation
+        """
+        key_files = [APT_LOCAL_KEYS] if os.path.isfile(APT_LOCAL_KEYS) else []
+
+        for file in os.listdir(APT_TRUSTED_GPG_DIR):
+            if file.endswith('.gpg') or file.endswith('.asc'):
+                key_files.append(APT_TRUSTED_GPG_DIR + file)
+        return key_files if key_files else ''
+
+    def apt_key_add():
+        """apt-key add <file>
+
+        returns filepath to new keyring, or '/dev/null' when an error occurs
+        """
+        file_name = '/dev/null'
+        if not output_file:
+            util.logexc(
+                LOG, 'Unknown filename, failed to add key: "{}"'.format(data))
+        else:
+            try:
+                key_dir = \
+                    CLOUD_INIT_GPG_DIR if hardened else APT_TRUSTED_GPG_DIR
+                stdout = gpg.dearmor(data)
+                file_name = '{}{}.gpg'.format(key_dir, output_file)
+                util.write_file(file_name, stdout)
+            except subp.ProcessExecutionError:
+                util.logexc(LOG, 'Gpg error, failed to add key: {}'.format(
+                    data))
+            except UnicodeDecodeError:
+                util.logexc(LOG, 'Decode error, failed to add key: {}'.format(
+                    data))
+        return file_name
+
+    def apt_key_list():
+        """apt-key list
+
+        returns string of all trusted keys (in /etc/apt/trusted.gpg and
+        /etc/apt/trusted.gpg.d/)
+        """
+        key_list = []
+        for key_file in _get_key_files():
+            try:
+                key_list.append(gpg.list(key_file, human_output=human_output))
+            except subp.ProcessExecutionError as error:
+                LOG.warning('Failed to list key "%s": %s', key_file, error)
+        return '\n'.join(key_list)
+
+    if command == 'add':
+        return apt_key_add()
+    elif command == 'finger' or command == 'list':
+        return apt_key_list()
+    else:
+        raise ValueError(
+            'apt_key() commands add, list, and finger are currently supported')
+
+
 CONFIG_CLEANERS = {
     'cloud-init': clean_cloud_init,
 }

cloudinit/gpg.py

@@ -14,6 +14,9 @@
 
 LOG = logging.getLogger(__name__)
 
+GPG_LIST = ['gpg', '--with-fingerprint', '--no-default-keyring', '--list-keys',
+            '--keyring']
+
 
 def export_armour(key):
     """Export gpg key, armoured key gets returned"""
@@ -27,6 +30,33 @@ def export_armour(key):
     return armour
 
 
+def dearmor(key):
+    """Dearmor gpg key, dearmored key gets returned
+
+        note: man gpg(1) makes no mention of an --armour spelling, only --armor
+    """
+    return subp.subp(["gpg", "--dearmor"], data=key, decode=False)[0]
+
+
+def list(key_file, human_output=False):
+    """List keys from a keyring with fingerprints. Default to a stable machine
+    parseable format.
+
+    @param key_file: a string containing a filepath to a key
+    @param human_output: return output intended for human parsing
+    """
+    cmd = []
+    cmd.extend(GPG_LIST)
+    if not human_output:
+        cmd.append('--with-colons')
+
+    cmd.append(key_file)
+    (stdout, stderr) = subp.subp(cmd, capture=True)
+    if stderr:
+        LOG.warning('Failed to export armoured key "%s": %s', key_file, stderr)
+    return stdout
+
+
 def recv_key(key, keyserver, retries=(1, 1)):
     """Receive gpg key from the specified keyserver.
 

doc/examples/cloud-config-apt.txt

@@ -149,6 +149,7 @@ apt:
   # security is optional, if not defined it is set to the same value as primary
   security:
     - uri: http://security.ubuntu.com/ubuntu
+    - arches: [default]
   # If search_dns is set for security the searched pattern is:
   #   <distro>-security-mirror
 
@@ -212,14 +213,14 @@ apt:
   #
   # The key of each source entry is the filename and will be prepended by
   # /etc/apt/sources.list.d/ if it doesn't start with a '/'.
-  # If it doesn't end with .list it will be appended so that apt picks up it's
+  # If it doesn't end with .list it will be appended so that apt picks up its
   # configuration.
   #
   # Whenever there is no content to be written into such a file, the key is
   # not used as filename - yet it can still be used as index for merging
   # configuration.
   #
-  # The values inside the entries consost of the following optional entries:
+  # The values inside the entries consist of the following optional entries:
   #   'source': a sources.list entry (some variable replacements apply)
   #   'keyid': providing a key to import via shortid or fingerprint
   #   'key': providing a raw PGP key
@@ -276,13 +277,14 @@ apt:
     my-repo2.list:
       # 2.4 replacement variables
       #
-      # sources can use $MIRROR, $PRIMARY, $SECURITY and $RELEASE replacement
-      # variables.
+      # sources can use $MIRROR, $PRIMARY, $SECURITY, $RELEASE and $KEY_FILE
+      # replacement variables.
       # They will be replaced with the default or specified mirrors and the
       # running release.
       # The entry below would be possibly turned into:
       #   source: deb http://archive.ubuntu.com/ubuntu xenial multiverse
-      source: deb $MIRROR $RELEASE multiverse
+      source: deb [signed-by=$KEY_FILE] $MIRROR $RELEASE multiverse
+      keyid: F430BBA5
 
     my-repo3.list:
       # this would have the same end effect as 'ppa:curtin-dev/test-archive'
@@ -310,9 +312,19 @@ apt:
       keyid: B59D 5F15 97A5 04B7 E230  6DCA 0620 BBCF 0368 3F77
       keyserver: pgp.mit.edu
 
+    ignored5:
+      # 2.8 signed-by
+      #
+      # One can specify [signed-by=$KEY_FILE] in the source definition, which
+      # will make the key be installed in the directory /etc/cloud-init.gpg.d/
+      # and the $KEY_FILE replacement variable will be replaced with the path
+      # to the specified key. If $KEY_FILE is used, but no key is specified,
+      # apt update will (rightfully) fail due to an invalid value.
+      source: deb [signed-by=$KEY_FILE] $MIRROR $RELEASE multiverse
+      keyid: B59D 5F15 97A5 04B7 E230  6DCA 0620 BBCF 0368 3F77
 
     my-repo4.list:
-      # 2.8 raw key
+      # 2.9 raw key
       #
       # The apt signing key can also be specified by providing a pgp public key
       # block. Providing the PGP key this way is the most robust method for