Move subp into its own module.

cloudinit/analyze/dump.py

@@ -4,6 +4,7 @@
 from datetime import datetime
 import sys
 
+from cloudinit import subp
 from cloudinit import util
 
 stage_to_description = {
@@ -51,7 +52,7 @@ def parse_timestamp(timestampstr):
 
 
 def parse_timestamp_from_date(timestampstr):
-    out, _ = util.subp(['date', '+%s.%3N', '-d', timestampstr])
+    out, _ = subp.subp(['date', '+%s.%3N', '-d', timestampstr])
     timestamp = out.strip()
     return float(timestamp)
 

cloudinit/analyze/show.py

@@ -11,6 +11,7 @@
 import time
 import sys
 
+from cloudinit import subp
 from cloudinit import util
 from cloudinit.distros import uses_systemd
 
@@ -155,7 +156,7 @@ def subp(self):
         :return: whether the subp call failed or not
         '''
         try:
-            value, err = util.subp(self.args, capture=True)
+            value, err = subp.subp(self.args, capture=True)
             if err:
                 return err
             self.epoch = value
@@ -215,7 +216,7 @@ def gather_timestamps_using_dmesg():
     with gather_timestamps_using_systemd
     '''
     try:
-        data, _ = util.subp(['dmesg'], capture=True)
+        data, _ = subp.subp(['dmesg'], capture=True)
         split_entries = data[0].splitlines()
         for i in split_entries:
             if i.decode('UTF-8').find('user') != -1:

cloudinit/analyze/tests/test_boot.py

@@ -25,7 +25,7 @@ def test_freebsd_gentoo_cant_find(self, m_sys_info,
                                       m_get_linux_distro, m_is_FreeBSD):
         self.assertEqual(err_code, dist_check_timestamp())
 
-    @mock.patch('cloudinit.util.subp', return_value=(0, 1))
+    @mock.patch('cloudinit.subp.subp', return_value=(0, 1))
     def test_subp_fails(self, m_subp):
         self.assertEqual(err_code, dist_check_timestamp())
 
@@ -42,20 +42,20 @@ def test_systemctl_invalid_parameter(self):
         with self.assertRaises(RuntimeError):
             reader.parse_epoch_as_float()
 
-    @mock.patch('cloudinit.util.subp', return_value=('U=1000000', None))
+    @mock.patch('cloudinit.subp.subp', return_value=('U=1000000', None))
     def test_systemctl_works_correctly_threshold(self, m_subp):
         reader = SystemctlReader('dummyProperty', 'dummyParameter')
         self.assertEqual(1.0, reader.parse_epoch_as_float())
         thresh = 1.0 - reader.parse_epoch_as_float()
         self.assertTrue(thresh < 1e-6)
         self.assertTrue(thresh > (-1 * 1e-6))
 
-    @mock.patch('cloudinit.util.subp', return_value=('U=0', None))
+    @mock.patch('cloudinit.subp.subp', return_value=('U=0', None))
     def test_systemctl_succeed_zero(self, m_subp):
         reader = SystemctlReader('dummyProperty', 'dummyParameter')
         self.assertEqual(0.0, reader.parse_epoch_as_float())
 
-    @mock.patch('cloudinit.util.subp', return_value=('U=1', None))
+    @mock.patch('cloudinit.subp.subp', return_value=('U=1', None))
     def test_systemctl_succeed_distinct(self, m_subp):
         reader = SystemctlReader('dummyProperty', 'dummyParameter')
         val1 = reader.parse_epoch_as_float()
@@ -64,13 +64,13 @@ def test_systemctl_succeed_distinct(self, m_subp):
         val2 = reader2.parse_epoch_as_float()
         self.assertNotEqual(val1, val2)
 
-    @mock.patch('cloudinit.util.subp', return_value=('100', None))
+    @mock.patch('cloudinit.subp.subp', return_value=('100', None))
     def test_systemctl_epoch_not_splittable(self, m_subp):
         reader = SystemctlReader('dummyProperty', 'dummyParameter')
         with self.assertRaises(IndexError):
             reader.parse_epoch_as_float()
 
-    @mock.patch('cloudinit.util.subp', return_value=('U=foobar', None))
+    @mock.patch('cloudinit.subp.subp', return_value=('U=foobar', None))
     def test_systemctl_cannot_convert_epoch_to_float(self, m_subp):
         reader = SystemctlReader('dummyProperty', 'dummyParameter')
         with self.assertRaises(ValueError):
@@ -130,7 +130,7 @@ def test_boot_invalid_distro(self, m_dist_check_timestamp):
         self.assertEqual(err_string, data)
 
     @mock.patch("cloudinit.util.is_container", return_value=True)
-    @mock.patch('cloudinit.util.subp', return_value=('U=1000000', None))
+    @mock.patch('cloudinit.subp.subp', return_value=('U=1000000', None))
     def test_container_no_ci_log_line(self, m_is_container, m_subp):
         path = os.path.dirname(os.path.abspath(__file__))
         log_path = path + '/boot-test.log'
@@ -148,7 +148,7 @@ def test_container_no_ci_log_line(self, m_is_container, m_subp):
         self.assertEqual(FAIL_CODE, finish_code)
 
     @mock.patch("cloudinit.util.is_container", return_value=True)
-    @mock.patch('cloudinit.util.subp', return_value=('U=1000000', None))
+    @mock.patch('cloudinit.subp.subp', return_value=('U=1000000', None))
     @mock.patch('cloudinit.analyze.__main__._get_events', return_value=[{
         'name': 'init-local', 'description': 'starting search', 'timestamp':
         100000}])

cloudinit/analyze/tests/test_dump.py

@@ -5,7 +5,8 @@
 
 from cloudinit.analyze.dump import (
     dump_events, parse_ci_logline, parse_timestamp)
-from cloudinit.util import which, write_file
+from cloudinit.util import write_file
+from cloudinit.subp import which
 from cloudinit.tests.helpers import CiTestCase, mock, skipIf
 
 

cloudinit/cmd/clean.py

@@ -10,9 +10,8 @@
 import sys
 
 from cloudinit.stages import Init
-from cloudinit.util import (
-    ProcessExecutionError, del_dir, del_file, get_config_logfiles,
-    is_link, subp)
+from cloudinit.subp import (ProcessExecutionError, subp)
+from cloudinit.util import (del_dir, del_file, get_config_logfiles, is_link)
 
 
 def error(msg):

cloudinit/cmd/devel/logs.py

@@ -12,8 +12,8 @@
 
 from cloudinit.sources import INSTANCE_JSON_SENSITIVE_FILE
 from cloudinit.temp_utils import tempdir
-from cloudinit.util import (
-    ProcessExecutionError, chdir, copy, ensure_dir, subp, write_file)
+from cloudinit.subp import (ProcessExecutionError, subp)
+from cloudinit.util import (chdir, copy, ensure_dir, write_file)
 
 
 CLOUDINIT_LOGS = ['/var/log/cloud-init.log', '/var/log/cloud-init-output.log']

cloudinit/cmd/devel/tests/test_logs.py

@@ -8,7 +8,8 @@
 from cloudinit.sources import INSTANCE_JSON_SENSITIVE_FILE
 from cloudinit.tests.helpers import (
     FilesystemMockingTestCase, mock, wrap_and_call)
-from cloudinit.util import ensure_dir, load_file, subp, write_file
+from cloudinit.subp import subp
+from cloudinit.util import ensure_dir, load_file, write_file
 
 
 @mock.patch('cloudinit.cmd.devel.logs.os.getuid')

cloudinit/config/cc_apt_configure.py

@@ -17,6 +17,7 @@
     get_schema_doc, validate_cloudconfig_schema)
 from cloudinit import gpg
 from cloudinit import log as logging
+from cloudinit import subp
 from cloudinit import templater
 from cloudinit import util
 from cloudinit.settings import PER_INSTANCE
@@ -431,7 +432,7 @@ def _should_configure_on_empty_apt():
     # if no config was provided, should apt configuration be done?
     if util.system_is_snappy():
         return False, "system is snappy."
-    if not (util.which('apt-get') or util.which('apt')):
+    if not (subp.which('apt-get') or subp.which('apt')):
         return False, "no apt commands."
     return True, "Apt is available."
 
@@ -478,7 +479,7 @@ def apply_apt(cfg, cloud, target):
 def debconf_set_selections(selections, target=None):
     if not selections.endswith(b'\n'):
         selections += b'\n'
-    util.subp(['debconf-set-selections'], data=selections, target=target,
+    subp.subp(['debconf-set-selections'], data=selections, target=target,
               capture=True)
 
 
@@ -503,7 +504,7 @@ def dpkg_reconfigure(packages, target=None):
                     "but cannot be unconfigured: %s", unhandled)
 
     if len(to_config):
-        util.subp(['dpkg-reconfigure', '--frontend=noninteractive'] +
+        subp.subp(['dpkg-reconfigure', '--frontend=noninteractive'] +
                   list(to_config), data=None, target=target, capture=True)
 
 
@@ -546,7 +547,7 @@ def apply_debconf_selections(cfg, target=None):
 def clean_cloud_init(target):
     """clean out any local cloud-init config"""
     flist = glob.glob(
-        util.target_path(target, "/etc/cloud/cloud.cfg.d/*dpkg*"))
+        subp.target_path(target, "/etc/cloud/cloud.cfg.d/*dpkg*"))
 
     LOG.debug("cleaning cloud-init config from: %s", flist)
     for dpkg_cfg in flist:
@@ -575,7 +576,7 @@ def rename_apt_lists(new_mirrors, target, arch):
     """rename_apt_lists - rename apt lists to preserve old cache data"""
     default_mirrors = get_default_mirrors(arch)
 
-    pre = util.target_path(target, APT_LISTS)
+    pre = subp.target_path(target, APT_LISTS)
     for (name, omirror) in default_mirrors.items():
         nmirror = new_mirrors.get(name)
         if not nmirror:
@@ -694,8 +695,8 @@ def add_apt_key_raw(key, target=None):
     """
     LOG.debug("Adding key:\n'%s'", key)
     try:
-        util.subp(['apt-key', 'add', '-'], data=key.encode(), target=target)
-    except util.ProcessExecutionError:
+        subp.subp(['apt-key', 'add', '-'], data=key.encode(), target=target)
+    except subp.ProcessExecutionError:
         LOG.exception("failed to add apt GPG Key to apt keyring")
         raise
 
@@ -758,13 +759,13 @@ def add_apt_sources(srcdict, cloud, target=None, template_params=None,
 
         if aa_repo_match(source):
             try:
-                util.subp(["add-apt-repository", source], target=target)
-            except util.ProcessExecutionError:
+                subp.subp(["add-apt-repository", source], target=target)
+            except subp.ProcessExecutionError:
                 LOG.exception("add-apt-repository failed.")
                 raise
             continue
 
-        sourcefn = util.target_path(target, ent['filename'])
+        sourcefn = subp.target_path(target, ent['filename'])
         try:
             contents = "%s\n" % (source)
             util.write_file(sourcefn, contents, omode="a")

cloudinit/config/cc_bootcmd.py

@@ -16,6 +16,7 @@
     get_schema_doc, validate_cloudconfig_schema)
 from cloudinit.settings import PER_ALWAYS
 from cloudinit import temp_utils
+from cloudinit import subp
 from cloudinit import util
 
 frequency = PER_ALWAYS
@@ -99,7 +100,7 @@ def handle(name, cfg, cloud, log, _args):
             if iid:
                 env['INSTANCE_ID'] = str(iid)
             cmd = ['/bin/sh', tmpf.name]
-            util.subp(cmd, env=env, capture=False)
+            subp.subp(cmd, env=env, capture=False)
         except Exception:
             util.logexc(log, "Failed to run bootcmd module %s", name)
             raise

cloudinit/config/cc_byobu.py

@@ -39,6 +39,7 @@
 """
 
 from cloudinit.distros import ug_util
+from cloudinit import subp
 from cloudinit import util
 
 distros = ['ubuntu', 'debian']
@@ -93,6 +94,6 @@ def handle(name, cfg, cloud, log, args):
     if len(shcmd):
         cmd = ["/bin/sh", "-c", "%s %s %s" % ("X=0;", shcmd, "exit $X")]
         log.debug("Setting byobu to %s", value)
-        util.subp(cmd, capture=False)
+        subp.subp(cmd, capture=False)
 
 # vi: ts=4 expandtab

cloudinit/config/cc_ca_certs.py

@@ -36,6 +36,7 @@
 
 import os
 
+from cloudinit import subp
 from cloudinit import util
 
 CA_CERT_PATH = "/usr/share/ca-certificates/"
@@ -51,7 +52,7 @@ def update_ca_certs():
     """
     Updates the CA certificate cache on the current machine.
     """
-    util.subp(["update-ca-certificates"], capture=False)
+    subp.subp(["update-ca-certificates"], capture=False)
 
 
 def add_ca_certs(certs):
@@ -85,7 +86,7 @@ def remove_default_ca_certs():
     util.delete_dir_contents(CA_CERT_SYSTEM_PATH)
     util.write_file(CA_CERT_CONFIG, "", mode=0o644)
     debconf_sel = "ca-certificates ca-certificates/trust_new_crts select no"
-    util.subp(('debconf-set-selections', '-'), debconf_sel)
+    subp.subp(('debconf-set-selections', '-'), debconf_sel)
 
 
 def handle(name, cfg, _cloud, log, _args):

cloudinit/config/cc_chef.py

@@ -76,7 +76,9 @@
 import json
 import os
 
+from cloudinit import subp
 from cloudinit import templater
+from cloudinit import temp_utils
 from cloudinit import url_helper
 from cloudinit import util
 
@@ -282,7 +284,32 @@ def run_chef(chef_cfg, log):
             cmd.extend(CHEF_EXEC_DEF_ARGS)
     else:
         cmd.extend(CHEF_EXEC_DEF_ARGS)
-    util.subp(cmd, capture=False)
+    subp.subp(cmd, capture=False)
+
+
+def subp_blob_in_tempfile(blob, *args, **kwargs):
+    """Write blob to a tempfile, and call subp with args, kwargs. Then cleanup.
+
+    'basename' as a kwarg allows providing the basename for the file.
+    The 'args' argument to subp will be updated with the full path to the
+    filename as the first argument.
+    """
+    basename = kwargs.pop('basename', "subp_blob")
+
+    if len(args) == 0 and 'args' not in kwargs:
+        args = [tuple()]
+
+    # Use tmpdir over tmpfile to avoid 'text file busy' on execute
+    with temp_utils.tempdir(needs_exe=True) as tmpd:
+        tmpf = os.path.join(tmpd, basename)
+        if 'args' in kwargs:
+            kwargs['args'] = [tmpf] + list(kwargs['args'])
+        else:
+            args = list(args)
+            args[0] = [tmpf] + args[0]
+
+        util.write_file(tmpf, blob, mode=0o700)
+        return subp.subp(*args, **kwargs)
 
 
 def install_chef_from_omnibus(url=None, retries=None, omnibus_version=None):
@@ -305,7 +332,7 @@ def install_chef_from_omnibus(url=None, retries=None, omnibus_version=None):
     else:
         args = ['-v', omnibus_version]
     content = url_helper.readurl(url=url, retries=retries).contents
-    return util.subp_blob_in_tempfile(
+    return subp_blob_in_tempfile(
         blob=content, args=args,
         basename='chef-omnibus-install', capture=False)
 
@@ -354,11 +381,11 @@ def install_chef_from_gems(ruby_version, chef_version, distro):
     if not os.path.exists('/usr/bin/ruby'):
         util.sym_link('/usr/bin/ruby%s' % ruby_version, '/usr/bin/ruby')
     if chef_version:
-        util.subp(['/usr/bin/gem', 'install', 'chef',
+        subp.subp(['/usr/bin/gem', 'install', 'chef',
                    '-v %s' % chef_version, '--no-ri',
                    '--no-rdoc', '--bindir', '/usr/bin', '-q'], capture=False)
     else:
-        util.subp(['/usr/bin/gem', 'install', 'chef',
+        subp.subp(['/usr/bin/gem', 'install', 'chef',
                    '--no-ri', '--no-rdoc', '--bindir',
                    '/usr/bin', '-q'], capture=False)
 

cloudinit/config/cc_disable_ec2_metadata.py

@@ -26,6 +26,7 @@
     disable_ec2_metadata: <true/false>
 """
 
+from cloudinit import subp
 from cloudinit import util
 
 from cloudinit.settings import PER_ALWAYS
@@ -40,15 +41,15 @@ def handle(name, cfg, _cloud, log, _args):
     disabled = util.get_cfg_option_bool(cfg, "disable_ec2_metadata", False)
     if disabled:
         reject_cmd = None
-        if util.which('ip'):
+        if subp.which('ip'):
             reject_cmd = REJECT_CMD_IP
-        elif util.which('ifconfig'):
+        elif subp.which('ifconfig'):
             reject_cmd = REJECT_CMD_IF
         else:
             log.error(('Neither "route" nor "ip" command found, unable to '
                        'manipulate routing table'))
             return
-        util.subp(reject_cmd, capture=False)
+        subp.subp(reject_cmd, capture=False)
     else:
         log.debug(("Skipping module named %s,"
                    " disabling the ec2 route not enabled"), name)