From 858e0a7a58a00a6573019b0eae0a2b1fb4f83214 Mon Sep 17 00:00:00 2001
From: Paride Legovini <paride.legovini@canonical.com>
Date: Fri, 28 Feb 2020 16:42:32 +0100
Subject: [PATCH 01/17] debian/rules.in: drop PYVER as python3 is the only
 supported version

---
 packages/debian/rules.in | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/packages/debian/rules.in b/packages/debian/rules.in
index e542c7f1737..021e4913dd9 100755
--- a/packages/debian/rules.in
+++ b/packages/debian/rules.in
@@ -2,11 +2,10 @@
 #!/usr/bin/make -f
 INIT_SYSTEM ?= systemd
 export PYBUILD_INSTALL_ARGS=--init-system=$(INIT_SYSTEM)
-PYVER ?= python${pyver}
 DEB_VERSION := $(shell dpkg-parsechangelog --show-field=Version)
 
 %:
-	dh $@ --with $(PYVER),systemd --buildsystem pybuild
+	dh $@ --with python3,systemd --buildsystem pybuild
 
 override_dh_install:
 	dh_install
@@ -19,7 +18,7 @@ override_dh_install:
 
 override_dh_auto_test:
 ifeq (,$(findstring nocheck,$(DEB_BUILD_OPTIONS)))
-	http_proxy= make PYVER=python${pyver} check
+	http_proxy= make check
 else
 	@echo check disabled by DEB_BUILD_OPTIONS=$(DEB_BUILD_OPTIONS)
 endif

From 9cf4b0a957285d51aa29764cc63e59a88db4614a Mon Sep 17 00:00:00 2001
From: Paride Legovini <paride.legovini@canonical.com>
Date: Sun, 1 Mar 2020 19:43:20 +0100
Subject: [PATCH 02/17] Makefile: drop everything related to Python 2

---
 Makefile | 51 ++++++++++++++-------------------------------------
 1 file changed, 14 insertions(+), 37 deletions(-)

diff --git a/Makefile b/Makefile
index 5f575633a01..a746b2a9342 100644
--- a/Makefile
+++ b/Makefile
@@ -1,38 +1,21 @@
 CWD=$(shell pwd)
-PYVER ?= $(shell for p in python3 python2; do \
-	out=$$(command -v $$p 2>&1) && echo $$p && exit; done; exit 1)
 
 YAML_FILES=$(shell find cloudinit tests tools -name "*.yaml" -type f )
 YAML_FILES+=$(shell find doc/examples -name "cloud-config*.txt" -type f )
 
 PIP_INSTALL := pip install
 
-ifeq ($(PYVER),python3)
-  pyflakes = pyflakes3
-  unittests = unittest3
-  yaml = yaml
-else
-ifeq ($(PYVER),python2)
-  pyflakes = pyflakes
-  unittests = unittest
-else
-  pyflakes = pyflakes pyflakes3
-  unittests = unittest unittest3
-endif
-endif
-
 ifeq ($(distro),)
   distro = redhat
 endif
 
-READ_VERSION=$(shell $(PYVER) $(CWD)/tools/read-version || \
-  echo read-version-failed)
-CODE_VERSION=$(shell $(PYVER) -c "from cloudinit import version; print(version.version_string())")
+READ_VERSION=$(shell python3 $(CWD)/tools/read-version || echo read-version-failed)
+CODE_VERSION=$(shell python3 -c "from cloudinit import version; print(version.version_string())")
 
 
 all: check
 
-check: check_version test $(yaml)
+check: check_version test yaml
 
 style-check: pep8 $(pyflakes)
 
@@ -40,22 +23,16 @@ pep8:
 	@$(CWD)/tools/run-pep8
 
 pyflakes:
-	@$(CWD)/tools/run-pyflakes
-
-pyflakes3:
 	@$(CWD)/tools/run-pyflakes3
 
 unittest: clean_pyc
-	python -m pytest -v tests/unittests cloudinit
-
-unittest3: clean_pyc
 	python3 -m pytest -v tests/unittests cloudinit
 
 ci-deps-ubuntu:
-	@$(PYVER) $(CWD)/tools/read-dependencies --distro ubuntu --test-distro
+	@python3 $(CWD)/tools/read-dependencies --distro ubuntu --test-distro
 
 ci-deps-centos:
-	@$(PYVER) $(CWD)/tools/read-dependencies --distro centos --test-distro
+	@python3 $(CWD)/tools/read-dependencies --distro centos --test-distro
 
 pip-requirements:
 	@echo "Installing cloud-init dependencies..."
@@ -65,7 +42,7 @@ pip-test-requirements:
 	@echo "Installing cloud-init test dependencies..."
 	$(PIP_INSTALL) -r "$@.txt" -q
 
-test: $(unittests)
+test: unittest
 
 check_version:
 	@if [ "$(READ_VERSION)" != "$(CODE_VERSION)" ]; then \
@@ -74,7 +51,7 @@ check_version:
 	    else true; fi
 
 config/cloud.cfg:
-	$(PYVER) ./tools/render-cloudcfg config/cloud.cfg.tmpl config/cloud.cfg
+	python3 ./tools/render-cloudcfg config/cloud.cfg.tmpl config/cloud.cfg
 
 clean_pyc:
 	@find . -type f -name "*.pyc" -delete
@@ -84,30 +61,30 @@ clean: clean_pyc
 	rm -rf doc/rtd_html .tox .coverage
 
 yaml:
-	@$(PYVER) $(CWD)/tools/validate-yaml.py $(YAML_FILES)
+	@python3 $(CWD)/tools/validate-yaml.py $(YAML_FILES)
 
 rpm:
-	$(PYVER) ./packages/brpm --distro=$(distro)
+	python3 ./packages/brpm --distro=$(distro)
 
 srpm:
-	$(PYVER) ./packages/brpm --srpm --distro=$(distro)
+	python3 ./packages/brpm --srpm --distro=$(distro)
 
 deb:
 	@which debuild || \
 		{ echo "Missing devscripts dependency. Install with:"; \
 		  echo sudo apt-get install devscripts; exit 1; }
 
-	$(PYVER) ./packages/bddeb
+	python3 ./packages/bddeb
 
 deb-src:
 	@which debuild || \
 		{ echo "Missing devscripts dependency. Install with:"; \
 		  echo sudo apt-get install devscripts; exit 1; }
-	$(PYVER) ./packages/bddeb -S -d
+	python3 ./packages/bddeb -S -d
 
 doc:
 	tox -e doc
 
-.PHONY: test pyflakes pyflakes3 clean pep8 rpm srpm deb deb-src yaml
+.PHONY: test pyflakes clean pep8 rpm srpm deb deb-src yaml
 .PHONY: check_version pip-test-requirements pip-requirements clean_pyc
-.PHONY: unittest unittest3 style-check doc
+.PHONY: unittest style-check doc

From 14c9da8483c61bc0c3bff42ebca48bdb6890bbc9 Mon Sep 17 00:00:00 2001
From: Paride Legovini <paride.legovini@canonical.com>
Date: Mon, 2 Mar 2020 18:02:08 +0100
Subject: [PATCH 03/17] run-container: install the CI deps only on
 ubuntu|debian

---
 tools/run-container | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/tools/run-container b/tools/run-container
index a7a552ec1cb..766114e0821 100755
--- a/tools/run-container
+++ b/tools/run-container
@@ -463,9 +463,9 @@ main() {
         return
     }
 
-    inside_as_cd "$name" root "$cdir" \
-        python3 ./tools/read-dependencies "--distro=${OS_NAME}" \
-            --test-distro || {
+    local rdcmd=(python3 tools/read-dependencies "--distro=${OS_NAME}" --install)
+    [[ ${OS_NAME} =~ (ubuntu|debian) ]] && rdcmd+=('--test-distro')
+    inside_as_cd "$name" root "$cdir" "${rdcmd[@]}" || {
         errorrc "FAIL: failed to install dependencies with read-dependencies"
         return
     }

From d69c200fc232de21572e664a5d8166e55077645f Mon Sep 17 00:00:00 2001
From: Paride Legovini <paride.legovini@canonical.com>
Date: Mon, 2 Mar 2020 18:05:23 +0100
Subject: [PATCH 04/17] brpm: read_dependencies(): drop unused argument

---
 packages/brpm | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/brpm b/packages/brpm
index 4004fd0e59b..1be8804cddb 100755
--- a/packages/brpm
+++ b/packages/brpm
@@ -42,7 +42,7 @@ def run_helper(helper, args=None, strip=True):
     return stdout
 
 
-def read_dependencies(distro, requirements_file='requirements.txt'):
+def read_dependencies(distro):
     """Returns the Python package depedencies from requirements.txt files.
 
     @returns a tuple of (requirements, test_requirements)

From 57f9244462992718294b39c94e69f17a281ab718 Mon Sep 17 00:00:00 2001
From: Paride Legovini <paride.legovini@canonical.com>
Date: Mon, 2 Mar 2020 18:15:06 +0100
Subject: [PATCH 05/17] read-dependencies: switch to Py3 and drop the
 --python-version option

---
 tools/read-dependencies | 29 ++++++++---------------------
 1 file changed, 8 insertions(+), 21 deletions(-)

diff --git a/tools/read-dependencies b/tools/read-dependencies
index 75d08a34253..693d211301c 100755
--- a/tools/read-dependencies
+++ b/tools/read-dependencies
@@ -9,7 +9,7 @@ try:
     from argparse import ArgumentParser
 except ImportError:
     raise RuntimeError(
-        'Could not import python-argparse. Please install python-argparse '
+        'Could not import argparse. Please install python3-argparse '
         'package to continue')
 
 import json
@@ -73,8 +73,8 @@ DISTRO_INSTALL_PKG_CMD = {
 # List of base system packages required to enable ci automation
 CI_SYSTEM_BASE_PKGS = {
     'common': ['make', 'sudo', 'tar'],
-    'redhat': ['python-tox'],
-    'centos': ['python-tox'],
+    'redhat': ['python3-tox'],
+    'centos': ['python3-tox'],
     'ubuntu': ['devscripts', 'python3-dev', 'libssl-dev', 'tox', 'sbuild'],
     'debian': ['devscripts', 'python3-dev', 'libssl-dev', 'tox', 'sbuild']}
 
@@ -99,7 +99,7 @@ def get_parser():
     parser.add_argument(
         '-s', '--system-pkg-names', action='store_true', default=False,
         dest='system_pkg_names',
-        help='The name of the distro to generate package deps for.')
+        help='Generate distribution package names (python3-pkgname).')
     parser.add_argument(
         '-i', '--install', action='store_true', default=False,
         dest='install',
@@ -109,12 +109,6 @@ def get_parser():
         dest='test_distro',
         help='Additionally install continuous integration system packages '
              'required for build and test automation.')
-    parser.add_argument(
-        '-v', '--python-version', type=str, dest='python_version',
-        default=None, choices=["2", "3"],
-        help='Override the version of python we want to generate system '
-             'package dependencies for. Defaults to the version of python '
-             'this script is called with')
     return parser
 
 
@@ -155,27 +149,20 @@ def parse_pip_requirements(requirements_path):
     return dep_names
 
 
-def translate_pip_to_system_pkg(pip_requires, renames, python_ver):
+def translate_pip_to_system_pkg(pip_requires, renames):
     """Translate pip package names to distro-specific package names.
 
     @param pip_requires: List of versionless pip package names to translate.
     @param renames: Dict containg special case renames from pip name to system
         package name for the distro.
-    @param python_ver: Optional python version string "2" or "3". When None,
-     use the python version that is calling this script via sys.version_info.
     """
-    if python_ver is None:
-        python_ver = str(sys.version_info[0])
-    if python_ver == "2":
-        prefix = "python-"
-    else:
-        prefix = "python3-"
+    prefix = "python3-"
     standard_pkg_name = "{0}{1}"
     translated_names = []
     for pip_name in pip_requires:
         pip_name = pip_name.lower()
         # Find a rename if present for the distro package and python version
-        rename = renames.get(pip_name, {}).get(python_ver, None)
+        rename = renames.get(pip_name, {})
         if rename:
             translated_names.append(rename)
         else:
@@ -222,7 +209,7 @@ def main(distro):
     deps_from_json = get_package_deps_from_json(topd, args.distro)
     renames = deps_from_json.get('renames', {})
     translated_pip_names = translate_pip_to_system_pkg(
-        pip_pkg_names, renames, args.python_version)
+        pip_pkg_names, renames)
     all_deps = []
     if args.distro:
         all_deps.extend(

From 099974b8bc9d82b5a839a89b3200ede7da4b74c7 Mon Sep 17 00:00:00 2001
From: Paride Legovini <paride.legovini@canonical.com>
Date: Mon, 2 Mar 2020 18:17:32 +0100
Subject: [PATCH 06/17] pkg-deps.json: drop the Python version field and update
 the redhat deps

As Python 3 is the only supported version the Python version field can
be dropped from pkg-deps.json.

In CentOS 8 and in the newer Fedora releases the Python 3 packages are
named python3-<pkgname>, just like in Ubuntu, so we can drop most of the
renames

The httpretty, unittest2 and contextlib2 packages are renamed to the
empty string as they are not available on CentOS 8. They are test
dependencies, and since we are not running the unit tests when building
the RPM package they can be safely ignored.
---
 packages/pkg-deps.json | 42 ++++++------------------------------------
 1 file changed, 6 insertions(+), 36 deletions(-)

diff --git a/packages/pkg-deps.json b/packages/pkg-deps.json
index f02e8348605..dbb5fbcddd2 100644
--- a/packages/pkg-deps.json
+++ b/packages/pkg-deps.json
@@ -6,14 +6,8 @@
          "dh-systemd"
       ],
       "renames" : {
-         "pyyaml" : {
-            "2" : "python-yaml",
-            "3" : "python3-yaml"
-         },
-         "pyserial" : {
-            "2" : "python-serial",
-            "3" : "python3-serial"
-         }
+         "pyyaml" : "python3-yaml",
+         "pyserial" : "python3-serial"
       },
       "requires" : [
          "procps"
@@ -21,33 +15,12 @@
    },
    "redhat" : {
       "build-requires" : [
-         "python-devel",
-         "python-setuptools"
+         "python3-devel"
       ],
       "renames" : {
-         "jinja2" : {
-            "3" : "python36-jinja2"
-         },
-         "jsonschema" : {
-            "3" : "python36-jsonschema"
-         },
-         "pyflakes" : {
-            "2" : "pyflakes",
-            "3" : "python36-pyflakes"
-         },
-         "pyyaml" : {
-            "2" : "PyYAML",
-            "3" : "python36-PyYAML"
-         },
-         "pyserial" : {
-            "2" : "pyserial"
-         },
-         "pytest": {
-             "3": "python36-pytest"
-         },
-         "requests" : {
-            "3" : "python36-requests"
-         }
+         "httpretty" : "",
+         "unittest2" : "",
+         "contextlib2" : ""
       },
       "requires" : [
          "e2fsprogs",
@@ -61,9 +34,6 @@
    },
    "suse" : {
       "renames" : {
-         "pyyaml" : {
-            "2" : "python-yaml"
-         }
       },
       "build-requires" : [
          "fdupes",

From 35e29ec07545fbdb4aad13bbf770e80c362e0166 Mon Sep 17 00:00:00 2001
From: Paride Legovini <paride.legovini@canonical.com>
Date: Mon, 2 Mar 2020 18:27:08 +0100
Subject: [PATCH 07/17] Update the spec file to use Python 3 when building the
 RPM

---
 packages/redhat/cloud-init.spec.in | 10 ++++------
 1 file changed, 4 insertions(+), 6 deletions(-)

diff --git a/packages/redhat/cloud-init.spec.in b/packages/redhat/cloud-init.spec.in
index 057a57845b6..4cff2c97603 100644
--- a/packages/redhat/cloud-init.spec.in
+++ b/packages/redhat/cloud-init.spec.in
@@ -1,6 +1,4 @@
 ## template: jinja
-%{!?python_sitelib: %global python_sitelib %(%{__python} -c "from distutils.sysconfig import get_python_lib; print get_python_lib()")}
-
 %define use_systemd (0%{?fedora} && 0%{?fedora} >= 18) || (0%{?rhel} && 0%{?rhel} >= 7)
 
 %if %{use_systemd}
@@ -94,11 +92,11 @@ ssh keys and to let the user run various scripts.
 {% endfor %}
 
 %build
-%{__python} setup.py build
+%{__python3} setup.py build
 
 %install
 
-%{__python} setup.py install -O1 \
+%{__python3} setup.py install -O1 \
             --skip-build --root $RPM_BUILD_ROOT \
             --init-system=%{init_system}
 
@@ -109,7 +107,7 @@ cp -p tools/21-cloudinit.conf \
       $RPM_BUILD_ROOT/%{_sysconfdir}/rsyslog.d/21-cloudinit.conf
 
 # Remove the tests
-rm -rf $RPM_BUILD_ROOT%{python_sitelib}/tests
+rm -rf $RPM_BUILD_ROOT%{python3_sitelib}/tests
 
 # Required dirs...
 mkdir -p $RPM_BUILD_ROOT/%{_sharedstatedir}/cloud
@@ -213,4 +211,4 @@ fi
 %dir %{_sharedstatedir}/cloud
 
 # Python code is here...
-%{python_sitelib}/*
+%{python3_sitelib}/*

From f28b0b5bb6c6368fc5fa300263ab8957bef0902f Mon Sep 17 00:00:00 2001
From: Paride Legovini <paride.legovini@canonical.com>
Date: Tue, 3 Mar 2020 15:36:59 +0100
Subject: [PATCH 08/17] Make the deb packaging tools Python 3 only

Changes:
 - bddeb: drop support for Python 2
 - Do not template d/rules (not needed anymore)
 - d/rules: drop XB-Python-Version (deprecated, [1])

[1] https://www.debian.org/doc/packaging-manuals/python-policy/ch-module_packages.html
---
 packages/bddeb                      | 26 ++++++--------------------
 packages/debian/control.in          |  3 +--
 packages/debian/{rules.in => rules} |  1 -
 3 files changed, 7 insertions(+), 23 deletions(-)
 rename packages/debian/{rules.in => rules} (98%)

diff --git a/packages/bddeb b/packages/bddeb
index 209765a5d14..02ac2975045 100755
--- a/packages/bddeb
+++ b/packages/bddeb
@@ -59,15 +59,9 @@ def run_helper(helper, args=None, strip=True):
     return stdout
 
 
-def write_debian_folder(root, templ_data, is_python2, cloud_util_deps):
+def write_debian_folder(root, templ_data, cloud_util_deps):
     """Create a debian package directory with all rendered template files."""
     print("Creating a debian/ folder in %r" % (root))
-    if is_python2:
-        pyver = "2"
-        python = "python"
-    else:
-        pyver = "3"
-        python = "python3"
 
     deb_dir = util.abs_join(root, 'debian')
 
@@ -83,30 +77,23 @@ def write_debian_folder(root, templ_data, is_python2, cloud_util_deps):
 
     # Write out the control file template
     reqs_output = run_helper(
-        'read-dependencies',
-        args=['--distro', 'debian', '--python-version', pyver])
+        'read-dependencies', args=['--distro', 'debian'])
     reqs = reqs_output.splitlines()
     test_reqs = run_helper(
         'read-dependencies',
         ['--requirements-file', 'test-requirements.txt',
-         '--system-pkg-names', '--python-version', pyver]).splitlines()
+         '--system-pkg-names']).splitlines()
 
     requires = ['cloud-utils | cloud-guest-utils'] if cloud_util_deps else []
     # We consolidate all deps as Build-Depends as our package build runs all
     # tests so we need all runtime dependencies anyway.
     # NOTE: python package was moved to the front after debuild -S would fail with
     # 'Please add apropriate interpreter' errors (as in debian bug 861132)
-    requires.extend([python] + reqs + test_reqs)
+    requires.extend(['python3'] + reqs + test_reqs)
     templater.render_to_file(util.abs_join(find_root(),
                                            'packages', 'debian', 'control.in'),
                              util.abs_join(deb_dir, 'control'),
-                             params={'build_depends': ','.join(requires),
-                                     'python': python})
-
-    templater.render_to_file(util.abs_join(find_root(),
-                                           'packages', 'debian', 'rules.in'),
-                             util.abs_join(deb_dir, 'rules'),
-                             params={'python': python, 'pyver': pyver})
+                             params={'build_depends': ','.join(requires)})
 
 
 def read_version():
@@ -208,8 +195,7 @@ def main():
         xdir = util.abs_join(tdir, "cloud-init-%s" % ver_data['version_long'])
         templ_data.update(ver_data)
 
-        write_debian_folder(xdir, templ_data, is_python2=args.python2,
-                            cloud_util_deps=args.cloud_utils)
+        write_debian_folder(xdir, templ_data, cloud_util_deps=args.cloud_utils)
 
         print("Running 'debuild %s' in %r" % (' '.join(args.debuild_args),
                                               xdir))
diff --git a/packages/debian/control.in b/packages/debian/control.in
index e9ed64f348b..72895b47af1 100644
--- a/packages/debian/control.in
+++ b/packages/debian/control.in
@@ -10,11 +10,10 @@ Standards-Version: 3.9.6
 Package: cloud-init
 Architecture: all
 Depends: ${misc:Depends},
-         ${${python}:Depends},
+         ${python3:Depends},
          iproute2,
          isc-dhcp-client
 Recommends: eatmydata, sudo, software-properties-common, gdisk
-XB-Python-Version: ${python:Versions}
 Description: Init scripts for cloud instances
  Cloud instances need special scripts to run during initialisation
  to retrieve and install ssh keys and to let the user run various scripts.
diff --git a/packages/debian/rules.in b/packages/debian/rules
similarity index 98%
rename from packages/debian/rules.in
rename to packages/debian/rules
index 021e4913dd9..d138deebe35 100755
--- a/packages/debian/rules.in
+++ b/packages/debian/rules
@@ -1,4 +1,3 @@
-## template:basic
 #!/usr/bin/make -f
 INIT_SYSTEM ?= systemd
 export PYBUILD_INSTALL_ARGS=--init-system=$(INIT_SYSTEM)

From 72fe16e725c5f624107821d65e1f379d801cad3b Mon Sep 17 00:00:00 2001
From: Paride Legovini <paride.legovini@canonical.com>
Date: Thu, 5 Mar 2020 15:39:30 +0100
Subject: [PATCH 09/17] Makefile: set a PYTHON variable with the Python
 interpterer to use

---
 Makefile | 21 +++++++++++----------
 1 file changed, 11 insertions(+), 10 deletions(-)

diff --git a/Makefile b/Makefile
index a746b2a9342..340c9e2d400 100644
--- a/Makefile
+++ b/Makefile
@@ -3,14 +3,15 @@ CWD=$(shell pwd)
 YAML_FILES=$(shell find cloudinit tests tools -name "*.yaml" -type f )
 YAML_FILES+=$(shell find doc/examples -name "cloud-config*.txt" -type f )
 
+PYTHON = python3
 PIP_INSTALL := pip install
 
 ifeq ($(distro),)
   distro = redhat
 endif
 
-READ_VERSION=$(shell python3 $(CWD)/tools/read-version || echo read-version-failed)
-CODE_VERSION=$(shell python3 -c "from cloudinit import version; print(version.version_string())")
+READ_VERSION=$(shell $(PYTHON) $(CWD)/tools/read-version || echo read-version-failed)
+CODE_VERSION=$(shell $(PYTHON) -c "from cloudinit import version; print(version.version_string())")
 
 
 all: check
@@ -29,10 +30,10 @@ unittest: clean_pyc
 	python3 -m pytest -v tests/unittests cloudinit
 
 ci-deps-ubuntu:
-	@python3 $(CWD)/tools/read-dependencies --distro ubuntu --test-distro
+	@$(PYTHON) $(CWD)/tools/read-dependencies --distro ubuntu --test-distro
 
 ci-deps-centos:
-	@python3 $(CWD)/tools/read-dependencies --distro centos --test-distro
+	@$(PYTHON) $(CWD)/tools/read-dependencies --distro centos --test-distro
 
 pip-requirements:
 	@echo "Installing cloud-init dependencies..."
@@ -51,7 +52,7 @@ check_version:
 	    else true; fi
 
 config/cloud.cfg:
-	python3 ./tools/render-cloudcfg config/cloud.cfg.tmpl config/cloud.cfg
+	$(PYTHON) ./tools/render-cloudcfg config/cloud.cfg.tmpl config/cloud.cfg
 
 clean_pyc:
 	@find . -type f -name "*.pyc" -delete
@@ -61,26 +62,26 @@ clean: clean_pyc
 	rm -rf doc/rtd_html .tox .coverage
 
 yaml:
-	@python3 $(CWD)/tools/validate-yaml.py $(YAML_FILES)
+	@$(PYTHON) $(CWD)/tools/validate-yaml.py $(YAML_FILES)
 
 rpm:
-	python3 ./packages/brpm --distro=$(distro)
+	$(PYTHON) ./packages/brpm --distro=$(distro)
 
 srpm:
-	python3 ./packages/brpm --srpm --distro=$(distro)
+	$(PYTHON) ./packages/brpm --srpm --distro=$(distro)
 
 deb:
 	@which debuild || \
 		{ echo "Missing devscripts dependency. Install with:"; \
 		  echo sudo apt-get install devscripts; exit 1; }
 
-	python3 ./packages/bddeb
+	$(PYTHON) ./packages/bddeb
 
 deb-src:
 	@which debuild || \
 		{ echo "Missing devscripts dependency. Install with:"; \
 		  echo sudo apt-get install devscripts; exit 1; }
-	python3 ./packages/bddeb -S -d
+	$(PYTHON) ./packages/bddeb -S -d
 
 doc:
 	tox -e doc

From 75cbc055441bce88c7ab6f5b50a67db802b96bb3 Mon Sep 17 00:00:00 2001
From: Paride Legovini <paride.legovini@canonical.com>
Date: Thu, 5 Mar 2020 15:43:23 +0100
Subject: [PATCH 10/17] Makefile: call 'pip3 install' instead of 'pip install'

---
 Makefile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Makefile b/Makefile
index 340c9e2d400..9cc66f09008 100644
--- a/Makefile
+++ b/Makefile
@@ -4,7 +4,7 @@ YAML_FILES=$(shell find cloudinit tests tools -name "*.yaml" -type f )
 YAML_FILES+=$(shell find doc/examples -name "cloud-config*.txt" -type f )
 
 PYTHON = python3
-PIP_INSTALL := pip install
+PIP_INSTALL := pip3 install
 
 ifeq ($(distro),)
   distro = redhat

From 14066220fdf823082cdc54765de4c4b7aab66f95 Mon Sep 17 00:00:00 2001
From: Paride Legovini <paride.legovini@canonical.com>
Date: Thu, 5 Mar 2020 15:46:19 +0100
Subject: [PATCH 11/17] Make tools/run-pyflakes Python 3 only

---
 Makefile            | 2 +-
 tools/run-pyflakes  | 3 +--
 tools/run-pyflakes3 | 2 --
 3 files changed, 2 insertions(+), 5 deletions(-)
 delete mode 100755 tools/run-pyflakes3

diff --git a/Makefile b/Makefile
index 9cc66f09008..5fb0fcbfe8b 100644
--- a/Makefile
+++ b/Makefile
@@ -24,7 +24,7 @@ pep8:
 	@$(CWD)/tools/run-pep8
 
 pyflakes:
-	@$(CWD)/tools/run-pyflakes3
+	@$(CWD)/tools/run-pyflakes
 
 unittest: clean_pyc
 	python3 -m pytest -v tests/unittests cloudinit
diff --git a/tools/run-pyflakes b/tools/run-pyflakes
index b3759a9421a..179afebe8f3 100755
--- a/tools/run-pyflakes
+++ b/tools/run-pyflakes
@@ -1,6 +1,5 @@
 #!/bin/bash
 
-PYTHON_VERSION=${PYTHON_VERSION:-2}
 CR="
 "
 pycheck_dirs=( "cloudinit/" "tests/" "tools/" )
@@ -12,7 +11,7 @@ else
    files=( "$@" )
 fi
 
-cmd=( "python${PYTHON_VERSION}" -m "pyflakes" "${files[@]}" )
+cmd=( "python3" -m "pyflakes" "${files[@]}" )
 
 echo "Running: " "${cmd[@]}" 1>&2
 exec "${cmd[@]}"
diff --git a/tools/run-pyflakes3 b/tools/run-pyflakes3
deleted file mode 100755
index e9f0863d925..00000000000
--- a/tools/run-pyflakes3
+++ /dev/null
@@ -1,2 +0,0 @@
-#!/bin/sh
-PYTHON_VERSION=3 exec "${0%/*}/run-pyflakes" "$@"

From 02bdbce2a213d72092353437880d09d863249170 Mon Sep 17 00:00:00 2001
From: Paride Legovini <paride.legovini@canonical.com>
Date: Thu, 12 Mar 2020 15:48:49 +0100
Subject: [PATCH 12/17] run-container: print a warning when skipping
 --test-distro

run-container: print a warning when not passing --test-distro to
read-dependencies because the target OS is known to miss some test
dependencies. Only CentOS is covered at the moment, but in principle
other distros can be added.
---
 tools/run-container | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/tools/run-container b/tools/run-container
index 766114e0821..c25ebfc682f 100755
--- a/tools/run-container
+++ b/tools/run-container
@@ -464,7 +464,11 @@ main() {
     }
 
     local rdcmd=(python3 tools/read-dependencies "--distro=${OS_NAME}" --install)
-    [[ ${OS_NAME} =~ (ubuntu|debian) ]] && rdcmd+=('--test-distro')
+    if [[ ${OS_NAME} == centos ]]; then
+        error "WARNING: Can't setup CentOS for testing (missing deps, LP: #1867151)."
+    else
+        rdcmd+=('--test-distro')
+    fi
     inside_as_cd "$name" root "$cdir" "${rdcmd[@]}" || {
         errorrc "FAIL: failed to install dependencies with read-dependencies"
         return

From 29ab2bb0f0c43d66a38f1f8d2f1c58ae3482fd86 Mon Sep 17 00:00:00 2001
From: Paride Legovini <paride.legovini@canonical.com>
Date: Wed, 29 Apr 2020 15:56:50 +0200
Subject: [PATCH 13/17] run-container: always try to install all the test deps

---
 tools/run-container | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/tools/run-container b/tools/run-container
index c25ebfc682f..31a39b06920 100755
--- a/tools/run-container
+++ b/tools/run-container
@@ -463,11 +463,10 @@ main() {
         return
     }
 
-    local rdcmd=(python3 tools/read-dependencies "--distro=${OS_NAME}" --install)
+    local rdcmd=(python3 tools/read-dependencies "--distro=${OS_NAME}" --install --test-distro)
     if [[ ${OS_NAME} == centos ]]; then
-        error "WARNING: Can't setup CentOS for testing (missing deps, LP: #1867151)."
-    else
-        rdcmd+=('--test-distro')
+        error "WARNING: Unable to install all CentOS for test dependencies (missing deps, LP: #1867151)."
+        unittest=""
     fi
     inside_as_cd "$name" root "$cdir" "${rdcmd[@]}" || {
         errorrc "FAIL: failed to install dependencies with read-dependencies"

From 7ba5f1fb02c6a4123d3e70c82003f74c6923498f Mon Sep 17 00:00:00 2001
From: Paride Legovini <paride.legovini@canonical.com>
Date: Wed, 29 Apr 2020 15:57:59 +0200
Subject: [PATCH 14/17] pkg-deps: split centos and redhat; allow omitting deps

* Add a centos distro reference for renames since fedora/redhat are slightly
  different and have those missing deps
* Add an OMIT: in rename to pkg-deps.json for the missing deps and allow
  read-dependencies to skip specific deps
---
 packages/pkg-deps.json  | 21 +++++++++++++++++----
 tools/read-dependencies |  7 ++++++-
 2 files changed, 23 insertions(+), 5 deletions(-)

diff --git a/packages/pkg-deps.json b/packages/pkg-deps.json
index dbb5fbcddd2..114e921e4af 100644
--- a/packages/pkg-deps.json
+++ b/packages/pkg-deps.json
@@ -13,14 +13,13 @@
          "procps"
       ]
    },
-   "redhat" : {
+   "centos" : {
       "build-requires" : [
          "python3-devel"
       ],
       "renames" : {
-         "httpretty" : "",
-         "unittest2" : "",
-         "contextlib2" : ""
+         "unittest2" : "OMIT: LP: #1867151 Missing CentOS pkg deps",
+         "contextlib2" : "OMIT: LP: #1867151 Missing CentOS pkg deps"
       },
       "requires" : [
          "e2fsprogs",
@@ -32,6 +31,20 @@
          "sudo"
       ]
    },
+   "redhat" : {
+      "build-requires" : [
+         "python3-devel"
+      ],
+      "requires" : [
+         "e2fsprogs",
+         "iproute",
+         "net-tools",
+         "procps",
+         "rsyslog",
+         "shadow-utils",
+         "sudo"
+      ]
+   },
    "suse" : {
       "renames" : {
       },
diff --git a/tools/read-dependencies b/tools/read-dependencies
index 693d211301c..23d00163fb6 100755
--- a/tools/read-dependencies
+++ b/tools/read-dependencies
@@ -126,6 +126,9 @@ def get_package_deps_from_json(topdir, distro):
         deps = json.loads(stream.read())
     if distro is None:
         return {}
+    if deps.get(distro):  # If we have a specific distro defined, use it.
+        return deps[distro]
+    # Use generic distro dependency map via DISTRO_PKG_TYPE_MAP
     return deps[DISTRO_PKG_TYPE_MAP[distro]]
 
 
@@ -162,7 +165,9 @@ def translate_pip_to_system_pkg(pip_requires, renames):
     for pip_name in pip_requires:
         pip_name = pip_name.lower()
         # Find a rename if present for the distro package and python version
-        rename = renames.get(pip_name, {})
+        rename = renames.get(pip_name, "")
+        if rename.startswith("OMIT:"):
+            continue
         if rename:
             translated_names.append(rename)
         else:

From a939e3cb4f62b3a7a0681deb5e5ff4b545efc87d Mon Sep 17 00:00:00 2001
From: Paride Legovini <paride.legovini@canonical.com>
Date: Fri, 1 May 2020 15:17:06 +0200
Subject: [PATCH 15/17] pkg-deps: drop the unittest2 and contextlib2 renames

unittest2 and contextlib2 are not longer needed in the tip of master.
---
 packages/pkg-deps.json | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/packages/pkg-deps.json b/packages/pkg-deps.json
index 114e921e4af..8002839660c 100644
--- a/packages/pkg-deps.json
+++ b/packages/pkg-deps.json
@@ -17,10 +17,6 @@
       "build-requires" : [
          "python3-devel"
       ],
-      "renames" : {
-         "unittest2" : "OMIT: LP: #1867151 Missing CentOS pkg deps",
-         "contextlib2" : "OMIT: LP: #1867151 Missing CentOS pkg deps"
-      },
       "requires" : [
          "e2fsprogs",
          "iproute",

From 9179b289eb9b645805295b403178636fec5a9331 Mon Sep 17 00:00:00 2001
From: Paride Legovini <paride.legovini@canonical.com>
Date: Fri, 1 May 2020 15:18:13 +0200
Subject: [PATCH 16/17] read-dependencies: drop the OMIT logic

There are no packages to omit anymore, let's drop the shortly lived OMIT
logic.
---
 tools/read-dependencies | 2 --
 1 file changed, 2 deletions(-)

diff --git a/tools/read-dependencies b/tools/read-dependencies
index 23d00163fb6..666e24f541e 100755
--- a/tools/read-dependencies
+++ b/tools/read-dependencies
@@ -166,8 +166,6 @@ def translate_pip_to_system_pkg(pip_requires, renames):
         pip_name = pip_name.lower()
         # Find a rename if present for the distro package and python version
         rename = renames.get(pip_name, "")
-        if rename.startswith("OMIT:"):
-            continue
         if rename:
             translated_names.append(rename)
         else:

From 02359578aa5db42f17aeff7f98efbe30bbe857b2 Mon Sep 17 00:00:00 2001
From: Paride Legovini <paride.legovini@canonical.com>
Date: Fri, 1 May 2020 15:22:35 +0200
Subject: [PATCH 17/17] run-container: re-enable the unittests on CentOS

All the required dependencies should be available now.
---
 tools/run-container | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/tools/run-container b/tools/run-container
index 31a39b06920..7212550eb61 100755
--- a/tools/run-container
+++ b/tools/run-container
@@ -464,10 +464,6 @@ main() {
     }
 
     local rdcmd=(python3 tools/read-dependencies "--distro=${OS_NAME}" --install --test-distro)
-    if [[ ${OS_NAME} == centos ]]; then
-        error "WARNING: Unable to install all CentOS for test dependencies (missing deps, LP: #1867151)."
-        unittest=""
-    fi
     inside_as_cd "$name" root "$cdir" "${rdcmd[@]}" || {
         errorrc "FAIL: failed to install dependencies with read-dependencies"
         return