From bffadb684aba53e096afff3da42ba6c812c16030 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Mon, 18 May 2020 22:40:14 +0100 Subject: [PATCH 1/5] Fix python3 incompatibility iteritems doesn't exist in python 3. --- bin/dh_virtualenv | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/dh_virtualenv b/bin/dh_virtualenv index f4cb91f..a364e6a 100755 --- a/bin/dh_virtualenv +++ b/bin/dh_virtualenv @@ -38,7 +38,7 @@ log = logging.getLogger(__name__) def _shell_vars(**kwargs): """Convert the given values into the equivalent shell snippet defining them.""" return '\n'.join("dh_venv_{0}='{1}'".format(k, v.replace("'", r"'\''")) - for k, v in sorted(kwargs.iteritems())) + for k, v in sorted(kwargs.items())) def main(): From cf63ff7cfafdfef17935cd82c08952ff39ada48d Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Mon, 18 May 2020 20:19:30 +0100 Subject: [PATCH 2/5] Import debhelper.py This seems to have been removed from debian in recent versions, and it's not that big, so let's just bring it in here. --- bin/dh_virtualenv | 5 +- dh_virtualenv/debhelper.py | 210 +++++++++++++++++++++++++++++++++++++ 2 files changed, 211 insertions(+), 4 deletions(-) create mode 100644 dh_virtualenv/debhelper.py diff --git a/bin/dh_virtualenv b/bin/dh_virtualenv index a364e6a..8bafbcf 100755 --- a/bin/dh_virtualenv +++ b/bin/dh_virtualenv @@ -23,12 +23,9 @@ import logging import os import sys -# The debpython resides here -sys.path.insert(1, '/usr/share/python/') - -from debpython.debhelper import DebHelper from dh_virtualenv import Deployment from dh_virtualenv.cmdline import get_default_parser +from dh_virtualenv.debhelper import DebHelper logging.basicConfig(format='%(levelname).1s: %(module)s:%(lineno)d: ' '%(message)s') diff --git a/dh_virtualenv/debhelper.py b/dh_virtualenv/debhelper.py new file mode 100644 index 0000000..2b6b299 --- /dev/null +++ b/dh_virtualenv/debhelper.py @@ -0,0 +1,210 @@ +# -*- coding: UTF-8 -*- +# Copyright © 2010-2012 Piotr Ożarowski +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. + +import logging +from os import makedirs, chmod +from os.path import exists, join, dirname + +log = logging.getLogger(__name__) + + +class DebHelper(object): + """Reinvents the wheel / some dh functionality (Perl is ugly ;-P)""" + + def __init__(self, options): + self.options = options + self.packages = {} + self.python_version = None + source_section = True + binary_package = None + + pkgs = options.package + skip_pkgs = options.no_package + + try: + fp = open('debian/control', 'r') + except IOError: + raise Exception('cannot find debian/control file') + + xspv = xpv = False + for line in fp: + if not line.strip(): + source_section = False + binary_package = None + continue + if binary_package: + if binary_package.startswith('python3'): + continue + if pkgs and binary_package not in pkgs: + continue + if skip_pkgs and binary_package in skip_pkgs: + continue + if line.startswith('Architecture:'): + arch = line[13:].strip() + # TODO: if arch doesn't match current architecture: + #del self.packages[binary_package] + self.packages[binary_package]['arch'] = arch + continue + elif line.startswith('Package:'): + binary_package = line[8:].strip() + if binary_package.startswith('python3'): + log.debug('skipping Python 3.X package: %s', binary_package) + continue + if pkgs and binary_package not in pkgs: + continue + if skip_pkgs and binary_package in skip_pkgs: + continue + self.packages[binary_package] = {'substvars': {}, + 'autoscripts': {}, + 'rtupdates': [], + 'arch': 'any'} + elif line.startswith('Source:'): + self.source_name = line[7:].strip() + elif source_section: + if line.lower().startswith('xs-python-version:'): + xspv = True + if not self.python_version: + self.python_version = line[18:].strip() + if line.lower().startswith('x-python-version:'): + xpv = True + self.python_version = line[17:].strip() + + if xspv and xpv: + log.error('Please remove XS-Python-Version from debian/control') + + log.debug('source=%s, binary packages=%s', self.source_name, \ + self.packages.keys()) + + def addsubstvar(self, package, name, value): + """debhelper's addsubstvar""" + self.packages[package]['substvars'].setdefault(name, []).append(value) + + def autoscript(self, package, when, template, args): + """debhelper's autoscript""" + self.packages[package]['autoscripts'].setdefault(when, {})\ + .setdefault(template, []).append(args) + + def add_rtupdate(self, package, value): + self.packages[package]['rtupdates'].append(value) + + def save_autoscripts(self): + for package, settings in self.packages.iteritems(): + autoscripts = settings.get('autoscripts') + if not autoscripts: + continue + + for when, templates in autoscripts.iteritems(): + fn = "debian/%s.%s.debhelper" % (package, when) + if exists(fn): + data = open(fn, 'r').read() + else: + data = '' + + new_data = '' + for tpl_name, args in templates.iteritems(): + for i in args: + # try local one first (useful while testing dh_python2) + fpath = join(dirname(__file__), '..', + "autoscripts/%s" % tpl_name) + if not exists(fpath): + fpath = "/usr/share/debhelper/autoscripts/%s" % tpl_name + tpl = open(fpath, 'r').read() + if self.options.compile_all and args: + # TODO: should args be checked to contain dir name? + tpl = tpl.replace('#PACKAGE#', '') + else: + tpl = tpl.replace('#PACKAGE#', package) + tpl = tpl.replace('#ARGS#', i) + if tpl not in data and tpl not in new_data: + new_data += "\n%s" % tpl + if new_data: + data += "\n# Automatically added by dh_python2:" +\ + "%s\n# End automatically added section\n" % new_data + fp = open(fn, 'w') + fp.write(data) + fp.close() + + def save_substvars(self): + for package, settings in self.packages.iteritems(): + substvars = settings.get('substvars') + if not substvars: + continue + fn = "debian/%s.substvars" % package + if exists(fn): + data = open(fn, 'r').read() + else: + data = '' + for name, values in substvars.iteritems(): + p = data.find("%s=" % name) + if p > -1: # parse the line and remove it from data + e = data[p:].find('\n') + line = data[p + len("%s=" % name):\ + p + e if e > -1 else None] + items = [i.strip() for i in line.split(',') if i] + if e > -1 and data[p + e:].strip(): + data = "%s\n%s" % (data[:p], data[p + e:]) + else: + data = data[:p] + else: + items = [] + for j in values: + if j not in items: + items.append(j) + if items: + if data: + data += '\n' + data += "%s=%s\n" % (name, ', '.join(items)) + data = data.replace('\n\n', '\n') + if data: + fp = open(fn, 'w') + fp.write(data) + fp.close() + + def save_rtupdate(self): + for package, settings in self.packages.iteritems(): + pkg_arg = '' if self.options.compile_all else "-p %s" % package + values = settings.get('rtupdates') + if not values: + continue + d = "debian/%s/usr/share/python/runtime.d" % package + if not exists(d): + makedirs(d) + fn = "%s/%s.rtupdate" % (d, package) + if exists(fn): + data = open(fn, 'r').read() + else: + data = "#! /bin/sh\nset -e" + for dname, args in values: + cmd = 'if [ "$1" = rtupdate ]; then' +\ + "\n\tpyclean %s %s" % (pkg_arg, dname) +\ + "\n\tpycompile %s %s %s\nfi" % (pkg_arg, args, dname) + if cmd not in data: + data += "\n%s" % cmd + if data: + fp = open(fn, 'w') + fp.write(data) + fp.close() + chmod(fn, 0755) + + def save(self): + self.save_substvars() + self.save_autoscripts() + self.save_rtupdate() From 7955a0703b1b22ef78100d129b43345289f97325 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Mon, 18 May 2020 20:27:00 +0100 Subject: [PATCH 3/5] Fix python3 incompatibilities in debhelper.py --- dh_virtualenv/debhelper.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/dh_virtualenv/debhelper.py b/dh_virtualenv/debhelper.py index 2b6b299..3de8e03 100644 --- a/dh_virtualenv/debhelper.py +++ b/dh_virtualenv/debhelper.py @@ -106,12 +106,12 @@ def add_rtupdate(self, package, value): self.packages[package]['rtupdates'].append(value) def save_autoscripts(self): - for package, settings in self.packages.iteritems(): + for package, settings in self.packages.items(): autoscripts = settings.get('autoscripts') if not autoscripts: continue - for when, templates in autoscripts.iteritems(): + for when, templates in autoscripts.items(): fn = "debian/%s.%s.debhelper" % (package, when) if exists(fn): data = open(fn, 'r').read() @@ -119,7 +119,7 @@ def save_autoscripts(self): data = '' new_data = '' - for tpl_name, args in templates.iteritems(): + for tpl_name, args in templates.items(): for i in args: # try local one first (useful while testing dh_python2) fpath = join(dirname(__file__), '..', @@ -143,7 +143,7 @@ def save_autoscripts(self): fp.close() def save_substvars(self): - for package, settings in self.packages.iteritems(): + for package, settings in self.packages.items(): substvars = settings.get('substvars') if not substvars: continue @@ -152,7 +152,7 @@ def save_substvars(self): data = open(fn, 'r').read() else: data = '' - for name, values in substvars.iteritems(): + for name, values in substvars.items(): p = data.find("%s=" % name) if p > -1: # parse the line and remove it from data e = data[p:].find('\n') @@ -179,7 +179,7 @@ def save_substvars(self): fp.close() def save_rtupdate(self): - for package, settings in self.packages.iteritems(): + for package, settings in self.packages.items(): pkg_arg = '' if self.options.compile_all else "-p %s" % package values = settings.get('rtupdates') if not values: @@ -202,7 +202,7 @@ def save_rtupdate(self): fp = open(fn, 'w') fp.write(data) fp.close() - chmod(fn, 0755) + chmod(fn, 0o755) def save(self): self.save_substvars() From 2de6b917379a422e0b5a036ce9f262847f35a712 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Mon, 18 May 2020 16:28:30 +0100 Subject: [PATCH 4/5] Switch to pybuild buildsystem pybuild is a replacement for `python_distutils`, which requires python2. pybuild is part of `dh-python`, so it will be available wherever our build deps are installed. See also: * https://wiki.debian.org/Python/LibraryStyleGuide#Overview * https://wiki.debian.org/Python/Pybuild --- debian/rules | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/debian/rules b/debian/rules index a9fbe0f..e245850 100755 --- a/debian/rules +++ b/debian/rules @@ -6,7 +6,7 @@ ifeq (,$(findstring nodoc, $(DEB_BUILD_OPTIONS))) endif %: - dh $@ --with python2 $(DH_ARGS) + dh $@ --buildsystem=pybuild --with python2 $(DH_ARGS) override_dh_auto_clean: rm -rf doc/_build From c73d7dab6f1f2346cebc751ea8f90406c614833f Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Mon, 18 May 2020 17:07:07 +0100 Subject: [PATCH 5/5] Build debian package with python3 Switch to using python 3 for building the dh-virtualenv deb. Python2 is dead, and the existing build fails for Ubuntu 20.04 and debian testing (specifically, python-mock and python-sphinx no longer exist). --- Dockerfile | 4 ++-- debian/control | 8 ++++---- debian/rules | 4 ++-- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/Dockerfile b/Dockerfile index 5250c91..db932c3 100644 --- a/Dockerfile +++ b/Dockerfile @@ -11,8 +11,8 @@ RUN apt-get update -qq -o Acquire::Languages=none \ && env DEBIAN_FRONTEND=noninteractive apt-get install \ -yqq --no-install-recommends -o Dpkg::Options::=--force-unsafe-io \ build-essential debhelper devscripts equivs lsb-release libparse-debianchangelog-perl \ - python python-setuptools python-pip python-dev \ - python-sphinx python-mock dh-exec dh-python python-sphinx-rtd-theme \ + python3 python3-setuptools python3-pip python3-dev \ + python3-sphinx python3-mock dh-exec dh-python python3-sphinx-rtd-theme \ && apt-get clean && rm -rf "/var/lib/apt/lists"/* WORKDIR /dpkg-build COPY ./ ./ diff --git a/debian/control b/debian/control index c28beb7..c7c6dfb 100644 --- a/debian/control +++ b/debian/control @@ -2,8 +2,8 @@ Source: dh-virtualenv Section: python Priority: optional Maintainer: Jyrki Pulliainen -Build-Depends: debhelper (>= 9), python(>= 2.6.6-3~), - python-setuptools, python-sphinx, python-mock, dh-exec, dh-python, +Build-Depends: debhelper (>= 9), python3, + python3-setuptools, python3-sphinx, python3-mock, dh-exec, dh-python, # python-sphinx-rtd-theme doesn't exist in distributions # predating Debian Jessie and Ubuntu Xenial. On these legacy # systems: @@ -11,13 +11,13 @@ Build-Depends: debhelper (>= 9), python(>= 2.6.6-3~), # 2. pip install sphinx_rtd_theme. # 3. Proceed with your build process (typically dpkg-build). # See https://github.com/spotify/dh-virtualenv/issues/230 - python-sphinx-rtd-theme + python3-sphinx-rtd-theme Standards-Version: 4.2.1 Homepage: http://www.github.com/spotify/dh-virtualenv Package: dh-virtualenv Architecture: all -Depends: ${python:Depends}, ${misc:Depends}, ${sphinxdoc:Depends}, +Depends: ${python3:Depends}, ${misc:Depends}, ${sphinxdoc:Depends}, virtualenv | python-virtualenv (>= 1.7) | python3-venv Description: wrap and build python packages using virtualenv This package provides a dh sequencer that helps you to deploy your diff --git a/debian/rules b/debian/rules index e245850..1ea2faf 100755 --- a/debian/rules +++ b/debian/rules @@ -6,7 +6,7 @@ ifeq (,$(findstring nodoc, $(DEB_BUILD_OPTIONS))) endif %: - dh $@ --buildsystem=pybuild --with python2 $(DH_ARGS) + dh $@ --buildsystem=pybuild --with python3 $(DH_ARGS) override_dh_auto_clean: rm -rf doc/_build @@ -20,7 +20,7 @@ override_dh_auto_build: ifeq (,$(findstring nodoc, $(DEB_BUILD_OPTIONS))) override_dh_installdocs: - python setup.py build_sphinx + python3 setup.py build_sphinx dh_installdocs doc/_build/html # Make Sphinxdocs build on Jessie and Xenial