Skip to content

gh-110548: modernize venv EnvBuilder example #144745

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
Gh-Novel wants to merge 1 commit into
base: main
Choose a base branch
from
Open

gh-110548: modernize venv EnvBuilder example #144745

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
166 changes: 39 additions & 127 deletions Doc/library/venv.rst
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -543,142 +543,65 @@ An example of extending ``EnvBuilder``
--------------------------------------

The following script shows how to extend :class:`EnvBuilder` by implementing a
subclass which installs setuptools and pip into a created virtual environment::
subclass which pre-installs packages into a created virtual environment.

Since :mod:`!venv` bootstraps ``pip`` by default (via :mod:`ensurepip`), the
example uses ``pip`` to install additional packages after the environment is
created::

import os
import os.path
from subprocess import Popen, PIPE
import subprocess
import sys
from threading import Thread
from urllib.parse import urlsplit
from urllib.request import urlretrieve
import venv


class ExtendedEnvBuilder(venv.EnvBuilder):
"""
This builder installs setuptools and pip so that you can pip or
easy_install other packages into the created virtual environment.

:param nodist: If true, setuptools and pip are not installed into the
created virtual environment.
:param nopip: If true, pip is not installed into the created
virtual environment.
:param progress: If setuptools or pip are installed, the progress of the
installation can be monitored by passing a progress
callable. If specified, it is called with two
arguments: a string indicating some progress, and a
context indicating where the string is coming from.
The context argument can have one of three values:
'main', indicating that it is called from virtualize()
itself, and 'stdout' and 'stderr', which are obtained
by reading lines from the output streams of a subprocess
which is used to install the app.

If a callable is not specified, default progress
information is output to sys.stderr.
This builder installs additional packages into the created
virtual environment using pip.

:param packages: A list of packages to install after creating
the virtual environment.
:param verbose: If true, display the output from pip.
"""

def __init__(self, *args, **kwargs):
self.nodist = kwargs.pop('nodist', False)
self.nopip = kwargs.pop('nopip', False)
self.progress = kwargs.pop('progress', None)
self.packages = kwargs.pop('packages', [])
self.verbose = kwargs.pop('verbose', False)
super().__init__(*args, **kwargs)

def post_setup(self, context):
"""
Set up any packages which need to be pre-installed into the
virtual environment being created.
Install additional packages into the virtual environment.

:param context: The information for the virtual environment
creation request being processed.
"""
os.environ['VIRTUAL_ENV'] = context.env_dir
if not self.nodist:
self.install_setuptools(context)
# Can't install pip without setuptools
if not self.nopip and not self.nodist:
self.install_pip(context)

def reader(self, stream, context):
"""
Read lines from a subprocess' output stream and either pass to a progress
callable (if specified) or write progress information to sys.stderr.
"""
progress = self.progress
while True:
s = stream.readline()
if not s:
break
if progress is not None:
progress(s, context)
else:
if not self.verbose:
sys.stderr.write('.')
else:
sys.stderr.write(s.decode('utf-8'))
sys.stderr.flush()
stream.close()

def install_script(self, context, name, url):
_, _, path, _, _ = urlsplit(url)
fn = os.path.split(path)[-1]
binpath = context.bin_path
distpath = os.path.join(binpath, fn)
# Download script into the virtual environment's binaries folder
urlretrieve(url, distpath)
progress = self.progress
if self.verbose:
term = '\n'
else:
term = ''
if progress is not None:
progress('Installing %s ...%s' % (name, term), 'main')
else:
sys.stderr.write('Installing %s ...%s' % (name, term))
sys.stderr.flush()
# Install in the virtual environment
args = [context.env_exe, fn]
p = Popen(args, stdout=PIPE, stderr=PIPE, cwd=binpath)
t1 = Thread(target=self.reader, args=(p.stdout, 'stdout'))
t1.start()
t2 = Thread(target=self.reader, args=(p.stderr, 'stderr'))
t2.start()
p.wait()
t1.join()
t2.join()
if progress is not None:
progress('done.', 'main')
else:
sys.stderr.write('done.\n')
# Clean up - no longer needed
os.unlink(distpath)

def install_setuptools(self, context):
"""
Install setuptools in the virtual environment.
if self.packages:
self.install_packages(context)

:param context: The information for the virtual environment
creation request being processed.
"""
url = "https://bootstrap.pypa.io/ez_setup.py"
self.install_script(context, 'setuptools', url)
# clear up the setuptools archive which gets downloaded
pred = lambda o: o.startswith('setuptools-') and o.endswith('.tar.gz')
files = filter(pred, os.listdir(context.bin_path))
for f in files:
f = os.path.join(context.bin_path, f)
os.unlink(f)

def install_pip(self, context):
def install_packages(self, context):
"""
Install pip in the virtual environment.
Install the specified packages using pip.

:param context: The information for the virtual environment
creation request being processed.
"""
url = 'https://bootstrap.pypa.io/get-pip.py'
self.install_script(context, 'pip', url)
# Use the pip installed in the virtual environment
args = [context.env_exec_cmd, '-m', 'pip', 'install']
if not self.verbose:
args.append('-q')
args.extend(self.packages)
sys.stderr.write('Installing packages: %s ...\n'
% ', '.join(self.packages))
result = subprocess.run(args, capture_output=not self.verbose)
if result.returncode != 0:
sys.stderr.write('Package installation failed '
'(exit code %d).\n' % result.returncode)
if result.stderr:
sys.stderr.write(result.stderr.decode('utf-8'))
else:
sys.stderr.write('done.\n')


def main(args=None):
Expand All @@ -692,14 +615,9 @@ subclass which installs setuptools and pip into a created virtual environment::
parser.add_argument('dirs', metavar='ENV_DIR', nargs='+',
help='A directory in which to create the '
'virtual environment.')
parser.add_argument('--no-setuptools', default=False,
action='store_true', dest='nodist',
help="Don't install setuptools or pip in the "
"virtual environment.")
parser.add_argument('--no-pip', default=False,
action='store_true', dest='nopip',
help="Don't install pip in the virtual "
"environment.")
parser.add_argument('--packages', nargs='*', default=[],
help='Packages to install after environment '
'creation.')
parser.add_argument('--system-site-packages', default=False,
action='store_true', dest='system_site',
help='Give the virtual environment access to the '
Expand Down Expand Up @@ -728,17 +646,15 @@ subclass which installs setuptools and pip into a created virtual environment::
'in-place.')
parser.add_argument('--verbose', default=False, action='store_true',
dest='verbose', help='Display the output '
'from the scripts which '
'install setuptools and pip.')
'from pip install.')
options = parser.parse_args(args)
if options.upgrade and options.clear:
raise ValueError('you cannot supply --upgrade and --clear together.')
builder = ExtendedEnvBuilder(system_site_packages=options.system_site,
clear=options.clear,
symlinks=options.symlinks,
upgrade=options.upgrade,
nodist=options.nodist,
nopip=options.nopip,
packages=options.packages,
verbose=options.verbose)
for d in options.dirs:
builder.create(d)
Expand All @@ -751,7 +667,3 @@ subclass which installs setuptools and pip into a created virtual environment::
except Exception as e:
print('Error: %s' % e, file=sys.stderr)
sys.exit(rc)


This script is also available for download `online
<https://gist.github.com/vsajip/4673395>`_.
Loading