diff --git a/Doc/library/multiprocessing.rst b/Doc/library/multiprocessing.rst index a77815918d2b0b..8ac87393045fdc 100644 --- a/Doc/library/multiprocessing.rst +++ b/Doc/library/multiprocessing.rst @@ -943,8 +943,8 @@ Miscellaneous .. function:: freeze_support() Add support for when a program which uses :mod:`multiprocessing` has been - frozen to produce a Windows executable. (Has been tested with **py2exe**, - **PyInstaller** and **cx_Freeze**.) + frozen to produce an application executable. (Has been tested with + **py2exe**, **PyInstaller** and **cx_Freeze**.) One needs to call this function straight after the ``if __name__ == '__main__'`` line of the main module. For example:: @@ -959,12 +959,17 @@ Miscellaneous Process(target=f).start() If the ``freeze_support()`` line is omitted then trying to run the frozen - executable will raise :exc:`RuntimeError`. + executable will cause errors (e.g., :exc:`RuntimeError`). It is needed + when using the ``'spawn'`` and ``'forkserver'`` start methods. - Calling ``freeze_support()`` has no effect when invoked on any operating - system other than Windows. In addition, if the module is being run - normally by the Python interpreter on Windows (the program has not been - frozen), then ``freeze_support()`` has no effect. + + ``freeze_support()`` has no effect when invoked in a module that is being + run normally by the Python interpreter (i.e., instead of by a frozen + executable). + +.. versionchanged:: 3.7 + Now supported on Unix (for the ``'spawn'`` and ``'forkserver'`` start + methods) .. function:: get_all_start_methods() diff --git a/Lib/multiprocessing/context.py b/Lib/multiprocessing/context.py index 871746b1a047b3..9309e8f7e1a65a 100644 --- a/Lib/multiprocessing/context.py +++ b/Lib/multiprocessing/context.py @@ -144,7 +144,7 @@ def freeze_support(self): '''Check whether this is a fake forked process in a frozen executable. If so then run code specified by commandline and exit. ''' - if sys.platform == 'win32' and getattr(sys, 'frozen', False): + if getattr(sys, 'frozen', False): from .spawn import freeze_support freeze_support() diff --git a/Lib/multiprocessing/forkserver.py b/Lib/multiprocessing/forkserver.py index 040b46e66a0330..57ea6f584602c5 100644 --- a/Lib/multiprocessing/forkserver.py +++ b/Lib/multiprocessing/forkserver.py @@ -127,7 +127,10 @@ def ensure_running(self): cmd %= (listener.fileno(), alive_r, self._preload_modules, data) exe = spawn.get_executable() - args = [exe] + util._args_from_interpreter_flags() + args = [exe] + if getattr(sys, 'frozen', False): + args.append('--multiprocessing-forkserver') + args += util._args_from_interpreter_flags() args += ['-c', cmd] pid = util.spawnv_passfds(exe, args, fds_to_pass) except: diff --git a/Lib/multiprocessing/semaphore_tracker.py b/Lib/multiprocessing/semaphore_tracker.py index 82833bcf861a49..d842be4f64a819 100644 --- a/Lib/multiprocessing/semaphore_tracker.py +++ b/Lib/multiprocessing/semaphore_tracker.py @@ -75,7 +75,10 @@ def ensure_running(self): fds_to_pass.append(r) # process will out live us, so no need to wait on pid exe = spawn.get_executable() - args = [exe] + util._args_from_interpreter_flags() + args = [exe] + if getattr(sys, 'frozen', False): + args.append('--multiprocessing-semaphore-tracker') + args += util._args_from_interpreter_flags() args += ['-c', cmd % r] # bpo-33613: Register a signal mask that will block the signals. # This signal mask will be inherited by the child that is going diff --git a/Lib/multiprocessing/spawn.py b/Lib/multiprocessing/spawn.py index 73aa69471f29c3..0bc76ba373a867 100644 --- a/Lib/multiprocessing/spawn.py +++ b/Lib/multiprocessing/spawn.py @@ -8,6 +8,7 @@ # Licensed to PSF under a Contributor Agreement. # +import ast import os import sys import runpy @@ -49,6 +50,7 @@ def get_executable(): # # + def is_forking(argv): ''' Return whether commandline indicates we are forking @@ -59,19 +61,85 @@ def is_forking(argv): return False +def get_forking_args(argv): + ''' + If the command line indicated we are forking, return (args, kwargs) + suitable for passing to spawn_main. Otherwise return None. + ''' + if not is_forking(argv): + return None + + args = [] + kwds = {} + for arg in argv[2:]: + name, value = arg.split('=') + if value == 'None': + kwds[name] = None + else: + kwds[name] = int(value) + + return args, kwds + + +def get_semaphore_tracker_args(argv): + ''' + If the command line indicates we are running the semaphore tracker, + return (args, kwargs) suitable for passing to semaphore_tracker.main. + Otherwise return None. + ''' + if len(argv) < 2 or argv[1] != '--multiprocessing-semaphore-tracker': + return None + + # command ends with main(fd) - extract the fd. + r = int(argv[-1].rsplit('(')[1].split(')')[0]) + + args = [r] + kwds = {} + return args, kwds + + +def get_forkserver_args(argv): + ''' + If the command line indicates we are running the forkserver, return + (args, kwargs) suitable for passing to forkserver.main. Otherwise return + None. + ''' + if len(argv) < 2 or argv[1] != '--multiprocessing-forkserver': + return None + + # command ends with main(listener_fd, alive_r, preload, **kwds) - extract + # the args and kwargs. listener_fd and alive_r are integers. + # preload is a list. The kwds map strings to lists. + parsed = ast.parse(argv[-1]) + args = [ast.literal_eval(parsed.body[1].value.args[i]) for i in range(3)] + kwds = ast.literal_eval(parsed.body[1].value.keywords[0].value) + return args, kwds + + def freeze_support(): ''' - Run code for process object if this in not the main process + Run code for process object if this in not the main process. ''' - if is_forking(sys.argv): - kwds = {} - for arg in sys.argv[2:]: - name, value = arg.split('=') - if value == 'None': - kwds[name] = None - else: - kwds[name] = int(value) - spawn_main(**kwds) + argv = sys.argv + + forking_args = get_forking_args(argv) + if forking_args is not None: + args, kwds = forking_args + spawn_main(*args, **kwds) + sys.exit() + + semaphore_tracker_args = get_semaphore_tracker_args(argv) + if semaphore_tracker_args is not None: + from multiprocessing.semaphore_tracker import main + args, kwds = semaphore_tracker_args + main(*args, **kwds) + sys.exit() + + forkserver_args = get_forkserver_args(argv) + if get_forkserver_args(sys.argv): + from multiprocessing.forkserver import main + args, kwds = forkserver_args + main(*args, **kwds) sys.exit() diff --git a/Lib/test/_test_multiprocessing.py b/Lib/test/_test_multiprocessing.py index 7341131231a4f0..9ace4b9865cf7b 100644 --- a/Lib/test/_test_multiprocessing.py +++ b/Lib/test/_test_multiprocessing.py @@ -4665,6 +4665,76 @@ def test_empty(self): proc.join() +# +# Issue 32146: freeze_support for fork, spawn, and forkserver start methods +# + +class TestFreezeSupport(unittest.TestCase): + def setUp(self): + import multiprocessing.spawn + self.module = multiprocessing.spawn + + def test_get_forking_args(self): + # Too few args + self.assertIsNone(self.module.get_forking_args(['./embed'])) + + # Wrong second argument + self.assertIsNone( + self.module.get_forking_args(['./embed', '-h']) + ) + + # All correct + args, kwds = self.module.get_forking_args( + ['./embed', '--multiprocessing-fork', 'pipe_handle=6', 'key=None'] + ) + self.assertEqual(args, []) + self.assertEqual(kwds, {'pipe_handle': 6, 'key': None}) + + def test_get_semaphore_tracker_args(self): + # Too few args + self.assertIsNone(self.module.get_semaphore_tracker_args(['.embed'])) + + # Wrong second argument + self.assertIsNone(self.module.get_semaphore_tracker_args( + ['./embed', '-h']) + ) + + # All correct + argv = [ + './embed', + '--multiprocessing-semaphore-tracker', + 'from multiprocessing.semaphore_tracker import main;main(5)' + ] + args, kwds = self.module.get_semaphore_tracker_args(argv) + self.assertEqual(args, [5]) + self.assertEqual(kwds, {}) + + def test_get_forkserver_args(self): + # Too few args + self.assertFalse(self.module.get_forkserver_args(['./python-embed'])) + + # Wrong second argument + self.assertFalse( + self.module.get_forkserver_args(['./python-embed', '-h']) + ) + + # All correct + argv = [ + './embed', + '--multiprocessing-forkserver', + ( + "from multiprocessing.forkserver import main; " + "main(8, 9, ['__main__', 'other'], " + "**{'sys_path': ['/embed/lib', '/embed/lib/library.zip']})" + ) + ] + args, kwds = self.module.get_forkserver_args(argv) + self.assertEqual(args, [8, 9, ['__main__', 'other']]) + self.assertEqual( + kwds, {'sys_path': ['/embed/lib', '/embed/lib/library.zip']} + ) + + class TestPoolNotLeakOnFailure(unittest.TestCase): def test_release_unused_processes(self): @@ -4706,12 +4776,13 @@ def is_alive(self): any(process.is_alive() for process in forked_processes)) - class MiscTestCase(unittest.TestCase): def test__all__(self): # Just make sure names in blacklist are excluded support.check__all__(self, multiprocessing, extra=multiprocessing.__all__, blacklist=['SUBDEBUG', 'SUBWARNING']) + + # # Mixins # diff --git a/Misc/NEWS.d/next/Library/2018-01-14-16-34-26.bpo-32146.6pFDbn.rst b/Misc/NEWS.d/next/Library/2018-01-14-16-34-26.bpo-32146.6pFDbn.rst new file mode 100644 index 00000000000000..5c5636bf41ffe4 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2018-01-14-16-34-26.bpo-32146.6pFDbn.rst @@ -0,0 +1,2 @@ +``multiprocessing.freeze_support`` now works on non-Windows platforms as +well.