From e1a3a8fe4734f20536b9c858ea4c82e00047e1ae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Sun, 16 Aug 2026 10:23:02 +0200 Subject: [PATCH 1/4] gh-155717: use `spawn` as the default start method for read-only filesystems (GH-155827) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The "forkserver" start method (the default start method on non-Windows systems) requires the ability to write temporary files, which is not possible if TMPDIR is read-only (e.g., k8s containers mounted with `readOnlyRootFilesystem=True`). On such filesystems, the default start method changes from "forkserver" to "spawn". (cherry picked from commit c1447994a42004a81a8caca026218daf69d451c8) Co-authored-by: Bénédikt Tran <10796600+picnixz@users.noreply.github.com> --- Lib/multiprocessing/context.py | 8 ++++- Lib/multiprocessing/util.py | 31 ++++++++++++++-- Lib/test/_test_multiprocessing.py | 35 +++++++++++++++++++ ...-08-15-09-47-23.gh-issue-155717.jWFLR2.rst | 3 ++ 4 files changed, 73 insertions(+), 4 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2026-08-15-09-47-23.gh-issue-155717.jWFLR2.rst diff --git a/Lib/multiprocessing/context.py b/Lib/multiprocessing/context.py index 5fa6d7e48611f03..79c46af02425c64 100644 --- a/Lib/multiprocessing/context.py +++ b/Lib/multiprocessing/context.py @@ -4,6 +4,7 @@ from . import process from . import reduction +from . import util __all__ = () @@ -328,7 +329,12 @@ def _check_available(self): # bpo-33725: running arbitrary code after fork() is no longer reliable # on macOS since macOS 10.14 (Mojave). Use spawn by default instead. # gh-84559: We changed everyones default to a thread safeish one in 3.14. - if reduction.HAVE_SEND_HANDLE and sys.platform != 'darwin': + if ( + reduction.HAVE_SEND_HANDLE + and sys.platform != 'darwin' + # gh-155717: forkserver requires to write temporary files + and util._has_writeable_tempdir() + ): _default_context = DefaultContext(_concrete_contexts['forkserver']) else: _default_context = DefaultContext(_concrete_contexts['spawn']) diff --git a/Lib/multiprocessing/util.py b/Lib/multiprocessing/util.py index 549fb07c27549e0..cf7e0b2990598b7 100644 --- a/Lib/multiprocessing/util.py +++ b/Lib/multiprocessing/util.py @@ -10,6 +10,7 @@ import os import itertools import sys +import tempfile import weakref import atexit import threading # we want threading to install it's @@ -143,6 +144,7 @@ def is_abstract_socket_namespace(address): # On Windows platforms, we do not create AF_UNIX sockets. _SUN_PATH_MAX = None if os.name == 'nt' else 92 + def _remove_temp_dir(rmtree, tempdir): rmtree(tempdir) @@ -152,7 +154,8 @@ def _remove_temp_dir(rmtree, tempdir): if current_process is not None: current_process._config['tempdir'] = None -def _get_base_temp_dir(tempfile): + +def _get_base_temp_dir(): """Get a temporary directory where socket files will be created. To prevent additional imports, pass a pre-imported 'tempfile' module. @@ -208,12 +211,13 @@ def _get_base_temp_dir(tempfile): assert len(base_system_tempdir) + 14 + 14 < _SUN_PATH_MAX return base_system_tempdir + def get_temp_dir(): # get name of a temp directory which will be automatically cleaned up tempdir = process.current_process()._config.get('tempdir') if tempdir is None: - import shutil, tempfile - base_tempdir = _get_base_temp_dir(tempfile) + import shutil + base_tempdir = _get_base_temp_dir() tempdir = tempfile.mkdtemp(prefix='pymp-', dir=base_tempdir) info('created temp directory %s', tempdir) # keep a strong reference to shutil.rmtree(), since the finalizer @@ -223,6 +227,27 @@ def get_temp_dir(): process.current_process()._config['tempdir'] = tempdir return tempdir + +def _has_writeable_tempdir(): + # 'forkserver' requires writeable temporary files. This function is + # called to determine the default context's start method. + # + # See: https://github.com/python/cpython/issues/155717. + + path = _get_base_temp_dir() + if path is None: + return False + + # os.access() is advisory and racy. It can lie on read-only filesystems, + # NFS/network mounts, containers, and immutable-flag files, so we simply + # try to create a file to check if this works and delete it otherwise. + try: + with tempfile.NamedTemporaryFile(dir=path): + return True + except OSError: + return False + + # # Support for reinitialization of objects when bootstrapping a child process # diff --git a/Lib/test/_test_multiprocessing.py b/Lib/test/_test_multiprocessing.py index fa970b80efd9c75..09d2e8beeaeec0b 100644 --- a/Lib/test/_test_multiprocessing.py +++ b/Lib/test/_test_multiprocessing.py @@ -26,6 +26,7 @@ import struct import tempfile import operator +import pathlib import pickle import weakref import warnings @@ -6051,6 +6052,40 @@ def test_nested_startmethod(self): # there is no synchronization in the test. self.assertSetEqual(set(results), set([2, 1])) + @unittest.skipIf(os.name == "nt", "requires POSIX") + @support.subTests("mode", [ + os.R_OK, # read-only directory + os.R_OK | os.X_OK, # read-only directory + os.W_OK # write-only directory _without_ permissions for creating files + ]) + def test_forkserver_requires_writeable_tempdir(self, mode): + # Regression test to ensure that the defualt start method is + # not 'forkserver' when the temporary directory is not writeable. + # + # See https://github.com/python/cpython/issues/155717. + + cmd = '''if 1: + import os, tempfile + # We fake the read-onlyiness of /tmp (which is a fallback when + # the user-defined TMPDIR is not acceptable) by hardcoding the + # temporary directory for this specific test. + tempfile.tempdir = os.environ["TMPDIR"] + + # Imported after patching 'tempfile' so that the default start + # method is deduced according to the permissions of TMPDIR. + import multiprocessing + if __name__ == "__main__": + print(multiprocessing.get_start_method()) + ''' + + with support.os_helper.temp_dir() as root: + TMPDIR = pathlib.Path(root, "TMPDIR") + TMPDIR.mkdir(mode=mode) + file = pathlib.Path(TMPDIR, "file") + self.assertRaises(OSError, file.touch) + _, out, err = script_helper.assert_python_ok('-c', cmd, TMPDIR=TMPDIR) + self.assertEqual(out.decode().strip(), "spawn") + @unittest.skipIf(sys.platform == "win32", "test semantics don't make sense on Windows") diff --git a/Misc/NEWS.d/next/Library/2026-08-15-09-47-23.gh-issue-155717.jWFLR2.rst b/Misc/NEWS.d/next/Library/2026-08-15-09-47-23.gh-issue-155717.jWFLR2.rst new file mode 100644 index 000000000000000..0994dc8d03051f4 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-08-15-09-47-23.gh-issue-155717.jWFLR2.rst @@ -0,0 +1,3 @@ +:mod:`multiprocessing`'s default start method on systems with non-writeable +tempfile filesystem is now :ref:`"spawn" ` +instead of ``"forkserver"``. Patch by Bénédikt Tran. From bab0322ecb0cc722afa633736ff920da32211d8e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Sun, 16 Aug 2026 11:43:18 +0200 Subject: [PATCH 2/4] Update Lib/test/_test_multiprocessing.py --- Lib/test/_test_multiprocessing.py | 1 + 1 file changed, 1 insertion(+) diff --git a/Lib/test/_test_multiprocessing.py b/Lib/test/_test_multiprocessing.py index 09d2e8beeaeec0b..71384d4d53b43fb 100644 --- a/Lib/test/_test_multiprocessing.py +++ b/Lib/test/_test_multiprocessing.py @@ -6053,6 +6053,7 @@ def test_nested_startmethod(self): self.assertSetEqual(set(results), set([2, 1])) @unittest.skipIf(os.name == "nt", "requires POSIX") + @support.requires_non_root_user @support.subTests("mode", [ os.R_OK, # read-only directory os.R_OK | os.X_OK, # read-only directory From d146c73bde595bcbf1db55762e831b9e35c295a9 Mon Sep 17 00:00:00 2001 From: "Gregory P. Smith" <68491+gpshead@users.noreply.github.com> Date: Mon, 12 Oct 2026 15:13:51 -0700 Subject: [PATCH 3/4] [3.14] gh-155717: Determine multiprocessing's default start method lazily (GH-159254) gh-155827 made `import multiprocessing` check whether the temporary directory is writeable in order to choose between "forkserver" and "spawn" as the default start method. That check ran at import time and pulled in `multiprocessing.util`, `subprocess`, `tempfile`, `shutil` and `random`, noticeably slowing down the import for every program, including ones that never start a process and every spawned child, the fork server and the resource tracker. The default start method is now determined the first time it is needed instead: on the first `get_context()`, `get_start_method()`, `get_all_start_methods()` or default `Process` start. `multiprocessing.util` goes back to importing `tempfile` only where it is used. Also include the fix from gh-158410 by Vyron Vasileiadis, along with a test adapted from that PR: when `tempfile` finds no usable temporary directory at all, the check now reports that instead of raising `FileNotFoundError`, so that "spawn" is chosen. The regression was reported, and the lazy approach suggested, by Leandro Damascena in https://github.com/python/cpython/issues/155717#issuecomment-6121012448 (cherry picked from commit 48ddda1987a17fa96d74409b9eccb7379f61816f) Co-authored-by: Vyron Vasileiadis --- Doc/library/multiprocessing.rst | 12 ++++++- Lib/multiprocessing/context.py | 48 +++++++++++++++++--------- Lib/multiprocessing/util.py | 17 +++++++--- Lib/test/_test_multiprocessing.py | 56 +++++++++++++++++++++++++++++++ 4 files changed, 111 insertions(+), 22 deletions(-) diff --git a/Doc/library/multiprocessing.rst b/Doc/library/multiprocessing.rst index 0ecaeaaa47ea61f..860707e2f6c67a3 100644 --- a/Doc/library/multiprocessing.rst +++ b/Doc/library/multiprocessing.rst @@ -160,11 +160,21 @@ to start a process. These *start methods* are No unnecessary resources are inherited. Available on POSIX platforms which support passing file descriptors over - Unix pipes such as Linux. The default on those. + Unix pipes such as Linux. The default on those, unless there is no + writable temporary directory (see :func:`tempfile.gettempdir`), which + the fork server requires; *spawn* is the default in that case. .. versionchanged:: 3.14 This became the default start method on POSIX platforms. + .. versionchanged:: next + *spawn* is now the default start method when there is no writable + temporary directory, for example on a read-only file system. + Previously *forkserver* remained the default there and starting a + process failed with an :exc:`OSError`. The default is determined + when it is first needed rather than when :mod:`!multiprocessing` + is imported. See :gh:`155717`. + .. versionchanged:: 3.4 *spawn* added on all POSIX platforms, and *forkserver* added for some POSIX platforms. diff --git a/Lib/multiprocessing/context.py b/Lib/multiprocessing/context.py index 79c46af02425c64..7226be77446aa1a 100644 --- a/Lib/multiprocessing/context.py +++ b/Lib/multiprocessing/context.py @@ -1,10 +1,10 @@ +import functools import os import sys import threading from . import process from . import reduction -from . import util __all__ = () @@ -237,10 +237,19 @@ def _after_fork(): class DefaultContext(BaseContext): Process = Process - def __init__(self, context): - self._default_context = context + def __init__(self, context=None): + if context is not None: + self._default_context = context self._actual_context = None + # The platform default is only determined when first needed, as doing + # so can be costly; importing multiprocessing must stay cheap. + # A cached_property rather than a property so that the attribute can + # still be assigned, deleted and patched like a plain one. + @functools.cached_property + def _default_context(self): + return _get_platform_default_context() + def get_context(self, method=None): if method is None: if self._actual_context is None: @@ -326,18 +335,22 @@ def _check_available(self): 'spawn': SpawnContext(), 'forkserver': ForkServerContext(), } - # bpo-33725: running arbitrary code after fork() is no longer reliable - # on macOS since macOS 10.14 (Mojave). Use spawn by default instead. - # gh-84559: We changed everyones default to a thread safeish one in 3.14. - if ( - reduction.HAVE_SEND_HANDLE - and sys.platform != 'darwin' - # gh-155717: forkserver requires to write temporary files - and util._has_writeable_tempdir() - ): - _default_context = DefaultContext(_concrete_contexts['forkserver']) - else: - _default_context = DefaultContext(_concrete_contexts['spawn']) + + def _get_platform_default_context(): + # bpo-33725: running arbitrary code after fork() is no longer reliable + # on macOS since macOS 10.14 (Mojave). Use spawn by default instead. + # gh-84559: We changed everyones default to a thread safeish one in 3.14. + if reduction.HAVE_SEND_HANDLE and sys.platform != 'darwin': + # gh-155717: forkserver requires to write temporary files. + # + # Imported here rather than at the top level as util is costly + # to import and is not otherwise needed to import this module. + from . import util + if util._has_writeable_tempdir(): + return _concrete_contexts['forkserver'] + return _concrete_contexts['spawn'] + + _default_context = DefaultContext() else: # Windows @@ -360,7 +373,10 @@ class SpawnContext(BaseContext): _concrete_contexts = { 'spawn': SpawnContext(), } - _default_context = DefaultContext(_concrete_contexts['spawn']) + def _get_platform_default_context(): + return _concrete_contexts['spawn'] + + _default_context = DefaultContext() # # Force the start method diff --git a/Lib/multiprocessing/util.py b/Lib/multiprocessing/util.py index cf7e0b2990598b7..a84546bbb8fe1e3 100644 --- a/Lib/multiprocessing/util.py +++ b/Lib/multiprocessing/util.py @@ -10,7 +10,6 @@ import os import itertools import sys -import tempfile import weakref import atexit import threading # we want threading to install it's @@ -155,7 +154,7 @@ def _remove_temp_dir(rmtree, tempdir): current_process._config['tempdir'] = None -def _get_base_temp_dir(): +def _get_base_temp_dir(tempfile): """Get a temporary directory where socket files will be created. To prevent additional imports, pass a pre-imported 'tempfile' module. @@ -216,8 +215,9 @@ def get_temp_dir(): # get name of a temp directory which will be automatically cleaned up tempdir = process.current_process()._config.get('tempdir') if tempdir is None: - import shutil - base_tempdir = _get_base_temp_dir() + # Deferred until first use to keep importing this module cheap. + import shutil, tempfile + base_tempdir = _get_base_temp_dir(tempfile) tempdir = tempfile.mkdtemp(prefix='pymp-', dir=base_tempdir) info('created temp directory %s', tempdir) # keep a strong reference to shutil.rmtree(), since the finalizer @@ -234,7 +234,14 @@ def _has_writeable_tempdir(): # # See: https://github.com/python/cpython/issues/155717. - path = _get_base_temp_dir() + # Deferred until first use to keep importing this module cheap. + import tempfile + + try: + path = _get_base_temp_dir(tempfile) + except OSError: + # No usable temporary directory at all. + return False if path is None: return False diff --git a/Lib/test/_test_multiprocessing.py b/Lib/test/_test_multiprocessing.py index 71384d4d53b43fb..f30d9f779dc3492 100644 --- a/Lib/test/_test_multiprocessing.py +++ b/Lib/test/_test_multiprocessing.py @@ -6087,6 +6087,62 @@ def test_forkserver_requires_writeable_tempdir(self, mode): _, out, err = script_helper.assert_python_ok('-c', cmd, TMPDIR=TMPDIR) self.assertEqual(out.decode().strip(), "spawn") + @unittest.skipIf(os.name == "nt", "requires POSIX") + @support.requires_non_root_user + def test_default_start_method_without_usable_tempdir(self): + # Neither importing multiprocessing nor determining the default + # start method may fail when tempfile finds no usable temporary + # directory at all, and the default start method must be 'spawn'. + # + # See https://github.com/python/cpython/issues/155717. + + cmd = '''if 1: + import os, tempfile + # Make TMPDIR the only candidate, so that tempfile.gettempdir() + # raises FileNotFoundError instead of falling back to /tmp. + tempfile._candidate_tempdir_list = lambda: [os.environ["TMPDIR"]] + try: + tempfile.gettempdir() + except FileNotFoundError: + pass + else: + raise AssertionError("TMPDIR is usable") + + import multiprocessing + if __name__ == "__main__": + print(multiprocessing.get_start_method(), + multiprocessing.get_all_start_methods()[0]) + ''' + + with support.os_helper.temp_dir() as root: + TMPDIR = pathlib.Path(root, "TMPDIR") + TMPDIR.mkdir(mode=os.R_OK | os.X_OK) + _, out, err = script_helper.assert_python_ok('-c', cmd, TMPDIR=TMPDIR) + self.assertEqual(out.decode().split(), ["spawn", "spawn"]) + + def test_import_does_not_determine_default_start_method(self): + # Determining the default start method requires checking that the + # temporary directory is writeable. That must not happen, nor may + # the modules it needs be loaded, when importing multiprocessing. + import_helper.ensure_lazy_imports( + "multiprocessing", {"tempfile", "shutil", "subprocess"}) + + cmd = '''if 1: + import sys + import multiprocessing + print(multiprocessing.get_start_method(allow_none=True), + "tempfile" in sys.modules, + multiprocessing.get_start_method(), + multiprocessing.get_all_start_methods()[0]) + ''' + # -S: site customizations could import tempfile themselves. + _, out, err = script_helper.assert_python_ok('-S', '-c', cmd) + unset, probed, method, first = out.decode().split() + self.assertEqual(unset, "None") + self.assertEqual(probed, "False") + self.assertEqual(method, first) + self.assertIn(method, multiprocessing.get_all_start_methods()) + @unittest.skipIf(sys.platform == "win32", "test semantics don't make sense on Windows") From f65fc7b9cae70625b0169d5c5c7875db35d3580e Mon Sep 17 00:00:00 2001 From: "Gregory P. Smith" Date: Mon, 12 Oct 2026 22:22:38 +0000 Subject: [PATCH 4/4] gh-155717: Reword the NEWS entry for the default start method change Say that the change applies when there is no writable temporary directory and that starting a process previously failed there. --- .../2026-08-15-09-47-23.gh-issue-155717.jWFLR2.rst | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/Misc/NEWS.d/next/Library/2026-08-15-09-47-23.gh-issue-155717.jWFLR2.rst b/Misc/NEWS.d/next/Library/2026-08-15-09-47-23.gh-issue-155717.jWFLR2.rst index 0994dc8d03051f4..671da00610e8247 100644 --- a/Misc/NEWS.d/next/Library/2026-08-15-09-47-23.gh-issue-155717.jWFLR2.rst +++ b/Misc/NEWS.d/next/Library/2026-08-15-09-47-23.gh-issue-155717.jWFLR2.rst @@ -1,3 +1,5 @@ -:mod:`multiprocessing`'s default start method on systems with non-writeable -tempfile filesystem is now :ref:`"spawn" ` -instead of ``"forkserver"``. Patch by Bénédikt Tran. +:mod:`multiprocessing`'s default start method is now +:ref:`"spawn" ` instead of ``"forkserver"`` +when there is no writable temporary directory, such as on a read-only file +system. Previously starting a process failed with an :exc:`OSError` there. +Patch by Bénédikt Tran.