Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 0 additions & 1 deletion Lib/test/lazy_imports_all_exclude.txt
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,5 @@ test_pyrepl
test_subprocess
test_symtable
test_tools
test_trace
test_type_annotations
test_unittest
75 changes: 73 additions & 2 deletions Lib/test/test_lazy_import/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -724,10 +724,17 @@ def test_non_package_lazily_imported(self):
assert_python_ok("-c", code)

def test_non_package_lazily_imported_as(self):
"""Doing a dotted lazy import as still works"""
"""A dotted lazy import as raises when the name is not a module."""
# gh-157757: the eager statement raises, so the lazy one raises too.
code = textwrap.dedent("""
lazy import math.pi as pi
pi

try:
pi
except ModuleNotFoundError:
pass
else:
raise AssertionError("ModuleNotFoundError was not raised")
""")
assert_python_ok("-c", code)

Expand Down Expand Up @@ -1170,6 +1177,22 @@ def test_accessing_one_name_leaves_others_as_proxies(self):
self.assertEqual(result.returncode, 0, f"stdout: {result.stdout}, stderr: {result.stderr}")
self.assertIn("OK", result.stdout)

def test_accessing_one_name_imports_only_its_submodule(self):
"""Accessing one name should not import the other names' submodules."""
code = textwrap.dedent("""
import sys

lazy from test.test_lazy_import.data.pkg import b, bar, broken

# Importing bar prints, and importing broken raises.
b.foo()

assert "test.test_lazy_import.data.pkg.bar" not in sys.modules
assert "test.test_lazy_import.data.pkg.broken" not in sys.modules
""")
rc, out, err = assert_python_ok("-c", code)
self.assertEqual(out, b"")

def test_all_names_reified_after_all_accessed(self):
"""All names should be reified after each is accessed."""
code = textwrap.dedent("""
Expand Down Expand Up @@ -2209,6 +2232,54 @@ def test_import_after_variable_wins(self):
]
self.assertIs(module_same_name_var_order2.bar, bar_mod)

def test_lazy_import_as_wins_over_variable(self):
"""A dotted lazy import as imports the submodule the variable hides."""
# gh-157757: importing pkg.b rebinds pkg.b from the variable to the
# module, eagerly and lazily alike.
code = textwrap.dedent("""
import sys
import test.test_lazy_import.data.pkg as pkg
pkg.b = "hides the b submodule"

lazy import test.test_lazy_import.data.pkg.b as b
lazy import test.test_lazy_import.data.metasyntactic.foo.bar as bar

assert b is sys.modules["test.test_lazy_import.data.pkg.b"], b
assert bar is sys.modules[
"test.test_lazy_import.data.metasyntactic.foo.bar"], bar
""")
assert_python_ok("-c", code)

def test_dotted_as_of_loaded_module(self):
"""A dotted lazy import as binds the module, not a same-named attribute."""
# importlib.metadata is already loaded and has a `metadata` attribute.
code = textwrap.dedent("""
import importlib.metadata
import importlib.metadata as eager

lazy import importlib.metadata as lazily

assert lazily is eager, lazily
""")
assert_python_ok("-c", code)

def test_dotted_as_replays_lookups_on_custom_placeholder(self):
"""A dotted lazy import as looks up its names on what the hook returned."""
code = textwrap.dedent("""
import builtins
import xml.dom

# In a list, so the hook reading it does not resolve it.
placeholder = [__lazy_import__("xml")]
default = builtins.__lazy_import__
builtins.__lazy_import__ = lambda *args: placeholder[0]
lazy import fake.dom as dom
builtins.__lazy_import__ = default

assert dom is xml.dom, dom
""")
assert_python_ok("-c", code)


class DeletedModuleReimportTests(unittest.TestCase):
"""Tests for reimporting after module deletion from sys.modules."""
Expand Down
2 changes: 2 additions & 0 deletions Lib/test/test_lazy_import/data/pkg/broken.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
# Submodule that raises an error during import
raise ValueError("This module always fails to import")
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
Fix a lazy ``import a.b as c`` reading ``b`` off ``a`` instead of importing
the module ``a.b``. It now binds the submodule, and raises
:exc:`ModuleNotFoundError` when no module backs the name.
43 changes: 35 additions & 8 deletions Objects/lazyimportobject.c
Original file line number Diff line number Diff line change
Expand Up @@ -14,8 +14,8 @@ PyObject *
_PyLazyImport_New(_PyInterpreterFrame *frame, PyObject *builtins, PyObject *name, PyObject *fromlist)
{
PyLazyImportObject *m;
if (!name || !PyUnicode_Check(name)) {
PyErr_SetString(PyExc_TypeError, "expected str for name");
if (!name || !(PyUnicode_Check(name) || PyLazyImport_CheckExact(name))) {
PyErr_SetString(PyExc_TypeError, "expected str or lazy_import for name");
return NULL;
}
if (fromlist == Py_None || fromlist == NULL) {
Expand Down Expand Up @@ -104,16 +104,43 @@ lazy_import_getattro(PyObject *op, PyObject *name)
return value;
}

// The dotted name of the object that resolving the placeholder returns.
static PyObject *
lazy_import_path(PyLazyImportObject *m)
{
if (PyLazyImport_CheckExact(m->lz_from)) {
PyObject *base = lazy_import_path((PyLazyImportObject *)m->lz_from);
if (base == NULL) {
return NULL;
}
PyObject *res = PyUnicode_FromFormat("%U.%U", base, m->lz_attr);
Py_DECREF(base);
return res;
}
if (m->lz_attr != NULL) {
return Py_NewRef(m->lz_from);
}
// __import__("a.b") returns the top-level package `a`.
Py_ssize_t dot = PyUnicode_FindChar(
m->lz_from, '.', 0, PyUnicode_GET_LENGTH(m->lz_from), 1
);
if (dot == -2) {
return NULL;
}
if (dot < 0) {
return Py_NewRef(m->lz_from);
}
return PyUnicode_Substring(m->lz_from, 0, dot);
}

static PyObject *
lazy_import_name(PyLazyImportObject *m)
{
if (PyLazyImport_CheckExact(m->lz_from)) {
return lazy_import_path(m);
}
if (m->lz_attr != NULL) {
if (PyUnicode_Check(m->lz_attr)) {
return PyUnicode_FromFormat("%U.%U", m->lz_from, m->lz_attr);
}
else {
return PyUnicode_FromFormat("%U...", m->lz_from);
}
return PyUnicode_FromFormat("%U...", m->lz_from);
}
return Py_NewRef(m->lz_from);
}
Expand Down
36 changes: 7 additions & 29 deletions Python/ceval.c
Original file line number Diff line number Diff line change
Expand Up @@ -3333,7 +3333,12 @@ _PyEval_LazyImportFrom(PyThreadState *tstate, _PyInterpreterFrame *frame, PyObje
assert(PyUnicode_Check(name));
PyObject *ret;
PyLazyImportObject *d = (PyLazyImportObject *)v;
PyObject *mod = PyImport_GetModule(d->lz_from);
PyObject *mod = NULL;
// Only `from a import b` can take b off an already imported a;
// `import a.b as c` has to import a.b first.
if (d->lz_attr != NULL && PyTuple_Check(d->lz_attr)) {
mod = PyImport_GetModule(d->lz_from);
}
if (mod != NULL) {
// Check if the module already has the attribute, if so, resolve it
// eagerly.
Expand All @@ -3353,34 +3358,7 @@ _PyEval_LazyImportFrom(PyThreadState *tstate, _PyInterpreterFrame *frame, PyObje
Py_DECREF(mod);
}

if (d->lz_attr != NULL) {
if (PyUnicode_Check(d->lz_attr)) {
PyObject *from = PyUnicode_FromFormat(
"%U.%U", d->lz_from, d->lz_attr);
if (from == NULL) {
return NULL;
}
ret = _PyLazyImport_New(frame, d->lz_builtins, from, name);
Py_DECREF(from);
return ret;
}
}
else {
Py_ssize_t dot = PyUnicode_FindChar(
d->lz_from, '.', 0, PyUnicode_GET_LENGTH(d->lz_from), 1
);
if (dot >= 0) {
PyObject *from = PyUnicode_Substring(d->lz_from, 0, dot);
if (from == NULL) {
return NULL;
}
ret = _PyLazyImport_New(frame, d->lz_builtins, from, name);
Py_DECREF(from);
return ret;
}
}
ret = _PyLazyImport_New(frame, d->lz_builtins, d->lz_from, name);
return ret;
return _PyLazyImport_New(frame, d->lz_builtins, v, name);
}

#define CANNOT_CATCH_MSG "catching classes that do not inherit from "\
Expand Down
62 changes: 40 additions & 22 deletions Python/import.c
Original file line number Diff line number Diff line change
Expand Up @@ -3898,6 +3898,25 @@ _PyImport_ResolveName(PyThreadState *tstate, PyObject *name,
return resolve_name(tstate, name, globals, level);
}

// Look up, in order, the attributes recorded from the root placeholder to lz
// on the module the root's import returned.
static PyObject *
lazy_import_replay_from(PyThreadState *tstate, PyObject *mod,
PyLazyImportObject *lz)
{
if (!PyLazyImport_CheckExact(lz->lz_from)) {
return Py_NewRef(mod);
}
PyObject *from = lazy_import_replay_from(
tstate, mod, (PyLazyImportObject *)lz->lz_from);
if (from == NULL) {
return NULL;
}
PyObject *obj = _PyEval_ImportFrom(tstate, from, lz->lz_attr);
Py_DECREF(from);
return obj;
}

PyObject *
_PyImport_LoadLazyImportTstate(PyThreadState *tstate, PyObject *lazy_import)
{
Expand All @@ -3910,6 +3929,13 @@ _PyImport_LoadLazyImportTstate(PyThreadState *tstate, PyObject *lazy_import)
PyLazyImportObject *lz = (PyLazyImportObject *)lazy_import;
PyInterpreterState *interp = tstate->interp;

// Walk back to the placeholder IMPORT_NAME left, and the first lookup on it.
PyLazyImportObject *root = lz, *first = NULL;
while (PyLazyImport_CheckExact(root->lz_from)) {
first = root;
root = (PyLazyImportObject *)root->lz_from;
}

// Acquire the global import lock to serialize reification
_PyImport_AcquireLock(interp);

Expand Down Expand Up @@ -3946,7 +3972,7 @@ _PyImport_LoadLazyImportTstate(PyThreadState *tstate, PyObject *lazy_import)
return NULL;
}
PyErr_SetImportErrorSubclass(PyExc_ImportCycleError, errmsg,
lz->lz_from, NULL);
root->lz_from, NULL);
Py_DECREF(errmsg);
Py_DECREF(name);
_PyImport_ReleaseLock(interp);
Expand All @@ -3956,24 +3982,18 @@ _PyImport_LoadLazyImportTstate(PyThreadState *tstate, PyObject *lazy_import)
goto error;
}

if (lz->lz_attr != NULL) {
if (PyUnicode_Check(lz->lz_attr)) {
fromlist = PyTuple_New(1);
if (fromlist == NULL) {
goto error;
}
Py_INCREF(lz->lz_attr);
PyTuple_SET_ITEM(fromlist, 0, lz->lz_attr);
}
else {
Py_INCREF(lz->lz_attr);
fromlist = lz->lz_attr;
if (root->lz_attr != NULL) {
// `from a import b, c`: import only the name being resolved.
fromlist = first ? PyTuple_Pack(1, first->lz_attr)

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think an empty fromlist here should behave like None. __lazy_import__(name) defaults to fromlist=(), and __import__("a.b", fromlist=()) returns a, but once something is chained on it we pass (attr,) and replay on a.b instead. _PyEval_LazyImportFrom (the PyTuple_Check) and lazy_import_path have the same problem. Maybe check PyTuple_GET_SIZE(root->lz_attr) > 0 in all three places and add a small test with a hook that calls __lazy_import__(name) for a dotted name?

: Py_NewRef(root->lz_attr);
if (fromlist == NULL) {
goto error;
}
}

PyObject *globals = PyEval_GetGlobals();

if (PyMapping_GetOptionalItem(lz->lz_builtins, &_Py_ID(__import__),
if (PyMapping_GetOptionalItem(root->lz_builtins, &_Py_ID(__import__),
&import_func) < 0) {
goto error;
}
Expand All @@ -3983,19 +4003,17 @@ _PyImport_LoadLazyImportTstate(PyThreadState *tstate, PyObject *lazy_import)
}
obj = _PyEval_ImportNameWithImport(
tstate, import_func, globals, globals,
lz->lz_from, fromlist, _PyLong_GetZero()
root->lz_from, fromlist, _PyLong_GetZero()
);
if (obj == NULL) {
goto error;
}

if (lz->lz_attr != NULL && PyUnicode_Check(lz->lz_attr)) {
PyObject *from = obj;
obj = _PyEval_ImportFrom(tstate, from, lz->lz_attr);
Py_DECREF(from);
if (obj == NULL) {
goto error;
}
PyObject *from = obj;
obj = lazy_import_replay_from(tstate, from, lz);
Py_DECREF(from);
if (obj == NULL) {
goto error;
}

assert(!PyLazyImport_CheckExact(obj));
Expand Down
Loading