Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions Doc/library/types.rst
Original file line number Diff line number Diff line change
Expand Up @@ -358,6 +358,12 @@ Standard names are defined for the following types:

Reify the lazy import and return the "real" object being imported.

Once a lazy import has been reified successfully, repeated calls, and
any later access to a name still bound to the proxy, return the same
object, even if the module has since been removed from
:data:`sys.modules`. A reification that raises is not remembered, so
the next access retries the import.


.. class:: GetSetDescriptorType

Expand Down
116 changes: 116 additions & 0 deletions Lib/test/test_lazy_import/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@
import contextlib

from test import support
from test.support import threading_helper
from test.support.script_helper import assert_python_ok, assert_python_failure

try:
Expand Down Expand Up @@ -295,6 +296,121 @@ def test_lazy_import_type_attribute_error_message(self):
""")
assert_python_ok("-c", code)

# Setup for the two tests below. A repeat import is visible both in
# ``calls`` and as a different module object. They read the proxy out of
# the module dict because a plain global load would reify it first.
_COUNTING_IMPORT = """
import builtins
import types

real_import = builtins.__import__
calls = []
fail_import = False

lazy import target_module as target

def counting_import(name, *args, **kwargs):
if name != "target_module":
return real_import(name, *args, **kwargs)
calls.append(name)
if fail_import:
raise ImportError("no target_module for you")
return types.ModuleType(name)

builtins.__import__ = counting_import
"""

def _assert_counting_import_ok(self, body):
"""Run *body* in a subprocess with the counting __import__ installed."""
code = textwrap.dedent(self._COUNTING_IMPORT) + textwrap.dedent(body)
return assert_python_ok("-c", code)

@support.requires_subprocess()
def test_proxy_reifies_once_whichever_path_reaches_it(self):
"""Every reification of one proxy yields the first imported object."""
self._assert_counting_import_ok("""
def main():
proxy = globals()["target"]
resolved = proxy.resolve()
assert proxy.resolve() is resolved
# The global is still bound to the proxy, so loading it reifies.
assert target is resolved, (target, resolved)
# So does loading a copy of the proxy from another namespace.
namespace = {"__builtins__": builtins, "alias": proxy}
exec("alias", namespace)
assert namespace["alias"] is resolved
assert calls == ["target_module"], calls

main()
""")

@support.requires_subprocess()
def test_failed_resolve_is_not_cached(self):
"""A failed reification is retried rather than remembered."""
self._assert_counting_import_ok("""
def main():
global fail_import
fail_import = True
proxy = globals()["target"]
try:
proxy.resolve()
except ImportError:
pass
else:
assert False, 'ImportError is not raised'
fail_import = False
resolved = proxy.resolve()
assert proxy.resolve() is resolved
assert calls == ["target_module"] * 2, calls

main()
""")

@threading_helper.requires_working_threading()
def test_concurrent_resolve_returns_one_object(self):
"""Concurrent reifications of one proxy all return the same object."""
import builtins
ns = {"__builtins__": {
"__lazy_import__": builtins.__lazy_import__,
"__import__": lambda name, *args: types.ModuleType(name),
}}
exec("lazy import target_module", ns)
proxy = ns["target_module"]
barrier = threading.Barrier(8)
results = []

def worker():
barrier.wait()
results.append(proxy.resolve())

with threading_helper.start_threads(
threading.Thread(target=worker) for _ in range(8)):
pass
self.assertEqual(len(results), 8)
self.assertTrue(all(r is results[0] for r in results))

@support.requires_subprocess()
def test_from_import_proxy_remembers_the_attribute(self):
"""A `lazy from` proxy binds the attribute, not the module."""
code = textwrap.dedent("""
import sys

lazy from test.test_lazy_import.data.basic2 import f

def main():
proxy = globals()["f"]
resolved = proxy.resolve()
module = sys.modules["test.test_lazy_import.data.basic2"]
assert resolved is module.f, (resolved, module.f)
# Rebinding on the source module does not retarget the proxy,
# just as it does not for an eager ``from ... import``.
module.f = lambda: None
assert proxy.resolve() is resolved

main()
""")
assert_python_ok("-c", code)


class SyntaxRestrictionTests(LazyImportTestCase):
"""Tests for syntax restrictions on lazy imports."""
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
Reify a lazy import at most once. Previously
:meth:`!types.LazyImportType.resolve` followed by a normal access to the
still-lazy global imported twice and bound two different objects.
18 changes: 18 additions & 0 deletions Objects/lazyimportobject.c
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,8 @@ typedef struct {
// in lz_from, and the attribute to import from (PyUnicode) it in lz_attr.
PyObject *lz_from;
PyObject *lz_attr;
// Result of the first successful resolution, or NULL.
PyObject *lz_resolved;
// Declaration location.
PyCodeObject *lz_code;
int lz_instr_offset;
Expand Down Expand Up @@ -70,6 +72,7 @@ _PyLazyImport_New(_PyInterpreterFrame *frame, PyObject *builtins,
m->lz_builtins = Py_XNewRef(builtins);
m->lz_from = Py_NewRef(name);
m->lz_attr = Py_XNewRef(fromlist);
m->lz_resolved = NULL;

m->lz_code = NULL;
m->lz_instr_offset = -1;
Expand Down Expand Up @@ -156,6 +159,7 @@ lazy_import_traverse(PyObject *op, visitproc visit, void *arg)
Py_VISIT(m->lz_builtins);
Py_VISIT(m->lz_from);
Py_VISIT(m->lz_attr);
Py_VISIT(m->lz_resolved);
Py_VISIT(m->lz_code);
return 0;
}
Expand All @@ -167,6 +171,7 @@ lazy_import_clear(PyObject *op)
Py_CLEAR(m->lz_builtins);
Py_CLEAR(m->lz_from);
Py_CLEAR(m->lz_attr);
Py_CLEAR(m->lz_resolved);
Py_CLEAR(m->lz_code);
return 0;
}
Expand Down Expand Up @@ -368,6 +373,10 @@ lazy_import_resolve_impl(PyThreadState *tstate, PyObject *lazy_import,
assert(PyLazyImport_CheckExact(lazy_import));

PyLazyImportObject *lz = (PyLazyImportObject *)lazy_import;
PyObject *resolved = FT_ATOMIC_LOAD_PTR_ACQUIRE(lz->lz_resolved);
if (resolved != NULL) {
return Py_NewRef(resolved);
}

// Walk back to the placeholder IMPORT_NAME left, and the first lookup on it.
PyLazyImportObject *root = lz, *first = NULL;
Expand Down Expand Up @@ -535,6 +544,15 @@ lazy_import_resolve_impl(PyThreadState *tstate, PyObject *lazy_import,
}
Py_XDECREF(name);
}
if (obj != NULL) {
// Concurrent resolutions return the first result that was stored.
PyObject *expected = NULL;
if (!_Py_atomic_compare_exchange_ptr(&lz->lz_resolved, &expected,
Py_NewRef(obj))) {
Py_DECREF(obj);
Py_SETREF(obj, Py_NewRef(expected));
}
}
if (resolving != NULL) {
// A failed set resize can leave the placeholder inserted. Removing by
// identity also permits greenlets to finish in a different order.
Expand Down
Loading