From 7cfb6552ffd10068cf75abde2d653b36878cc8d5 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 23 Sep 2026 11:17:39 +0200 Subject: [PATCH 1/6] gh-148233: Add Py_HashBuffer() to the limited C API (#157966) --- Doc/data/stable_abi.dat | 1 + Doc/whatsnew/3.16.rst | 3 +- Include/cpython/pyhash.h | 2 - Include/pyhash.h | 4 ++ Lib/test/test_capi/test_hash.py | 5 ++- Lib/test/test_stable_abi_ctypes.py | 1 + ...-09-22-23-20-53.gh-issue-148233.U_2x-F.rst | 1 + Misc/stable_abi.toml | 2 + Modules/Setup.stdlib.in | 2 +- Modules/_testcapi/hash.c | 15 ------- Modules/_testlimitedcapi.c | 3 ++ Modules/_testlimitedcapi/hash.c | 40 +++++++++++++++++++ Modules/_testlimitedcapi/parts.h | 1 + PC/python3dll.c | 1 + PCbuild/_testlimitedcapi.vcxproj | 1 + PCbuild/_testlimitedcapi.vcxproj.filters | 1 + 16 files changed, 63 insertions(+), 20 deletions(-) create mode 100644 Misc/NEWS.d/next/C_API/2026-09-22-23-20-53.gh-issue-148233.U_2x-F.rst create mode 100644 Modules/_testlimitedcapi/hash.c diff --git a/Doc/data/stable_abi.dat b/Doc/data/stable_abi.dat index 86080fac7163838..3fbff9a0d65362b 100644 --- a/Doc/data/stable_abi.dat +++ b/Doc/data/stable_abi.dat @@ -969,6 +969,7 @@ func,Py_GetPlatform,3.2,, func,Py_GetRecursionLimit,3.2,, func,Py_GetVersion,3.2,, data,Py_HasFileSystemDefaultEncoding,3.2,, +func,Py_HashBuffer,3.16,, func,Py_IS_TYPE,3.15,, func,Py_IncRef,3.2,, func,Py_Initialize,3.2,, diff --git a/Doc/whatsnew/3.16.rst b/Doc/whatsnew/3.16.rst index 7e02924babe9523..8362b1ef7e312b0 100644 --- a/Doc/whatsnew/3.16.rst +++ b/Doc/whatsnew/3.16.rst @@ -1084,7 +1084,8 @@ C API changes New features ------------ -* TODO +* Add :c:func:`Py_HashBuffer` to the limited C API. + (Contributed by Victor Stinner in :gh:`148233`.) Porting to Python 3.16 ---------------------- diff --git a/Include/cpython/pyhash.h b/Include/cpython/pyhash.h index dac223368dbe5e4..39ccffa6f2b90f1 100644 --- a/Include/cpython/pyhash.h +++ b/Include/cpython/pyhash.h @@ -50,5 +50,3 @@ _Py_HashPointer(const void *ptr) } PyAPI_FUNC(Py_hash_t) PyObject_GenericHash(PyObject *); - -PyAPI_FUNC(Py_hash_t) Py_HashBuffer(const void *ptr, Py_ssize_t len); diff --git a/Include/pyhash.h b/Include/pyhash.h index 3e23e2758808d79..acadf28f63c5c36 100644 --- a/Include/pyhash.h +++ b/Include/pyhash.h @@ -47,6 +47,10 @@ extern "C" { # endif /* uint64_t && uint32_t && aligned */ #endif /* Py_HASH_ALGORITHM */ +#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= _Py_PACK_VERSION(3, 16) +PyAPI_FUNC(Py_hash_t) Py_HashBuffer(const void *ptr, Py_ssize_t len); +#endif + #ifndef Py_LIMITED_API # define Py_CPYTHON_HASH_H # include "cpython/pyhash.h" diff --git a/Lib/test/test_capi/test_hash.py b/Lib/test/test_capi/test_hash.py index f553ffb0d90f33b..d03c3e88087df57 100644 --- a/Lib/test/test_capi/test_hash.py +++ b/Lib/test/test_capi/test_hash.py @@ -1,7 +1,9 @@ import sys import unittest from test.support import import_helper + _testcapi = import_helper.import_module('_testcapi') +_testlimitedcapi = import_helper.import_module('_testlimitedcapi') SIZEOF_VOID_P = _testcapi.SIZEOF_VOID_P @@ -79,7 +81,8 @@ def python_hash_pointer(x): self.assertEqual(hash_pointer(VOID_P_MAX), -2) def test_hash_buffer(self): - hash_buffer = _testcapi.hash_buffer + # Test Py_HashBuffer() + hash_buffer = _testlimitedcapi.hash_buffer def check(data): self.assertEqual(hash_buffer(data), hash(data)) diff --git a/Lib/test/test_stable_abi_ctypes.py b/Lib/test/test_stable_abi_ctypes.py index 09ee2d53f98f585..aed7451e89b1563 100644 --- a/Lib/test/test_stable_abi_ctypes.py +++ b/Lib/test/test_stable_abi_ctypes.py @@ -929,6 +929,7 @@ def test_windows_feature_macros(self): "Py_GetRecursionLimit", "Py_GetVersion", "Py_HasFileSystemDefaultEncoding", + "Py_HashBuffer", "Py_IS_TYPE", "Py_IncRef", "Py_Initialize", diff --git a/Misc/NEWS.d/next/C_API/2026-09-22-23-20-53.gh-issue-148233.U_2x-F.rst b/Misc/NEWS.d/next/C_API/2026-09-22-23-20-53.gh-issue-148233.U_2x-F.rst new file mode 100644 index 000000000000000..069d547a3022b38 --- /dev/null +++ b/Misc/NEWS.d/next/C_API/2026-09-22-23-20-53.gh-issue-148233.U_2x-F.rst @@ -0,0 +1 @@ +Add :c:func:`Py_HashBuffer` to the limited C API. Patch by Victor Stinner. diff --git a/Misc/stable_abi.toml b/Misc/stable_abi.toml index d59a7c788fa9e02..36d4f094f9e85ac 100644 --- a/Misc/stable_abi.toml +++ b/Misc/stable_abi.toml @@ -2773,6 +2773,8 @@ added = '3.15' [const.Py_tp_module] added = '3.15' +[function.Py_HashBuffer] + added = '3.16' # PEP 757 import/export API. diff --git a/Modules/Setup.stdlib.in b/Modules/Setup.stdlib.in index d10ed146db92b6b..696fec1290e5f3e 100644 --- a/Modules/Setup.stdlib.in +++ b/Modules/Setup.stdlib.in @@ -174,7 +174,7 @@ @MODULE__TESTBUFFER_TRUE@_testbuffer _testbuffer.c @MODULE__TESTINTERNALCAPI_TRUE@_testinternalcapi _testinternalcapi.c _testinternalcapi/test_lock.c _testinternalcapi/pytime.c _testinternalcapi/set.c _testinternalcapi/test_critical_sections.c _testinternalcapi/complex.c _testinternalcapi/interpreter.c _testinternalcapi/tokenizer.c _testinternalcapi/tuple.c _testinternalcapi/typecache.c @MODULE__TESTCAPI_TRUE@_testcapi _testcapimodule.c _testcapi/vectorcall.c _testcapi/heaptype.c _testcapi/abstract.c _testcapi/unicode.c _testcapi/dict.c _testcapi/set.c _testcapi/list.c _testcapi/tuple.c _testcapi/getargs.c _testcapi/datetime.c _testcapi/docstring.c _testcapi/mem.c _testcapi/watchers.c _testcapi/long.c _testcapi/float.c _testcapi/complex.c _testcapi/numbers.c _testcapi/structmember.c _testcapi/exceptions.c _testcapi/code.c _testcapi/buffer.c _testcapi/pyatomic.c _testcapi/run.c _testcapi/file.c _testcapi/codec.c _testcapi/immortal.c _testcapi/gc.c _testcapi/hash.c _testcapi/time.c _testcapi/bytes.c _testcapi/object.c _testcapi/modsupport.c _testcapi/monitoring.c _testcapi/config.c _testcapi/import.c _testcapi/frame.c _testcapi/type.c _testcapi/function.c _testcapi/module.c _testcapi/weakref.c _testcapi/marshal.c -@MODULE__TESTLIMITEDCAPI_TRUE@_testlimitedcapi _testlimitedcapi.c _testlimitedcapi/abstract.c _testlimitedcapi/bytearray.c _testlimitedcapi/bytes.c _testlimitedcapi/capsule.c _testlimitedcapi/codec.c _testlimitedcapi/complex.c _testlimitedcapi/dict.c _testlimitedcapi/eval.c _testlimitedcapi/float.c _testlimitedcapi/heaptype_relative.c _testlimitedcapi/import.c _testlimitedcapi/list.c _testlimitedcapi/long.c _testlimitedcapi/object.c _testlimitedcapi/pyos.c _testlimitedcapi/set.c _testlimitedcapi/slots.c _testlimitedcapi/sys.c _testlimitedcapi/threadstate.c _testlimitedcapi/tuple.c _testlimitedcapi/unicode.c _testlimitedcapi/vectorcall_limited.c _testlimitedcapi/version.c _testlimitedcapi/file.c _testlimitedcapi/weakref.c _testlimitedcapi/run.c _testlimitedcapi/type.c +@MODULE__TESTLIMITEDCAPI_TRUE@_testlimitedcapi _testlimitedcapi.c _testlimitedcapi/abstract.c _testlimitedcapi/bytearray.c _testlimitedcapi/bytes.c _testlimitedcapi/capsule.c _testlimitedcapi/codec.c _testlimitedcapi/complex.c _testlimitedcapi/dict.c _testlimitedcapi/eval.c _testlimitedcapi/float.c _testlimitedcapi/heaptype_relative.c _testlimitedcapi/import.c _testlimitedcapi/list.c _testlimitedcapi/long.c _testlimitedcapi/object.c _testlimitedcapi/pyos.c _testlimitedcapi/set.c _testlimitedcapi/slots.c _testlimitedcapi/sys.c _testlimitedcapi/threadstate.c _testlimitedcapi/tuple.c _testlimitedcapi/unicode.c _testlimitedcapi/vectorcall_limited.c _testlimitedcapi/version.c _testlimitedcapi/file.c _testlimitedcapi/weakref.c _testlimitedcapi/run.c _testlimitedcapi/type.c _testlimitedcapi/hash.c @MODULE__TESTCLINIC_TRUE@_testclinic _testclinic.c @MODULE__TESTCLINIC_LIMITED_TRUE@_testclinic_limited _testclinic_limited.c diff --git a/Modules/_testcapi/hash.c b/Modules/_testcapi/hash.c index 1525344a93fbcf0..87d535c585cc27a 100644 --- a/Modules/_testcapi/hash.c +++ b/Modules/_testcapi/hash.c @@ -66,20 +66,6 @@ hash_pointer(PyObject *Py_UNUSED(module), PyObject *arg) } -static PyObject * -hash_buffer(PyObject *Py_UNUSED(module), PyObject *args) -{ - char *ptr; - Py_ssize_t len; - if (!PyArg_ParseTuple(args, "y#", &ptr, &len)) { - return NULL; - } - - Py_hash_t hash = Py_HashBuffer(ptr, len); - return long_from_hash(hash); -} - - static PyObject * object_generichash(PyObject *Py_UNUSED(module), PyObject *arg) { @@ -92,7 +78,6 @@ object_generichash(PyObject *Py_UNUSED(module), PyObject *arg) static PyMethodDef test_methods[] = { {"hash_getfuncdef", hash_getfuncdef, METH_NOARGS}, {"hash_pointer", hash_pointer, METH_O}, - {"hash_buffer", hash_buffer, METH_VARARGS}, {"object_generichash", object_generichash, METH_O}, {NULL}, }; diff --git a/Modules/_testlimitedcapi.c b/Modules/_testlimitedcapi.c index 1f7392dd430d4cb..b30e32c56c57041 100644 --- a/Modules/_testlimitedcapi.c +++ b/Modules/_testlimitedcapi.c @@ -91,6 +91,9 @@ module_exec(PyObject *mod) if (_PyTestLimitedCAPI_Init_Type(mod) < 0) { return -1; } + if (_PyTestLimitedCAPI_Init_Hash(mod) < 0) { + return -1; + } return 0; } diff --git a/Modules/_testlimitedcapi/hash.c b/Modules/_testlimitedcapi/hash.c new file mode 100644 index 000000000000000..a17359d5aa45fe5 --- /dev/null +++ b/Modules/_testlimitedcapi/hash.c @@ -0,0 +1,40 @@ +// Need limited C API version 3.16 for Py_HashBuffer() +#define Py_LIMITED_API 0x03100000 + +#include "parts.h" +#include "util.h" + + +static PyObject* +long_from_hash(Py_hash_t hash) +{ + Py_BUILD_ASSERT(sizeof(long long) >= sizeof(hash)); + return PyLong_FromLongLong(hash); +} + + +// Test Py_HashBuffer() +static PyObject* +hash_buffer(PyObject *Py_UNUSED(module), PyObject *args) +{ + char *ptr; + Py_ssize_t len; + if (!PyArg_ParseTuple(args, "y#", &ptr, &len)) { + return NULL; + } + + Py_hash_t hash = Py_HashBuffer(ptr, len); + return long_from_hash(hash); +} + + +static PyMethodDef test_methods[] = { + {"hash_buffer", hash_buffer, METH_VARARGS}, + {NULL}, +}; + +int +_PyTestLimitedCAPI_Init_Hash(PyObject *m) +{ + return PyModule_AddFunctions(m, test_methods); +} diff --git a/Modules/_testlimitedcapi/parts.h b/Modules/_testlimitedcapi/parts.h index e34f7dd68095723..e8a2d82fb94aea4 100644 --- a/Modules/_testlimitedcapi/parts.h +++ b/Modules/_testlimitedcapi/parts.h @@ -51,5 +51,6 @@ int _PyTestLimitedCAPI_Init_File(PyObject *module); int _PyTestLimitedCAPI_Init_Weakref(PyObject *module); int _PyTestLimitedCAPI_Init_Run(PyObject *module); int _PyTestLimitedCAPI_Init_Type(PyObject *module); +int _PyTestLimitedCAPI_Init_Hash(PyObject *module); #endif // Py_TESTLIMITEDCAPI_PARTS_H diff --git a/PC/python3dll.c b/PC/python3dll.c index e0be9d65a93cda6..d1b7bafe6abb3cb 100755 --- a/PC/python3dll.c +++ b/PC/python3dll.c @@ -67,6 +67,7 @@ EXPORT_FUNC(Py_GetProgramName) EXPORT_FUNC(Py_GetPythonHome) EXPORT_FUNC(Py_GetRecursionLimit) EXPORT_FUNC(Py_GetVersion) +EXPORT_FUNC(Py_HashBuffer) EXPORT_FUNC(Py_IncRef) EXPORT_FUNC(Py_Initialize) EXPORT_FUNC(Py_InitializeEx) diff --git a/PCbuild/_testlimitedcapi.vcxproj b/PCbuild/_testlimitedcapi.vcxproj index c2bdea923f0f6d9..4218e0ed3945d27 100644 --- a/PCbuild/_testlimitedcapi.vcxproj +++ b/PCbuild/_testlimitedcapi.vcxproj @@ -121,6 +121,7 @@ + diff --git a/PCbuild/_testlimitedcapi.vcxproj.filters b/PCbuild/_testlimitedcapi.vcxproj.filters index 4c999949474b976..ddef60e599d4853 100644 --- a/PCbuild/_testlimitedcapi.vcxproj.filters +++ b/PCbuild/_testlimitedcapi.vcxproj.filters @@ -37,6 +37,7 @@ + From 71f6c005dc205f68d4435af319142e88a1405a80 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 23 Sep 2026 11:20:30 +0200 Subject: [PATCH 2/6] gh-157710: Defer allocation in PyUnicodeWriter_Create() (#157969) PyUnicodeWriter_Create(length) no longer allocates 'length' characters immediately. The allocation of the buffer is now done lazily at the first write, except if the read-only optimization is used. So the read-only optimization can also be used even if length is greater than 0. No longer overallocate the first buffer (at the first write). Only overallocate when the buffer is resized (at the second write). It avoids the need to truncate in PyUnicodeWriter_Finish() when PyUnicodeWriter_Create(length) used the exact output length. Add get_buffer() method to writer tests. --- Lib/test/test_capi/test_unicode.py | 122 +++++++++++++++++++++-------- Modules/_testcapi/unicode.c | 26 ++++++ Objects/unicode_writer.c | 59 +++++++------- 3 files changed, 144 insertions(+), 63 deletions(-) diff --git a/Lib/test/test_capi/test_unicode.py b/Lib/test/test_capi/test_unicode.py index 9bfb148f87b585d..f4bd961017b0ede 100644 --- a/Lib/test/test_capi/test_unicode.py +++ b/Lib/test/test_capi/test_unicode.py @@ -1880,6 +1880,36 @@ def test_basic(self): self.assertEqual(writer.finish(), "var=long value 'repr'") + def test_create(self): + # Test PyUnicodeWriter_Create() with non-zero size + s = 'Monty Python' + + # Preallocate the exact length. Use 2 writes to force the creation + # of a buffer: + # 1. Use the read-only optimization. + # 2. Allocate a buffer of length character. + # No resize needed in finish(). + writer = self.create_writer(len(s)) + writer.write_str(s[:5]) + self.assertEqual(writer.get_buffer(), (5, 127, True)) + writer.write_str(s[5:]) + self.assertEqual(writer.get_buffer(), (len(s), 127, False)) + self.assertEqual(writer.finish(), s) + + # Preallocate len(s)-1 characters. Use 3 writes: + # 1. Use read-only optimization. + # 2. Allocate a buffer of len-1 characters. + # 3. Resize the buffer with overallocation. + # finish() has to truncate the buffer. + writer = self.create_writer(len(s) - 1) + writer.write_str(s[:2]) + self.assertEqual(writer.get_buffer(), (2, 127, True)) + writer.write_str(s[2:5]) + self.assertEqual(writer.get_buffer(), (len(s) - 1, 127, False)) + writer.write_str(s[5:]) + self.assertGreater(writer.get_buffer()[0], len(s)) + self.assertEqual(writer.finish(), s) + def test_repr_null(self): writer = self.create_writer(0) writer.write_utf8(b'var=', -1) @@ -2087,32 +2117,39 @@ def test_substring_empty(self): def test_singletons(self): for size in (0, 123): with self.subTest(size=size): + # PyUnicodeWriter_Finish() returns the empty string singleton + # if no character has been written. writer = self.create_writer(size) writer.write_utf8(b'utf8', 0) writer.write_ascii(b'ascii', 0) writer.write_widechar(b'wstr', 0) writer.write_ucs4(b'ucs4', 0) - writer.write_substring('text', 0, 0) + writer.write_substring('text', 2, 2) + self.assertEqual(writer.get_buffer(), (None, 127, False)) self.assertIs(writer.finish(), '') for size in (0, 123): for ch in range(256): with self.subTest(size=size, ch=ch): ch = chr(ch) + maxchar = (255 if ord(ch) >= 128 else 127) - # If the first write is a Latin1 character and no buffer - # was allocated yet, use the singleton as the read-only - # buffer + # PyUnicodeWriter_WriteChar(ch) uses the read-only + # optimization with the character singleton if ch is a + # Latin1 character and no buffer was allocated yet. writer = self.create_writer(size) writer.write_char(ord(ch)) + self.assertEqual(writer.get_buffer(), + (1, maxchar, True)) self.assertIs(writer.finish(), ch) - # PyUnicodeWriter_Finish() replaces the buffer - # with the singleton + # PyUnicodeWriter_Finish() replaces the buffer with the + # singleton. Use PyUnicodeWriter_WriteSubstring() to avoid + # the read-only buffer optimization. writer = self.create_writer(size) - # Use PyUnicodeWriter_WriteSubstring() to avoid - # the read-only buffer optimization - writer.write_substring(ch + 'xxx', 0, 1) + writer.write_substring('xxx' + ch + 'y', 3, 4) + self.assertEqual(writer.get_buffer(), + (size or 1, maxchar, False)) self.assertIs(writer.finish(), ch) @unittest.skipUnless(support.Py_DEBUG, 'need debug build (Py_DEBUG)') @@ -2135,7 +2172,8 @@ def test_detect_overflow(self): def test_memory_error(self): # Inject MemoryError in PyUnicodeWriter_WriteStr() writer = self.create_writer(0) - writer.write_str("start") + writer.write_utf8(b"start", -1) + self.assertEqual(writer.get_buffer(), (5, 127, False)) with self.assertRaises(MemoryError): with support.inject_memory_error_cm(): # Resize the internal str object @@ -2143,24 +2181,35 @@ def test_memory_error(self): writer.write_str(" end") self.assertEqual(writer.finish(), "start end") - # Inject MemoryError in PyUnicodeWriter_Finish() + # Inject MemoryError in PyUnicodeWriter_Finish(). Use write_utf8() to + # allocate a buffer of 1024 character. finish() needs to truncate the + # buffer to 3 characters. writer = self.create_writer(1024) - writer.write_str("abc") + writer.write_utf8(b"abc", -1) + self.assertEqual(writer.get_buffer(), (1024, 127, False)) with self.assertRaises(MemoryError): with support.inject_memory_error_cm(): - # Need to truncate the internal str object writer.finish() def test_change_kind(self): writer = self.create_writer(0) + # Create an ASCII buffer writer.write_str('ascii ') + self.assertEqual(writer.get_buffer()[1], 127) + # Change the buffer to UCS1 writer.write_str('latin1:\xe9 ') + self.assertEqual(writer.get_buffer()[1], 255) + # Change the buffer to UCS2 writer.write_str('ucs2:\u20ac ') + self.assertEqual(writer.get_buffer()[1], 0xffff) + # Change the buffer to UCS4 writer.write_str('ucs4:\U0010ffff') + self.assertEqual(writer.get_buffer()[1], 0x10_ffff) + self.assertEqual(writer.finish(), 'ascii latin1:\xe9 ucs2:\u20ac ucs4:\U0010ffff') @@ -2168,27 +2217,38 @@ def test_readonly_optim(self): # Read-only optimization: if the first and only write is a Python str # object and no buffer was allocated yet, return the object unchanged unique_string = 'unique string' - writer = self.create_writer(0) - writer.write_str(unique_string) - self.assertIs(writer.finish(), unique_string) + expected = (len(unique_string), 127, True) + for size in (0, 123): + with self.subTest(size=size): + # PyUnicodeWriter_WriteStr() optimization + writer = self.create_writer(size) + writer.write_str(unique_string) + self.assertEqual(writer.get_buffer(), expected) + self.assertIs(writer.finish(), unique_string) - writer = self.create_writer(0) - writer.write_substring(unique_string, 0, len(unique_string)) - self.assertIs(writer.finish(), unique_string) + # PyUnicodeWriter_WriteSubstring() optimization + writer = self.create_writer(size) + writer.write_substring(unique_string, 0, len(unique_string)) + self.assertEqual(writer.get_buffer(), expected) + self.assertIs(writer.finish(), unique_string) - class MyStr: - def __str__(self): - return unique_string - writer = self.create_writer(0) - writer.write_str(MyStr()) - self.assertIs(writer.finish(), unique_string) + # PyUnicodeWriter_WriteStr() optimization + class MyStr: + def __str__(self): + return unique_string + writer = self.create_writer(size) + writer.write_str(MyStr()) + self.assertEqual(writer.get_buffer(), expected) + self.assertIs(writer.finish(), unique_string) - class MyRepr: - def __repr__(self): - return unique_string - writer = self.create_writer(0) - writer.write_repr(MyRepr()) - self.assertIs(writer.finish(), unique_string) + # PyUnicodeWriter_WriteRepr() optimization + class MyRepr: + def __repr__(self): + return unique_string + writer = self.create_writer(size) + writer.write_repr(MyRepr()) + self.assertEqual(writer.get_buffer(), expected) + self.assertIs(writer.finish(), unique_string) # Test PyUnicodeWriter_Format() diff --git a/Modules/_testcapi/unicode.c b/Modules/_testcapi/unicode.c index 8c9e3e9b5321d4b..c62813f4f3768d7 100644 --- a/Modules/_testcapi/unicode.c +++ b/Modules/_testcapi/unicode.c @@ -730,6 +730,31 @@ writer_get_pointer(PyObject *self_raw, PyObject *args) } +static PyObject* +writer_get_buffer(PyObject *self_raw, PyObject *args) +{ + WriterObject *self = (WriterObject *)self_raw; + if (writer_check(self) < 0) { + return NULL; + } + + _PyUnicodeWriter *writer = (_PyUnicodeWriter*)self->writer; + PyObject *allocated; + Py_UCS4 maxchar; + if (writer->buffer) { + allocated = PyLong_FromSsize_t(PyUnicode_GET_LENGTH(writer->buffer)); + maxchar = PyUnicode_MAX_CHAR_VALUE(writer->buffer); + } + else { + allocated = Py_None; + maxchar = writer->min_char; + } + return Py_BuildValue("(NkN)", + allocated, (unsigned long)maxchar, + PyBool_FromLong(writer->readonly)); +} + + static PyObject* writer_finish(PyObject *self_raw, PyObject *Py_UNUSED(args)) { @@ -755,6 +780,7 @@ static PyMethodDef writer_methods[] = { {"write_substring", _PyCFunction_CAST(writer_write_substring), METH_VARARGS}, {"decodeutf8stateful", _PyCFunction_CAST(writer_decodeutf8stateful), METH_VARARGS}, {"get_pointer", _PyCFunction_CAST(writer_get_pointer), METH_VARARGS}, + {"get_buffer", _PyCFunction_CAST(writer_get_buffer), METH_VARARGS}, {"finish", _PyCFunction_CAST(writer_finish), METH_NOARGS}, {NULL, NULL} /* sentinel */ }; diff --git a/Objects/unicode_writer.c b/Objects/unicode_writer.c index fc3a95cd97e4213..d6564ce84ed54eb 100644 --- a/Objects/unicode_writer.c +++ b/Objects/unicode_writer.c @@ -151,10 +151,9 @@ PyUnicodeWriter_Create(Py_ssize_t length) _PyUnicodeWriter *writer = (_PyUnicodeWriter *)pub_writer; _PyUnicodeWriter_Init(writer); - if (_PyUnicodeWriter_Prepare(writer, length, 127) < 0) { - PyUnicodeWriter_Discard(pub_writer); - return NULL; - } + // The buffer is created lazily at the first write, except if + // the read-only optimization is used. + writer->min_length = length; writer->overallocate = 1; return pub_writer; @@ -189,9 +188,6 @@ int _PyUnicodeWriter_PrepareInternal(_PyUnicodeWriter *writer, Py_ssize_t length, Py_UCS4 maxchar) { - Py_ssize_t newlen; - PyObject *newbuffer; - assert(length >= 0); assert(maxchar <= _Py_MAX_UNICODE); @@ -203,50 +199,50 @@ _PyUnicodeWriter_PrepareInternal(_PyUnicodeWriter *writer, PyErr_NoMemory(); return -1; } - newlen = writer->pos + length; + Py_ssize_t alloc = writer->pos + length; maxchar = Py_MAX(maxchar, writer->min_char); + PyObject *newbuffer; if (writer->buffer == NULL) { assert(!writer->readonly); - if (writer->overallocate - && newlen <= (PY_SSIZE_T_MAX - newlen / OVERALLOCATE_FACTOR)) { - /* overallocate to limit the number of realloc() */ - newlen += newlen / OVERALLOCATE_FACTOR; - } - if (newlen < writer->min_length) - newlen = writer->min_length; + // Do not overallocate at the first allocation, but use min_length + if (alloc < writer->min_length) + alloc = writer->min_length; - writer->buffer = PyUnicode_New(newlen, maxchar); + writer->buffer = PyUnicode_New(alloc, maxchar); if (writer->buffer == NULL) return -1; } - else if (newlen > writer->size) { - if (writer->overallocate - && newlen <= (PY_SSIZE_T_MAX - newlen / OVERALLOCATE_FACTOR)) { + else if (alloc > writer->size) { + // Do not overallocate at the first allocation, but use min_length + int overallocate = (writer->overallocate && !writer->readonly); + if (overallocate + && alloc <= (PY_SSIZE_T_MAX - alloc / OVERALLOCATE_FACTOR)) { /* overallocate to limit the number of realloc() */ - newlen += newlen / OVERALLOCATE_FACTOR; + alloc += alloc / OVERALLOCATE_FACTOR; } - if (newlen < writer->min_length) - newlen = writer->min_length; + if (alloc < writer->min_length) + alloc = writer->min_length; if (maxchar > writer->maxchar || writer->readonly) { /* resize + widen */ maxchar = Py_MAX(maxchar, writer->maxchar); - newbuffer = PyUnicode_New(newlen, maxchar); + newbuffer = PyUnicode_New(alloc, maxchar); if (newbuffer == NULL) return -1; _PyUnicode_FastCopyCharacters(newbuffer, 0, writer->buffer, 0, writer->pos); - Py_DECREF(writer->buffer); writer->readonly = 0; + Py_DECREF(writer->buffer); + writer->buffer = newbuffer; } else { - newbuffer = _PyUnicode_ResizeCompact(writer->buffer, newlen); + newbuffer = _PyUnicode_ResizeCompact(writer->buffer, alloc); if (newbuffer == NULL) return -1; + writer->buffer = newbuffer; } - writer->buffer = newbuffer; } else if (maxchar > writer->maxchar) { assert(!writer->readonly); @@ -310,13 +306,12 @@ _PyUnicodeWriter_WriteStr(_PyUnicodeWriter *writer, PyObject *str) { assert(PyUnicode_Check(str)); - Py_UCS4 maxchar; - Py_ssize_t len; - - len = PyUnicode_GET_LENGTH(str); - if (len == 0) + Py_ssize_t len = PyUnicode_GET_LENGTH(str); + if (len == 0) { return 0; - maxchar = PyUnicode_MAX_CHAR_VALUE(str); + } + Py_UCS4 maxchar = PyUnicode_MAX_CHAR_VALUE(str); + if (maxchar > writer->maxchar || len > writer->size - writer->pos) { if (writer->buffer == NULL) { assert(_PyUnicode_CheckConsistency(str, 1)); From 55ecc0a40918ba0346bc83ef639b655b8a883645 Mon Sep 17 00:00:00 2001 From: stratakis Date: Wed, 23 Sep 2026 12:01:48 +0200 Subject: [PATCH 3/6] gh-156722, test_dtrace: Terminate tracing processes gracefully on timeout (#156753) Send SIGTERM to the tracing process group before falling back to SIGKILL. --- Lib/test/test_dtrace.py | 26 +++++++++++++++++++------- 1 file changed, 19 insertions(+), 7 deletions(-) diff --git a/Lib/test/test_dtrace.py b/Lib/test/test_dtrace.py index 126e47a20cb7c61..946f6dc3a70a5c1 100644 --- a/Lib/test/test_dtrace.py +++ b/Lib/test/test_dtrace.py @@ -84,21 +84,33 @@ def normalize_trace_output(output): USE_PROCESS_GROUP = (hasattr(os, "setsid") and hasattr(os, "killpg")) +TERMINATE_TIMEOUT = 10 def create_process_group(*args, **kwargs): if USE_PROCESS_GROUP: kwargs['start_new_session'] = True return subprocess.Popen(*args, **kwargs) -def kill_process_group(proc): +def terminate_process_group(proc): if USE_PROCESS_GROUP: try: - os.killpg(proc.pid, signal.SIGKILL) + os.killpg(proc.pid, signal.SIGTERM) except ProcessLookupError: pass else: - proc.kill() - proc.communicate() # Clean up + proc.terminate() + + try: + proc.communicate(timeout=TERMINATE_TIMEOUT) + except subprocess.TimeoutExpired: + if USE_PROCESS_GROUP: + try: + os.killpg(proc.pid, signal.SIGKILL) + except ProcessLookupError: + pass + else: + proc.kill() + proc.communicate(timeout=TERMINATE_TIMEOUT) # Clean up def run_readelf(cmd): @@ -163,7 +175,7 @@ def trace(self, script_file, subcommand=None, *, timeout=None, try: stdout, _ = proc.communicate(timeout=timeout) except subprocess.TimeoutExpired: - kill_process_group(proc) + terminate_process_group(proc) raise if check_returncode and proc.returncode: raise AssertionError( @@ -361,7 +373,7 @@ def run_case(self, name, optimize_python=None): ) stdout, stderr = proc.communicate(timeout=60) except subprocess.TimeoutExpired: - kill_process_group(proc) + terminate_process_group(proc) raise AssertionError("bpftrace timed out") except (FileNotFoundError, PermissionError) as e: raise unittest.SkipTest(f"bpftrace not available: {e}") @@ -400,7 +412,7 @@ def assert_usable(self): ) stdout, stderr = proc.communicate(timeout=10) except subprocess.TimeoutExpired: - kill_process_group(proc) + terminate_process_group(proc) raise unittest.SkipTest("bpftrace timed out during usability check") except OSError as e: raise unittest.SkipTest(f"bpftrace not available: {e}") From 4532b36db5b9a43f63bf295aaf51df426e4c488d Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Wed, 23 Sep 2026 13:09:32 +0300 Subject: [PATCH 4/6] gh-157672: Keep Python's signal handlers when Tk is initialized on macOS (GH-157673) Tk on macOS replaces the SIGINT, SIGHUP and SIGTERM handlers with its own, which exits the process, so Ctrl-C terminated a tkinter program instead of raising KeyboardInterrupt. Save the handlers installed by Python before Tk_Init() and restore them after it. This affects Tk 8.6.11 to 8.6.18 and 9.0 to 9.0.4. Tk itself is fixed in 8.6.19 and 9.0.5. --- Lib/test/test_tkinter/test_misc.py | 14 ++++++++++++ ...-09-17-14-00-00.gh-issue-157672.tksigs.rst | 4 ++++ Modules/_tkinter.c | 22 ++++++++++++++++++- 3 files changed, 39 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Library/2026-09-17-14-00-00.gh-issue-157672.tksigs.rst diff --git a/Lib/test/test_tkinter/test_misc.py b/Lib/test/test_tkinter/test_misc.py index c9d402e11a8826f..5c1eea786698595 100644 --- a/Lib/test/test_tkinter/test_misc.py +++ b/Lib/test/test_tkinter/test_misc.py @@ -3,6 +3,7 @@ import os import gc import platform +import signal import sys import textwrap import time @@ -13,6 +14,7 @@ import enum from test import support from test.support import os_helper +from test.support.isolation import runInSubprocess from test.support.script_helper import assert_python_ok from test.test_tkinter.support import setUpModule # noqa: F401 from test.test_tkinter.support import (AbstractTkTest, AbstractDefaultRootTest, @@ -2149,6 +2151,18 @@ def _info_commands(widget, pattern=None): return widget.tk.splitlist(widget.tk.call('info', 'commands', pattern)) +class SignalTest(unittest.TestCase): + + @runInSubprocess() + def test_sigint_handler(self): + # gh-157672: Tk on macOS replaced the SIGINT handler with its own, + # which exits the process. + root = tkinter.Tk() + self.addCleanup(root.destroy) + with self.assertRaises(KeyboardInterrupt): + signal.raise_signal(signal.SIGINT) + + class TclObjTypeTest(AbstractTkTest, unittest.TestCase): # See FromObj() in Modules/_tkinter.c: Tcl object types are converted to # appropriate Python types. These conversions only happen in the object diff --git a/Misc/NEWS.d/next/Library/2026-09-17-14-00-00.gh-issue-157672.tksigs.rst b/Misc/NEWS.d/next/Library/2026-09-17-14-00-00.gh-issue-157672.tksigs.rst new file mode 100644 index 000000000000000..a751a2f9dc406b1 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-09-17-14-00-00.gh-issue-157672.tksigs.rst @@ -0,0 +1,4 @@ +Fix :mod:`tkinter` on macOS: creating a :class:`~tkinter.Tk` instance no +longer replaces the Python handlers of SIGINT, SIGHUP and SIGTERM with Tk's +handler which exits the process, so Ctrl-C raises :exc:`KeyboardInterrupt` +again. diff --git a/Modules/_tkinter.c b/Modules/_tkinter.c index f5fb1841f0d5940..746c39e6d3f4a20 100644 --- a/Modules/_tkinter.c +++ b/Modules/_tkinter.c @@ -33,6 +33,8 @@ Copyright (C) 1994 Steen Lumholt. #include "pycore_long.h" // _PyLong_IsNegative() #include "pycore_unicodeobject.h" // _PyUnicode_AsUTF8String +#include // SIGINT + #ifdef MS_WINDOWS # include #endif @@ -227,7 +229,25 @@ Tkinter_TkInit(Tcl_Interp *interp) does not search. Mount the DLL using Zipfs if possible. */ mount_tk_dll_zip(); #endif - return Tk_Init(interp); +#ifdef __APPLE__ + /* Tk on macOS replaces the handlers of these signals with its own, + which exits the process. Keep the handlers installed by Python + (gh-157672). */ + static const int signals[] = {SIGINT, SIGHUP, SIGTERM}; + PyOS_sighandler_t handlers[Py_ARRAY_LENGTH(signals)]; + for (size_t i = 0; i < Py_ARRAY_LENGTH(signals); i++) { + handlers[i] = PyOS_getsig(signals[i]); + } +#endif + int result = Tk_Init(interp); +#ifdef __APPLE__ + for (size_t i = 0; i < Py_ARRAY_LENGTH(signals); i++) { + if (handlers[i] != SIG_DFL) { + PyOS_setsig(signals[i], handlers[i]); + } + } +#endif + return result; } /* The threading situation is complicated. Tcl is not thread-safe, except From e1be6132b20fc7a65edac8c671420fa61eabe46c Mon Sep 17 00:00:00 2001 From: stratakis Date: Wed, 23 Sep 2026 12:10:09 +0200 Subject: [PATCH 5/6] gh-156701: Increase the SystemTap usability timeout (#156720) SystemTap may compile a kernel module during its usability check (if no cached module is available). We need to allow a larger timeout for that case. --- Lib/test/test_dtrace.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_dtrace.py b/Lib/test/test_dtrace.py index 946f6dc3a70a5c1..d398c0789d24a1d 100644 --- a/Lib/test/test_dtrace.py +++ b/Lib/test/test_dtrace.py @@ -144,6 +144,7 @@ class TraceBackend: EXTENSION = None COMMAND = None COMMAND_ARGS = [] + USABILITY_TIMEOUT = 10 def run_case(self, name, optimize_python=None): try: @@ -195,7 +196,7 @@ def trace_python(self, script_file, python_file, optimize_python=None): def assert_usable(self): try: output = self.trace(abspath("assert_usable" + self.EXTENSION), - timeout=10) + timeout=self.USABILITY_TIMEOUT) output = output.strip() except subprocess.TimeoutExpired: raise unittest.SkipTest( @@ -220,6 +221,7 @@ class SystemTapBackend(TraceBackend): EXTENSION = ".stp" COMMAND = ["stap", "-g"] PROBE_PLACEHOLDER = "@PYTHON_SYSTEMTAP_PROBE@" + USABILITY_TIMEOUT = 60 @staticmethod def quote_systemtap_string(value): From 68ed7a50226f8981788f2673677e30f1431e18a9 Mon Sep 17 00:00:00 2001 From: Locked-chess-official <13140752715@163.com> Date: Wed, 23 Sep 2026 18:54:26 +0800 Subject: [PATCH 6/6] gh-151471: Colorize the soft keyword `type` in IDLE (GH-151700) `type` is now colored as a keyword when it starts a statement and is followed by a name, as in `type Point = tuple[float, float]`. It stays a builtin elsewhere, for example in `type = type(1)`, `type(x)` or `type in (int, str)`. --- Lib/idlelib/colorizer.py | 8 +++++++- Lib/idlelib/idle_test/test_colorizer.py | 6 +++++- .../IDLE/2026-09-17-08-53-12.gh-issue-151471.-yvE03.rst | 1 + 3 files changed, 13 insertions(+), 2 deletions(-) create mode 100644 Misc/NEWS.d/next/IDLE/2026-09-17-08-53-12.gh-issue-151471.-yvE03.rst diff --git a/Lib/idlelib/colorizer.py b/Lib/idlelib/colorizer.py index ac12d8ace3575f3..5f18a22d2f05078 100644 --- a/Lib/idlelib/colorizer.py +++ b/Lib/idlelib/colorizer.py @@ -42,6 +42,11 @@ def make_pat(): ]) + r"))" ) + type_softkw = ( + r"^[ \t]*" + # at beginning of line + possible indentation + r"(?Ptype)" + + r"(?=[ \t]+(?!(?:" + "|".join(keyword.kwlist) + r")\b)[^\W\d])" + ) lazy_softkw = ( # lazy new in 3.15 (+ 2 lines below). r"^[ \t]*" + # at beginning of line + possible indentation r"(?Plazy)" + @@ -59,7 +64,7 @@ def make_pat(): dq3string = stringprefix + r'"""[^"\\]*((\\.|"(?!""))[^"\\]*)*(""")?' string = any("STRING", [sq3string, dq3string, sqstring, dqstring]) prog = re.compile("|".join([ - builtin, comment, string, kw, + type_softkw, builtin, comment, string, kw, match_softkw, case_default, case_softkw_and_pattern, lazy_softkw, any("SYNC", [r"\n"]), @@ -75,6 +80,7 @@ def make_pat(): "CASE_SOFTKW": "KEYWORD", "CASE_DEFAULT_UNDERSCORE": "KEYWORD", "CASE_SOFTKW2": "KEYWORD", + "TYPE_SOFTKW": "KEYWORD", "LAZY_SOFTKW": "KEYWORD", } diff --git a/Lib/idlelib/idle_test/test_colorizer.py b/Lib/idlelib/idle_test/test_colorizer.py index 8a6ab527f9387f4..5bb4f5b3ff36e64 100644 --- a/Lib/idlelib/idle_test/test_colorizer.py +++ b/Lib/idlelib/idle_test/test_colorizer.py @@ -52,6 +52,8 @@ async def f(): await g() ''' case _:''' "match x:" + type Point = tuple[float, float] + type = type(1) """) @@ -404,6 +406,8 @@ def test_recolorize_main(self, mock_notify): ('28.25', ('STRING',)), ('28.38', ('STRING',)), ('30.0', ('STRING',)), ('31.1', ('STRING',)), + ('32.0', ('KEYWORD',)), + ('33.0', ('BUILTIN',)), ('33.1', ('BUILTIN',)), # SYNC at the end of every line. ('1.55', ('SYNC',)), ('2.50', ('SYNC',)), ('3.34', ('SYNC',)), ) @@ -434,7 +438,7 @@ def test_recolorize_main(self, mock_notify): eq(text.tag_nextrange('STRING', '8.12'), ('8.14', '8.17')) eq(text.tag_nextrange('STRING', '8.17'), ('8.19', '8.26')) eq(text.tag_nextrange('SYNC', '8.0'), ('8.26', '9.0')) - eq(text.tag_nextrange('SYNC', '31.0'), ('31.10', '33.0')) + eq(text.tag_nextrange('SYNC', '31.0'), ('31.10', '32.0')) def _assert_highlighting(self, source, tag_ranges): """Check highlighting of a given piece of code. diff --git a/Misc/NEWS.d/next/IDLE/2026-09-17-08-53-12.gh-issue-151471.-yvE03.rst b/Misc/NEWS.d/next/IDLE/2026-09-17-08-53-12.gh-issue-151471.-yvE03.rst new file mode 100644 index 000000000000000..cc04822f6d33216 --- /dev/null +++ b/Misc/NEWS.d/next/IDLE/2026-09-17-08-53-12.gh-issue-151471.-yvE03.rst @@ -0,0 +1 @@ +Colorize the soft keyword ``type`` in IDLE.