Issue #139123: Reimplement base UUID type, uuid4(), and uuid7() in C

The C implementation considerably boosts the performance of the key UUID
operations:

------------------------------------
Operation                    Speedup
------------------------------------
uuid4() generation            15.01x
uuid7() generation            29.64x
UUID from string               6.76x
UUID from bytes                5.16x
str(uuid) conversion           6.66x
------------------------------------

Summary of changes:

* The UUID type is reimplemented in C in its entirety.

* The pure-Python is kept around and is used of the C implementation
  isn't available for some reason.

* Both implementations are tested extensively; additional tests are
  added to ensure that the C implementation of the type follows the pure
  Python implementation fully.

* The Python implementation stores UUID values as int objects. The C
  implementation stores them as `uint8_t[16]` array.

* The C implementation supports unpickling of UUIDs created with Python
  2 using protocols starting with 0. That necessitated a small fix to
  the `copyreg` module (the change is only affecting legacy pickle
  pathway).

* The C implementation has faster hash() implementation but also caches
  the computed hash value to speedup cases when UUIDs are used as
  set/dict keys.

* The C implementation has a freelist to make new UUID object
  instantiation as fast as possible.

* uuid4() and uuid7() are now implmented in C. The most performance
  boost (10x) comes from overfetching entropy to decrease the number of
  _PyOS_URandom() calls. On its own it's a safe optimization with the
  edge case that Unix fork needs to be explicitly handled. We do that by
  comparing the current PID to the PID of when the random buffer was
  populated.

* Portions of code are coming from my implementation of faster UUID
  in gel-python [1]. I did use AI during the development, but basically
  had to rewrite the code it generated to be more idiomatic and
  efficient.

* The benchmark can be found here [2].

* This PR makes Python UUID operations as fast as they are in NodeJS and
  Bun runtimes.

[1] https://github.com/MagicStack/py-pgproto/blob/b8109fb311a59f30f9947567a13508da9a776564/uuid.pyx

[2] https://gist.github.com/1st1/f03e816f34a61e4d46c78ff98baf4818

Author: 1st1

Include/internal/pycore_pylifecycle.h

@@ -98,8 +98,8 @@ extern const char* _Py_gitversion(void);
 // Export for '_asyncio' shared extension
 PyAPI_FUNC(int) _Py_IsInterpreterFinalizing(PyInterpreterState *interp);
 
-/* Random */
-extern int _PyOS_URandom(void *buffer, Py_ssize_t size);
+// Export for '_uuid' shared extension
+PyAPI_FUNC(int) _PyOS_URandom(void *buffer, Py_ssize_t size);
 
 // Export for '_random' shared extension
 PyAPI_FUNC(int) _PyOS_URandomNonblock(void *buffer, Py_ssize_t size);

Lib/copyreg.py

@@ -44,7 +44,13 @@ def pickle_super(obj):
 # Support for pickling new-style objects
 
 def _reconstructor(cls, base, state):
-    if base is object:
+    if cls.__module__ == '_uuid' and cls.__name__ == 'UUID' and base is object:
+        # Compatibility with ancient pickled data -- prior to uuid rewritten in C
+        # object.__new__ would work, but it can't work anymore.
+        obj = cls.__new__(cls)
+        return obj
+
+    if base is object and cls.__name__ != 'UUID':
         obj = object.__new__(cls)
     else:
         obj = base.__new__(cls, state)

Lib/test/test_uuid.py

@@ -35,6 +35,7 @@ def get_command_stdout(command, args):
 
 class BaseTestUUID:
     uuid = None
+    is_c_uuid = False
 
     def test_nil_uuid(self):
         nil_uuid = self.uuid.NIL
@@ -282,14 +283,16 @@ def test_exceptions(self):
         badvalue(lambda: self.uuid.UUID('123456781234567812345678z2345678'))
 
         # Badly formed bytes.
-        badvalue(lambda: self.uuid.UUID(bytes='abc'))
-        badvalue(lambda: self.uuid.UUID(bytes='\0'*15))
-        badvalue(lambda: self.uuid.UUID(bytes='\0'*17))
+        badtype(lambda: self.uuid.UUID(bytes='abc'))
+        badvalue(lambda: self.uuid.UUID(bytes=b'abc'))
+        badvalue(lambda: self.uuid.UUID(bytes=b'\0'*15))
+        badvalue(lambda: self.uuid.UUID(bytes=b'\0'*17))
 
         # Badly formed bytes_le.
-        badvalue(lambda: self.uuid.UUID(bytes_le='abc'))
-        badvalue(lambda: self.uuid.UUID(bytes_le='\0'*15))
-        badvalue(lambda: self.uuid.UUID(bytes_le='\0'*17))
+        badtype(lambda: self.uuid.UUID(bytes_le='abc'))
+        badvalue(lambda: self.uuid.UUID(bytes_le=b'abc'))
+        badvalue(lambda: self.uuid.UUID(bytes_le=b'\0'*15))
+        badvalue(lambda: self.uuid.UUID(bytes_le=b'\0'*17))
 
         # Badly formed fields.
         badvalue(lambda: self.uuid.UUID(fields=(1,)))
@@ -877,12 +880,18 @@ def test_uuid6_test_vectors(self):
             equal((u.int >> 80) & 0xffff, 0x232a)
             equal((u.int >> 96) & 0xffff_ffff, 0x1ec9_414c)
 
-    def test_uuid7(self):
+    def test_uuid7_functional(self):
         equal = self.assertEqual
         u = self.uuid.uuid7()
         equal(u.variant, self.uuid.RFC_4122)
         equal(u.version, 7)
 
+    def test_uuid7_mock(self):
+        if self.is_c_uuid:
+            self.skipTest("C implementation of uuid7 cannot be tested with mocks")
+
+        equal = self.assertEqual
+
         # 1 Jan 2023 12:34:56.123_456_789
         timestamp_ns = 1672533296_123_456_789  # ns precision
         timestamp_ms, _ = divmod(timestamp_ns, 1_000_000)
@@ -940,7 +949,15 @@ def test_uuid7_uniqueness(self):
         versions = {u.version for u in uuids}
         self.assertSetEqual(versions, {7})
 
-    def test_uuid7_monotonicity(self):
+    def test_uuid7_monotonicity_functional(self):
+        equal = self.assertEqual
+        us = [self.uuid.uuid7() for _ in range(10_000)]
+        equal(us, sorted(us))
+
+    def test_uuid7_monotonicity_mock(self):
+        if self.is_c_uuid:
+            self.skipTest("C implementation of uuid7 cannot be tested with mocks")
+
         equal = self.assertEqual
 
         us = [self.uuid.uuid7() for _ in range(10_000)]
@@ -1003,7 +1020,10 @@ def test_uuid7_monotonicity(self):
 
             self.assertLess(u1, u2)
 
-    def test_uuid7_timestamp_backwards(self):
+    def test_uuid7_timestamp_backwards_mock(self):
+        if self.is_c_uuid:
+            self.skipTest("C implementation of uuid7 cannot be tested with mocks")
+
         equal = self.assertEqual
         # 1 Jan 2023 12:34:56.123_456_789
         timestamp_ns = 1672533296_123_456_789  # ns precision
@@ -1043,7 +1063,10 @@ def test_uuid7_timestamp_backwards(self):
             equal((u.int >> 32) & 0x3fff_ffff, counter_lo + 1)
             equal(u.int & 0xffff_ffff, tail)
 
-    def test_uuid7_overflow_counter(self):
+    def test_uuid7_overflow_counter_mock(self):
+        if self.is_c_uuid:
+            self.skipTest("C implementation of uuid7 cannot be tested with mocks")
+
         equal = self.assertEqual
         # 1 Jan 2023 12:34:56.123_456_789
         timestamp_ns = 1672533296_123_456_789  # ns precision
@@ -1149,6 +1172,7 @@ def test_uuid_weakref(self):
 
 class CommandLineTestCases:
     uuid = None  # to be defined in subclasses
+    is_c_uuid = False
 
     def do_test_standalone_uuid(self, version):
         stdout = io.StringIO()
@@ -1257,6 +1281,7 @@ class TestUUIDWithoutExtModule(CommandLineTestCases, BaseTestUUID, unittest.Test
 @unittest.skipUnless(c_uuid, 'requires the C _uuid module')
 class TestUUIDWithExtModule(CommandLineTestCases, BaseTestUUID, unittest.TestCase):
     uuid = c_uuid
+    is_c_uuid = True
 
     def check_has_stable_libuuid_extractable_node(self):
         if not self.uuid._has_stable_extractable_node:
@@ -1287,6 +1312,7 @@ def test_windows_getnode_from_libuuid(self):
 
 class BaseTestInternals:
     _uuid = py_uuid
+    is_c_uuid = False
 
     def check_parse_mac(self, aix):
         if not aix:
@@ -1480,6 +1506,7 @@ class TestInternalsWithoutExtModule(BaseTestInternals, unittest.TestCase):
 @unittest.skipUnless(c_uuid, 'requires the C _uuid module')
 class TestInternalsWithExtModule(BaseTestInternals, unittest.TestCase):
     uuid = c_uuid
+    is_c_uuid = True
 
     @unittest.skipUnless(os.name == 'posix', 'requires Posix')
     def test_unix_getnode(self):
@@ -1497,5 +1524,78 @@ def test_windll_getnode(self):
         self.check_node(node)
 
 
+@unittest.skipUnless(c_uuid, "requires the C _uuid module")
+class TestCImplementationCompat(unittest.TestCase):
+    def test_compatibility(self):
+        import uuid
+
+        PU = uuid._py_UUID
+        CU = uuid._c_UUID
+        N = 1000
+
+        uuids = [
+            "00000000-0000-0000-0000-000000000000",
+            "ffffffff-ffff-ffff-ffff-ffffffffffff",
+            "c0bec4fd-e4e3-050c-a362-da3f734ffd56",  # regression
+            *(str(uuid.uuid4()) for _ in range(N)),
+            *(str(uuid.uuid7()) for _ in range(N)),
+            *(str(uuid.uuid1()) for _ in range(N)),
+            *(str(uuid.UUID(bytes=os.urandom(16))) for _ in range(N)),
+        ]
+
+        def full_test(p, u):
+            self.assertEqual(p, u)
+            self.assertEqual(p.hex, u.hex)
+            self.assertEqual(p.int, u.int)
+            self.assertEqual(p.variant, u.variant)
+            self.assertEqual(p.version, u.version)
+            self.assertEqual(p.is_safe, u.is_safe)
+            self.assertEqual(p.bytes, u.bytes)
+            self.assertEqual(p.bytes_le, u.bytes_le)
+            self.assertEqual(p.fields, u.fields)
+            self.assertEqual(p.time_low, u.time_low)
+            self.assertEqual(p.time_mid, u.time_mid)
+            self.assertEqual(p.time_hi_version, u.time_hi_version)
+            self.assertEqual(p.clock_seq_hi_variant, u.clock_seq_hi_variant)
+            self.assertEqual(p.clock_seq_low, u.clock_seq_low)
+            self.assertEqual(p.node, u.node)
+
+        all_ps = set()
+        all_us = set()
+        for uuid_str in uuids:
+            with self.subTest(uuid=uuid_str):
+                p = PU(uuid_str)
+                u = CU(uuid_str)
+                full_test(p, u)
+
+                u2 = CU(bytes_le=p.bytes_le)
+                full_test(p, u2)
+
+                u3 = CU(fields=p.fields)
+                full_test(p, u3)
+
+                u4 = CU(int=p.int)
+                full_test(p, u4)
+
+                u5 = CU(
+                    hex=p.hex,
+                    is_safe=uuid.SafeUUID.safe,
+                )
+                full_test(
+                    PU(
+                        uuid_str,
+                        is_safe=uuid.SafeUUID.safe,
+                    ),
+                    u5,
+                )
+
+                all_ps.add(p)
+                all_us.add(u)
+
+        self.assertEqual(len(all_ps), len(all_us))
+        self.assertEqual(len(all_ps), len(uuids))
+
+
+
 if __name__ == '__main__':
     unittest.main()

Lib/uuid.py

@@ -99,6 +99,7 @@ class SafeUUID:
 
 
 _UINT_128_MAX = (1 << 128) - 1
+
 # 128-bit mask to clear the variant and version bits of a UUID integral value
 _RFC_4122_CLEARFLAGS_MASK = ~((0xf000 << 64) | (0xc000 << 48))
 # RFC 4122 variant bits and version bits to activate on a UUID integral value.
@@ -111,6 +112,28 @@ class SafeUUID:
 _RFC_4122_VERSION_8_FLAGS = ((8 << 76) | (0x8000 << 48))
 
 
+def _unpickle(state):
+    """Internal function to unpickle a UUID from a state dictionary."""
+    # This is also used by the C extension module for pickle compatibility
+    # with protocols 0 & 1.
+    obj = UUID.__new__(UUID)
+    obj.__setstate__(state)
+    return obj
+
+
+# Import optional C extension at toplevel, to help disabling it when testing
+try:
+    import _uuid
+    _generate_time_safe = getattr(_uuid, "generate_time_safe", None)
+    _has_stable_extractable_node = _uuid.has_stable_extractable_node
+    _UuidCreate = getattr(_uuid, "UuidCreate", None)
+except ImportError:
+    _uuid = None
+    _generate_time_safe = None
+    _has_stable_extractable_node = False
+    _UuidCreate = None
+
+
 class UUID:
     """Instances of the UUID class represent UUIDs as specified in RFC 4122.
     UUID objects are immutable, hashable, and usable as dictionary keys.
@@ -219,13 +242,21 @@ def __init__(self, hex=None, bytes=None, bytes_le=None, fields=None,
                 raise ValueError('badly formed hexadecimal UUID string')
             int = int_(hex, 16)
         elif bytes_le is not None:
+            if not isinstance(bytes_le, bytes_):
+                raise TypeError(
+                    f'a bytes-like object is required, not {type(bytes_le).__name__!r}'
+                )
             if len(bytes_le) != 16:
                 raise ValueError('bytes_le is not a 16-char string')
             assert isinstance(bytes_le, bytes_), repr(bytes_le)
             bytes = (bytes_le[4-1::-1] + bytes_le[6-1:4-1:-1] +
                      bytes_le[8-1:6-1:-1] + bytes_le[8:])
             int = int_.from_bytes(bytes)  # big endian
         elif bytes is not None:
+            if not isinstance(bytes, bytes_):
+                raise TypeError(
+                    f'a bytes-like object is required, not {type(bytes).__name__!r}'
+                )
             if len(bytes) != 16:
                 raise ValueError('bytes is not a 16-char string')
             assert isinstance(bytes, bytes_), repr(bytes)
@@ -234,7 +265,7 @@ def __init__(self, hex=None, bytes=None, bytes_le=None, fields=None,
             if len(fields) != 6:
                 raise ValueError('fields is not a 6-tuple')
             (time_low, time_mid, time_hi_version,
-             clock_seq_hi_variant, clock_seq_low, node) = fields
+                clock_seq_hi_variant, clock_seq_low, node) = fields
             if not 0 <= time_low < (1 << 32):
                 raise ValueError('field 1 out of range (need a 32-bit value)')
             if not 0 <= time_mid < (1 << 16):
@@ -249,7 +280,7 @@ def __init__(self, hex=None, bytes=None, bytes_le=None, fields=None,
                 raise ValueError('field 6 out of range (need a 48-bit value)')
             clock_seq = (clock_seq_hi_variant << 8) | clock_seq_low
             int = ((time_low << 96) | (time_mid << 80) |
-                   (time_hi_version << 64) | (clock_seq << 48) | node)
+                    (time_hi_version << 64) | (clock_seq << 48) | node)
         if not 0 <= int <= _UINT_128_MAX:
             raise ValueError('int is out of range (need a 128-bit value)')
         if version is not None:
@@ -273,6 +304,11 @@ def _from_int(cls, value):
         object.__setattr__(self, 'is_safe', SafeUUID.unknown)
         return self
 
+    def __reduce_ex__(self, protocol):
+        # Primarily we define __reduce_ex__ to make the C implementation
+        # compatible with protocols 0 & 1.
+        return _unpickle, (self.__getstate__(),)
+
     def __getstate__(self):
         d = {'int': self.int}
         if self.is_safe != SafeUUID.unknown:
@@ -629,19 +665,6 @@ def _netstat_getnode():
     return _find_mac_under_heading('netstat', '-ian', b'Address')
 
 
-# Import optional C extension at toplevel, to help disabling it when testing
-try:
-    import _uuid
-    _generate_time_safe = getattr(_uuid, "generate_time_safe", None)
-    _has_stable_extractable_node = _uuid.has_stable_extractable_node
-    _UuidCreate = getattr(_uuid, "UuidCreate", None)
-except ImportError:
-    _uuid = None
-    _generate_time_safe = None
-    _has_stable_extractable_node = False
-    _UuidCreate = None
-
-
 def _unix_getnode():
     """Get the hardware address on Unix using the _uuid extension module."""
     if _generate_time_safe and _has_stable_extractable_node:
@@ -932,6 +955,20 @@ def uuid8(a=None, b=None, c=None):
     return UUID._from_int(int_uuid_8)
 
 
+_py_uuid4 = uuid4
+_py_uuid7 = uuid7
+_py_UUID = UUID
+try:
+    from _uuid import UUID, uuid4, uuid7
+except ImportError:
+    _c_UUID = None
+    _c_uuid4 = None
+    _c_uuid7 = None
+else:
+    _c_UUID = UUID
+    _c_uuid4 = uuid4
+    _c_uuid7 = uuid7
+
 def main():
     """Run the uuid command line interface."""
     uuid_funcs = {