gh-117139: Add header for tagged pointers

Include/internal/pycore_object.h

@@ -159,20 +159,6 @@ static inline void _Py_ClearImmortal(PyObject *op)
         op = NULL; \
     } while (0)
 
-// Mark an object as supporting deferred reference counting. This is a no-op
-// in the default (with GIL) build. Objects that use deferred reference
-// counting should be tracked by the GC so that they are eventually collected.
-extern void _PyObject_SetDeferredRefcount(PyObject *op);
-
-static inline int
-_PyObject_HasDeferredRefcount(PyObject *op)
-{
-#ifdef Py_GIL_DISABLED
-    return (op->ob_gc_bits & _PyGC_BITS_DEFERRED) != 0;
-#else
-    return 0;
-#endif
-}
 
 #if !defined(Py_GIL_DISABLED)
 static inline void

Include/internal/pycore_tagged.h

@@ -0,0 +1,194 @@
+#ifndef Py_INTERNAL_TAGGED_H
+#define Py_INTERNAL_TAGGED_H
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#ifndef Py_BUILD_CORE
+#  error "this header requires Py_BUILD_CORE define"
+#endif
+
+#include <stddef.h>
+
+// Mark an object as supporting deferred reference counting. This is a no-op
+// in the default (with GIL) build. Objects that use deferred reference
+// counting should be tracked by the GC so that they are eventually collected.
+extern void _PyObject_SetDeferredRefcount(PyObject *op);
+
+static inline int
+_PyObject_HasDeferredRefcount(PyObject *op)
+{
+#ifdef Py_GIL_DISABLED
+    return (op->ob_gc_bits & _PyGC_BITS_DEFERRED) != 0;
+#else
+    return 0;
+#endif
+}
+
+typedef union {
+    uintptr_t bits;
+} _PyStackRef;
+
+#ifdef Py_GIL_DISABLED
+    #define Py_TAG_DEFERRED (0b1)
+    #define Py_TAG (Py_TAG_DEFERRED)
+#else
+    // Define this to test for tagged pointer leakage in the ceval stack.
+    // #define Py_TAG_TEST (0b1)
+    #define Py_TAG 0
+#endif
+
+#if defined(Py_TAG_TEST)
+    #define Py_STACKREF_UNTAG_BORROWED(tagged) ((PyObject *)(uintptr_t)((tagged).bits & (~Py_TAG_TEST)))
+#elif defined(Py_GIL_DISABLED)
+    static inline PyObject *
+    _Py_STACKREF_UNTAG_BORROWED(_PyStackRef tagged) {
+        PyObject *cleared = ((PyObject *)((tagged).bits & (~Py_TAG)));
+        return cleared;
+    }
+    #define Py_STACKREF_UNTAG_BORROWED(tagged) _Py_STACKREF_UNTAG_BORROWED(tagged)
+#else
+    #define Py_STACKREF_UNTAG_BORROWED(tagged) ((PyObject *)(uintptr_t)((tagged).bits))
+#endif
+
+#if defined(Py_TAG_TEST)
+    #define Py_STACKREF_TAG(obj) ((_PyStackRef){.bits = ((uintptr_t)(obj) | Py_TAG_TEST)})
+#elif defined(Py_GIL_DISABLED)
+    static inline _PyStackRef
+    _Py_STACKREF_TAG(PyObject *obj) {
+        // Make sure we don't take an already tagged value.
+        assert(Py_STACKREF_UNTAG_BORROWED(((_PyStackRef){.bits = ((uintptr_t)(obj))})) == obj);
+        return ((_PyStackRef){.bits = ((uintptr_t)(obj))});
+    }
+    #define Py_STACKREF_TAG(obj) _Py_STACKREF_TAG(_PyObject_CAST(obj))
+#else
+    #define Py_STACKREF_TAG(obj) ((_PyStackRef){.bits = ((uintptr_t)(obj))})
+#endif
+
+#if defined(Py_TAG_TEST)
+    #define Py_STACKREF_TAG_DEFERRED(obj) ((_PyStackRef){.bits = ((uintptr_t)(obj) | Py_TAG_TEST)})
+#elif defined(Py_GIL_DISABLED)
+    static inline _PyStackRef
+    _Py_STACKREF_TAG_DEFERRED(PyObject *obj) {
+        // Make sure we don't take an already tagged value.
+        assert(Py_STACKREF_UNTAG_BORROWED(((_PyStackRef){.bits = ((uintptr_t)(obj))})) == obj);
+        int is_deferred = (obj != NULL && _PyObject_HasDeferredRefcount(obj));
+        int tag = (is_deferred ? Py_TAG_DEFERRED : 0);
+        return ((_PyStackRef){.bits = ((uintptr_t)(obj) | tag)});
+    }
+    #define Py_STACKREF_TAG_DEFERRED(obj) _Py_STACKREF_TAG_DEFERRED(_PyObject_CAST(obj))
+#else
+    #define Py_STACKREF_TAG_DEFERRED(obj) ((_PyStackRef){.bits = ((uintptr_t)(obj))})
+#endif
+
+#if defined(Py_TAG_TEST)
+    #define Py_STACKREF_UNTAG_OWNED(tagged) Py_STACKREF_UNTAG_BORROWED(tagged)
+#elif defined(Py_GIL_DISABLED)
+    static inline PyObject *
+    _Py_STACKREF_UNTAG_OWNED(_PyStackRef tagged) {
+        if ((tagged.bits & Py_TAG_DEFERRED) == Py_TAG_DEFERRED) {
+            assert(_PyObject_HasDeferredRefcount(Py_STACKREF_UNTAG_BORROWED(tagged)));
+            return Py_NewRef(Py_STACKREF_UNTAG_BORROWED(tagged));
+        }
+        return Py_STACKREF_UNTAG_BORROWED(tagged);
+    }
+    #define Py_STACKREF_UNTAG_OWNED(tagged) _Py_STACKREF_UNTAG_OWNED(tagged)
+#else
+    #define Py_STACKREF_UNTAG_OWNED(tagged) Py_STACKREF_UNTAG_BORROWED(tagged)
+#endif
+
+#define MAX_UNTAG_SCRATCH 10
+
+static inline void
+_Py_untag_stack_borrowed(PyObject **dst, const _PyStackRef *src, size_t length) {
+    for (size_t i = 0; i < length; i++) {
+        dst[i] = Py_STACKREF_UNTAG_BORROWED(src[i]);
+    }
+}
+
+static inline void
+_Py_untag_stack_owned(PyObject **dst, const _PyStackRef *src, size_t length) {
+    for (size_t i = 0; i < length; i++) {
+        dst[i] = Py_STACKREF_UNTAG_OWNED(src[i]);
+    }
+}
+
+
+#define Py_STACKREF_XSETREF(dst, src) \
+    do { \
+        _PyStackRef *_tmp_dst_ptr = _Py_CAST(_PyStackRef*, &(dst)); \
+        _PyStackRef _tmp_old_dst = (*_tmp_dst_ptr); \
+        *_tmp_dst_ptr = (src); \
+        Py_STACKREF_XDECREF(_tmp_old_dst); \
+    } while (0)
+
+#define Py_STACKREF_SETREF(dst, src) \
+    do { \
+        _PyStackRef *_tmp_dst_ptr = _Py_CAST(_PyStackRef*, &(dst)); \
+        _PyStackRef _tmp_old_dst = (*_tmp_dst_ptr); \
+        *_tmp_dst_ptr = (src); \
+        Py_STACKREF_DECREF(_tmp_old_dst); \
+    } while (0)
+
+#define Py_STACKREF_CLEAR(op) \
+    do { \
+        _PyStackRef *_tmp_op_ptr = _Py_CAST(_PyStackRef*, &(op)); \
+        _PyStackRef _tmp_old_op = (*_tmp_op_ptr); \
+        if (Py_STACKREF_UNTAG_BORROWED(_tmp_old_op) != NULL) { \
+            *_tmp_op_ptr = Py_STACKREF_TAG(_Py_NULL); \
+            Py_STACKREF_DECREF(_tmp_old_op); \
+        } \
+    } while (0)
+
+#if defined(Py_GIL_DISABLED)
+    static inline void
+    _Py_STACKREF_DECREF(_PyStackRef tagged) {
+        if ((tagged.bits & Py_TAG_DEFERRED) == Py_TAG_DEFERRED) {
+            return;
+        }
+        Py_DECREF(Py_STACKREF_UNTAG_BORROWED(tagged));
+    }
+    #define Py_STACKREF_DECREF(op) _Py_STACKREF_DECREF(op)
+#else
+    #define Py_STACKREF_DECREF(op) Py_DECREF(Py_STACKREF_UNTAG_BORROWED(op))
+#endif
+
+#define Py_STACKREF_DECREF_OWNED(op) Py_DECREF(Py_STACKREF_UNTAG_BORROWED(op));
+
+#if defined(Py_GIL_DISABLED)
+    static inline void
+    _Py_STACKREF_INCREF(_PyStackRef tagged) {
+        if ((tagged.bits & Py_TAG_DEFERRED) == Py_TAG_DEFERRED) {
+            assert(_PyObject_HasDeferredRefcount(Py_STACKREF_UNTAG_BORROWED(tagged)));
+            return;
+        }
+        Py_INCREF(Py_STACKREF_UNTAG_BORROWED(tagged));
+    }
+    #define Py_STACKREF_INCREF(op) _Py_STACKREF_INCREF(op)
+#else
+    #define Py_STACKREF_INCREF(op) Py_INCREF(Py_STACKREF_UNTAG_BORROWED(op))
+#endif
+
+static inline void
+_Py_STACKREF_XDECREF(_PyStackRef op)
+{
+    if (Py_STACKREF_UNTAG_BORROWED(op) != NULL) {
+        Py_STACKREF_DECREF(op);
+    }
+}
+
+#define Py_STACKREF_XDECREF(op) _Py_STACKREF_XDECREF(op)
+
+static inline _PyStackRef
+_Py_StackRef_NewRef(_PyStackRef obj)
+{
+    Py_STACKREF_INCREF(obj);
+    return obj;
+}
+
+#define Py_StackRef_NewRef(op) _Py_StackRef_NewRef(op)
+
+#ifdef __cplusplus
+}
+#endif
+#endif /* !Py_INTERNAL_TAGGED_H */

Makefile.pre.in

@@ -1225,6 +1225,7 @@ PYTHON_HEADERS= \
 		$(srcdir)/Include/internal/pycore_structseq.h \
 		$(srcdir)/Include/internal/pycore_symtable.h \
 		$(srcdir)/Include/internal/pycore_sysmodule.h \
+		$(srcdir)/Include/internal/pycore_tagged.h \
 		$(srcdir)/Include/internal/pycore_time.h \
 		$(srcdir)/Include/internal/pycore_token.h \
 		$(srcdir)/Include/internal/pycore_traceback.h \

Objects/codeobject.c

@@ -11,6 +11,7 @@
 #include "pycore_opcode_utils.h"  // RESUME_AT_FUNC_START
 #include "pycore_pystate.h"       // _PyInterpreterState_GET()
 #include "pycore_setobject.h"     // _PySet_NextEntry()
+#include "pycore_tagged.h"        // _PyObject_SetDeferredRefcount
 #include "pycore_tuple.h"         // _PyTuple_ITEMS()
 #include "clinic/codeobject.c.h"
 

Objects/descrobject.c

@@ -9,6 +9,7 @@
 #include "pycore_modsupport.h"    // _PyArg_UnpackStack()
 #include "pycore_object.h"        // _PyObject_GC_UNTRACK()
 #include "pycore_pystate.h"       // _PyThreadState_GET()
+#include "pycore_tagged.h"        // _PyObject_SetDeferredRefcount
 #include "pycore_tuple.h"         // _PyTuple_ITEMS()
 
 

Objects/funcobject.c

@@ -6,6 +6,7 @@
 #include "pycore_modsupport.h"    // _PyArg_NoKeywords()
 #include "pycore_object.h"        // _PyObject_GC_UNTRACK()
 #include "pycore_pyerrors.h"      // _PyErr_Occurred()
+#include "pycore_tagged.h"        // _PyObject_SetDeferredRefcount
 
 
 static const char *

Objects/moduleobject.c

@@ -10,6 +10,7 @@
 #include "pycore_object.h"        // _PyType_AllocNoTrack
 #include "pycore_pyerrors.h"      // _PyErr_FormatFromCause()
 #include "pycore_pystate.h"       // _PyInterpreterState_GET()
+#include "pycore_tagged.h"        // _PyObject_SetDeferredRefcount
 
 #include "osdefs.h"               // MAXPATHLEN
 

Objects/typeobject.c

@@ -16,6 +16,7 @@
 #include "pycore_pyerrors.h"      // _PyErr_Occurred()
 #include "pycore_pystate.h"       // _PyThreadState_GET()
 #include "pycore_symtable.h"      // _Py_Mangle()
+#include "pycore_tagged.h"        // _PyObject_SetDeferredRefcount
 #include "pycore_typeobject.h"    // struct type_cache
 #include "pycore_unionobject.h"   // _Py_union_type_or
 #include "pycore_weakref.h"       // _PyWeakref_GET_REF()

PCbuild/pythoncore.vcxproj

@@ -291,6 +291,7 @@
     <ClInclude Include="..\Include\internal\pycore_structseq.h" />
     <ClInclude Include="..\Include\internal\pycore_sysmodule.h" />
     <ClInclude Include="..\Include\internal\pycore_symtable.h" />
+    <ClInclude Include="..\Include\internal\pycore_tagged.h" />
     <ClInclude Include="..\Include\internal\pycore_time.h" />
     <ClInclude Include="..\Include\internal\pycore_token.h" />
     <ClInclude Include="..\Include\internal\pycore_traceback.h" />

PCbuild/pythoncore.vcxproj.filters

@@ -789,6 +789,9 @@
     <ClInclude Include="..\Include\internal\pycore_symtable.h">
       <Filter>Include\internal</Filter>
     </ClInclude>
+    <ClInclude Include="..\Include\internal\pycore_tagged.h">
+      <Filter>Include\internal</Filter>
+    </ClInclude>
     <ClInclude Include="..\Include\internal\pycore_time.h">
       <Filter>Include\internal</Filter>
     </ClInclude>

Python/gc_free_threading.c

@@ -11,6 +11,7 @@
 #include "pycore_object_stack.h"
 #include "pycore_pyerrors.h"
 #include "pycore_pystate.h"       // _PyThreadState_GET()
+#include "pycore_tagged.h"        // _PyObject_SetDeferredRefcount
 #include "pycore_time.h"          // _PyTime_GetPerfCounter()
 #include "pycore_tstate.h"        // _PyThreadStateImpl
 #include "pycore_weakref.h"       // _PyWeakref_ClearRef()