From 09f99614594371e1fdabda245e6744ce1cd72c6f Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Mon, 9 Mar 2026 22:33:31 +0900 Subject: [PATCH 01/92] type lock --- crates/vm/src/builtins/type.rs | 419 ++++++++++++++-------- crates/vm/src/frame.rs | 45 +-- crates/vm/src/stdlib/_ctypes/base.rs | 10 +- crates/vm/src/stdlib/_ctypes/structure.rs | 6 +- crates/vm/src/stdlib/_ctypes/union.rs | 3 +- crates/vm/src/stdlib/posix.rs | 1 + crates/vm/src/vm/interpreter.rs | 1 + crates/vm/src/vm/mod.rs | 2 + 8 files changed, 305 insertions(+), 182 deletions(-) diff --git a/crates/vm/src/builtins/type.rs b/crates/vm/src/builtins/type.rs index 19fca5cf473..303529894b0 100644 --- a/crates/vm/src/builtins/type.rs +++ b/crates/vm/src/builtins/type.rs @@ -460,9 +460,15 @@ fn is_subtype_with_mro(a_mro: &[PyTypeRef], a: &Py, b: &Py) -> b } impl PyType { + #[inline] + fn with_type_lock(vm: &VirtualMachine, f: impl FnOnce() -> R) -> R { + let _guard = vm.state.type_mutex.lock(); + f() + } + /// Assign a fresh version tag. Returns 0 if the version counter has been /// exhausted, in which case no new cache entries can be created. - pub fn assign_version_tag(&self) -> u32 { + fn assign_version_tag_inner(&self) -> u32 { let v = self.tp_version_tag.load(Ordering::Acquire); if v != 0 { return v; @@ -470,7 +476,7 @@ impl PyType { // Assign versions to all direct bases first (MRO invariant). for base in self.bases.read().iter() { - if base.assign_version_tag() == 0 { + if base.assign_version_tag_inner() == 0 { return 0; } } @@ -490,8 +496,23 @@ impl PyType { } } + pub fn assign_version_tag(&self) -> u32 { + self.assign_version_tag_inner() + } + + pub(crate) fn version_for_specialization(&self, vm: &VirtualMachine) -> u32 { + Self::with_type_lock(vm, || { + let version = self.tp_version_tag.load(Ordering::Acquire); + if version == 0 { + self.assign_version_tag_inner() + } else { + version + } + }) + } + /// Invalidate this type's version tag and cascade to all subclasses. - pub fn modified(&self) { + fn modified_inner(&self) { if let Some(ext) = self.heaptype_ext.as_ref() { ext.specialization_cache.invalidate_for_type_modified(); } @@ -508,11 +529,15 @@ impl PyType { let subclasses = self.subclasses.read(); for weak_ref in subclasses.iter() { if let Some(sub) = weak_ref.upgrade() { - sub.downcast_ref::().unwrap().modified(); + sub.downcast_ref::().unwrap().modified_inner(); } } } + pub fn modified(&self) { + self.modified_inner(); + } + pub fn new_simple_heap( name: &str, base: &Py, @@ -974,6 +999,74 @@ impl PyType { self.find_name_in_mro(attr_name) } + /// CPython-style `_PyType_LookupRefAndVersion` equivalent for interned names. + /// Returns the observed lookup result and the type version used for the lookup. + pub(crate) fn lookup_ref_and_version_interned( + &self, + name: &'static PyStrInterned, + vm: &VirtualMachine, + ) -> (Option, u32) { + let version = self.tp_version_tag.load(Ordering::Acquire); + if version != 0 { + let idx = type_cache_hash(version, name); + let entry = &TYPE_CACHE[idx]; + let name_ptr = name as *const _ as *mut _; + loop { + let seq1 = entry.begin_read(); + let entry_version = entry.version.load(Ordering::Acquire); + let type_version = self.tp_version_tag.load(Ordering::Acquire); + if entry_version != type_version + || !core::ptr::eq(entry.name.load(Ordering::Relaxed), name_ptr) + { + break; + } + let ptr = entry.value.load(Ordering::Acquire); + if ptr.is_null() { + if entry.end_read(seq1) { + return (None, entry_version); + } + continue; + } + if let Some(cloned) = unsafe { PyObject::try_to_owned_from_ptr(ptr) } { + let same_ptr = core::ptr::eq(entry.value.load(Ordering::Relaxed), ptr); + if same_ptr && entry.end_read(seq1) { + return (Some(cloned), entry_version); + } + drop(cloned); + continue; + } + break; + } + } + + Self::with_type_lock(vm, || { + let assigned = if self.tp_version_tag.load(Ordering::Acquire) == 0 { + self.assign_version_tag_inner() + } else { + self.tp_version_tag.load(Ordering::Acquire) + }; + let result = self.find_name_in_mro_uncached(name); + if assigned != 0 + && !TYPE_CACHE_CLEARING.load(Ordering::Acquire) + && self.tp_version_tag.load(Ordering::Acquire) == assigned + { + let idx = type_cache_hash(assigned, name); + let entry = &TYPE_CACHE[idx]; + let name_ptr = name as *const _ as *mut _; + entry.begin_write(); + entry.version.store(0, Ordering::Release); + let new_ptr = result.as_ref().map_or(core::ptr::null_mut(), |found| { + &**found as *const PyObject as *mut _ + }); + entry.value.store(new_ptr, Ordering::Relaxed); + entry.name.store(name_ptr, Ordering::Relaxed); + entry.version.store(assigned, Ordering::Release); + entry.end_write(); + } + (result, assigned) + }) + } + /// Cache __init__ for CALL_ALLOC_AND_ENTER_INIT specialization. /// The cache is valid only when guarded by the type version check. pub(crate) fn cache_init_for_specialization( @@ -988,15 +1081,17 @@ impl PyType { if tp_version == 0 { return false; } - if self.tp_version_tag.load(Ordering::Acquire) != tp_version { - return false; - } - let _guard = ext.specialization_cache.write_lock.lock(); - if self.tp_version_tag.load(Ordering::Acquire) != tp_version { - return false; - } - ext.specialization_cache.swap_init(Some(init), Some(vm)); - true + Self::with_type_lock(vm, || { + if self.tp_version_tag.load(Ordering::Acquire) != tp_version { + return false; + } + let _guard = ext.specialization_cache.write_lock.lock(); + if self.tp_version_tag.load(Ordering::Acquire) != tp_version { + return false; + } + ext.specialization_cache.swap_init(Some(init), Some(vm)); + true + }) } /// Read cached __init__ for CALL_ALLOC_AND_ENTER_INIT specialization. @@ -1030,20 +1125,22 @@ impl PyType { if tp_version == 0 { return false; } - let _guard = ext.specialization_cache.write_lock.lock(); - if self.tp_version_tag.load(Ordering::Acquire) != tp_version { - return false; - } - let func_version = getitem.get_version_for_current_state(); - if func_version == 0 { - return false; - } - ext.specialization_cache - .swap_getitem(Some(getitem), Some(vm)); - ext.specialization_cache - .getitem_version - .store(func_version, Ordering::Relaxed); - true + Self::with_type_lock(vm, || { + let _guard = ext.specialization_cache.write_lock.lock(); + if self.tp_version_tag.load(Ordering::Acquire) != tp_version { + return false; + } + let func_version = getitem.get_version_for_current_state(); + if func_version == 0 { + return false; + } + ext.specialization_cache + .getitem_version + .store(func_version, Ordering::Release); + ext.specialization_cache + .swap_getitem(Some(getitem), Some(vm)); + true + }) } /// Read cached __getitem__ for BINARY_OP_SUBSCR_GETITEM specialization. @@ -1410,38 +1507,41 @@ impl PyType { // // TODO: how to uniquely identify the subclasses to remove? // } - *zelf.bases.write() = bases; - // Recursively update the mros of this class and all subclasses - fn update_mro_recursively(cls: &PyType, vm: &VirtualMachine) -> PyResult<()> { - let mut mro = - PyType::resolve_mro(&cls.bases.read()).map_err(|msg| vm.new_type_error(msg))?; - // Preserve self (mro[0]) when updating MRO - mro.insert(0, cls.mro.read()[0].to_owned()); - *cls.mro.write() = mro; - for subclass in cls.subclasses.write().iter() { - let subclass = subclass.upgrade().unwrap(); - let subclass: &Py = subclass.downcast_ref().unwrap(); - update_mro_recursively(subclass, vm)?; + Self::with_type_lock(vm, || { + *zelf.bases.write() = bases; + // Recursively update the mros of this class and all subclasses + fn update_mro_recursively(cls: &PyType, vm: &VirtualMachine) -> PyResult<()> { + let mut mro = + PyType::resolve_mro(&cls.bases.read()).map_err(|msg| vm.new_type_error(msg))?; + // Preserve self (mro[0]) when updating MRO + mro.insert(0, cls.mro.read()[0].to_owned()); + *cls.mro.write() = mro; + for subclass in cls.subclasses.write().iter() { + let subclass = subclass.upgrade().unwrap(); + let subclass: &Py = subclass.downcast_ref().unwrap(); + update_mro_recursively(subclass, vm)?; + } + Ok(()) } - Ok(()) - } - update_mro_recursively(zelf, vm)?; + update_mro_recursively(zelf, vm)?; - // Invalidate inline caches - zelf.modified(); + // Invalidate inline caches + zelf.modified_inner(); - // TODO: do any old slots need to be cleaned up first? - zelf.init_slots(&vm.ctx); + // TODO: do any old slots need to be cleaned up first? + zelf.init_slots(&vm.ctx); - // Register this type as a subclass of its new bases - let weakref_type = super::PyWeak::static_type(); - for base in zelf.bases.read().iter() { - base.subclasses.write().push( - zelf.as_object() - .downgrade_with_weakref_typ_opt(None, weakref_type.to_owned()) - .unwrap(), - ); - } + // Register this type as a subclass of its new bases + let weakref_type = super::PyWeak::static_type(); + for base in zelf.bases.read().iter() { + base.subclasses.write().push( + zelf.as_object() + .downgrade_with_weakref_typ_opt(None, weakref_type.to_owned()) + .unwrap(), + ); + } + Ok(()) + })?; Ok(()) } @@ -1533,20 +1633,30 @@ impl PyType { ))); } - let mut attrs = self.attributes.write(); - // First try __annotate__, in case that's been set explicitly - if let Some(annotate) = attrs.get(identifier!(vm, __annotate__)).cloned() { + let annotate_key = identifier!(vm, __annotate__); + let annotate_func_key = identifier!(vm, __annotate_func__); + let attrs = self.attributes.read(); + if let Some(annotate) = attrs.get(annotate_key).cloned() { return Ok(annotate); } - // Then try __annotate_func__ - if let Some(annotate) = attrs.get(identifier!(vm, __annotate_func__)).cloned() { - // TODO: Apply descriptor tp_descr_get if needed + if let Some(annotate) = attrs.get(annotate_func_key).cloned() { return Ok(annotate); } - // Set __annotate_func__ = None and return None + drop(attrs); + let none = vm.ctx.none(); - attrs.insert(identifier!(vm, __annotate_func__), none.clone()); - Ok(none) + Ok(Self::with_type_lock(vm, || { + let mut attrs = self.attributes.write(); + if let Some(annotate) = attrs.get(annotate_key).cloned() { + return annotate; + } + if let Some(annotate) = attrs.get(annotate_func_key).cloned() { + return annotate; + } + self.modified_inner(); + attrs.insert(annotate_func_key, none.clone()); + none + })) } #[pygetset(setter)] @@ -1569,20 +1679,25 @@ impl PyType { return Err(vm.new_type_error("__annotate__ must be callable or None")); } - let mut attrs = self.attributes.write(); - // Clear cached annotations only when setting to a new callable - if !vm.is_none(&value) { - attrs.swap_remove(identifier!(vm, __annotations_cache__)); - } - attrs.insert(identifier!(vm, __annotate_func__), value); + Self::with_type_lock(vm, || { + self.modified_inner(); + let mut attrs = self.attributes.write(); + // Clear cached annotations only when setting to a new callable + if !vm.is_none(&value) { + attrs.swap_remove(identifier!(vm, __annotations_cache__)); + } + attrs.insert(identifier!(vm, __annotate_func__), value); + }); Ok(()) } #[pygetset] fn __annotations__(&self, vm: &VirtualMachine) -> PyResult { + let annotations_key = identifier!(vm, __annotations__); + let annotations_cache_key = identifier!(vm, __annotations_cache__); let attrs = self.attributes.read(); - if let Some(annotations) = attrs.get(identifier!(vm, __annotations__)).cloned() { + if let Some(annotations) = attrs.get(annotations_key).cloned() { // Ignore the __annotations__ descriptor stored on type itself. if !annotations.class().is(vm.ctx.types.getset_type) { if vm.is_none(&annotations) @@ -1597,8 +1712,7 @@ impl PyType { ))); } } - // Then try __annotations_cache__ - if let Some(annotations) = attrs.get(identifier!(vm, __annotations_cache__)).cloned() { + if let Some(annotations) = attrs.get(annotations_cache_key).cloned() { if vm.is_none(&annotations) || annotations.class().is(vm.ctx.types.dict_type) || self.slots.flags.has_feature(PyTypeFlags::HEAPTYPE) @@ -1635,11 +1749,20 @@ impl PyType { vm.ctx.new_dict().into() }; - // Cache the result in __annotations_cache__ - self.attributes - .write() - .insert(identifier!(vm, __annotations_cache__), annotations.clone()); - Ok(annotations) + Ok(Self::with_type_lock(vm, || { + let mut attrs = self.attributes.write(); + if let Some(existing) = attrs.get(annotations_key).cloned() + && !existing.class().is(vm.ctx.types.getset_type) + { + return existing; + } + if let Some(existing) = attrs.get(annotations_cache_key).cloned() { + return existing; + } + self.modified_inner(); + attrs.insert(annotations_cache_key, annotations.clone()); + annotations + })) } #[pygetset(setter)] @@ -1655,43 +1778,45 @@ impl PyType { ))); } - let mut attrs = self.attributes.write(); - let has_annotations = attrs.contains_key(identifier!(vm, __annotations__)); - - match value { - crate::function::PySetterValue::Assign(value) => { - // SET path: store the value (including None) - let key = if has_annotations { - identifier!(vm, __annotations__) - } else { - identifier!(vm, __annotations_cache__) - }; - attrs.insert(key, value); - if has_annotations { - attrs.swap_remove(identifier!(vm, __annotations_cache__)); - } - } - crate::function::PySetterValue::Delete => { - // DELETE path: remove the key - let removed = if has_annotations { - attrs - .swap_remove(identifier!(vm, __annotations__)) - .is_some() - } else { - attrs - .swap_remove(identifier!(vm, __annotations_cache__)) - .is_some() - }; - if !removed { - return Err(vm.new_attribute_error("__annotations__")); + Self::with_type_lock(vm, || { + self.modified_inner(); + let mut attrs = self.attributes.write(); + let has_annotations = attrs.contains_key(identifier!(vm, __annotations__)); + + match value { + crate::function::PySetterValue::Assign(value) => { + let key = if has_annotations { + identifier!(vm, __annotations__) + } else { + identifier!(vm, __annotations_cache__) + }; + attrs.insert(key, value); + if has_annotations { + attrs.swap_remove(identifier!(vm, __annotations_cache__)); + } } - if has_annotations { - attrs.swap_remove(identifier!(vm, __annotations_cache__)); + crate::function::PySetterValue::Delete => { + let removed = if has_annotations { + attrs + .swap_remove(identifier!(vm, __annotations__)) + .is_some() + } else { + attrs + .swap_remove(identifier!(vm, __annotations_cache__)) + .is_some() + }; + if !removed { + return Err(vm.new_attribute_error("__annotations__")); + } + if has_annotations { + attrs.swap_remove(identifier!(vm, __annotations_cache__)); + } } } - } - attrs.swap_remove(identifier!(vm, __annotate_func__)); - attrs.swap_remove(identifier!(vm, __annotate__)); + attrs.swap_remove(identifier!(vm, __annotate_func__)); + attrs.swap_remove(identifier!(vm, __annotate__)); + Ok(()) + })?; Ok(()) } @@ -1724,9 +1849,12 @@ impl PyType { #[pygetset(setter)] fn set___module__(&self, value: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { self.check_set_special_type_attr(identifier!(vm, __module__), vm)?; - let mut attributes = self.attributes.write(); - attributes.swap_remove(identifier!(vm, __firstlineno__)); - attributes.insert(identifier!(vm, __module__), value); + Self::with_type_lock(vm, || { + self.modified_inner(); + self.attributes + .write() + .insert(identifier!(vm, __module__), value); + }); Ok(()) } @@ -1848,24 +1976,26 @@ impl PyType { value: PySetterValue, vm: &VirtualMachine, ) -> PyResult<()> { + let key = identifier!(vm, __type_params__); match value { - PySetterValue::Assign(ref val) => { - let key = identifier!(vm, __type_params__); + PySetterValue::Assign(val) => { self.check_set_special_type_attr(key, vm)?; - self.modified(); - self.attributes.write().insert(key, val.clone().into()); + Self::with_type_lock(vm, || { + self.modified_inner(); + self.attributes.write().insert(key, val.into()); + }); } PySetterValue::Delete => { - // For delete, we still need to check if the type is immutable if self.slots.flags.has_feature(PyTypeFlags::IMMUTABLETYPE) { return Err(vm.new_type_error(format!( "cannot delete '__type_params__' attribute of immutable type '{}'", self.slot_name() ))); } - let key = identifier!(vm, __type_params__); - self.modified(); - self.attributes.write().shift_remove(&key); + Self::with_type_lock(vm, || { + self.modified_inner(); + self.attributes.write().shift_remove(&key); + }); } } Ok(()) @@ -2487,10 +2617,12 @@ impl Py { // Check if we can set this special type attribute self.check_set_special_type_attr(identifier!(vm, __doc__), vm)?; - // Set the __doc__ in the type's dict - self.attributes - .write() - .insert(identifier!(vm, __doc__), value); + PyType::with_type_lock(vm, || { + self.modified_inner(); + self.attributes + .write() + .insert(identifier!(vm, __doc__), value); + }); Ok(()) } @@ -2552,23 +2684,26 @@ impl SetAttr for PyType { } let assign = value.is_assign(); - // Invalidate inline caches before modifying attributes. - // This ensures other threads see the version invalidation before - // any attribute changes, preventing use-after-free of cached descriptors. - zelf.modified(); + Self::with_type_lock(vm, || { + // Invalidate inline caches before modifying attributes. + // This ensures other threads see the version invalidation before + // any attribute changes, preventing use-after-free of cached descriptors. + zelf.modified_inner(); - if let PySetterValue::Assign(value) = value { - zelf.attributes.write().insert(attr_name, value); - } else { - let prev_value = zelf.attributes.write().shift_remove(attr_name); // TODO: swap_remove applicable? - if prev_value.is_none() { - return Err(vm.new_attribute_error(format!( - "type object '{}' has no attribute '{}'", - zelf.name(), - attr_name, - ))); + if let PySetterValue::Assign(value) = value { + zelf.attributes.write().insert(attr_name, value); + } else { + let prev_value = zelf.attributes.write().shift_remove(attr_name); // TODO: swap_remove applicable? + if prev_value.is_none() { + return Err(vm.new_attribute_error(format!( + "type object '{}' has no attribute '{}'", + zelf.name(), + attr_name, + ))); + } } - } + Ok(()) + })?; if attr_name.as_wtf8().starts_with("__") && attr_name.as_wtf8().ends_with("__") { if assign { diff --git a/crates/vm/src/frame.rs b/crates/vm/src/frame.rs index 85b15aaac49..a78e6d37c56 100644 --- a/crates/vm/src/frame.rs +++ b/crates/vm/src/frame.rs @@ -7530,10 +7530,7 @@ impl ExecutingFrame<'_> { .load() .is_some_and(|f| f as usize == PyBaseObject::getattro as *const () as usize); if !is_default_getattro { - let mut type_version = cls.tp_version_tag.load(Acquire); - if type_version == 0 { - type_version = cls.assign_version_tag(); - } + let type_version = cls.version_for_specialization(_vm); if type_version != 0 && !oparg.is_method() && !self.specialization_eval_frame_active(_vm) @@ -7571,10 +7568,7 @@ impl ExecutingFrame<'_> { } // Get or assign type version - let mut type_version = cls.tp_version_tag.load(Acquire); - if type_version == 0 { - type_version = cls.assign_version_tag(); - } + let type_version = cls.version_for_specialization(_vm); if type_version == 0 { // Version counter overflow — backoff to avoid re-attempting every execution unsafe { @@ -7794,10 +7788,7 @@ impl ExecutingFrame<'_> { let owner_type = obj.downcast_ref::().unwrap(); // Get or assign type version for the type object itself - let mut type_version = owner_type.tp_version_tag.load(Acquire); - if type_version == 0 { - type_version = owner_type.assign_version_tag(); - } + let type_version = owner_type.version_for_specialization(_vm); if type_version == 0 { unsafe { self.code.instructions.write_adaptive_counter( @@ -7832,10 +7823,7 @@ impl ExecutingFrame<'_> { } let mut metaclass_version = 0; if !mcl.slots.flags.has_feature(PyTypeFlags::IMMUTABLETYPE) { - metaclass_version = mcl.tp_version_tag.load(Acquire); - if metaclass_version == 0 { - metaclass_version = mcl.assign_version_tag(); - } + metaclass_version = mcl.version_for_specialization(_vm); if metaclass_version == 0 { unsafe { self.code.instructions.write_adaptive_counter( @@ -8022,16 +8010,14 @@ impl ExecutingFrame<'_> { Some(Instruction::BinaryOpSubscrListSlice) } else { let cls = a.class(); + let (getitem, type_version) = + cls.lookup_ref_and_version_interned(identifier!(vm, __getitem__), vm); if cls.slots.flags.has_feature(PyTypeFlags::HEAPTYPE) && !self.specialization_eval_frame_active(vm) - && let Some(_getitem) = cls.get_attr(identifier!(vm, __getitem__)) + && let Some(_getitem) = getitem && let Some(func) = _getitem.downcast_ref_if_exact::(vm) && func.can_specialize_call(2) { - let mut type_version = cls.tp_version_tag.load(Acquire); - if type_version == 0 { - type_version = cls.assign_version_tag(); - } if type_version != 0 { if cls.cache_getitem_for_specialization( func.to_owned(), @@ -8570,11 +8556,8 @@ impl ExecutingFrame<'_> { && cls_new_fn as usize == obj_new_fn as usize && cls_alloc_fn as usize == obj_alloc_fn as usize { - let init = cls.get_attr(identifier!(vm, __init__)); - let mut version = cls.tp_version_tag.load(Acquire); - if version == 0 { - version = cls.assign_version_tag(); - } + let (init, version) = + cls.lookup_ref_and_version_interned(identifier!(vm, __init__), vm); if version == 0 { unsafe { self.code.instructions.write_adaptive_counter( @@ -8894,10 +8877,7 @@ impl ExecutingFrame<'_> { && cls.slots.as_sequence.length.load().is_none() { // Cache type version for ToBoolAlwaysTrue guard - let mut type_version = cls.tp_version_tag.load(Acquire); - if type_version == 0 { - type_version = cls.assign_version_tag(); - } + let type_version = cls.version_for_specialization(vm); if type_version != 0 { unsafe { self.code @@ -9232,10 +9212,7 @@ impl ExecutingFrame<'_> { } // Get or assign type version - let mut type_version = cls.tp_version_tag.load(Acquire); - if type_version == 0 { - type_version = cls.assign_version_tag(); - } + let type_version = cls.version_for_specialization(vm); if type_version == 0 { unsafe { self.code.instructions.write_adaptive_counter( diff --git a/crates/vm/src/stdlib/_ctypes/base.rs b/crates/vm/src/stdlib/_ctypes/base.rs index 62856c4cef8..45c0e537452 100644 --- a/crates/vm/src/stdlib/_ctypes/base.rs +++ b/crates/vm/src/stdlib/_ctypes/base.rs @@ -2177,10 +2177,11 @@ fn make_fields( } let new_descr = super::PyCField::new_from_field(fdescr, index, offset); - cls.set_attr( + cls.as_object().set_attr( vm.ctx.intern_str(fname.as_wtf8()), new_descr.to_pyobject(vm), - ); + vm, + )?; } Ok(()) @@ -2219,10 +2220,11 @@ pub(super) fn make_anon_fields(cls: &Py, vm: &VirtualMachine) -> PyResul let mut new_descr = super::PyCField::new_from_field(descr, 0, 0); new_descr.set_anonymous(true); - cls.set_attr( + cls.as_object().set_attr( vm.ctx.intern_str(fname.as_wtf8()), new_descr.to_pyobject(vm), - ); + vm, + )?; make_fields(cls, descr, descr.index, descr.offset, vm)?; } diff --git a/crates/vm/src/stdlib/_ctypes/structure.rs b/crates/vm/src/stdlib/_ctypes/structure.rs index 96321cd7d55..fe30f30d9b2 100644 --- a/crates/vm/src/stdlib/_ctypes/structure.rs +++ b/crates/vm/src/stdlib/_ctypes/structure.rs @@ -498,7 +498,11 @@ impl PyCStructType { }; // Set the CField as a class attribute - cls.set_attr(vm.ctx.intern_str(name.clone()), c_field.to_pyobject(vm)); + cls.as_object().set_attr( + vm.ctx.intern_str(name.clone()), + c_field.to_pyobject(vm), + vm, + )?; // Update tracking - don't advance offset for packed bitfields if field_advances_offset { diff --git a/crates/vm/src/stdlib/_ctypes/union.rs b/crates/vm/src/stdlib/_ctypes/union.rs index e0b4900cbd5..0391291e32e 100644 --- a/crates/vm/src/stdlib/_ctypes/union.rs +++ b/crates/vm/src/stdlib/_ctypes/union.rs @@ -312,7 +312,8 @@ impl PyCUnionType { PyCField::new(name.clone(), field_type_ref, 0, size as isize, index) }; - cls.set_attr(vm.ctx.intern_str(name), c_field.to_pyobject(vm)); + cls.as_object() + .set_attr(vm.ctx.intern_str(name), c_field.to_pyobject(vm), vm)?; } // Calculate total_align and aligned_size diff --git a/crates/vm/src/stdlib/posix.rs b/crates/vm/src/stdlib/posix.rs index 21576e8da2d..e4fd42e64cf 100644 --- a/crates/vm/src/stdlib/posix.rs +++ b/crates/vm/src/stdlib/posix.rs @@ -694,6 +694,7 @@ pub mod module { reinit_mutex_after_fork(&vm.state.atexit_funcs); reinit_mutex_after_fork(&vm.state.global_trace_func); reinit_mutex_after_fork(&vm.state.global_profile_func); + reinit_mutex_after_fork(&vm.state.type_mutex); reinit_mutex_after_fork(&vm.state.monitoring); // PyGlobalState parking_lot::Mutex locks diff --git a/crates/vm/src/vm/interpreter.rs b/crates/vm/src/vm/interpreter.rs index 56c9606f7de..29cfcef6783 100644 --- a/crates/vm/src/vm/interpreter.rs +++ b/crates/vm/src/vm/interpreter.rs @@ -122,6 +122,7 @@ where switch_interval: AtomicCell::new(0.005), global_trace_func: PyMutex::default(), global_profile_func: PyMutex::default(), + type_mutex: PyMutex::default(), #[cfg(feature = "threading")] main_thread_ident: AtomicCell::new(0), #[cfg(feature = "threading")] diff --git a/crates/vm/src/vm/mod.rs b/crates/vm/src/vm/mod.rs index 8ad8a0d0bca..4c32f616042 100644 --- a/crates/vm/src/vm/mod.rs +++ b/crates/vm/src/vm/mod.rs @@ -637,6 +637,8 @@ pub struct PyGlobalState { pub global_trace_func: PyMutex>, /// Global profile function for all threads (set by sys._setprofileallthreads) pub global_profile_func: PyMutex>, + /// Global type mutation/versioning mutex for CPython-style FT type operations. + pub type_mutex: PyMutex<()>, /// Main thread identifier (pthread_self on Unix) #[cfg(feature = "threading")] pub main_thread_ident: AtomicCell, From 2a9b982591e0af65850b337dfe88eeb118806de3 Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Tue, 17 Mar 2026 21:00:02 +0900 Subject: [PATCH 02/92] Drop old PyObjectRef outside type lock to prevent deadlock Dropping values inside with_type_lock can trigger weakref callbacks, which may access attributes (LOAD_ATTR specialization) and re-acquire the non-reentrant type mutex, causing deadlock. Return old values from lock closures so they drop after lock release. --- crates/vm/src/builtins/type.rs | 106 +++++++++++----------- crates/vm/src/stdlib/_ctypes/base.rs | 10 +- crates/vm/src/stdlib/_ctypes/structure.rs | 6 +- crates/vm/src/stdlib/_ctypes/union.rs | 3 +- 4 files changed, 58 insertions(+), 67 deletions(-) diff --git a/crates/vm/src/builtins/type.rs b/crates/vm/src/builtins/type.rs index 303529894b0..7cbeaba4acb 100644 --- a/crates/vm/src/builtins/type.rs +++ b/crates/vm/src/builtins/type.rs @@ -18,7 +18,7 @@ use crate::{ common::{ ascii, borrow::BorrowedValue, - lock::{PyMutex, PyRwLock, PyRwLockReadGuard}, + lock::{PyRwLock, PyRwLockReadGuard}, }, function::{FuncArgs, KwArgs, OptionalArg, PyMethodDef, PySetterValue}, object::{Traverse, TraverseFn}, @@ -277,8 +277,6 @@ pub struct TypeSpecializationCache { pub init: PyAtomicRef>, pub getitem: PyAtomicRef>, pub getitem_version: AtomicU32, - // Serialize cache writes/invalidation similar to CPython's BEGIN_TYPE_LOCK. - write_lock: PyMutex<()>, retired: PyRwLock>, } @@ -288,7 +286,6 @@ impl TypeSpecializationCache { init: PyAtomicRef::from(None::>), getitem: PyAtomicRef::from(None::>), getitem_version: AtomicU32::new(0), - write_lock: PyMutex::new(()), retired: PyRwLock::new(Vec::new()), } } @@ -329,9 +326,6 @@ impl TypeSpecializationCache { #[inline] fn invalidate_for_type_modified(&self) { - let _guard = self.write_lock.lock(); - // _spec_cache contract: type modification invalidates all cached - // specialization functions. self.swap_init(None, None); self.swap_getitem(None, None); } @@ -351,7 +345,6 @@ impl TypeSpecializationCache { } fn clear_into(&self, out: &mut Vec) { - let _guard = self.write_lock.lock(); let old_init = unsafe { self.init.swap(None) }; if let Some(old_init) = old_init { out.push(old_init.into()); @@ -1085,7 +1078,6 @@ impl PyType { if self.tp_version_tag.load(Ordering::Acquire) != tp_version { return false; } - let _guard = ext.specialization_cache.write_lock.lock(); if self.tp_version_tag.load(Ordering::Acquire) != tp_version { return false; } @@ -1126,7 +1118,6 @@ impl PyType { return false; } Self::with_type_lock(vm, || { - let _guard = ext.specialization_cache.write_lock.lock(); if self.tp_version_tag.load(Ordering::Acquire) != tp_version { return false; } @@ -1645,18 +1636,19 @@ impl PyType { drop(attrs); let none = vm.ctx.none(); - Ok(Self::with_type_lock(vm, || { + let (result, _prev) = Self::with_type_lock(vm, || { let mut attrs = self.attributes.write(); if let Some(annotate) = attrs.get(annotate_key).cloned() { - return annotate; + return (annotate, None); } if let Some(annotate) = attrs.get(annotate_func_key).cloned() { - return annotate; + return (annotate, None); } self.modified_inner(); - attrs.insert(annotate_func_key, none.clone()); - none - })) + let prev = attrs.insert(annotate_func_key, none.clone()); + (none, prev) + }); + Ok(result) } #[pygetset(setter)] @@ -1679,14 +1671,17 @@ impl PyType { return Err(vm.new_type_error("__annotate__ must be callable or None")); } - Self::with_type_lock(vm, || { + let _prev_values = Self::with_type_lock(vm, || { self.modified_inner(); let mut attrs = self.attributes.write(); // Clear cached annotations only when setting to a new callable - if !vm.is_none(&value) { - attrs.swap_remove(identifier!(vm, __annotations_cache__)); - } - attrs.insert(identifier!(vm, __annotate_func__), value); + let removed = if !vm.is_none(&value) { + attrs.swap_remove(identifier!(vm, __annotations_cache__)) + } else { + None + }; + let prev = attrs.insert(identifier!(vm, __annotate_func__), value); + (removed, prev) }); Ok(()) @@ -1749,20 +1744,21 @@ impl PyType { vm.ctx.new_dict().into() }; - Ok(Self::with_type_lock(vm, || { + let (result, _prev) = Self::with_type_lock(vm, || { let mut attrs = self.attributes.write(); if let Some(existing) = attrs.get(annotations_key).cloned() && !existing.class().is(vm.ctx.types.getset_type) { - return existing; + return (existing, None); } if let Some(existing) = attrs.get(annotations_cache_key).cloned() { - return existing; + return (existing, None); } self.modified_inner(); - attrs.insert(annotations_cache_key, annotations.clone()); - annotations - })) + let prev = attrs.insert(annotations_cache_key, annotations.clone()); + (annotations, prev) + }); + Ok(result) } #[pygetset(setter)] @@ -1778,11 +1774,12 @@ impl PyType { ))); } - Self::with_type_lock(vm, || { + let _prev_values = Self::with_type_lock(vm, || { self.modified_inner(); let mut attrs = self.attributes.write(); let has_annotations = attrs.contains_key(identifier!(vm, __annotations__)); + let mut prev = Vec::new(); match value { crate::function::PySetterValue::Assign(value) => { let key = if has_annotations { @@ -1790,32 +1787,29 @@ impl PyType { } else { identifier!(vm, __annotations_cache__) }; - attrs.insert(key, value); + prev.extend(attrs.insert(key, value)); if has_annotations { - attrs.swap_remove(identifier!(vm, __annotations_cache__)); + prev.extend(attrs.swap_remove(identifier!(vm, __annotations_cache__))); } } crate::function::PySetterValue::Delete => { let removed = if has_annotations { - attrs - .swap_remove(identifier!(vm, __annotations__)) - .is_some() + attrs.swap_remove(identifier!(vm, __annotations__)) } else { - attrs - .swap_remove(identifier!(vm, __annotations_cache__)) - .is_some() + attrs.swap_remove(identifier!(vm, __annotations_cache__)) }; - if !removed { + if removed.is_none() { return Err(vm.new_attribute_error("__annotations__")); } + prev.extend(removed); if has_annotations { - attrs.swap_remove(identifier!(vm, __annotations_cache__)); + prev.extend(attrs.swap_remove(identifier!(vm, __annotations_cache__))); } } } - attrs.swap_remove(identifier!(vm, __annotate_func__)); - attrs.swap_remove(identifier!(vm, __annotate__)); - Ok(()) + prev.extend(attrs.swap_remove(identifier!(vm, __annotate_func__))); + prev.extend(attrs.swap_remove(identifier!(vm, __annotate__))); + Ok(prev) })?; Ok(()) @@ -1849,11 +1843,12 @@ impl PyType { #[pygetset(setter)] fn set___module__(&self, value: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { self.check_set_special_type_attr(identifier!(vm, __module__), vm)?; - Self::with_type_lock(vm, || { + let _prev_values = Self::with_type_lock(vm, || { self.modified_inner(); - self.attributes - .write() - .insert(identifier!(vm, __module__), value); + let mut attributes = self.attributes.write(); + let removed = attributes.swap_remove(identifier!(vm, __firstlineno__)); + let prev = attributes.insert(identifier!(vm, __module__), value); + (removed, prev) }); Ok(()) } @@ -1980,9 +1975,9 @@ impl PyType { match value { PySetterValue::Assign(val) => { self.check_set_special_type_attr(key, vm)?; - Self::with_type_lock(vm, || { + let _prev_value = Self::with_type_lock(vm, || { self.modified_inner(); - self.attributes.write().insert(key, val.into()); + self.attributes.write().insert(key, val.into()) }); } PySetterValue::Delete => { @@ -1992,9 +1987,9 @@ impl PyType { self.slot_name() ))); } - Self::with_type_lock(vm, || { + let _prev_value = Self::with_type_lock(vm, || { self.modified_inner(); - self.attributes.write().shift_remove(&key); + self.attributes.write().shift_remove(&key) }); } } @@ -2617,11 +2612,11 @@ impl Py { // Check if we can set this special type attribute self.check_set_special_type_attr(identifier!(vm, __doc__), vm)?; - PyType::with_type_lock(vm, || { + let _prev_value = PyType::with_type_lock(vm, || { self.modified_inner(); self.attributes .write() - .insert(identifier!(vm, __doc__), value); + .insert(identifier!(vm, __doc__), value) }); Ok(()) @@ -2684,14 +2679,17 @@ impl SetAttr for PyType { } let assign = value.is_assign(); - Self::with_type_lock(vm, || { + // Drop old value OUTSIDE the type lock to avoid deadlock: + // dropping may trigger weakref callbacks → method calls → + // LOAD_ATTR specialization → version_for_specialization → type lock. + let _prev_value = Self::with_type_lock(vm, || { // Invalidate inline caches before modifying attributes. // This ensures other threads see the version invalidation before // any attribute changes, preventing use-after-free of cached descriptors. zelf.modified_inner(); if let PySetterValue::Assign(value) = value { - zelf.attributes.write().insert(attr_name, value); + Ok(zelf.attributes.write().insert(attr_name, value)) } else { let prev_value = zelf.attributes.write().shift_remove(attr_name); // TODO: swap_remove applicable? if prev_value.is_none() { @@ -2701,8 +2699,8 @@ impl SetAttr for PyType { attr_name, ))); } + Ok(prev_value) } - Ok(()) })?; if attr_name.as_wtf8().starts_with("__") && attr_name.as_wtf8().ends_with("__") { diff --git a/crates/vm/src/stdlib/_ctypes/base.rs b/crates/vm/src/stdlib/_ctypes/base.rs index 45c0e537452..62856c4cef8 100644 --- a/crates/vm/src/stdlib/_ctypes/base.rs +++ b/crates/vm/src/stdlib/_ctypes/base.rs @@ -2177,11 +2177,10 @@ fn make_fields( } let new_descr = super::PyCField::new_from_field(fdescr, index, offset); - cls.as_object().set_attr( + cls.set_attr( vm.ctx.intern_str(fname.as_wtf8()), new_descr.to_pyobject(vm), - vm, - )?; + ); } Ok(()) @@ -2220,11 +2219,10 @@ pub(super) fn make_anon_fields(cls: &Py, vm: &VirtualMachine) -> PyResul let mut new_descr = super::PyCField::new_from_field(descr, 0, 0); new_descr.set_anonymous(true); - cls.as_object().set_attr( + cls.set_attr( vm.ctx.intern_str(fname.as_wtf8()), new_descr.to_pyobject(vm), - vm, - )?; + ); make_fields(cls, descr, descr.index, descr.offset, vm)?; } diff --git a/crates/vm/src/stdlib/_ctypes/structure.rs b/crates/vm/src/stdlib/_ctypes/structure.rs index fe30f30d9b2..96321cd7d55 100644 --- a/crates/vm/src/stdlib/_ctypes/structure.rs +++ b/crates/vm/src/stdlib/_ctypes/structure.rs @@ -498,11 +498,7 @@ impl PyCStructType { }; // Set the CField as a class attribute - cls.as_object().set_attr( - vm.ctx.intern_str(name.clone()), - c_field.to_pyobject(vm), - vm, - )?; + cls.set_attr(vm.ctx.intern_str(name.clone()), c_field.to_pyobject(vm)); // Update tracking - don't advance offset for packed bitfields if field_advances_offset { diff --git a/crates/vm/src/stdlib/_ctypes/union.rs b/crates/vm/src/stdlib/_ctypes/union.rs index 0391291e32e..e0b4900cbd5 100644 --- a/crates/vm/src/stdlib/_ctypes/union.rs +++ b/crates/vm/src/stdlib/_ctypes/union.rs @@ -312,8 +312,7 @@ impl PyCUnionType { PyCField::new(name.clone(), field_type_ref, 0, size as isize, index) }; - cls.as_object() - .set_attr(vm.ctx.intern_str(name), c_field.to_pyobject(vm), vm)?; + cls.set_attr(vm.ctx.intern_str(name), c_field.to_pyobject(vm)); } // Calculate total_align and aligned_size From 4b71002a632dc919984ee1539abb7321088b8a0b Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Tue, 14 Apr 2026 00:54:04 +0900 Subject: [PATCH 03/92] Align type lock behavior with CPython --- crates/vm/src/builtins/type.rs | 115 +++++++++++++++++++++------------ crates/vm/src/frame.rs | 89 +++++++++++++------------ crates/vm/src/stdlib/posix.rs | 3 + crates/vm/src/vm/thread.rs | 9 +++ 4 files changed, 131 insertions(+), 85 deletions(-) diff --git a/crates/vm/src/builtins/type.rs b/crates/vm/src/builtins/type.rs index 7cbeaba4acb..7bf9fc887ce 100644 --- a/crates/vm/src/builtins/type.rs +++ b/crates/vm/src/builtins/type.rs @@ -217,6 +217,24 @@ pub(crate) fn type_cache_clear() { TYPE_CACHE_CLEARING.store(false, Ordering::Release); } +/// Repair type-cache SeqLock state in the post-fork child. +/// +/// If fork happens while a writer holds an entry SeqLock, the child inherits +/// the odd sequence value with no surviving writer to release it. Clear only +/// those in-progress entries, matching CPython's `_PyTypes_AfterFork()`. +pub unsafe fn type_cache_after_fork() { + for entry in TYPE_CACHE.iter() { + let seq = entry.sequence.load(Ordering::Relaxed); + if (seq & 1) == 0 { + continue; + } + entry.value.store(core::ptr::null_mut(), Ordering::Relaxed); + entry.name.store(core::ptr::null_mut(), Ordering::Relaxed); + entry.version.store(0, Ordering::Relaxed); + entry.sequence.store(0, Ordering::Relaxed); + } +} + unsafe impl crate::object::Traverse for PyType { fn traverse(&self, tracer_fn: &mut crate::object::TraverseFn<'_>) { self.base.traverse(tracer_fn); @@ -490,10 +508,28 @@ impl PyType { } pub fn assign_version_tag(&self) -> u32 { - self.assign_version_tag_inner() + let version = self.tp_version_tag.load(Ordering::Acquire); + if version != 0 { + return version; + } + crate::vm::thread::try_with_current_vm(|vm| { + Self::with_type_lock(vm, || { + let version = self.tp_version_tag.load(Ordering::Acquire); + if version == 0 { + self.assign_version_tag_inner() + } else { + version + } + }) + }) + .unwrap_or_else(|| self.assign_version_tag_inner()) } pub(crate) fn version_for_specialization(&self, vm: &VirtualMachine) -> u32 { + let version = self.tp_version_tag.load(Ordering::Acquire); + if version != 0 { + return version; + } Self::with_type_lock(vm, || { let version = self.tp_version_tag.load(Ordering::Acquire); if version == 0 { @@ -506,28 +542,34 @@ impl PyType { /// Invalidate this type's version tag and cascade to all subclasses. fn modified_inner(&self) { - if let Some(ext) = self.heaptype_ext.as_ref() { - ext.specialization_cache.invalidate_for_type_modified(); - } - // If already invalidated, all subclasses must also be invalidated - // (guaranteed by the MRO invariant in assign_version_tag). let old_version = self.tp_version_tag.load(Ordering::Acquire); if old_version == 0 { return; } - self.tp_version_tag.store(0, Ordering::SeqCst); - // Nullify borrowed pointers in cache entries for this version - // so they don't dangle after the dict is modified. - type_cache_clear_version(old_version); let subclasses = self.subclasses.read(); for weak_ref in subclasses.iter() { if let Some(sub) = weak_ref.upgrade() { sub.downcast_ref::().unwrap().modified_inner(); } } + self.tp_version_tag.store(0, Ordering::SeqCst); + // Nullify borrowed pointers in cache entries for this version + // so they don't dangle after the dict is modified. + type_cache_clear_version(old_version); + if let Some(ext) = self.heaptype_ext.as_ref() { + ext.specialization_cache.invalidate_for_type_modified(); + } } pub fn modified(&self) { + if self.tp_version_tag.load(Ordering::Acquire) == 0 { + return; + } + if let Some(()) = crate::vm::thread::try_with_current_vm(|vm| { + Self::with_type_lock(vm, || self.modified_inner()); + }) { + return; + } self.modified_inner(); } @@ -1125,11 +1167,11 @@ impl PyType { if func_version == 0 { return false; } + ext.specialization_cache + .swap_getitem(Some(getitem), Some(vm)); ext.specialization_cache .getitem_version .store(func_version, Ordering::Release); - ext.specialization_cache - .swap_getitem(Some(getitem), Some(vm)); true }) } @@ -1152,18 +1194,7 @@ impl PyType { Some((getitem, cached_version)) } - pub fn get_direct_attr(&self, attr_name: &'static PyStrInterned) -> Option { - self.attributes.read().get(attr_name).cloned() - } - - /// find_name_in_mro with method cache (MCACHE). - /// Looks in tp_dict of types in MRO, bypasses descriptors. - /// - /// Uses a lock-free SeqLock-style pattern: - /// Read: load sequence/version/name → load value + try_to_owned → - /// validate value pointer + sequence - /// Write: sequence(begin) → version=0 → swap value/name → version=assigned → sequence(end) - fn find_name_in_mro(&self, name: &'static PyStrInterned) -> Option { + fn find_name_in_mro_without_vm(&self, name: &'static PyStrInterned) -> Option { let version = self.tp_version_tag.load(Ordering::Acquire); if version != 0 { let idx = type_cache_hash(version, name); @@ -1185,8 +1216,6 @@ impl PyType { } continue; } - // _Py_TryIncrefCompare-style validation: - // safe_inc via raw pointer, then ensure source is unchanged. if let Some(cloned) = unsafe { PyObject::try_to_owned_from_ptr(ptr) } { let same_ptr = core::ptr::eq(entry.value.load(Ordering::Relaxed), ptr); if same_ptr && entry.end_read(seq1) { @@ -1199,20 +1228,12 @@ impl PyType { } } - // Assign version BEFORE the MRO walk so that any concurrent - // modified() call during the walk invalidates this version. let assigned = if version == 0 { self.assign_version_tag() } else { version }; - - // MRO walk let result = self.find_name_in_mro_uncached(name); - - // Only cache positive results. Negative results are not cached to - // avoid stale entries from transient MRO walk failures during - // concurrent type modifications. if let Some(ref found) = result && assigned != 0 && !TYPE_CACHE_CLEARING.load(Ordering::Acquire) @@ -1222,20 +1243,34 @@ impl PyType { let entry = &TYPE_CACHE[idx]; let name_ptr = name as *const _ as *mut _; entry.begin_write(); - // Invalidate first to prevent readers from seeing partial state entry.version.store(0, Ordering::Release); - // Store borrowed pointer (no refcount increment). let new_ptr = &**found as *const PyObject as *mut PyObject; entry.value.store(new_ptr, Ordering::Relaxed); entry.name.store(name_ptr, Ordering::Relaxed); - // Activate entry — Release ensures value/name writes are visible entry.version.store(assigned, Ordering::Release); entry.end_write(); } - result } + pub fn get_direct_attr(&self, attr_name: &'static PyStrInterned) -> Option { + self.attributes.read().get(attr_name).cloned() + } + + /// find_name_in_mro with method cache (MCACHE). + /// Looks in tp_dict of types in MRO, bypasses descriptors. + /// + /// Uses a lock-free SeqLock-style pattern: + /// Read: load sequence/version/name → load value + try_to_owned → + /// validate value pointer + sequence + /// Write: sequence(begin) → version=0 → swap value/name → version=assigned → sequence(end) + fn find_name_in_mro(&self, name: &'static PyStrInterned) -> Option { + crate::vm::thread::try_with_current_vm(|vm| { + self.lookup_ref_and_version_interned(name, vm).0 + }) + .unwrap_or_else(|| self.find_name_in_mro_without_vm(name)) + } + /// Raw MRO walk without cache. fn find_name_in_mro_uncached(&self, name: &'static PyStrInterned) -> Option { for cls in self.mro.read().iter() { @@ -1249,7 +1284,7 @@ impl PyType { /// _PyType_LookupRef: look up a name through the MRO without setting an exception. pub fn lookup_ref(&self, name: &Py, vm: &VirtualMachine) -> Option { let interned_name = vm.ctx.interned_str(name)?; - self.find_name_in_mro(interned_name) + self.lookup_ref_and_version_interned(interned_name, vm).0 } pub fn get_super_attr(&self, attr_name: &'static PyStrInterned) -> Option { @@ -1507,7 +1542,7 @@ impl PyType { // Preserve self (mro[0]) when updating MRO mro.insert(0, cls.mro.read()[0].to_owned()); *cls.mro.write() = mro; - for subclass in cls.subclasses.write().iter() { + for subclass in cls.subclasses.read().iter() { let subclass = subclass.upgrade().unwrap(); let subclass: &Py = subclass.downcast_ref().unwrap(); update_mro_recursively(subclass, vm)?; diff --git a/crates/vm/src/frame.rs b/crates/vm/src/frame.rs index a78e6d37c56..0545b6b2d66 100644 --- a/crates/vm/src/frame.rs +++ b/crates/vm/src/frame.rs @@ -7530,12 +7530,13 @@ impl ExecutingFrame<'_> { .load() .is_some_and(|f| f as usize == PyBaseObject::getattro as *const () as usize); if !is_default_getattro { - let type_version = cls.version_for_specialization(_vm); + let (getattribute, type_version) = + cls.lookup_ref_and_version_interned(identifier!(_vm, __getattribute__), _vm); if type_version != 0 && !oparg.is_method() && !self.specialization_eval_frame_active(_vm) && cls.get_attr(identifier!(_vm, __getattr__)).is_none() - && let Some(getattribute) = cls.get_attr(identifier!(_vm, __getattribute__)) + && let Some(getattribute) = getattribute && let Some(func) = getattribute.downcast_ref_if_exact::(_vm) && func.can_specialize_call(2) { @@ -7567,27 +7568,24 @@ impl ExecutingFrame<'_> { return; } - // Get or assign type version - let type_version = cls.version_for_specialization(_vm); - if type_version == 0 { - // Version counter overflow — backoff to avoid re-attempting every execution - unsafe { - self.code.instructions.write_adaptive_counter( - cache_base, - bytecode::adaptive_counter_backoff( - self.code.instructions.read_adaptive_counter(cache_base), - ), - ); - } - return; - } - let attr_name = self.code.names[oparg.name_idx() as usize]; // Match CPython: only specialize module attribute loads when the // current module dict has no __getattr__ override and the attribute is // already present. if let Some(module) = obj.downcast_ref_if_exact::(_vm) { + let type_version = cls.version_for_specialization(_vm); + if type_version == 0 { + unsafe { + self.code.instructions.write_adaptive_counter( + cache_base, + bytecode::adaptive_counter_backoff( + self.code.instructions.read_adaptive_counter(cache_base), + ), + ); + } + return; + } let module_dict = module.dict(); match ( module_dict.get_item_opt(identifier!(_vm, __getattr__), _vm), @@ -7614,8 +7612,18 @@ impl ExecutingFrame<'_> { return; } - // Look up attr in class via MRO - let cls_attr = cls.get_attr(attr_name); + let (cls_attr, type_version) = cls.lookup_ref_and_version_interned(attr_name, _vm); + if type_version == 0 { + unsafe { + self.code.instructions.write_adaptive_counter( + cache_base, + bytecode::adaptive_counter_backoff( + self.code.instructions.read_adaptive_counter(cache_base), + ), + ); + } + return; + } let class_has_dict = cls.slots.flags.has_feature(PyTypeFlags::HAS_DICT); if oparg.is_method() { @@ -7786,26 +7794,11 @@ impl ExecutingFrame<'_> { ) { let obj = self.top_value(); let owner_type = obj.downcast_ref::().unwrap(); - - // Get or assign type version for the type object itself - let type_version = owner_type.version_for_specialization(_vm); - if type_version == 0 { - unsafe { - self.code.instructions.write_adaptive_counter( - cache_base, - bytecode::adaptive_counter_backoff( - self.code.instructions.read_adaptive_counter(cache_base), - ), - ); - } - return; - } - let attr_name = self.code.names[oparg.name_idx() as usize]; // Check metaclass: ensure no data descriptor on metaclass for this name let mcl = obj.class(); - let mcl_attr = mcl.get_attr(attr_name); + let (mcl_attr, mut metaclass_version) = mcl.lookup_ref_and_version_interned(attr_name, _vm); if let Some(ref attr) = mcl_attr { let attr_class = attr.class(); if attr_class.slots.descr_set.load().is_some() { @@ -7821,9 +7814,7 @@ impl ExecutingFrame<'_> { return; } } - let mut metaclass_version = 0; if !mcl.slots.flags.has_feature(PyTypeFlags::IMMUTABLETYPE) { - metaclass_version = mcl.version_for_specialization(_vm); if metaclass_version == 0 { unsafe { self.code.instructions.write_adaptive_counter( @@ -7835,10 +7826,22 @@ impl ExecutingFrame<'_> { } return; } + } else { + metaclass_version = 0; } - // Look up attr in the type's own MRO - let cls_attr = owner_type.get_attr(attr_name); + let (cls_attr, type_version) = owner_type.lookup_ref_and_version_interned(attr_name, _vm); + if type_version == 0 { + unsafe { + self.code.instructions.write_adaptive_counter( + cache_base, + bytecode::adaptive_counter_backoff( + self.code.instructions.read_adaptive_counter(cache_base), + ), + ); + } + return; + } if let Some(ref descr) = cls_attr { let descr_class = descr.class(); let has_descr_get = descr_class.slots.descr_get.load().is_some(); @@ -9211,8 +9214,8 @@ impl ExecutingFrame<'_> { return; } - // Get or assign type version - let type_version = cls.version_for_specialization(vm); + let attr_name = self.code.names[attr_idx as usize]; + let (cls_attr, type_version) = cls.lookup_ref_and_version_interned(attr_name, vm); if type_version == 0 { unsafe { self.code.instructions.write_adaptive_counter( @@ -9224,10 +9227,6 @@ impl ExecutingFrame<'_> { } return; } - - // Check for data descriptor - let attr_name = self.code.names[attr_idx as usize]; - let cls_attr = cls.get_attr(attr_name); let has_data_descr = cls_attr.as_ref().is_some_and(|descr| { let descr_cls = descr.class(); descr_cls.slots.descr_get.load().is_some() && descr_cls.slots.descr_set.load().is_some() diff --git a/crates/vm/src/stdlib/posix.rs b/crates/vm/src/stdlib/posix.rs index e4fd42e64cf..0eaad949f15 100644 --- a/crates/vm/src/stdlib/posix.rs +++ b/crates/vm/src/stdlib/posix.rs @@ -655,6 +655,9 @@ pub mod module { #[cfg(feature = "threading")] crate::object::reset_weakref_locks_after_fork(); + // Repair any type-cache entries left mid-update at fork time. + unsafe { crate::builtins::type_::type_cache_after_fork() }; + // Phase 3: Clean up thread state. Locks are now reinit'd so we can // acquire them normally instead of using try_lock(). #[cfg(feature = "threading")] diff --git a/crates/vm/src/vm/thread.rs b/crates/vm/src/vm/thread.rs index 5e73cc5f618..9bc82006608 100644 --- a/crates/vm/src/vm/thread.rs +++ b/crates/vm/src/vm/thread.rs @@ -109,6 +109,15 @@ fn set_current_vm(vm: &VirtualMachine, f: impl FnOnce() -> R) -> R { }) } +pub fn try_with_current_vm(f: impl FnOnce(&VirtualMachine) -> R) -> Option { + VM_STACK.with(|vms| { + let vm = vms.borrow().last().copied()?; + // SAFETY: entries in VM_STACK either borrow a VM for the dynamic + // scope of a set_current_vm()/enter_vm() call or point at GILSTATE_VM. + Some(f(unsafe { vm.as_ref() })) + }) +} + pub fn enter_vm(vm: &VirtualMachine, f: impl FnOnce() -> R) -> R { // Outermost enter_vm: transition DETACHED → ATTACHED #[cfg(all(unix, feature = "threading"))] From 9196dd4bae852ba4dc692b14bc434b5c3d09854b Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Sun, 5 Jul 2026 18:35:32 +0900 Subject: [PATCH 04/92] Add PUBLISHED flag bit to RefCount state word Assisted-by: Claude --- crates/common/src/refcount.rs | 42 ++++++++++++++++++++++++++++++++--- 1 file changed, 39 insertions(+), 3 deletions(-) diff --git a/crates/common/src/refcount.rs b/crates/common/src/refcount.rs index c589ead40f6..4d52e1382e6 100644 --- a/crates/common/src/refcount.rs +++ b/crates/common/src/refcount.rs @@ -1,10 +1,14 @@ use crate::atomic::{Ordering, PyAtomic, Radium}; // State layout (usize): -// [1 bit: destructed] [1 bit: reserved] [1 bit: leaked] [N bits: weak_count] [M bits: strong_count] +// [1 bit: destructed] [1 bit: published] [1 bit: leaked] [N bits: weak_count] [M bits: strong_count] // 64-bit: N=30, M=31. 32-bit: N=14, M=15. const FLAG_BITS: u32 = 3; const DESTRUCTED: usize = 1 << (usize::BITS - 1); +/// Object was published to a lock-free cache; memory reclamation is +/// deferred through QSBR so concurrent try-incref readers never touch +/// freed memory. Sticky once set. +const PUBLISHED: usize = 1 << (usize::BITS - 2); const LEAKED: usize = 1 << (usize::BITS - 3); const TOTAL_COUNT_WIDTH: u32 = usize::BITS - FLAG_BITS; const WEAK_WIDTH: u32 = TOTAL_COUNT_WIDTH / 2; @@ -72,8 +76,8 @@ impl State { /// Reference count using state layout with LEAKED support. /// /// State layout (usize): -/// 64-bit: [1 bit: destructed] [1 bit: reserved] [1 bit: leaked] [30 bits: weak_count] [31 bits: strong_count] -/// 32-bit: [1 bit: destructed] [1 bit: reserved] [1 bit: leaked] [14 bits: weak_count] [15 bits: strong_count] +/// 64-bit: [1 bit: destructed] [1 bit: published] [1 bit: leaked] [30 bits: weak_count] [31 bits: strong_count] +/// 32-bit: [1 bit: destructed] [1 bit: published] [1 bit: leaked] [14 bits: weak_count] [15 bits: strong_count] pub struct RefCount { state: PyAtomic, } @@ -187,6 +191,17 @@ impl RefCount { pub fn is_leaked(&self) -> bool { State::from_raw(self.state.load(Ordering::Acquire)).leaked() } + + /// Mark the object as published to a lock-free cache (sticky). + #[inline] + pub fn mark_published(&self) { + self.state.fetch_or(PUBLISHED, Ordering::Release); + } + + #[inline] + pub fn is_published(&self) -> bool { + (self.state.load(Ordering::Acquire) & PUBLISHED) != 0 + } } // Deferred Drop Infrastructure @@ -279,3 +294,24 @@ pub fn flush_deferred_drops() { } }); } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn published_bit_survives_refcount_traffic() { + let rc = RefCount::new(); // strong = 1 + assert!(!rc.is_published()); + rc.mark_published(); + assert!(rc.is_published()); + rc.inc(); // strong = 2 + assert!(rc.is_published()); + assert!(!rc.dec()); // strong = 1 + assert!(rc.is_published()); + assert!(rc.safe_inc()); // strong = 2 + assert!(!rc.dec()); // strong = 1 + assert!(rc.dec()); // strong = 0 -> true + assert!(rc.is_published()); + } +} From c32cb0e3693ea4d409b593d73a326ef3ee9146c5 Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Sun, 5 Jul 2026 18:42:07 +0900 Subject: [PATCH 05/92] Add QSBR module for deferred memory reclamation Assisted-by: Claude --- crates/vm/src/object/mod.rs | 1 + crates/vm/src/object/qsbr.rs | 266 +++++++++++++++++++++++++++++++++++ 2 files changed, 267 insertions(+) create mode 100644 crates/vm/src/object/qsbr.rs diff --git a/crates/vm/src/object/mod.rs b/crates/vm/src/object/mod.rs index 56db97aef1d..3d4f0fc9a3b 100644 --- a/crates/vm/src/object/mod.rs +++ b/crates/vm/src/object/mod.rs @@ -3,6 +3,7 @@ mod ext; mod payload; mod traverse; mod traverse_object; +pub(crate) mod qsbr; pub use self::core::*; pub use self::ext::*; diff --git a/crates/vm/src/object/qsbr.rs b/crates/vm/src/object/qsbr.rs new file mode 100644 index 00000000000..a88c6b88b72 --- /dev/null +++ b/crates/vm/src/object/qsbr.rs @@ -0,0 +1,266 @@ +//! Quiescent-state-based reclamation (QSBR) for lock-free caches. +//! +//! Objects published to lock-free caches (type method cache, type +//! specialization caches) are read via borrowed pointers plus try-incref. +//! Their memory must stay mapped until every thread that could hold such a +//! borrowed pointer has passed a quiescent state. Destructors run at the +//! normal drop point; only the final deallocation is deferred. +//! +//! Mirrors _Py_qsbr (Python/qsbr.c): a global write sequence advances on +//! each retirement; each thread records the last sequence it observed at a +//! quiescent point (eval-breaker checkpoint, attach/detach). A retired +//! allocation is freed once every online thread's sequence passes its goal. + +use core::alloc::Layout; + +/// Sequence value of an offline (detached) thread. +#[cfg(feature = "threading")] +const QSBR_OFFLINE: u64 = 0; +/// Initial write sequence value. +#[cfg(feature = "threading")] +const QSBR_INITIAL: u64 = 1; +/// Write sequence increment. +#[cfg(feature = "threading")] +const QSBR_INCR: u64 = 2; + +#[cfg(feature = "threading")] +pub use threading::*; + +#[cfg(feature = "threading")] +mod threading { + use super::*; + use core::sync::atomic::{AtomicBool, AtomicU64, Ordering}; + use std::sync::{Arc, Mutex, Weak}; + + /// Per-thread QSBR state, owned by the thread's `ThreadSlot`. + pub struct QsbrSlot { + /// Last write sequence observed at a quiescent point; + /// `QSBR_OFFLINE` while the thread is detached. + seq: AtomicU64, + /// Set when this thread should pass a checkpoint (eval-breaker bit). + pub requested: AtomicBool, + } + + struct Retired { + ptr: *mut u8, + layout: Layout, + goal: u64, + } + // SAFETY: `ptr` is an exclusively owned dead allocation; only the + // processing thread touches it. + unsafe impl Send for Retired {} + + pub struct Qsbr { + /// Global write sequence (_Py_qsbr wr_seq). + wr_seq: AtomicU64, + /// Cached minimum observed read sequence (rd_seq). + rd_seq: AtomicU64, + threads: Mutex>>, + queue: Mutex>, + } + + pub static QSBR: Qsbr = Qsbr::new(); + + impl Qsbr { + const fn new() -> Self { + Self { + wr_seq: AtomicU64::new(QSBR_INITIAL), + rd_seq: AtomicU64::new(QSBR_INITIAL), + threads: Mutex::new(Vec::new()), + queue: Mutex::new(Vec::new()), + } + } + + /// Register the calling thread. The returned slot is stored in the + /// thread's `ThreadSlot`; dropping it unregisters the thread. + pub fn register(&self) -> Arc { + let slot = Arc::new(QsbrSlot { + seq: AtomicU64::new(self.wr_seq.load(Ordering::Acquire)), + requested: AtomicBool::new(false), + }); + self.threads.lock().unwrap().push(Arc::downgrade(&slot)); + slot + } + + /// Advance the write sequence; returns the goal a retirement must + /// wait for (_Py_qsbr_advance). + fn advance(&self) -> u64 { + self.wr_seq.fetch_add(QSBR_INCR, Ordering::AcqRel) + QSBR_INCR + } + + /// Record that the calling thread is at a quiescent point: it holds + /// no borrowed cache pointers (_Py_qsbr_quiescent_state). + pub fn quiescent_state(&self, slot: &QsbrSlot) { + slot.seq + .store(self.wr_seq.load(Ordering::Acquire), Ordering::Release); + } + + /// Mark a thread offline (detached); it no longer delays grace + /// periods (_Py_qsbr_detach). The thread must not perform lock-free + /// cache reads while offline. + pub fn offline(&self, slot: &QsbrSlot) { + slot.seq.store(QSBR_OFFLINE, Ordering::Release); + } + + /// Mark a thread online again (_Py_qsbr_attach). + pub fn online(&self, slot: &QsbrSlot) { + self.quiescent_state(slot); + } + + /// Whether every online thread has passed `goal` (_Py_qsbr_poll). + fn poll(&self, goal: u64) -> bool { + if self.rd_seq.load(Ordering::Acquire) >= goal { + return true; + } + self.poll_scan() >= goal + } + + /// Recompute the minimum sequence over all live online threads, + /// pruning dead ones. + fn poll_scan(&self) -> u64 { + let mut min_seq = self.wr_seq.load(Ordering::Acquire); + let mut threads = self.threads.lock().unwrap(); + threads.retain(|weak| match weak.upgrade() { + Some(slot) => { + let seq = slot.seq.load(Ordering::Acquire); + if seq != QSBR_OFFLINE { + min_seq = min_seq.min(seq); + } + true + } + None => false, + }); + drop(threads); + self.rd_seq.fetch_max(min_seq, Ordering::AcqRel); + min_seq + } + + /// Defer deallocation of a dead object's memory until a grace + /// period passes (_PyMem_FreeDelayed). + /// + /// # Safety + /// `ptr`/`layout` must describe an allocation whose contents have + /// been dropped and which nothing accesses afterwards except the + /// racing try-incref reads this mechanism protects against. + pub unsafe fn free_delayed(&self, ptr: *mut u8, layout: Layout) { + let goal = self.advance(); + self.queue.lock().unwrap().push(Retired { ptr, layout, goal }); + // Ask every registered thread to pass a checkpoint. + for weak in self.threads.lock().unwrap().iter() { + if let Some(slot) = weak.upgrade() { + slot.requested.store(true, Ordering::Release); + } + } + } + + /// Free retired allocations whose grace period has passed + /// (_PyMem_ProcessDelayed). + pub fn process(&self) { + let Ok(mut queue) = self.queue.try_lock() else { + // Another thread is already processing. + return; + }; + // Goals are monotonically increasing in push order: free the + // prefix whose grace period has passed. + let safe_prefix = queue + .iter() + .position(|item| !self.poll(item.goal)) + .unwrap_or(queue.len()); + for item in queue.drain(..safe_prefix) { + // SAFETY: grace period passed; no reader can hold `ptr`. + unsafe { alloc::alloc::dealloc(item.ptr, item.layout) }; + } + } + + /// Free all retired allocations immediately. + /// + /// # Safety + /// Only sound when no other thread can be mid-read: the post-fork + /// child, or teardown after all threads exited. + pub unsafe fn drain_all(&self) { + let mut queue = self.queue.lock().unwrap(); + for item in queue.drain(..) { + // SAFETY: guaranteed single-threaded by the caller. + unsafe { alloc::alloc::dealloc(item.ptr, item.layout) }; + } + } + + #[cfg(test)] + fn pending(&self) -> usize { + self.queue.lock().unwrap().len() + } + } + + #[cfg(test)] + mod tests { + use super::*; + + #[test] + fn poll_requires_all_online_threads() { + let q = Qsbr::new(); + let a = q.register(); + let b = q.register(); + let goal = q.advance(); + assert!(!q.poll(goal)); + q.quiescent_state(&a); + assert!(!q.poll(goal)); + q.quiescent_state(&b); + assert!(q.poll(goal)); + } + + #[test] + fn offline_thread_does_not_delay_grace() { + let q = Qsbr::new(); + let a = q.register(); + let b = q.register(); + let goal = q.advance(); + q.quiescent_state(&a); + q.offline(&b); + assert!(q.poll(goal)); + } + + #[test] + fn dead_thread_is_pruned() { + let q = Qsbr::new(); + let a = q.register(); + let b = q.register(); + drop(b); + let goal = q.advance(); + q.quiescent_state(&a); + assert!(q.poll(goal)); + } + + #[test] + fn process_frees_only_after_grace() { + let q = Qsbr::new(); + let a = q.register(); + let layout = Layout::new::(); + let ptr = unsafe { alloc::alloc::alloc(layout) }; + unsafe { q.free_delayed(ptr, layout) }; + assert!(a.requested.load(Ordering::Acquire)); + q.process(); + assert_eq!(q.pending(), 1); // grace period not passed yet + q.quiescent_state(&a); + q.process(); + assert_eq!(q.pending(), 0); + } + } +} + +/// Defer (threading) or immediately perform (non-threading) deallocation +/// of a dead published object's memory. +/// +/// # Safety +/// Same contract as [`Qsbr::free_delayed`]. +#[inline] +pub(crate) unsafe fn free_delayed(ptr: *mut u8, layout: Layout) { + #[cfg(feature = "threading")] + unsafe { + QSBR.free_delayed(ptr, layout) + }; + #[cfg(not(feature = "threading"))] + // No concurrent readers can exist without threads. + unsafe { + alloc::alloc::dealloc(ptr, layout) + }; +} From 9bfa0701cf6498c738b93a9680f1bcabdf2a7a5f Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Sun, 5 Jul 2026 18:51:00 +0900 Subject: [PATCH 06/92] Defer memory free of cache-published objects via QSBR Assisted-by: Claude --- crates/vm/src/object/core.rs | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/crates/vm/src/object/core.rs b/crates/vm/src/object/core.rs index 88ca646a4f1..49c63d8516a 100644 --- a/crates/vm/src/object/core.rs +++ b/crates/vm/src/object/core.rs @@ -1005,6 +1005,9 @@ impl PyInner { let has_ext = flags.has_feature(crate::types::PyTypeFlags::HAS_DICT) || member_count > 0; let has_weakref = flags.has_feature(crate::types::PyTypeFlags::HAS_WEAKREF); + // Objects published to lock-free caches keep their memory mapped + // until a QSBR grace period passes; destructors still run now. + let published = (*ptr).ref_count.is_published(); if has_ext || has_weakref { // Reconstruct the same layout used in new() @@ -1037,7 +1040,15 @@ impl PyInner { } // WeakRefList has no Drop (just raw pointers), no drop_in_place needed - alloc::alloc::dealloc(alloc_ptr, combined); + if published { + crate::object::qsbr::free_delayed(alloc_ptr, combined); + } else { + alloc::alloc::dealloc(alloc_ptr, combined); + } + } else if published { + let layout = core::alloc::Layout::new::(); + core::ptr::drop_in_place(ptr); + crate::object::qsbr::free_delayed(ptr as *mut u8, layout); } else { drop(Box::from_raw(ptr)); } @@ -1287,6 +1298,13 @@ impl PyObject { None } } + + /// Mark this object as published to a lock-free cache. Its memory + /// reclamation is deferred through QSBR (see `object::qsbr`) so that + /// concurrent try-incref readers never touch freed memory. + pub(crate) fn mark_cache_published(&self) { + self.0.ref_count.mark_published(); + } } impl PyObjectRef { From 59a31276fc313ede19a8edf2c095626f983ec859 Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Sun, 5 Jul 2026 18:58:26 +0900 Subject: [PATCH 07/92] Wire QSBR checkpoints into thread lifecycle and eval breaker Assisted-by: Claude --- crates/vm/src/gc_state.rs | 4 +++ crates/vm/src/vm/interpreter.rs | 11 +++++-- crates/vm/src/vm/mod.rs | 11 +++++++ crates/vm/src/vm/thread.rs | 55 ++++++++++++++++++++++++++++++++- 4 files changed, 78 insertions(+), 3 deletions(-) diff --git a/crates/vm/src/gc_state.rs b/crates/vm/src/gc_state.rs index e20cbcb8ecf..ef6b8eed2e1 100644 --- a/crates/vm/src/gc_state.rs +++ b/crates/vm/src/gc_state.rs @@ -409,6 +409,10 @@ impl GcState { // might prevent cycle collection (_PyType_ClearCache). crate::builtins::type_::type_cache_clear(); + // Backstop for QSBR reclamation (threads may have missed requests). + #[cfg(feature = "threading")] + crate::object::qsbr::QSBR.process(); + // Step 1: Gather objects from generations 0..=generation // Hold read locks for the entire scan to prevent concurrent modifications. let gen_locks: Vec<_> = (0..=generation) diff --git a/crates/vm/src/vm/interpreter.rs b/crates/vm/src/vm/interpreter.rs index 29cfcef6783..31aeb42a22d 100644 --- a/crates/vm/src/vm/interpreter.rs +++ b/crates/vm/src/vm/interpreter.rs @@ -462,11 +462,18 @@ impl Interpreter { } // Match CPython: if exit_code is 0 and stdout flush failed, exit 120 - if exit_code == 0 && flush_status < 0 { + let exit_code = if exit_code == 0 && flush_status < 0 { EXITCODE_FLUSH_FAILURE } else { exit_code - } + }; + + // Daemon threads may still exist, so use the safe `process()`, + // not `drain_all()`. + #[cfg(feature = "threading")] + crate::object::qsbr::QSBR.process(); + + exit_code }) } } diff --git a/crates/vm/src/vm/mod.rs b/crates/vm/src/vm/mod.rs index 4c32f616042..3e40d235809 100644 --- a/crates/vm/src/vm/mod.rs +++ b/crates/vm/src/vm/mod.rs @@ -2041,6 +2041,11 @@ impl VirtualMachine { return true; } + #[cfg(feature = "threading")] + if thread::qsbr_break_requested() { + return true; + } + #[cfg(not(target_arch = "wasm32"))] if crate::signal::is_triggered() { return true; @@ -2064,6 +2069,12 @@ impl VirtualMachine { #[cfg(all(unix, feature = "threading"))] thread::suspend_if_needed(&self.state.stop_the_world); + // Pass a QSBR checkpoint if requested (deferred memory reclamation). + #[cfg(feature = "threading")] + if thread::qsbr_break_requested() { + thread::qsbr_checkpoint(); + } + #[cfg(not(target_arch = "wasm32"))] crate::signal::check_signals(self)?; diff --git a/crates/vm/src/vm/thread.rs b/crates/vm/src/vm/thread.rs index 9bc82006608..a1c920af916 100644 --- a/crates/vm/src/vm/thread.rs +++ b/crates/vm/src/vm/thread.rs @@ -43,6 +43,11 @@ pub struct ThreadSlot { /// Handle for waking this thread from park in stop-the-world paths. #[cfg(unix)] pub thread: std::thread::Thread, + /// QSBR state for deferred memory reclamation. On non-unix threading + /// builds there is no attach/detach state machine, so the slot stays + /// online from registration to thread exit; a thread blocked in native + /// code simply delays reclamation until it runs again. + pub qsbr: Arc, } #[cfg(feature = "threading")] @@ -226,6 +231,7 @@ fn init_thread_slot_if_needed(vm: &VirtualMachine) { stop_requested: core::sync::atomic::AtomicBool::new(false), #[cfg(unix)] thread: std::thread::current(), + qsbr: crate::object::qsbr::QSBR.register(), }); registry.insert(thread_id, new_slot.clone()); drop(registry); @@ -259,6 +265,7 @@ fn attach_thread(vm: &VirtualMachine) { Ordering::Relaxed, ) { Ok(_) => { + crate::object::qsbr::QSBR.online(&s.qsbr); super::stw_trace(format_args!("attach DETACHED->ATTACHED")); break; } @@ -290,7 +297,9 @@ fn detach_thread() { Ordering::AcqRel, Ordering::Acquire, ) { - Ok(_) => {} + Ok(_) => { + crate::object::qsbr::QSBR.offline(&s.qsbr); + } Err(THREAD_DETACHED) => { debug_assert!(false, "detach called while already DETACHED"); return; @@ -451,6 +460,49 @@ pub fn stop_requested_for_current_thread() -> bool { }) } +/// Whether the QSBR subsystem asked this thread to pass a checkpoint. +/// A missed or racing read of this flag is harmless: the pending +/// retirement is still processed at the next checkpoint or by the GC +/// backstop. +#[cfg(feature = "threading")] +pub(crate) fn qsbr_break_requested() -> bool { + CURRENT_THREAD_SLOT.with(|slot| { + slot.borrow() + .as_ref() + .is_some_and(|s| s.qsbr.requested.load(Ordering::Relaxed)) + }) +} + +/// Pass a QSBR checkpoint: the calling thread holds no borrowed cache +/// pointers here (instruction boundary), so mark it quiescent and try to +/// free retired allocations. +#[cfg(feature = "threading")] +pub(crate) fn qsbr_checkpoint() { + use crate::object::qsbr::QSBR; + CURRENT_THREAD_SLOT.with(|slot| { + if let Some(s) = slot.borrow().as_ref() { + s.qsbr.requested.store(false, Ordering::Relaxed); + QSBR.quiescent_state(&s.qsbr); + } + }); + QSBR.process(); +} + +/// Debug check: lock-free type-cache reads are only sound on threads that +/// are registered with QSBR and currently ATTACHED. +#[cfg(all(unix, feature = "threading"))] +pub(crate) fn debug_assert_current_thread_attached() { + CURRENT_THREAD_SLOT.with(|slot| { + if let Some(s) = slot.borrow().as_ref() { + debug_assert_eq!( + s.state.load(Ordering::Relaxed), + THREAD_ATTACHED, + "type cache read while thread not ATTACHED" + ); + } + }); +} + /// Push a frame pointer onto the current thread's shared frame stack. /// The pointed-to frame must remain alive until the matching pop. #[cfg(feature = "threading")] @@ -584,6 +636,7 @@ pub fn reinit_frame_slot_after_fork(vm: &VirtualMachine) { stop_requested: core::sync::atomic::AtomicBool::new(false), #[cfg(unix)] thread: std::thread::current(), + qsbr: crate::object::qsbr::QSBR.register(), }); // Lock is safe: reinit_locks_after_fork() already reset it to unlocked. From 3f93b63bdb5a2369aef767688a006fdf1de18683 Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Sun, 5 Jul 2026 19:21:16 +0900 Subject: [PATCH 08/92] Publish type-cache values to QSBR and bypass cache without a VM Mark cached type-method values as QSBR-published before storing their pointer in TYPE_CACHE, so racing readers never try-incref freed memory. Debug-assert that the current thread is ATTACHED whenever a lock-free cache read happens. Delete find_name_in_mro_without_vm and its unsound direct SeqLock read outside any VM/thread registration; find_name_in_mro now falls back to the locked, uncached MRO walk when no VM is current. This also removes assign_version_tag()'s last caller, so its unlocked fallback is deleted along with it (version_for_specialization keeps the locked path). VirtualMachine::initialize() runs Python bytecode (e.g. importing codecs/encodings) before any enter_vm scope exists, which left the bootstrap thread not ATTACHED for cache reads during startup. Add thread::VmBootstrapGuard, an RAII counterpart to enter_vm usable across statements that need &mut VirtualMachine, and wrap initialize() with it. Assisted-by: Claude --- crates/vm/src/builtins/type.rs | 102 ++++++-------------------------- crates/vm/src/vm/interpreter.rs | 5 ++ crates/vm/src/vm/thread.rs | 55 +++++++++++++++++ 3 files changed, 77 insertions(+), 85 deletions(-) diff --git a/crates/vm/src/builtins/type.rs b/crates/vm/src/builtins/type.rs index 7bf9fc887ce..d489d439748 100644 --- a/crates/vm/src/builtins/type.rs +++ b/crates/vm/src/builtins/type.rs @@ -57,7 +57,7 @@ pub struct PyType { } /// Monotonic counter for type version tags. Once it reaches `u32::MAX`, -/// `assign_version_tag()` returns 0 permanently, disabling new inline-cache +/// version assignment returns 0 permanently, disabling new inline-cache /// entries but not invalidating correctness (cache misses fall back to the /// generic path). static NEXT_TYPE_VERSION: AtomicU32 = AtomicU32::new(1); @@ -507,24 +507,6 @@ impl PyType { } } - pub fn assign_version_tag(&self) -> u32 { - let version = self.tp_version_tag.load(Ordering::Acquire); - if version != 0 { - return version; - } - crate::vm::thread::try_with_current_vm(|vm| { - Self::with_type_lock(vm, || { - let version = self.tp_version_tag.load(Ordering::Acquire); - if version == 0 { - self.assign_version_tag_inner() - } else { - version - } - }) - }) - .unwrap_or_else(|| self.assign_version_tag_inner()) - } - pub(crate) fn version_for_specialization(&self, vm: &VirtualMachine) -> u32 { let version = self.tp_version_tag.load(Ordering::Acquire); if version != 0 { @@ -1034,13 +1016,21 @@ impl PyType { self.find_name_in_mro(attr_name) } - /// CPython-style `_PyType_LookupRefAndVersion` equivalent for interned names. + /// `_PyType_LookupRefAndVersion` equivalent for interned names. /// Returns the observed lookup result and the type version used for the lookup. + /// + /// Uses a lock-free SeqLock-style pattern: + /// Read: load sequence/version/name → load value + try_to_owned → + /// validate value pointer + sequence + /// Write: sequence(begin) → version=0 → swap value/name → version=assigned → sequence(end) pub(crate) fn lookup_ref_and_version_interned( &self, name: &'static PyStrInterned, vm: &VirtualMachine, ) -> (Option, u32) { + #[cfg(all(unix, feature = "threading", debug_assertions))] + crate::vm::thread::debug_assert_current_thread_attached(); + let version = self.tp_version_tag.load(Ordering::Acquire); if version != 0 { let idx = type_cache_hash(version, name); @@ -1091,6 +1081,9 @@ impl PyType { entry.begin_write(); entry.version.store(0, Ordering::Release); let new_ptr = result.as_ref().map_or(core::ptr::null_mut(), |found| { + // Defer memory reclamation of cached values via QSBR so + // racing readers never try-incref freed memory. + found.mark_cache_published(); &**found as *const PyObject as *mut _ }); entry.value.store(new_ptr, Ordering::Relaxed); @@ -1194,81 +1187,20 @@ impl PyType { Some((getitem, cached_version)) } - fn find_name_in_mro_without_vm(&self, name: &'static PyStrInterned) -> Option { - let version = self.tp_version_tag.load(Ordering::Acquire); - if version != 0 { - let idx = type_cache_hash(version, name); - let entry = &TYPE_CACHE[idx]; - let name_ptr = name as *const _ as *mut _; - loop { - let seq1 = entry.begin_read(); - let v1 = entry.version.load(Ordering::Acquire); - let type_version = self.tp_version_tag.load(Ordering::Acquire); - if v1 != type_version - || !core::ptr::eq(entry.name.load(Ordering::Relaxed), name_ptr) - { - break; - } - let ptr = entry.value.load(Ordering::Acquire); - if ptr.is_null() { - if entry.end_read(seq1) { - break; - } - continue; - } - if let Some(cloned) = unsafe { PyObject::try_to_owned_from_ptr(ptr) } { - let same_ptr = core::ptr::eq(entry.value.load(Ordering::Relaxed), ptr); - if same_ptr && entry.end_read(seq1) { - return Some(cloned); - } - drop(cloned); - continue; - } - break; - } - } - - let assigned = if version == 0 { - self.assign_version_tag() - } else { - version - }; - let result = self.find_name_in_mro_uncached(name); - if let Some(ref found) = result - && assigned != 0 - && !TYPE_CACHE_CLEARING.load(Ordering::Acquire) - && self.tp_version_tag.load(Ordering::Acquire) == assigned - { - let idx = type_cache_hash(assigned, name); - let entry = &TYPE_CACHE[idx]; - let name_ptr = name as *const _ as *mut _; - entry.begin_write(); - entry.version.store(0, Ordering::Release); - let new_ptr = &**found as *const PyObject as *mut PyObject; - entry.value.store(new_ptr, Ordering::Relaxed); - entry.name.store(name_ptr, Ordering::Relaxed); - entry.version.store(assigned, Ordering::Release); - entry.end_write(); - } - result - } - pub fn get_direct_attr(&self, attr_name: &'static PyStrInterned) -> Option { self.attributes.read().get(attr_name).cloned() } /// find_name_in_mro with method cache (MCACHE). /// Looks in tp_dict of types in MRO, bypasses descriptors. - /// - /// Uses a lock-free SeqLock-style pattern: - /// Read: load sequence/version/name → load value + try_to_owned → - /// validate value pointer + sequence - /// Write: sequence(begin) → version=0 → swap value/name → version=assigned → sequence(end) fn find_name_in_mro(&self, name: &'static PyStrInterned) -> Option { crate::vm::thread::try_with_current_vm(|vm| { self.lookup_ref_and_version_interned(name, vm).0 }) - .unwrap_or_else(|| self.find_name_in_mro_without_vm(name)) + // No current VM: this thread is not registered for QSBR, so the + // lock-free cache read protocol is not sound here. Walk the MRO + // under the attributes locks instead (the dicts hold strong refs). + .unwrap_or_else(|| self.find_name_in_mro_uncached(name)) } /// Raw MRO walk without cache. diff --git a/crates/vm/src/vm/interpreter.rs b/crates/vm/src/vm/interpreter.rs index 31aeb42a22d..43d9560244e 100644 --- a/crates/vm/src/vm/interpreter.rs +++ b/crates/vm/src/vm/interpreter.rs @@ -150,7 +150,12 @@ where // Call custom init function (can mutate vm.state) init(&mut vm); + // `initialize()` runs Python bytecode directly (e.g. importing `codecs` + // and `encodings`) before any `enter_vm` scope exists, so attach this + // thread for the duration so type cache reads see it as ATTACHED. + let vm_guard = thread::VmBootstrapGuard::new(&vm); vm.initialize(); + drop(vm_guard); // Clone global_state for Interpreter after all initialization is done let global_state = vm.state.clone(); diff --git a/crates/vm/src/vm/thread.rs b/crates/vm/src/vm/thread.rs index a1c920af916..91efa384e4d 100644 --- a/crates/vm/src/vm/thread.rs +++ b/crates/vm/src/vm/thread.rs @@ -148,6 +148,61 @@ pub fn enter_vm(vm: &VirtualMachine, f: impl FnOnce() -> R) -> R { set_current_vm(vm, f) } +/// RAII counterpart to `enter_vm`, for code that runs Python bytecode across +/// several statements interspersed with `&mut VirtualMachine` calls +/// (`VirtualMachine::initialize`), where a single closure-based `enter_vm` +/// scope cannot be expressed because the borrow checker won't let a closure +/// hold `&mut VirtualMachine` at the same time `enter_vm` reborrows it as +/// `&VirtualMachine`. Construction only needs a transient `&VirtualMachine` +/// borrow, so it can be dropped before subsequent `&mut` use. +/// +/// Without this, code that runs Python bytecode before any `enter_vm` scope +/// exists would leave the thread not ATTACHED, making lock-free type cache +/// reads unsound. +#[must_use] +pub(crate) struct VmBootstrapGuard { + #[cfg(all(unix, feature = "threading"))] + was_outermost: bool, +} + +impl VmBootstrapGuard { + pub(crate) fn new(vm: &VirtualMachine) -> Self { + // Outermost: transition DETACHED → ATTACHED + #[cfg(all(unix, feature = "threading"))] + let was_outermost = !current_vm_is_set(); + + // Initialize thread slot for this thread if not already done + #[cfg(feature = "threading")] + init_thread_slot_if_needed(vm); + + #[cfg(all(unix, feature = "threading"))] + if was_outermost { + attach_thread(vm); + } + + VM_STACK.with(|vms| vms.borrow_mut().push(vm.into())); + + Self { + #[cfg(all(unix, feature = "threading"))] + was_outermost, + } + } +} + +impl Drop for VmBootstrapGuard { + fn drop(&mut self) { + VM_STACK.with(|vms| { + vms.borrow_mut().pop(); + }); + + // Outermost exit: transition ATTACHED → DETACHED + #[cfg(all(unix, feature = "threading"))] + if self.was_outermost { + detach_thread(); + } + } +} + #[cfg(feature = "threading")] #[derive(Clone, Copy, Debug, Eq, PartialEq)] pub enum CurrentVmAttachState { From 06871c432e4d98c500eb3ec9899367d9b434fbf1 Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Sun, 5 Jul 2026 19:31:42 +0900 Subject: [PATCH 09/92] Use try-incref reads and QSBR-backed swaps in specialization cache Assisted-by: Claude --- crates/vm/src/builtins/type.rs | 75 ++++++++++++++-------------------- crates/vm/src/object/ext.rs | 29 +++++++++++++ 2 files changed, 59 insertions(+), 45 deletions(-) diff --git a/crates/vm/src/builtins/type.rs b/crates/vm/src/builtins/type.rs index d489d439748..800b658303d 100644 --- a/crates/vm/src/builtins/type.rs +++ b/crates/vm/src/builtins/type.rs @@ -295,7 +295,6 @@ pub struct TypeSpecializationCache { pub init: PyAtomicRef>, pub getitem: PyAtomicRef>, pub getitem_version: AtomicU32, - retired: PyRwLock>, } impl TypeSpecializationCache { @@ -304,48 +303,40 @@ impl TypeSpecializationCache { init: PyAtomicRef::from(None::>), getitem: PyAtomicRef::from(None::>), getitem_version: AtomicU32::new(0), - retired: PyRwLock::new(Vec::new()), } } #[inline] - fn retire_old_function(&self, old: Option>) { - if let Some(old) = old { - self.retired.write().push(old.into()); - } - } - - #[inline] - fn swap_init(&self, new_init: Option>, vm: Option<&VirtualMachine>) { - if let Some(vm) = vm { - // Keep replaced refs alive for the currently executing frame, matching - // CPython-style "old pointer remains valid during ongoing execution" - // without accumulating global retired refs. - self.init.swap_to_temporary_refs(new_init, vm); - return; + fn swap_init(&self, new_init: Option>) { + if let Some(new) = &new_init { + new.as_object().mark_cache_published(); } - // SAFETY: old value is moved to `retired`, so it stays alive while - // concurrent readers may still hold borrowed references. + // SAFETY: reclamation of published objects is deferred via QSBR; + // racing try_to_owned readers never touch freed memory. let old = unsafe { self.init.swap(new_init) }; - self.retire_old_function(old); + if let Some(old) = old { + // Dropping may run arbitrary Python; defer past the type lock. + rustpython_common::refcount::try_defer_drop(move || drop(old)); + } } #[inline] - fn swap_getitem(&self, new_getitem: Option>, vm: Option<&VirtualMachine>) { - if let Some(vm) = vm { - self.getitem.swap_to_temporary_refs(new_getitem, vm); - return; + fn swap_getitem(&self, new_getitem: Option>) { + if let Some(new) = &new_getitem { + new.as_object().mark_cache_published(); } - // SAFETY: old value is moved to `retired`, so it stays alive while - // concurrent readers may still hold borrowed references. + // SAFETY: as in swap_init. let old = unsafe { self.getitem.swap(new_getitem) }; - self.retire_old_function(old); + if let Some(old) = old { + rustpython_common::refcount::try_defer_drop(move || drop(old)); + } } #[inline] fn invalidate_for_type_modified(&self) { - self.swap_init(None, None); - self.swap_getitem(None, None); + self.swap_init(None); + self.swap_getitem(None); + self.getitem_version.store(0, Ordering::Release); } fn traverse(&self, tracer_fn: &mut TraverseFn<'_>) { @@ -355,11 +346,6 @@ impl TypeSpecializationCache { if let Some(getitem) = self.getitem.deref() { tracer_fn(getitem.as_object()); } - self.retired - .read() - .iter() - .map(|obj| obj.traverse(tracer_fn)) - .count(); } fn clear_into(&self, out: &mut Vec) { @@ -372,7 +358,6 @@ impl TypeSpecializationCache { out.push(old_getitem.into()); } self.getitem_version.store(0, Ordering::Release); - out.extend(self.retired.write().drain(..)); } } @@ -473,8 +458,12 @@ fn is_subtype_with_mro(a_mro: &[PyTypeRef], a: &Py, b: &Py) -> b impl PyType { #[inline] fn with_type_lock(vm: &VirtualMachine, f: impl FnOnce() -> R) -> R { - let _guard = vm.state.type_mutex.lock(); - f() + // Drops deferred via try_defer_drop inside the critical section run + // after the guard is released, outside the lock. + rustpython_common::refcount::with_deferred_drops(|| { + let _guard = vm.state.type_mutex.lock(); + f() + }) } /// Assign a fresh version tag. Returns 0 if the version counter has been @@ -1113,10 +1102,7 @@ impl PyType { if self.tp_version_tag.load(Ordering::Acquire) != tp_version { return false; } - if self.tp_version_tag.load(Ordering::Acquire) != tp_version { - return false; - } - ext.specialization_cache.swap_init(Some(init), Some(vm)); + ext.specialization_cache.swap_init(Some(init)); true }) } @@ -1135,7 +1121,7 @@ impl PyType { } ext.specialization_cache .init - .to_owned_ordering(Ordering::Acquire) + .try_to_owned(Ordering::Acquire) } /// Cache __getitem__ for BINARY_OP_SUBSCR_GETITEM specialization. @@ -1160,8 +1146,7 @@ impl PyType { if func_version == 0 { return false; } - ext.specialization_cache - .swap_getitem(Some(getitem), Some(vm)); + ext.specialization_cache.swap_getitem(Some(getitem)); ext.specialization_cache .getitem_version .store(func_version, Ordering::Release); @@ -1172,11 +1157,11 @@ impl PyType { /// Read cached __getitem__ for BINARY_OP_SUBSCR_GETITEM specialization. pub(crate) fn get_cached_getitem_for_specialization(&self) -> Option<(PyRef, u32)> { let ext = self.heaptype_ext.as_ref()?; - // Match CPython check order: pointer (Acquire) then function version. + // Check order: pointer (Acquire) then function version. let getitem = ext .specialization_cache .getitem - .to_owned_ordering(Ordering::Acquire)?; + .try_to_owned(Ordering::Acquire)?; let cached_version = ext .specialization_cache .getitem_version diff --git a/crates/vm/src/object/ext.rs b/crates/vm/src/object/ext.rs index d400de29c38..e576ac2c191 100644 --- a/crates/vm/src/object/ext.rs +++ b/crates/vm/src/object/ext.rs @@ -365,6 +365,35 @@ impl PyAtomicRef> { self.deref_ordering(ordering).map(|x| x.to_owned()) } + /// Try-incref read of the current value. + /// + /// Unlike [`Self::to_owned`], this never increfs a destructed object: + /// it uses a conditional incref and revalidates that the slot still + /// holds the same pointer. Returns `None` when the slot is empty. + /// + /// Soundness relies on published-object memory being reclaimed only + /// after a QSBR grace period (see `object::qsbr`), so the refcount + /// word of a concurrently swapped-out value stays readable. + pub fn try_to_owned(&self, ordering: Ordering) -> Option> { + loop { + let ptr = self.inner.load(ordering); + if ptr.is_null() { + return None; + } + if let Some(obj) = unsafe { PyObject::try_to_owned_from_ptr(ptr.cast::()) } { + if core::ptr::eq(self.inner.load(Ordering::Acquire), ptr) { + // SAFETY: the slot only ever stores `PyRef` values. + return Some(unsafe { obj.downcast_unchecked::() }); + } + drop(obj); + } + // Slot changed or the value was torn down mid-read; a failed + // incref with an unchanged slot is impossible (the slot's own + // strong ref keeps the value alive), so this loop progresses. + core::hint::spin_loop(); + } + } + /// # Safety /// The caller is responsible to keep the returned PyRef alive /// until no more reference can be used via PyAtomicRef::deref() From 6eecfe8494ff649af664e8de6423dcb994cd22e0 Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Sun, 5 Jul 2026 19:48:55 +0900 Subject: [PATCH 10/92] Reset QSBR thread registry after fork in the child Add Qsbr::reset_after_fork, which clears the registered thread slots before draining the retire queue. Call it in py_os_after_fork_child, right after type_cache_after_fork() and before _thread::after_fork_child() re-registers the surviving thread. Dead parent threads' Arc handles live on in the child's copied memory with no destructor ever running, so without this the registry keeps them 'online' forever and future grace periods never complete. Assisted-by: Claude --- crates/vm/src/object/qsbr.rs | 14 ++++++++++++++ crates/vm/src/stdlib/posix.rs | 8 ++++++++ 2 files changed, 22 insertions(+) diff --git a/crates/vm/src/object/qsbr.rs b/crates/vm/src/object/qsbr.rs index a88c6b88b72..965a85f8f6d 100644 --- a/crates/vm/src/object/qsbr.rs +++ b/crates/vm/src/object/qsbr.rs @@ -185,6 +185,20 @@ mod threading { } } + /// Reset after fork: drop all registered thread entries (dead + /// parent threads' slots would otherwise stay online forever and + /// stall every future grace period) and free all retired + /// allocations. + /// + /// # Safety + /// Only sound in the single-threaded post-fork child, before the + /// surviving thread re-registers. + pub unsafe fn reset_after_fork(&self) { + self.threads.lock().unwrap().clear(); + // SAFETY: single-threaded child, no concurrent reader exists. + unsafe { self.drain_all() }; + } + #[cfg(test)] fn pending(&self) -> usize { self.queue.lock().unwrap().len() diff --git a/crates/vm/src/stdlib/posix.rs b/crates/vm/src/stdlib/posix.rs index 0eaad949f15..e9b44a3786c 100644 --- a/crates/vm/src/stdlib/posix.rs +++ b/crates/vm/src/stdlib/posix.rs @@ -658,6 +658,14 @@ pub mod module { // Repair any type-cache entries left mid-update at fork time. unsafe { crate::builtins::type_::type_cache_after_fork() }; + // Reset QSBR: dead parent threads' slots would stall reclamation + // forever, and retired memory can be freed immediately in the + // single-threaded child. + #[cfg(feature = "threading")] + unsafe { + crate::object::qsbr::QSBR.reset_after_fork() + }; + // Phase 3: Clean up thread state. Locks are now reinit'd so we can // acquire them normally instead of using try_lock(). #[cfg(feature = "threading")] From 020edd33d5849be7de78a1d9dd68b33ec6030578 Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Sun, 5 Jul 2026 20:02:52 +0900 Subject: [PATCH 11/92] Add threaded stress test for type cache mutation races Assisted-by: Claude --- .../snippets/stdlib_threading_type_cache.py | 47 +++++++++++++++++++ 1 file changed, 47 insertions(+) create mode 100644 extra_tests/snippets/stdlib_threading_type_cache.py diff --git a/extra_tests/snippets/stdlib_threading_type_cache.py b/extra_tests/snippets/stdlib_threading_type_cache.py new file mode 100644 index 00000000000..d3130565fc7 --- /dev/null +++ b/extra_tests/snippets/stdlib_threading_type_cache.py @@ -0,0 +1,47 @@ +"""Stress the lock-free type method cache against concurrent type mutation. + +Readers hammer method lookups while a mutator continuously replaces and +deletes the method, dropping the old function objects. Guards against +use-after-free in the cache read protocol (QSBR deferred reclamation). +""" +import threading +import time + +class C: + def m(self): + return -1 + +DURATION = 1.5 + +def reader(stop): + obj = C() + while not stop.is_set(): + for _ in range(1000): + try: + obj.m() + except AttributeError: + pass + +def mutator(stop): + i = 0 + while not stop.is_set(): + def m(self, _i=i): + return _i + C.m = m + i += 1 + if i % 97 == 0: + try: + del C.m + except AttributeError: + pass + +stop = threading.Event() +threads = [threading.Thread(target=reader, args=(stop,)) for _ in range(4)] +threads.append(threading.Thread(target=mutator, args=(stop,))) +for t in threads: + t.start() +time.sleep(DURATION) +stop.set() +for t in threads: + t.join() +print("ok") From d565da5aee3fd3fdb3ba35cda4df89c02e3219cf Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Sun, 5 Jul 2026 20:09:34 +0900 Subject: [PATCH 12/92] Restrict qsbr internals to pub(crate) and fix clippy lints Downgrade pub items in the private qsbr::threading module (QsbrSlot, Qsbr, QSBR, and its methods) to pub(crate), since they were never reachable outside the crate. Downgrade ThreadSlot::qsbr accordingly to avoid a private-interfaces warning. Import Arc/Weak from alloc instead of std to match clippy::std_instead_of_alloc. Rewrite the process() comment: goals are not strictly ordered by push order since advance() and the queue push aren't atomic together, so concurrent free_delayed calls can interleave; the drained prefix stays sound because each item individually passed poll, with reordering only affecting reclamation latency. Assisted-by: Claude --- crates/vm/src/object/qsbr.rs | 39 +++++++++++++++++++++--------------- crates/vm/src/vm/thread.rs | 2 +- 2 files changed, 24 insertions(+), 17 deletions(-) diff --git a/crates/vm/src/object/qsbr.rs b/crates/vm/src/object/qsbr.rs index 965a85f8f6d..9b586eb2b65 100644 --- a/crates/vm/src/object/qsbr.rs +++ b/crates/vm/src/object/qsbr.rs @@ -24,21 +24,22 @@ const QSBR_INITIAL: u64 = 1; const QSBR_INCR: u64 = 2; #[cfg(feature = "threading")] -pub use threading::*; +pub(crate) use threading::*; #[cfg(feature = "threading")] mod threading { use super::*; + use alloc::sync::{Arc, Weak}; use core::sync::atomic::{AtomicBool, AtomicU64, Ordering}; - use std::sync::{Arc, Mutex, Weak}; + use std::sync::Mutex; /// Per-thread QSBR state, owned by the thread's `ThreadSlot`. - pub struct QsbrSlot { + pub(crate) struct QsbrSlot { /// Last write sequence observed at a quiescent point; /// `QSBR_OFFLINE` while the thread is detached. seq: AtomicU64, /// Set when this thread should pass a checkpoint (eval-breaker bit). - pub requested: AtomicBool, + pub(crate) requested: AtomicBool, } struct Retired { @@ -50,7 +51,7 @@ mod threading { // processing thread touches it. unsafe impl Send for Retired {} - pub struct Qsbr { + pub(crate) struct Qsbr { /// Global write sequence (_Py_qsbr wr_seq). wr_seq: AtomicU64, /// Cached minimum observed read sequence (rd_seq). @@ -59,7 +60,7 @@ mod threading { queue: Mutex>, } - pub static QSBR: Qsbr = Qsbr::new(); + pub(crate) static QSBR: Qsbr = Qsbr::new(); impl Qsbr { const fn new() -> Self { @@ -73,7 +74,7 @@ mod threading { /// Register the calling thread. The returned slot is stored in the /// thread's `ThreadSlot`; dropping it unregisters the thread. - pub fn register(&self) -> Arc { + pub(crate) fn register(&self) -> Arc { let slot = Arc::new(QsbrSlot { seq: AtomicU64::new(self.wr_seq.load(Ordering::Acquire)), requested: AtomicBool::new(false), @@ -90,7 +91,7 @@ mod threading { /// Record that the calling thread is at a quiescent point: it holds /// no borrowed cache pointers (_Py_qsbr_quiescent_state). - pub fn quiescent_state(&self, slot: &QsbrSlot) { + pub(crate) fn quiescent_state(&self, slot: &QsbrSlot) { slot.seq .store(self.wr_seq.load(Ordering::Acquire), Ordering::Release); } @@ -98,12 +99,12 @@ mod threading { /// Mark a thread offline (detached); it no longer delays grace /// periods (_Py_qsbr_detach). The thread must not perform lock-free /// cache reads while offline. - pub fn offline(&self, slot: &QsbrSlot) { + pub(crate) fn offline(&self, slot: &QsbrSlot) { slot.seq.store(QSBR_OFFLINE, Ordering::Release); } /// Mark a thread online again (_Py_qsbr_attach). - pub fn online(&self, slot: &QsbrSlot) { + pub(crate) fn online(&self, slot: &QsbrSlot) { self.quiescent_state(slot); } @@ -142,7 +143,7 @@ mod threading { /// `ptr`/`layout` must describe an allocation whose contents have /// been dropped and which nothing accesses afterwards except the /// racing try-incref reads this mechanism protects against. - pub unsafe fn free_delayed(&self, ptr: *mut u8, layout: Layout) { + pub(crate) unsafe fn free_delayed(&self, ptr: *mut u8, layout: Layout) { let goal = self.advance(); self.queue.lock().unwrap().push(Retired { ptr, layout, goal }); // Ask every registered thread to pass a checkpoint. @@ -155,13 +156,19 @@ mod threading { /// Free retired allocations whose grace period has passed /// (_PyMem_ProcessDelayed). - pub fn process(&self) { + pub(crate) fn process(&self) { let Ok(mut queue) = self.queue.try_lock() else { // Another thread is already processing. return; }; - // Goals are monotonically increasing in push order: free the - // prefix whose grace period has passed. + // Goals are usually increasing in push order, but concurrent + // `free_delayed` calls can interleave their `advance()` and + // queue push, so a smaller goal can occasionally land behind a + // larger one. Free the longest prefix whose grace period has + // passed; each drained item individually passed `poll`, so this + // is sound regardless of ordering. A goal stuck behind an + // out-of-order neighbor just waits for the next checkpoint or + // GC pass, not a correctness issue. let safe_prefix = queue .iter() .position(|item| !self.poll(item.goal)) @@ -177,7 +184,7 @@ mod threading { /// # Safety /// Only sound when no other thread can be mid-read: the post-fork /// child, or teardown after all threads exited. - pub unsafe fn drain_all(&self) { + pub(crate) unsafe fn drain_all(&self) { let mut queue = self.queue.lock().unwrap(); for item in queue.drain(..) { // SAFETY: guaranteed single-threaded by the caller. @@ -193,7 +200,7 @@ mod threading { /// # Safety /// Only sound in the single-threaded post-fork child, before the /// surviving thread re-registers. - pub unsafe fn reset_after_fork(&self) { + pub(crate) unsafe fn reset_after_fork(&self) { self.threads.lock().unwrap().clear(); // SAFETY: single-threaded child, no concurrent reader exists. unsafe { self.drain_all() }; diff --git a/crates/vm/src/vm/thread.rs b/crates/vm/src/vm/thread.rs index 91efa384e4d..bb7980004e6 100644 --- a/crates/vm/src/vm/thread.rs +++ b/crates/vm/src/vm/thread.rs @@ -47,7 +47,7 @@ pub struct ThreadSlot { /// builds there is no attach/detach state machine, so the slot stays /// online from registration to thread exit; a thread blocked in native /// code simply delays reclamation until it runs again. - pub qsbr: Arc, + pub(crate) qsbr: Arc, } #[cfg(feature = "threading")] From cbd20c0575c3c5e2ed16ba81f6f916804319457f Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Sun, 5 Jul 2026 20:18:31 +0900 Subject: [PATCH 13/92] Gate per-instruction QSBR check behind a global pending flag eval_breaker_tripped() and check_signals() called thread::qsbr_break_requested() on every bytecode instruction, which does a thread-local lookup, RefCell borrow, and atomic load even when no QSBR retirement is pending. This caused a measured 10-18% slowdown on tight interpreter loops. Add Qsbr::pending, a global AtomicBool set under the queue lock while free_delayed() holds a retired allocation, and cleared under the same lock by process()/drain_all() once the queue empties. The hot path now checks Qsbr::break_pending() (a single relaxed static load) before touching thread-local state, and only pays the TLS cost while a retirement is actually in flight. Assisted-by: Claude --- crates/vm/src/object/qsbr.rs | 28 +++++++++++++++++++++++++++- crates/vm/src/vm/mod.rs | 4 ++-- 2 files changed, 29 insertions(+), 3 deletions(-) diff --git a/crates/vm/src/object/qsbr.rs b/crates/vm/src/object/qsbr.rs index 9b586eb2b65..ddfd897b429 100644 --- a/crates/vm/src/object/qsbr.rs +++ b/crates/vm/src/object/qsbr.rs @@ -58,6 +58,10 @@ mod threading { rd_seq: AtomicU64, threads: Mutex>>, queue: Mutex>, + /// Set while the retire queue is non-empty; gates the per-instruction + /// eval-breaker check so the hot path pays only one relaxed static + /// load when nothing is pending. + pending: AtomicBool, } pub(crate) static QSBR: Qsbr = Qsbr::new(); @@ -69,9 +73,17 @@ mod threading { rd_seq: AtomicU64::new(QSBR_INITIAL), threads: Mutex::new(Vec::new()), queue: Mutex::new(Vec::new()), + pending: AtomicBool::new(false), } } + /// Whether retired allocations are pending. Cheap gate for the + /// per-instruction eval-breaker check. + #[inline] + pub(crate) fn break_pending(&self) -> bool { + self.pending.load(Ordering::Relaxed) + } + /// Register the calling thread. The returned slot is stored in the /// thread's `ThreadSlot`; dropping it unregisters the thread. pub(crate) fn register(&self) -> Arc { @@ -145,7 +157,14 @@ mod threading { /// racing try-incref reads this mechanism protects against. pub(crate) unsafe fn free_delayed(&self, ptr: *mut u8, layout: Layout) { let goal = self.advance(); - self.queue.lock().unwrap().push(Retired { ptr, layout, goal }); + { + let mut queue = self.queue.lock().unwrap(); + queue.push(Retired { ptr, layout, goal }); + // Set while still holding the queue lock, so this pairs with + // `process` clearing the flag under the same lock and no + // push can be left behind with the flag cleared. + self.pending.store(true, Ordering::Release); + } // Ask every registered thread to pass a checkpoint. for weak in self.threads.lock().unwrap().iter() { if let Some(slot) = weak.upgrade() { @@ -177,6 +196,9 @@ mod threading { // SAFETY: grace period passed; no reader can hold `ptr`. unsafe { alloc::alloc::dealloc(item.ptr, item.layout) }; } + if queue.is_empty() { + self.pending.store(false, Ordering::Release); + } } /// Free all retired allocations immediately. @@ -190,6 +212,7 @@ mod threading { // SAFETY: guaranteed single-threaded by the caller. unsafe { alloc::alloc::dealloc(item.ptr, item.layout) }; } + self.pending.store(false, Ordering::Release); } /// Reset after fork: drop all registered thread entries (dead @@ -259,11 +282,14 @@ mod threading { let ptr = unsafe { alloc::alloc::alloc(layout) }; unsafe { q.free_delayed(ptr, layout) }; assert!(a.requested.load(Ordering::Acquire)); + assert!(q.break_pending()); q.process(); assert_eq!(q.pending(), 1); // grace period not passed yet + assert!(q.break_pending()); q.quiescent_state(&a); q.process(); assert_eq!(q.pending(), 0); + assert!(!q.break_pending()); } } } diff --git a/crates/vm/src/vm/mod.rs b/crates/vm/src/vm/mod.rs index 3e40d235809..79e1573538e 100644 --- a/crates/vm/src/vm/mod.rs +++ b/crates/vm/src/vm/mod.rs @@ -2042,7 +2042,7 @@ impl VirtualMachine { } #[cfg(feature = "threading")] - if thread::qsbr_break_requested() { + if crate::object::qsbr::QSBR.break_pending() && thread::qsbr_break_requested() { return true; } @@ -2071,7 +2071,7 @@ impl VirtualMachine { // Pass a QSBR checkpoint if requested (deferred memory reclamation). #[cfg(feature = "threading")] - if thread::qsbr_break_requested() { + if crate::object::qsbr::QSBR.break_pending() && thread::qsbr_break_requested() { thread::qsbr_checkpoint(); } From eb633b2167f1efefcbac2f9a7930babec582e828 Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Sun, 5 Jul 2026 20:29:43 +0900 Subject: [PATCH 14/92] Merge signal and QSBR eval-breaker flags into one atomic word Replace the separate ANY_TRIGGERED AtomicBool and QSBR pending AtomicBool checks in the per-instruction eval-breaker path with a single AtomicU8 (EVAL_BREAKER) holding a bit per source. The global QSBR instance mirrors its pending state into the QSBR bit under the same queue-lock critical sections that already toggle pending, while local QSBR instances used by unit tests are left untouched. eval_breaker_tripped() and check_signals() now read/act on the merged word (eval_breaker_pending(), qsbr_bit_set()) instead of issuing two separate atomic loads per instruction. set_triggered() switches from store to fetch_or so a signal handler never clobbers the QSBR bit. Removed the now-unreferenced is_triggered() helper; break_pending() is kept for unit tests only and marked allow(dead_code) outside test builds. Assisted-by: Claude --- crates/vm/src/object/qsbr.rs | 22 ++++++++++++++++-- crates/vm/src/signal.rs | 44 ++++++++++++++++++++++++++++-------- crates/vm/src/vm/mod.rs | 11 ++++----- 3 files changed, 59 insertions(+), 18 deletions(-) diff --git a/crates/vm/src/object/qsbr.rs b/crates/vm/src/object/qsbr.rs index ddfd897b429..c4e84208dba 100644 --- a/crates/vm/src/object/qsbr.rs +++ b/crates/vm/src/object/qsbr.rs @@ -77,13 +77,28 @@ mod threading { } } - /// Whether retired allocations are pending. Cheap gate for the - /// per-instruction eval-breaker check. + /// Whether retired allocations are pending. The hot path now reads + /// the mirrored bit in the eval-breaker word instead; this stays + /// only for unit tests that exercise local, non-global instances. #[inline] + #[cfg_attr(not(test), allow(dead_code))] pub(crate) fn break_pending(&self) -> bool { self.pending.load(Ordering::Relaxed) } + /// Mirror `pending` into the global eval-breaker word — only for + /// the global QSBR instance, so unit-test instances never touch + /// process-global state. + fn update_breaker_bit(&self, on: bool) { + if core::ptr::eq(self, &QSBR) { + if on { + crate::signal::set_qsbr_bit(); + } else { + crate::signal::clear_qsbr_bit(); + } + } + } + /// Register the calling thread. The returned slot is stored in the /// thread's `ThreadSlot`; dropping it unregisters the thread. pub(crate) fn register(&self) -> Arc { @@ -164,6 +179,7 @@ mod threading { // `process` clearing the flag under the same lock and no // push can be left behind with the flag cleared. self.pending.store(true, Ordering::Release); + self.update_breaker_bit(true); } // Ask every registered thread to pass a checkpoint. for weak in self.threads.lock().unwrap().iter() { @@ -198,6 +214,7 @@ mod threading { } if queue.is_empty() { self.pending.store(false, Ordering::Release); + self.update_breaker_bit(false); } } @@ -213,6 +230,7 @@ mod threading { unsafe { alloc::alloc::dealloc(item.ptr, item.layout) }; } self.pending.store(false, Ordering::Release); + self.update_breaker_bit(false); } /// Reset after fork: drop all registered thread entries (dead diff --git a/crates/vm/src/signal.rs b/crates/vm/src/signal.rs index eea42f4a87e..05e655680a5 100644 --- a/crates/vm/src/signal.rs +++ b/crates/vm/src/signal.rs @@ -2,7 +2,7 @@ use core::{ cell::{Cell, RefCell}, fmt, ops::{Deref, DerefMut, Index, IndexMut, Range}, - sync::atomic::{AtomicBool, Ordering}, + sync::atomic::{AtomicBool, AtomicU8, Ordering}, }; use std::sync::mpsc; @@ -13,7 +13,16 @@ use crate::{PyObjectRef, PyResult, TryFromBorrowedObject, TryFromObject, Virtual pub(crate) const NSIG: usize = 64; -static ANY_TRIGGERED: AtomicBool = AtomicBool::new(false); +/// Eval-breaker word: bit flags checked once per bytecode instruction. +/// Signal handlers and QSBR set bits with fetch_or (async-signal-safe, +/// lock-free); consumers clear only their own bit with fetch_and. +static EVAL_BREAKER: AtomicU8 = AtomicU8::new(0); + +/// A signal handler recorded a pending signal. +const SIGNAL_BIT: u8 = 1 << 0; +/// QSBR has retired allocations pending reclamation. +#[cfg(feature = "threading")] +const QSBR_BIT: u8 = 1 << 1; #[expect( clippy::declare_interior_mutable_const, @@ -49,12 +58,12 @@ pub fn check_signals(vm: &VirtualMachine) -> PyResult<()> { // Read-only check first: avoids cache-line invalidation on every // instruction when no signal is pending (the common case). - if !ANY_TRIGGERED.load(Ordering::Relaxed) { + if EVAL_BREAKER.load(Ordering::Relaxed) & SIGNAL_BIT == 0 { return Ok(()); } // Atomic RMW only when a signal is actually pending. - if !ANY_TRIGGERED.swap(false, Ordering::Acquire) { + if EVAL_BREAKER.fetch_and(!SIGNAL_BIT, Ordering::Acquire) & SIGNAL_BIT == 0 { return Ok(()); } @@ -101,20 +110,37 @@ fn trigger_signals(vm: &VirtualMachine) -> PyResult<()> { } pub(crate) fn set_triggered() { - ANY_TRIGGERED.store(true, Ordering::Release); + // fetch_or (not store) so a signal handler never clobbers the QSBR bit; + // this compiles to a lock-free RMW, safe to call from a signal handler. + EVAL_BREAKER.fetch_or(SIGNAL_BIT, Ordering::Release); } +/// Any eval-breaker bit pending? One relaxed load; checked per instruction. #[inline(always)] -#[cfg(not(target_arch = "wasm32"))] -pub(crate) fn is_triggered() -> bool { - ANY_TRIGGERED.load(Ordering::Relaxed) +pub(crate) fn eval_breaker_pending() -> bool { + EVAL_BREAKER.load(Ordering::Relaxed) != 0 +} + +#[cfg(feature = "threading")] +pub(crate) fn set_qsbr_bit() { + EVAL_BREAKER.fetch_or(QSBR_BIT, Ordering::Release); +} + +#[cfg(feature = "threading")] +pub(crate) fn clear_qsbr_bit() { + EVAL_BREAKER.fetch_and(!QSBR_BIT, Ordering::Release); +} + +#[cfg(feature = "threading")] +pub(crate) fn qsbr_bit_set() -> bool { + EVAL_BREAKER.load(Ordering::Relaxed) & QSBR_BIT != 0 } /// Reset all signal trigger state after fork in child process. /// Stale triggers from the parent must not fire in the child. #[cfg(all(unix, feature = "host_env"))] pub(crate) fn clear_after_fork() { - ANY_TRIGGERED.store(false, Ordering::Release); + EVAL_BREAKER.fetch_and(!SIGNAL_BIT, Ordering::Release); for trigger in &TRIGGERS { trigger.store(false, Ordering::Relaxed); } diff --git a/crates/vm/src/vm/mod.rs b/crates/vm/src/vm/mod.rs index 79e1573538e..40a97e40862 100644 --- a/crates/vm/src/vm/mod.rs +++ b/crates/vm/src/vm/mod.rs @@ -2041,13 +2041,10 @@ impl VirtualMachine { return true; } - #[cfg(feature = "threading")] - if crate::object::qsbr::QSBR.break_pending() && thread::qsbr_break_requested() { - return true; - } - + // Signal and QSBR bits share one word: a single relaxed load per + // instruction covers both. #[cfg(not(target_arch = "wasm32"))] - if crate::signal::is_triggered() { + if crate::signal::eval_breaker_pending() { return true; } @@ -2071,7 +2068,7 @@ impl VirtualMachine { // Pass a QSBR checkpoint if requested (deferred memory reclamation). #[cfg(feature = "threading")] - if crate::object::qsbr::QSBR.break_pending() && thread::qsbr_break_requested() { + if crate::signal::qsbr_bit_set() && thread::qsbr_break_requested() { thread::qsbr_checkpoint(); } From 9baa344c1381a9c70da5bef4dd4752f1fe55f99f Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Sun, 5 Jul 2026 20:50:16 +0900 Subject: [PATCH 15/92] Skip freelist reuse for published objects in default_dealloc Freelist-eligible payloads (tuple, int, list, dict, float, ...) were routed to T::freelist_push before ever reaching PyInner::dealloc, so a published object (e.g. a tuple cached as a type attribute) bypassed the is_published -> free_delayed QSBR hook entirely. That let PyRef::new_ref reuse the slot and rewrite the refcount word with a non-atomic core::ptr::write racing a reader's atomic try-incref, and let FreeList::drop call alloc::dealloc directly while a reader could still be mid-read. Gate the freelist branch in default_dealloc on RefCount::is_published(), checked before any teardown. A published object now always falls through to PyInner::dealloc, whose existing hook defers the memory free via QSBR. Document the non-unix (e.g. Windows) reclamation-latency limitation in the QSBR design doc's Lifecycle integration section (previously only noted in a code comment in vm/thread.rs). Extend the type-cache stress test so the mutator also churns a freelist-eligible published tuple class attribute (C.shape), read by readers each inner loop and deleted on the same cadence as C.m, to exercise the fixed bypass. Assisted-by: Claude --- crates/vm/src/object/core.rs | 6 ++++++ .../snippets/stdlib_threading_type_cache.py | 14 ++++++++++++++ 2 files changed, 20 insertions(+) diff --git a/crates/vm/src/object/core.rs b/crates/vm/src/object/core.rs index 49c63d8516a..23ec2f752eb 100644 --- a/crates/vm/src/object/core.rs +++ b/crates/vm/src/object/core.rs @@ -191,10 +191,16 @@ pub(super) unsafe fn default_dealloc(obj: *mut PyObject) { // Try to store in freelist for reuse BEFORE tp_clear, so that // size-based freelists (e.g. PyTuple) can read the payload directly. // Only exact base types (not heaptype or structseq subtypes) go into the freelist. + // Published objects (e.g. a tuple stored as a type attribute) must skip the + // freelist: `PyRef::new_ref` would reuse the slot and overwrite the refcount + // word with a non-atomic write, racing a reader's atomic try-incref. Route + // them through `PyInner::dealloc` instead, whose QSBR hook defers the actual + // memory free until readers can no longer observe it. let typ = obj_ref.class(); let pushed = if T::HAS_FREELIST && typ.heaptype_ext.is_none() && core::ptr::eq(typ, T::class(crate::vm::Context::genesis())) + && !obj_ref.0.ref_count.is_published() { unsafe { T::freelist_push(obj) } } else { diff --git a/extra_tests/snippets/stdlib_threading_type_cache.py b/extra_tests/snippets/stdlib_threading_type_cache.py index d3130565fc7..de96130a284 100644 --- a/extra_tests/snippets/stdlib_threading_type_cache.py +++ b/extra_tests/snippets/stdlib_threading_type_cache.py @@ -3,6 +3,11 @@ Readers hammer method lookups while a mutator continuously replaces and deletes the method, dropping the old function objects. Guards against use-after-free in the cache read protocol (QSBR deferred reclamation). + +Also churns a freelist-eligible published value (a tuple class attribute): +tuples normally go back through the freelist on dealloc, but once one is +published to the type cache it must instead go through the QSBR-deferred +reclamation path, so this exercises that bypass. """ import threading import time @@ -21,6 +26,10 @@ def reader(stop): obj.m() except AttributeError: pass + try: + obj.shape + except AttributeError: + pass def mutator(stop): i = 0 @@ -28,12 +37,17 @@ def mutator(stop): def m(self, _i=i): return _i C.m = m + C.shape = (i, i + 1) i += 1 if i % 97 == 0: try: del C.m except AttributeError: pass + try: + del C.shape + except AttributeError: + pass stop = threading.Event() threads = [threading.Thread(target=reader, args=(stop,)) for _ in range(4)] From 97dbfcc0687e6c4e4e47f8026b71a28f4e11e7b0 Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Sun, 5 Jul 2026 23:59:28 +0900 Subject: [PATCH 16/92] Use itoa for PyInt::to_str_radix_10 i64 fast path Format small integers with itoa::Buffer instead of i64::to_string() in PyInt::to_str_radix_10(), which backs both str(int)/repr(int) for exact int and PyObject::str()'s exact-int fast path. Adds the itoa crate as a workspace dependency. Assisted-by: Claude --- Cargo.lock | 1 + Cargo.toml | 1 + crates/vm/Cargo.toml | 1 + crates/vm/src/builtins/int.rs | 2 +- 4 files changed, 4 insertions(+), 1 deletion(-) diff --git a/Cargo.lock b/Cargo.lock index 5eb95bea29c..aeb2426e52a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3754,6 +3754,7 @@ dependencies = [ "indexmap", "is-macro", "itertools 0.15.0", + "itoa", "libc", "log", "malachite-bigint", diff --git a/Cargo.toml b/Cargo.toml index d8081f50166..540324a4aaf 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -229,6 +229,7 @@ hmac = "0.13" indexmap = { version = "2.14.0", features = ["std"] } insta = "1.47" itertools = { version = "0.15.0", default-features = false, features = ["use_alloc"] } +itoa = "1" is-macro = "0.3.7" js-sys = "0.3" junction = "2.0.0" diff --git a/crates/vm/Cargo.toml b/crates/vm/Cargo.toml index 0b2feed50a3..24006c8b3b9 100644 --- a/crates/vm/Cargo.toml +++ b/crates/vm/Cargo.toml @@ -54,6 +54,7 @@ flame = { workspace = true, optional = true } hex = { workspace = true } indexmap = { workspace = true } itertools = { workspace = true } +itoa = { workspace = true } is-macro = { workspace = true } libc = { workspace = true } log = { workspace = true } diff --git a/crates/vm/src/builtins/int.rs b/crates/vm/src/builtins/int.rs index b9246149731..c5aa607d023 100644 --- a/crates/vm/src/builtins/int.rs +++ b/crates/vm/src/builtins/int.rs @@ -310,7 +310,7 @@ impl PyInt { #[must_use] pub fn to_str_radix_10(&self) -> String { match self.value.to_i64() { - Some(i) => i.to_string(), + Some(i) => itoa::Buffer::new().format(i).to_owned(), None => self.value.to_string(), } } From e4ddae6ff88a52733de60118e789806a813d245f Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Mon, 6 Jul 2026 03:30:17 +0900 Subject: [PATCH 17/92] Make object attribute dunders wrapper descriptors Remove the __getattribute__, __setattr__ and __delattr__ #[pymethod]s from PyBaseObject so add_operators installs PyWrapper slot wrappers for them from the #[pyslot] functions instead of method descriptors. lookup_slot_in_mro now classifies these entries as NativeSlot, so heap types without Python-level overrides keep the native getattro/setattro slot functions, which lets LOAD_ATTR/STORE_ATTR specialization apply to instances of Python-defined classes. Since __setattr__ and __delattr__ share the setattro slot, resolve both names together in update_one_slot: overriding only one of them no longer lets the other name's native resolution overwrite the dispatching wrapper. Assisted-by: Claude --- crates/vm/src/builtins/object.rs | 25 +++---------------------- crates/vm/src/types/slot.rs | 31 ++++++++++++++++++++++++------- 2 files changed, 27 insertions(+), 29 deletions(-) diff --git a/crates/vm/src/builtins/object.rs b/crates/vm/src/builtins/object.rs index 147e215a0cb..fcdd0ffeb34 100644 --- a/crates/vm/src/builtins/object.rs +++ b/crates/vm/src/builtins/object.rs @@ -346,23 +346,7 @@ impl PyBaseObject { Ok(res) } - /// Implement setattr(self, name, value). - #[pymethod] - fn __setattr__( - obj: PyObjectRef, - name: PyStrRef, - value: PyObjectRef, - vm: &VirtualMachine, - ) -> PyResult<()> { - obj.generic_setattr(&name, PySetterValue::Assign(value), vm) - } - - /// Implement delattr(self, name). - #[pymethod] - fn __delattr__(obj: PyObjectRef, name: PyStrRef, vm: &VirtualMachine) -> PyResult<()> { - obj.generic_setattr(&name, PySetterValue::Delete, vm) - } - + // __setattr__ and __delattr__ are added as slot wrappers by add_operators. #[pyslot] pub(crate) fn slot_setattro( obj: &PyObject, @@ -513,17 +497,14 @@ impl PyBaseObject { } /// Return getattr(self, name). + /// + /// __getattribute__ is added as a slot wrapper by add_operators. #[pyslot] pub(crate) fn getattro(obj: &PyObject, name: &Py, vm: &VirtualMachine) -> PyResult { vm_trace!("object.__getattribute__({:?}, {:?})", obj, name); obj.as_object().generic_getattr(name, vm) } - #[pymethod] - fn __getattribute__(obj: PyObjectRef, name: PyStrRef, vm: &VirtualMachine) -> PyResult { - Self::getattro(&obj, &name, vm) - } - #[pymethod] fn __reduce__(obj: PyObjectRef, vm: &VirtualMachine) -> PyResult { common_reduce(obj, 0, vm) diff --git a/crates/vm/src/types/slot.rs b/crates/vm/src/types/slot.rs index 83d9706cb33..6f21800c5a8 100644 --- a/crates/vm/src/types/slot.rs +++ b/crates/vm/src/types/slot.rs @@ -897,19 +897,36 @@ impl PyType { } } SlotAccessor::TpSetattro => { - // __setattr__ and __delattr__ share the same slot + // __setattr__ and __delattr__ share the same slot, so both + // names must be resolved together: a Python-level override of + // either one forces the dispatching wrapper, and the native + // slot is only usable when every resolved name agrees on it. if ADD { - match self.lookup_slot_in_mro(name, ctx, |sf| match sf { + let extract = |sf: &SlotFunc| match sf { SlotFunc::SetAttro(f) | SlotFunc::DelAttro(f) => Some(*f), _ => None, - }) { - SlotLookupResult::NativeSlot(func) => { + }; + let setattr = + self.lookup_slot_in_mro(identifier!(ctx, __setattr__), ctx, extract); + let delattr = + self.lookup_slot_in_mro(identifier!(ctx, __delattr__), ctx, extract); + use SlotLookupResult::{NativeSlot, NotFound, PythonMethod}; + match (setattr, delattr) { + (PythonMethod, _) | (_, PythonMethod) => { + self.slots.setattro.store(Some(setattro_wrapper)); + } + (NativeSlot(set), NativeSlot(del)) => { + let func = if set as usize == del as usize { + set + } else { + setattro_wrapper + }; self.slots.setattro.store(Some(func)); } - SlotLookupResult::PythonMethod => { - self.slots.setattro.store(Some(setattro_wrapper)); + (NativeSlot(func), NotFound) | (NotFound, NativeSlot(func)) => { + self.slots.setattro.store(Some(func)); } - SlotLookupResult::NotFound => { + (NotFound, NotFound) => { accessor.inherit_from_mro(self); } } From 1ac47f358705a526118283914369261cd4041590 Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Mon, 6 Jul 2026 03:43:40 +0900 Subject: [PATCH 18/92] Resolve TpSetattro pair on attribute deletion, not just addition update_one_slot's TpSetattro branch only ran the combined __setattr__/__delattr__ resolution (added in the previous commit) when ADD was true. On deletion (e.g. del C.__setattr__), it fell back to inherit_from_mro, which copies the base class's slot and discards the class's own remaining Python-level override of the other name. lookup_slot_in_mro reads the current attribute dicts, so the same resolution is correct for both addition and deletion; the ADD-only guard is removed and the logic now always runs. Assisted-by: Claude --- crates/vm/src/types/slot.rs | 56 +++++++++++++++++-------------------- 1 file changed, 26 insertions(+), 30 deletions(-) diff --git a/crates/vm/src/types/slot.rs b/crates/vm/src/types/slot.rs index 6f21800c5a8..4e21b7f830d 100644 --- a/crates/vm/src/types/slot.rs +++ b/crates/vm/src/types/slot.rs @@ -901,37 +901,33 @@ impl PyType { // names must be resolved together: a Python-level override of // either one forces the dispatching wrapper, and the native // slot is only usable when every resolved name agrees on it. - if ADD { - let extract = |sf: &SlotFunc| match sf { - SlotFunc::SetAttro(f) | SlotFunc::DelAttro(f) => Some(*f), - _ => None, - }; - let setattr = - self.lookup_slot_in_mro(identifier!(ctx, __setattr__), ctx, extract); - let delattr = - self.lookup_slot_in_mro(identifier!(ctx, __delattr__), ctx, extract); - use SlotLookupResult::{NativeSlot, NotFound, PythonMethod}; - match (setattr, delattr) { - (PythonMethod, _) | (_, PythonMethod) => { - self.slots.setattro.store(Some(setattro_wrapper)); - } - (NativeSlot(set), NativeSlot(del)) => { - let func = if set as usize == del as usize { - set - } else { - setattro_wrapper - }; - self.slots.setattro.store(Some(func)); - } - (NativeSlot(func), NotFound) | (NotFound, NativeSlot(func)) => { - self.slots.setattro.store(Some(func)); - } - (NotFound, NotFound) => { - accessor.inherit_from_mro(self); - } + // This resolution reads the current attribute dicts, so it + // applies the same whether a name was just added or removed. + let extract = |sf: &SlotFunc| match sf { + SlotFunc::SetAttro(f) | SlotFunc::DelAttro(f) => Some(*f), + _ => None, + }; + let setattr = self.lookup_slot_in_mro(identifier!(ctx, __setattr__), ctx, extract); + let delattr = self.lookup_slot_in_mro(identifier!(ctx, __delattr__), ctx, extract); + use SlotLookupResult::{NativeSlot, NotFound, PythonMethod}; + match (setattr, delattr) { + (PythonMethod, _) | (_, PythonMethod) => { + self.slots.setattro.store(Some(setattro_wrapper)); + } + (NativeSlot(set), NativeSlot(del)) => { + let func = if set as usize == del as usize { + set + } else { + setattro_wrapper + }; + self.slots.setattro.store(Some(func)); + } + (NativeSlot(func), NotFound) | (NotFound, NativeSlot(func)) => { + self.slots.setattro.store(Some(func)); + } + (NotFound, NotFound) => { + accessor.inherit_from_mro(self); } - } else { - accessor.inherit_from_mro(self); } } SlotAccessor::TpDescrGet => update_main_slot!(descr_get, descr_get_wrapper, DescrGet), From 8f3245006e6c6867652844306c6fa9dd30241838 Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Mon, 6 Jul 2026 04:00:09 +0900 Subject: [PATCH 19/92] Generate Opcode::cache_entries/deopt as table lookups generate_rs_opcode_metadata.py now emits a 256-entry const array for each function instead of a chained match over opcode names. Opcode::deopt() indexes an [Option; 256] table built from the existing specialization-to-base mapping, and Opcode::cache_entries() indexes a [u8; 256] table that bakes in the same deopt/to_base composition deoptimize() previously performed before its own match. Regenerated opcode_metadata.rs from the updated script; PseudoOpcode's bodies are unchanged since it has no cache/deopt entries to table. Added an exhaustive test in instruction.rs that checks every valid u8 opcode against a frozen copy of the old chained-match bodies. Assisted-by: Claude --- .../compiler-core/src/bytecode/instruction.rs | 153 ++++++++ .../src/bytecode/opcode_metadata.rs | 371 +++++++++++++----- .../generate_rs_opcode_metadata.py | 81 +++- 3 files changed, 488 insertions(+), 117 deletions(-) diff --git a/crates/compiler-core/src/bytecode/instruction.rs b/crates/compiler-core/src/bytecode/instruction.rs index 76871b2c97e..8f488d233c0 100644 --- a/crates/compiler-core/src/bytecode/instruction.rs +++ b/crates/compiler-core/src/bytecode/instruction.rs @@ -1450,4 +1450,157 @@ mod tests { assert!(AnyInstruction::from(PseudoOpcode::Jump).is_no_fallthrough()); } + + /// Snapshot of the chained `match` implementations that `Opcode::deopt` + /// and `Opcode::cache_entries` used before they were rewritten as table + /// lookups. Exists only to pin the observable behavior of the table + /// lookups against the logic they replaced. + mod reference { + use super::Opcode; + + pub(super) const fn deopt(op: Opcode) -> Option { + Some(match op { + Opcode::ResumeCheck => Opcode::Resume, + Opcode::LoadConstMortal | Opcode::LoadConstImmortal => Opcode::LoadConst, + Opcode::ToBoolAlwaysTrue + | Opcode::ToBoolBool + | Opcode::ToBoolInt + | Opcode::ToBoolList + | Opcode::ToBoolNone + | Opcode::ToBoolStr => Opcode::ToBool, + Opcode::BinaryOpMultiplyInt + | Opcode::BinaryOpAddInt + | Opcode::BinaryOpSubtractInt + | Opcode::BinaryOpMultiplyFloat + | Opcode::BinaryOpAddFloat + | Opcode::BinaryOpSubtractFloat + | Opcode::BinaryOpAddUnicode + | Opcode::BinaryOpSubscrListInt + | Opcode::BinaryOpSubscrListSlice + | Opcode::BinaryOpSubscrTupleInt + | Opcode::BinaryOpSubscrStrInt + | Opcode::BinaryOpSubscrDict + | Opcode::BinaryOpSubscrGetitem + | Opcode::BinaryOpExtend + | Opcode::BinaryOpInplaceAddUnicode => Opcode::BinaryOp, + Opcode::StoreSubscrDict | Opcode::StoreSubscrListInt => Opcode::StoreSubscr, + Opcode::SendGen => Opcode::Send, + Opcode::UnpackSequenceTwoTuple + | Opcode::UnpackSequenceTuple + | Opcode::UnpackSequenceList => Opcode::UnpackSequence, + Opcode::StoreAttrInstanceValue + | Opcode::StoreAttrSlot + | Opcode::StoreAttrWithHint => Opcode::StoreAttr, + Opcode::LoadGlobalModule | Opcode::LoadGlobalBuiltin => Opcode::LoadGlobal, + Opcode::LoadSuperAttrAttr | Opcode::LoadSuperAttrMethod => Opcode::LoadSuperAttr, + Opcode::LoadAttrInstanceValue + | Opcode::LoadAttrModule + | Opcode::LoadAttrWithHint + | Opcode::LoadAttrSlot + | Opcode::LoadAttrClass + | Opcode::LoadAttrClassWithMetaclassCheck + | Opcode::LoadAttrProperty + | Opcode::LoadAttrGetattributeOverridden + | Opcode::LoadAttrMethodWithValues + | Opcode::LoadAttrMethodNoDict + | Opcode::LoadAttrMethodLazyDict + | Opcode::LoadAttrNondescriptorWithValues + | Opcode::LoadAttrNondescriptorNoDict => Opcode::LoadAttr, + Opcode::CompareOpFloat | Opcode::CompareOpInt | Opcode::CompareOpStr => { + Opcode::CompareOp + } + Opcode::ContainsOpSet | Opcode::ContainsOpDict => Opcode::ContainsOp, + Opcode::JumpBackwardNoJit | Opcode::JumpBackwardJit => Opcode::JumpBackward, + Opcode::ForIterList + | Opcode::ForIterTuple + | Opcode::ForIterRange + | Opcode::ForIterGen => Opcode::ForIter, + Opcode::CallBoundMethodExactArgs + | Opcode::CallPyExactArgs + | Opcode::CallType1 + | Opcode::CallStr1 + | Opcode::CallTuple1 + | Opcode::CallBuiltinClass + | Opcode::CallBuiltinO + | Opcode::CallBuiltinFast + | Opcode::CallBuiltinFastWithKeywords + | Opcode::CallLen + | Opcode::CallIsinstance + | Opcode::CallListAppend + | Opcode::CallMethodDescriptorO + | Opcode::CallMethodDescriptorFastWithKeywords + | Opcode::CallMethodDescriptorNoargs + | Opcode::CallMethodDescriptorFast + | Opcode::CallAllocAndEnterInit + | Opcode::CallPyGeneral + | Opcode::CallBoundMethodGeneral + | Opcode::CallNonPyGeneral => Opcode::Call, + Opcode::CallKwBoundMethod | Opcode::CallKwPy | Opcode::CallKwNonPy => { + Opcode::CallKw + } + _ => return None, + }) + } + + pub(super) const fn deoptimize(op: Opcode) -> Opcode { + match deopt(op) { + Some(v) => v, + None => match op.to_base() { + Some(v) => v, + None => op, + }, + } + } + + pub(super) const fn cache_entries(op: Opcode) -> usize { + match deoptimize(op) { + Opcode::StoreSubscr => 1, + Opcode::ToBool => 3, + Opcode::BinaryOp => 5, + Opcode::Call => 3, + Opcode::CallKw => 3, + Opcode::CompareOp => 1, + Opcode::ContainsOp => 1, + Opcode::ForIter => 1, + Opcode::JumpBackward => 1, + Opcode::LoadAttr => 9, + Opcode::LoadGlobal => 4, + Opcode::LoadSuperAttr => 1, + Opcode::PopJumpIfFalse => 1, + Opcode::PopJumpIfNone => 1, + Opcode::PopJumpIfNotNone => 1, + Opcode::PopJumpIfTrue => 1, + Opcode::Send => 1, + Opcode::StoreAttr => 4, + Opcode::UnpackSequence => 1, + _ => 0, + } + } + } + + #[test] + fn cache_entries_and_deopt_tables_match_reference_impl() { + let mut checked = 0; + for byte in 0u8..=255 { + let Ok(op) = Opcode::try_from_u8(byte) else { + continue; + }; + + assert_eq!( + op.deopt(), + reference::deopt(op), + "deopt() mismatch for {op:?}" + ); + assert_eq!( + op.cache_entries(), + reference::cache_entries(op), + "cache_entries() mismatch for {op:?}" + ); + checked += 1; + } + + // Sanity check that the loop actually exercised opcodes rather than + // silently skipping all of them. + assert!(checked > 200); + } } diff --git a/crates/compiler-core/src/bytecode/opcode_metadata.rs b/crates/compiler-core/src/bytecode/opcode_metadata.rs index 64c8d3c5330..050b53c1348 100644 --- a/crates/compiler-core/src/bytecode/opcode_metadata.rs +++ b/crates/compiler-core/src/bytecode/opcode_metadata.rs @@ -12,108 +12,283 @@ impl super::Opcode { #[must_use] pub const fn cache_entries(self) -> usize { - match self.deoptimize() { - Self::StoreSubscr => 1, - Self::ToBool => 3, - Self::BinaryOp => 5, - Self::Call => 3, - Self::CallKw => 3, - Self::CompareOp => 1, - Self::ContainsOp => 1, - Self::ForIter => 1, - Self::JumpBackward => 1, - Self::LoadAttr => 9, - Self::LoadGlobal => 4, - Self::LoadSuperAttr => 1, - Self::PopJumpIfFalse => 1, - Self::PopJumpIfNone => 1, - Self::PopJumpIfNotNone => 1, - Self::PopJumpIfTrue => 1, - Self::Send => 1, - Self::StoreAttr => 4, - Self::UnpackSequence => 1, - _ => 0, - } + const CACHE_ENTRIES: [u8; 256] = [ + 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 3, 1, 1, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 4, 0, 0, 0, 1, 0, 0, 0, 1, 1, 1, 1, 0, 0, 1, 0, 0, 0, 4, 0, 0, 0, 0, 0, + 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 3, 3, + 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 4, 4, 1, 1, 0, 1, 4, 4, 4, 1, 1, + 3, 3, 3, 3, 3, 3, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 1, 1, 1, 0, 0, 0, 0, 1, 3, 3, 0, 1, 0, 0, + ]; + + CACHE_ENTRIES[self.as_numeric() as usize] as usize } #[must_use] pub const fn deopt(self) -> Option { - Some(match self { - Self::ResumeCheck => Self::Resume, - Self::LoadConstMortal | Self::LoadConstImmortal => Self::LoadConst, - Self::ToBoolAlwaysTrue - | Self::ToBoolBool - | Self::ToBoolInt - | Self::ToBoolList - | Self::ToBoolNone - | Self::ToBoolStr => Self::ToBool, - Self::BinaryOpMultiplyInt - | Self::BinaryOpAddInt - | Self::BinaryOpSubtractInt - | Self::BinaryOpMultiplyFloat - | Self::BinaryOpAddFloat - | Self::BinaryOpSubtractFloat - | Self::BinaryOpAddUnicode - | Self::BinaryOpSubscrListInt - | Self::BinaryOpSubscrListSlice - | Self::BinaryOpSubscrTupleInt - | Self::BinaryOpSubscrStrInt - | Self::BinaryOpSubscrDict - | Self::BinaryOpSubscrGetitem - | Self::BinaryOpExtend - | Self::BinaryOpInplaceAddUnicode => Self::BinaryOp, - Self::StoreSubscrDict | Self::StoreSubscrListInt => Self::StoreSubscr, - Self::SendGen => Self::Send, - Self::UnpackSequenceTwoTuple | Self::UnpackSequenceTuple | Self::UnpackSequenceList => { - Self::UnpackSequence - } - Self::StoreAttrInstanceValue | Self::StoreAttrSlot | Self::StoreAttrWithHint => { - Self::StoreAttr - } - Self::LoadGlobalModule | Self::LoadGlobalBuiltin => Self::LoadGlobal, - Self::LoadSuperAttrAttr | Self::LoadSuperAttrMethod => Self::LoadSuperAttr, - Self::LoadAttrInstanceValue - | Self::LoadAttrModule - | Self::LoadAttrWithHint - | Self::LoadAttrSlot - | Self::LoadAttrClass - | Self::LoadAttrClassWithMetaclassCheck - | Self::LoadAttrProperty - | Self::LoadAttrGetattributeOverridden - | Self::LoadAttrMethodWithValues - | Self::LoadAttrMethodNoDict - | Self::LoadAttrMethodLazyDict - | Self::LoadAttrNondescriptorWithValues - | Self::LoadAttrNondescriptorNoDict => Self::LoadAttr, - Self::CompareOpFloat | Self::CompareOpInt | Self::CompareOpStr => Self::CompareOp, - Self::ContainsOpSet | Self::ContainsOpDict => Self::ContainsOp, - Self::JumpBackwardNoJit | Self::JumpBackwardJit => Self::JumpBackward, - Self::ForIterList | Self::ForIterTuple | Self::ForIterRange | Self::ForIterGen => { - Self::ForIter - } - Self::CallBoundMethodExactArgs - | Self::CallPyExactArgs - | Self::CallType1 - | Self::CallStr1 - | Self::CallTuple1 - | Self::CallBuiltinClass - | Self::CallBuiltinO - | Self::CallBuiltinFast - | Self::CallBuiltinFastWithKeywords - | Self::CallLen - | Self::CallIsinstance - | Self::CallListAppend - | Self::CallMethodDescriptorO - | Self::CallMethodDescriptorFastWithKeywords - | Self::CallMethodDescriptorNoargs - | Self::CallMethodDescriptorFast - | Self::CallAllocAndEnterInit - | Self::CallPyGeneral - | Self::CallBoundMethodGeneral - | Self::CallNonPyGeneral => Self::Call, - Self::CallKwBoundMethod | Self::CallKwPy | Self::CallKwNonPy => Self::CallKw, - _ => return None, - }) + const DEOPT: [Option; 256] = [ + None, + None, + None, + Some(super::Opcode::BinaryOp), + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + Some(super::Opcode::BinaryOp), + Some(super::Opcode::BinaryOp), + Some(super::Opcode::BinaryOp), + Some(super::Opcode::BinaryOp), + Some(super::Opcode::BinaryOp), + Some(super::Opcode::BinaryOp), + Some(super::Opcode::BinaryOp), + Some(super::Opcode::BinaryOp), + Some(super::Opcode::BinaryOp), + Some(super::Opcode::BinaryOp), + Some(super::Opcode::BinaryOp), + Some(super::Opcode::BinaryOp), + Some(super::Opcode::BinaryOp), + Some(super::Opcode::BinaryOp), + Some(super::Opcode::Call), + Some(super::Opcode::Call), + Some(super::Opcode::Call), + Some(super::Opcode::Call), + Some(super::Opcode::Call), + Some(super::Opcode::Call), + Some(super::Opcode::Call), + Some(super::Opcode::Call), + Some(super::Opcode::CallKw), + Some(super::Opcode::CallKw), + Some(super::Opcode::CallKw), + Some(super::Opcode::Call), + Some(super::Opcode::Call), + Some(super::Opcode::Call), + Some(super::Opcode::Call), + Some(super::Opcode::Call), + Some(super::Opcode::Call), + Some(super::Opcode::Call), + Some(super::Opcode::Call), + Some(super::Opcode::Call), + Some(super::Opcode::Call), + Some(super::Opcode::Call), + Some(super::Opcode::Call), + Some(super::Opcode::CompareOp), + Some(super::Opcode::CompareOp), + Some(super::Opcode::CompareOp), + Some(super::Opcode::ContainsOp), + Some(super::Opcode::ContainsOp), + Some(super::Opcode::ForIter), + Some(super::Opcode::ForIter), + Some(super::Opcode::ForIter), + Some(super::Opcode::ForIter), + Some(super::Opcode::JumpBackward), + Some(super::Opcode::JumpBackward), + Some(super::Opcode::LoadAttr), + Some(super::Opcode::LoadAttr), + Some(super::Opcode::LoadAttr), + Some(super::Opcode::LoadAttr), + Some(super::Opcode::LoadAttr), + Some(super::Opcode::LoadAttr), + Some(super::Opcode::LoadAttr), + Some(super::Opcode::LoadAttr), + Some(super::Opcode::LoadAttr), + Some(super::Opcode::LoadAttr), + Some(super::Opcode::LoadAttr), + Some(super::Opcode::LoadAttr), + Some(super::Opcode::LoadAttr), + Some(super::Opcode::LoadConst), + Some(super::Opcode::LoadConst), + Some(super::Opcode::LoadGlobal), + Some(super::Opcode::LoadGlobal), + Some(super::Opcode::LoadSuperAttr), + Some(super::Opcode::LoadSuperAttr), + Some(super::Opcode::Resume), + Some(super::Opcode::Send), + Some(super::Opcode::StoreAttr), + Some(super::Opcode::StoreAttr), + Some(super::Opcode::StoreAttr), + Some(super::Opcode::StoreSubscr), + Some(super::Opcode::StoreSubscr), + Some(super::Opcode::ToBool), + Some(super::Opcode::ToBool), + Some(super::Opcode::ToBool), + Some(super::Opcode::ToBool), + Some(super::Opcode::ToBool), + Some(super::Opcode::ToBool), + Some(super::Opcode::UnpackSequence), + Some(super::Opcode::UnpackSequence), + Some(super::Opcode::UnpackSequence), + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + ]; + + DEOPT[self.as_numeric() as usize] } /// Does this opcode have 'HAS_ARG_FLAG' set. diff --git a/tools/opcode_metadata/generate_rs_opcode_metadata.py b/tools/opcode_metadata/generate_rs_opcode_metadata.py index fd13e026613..17db77c94e1 100644 --- a/tools/opcode_metadata/generate_rs_opcode_metadata.py +++ b/tools/opcode_metadata/generate_rs_opcode_metadata.py @@ -146,21 +146,25 @@ def fn_to_instrumented(self) -> str: @property def fn_deopt(self) -> str: - arms = "" - for target, specialized in self.info.deopts.items(): - ops = "|".join(f"Self::{op}" for op in specialized) - arms += f"{ops} => Self::{target},\n" + specialized_to_base = self.specialized_to_base - arms = arms.strip() - - if not arms: + if not specialized_to_base: inner = "None" else: + table_type = f"super::{self.info.enum_name}" + entries = ",\n".join( + f"Some({table_type}::{specialized_to_base[name]})" + if name in specialized_to_base + else "None" + for name in self.rust_names_by_id + ) + inner = f""" - Some(match self {{ - {arms} - _ => return None, - }}) + const DEOPT: [Option<{table_type}>; {self.table_size}] = [ + {entries} + ]; + + DEOPT[self.as_numeric() as usize] """ return f""" @@ -172,7 +176,7 @@ def fn_deopt(self) -> str: @property def fn_cache_entries(self) -> str: - arms = "" + entries_by_base: dict[str, int] = {} for opcode in self: name = opcode.rust_name if opcode.is_instrumented: @@ -186,17 +190,20 @@ def fn_cache_entries(self) -> str: continue if size > 1: - arms += f"Self::{name} => {size - 1},\n" + entries_by_base[name] = size - 1 - arms = arms.strip() - if not arms: + if not entries_by_base: inner = "0" else: + entries = ", ".join( + str(entries_by_base.get(self.resolve_deoptimized(name), 0)) + for name in self.rust_names_by_id + ) + inner = f""" - match self.deoptimize() {{ - {arms} - _ => 0, - }} + const CACHE_ENTRIES: [u8; {self.table_size}] = [{entries}]; + + CACHE_ENTRIES[self.as_numeric() as usize] as usize """ return f""" @@ -323,6 +330,42 @@ def instrumented_mapping(self) -> dict[str, str]: return res + @property + def specialized_to_base(self) -> dict[str, str]: + """Maps a specialized opcode's name to its family's base opcode name.""" + res = {} + for target, specialized in self.info.deopts.items(): + for name in specialized: + res[name] = target + + return res + + @property + def instrumented_to_base(self) -> dict[str, str]: + """Maps an instrumented opcode's name to its base opcode name.""" + return {iname: name for name, iname in self.instrumented_mapping.items()} + + def resolve_deoptimized(self, name: str) -> str: + """ + Mirrors `deoptimize`: resolves a specialized opcode to its family's + base, an instrumented opcode to its base, or returns the name + unchanged. + """ + if name in self.specialized_to_base: + return self.specialized_to_base[name] + + return self.instrumented_to_base.get(name, name) + + @property + def table_size(self) -> int: + return {"u8": 256, "u16": 65536}[self.size] + + @property + def rust_names_by_id(self) -> list[str | None]: + """The opcode name at each numeric id, `None` where no opcode is assigned.""" + names_by_id = {opcode.id: opcode.rust_name for opcode in self} + return [names_by_id.get(i) for i in range(self.table_size)] + @property def size(self) -> str: return self.info.size From efc62e31861367295f6b4cd26aedd44d7b513242 Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Mon, 6 Jul 2026 04:25:15 +0900 Subject: [PATCH 20/92] Make Opcode/Instruction numeric conversions O(1) Opcode::as_numeric, Opcode::as_instruction and Instruction::as_opcode were each a per-variant match over ~230 arms in the define_opcodes! macro. Opcode now carries the same explicit $op_id discriminants and #[repr($typ)] as Instruction, which makes the conversions sound as: - Opcode::as_numeric: a plain `self as $typ` identity cast. - Opcode::as_instruction / Instruction::as_opcode: a mem::transmute, relying on both enums sharing one-$typ-wide layout (Instruction's payload fields are all the zero-sized Arg marker, checked by a new per-instantiation size_of assertion). try_from_numeric is left as a match: $op_id values have gaps (specialized/instrumented opcode ranges), so a range check can't replace it. Adds #[inline] to the rewritten conversions plus deoptimize, and to the small generated wrapper/table-lookup functions from the previous opcode-metadata change (as_u8/as_u16, cache_entries, deopt, to_base) via generate_rs_opcode_metadata.py, then regenerates opcode_metadata.rs. Extends the instruction.rs equivalence-test pattern with two exhaustive tests (u8 and u16 instantiations) checking the new identity-cast/transmute conversions against the untouched try_from_numeric/TryFrom paths. Assisted-by: Claude --- .../compiler-core/src/bytecode/instruction.rs | 118 +++++++++++++++--- .../src/bytecode/opcode_metadata.rs | 8 ++ .../generate_rs_opcode_metadata.py | 4 + 3 files changed, 116 insertions(+), 14 deletions(-) diff --git a/crates/compiler-core/src/bytecode/instruction.rs b/crates/compiler-core/src/bytecode/instruction.rs index 8f488d233c0..63e29222570 100644 --- a/crates/compiler-core/src/bytecode/instruction.rs +++ b/crates/compiler-core/src/bytecode/instruction.rs @@ -18,23 +18,29 @@ macro_rules! define_opcodes { } ) => { #[derive(Clone, Copy, Debug, Eq, PartialEq)] + #[repr($typ)] $opcode_vis enum $opcode_name { - $($op_name),* + $($op_name = $op_id),* } impl $opcode_name { #[doc = concat!("Converts this opcode to [`", stringify!($instr_name), "`].")] #[must_use] + #[inline] $opcode_vis const fn as_instruction(&self) -> $instr_name { - match self { - $( - Self::$op_name => $instr_name::$op_name $({ $arg_name: Arg::marker() })?, - )* - } + // SAFETY: `$opcode_name` and `$instr_name` are both `#[repr($typ)]` + // enums sharing identical explicit discriminants, and every + // `$instr_name` payload field is the zero-sized `Arg` marker + // (see the `size_of` assertion near its definition), so both + // enums have the same one-`$typ`-wide representation: just the + // discriminant. Converting a live `$opcode_name` value therefore + // yields the `$instr_name` variant with the matching discriminant. + unsafe { core::mem::transmute(*self) } } /// Map a specialized or instrumented opcode back to its adaptive (base) variant. #[must_use] + #[inline] $opcode_vis const fn deoptimize(self) -> Self { match self.deopt() { Some(v) => v, @@ -49,6 +55,9 @@ macro_rules! define_opcodes { } // NOTE: Keep private. Will be exposed under `try_from_u8/try_from_u16`. + // Kept as a match rather than a range check + transmute: `$op_id` + // values are not contiguous (specialized/instrumented opcodes leave + // gaps), so validity can't be expressed as a simple bound. pub(super) const fn try_from_numeric(value: $typ) -> Result { match value { $($op_id => Ok(Self::$op_name),)* @@ -58,10 +67,11 @@ macro_rules! define_opcodes { // NOTE: Keep private. Will be exposed under `as_u8/as_u16`. #[must_use] + #[inline] pub(super) const fn as_numeric(self) -> $typ { - match self { - $(Self::$op_name => $op_id,)* - } + // `$opcode_name` is `#[repr($typ)]` with an explicit `$op_id` + // discriminant on every variant, so this is a plain identity cast. + self as $typ } } @@ -95,15 +105,24 @@ macro_rules! define_opcodes { ),* } + // Every `$instr_name` payload field is the zero-sized `Arg` marker, so + // (combined with the `#[repr($typ)]` above) each variant's representation + // is exactly its `$typ` discriminant with no padding. `as_opcode` and + // `$opcode_name::as_instruction` rely on this to convert via + // `mem::transmute` instead of a per-variant match. + const _: () = assert!(core::mem::size_of::<$instr_name>() == core::mem::size_of::<$typ>()); + impl $instr_name { #[doc = concat!("Get the corresponding [`", stringify!($opcode_name), "`].")] #[must_use] + #[inline] $instr_vis const fn as_opcode(&self) -> $opcode_name { - match self { - $( - Self::$op_name $({ $arg_name: _ })? => $opcode_name::$op_name, - )* - } + // SAFETY: symmetric to `$opcode_name::as_instruction` above: + // `*self`'s representation is exactly its `$typ` discriminant + // (checked by the `size_of` assertion above this impl), and that + // discriminant is always a valid `$opcode_name` discriminant + // because both enums share the same explicit `$op_id` list. + unsafe { core::mem::transmute(*self) } } #[must_use] @@ -1603,4 +1622,75 @@ mod tests { // silently skipping all of them. assert!(checked > 200); } + + /// `Opcode::as_numeric`, `Opcode::as_instruction` and + /// `Instruction::as_opcode` used to be per-variant matches; they are now + /// an identity cast and two `mem::transmute`s respectively. `byte` (an + /// input independent of any of those three functions) together with the + /// untouched `try_from_u8`/`TryFrom` conversions serve as the + /// reference: every opcode reachable from a byte must convert back to + /// that exact byte and round-trip through `Instruction`. + #[test] + fn opcode_instruction_numeric_conversions_match_try_from_numeric() { + let mut checked = 0; + for byte in 0u8..=255 { + let Ok(op) = Opcode::try_from_u8(byte) else { + continue; + }; + + assert_eq!(op.as_numeric(), byte, "as_numeric() mismatch for {op:?}"); + + let instr = op.as_instruction(); + assert_eq!( + instr.as_opcode(), + op, + "as_instruction()/as_opcode() round trip mismatch for {op:?}" + ); + + let instr_via_try_from = Instruction::try_from(byte).unwrap(); + assert_eq!( + instr_via_try_from.as_opcode(), + op, + "Instruction::try_from({byte}) mismatch" + ); + + checked += 1; + } + + assert!(checked > 200); + } + + /// Same as [`opcode_instruction_numeric_conversions_match_try_from_numeric`] + /// but for the `u16`-discriminant pseudo-opcode instantiation of + /// `define_opcodes!`. + #[test] + fn pseudo_opcode_instruction_numeric_conversions_match_try_from_numeric() { + let mut checked = 0; + for value in 0u16..=u16::MAX { + let Ok(op) = PseudoOpcode::try_from_u16(value) else { + continue; + }; + + assert_eq!(op.as_numeric(), value, "as_numeric() mismatch for {op:?}"); + + let instr = op.as_instruction(); + assert_eq!( + instr.as_opcode(), + op, + "as_instruction()/as_opcode() round trip mismatch for {op:?}" + ); + + let instr_via_try_from = PseudoInstruction::try_from(value).unwrap(); + assert_eq!( + instr_via_try_from.as_opcode(), + op, + "PseudoInstruction::try_from({value}) mismatch" + ); + + checked += 1; + } + + // All 11 `PseudoInstruction` variants should have been exercised. + assert_eq!(checked, 11); + } } diff --git a/crates/compiler-core/src/bytecode/opcode_metadata.rs b/crates/compiler-core/src/bytecode/opcode_metadata.rs index 050b53c1348..16db49bea0d 100644 --- a/crates/compiler-core/src/bytecode/opcode_metadata.rs +++ b/crates/compiler-core/src/bytecode/opcode_metadata.rs @@ -6,11 +6,13 @@ use crate::{bytecode::instruction::StackEffect, marshal::MarshalError}; impl super::Opcode { /// Returns [`Self`] as [`u8`]. #[must_use] + #[inline] pub const fn as_u8(self) -> u8 { self.as_numeric() } #[must_use] + #[inline] pub const fn cache_entries(self) -> usize { const CACHE_ENTRIES: [u8; 256] = [ 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -28,6 +30,7 @@ impl super::Opcode { } #[must_use] + #[inline] pub const fn deopt(self) -> Option { const DEOPT: [Option; 256] = [ None, @@ -839,6 +842,7 @@ impl super::Opcode { } #[must_use] + #[inline] pub const fn to_base(self) -> Option { Some(match self { Self::InstrumentedCall => Self::Call, @@ -898,16 +902,19 @@ impl super::Opcode { impl super::PseudoOpcode { /// Returns [`Self`] as [`u16`]. #[must_use] + #[inline] pub const fn as_u16(self) -> u16 { self.as_numeric() } #[must_use] + #[inline] pub const fn cache_entries(self) -> usize { 0 } #[must_use] + #[inline] pub const fn deopt(self) -> Option { None } @@ -993,6 +1000,7 @@ impl super::PseudoOpcode { } #[must_use] + #[inline] pub const fn to_base(self) -> Option { None } diff --git a/tools/opcode_metadata/generate_rs_opcode_metadata.py b/tools/opcode_metadata/generate_rs_opcode_metadata.py index 17db77c94e1..97482f337a1 100644 --- a/tools/opcode_metadata/generate_rs_opcode_metadata.py +++ b/tools/opcode_metadata/generate_rs_opcode_metadata.py @@ -28,6 +28,7 @@ def fn_as_info_size(self) -> str: return f""" /// Returns [`Self`] as [`{self.size}`]. #[must_use] + #[inline] pub const fn as_{self.size}(self) -> {self.size} {{ self.as_numeric() }} @@ -113,6 +114,7 @@ def fn_to_base(self) -> str: return f""" #[must_use] + #[inline] pub const fn to_base(self) -> Option {{ {inner} }} @@ -169,6 +171,7 @@ def fn_deopt(self) -> str: return f""" #[must_use] + #[inline] pub const fn deopt(self) -> Option {{ {inner} }} @@ -208,6 +211,7 @@ def fn_cache_entries(self) -> str: return f""" #[must_use] + #[inline] pub const fn cache_entries(self) -> usize {{ {inner} }} From 7e4c76595fe7a34189bf55f7153916da0e9c68fe Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Mon, 6 Jul 2026 04:47:53 +0900 Subject: [PATCH 21/92] Pop CallIsinstance arguments directly instead of collecting into Vecs The specialized handler built two heap-allocated Vecs per call (pop_multiple().collect() plus a with_capacity(2) buffer) before invoking is_instance. Pop the class and instance straight off the stack for the effective two-argument case, removing the per-call allocations. Assisted-by: Claude --- crates/vm/src/frame.rs | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/crates/vm/src/frame.rs b/crates/vm/src/frame.rs index 0545b6b2d66..5264bc6a4b8 100644 --- a/crates/vm/src/frame.rs +++ b/crates/vm/src/frame.rs @@ -4532,16 +4532,18 @@ impl ExecutingFrame<'_> { .as_ref() .is_some_and(|isinstance_callable| callable.is(isinstance_callable)) { - let nargs_usize = nargs as usize; - let pos_args: Vec = self.pop_multiple(nargs_usize).collect(); - let self_or_null = self.pop_value_opt(); + // Stack: [callable, self_or_null, args...]; effective_nargs == 2, + // so the instance is either the first positional arg or self_or_null. + let cls = self.pop_value(); + let inst = if nargs == 2 { + let inst = self.pop_value(); + self.pop_value_opt(); // null + inst + } else { + self.pop_value() // self_or_null holds the instance + }; self.pop_value(); // callable - let mut all_args = Vec::with_capacity(2); - if let Some(self_val) = self_or_null { - all_args.push(self_val); - } - all_args.extend(pos_args); - let result = all_args[0].is_instance(&all_args[1], vm)?; + let result = inst.is_instance(&cls, vm)?; self.push_value(vm.ctx.new_bool(result).into()); return Ok(None); } From d9796eafefac2c95144f28f61a98d7180e17315e Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Mon, 6 Jul 2026 05:21:18 +0900 Subject: [PATCH 22/92] Inherit tp_new instead of always installing new_wrapper update_one_slot's TpNew branch stored new_wrapper for every ADD, so all heap types had slots.new == new_wrapper and the CallAllocAndEnterInit specialization never fired. Now __new__ is resolved through the MRO dicts: an own or Python-level definition installs new_wrapper, while a builtin __new__ entry (or none) inherits slots.new from the solid base. The now-reachable CallAllocAndEnterInit path gains the missing guards: the specializer requires co_argcount == oparg + 1 (via can_specialize_call) and rejects generator-like __init__, and the handler re-checks the argcount against the class-level init cache. Without the argcount check, __init__ defaults broke re._parser (UnboundLocalError in SubPattern.__init__). Remove the now-unused is_simple_for_call_specialization. Assisted-by: Claude --- crates/vm/src/builtins/function.rs | 20 ++++++++++---------- crates/vm/src/frame.rs | 4 +++- crates/vm/src/types/slot.rs | 25 ++++++++++++++++++++++--- 3 files changed, 35 insertions(+), 14 deletions(-) diff --git a/crates/vm/src/builtins/function.rs b/crates/vm/src/builtins/function.rs index 4125ef4c4c6..3e98d7898d6 100644 --- a/crates/vm/src/builtins/function.rs +++ b/crates/vm/src/builtins/function.rs @@ -634,16 +634,6 @@ impl Py { new_v } - /// function_kind(SIMPLE_FUNCTION) equivalent for CALL specialization. - /// Returns true if: CO_OPTIMIZED, no VARARGS, no VARKEYWORDS, no kwonly args. - pub(crate) fn is_simple_for_call_specialization(&self) -> bool { - let code: &Py = &self.code; - let flags = code.flags; - flags.contains(bytecode::CodeFlags::OPTIMIZED) - && !flags.intersects(bytecode::CodeFlags::VARARGS | bytecode::CodeFlags::VARKEYWORDS) - && code.kwonlyarg_count == 0 - } - /// Check if this function is eligible for exact-args call specialization. /// Returns true if: CO_OPTIMIZED, no VARARGS, no VARKEYWORDS, no kwonly args, /// and effective_nargs matches co_argcount. @@ -656,6 +646,16 @@ impl Py { && code.arg_count == effective_nargs } + /// True if the code object is a generator, coroutine or async generator. + #[inline] + pub(crate) fn is_generator_like(&self) -> bool { + self.code.flags.intersects( + bytecode::CodeFlags::GENERATOR + | bytecode::CodeFlags::COROUTINE + | bytecode::CodeFlags::ASYNC_GENERATOR, + ) + } + /// Runtime guard for CALL_*_EXACT_ARGS specialization: check only argcount. /// Other invariants are guaranteed by function versioning and specialization-time checks. #[inline] diff --git a/crates/vm/src/frame.rs b/crates/vm/src/frame.rs index 5264bc6a4b8..8f1af471cfc 100644 --- a/crates/vm/src/frame.rs +++ b/crates/vm/src/frame.rs @@ -4983,6 +4983,7 @@ impl ExecutingFrame<'_> { && let Some(cls) = callable.downcast_ref::() && cls.tp_version_tag.load(Acquire) == cached_version && let Some(init_func) = cls.get_cached_init_for_specialization(cached_version) + && init_func.has_exact_argcount(nargs + 1) && let Some(cls_alloc) = cls.slots.alloc.load() { // Match CPython's `code->co_framesize + _Py_InitCleanup.co_framesize` @@ -8576,7 +8577,8 @@ impl ExecutingFrame<'_> { } if let Some(init) = init && let Some(init_func) = init.downcast_ref_if_exact::(vm) - && init_func.is_simple_for_call_specialization() + && init_func.can_specialize_call(nargs + 1) + && !init_func.is_generator_like() && cls.cache_init_for_specialization(init_func.to_owned(), version, vm) { unsafe { diff --git a/crates/vm/src/types/slot.rs b/crates/vm/src/types/slot.rs index 4e21b7f830d..6e00c160975 100644 --- a/crates/vm/src/types/slot.rs +++ b/crates/vm/src/types/slot.rs @@ -843,12 +843,31 @@ impl PyType { } } SlotAccessor::TpNew => { - // __new__ is not wrapped via PyWrapper - if ADD { + // __new__ is a staticmethod, not a PyWrapper descriptor, so + // lookup_slot_in_mro cannot classify it. Resolve __new__ + // through the MRO dicts instead: a Python-level definition + // needs the dynamic new_wrapper, while a native type's + // builtin __new__ entry (or no entry at all) means the slot + // is inherited from the solid base, matching update_one_slot's + // tp_new special case over the tp_base-inherited value. + let needs_wrapper = if ADD && self.attributes.read().contains_key(name) { + true + } else { + // mro[0] is self, so skip it + self.mro.read()[1..] + .iter() + .find(|cls| cls.attributes.read().contains_key(name)) + .is_some_and(|cls| { + cls.slots.new.load().map(|f| f as usize) + == Some(new_wrapper as NewFunc as usize) + }) + }; + if needs_wrapper { self.slots.new.store(Some(new_wrapper)); self.slots.vectorcall.store(None); } else { - accessor.inherit_from_mro(self); + let inherited = self.base.as_ref().and_then(|base| base.slots.new.load()); + self.slots.new.store(inherited); } } SlotAccessor::TpDel => update_main_slot!(del, del_wrapper, Del), From dd72583e2cb98762c56a68b6a52267a0f9a9175d Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Mon, 6 Jul 2026 05:25:07 +0900 Subject: [PATCH 23/92] Skip locals dict allocation for the init-cleanup shim frame Mark the shim code object NEWLOCALS and create the shim frame with no locals mapping so FrameLocals::lazy() is used. The shim only executes ExitInitCheck/ReturnValue and never touches locals, so this removes one dict allocation per specialized instantiation. Assisted-by: Claude --- crates/vm/src/frame.rs | 8 ++++---- crates/vm/src/vm/context.rs | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/crates/vm/src/frame.rs b/crates/vm/src/frame.rs index 8f1af471cfc..a592b0bf67b 100644 --- a/crates/vm/src/frame.rs +++ b/crates/vm/src/frame.rs @@ -1402,12 +1402,12 @@ impl ExecutingFrame<'_> { } fn specialization_new_init_cleanup_frame(&self, vm: &VirtualMachine) -> FrameRef { + // The shim code has NEWLOCALS, so passing no locals selects + // FrameLocals::lazy() and no locals dict is allocated; the shim + // never touches locals. Frame::new( vm.ctx.init_cleanup_code.clone(), - Scope::new( - Some(ArgMapping::from_dict_exact(vm.ctx.new_dict())), - self.globals.clone(), - ), + Scope::new(None, self.globals.clone()), self.builtins.clone(), &[], None, diff --git a/crates/vm/src/vm/context.rs b/crates/vm/src/vm/context.rs index 226d5f1a1a7..a64d8410b61 100644 --- a/crates/vm/src/vm/context.rs +++ b/crates/vm/src/vm/context.rs @@ -421,7 +421,7 @@ impl Context { let code = bytecode::CodeObject { instructions: instructions.into(), locations: vec![(loc, loc); instructions.len()].into_boxed_slice(), - flags: CodeFlags::OPTIMIZED, + flags: CodeFlags::OPTIMIZED | CodeFlags::NEWLOCALS, posonlyarg_count: 0, arg_count: 0, kwonlyarg_count: 0, From da6633d67ccab95926f1a72ca78683ce47f4262c Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Mon, 6 Jul 2026 05:46:29 +0900 Subject: [PATCH 24/92] Guard CALL_ALLOC_AND_ENTER_INIT with cached function version The init specialization cache stored only the type version, so swapping __init__.__code__ (which zeroes func_version) did not deopt the warmed call site and stale code assumptions (argcount shape, no varargs/kwonly, not a generator) were applied to the new code object. Store get_version_for_current_state() in the specialization cache when caching __init__ and re-check func_version() in the handler before using the cached function, deopting to the generic call path on mismatch, the same way the getitem specialization cache does. Assisted-by: Claude --- crates/vm/src/builtins/type.rs | 27 ++++++++++++++++++++++++--- crates/vm/src/frame.rs | 4 +++- 2 files changed, 27 insertions(+), 4 deletions(-) diff --git a/crates/vm/src/builtins/type.rs b/crates/vm/src/builtins/type.rs index 800b658303d..7ac0b104d57 100644 --- a/crates/vm/src/builtins/type.rs +++ b/crates/vm/src/builtins/type.rs @@ -293,6 +293,7 @@ pub struct HeapTypeExt { pub struct TypeSpecializationCache { pub init: PyAtomicRef>, + pub init_version: AtomicU32, pub getitem: PyAtomicRef>, pub getitem_version: AtomicU32, } @@ -301,6 +302,7 @@ impl TypeSpecializationCache { fn new() -> Self { Self { init: PyAtomicRef::from(None::>), + init_version: AtomicU32::new(0), getitem: PyAtomicRef::from(None::>), getitem_version: AtomicU32::new(0), } @@ -335,6 +337,7 @@ impl TypeSpecializationCache { #[inline] fn invalidate_for_type_modified(&self) { self.swap_init(None); + self.init_version.store(0, Ordering::Release); self.swap_getitem(None); self.getitem_version.store(0, Ordering::Release); } @@ -353,6 +356,7 @@ impl TypeSpecializationCache { if let Some(old_init) = old_init { out.push(old_init.into()); } + self.init_version.store(0, Ordering::Release); let old_getitem = unsafe { self.getitem.swap(None) }; if let Some(old_getitem) = old_getitem { out.push(old_getitem.into()); @@ -1102,7 +1106,14 @@ impl PyType { if self.tp_version_tag.load(Ordering::Acquire) != tp_version { return false; } + let func_version = init.get_version_for_current_state(); + if func_version == 0 { + return false; + } ext.specialization_cache.swap_init(Some(init)); + ext.specialization_cache + .init_version + .store(func_version, Ordering::Release); true }) } @@ -1111,7 +1122,7 @@ impl PyType { pub(crate) fn get_cached_init_for_specialization( &self, tp_version: u32, - ) -> Option> { + ) -> Option<(PyRef, u32)> { let ext = self.heaptype_ext.as_ref()?; if tp_version == 0 { return None; @@ -1119,9 +1130,19 @@ impl PyType { if self.tp_version_tag.load(Ordering::Acquire) != tp_version { return None; } - ext.specialization_cache + // Check order: pointer (Acquire) then function version. + let init = ext + .specialization_cache .init - .try_to_owned(Ordering::Acquire) + .try_to_owned(Ordering::Acquire)?; + let cached_version = ext + .specialization_cache + .init_version + .load(Ordering::Relaxed); + if cached_version == 0 { + return None; + } + Some((init, cached_version)) } /// Cache __getitem__ for BINARY_OP_SUBSCR_GETITEM specialization. diff --git a/crates/vm/src/frame.rs b/crates/vm/src/frame.rs index a592b0bf67b..e8800fead9a 100644 --- a/crates/vm/src/frame.rs +++ b/crates/vm/src/frame.rs @@ -4982,7 +4982,9 @@ impl ExecutingFrame<'_> { && cached_version != 0 && let Some(cls) = callable.downcast_ref::() && cls.tp_version_tag.load(Acquire) == cached_version - && let Some(init_func) = cls.get_cached_init_for_specialization(cached_version) + && let Some((init_func, init_func_version)) = + cls.get_cached_init_for_specialization(cached_version) + && init_func.func_version() == init_func_version && init_func.has_exact_argcount(nargs + 1) && let Some(cls_alloc) = cls.slots.alloc.load() { From 6fe1fed40c0fd5c3e9c64b64fbeba989309156fb Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Mon, 6 Jul 2026 06:34:51 +0900 Subject: [PATCH 25/92] Apply rustfmt and ruff-format fixes to files from earlier commits Assisted-by: Claude --- crates/vm/src/object/mod.rs | 2 +- extra_tests/snippets/stdlib_threading_type_cache.py | 8 ++++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/crates/vm/src/object/mod.rs b/crates/vm/src/object/mod.rs index 3d4f0fc9a3b..b06957e1bc6 100644 --- a/crates/vm/src/object/mod.rs +++ b/crates/vm/src/object/mod.rs @@ -1,9 +1,9 @@ mod core; mod ext; mod payload; +pub(crate) mod qsbr; mod traverse; mod traverse_object; -pub(crate) mod qsbr; pub use self::core::*; pub use self::ext::*; diff --git a/extra_tests/snippets/stdlib_threading_type_cache.py b/extra_tests/snippets/stdlib_threading_type_cache.py index de96130a284..92368e8b83d 100644 --- a/extra_tests/snippets/stdlib_threading_type_cache.py +++ b/extra_tests/snippets/stdlib_threading_type_cache.py @@ -9,15 +9,19 @@ published to the type cache it must instead go through the QSBR-deferred reclamation path, so this exercises that bypass. """ + import threading import time + class C: def m(self): return -1 + DURATION = 1.5 + def reader(stop): obj = C() while not stop.is_set(): @@ -31,11 +35,14 @@ def reader(stop): except AttributeError: pass + def mutator(stop): i = 0 while not stop.is_set(): + def m(self, _i=i): return _i + C.m = m C.shape = (i, i + 1) i += 1 @@ -49,6 +56,7 @@ def m(self, _i=i): except AttributeError: pass + stop = threading.Event() threads = [threading.Thread(target=reader, args=(stop,)) for _ in range(4)] threads.append(threading.Thread(target=mutator, args=(stop,))) From d26a55387bf68a31839b61a8d86ac30b0bd1b9b7 Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Mon, 6 Jul 2026 06:34:51 +0900 Subject: [PATCH 26/92] Add frame object freelist Frame.iframe becomes FrameUnsafeCell> so a dead frame can be left as an empty husk. Traverse::clear extracts all owned child references (including localsplus values via LocalsPlus::clear_into) and empties the cell. Dead frame allocations are cached in a thread-local FreeList (up to 200 entries) and reused by PyRef::new_ref. clear_generator returns early when the frame was already cleared by cycle collection; generator drops can run after tp_clear of their frame. gc: skip tp_clear for objects saved to gc.garbage by DEBUG_SAVEALL, since they remain reachable from Python. Assisted-by: Claude --- crates/vm/src/frame.rs | 186 ++++++++++++++++++++++++++++++++++---- crates/vm/src/gc_state.rs | 14 ++- 2 files changed, 178 insertions(+), 22 deletions(-) diff --git a/crates/vm/src/frame.rs b/crates/vm/src/frame.rs index e8800fead9a..19f1afb2e5d 100644 --- a/crates/vm/src/frame.rs +++ b/crates/vm/src/frame.rs @@ -39,6 +39,7 @@ use crate::{ use alloc::fmt; use bstr::ByteSlice; use core::cell::UnsafeCell; +use core::ptr::NonNull; use core::sync::atomic; use core::sync::atomic::AtomicPtr; use core::sync::atomic::Ordering::{Acquire, Relaxed}; @@ -111,6 +112,12 @@ impl FrameUnsafeCell { unsafe fn get(&self) -> *mut T { self.0.get() } + + /// Safe exclusive access through `&mut self`. + #[inline(always)] + fn get_mut(&mut self) -> &mut T { + self.0.get_mut() + } } // SAFETY: Frame execution is single-threaded. See FrameUnsafeCell doc. @@ -220,6 +227,22 @@ impl LocalsPlus { } } + /// Extract all contained values into `out` without freeing the backing + /// storage. Used by tp_clear so that child references are dropped by the + /// caller instead of recursively inside the frame. + fn clear_into(&mut self, out: &mut Vec) { + while !self.stack_is_empty() { + if let Some(val) = self.stack_pop() { + out.push(val.to_pyobj()); + } + } + for slot in self.fastlocals_mut() { + if let Some(obj) = slot.take() { + out.push(obj); + } + } + } + // -- Data access helpers -- #[inline(always)] @@ -624,7 +647,55 @@ pub struct InterpreterFrame { /// Analogous to CPython's `PyFrameObject`. #[pyclass(module = false, name = "frame", traverse = "manual")] pub struct Frame { - pub(crate) iframe: FrameUnsafeCell, + /// Always `Some` while the frame is reachable from Python. Emptied only + /// by `Traverse::clear` during deallocation, leaving a trivially-droppable + /// husk that the freelist can cache. + pub(crate) iframe: FrameUnsafeCell>, +} + +impl Frame { + /// Shared access to the embedded interpreter frame. + /// + /// # Safety + /// Caller must ensure no concurrent mutable access (see `FrameUnsafeCell`) + /// and that the frame has not been cleared (i.e. it is still reachable + /// from Python; `Traverse::clear` only runs during deallocation). + #[inline(always)] + unsafe fn iframe_ref(&self) -> &InterpreterFrame { + let opt = unsafe { &*self.iframe.get() }; + #[cfg(debug_assertions)] + if opt.is_none() { + cleared_frame_access(); + } + // SAFETY: iframe is always Some while the frame is reachable (see above). + unsafe { opt.as_ref().unwrap_unchecked() } + } + + /// Exclusive access to the embedded interpreter frame. + /// + /// # Safety + /// Caller must ensure exclusive access (see `FrameUnsafeCell`) and that + /// the frame has not been cleared. + #[inline(always)] + #[allow(clippy::mut_from_ref)] + unsafe fn iframe_mut(&self) -> &mut InterpreterFrame { + let opt = unsafe { &mut *self.iframe.get() }; + #[cfg(debug_assertions)] + if opt.is_none() { + cleared_frame_access(); + } + // SAFETY: iframe is always Some while the frame is reachable (see above). + unsafe { opt.as_mut().unwrap_unchecked() } + } +} + +/// Out-of-line panic for the debug-only cleared-frame check, keeping the +/// inlined accessors' stack frames minimal. +#[cfg(debug_assertions)] +#[cold] +#[inline(never)] +fn cleared_frame_access() -> ! { + panic!("frame accessed after clear"); } impl core::ops::Deref for Frame { @@ -638,21 +709,66 @@ impl core::ops::Deref for Frame { /// are only mutated during single-threaded execution via `with_exec`. #[inline(always)] fn deref(&self) -> &InterpreterFrame { - unsafe { &*self.iframe.get() } + unsafe { self.iframe_ref() } } } +thread_local! { + /// Free list of dead frame objects for reuse. Entries are cleared husks + /// (`iframe == None`) whose child references were already released. + /// PyInner is fixed-size (localsplus storage is out-of-line), + /// so a single bucket suffices. + static FRAME_FREELIST: core::cell::Cell> = + const { core::cell::Cell::new(crate::object::FreeList::new()) }; +} + impl PyPayload for Frame { + const MAX_FREELIST: usize = 200; + const HAS_FREELIST: bool = true; + #[inline] fn class(ctx: &Context) -> &'static Py { ctx.types.frame_type } + + #[inline] + unsafe fn freelist_push(obj: *mut PyObject) -> bool { + FRAME_FREELIST + .try_with(|fl| { + let mut list = fl.take(); + let stored = if list.len() < Self::MAX_FREELIST { + list.push(obj); + true + } else { + false + }; + fl.set(list); + stored + }) + .unwrap_or(false) + } + + #[inline] + unsafe fn freelist_pop(_payload: &Self) -> Option> { + FRAME_FREELIST + .try_with(|fl| { + let mut list = fl.take(); + let result = list.pop().map(|p| unsafe { NonNull::new_unchecked(p) }); + fl.set(list); + result + }) + .ok() + .flatten() + } } unsafe impl Traverse for Frame { fn traverse(&self, tracer_fn: &mut TraverseFn<'_>) { // SAFETY: GC traversal does not run concurrently with frame execution. - let iframe = unsafe { &*self.iframe.get() }; + // A cleared frame (iframe == None) has no children to visit. + let Some(iframe) = (unsafe { &*self.iframe.get() }) else { + return; + }; iframe.code.traverse(tracer_fn); iframe.func_obj.traverse(tracer_fn); iframe.localsplus.traverse(tracer_fn); @@ -663,6 +779,29 @@ unsafe impl Traverse for Frame { iframe.temporary_refs.traverse(tracer_fn); iframe.f_locals_hidden_overlay.traverse(tracer_fn); } + + fn clear(&mut self, out: &mut Vec) { + let Some(mut iframe) = self.iframe.get_mut().take() else { + return; + }; + // Extract all owned child references for cycle breaking; the + // remaining non-reference fields (atomics, flags, backing storage) + // drop with `iframe` at the end of this scope, leaving the payload + // as a trivially-droppable husk for the freelist. + iframe.localsplus.clear_into(out); + out.push(iframe.code.into()); + out.extend(iframe.func_obj.take()); + if let Some(locals) = iframe.locals.inner.take() { + out.push(locals.into()); + } + out.push(iframe.globals.into()); + out.push(iframe.builtins); + out.push(iframe.trace.into_inner()); + out.append(iframe.temporary_refs.get_mut()); + if let Some(overlay) = iframe.f_locals_hidden_overlay.into_inner() { + out.push(overlay.into()); + } + } } // Running a frame can result in one of the below: @@ -747,7 +886,7 @@ impl Frame { pending_unwind_from_stack: Default::default(), }; Self { - iframe: FrameUnsafeCell::new(iframe), + iframe: FrameUnsafeCell::new(Some(iframe)), } } @@ -758,7 +897,7 @@ impl Frame { /// or called from the same thread during trace callback). #[inline(always)] pub unsafe fn fastlocals(&self) -> &[Option] { - unsafe { (*self.iframe.get()).localsplus.fastlocals() } + unsafe { self.iframe_ref().localsplus.fastlocals() } } /// Access fastlocals mutably. @@ -768,7 +907,7 @@ impl Frame { #[inline(always)] #[allow(clippy::mut_from_ref)] pub unsafe fn fastlocals_mut(&self) -> &mut [Option] { - unsafe { (*self.iframe.get()).localsplus.fastlocals_mut() } + unsafe { self.iframe_mut().localsplus.fastlocals_mut() } } /// Migrate data-stack-backed storage to the heap, preserving all values, @@ -779,7 +918,7 @@ impl Frame { /// Caller must ensure the frame is not executing and the returned /// pointer is passed to `VirtualMachine::datastack_pop()`. pub(crate) unsafe fn materialize_localsplus(&self) -> Option<*mut u8> { - unsafe { (*self.iframe.get()).localsplus.materialize_to_heap() } + unsafe { self.iframe_mut().localsplus.materialize_to_heap() } } /// Clear evaluation stack and state-owned cell/free references. @@ -788,7 +927,7 @@ impl Frame { // SAFETY: Called when frame is not executing (generator closed). // Cell refs in fastlocals[nlocals..] are cleared by clear_locals_and_stack(). unsafe { - (*self.iframe.get()).localsplus.stack_clear(); + self.iframe_mut().localsplus.stack_clear(); } } @@ -797,7 +936,7 @@ impl Frame { pub(crate) fn clear_locals_and_stack(&self) { self.clear_stack_and_cells(); // SAFETY: Frame is not executing (generator closed). - let fastlocals = unsafe { (*self.iframe.get()).localsplus.fastlocals_mut() }; + let fastlocals = unsafe { self.iframe_mut().localsplus.fastlocals_mut() }; for slot in fastlocals.iter_mut() { *slot = None; } @@ -807,7 +946,7 @@ impl Frame { /// Get cell contents by localsplus index. pub(crate) fn get_cell_contents(&self, localsplus_idx: usize) -> Option { // SAFETY: Frame not executing; no concurrent mutation. - let fastlocals = unsafe { (*self.iframe.get()).localsplus.fastlocals() }; + let fastlocals = unsafe { self.iframe_ref().localsplus.fastlocals() }; fastlocals .get(localsplus_idx) .and_then(|slot| slot.as_ref()) @@ -825,8 +964,17 @@ impl Frame { /// Clear the generator back-reference. Called when the generator is finalized. pub fn clear_generator(&self) { - self.generator.clear(); - self.owner + // The generator's drop may run after this frame was already cleared + // by cycle collection (both were garbage and the frame was cleared + // first); nothing to unlink then. + // SAFETY: shared access; the finalizing generator owns the frame, + // which is not executing. + let Some(iframe) = (unsafe { &*self.iframe.get() }) else { + return; + }; + iframe.generator.clear(); + iframe + .owner .store(FrameOwner::FrameObject as i8, atomic::Ordering::Release); } @@ -879,7 +1027,7 @@ impl Frame { .as_ref() .map_or_else(|| self.locals.mapping(vm), ArgMapping::mapping); // SAFETY: Called before generator resume; no concurrent access. - let fastlocals = unsafe { (*self.iframe.get()).localsplus.fastlocals_mut() }; + let fastlocals = unsafe { self.iframe_mut().localsplus.fastlocals_mut() }; for (i, &varname) in code.varnames.iter().enumerate() { if i >= fastlocals.len() { break; @@ -897,7 +1045,7 @@ impl Frame { fn has_active_hidden_locals(&self) -> bool { use rustpython_compiler_core::bytecode::{CO_FAST_CELL, CO_FAST_FREE, CO_FAST_HIDDEN}; let code = &**self.code; - let fastlocals = unsafe { (*self.iframe.get()).localsplus.fastlocals() }; + let fastlocals = unsafe { self.iframe_ref().localsplus.fastlocals() }; let is_optimized = code.flags.contains(bytecode::CodeFlags::OPTIMIZED); !is_optimized && code.localspluskinds.iter().enumerate().any(|(i, &kind)| { @@ -929,7 +1077,7 @@ impl Frame { // SAFETY: Either the frame is not executing (caller checked owner), // or we're in a trace callback on the same thread that's executing. let code = &**self.code; - let fastlocals = unsafe { (*self.iframe.get()).localsplus.fastlocals() }; + let fastlocals = unsafe { self.iframe_ref().localsplus.fastlocals() }; // Iterate through all localsplus slots using localspluskinds let nlocalsplus = code.localspluskinds.len(); @@ -1069,7 +1217,7 @@ impl Py { // SAFETY: Frame execution is single-threaded. Only one thread at a time // executes a given frame (enforced by the owner field and generator // running flag). Same safety argument as FastLocals (UnsafeCell). - let iframe = unsafe { &mut *self.iframe.get() }; + let iframe = unsafe { self.iframe_mut() }; let exec = ExecutingFrame { code: &iframe.code, localsplus: &mut iframe.localsplus, @@ -1129,7 +1277,7 @@ impl Py { return None; } // SAFETY: Frame is not executing, so UnsafeCell access is safe. - let iframe = unsafe { &mut *self.iframe.get() }; + let iframe = unsafe { self.iframe_mut() }; let exec = ExecutingFrame { code: &iframe.code, localsplus: &mut iframe.localsplus, @@ -9652,7 +9800,9 @@ impl fmt::Debug for Frame { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { // SAFETY: Debug is best-effort; concurrent mutation is unlikely // and would only affect debug output. - let iframe = unsafe { &*self.iframe.get() }; + let Some(iframe) = (unsafe { &*self.iframe.get() }) else { + return f.write_str("Frame Object { cleared }"); + }; let stack_str = iframe .localsplus diff --git a/crates/vm/src/gc_state.rs b/crates/vm/src/gc_state.rs index ef6b8eed2e1..be9d8bcf69a 100644 --- a/crates/vm/src/gc_state.rs +++ b/crates/vm/src/gc_state.rs @@ -731,11 +731,17 @@ impl GcState { if !truly_dead.is_empty() { // Break cycles by clearing references (tp_clear) // Use deferred drop context to prevent stack overflow. + // With DEBUG_SAVEALL the objects stay reachable through + // gc.garbage, so they must not be cleared (delete_garbage + // skips tp_clear for saved objects). + let save_all = debug.contains(GcDebugFlags::SAVEALL); rustpython_common::refcount::with_deferred_drops(|| { - for obj_ref in &truly_dead { - if obj_ref.gc_has_clear() { - let edges = unsafe { obj_ref.gc_clear() }; - drop(edges); + if !save_all { + for obj_ref in &truly_dead { + if obj_ref.gc_has_clear() { + let edges = unsafe { obj_ref.gc_clear() }; + drop(edges); + } } } drop(truly_dead); From 4c3bb64b247e8b3177d89813d2e826a24eb28ca6 Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Mon, 6 Jul 2026 06:45:39 +0900 Subject: [PATCH 27/92] Drop frame children directly in tp_clear instead of extracting Pushing the frame's ~10 child references into the clear buffer grew a Vec on every frame deallocation. Take the interpreter frame out of the cell and drop it in place; LocalsPlus::clear_into is removed. Assisted-by: Claude --- crates/vm/src/frame.rs | 46 +++++++++--------------------------------- 1 file changed, 9 insertions(+), 37 deletions(-) diff --git a/crates/vm/src/frame.rs b/crates/vm/src/frame.rs index 19f1afb2e5d..5c320fe9845 100644 --- a/crates/vm/src/frame.rs +++ b/crates/vm/src/frame.rs @@ -227,22 +227,6 @@ impl LocalsPlus { } } - /// Extract all contained values into `out` without freeing the backing - /// storage. Used by tp_clear so that child references are dropped by the - /// caller instead of recursively inside the frame. - fn clear_into(&mut self, out: &mut Vec) { - while !self.stack_is_empty() { - if let Some(val) = self.stack_pop() { - out.push(val.to_pyobj()); - } - } - for slot in self.fastlocals_mut() { - if let Some(obj) = slot.take() { - out.push(obj); - } - } - } - // -- Data access helpers -- #[inline(always)] @@ -780,27 +764,15 @@ unsafe impl Traverse for Frame { iframe.f_locals_hidden_overlay.traverse(tracer_fn); } - fn clear(&mut self, out: &mut Vec) { - let Some(mut iframe) = self.iframe.get_mut().take() else { - return; - }; - // Extract all owned child references for cycle breaking; the - // remaining non-reference fields (atomics, flags, backing storage) - // drop with `iframe` at the end of this scope, leaving the payload - // as a trivially-droppable husk for the freelist. - iframe.localsplus.clear_into(out); - out.push(iframe.code.into()); - out.extend(iframe.func_obj.take()); - if let Some(locals) = iframe.locals.inner.take() { - out.push(locals.into()); - } - out.push(iframe.globals.into()); - out.push(iframe.builtins); - out.push(iframe.trace.into_inner()); - out.append(iframe.temporary_refs.get_mut()); - if let Some(overlay) = iframe.f_locals_hidden_overlay.into_inner() { - out.push(overlay.into()); - } + fn clear(&mut self, _out: &mut Vec) { + // Drop the interpreter frame in place instead of extracting children + // into `_out`: pushing ~10 refs per frame would grow the buffer, a + // heap allocation on the hot dealloc path. Direct drops release the + // same references under the same recursion protection (trashcan in + // dealloc, deferred-drop context in cycle collection) as the payload + // drop did before the freelist existed. The payload is left as a + // trivially-droppable husk for the freelist. + drop(self.iframe.get_mut().take()); } } From c197615e57ce2812809078a0cafbde0b72d4a11f Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Mon, 6 Jul 2026 07:44:21 +0900 Subject: [PATCH 28/92] Untrack objects before tp_clear and guard cross-thread f_locals The cycle collector called tp_clear on unreachable objects while they were still linked into the GC generation lists, so another thread could obtain a strong reference to an already-cleared object (e.g. a frame husk with iframe == None) via gc.get_objects() and access the cleared payload. Untrack the dead set before the clear phase, mirroring the untrack-then-clear ordering of the refcount dealloc path. Objects that gained an external reference before untracking are found by comparing each object's strong count against the references coming from within the dead set; such late-resurrected objects and everything reachable from them are re-tracked and skipped, letting a later collection retry once the external references are released. Also reject f_locals access for a frame currently executing on another thread. Reading fastlocals of such a frame races with the executing thread overwriting the slots (torn reads of dropped values). Access from the executing thread itself (locals(), trace callbacks) is still allowed via the current-frame chain check. Assisted-by: Claude --- crates/vm/src/frame.rs | 26 ++++++++++++++ crates/vm/src/gc_state.rs | 74 +++++++++++++++++++++++++++++++++++++-- 2 files changed, 98 insertions(+), 2 deletions(-) diff --git a/crates/vm/src/frame.rs b/crates/vm/src/frame.rs index 5c320fe9845..bd4fbb1163e 100644 --- a/crates/vm/src/frame.rs +++ b/crates/vm/src/frame.rs @@ -1143,7 +1143,33 @@ impl Frame { Ok(()) } + /// Reject locals access for a frame that is executing on another thread. + /// + /// A thread-owned frame mutates `localsplus` without synchronization, so + /// reading fastlocals from a different thread would be a data race (the + /// executing thread overwrites slots and drops the old values while the + /// reader clones them). Access from the executing thread itself (locals() + /// builtin, trace callbacks) is fine: the frame sits on the current + /// thread's frame chain and is at a bytecode boundary. + fn check_locals_access(&self, vm: &VirtualMachine) -> PyResult<()> { + let owner = FrameOwner::from_i8(self.owner.load(atomic::Ordering::Acquire)); + if owner != FrameOwner::Thread { + return Ok(()); + } + let mut cur = crate::vm::thread::get_current_frame(); + while !cur.is_null() { + if core::ptr::eq(cur, self) { + return Ok(()); + } + cur = unsafe { (*cur).previous_frame() }; + } + Err(vm.new_runtime_error( + "cannot access frame locals while the frame is executing in another thread", + )) + } + pub fn f_locals_mapping(&self, vm: &VirtualMachine) -> PyResult { + self.check_locals_access(vm)?; if !self.has_active_hidden_locals() { self.f_locals_hidden_overlay.lock().take(); return self.locals(vm); diff --git a/crates/vm/src/gc_state.rs b/crates/vm/src/gc_state.rs index be9d8bcf69a..b8d0add67ba 100644 --- a/crates/vm/src/gc_state.rs +++ b/crates/vm/src/gc_state.rs @@ -735,11 +735,81 @@ impl GcState { // gc.garbage, so they must not be cleared (delete_garbage // skips tp_clear for saved objects). let save_all = debug.contains(GcDebugFlags::SAVEALL); + + // Untrack dead objects BEFORE clearing them, mirroring the + // untrack-then-clear ordering of the refcount dealloc path. + // A cleared object (e.g. a frame husk with iframe == None) must + // never be observable through the generation lists, or another + // thread could obtain a strong reference via gc.get_objects() + // and access the cleared payload. + let mut late_resurrected: HashSet = HashSet::new(); + if !save_all { + let mut expected_counts: std::collections::HashMap = + std::collections::HashMap::new(); + for obj_ref in &truly_dead { + let obj = obj_ref.as_ref(); + if obj.is_gc_tracked() { + unsafe { self.untrack_object(NonNull::from(obj)) }; + } + // One strong reference held by the `truly_dead` vec itself. + expected_counts.insert(GcPtr(NonNull::from(obj)), 1); + } + // With the objects out of the generation lists, no new external + // reference can appear. Count the references coming from within + // the dead set; any surplus in strong_count means another thread + // grabbed a reference before untracking (late resurrection) and + // the object must not be cleared. + let mut referents: std::collections::HashMap>> = + std::collections::HashMap::new(); + for obj_ref in &truly_dead { + let referent_ptrs = unsafe { obj_ref.gc_get_referent_ptrs() }; + for child_ptr in &referent_ptrs { + if let Some(n) = expected_counts.get_mut(&GcPtr(*child_ptr)) { + *n += 1; + } + } + referents.insert(GcPtr(NonNull::from(obj_ref.as_ref())), referent_ptrs); + } + let mut worklist: Vec = Vec::new(); + for obj_ref in &truly_dead { + let ptr = GcPtr(NonNull::from(obj_ref.as_ref())); + if obj_ref.strong_count() > expected_counts[&ptr] && late_resurrected.insert(ptr) + { + worklist.push(ptr); + } + } + // A holder of a late-resurrected object can reach its referents, + // so everything reachable from it must stay intact as well. + while let Some(ptr) = worklist.pop() { + let Some(referent_ptrs) = referents.get(&ptr) else { + continue; + }; + for child_ptr in referent_ptrs { + let child = GcPtr(*child_ptr); + if expected_counts.contains_key(&child) && late_resurrected.insert(child) { + worklist.push(child); + } + } + } + // Re-track late-resurrected objects so a future collection can + // retry once the external references are released. + #[expect( + clippy::iter_over_hash_type, + reason = "Iteration order doesn't matter here" + )] + for &ptr in &late_resurrected { + unsafe { self.track_object(ptr.0) }; + } + } rustpython_common::refcount::with_deferred_drops(|| { if !save_all { for obj_ref in &truly_dead { - if obj_ref.gc_has_clear() { - let edges = unsafe { obj_ref.gc_clear() }; + let obj = obj_ref.as_ref(); + if late_resurrected.contains(&GcPtr(NonNull::from(obj))) { + continue; + } + if obj.gc_has_clear() { + let edges = unsafe { obj.gc_clear() }; drop(edges); } } From cff6798786961c53a6d01cb1ef308ff6df7a200b Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Mon, 6 Jul 2026 08:11:09 +0900 Subject: [PATCH 29/92] Skip localsplus heap copy for uniquely referenced dying frames release_datastack_frame now checks the frame's strong count after untracking it from the GC generation lists. When the caller holds the only reference, localsplus values are dropped in place and the data stack storage is released without the heap copy; escaped frames (traceback, sys._getframe, trace callbacks) keep the copy path. The three inline materialize+pop call sites in function.rs are routed through release_datastack_frame. Assisted-by: Claude --- crates/vm/src/builtins/function.rs | 18 +----- crates/vm/src/frame.rs | 90 +++++++++++++++++++++++++++++- 2 files changed, 90 insertions(+), 18 deletions(-) diff --git a/crates/vm/src/builtins/function.rs b/crates/vm/src/builtins/function.rs index 3e98d7898d6..f3a8b719e23 100644 --- a/crates/vm/src/builtins/function.rs +++ b/crates/vm/src/builtins/function.rs @@ -598,11 +598,7 @@ impl Py { } else { let result = vm.run_frame(frame.clone()); // Release data stack memory after frame execution completes. - unsafe { - if let Some(base) = frame.materialize_localsplus() { - vm.datastack_pop(base); - } - } + crate::frame::release_datastack_frame(&frame, vm); result } } @@ -747,11 +743,7 @@ impl Py { let frame = self.prepare_exact_args_frame(args, vm); let result = vm.run_frame(frame.clone()); - unsafe { - if let Some(base) = frame.materialize_localsplus() { - vm.datastack_pop(base); - } - } + crate::frame::release_datastack_frame(&frame, vm); result } } @@ -1479,11 +1471,7 @@ pub(crate) fn vectorcall_function( let frame = zelf.prepare_exact_args_frame(args, vm); let result = vm.run_frame(frame.clone()); - unsafe { - if let Some(base) = frame.materialize_localsplus() { - vm.datastack_pop(base); - } - } + crate::frame::release_datastack_frame(&frame, vm); return result; } diff --git a/crates/vm/src/frame.rs b/crates/vm/src/frame.rs index bd4fbb1163e..1ce80aaaf5a 100644 --- a/crates/vm/src/frame.rs +++ b/crates/vm/src/frame.rs @@ -182,8 +182,9 @@ impl LocalsPlus { /// Create a new LocalsPlus backed by the thread data stack. /// All slots are zero-initialized. /// - /// The caller must call `materialize_localsplus()` when the frame finishes - /// to migrate data to the heap, then `datastack_pop()` to free the memory. + /// When the frame finishes, the caller must migrate data to the heap with + /// `materialize_localsplus()` (or drop it in place with + /// `release_localsplus()`), then `datastack_pop()` to free the memory. fn new_on_datastack(nlocalsplus: usize, stacksize: usize, vm: &VirtualMachine) -> Self { let capacity = nlocalsplus .checked_add(stacksize) @@ -218,6 +219,29 @@ impl LocalsPlus { } } + /// Drop all contained values and detach the data stack backing without + /// copying to the heap, leaving an empty heap-backed husk. + /// Returns the data stack base pointer for `DataStack::pop()`. + /// Returns `None` if already heap-backed. + /// + /// Only valid when the values can never be observed again (the enclosing + /// frame is uniquely referenced): the locals are gone afterwards. + fn release_datastack(&mut self) -> Option<*mut u8> { + let LocalsPlusData::DataStack { ptr, .. } = &self.data else { + return None; + }; + let base = *ptr as *mut u8; + // Drop values while the backing store is still valid. Value drops may + // run `__del__`, which can push nested data stack frames above `base`; + // those are popped before the caller pops `base` (LIFO preserved). + self.drop_values(); + self.data = LocalsPlusData::Heap(Box::default()); + // Keep the accessors consistent with the empty backing store. + // stack_top is already 0 after drop_values(). + self.nlocalsplus = 0; + Some(base) + } + /// Drop all contained values without freeing the backing storage. fn drop_values(&mut self) { self.stack_clear(); @@ -893,6 +917,19 @@ impl Frame { unsafe { self.iframe_mut().localsplus.materialize_to_heap() } } + /// Drop all localsplus values in place and detach the data stack backing + /// without the heap copy. Returns the data stack base pointer for + /// `VirtualMachine::datastack_pop()`, or `None` if heap-backed. + /// + /// # Safety + /// Caller must ensure the frame is not executing, that no other reference + /// to the frame exists or can be created (localsplus is unobservable + /// afterwards), and that the returned pointer is passed to + /// `VirtualMachine::datastack_pop()`. + pub(crate) unsafe fn release_localsplus(&self) -> Option<*mut u8> { + unsafe { self.iframe_mut().localsplus.release_datastack() } + } + /// Clear evaluation stack and state-owned cell/free references. /// For full local/cell cleanup, call `clear_locals_and_stack()`. pub(crate) fn clear_stack_and_cells(&self) { @@ -1366,7 +1403,54 @@ fn specialization_nonnegative_compact_index(i: &PyInt, vm: &VirtualMachine) -> O } } -fn release_datastack_frame(frame: &Py, vm: &VirtualMachine) { +/// Free a finished call frame's data stack storage. +/// +/// When the caller holds the only reference to the frame, the locals and +/// stack values are dropped in place and the storage is released without a +/// heap copy. Otherwise (the frame escaped through a traceback, +/// `sys._getframe`, a trace callback, ...) the values are copied to the heap +/// first so they stay readable through the escaped reference. +pub(crate) fn release_datastack_frame(frame: &Py, vm: &VirtualMachine) { + let frame_obj = frame.as_object(); + // Uniqueness argument: at this point the frame is already out of + // `vm.frames`, the thread-frames registry and the current-frame chain + // (all unlinked inside `with_frame_impl` before it returned), and the + // frame type has no weakref support, so with one exception no thread can + // mint a new reference without already holding one. The exception is the + // GC generation lists (`gc.get_objects`, `gc.get_referrers`, collection + // survivor refs), whose readers incref under the generation-list read + // lock. Untracking the frame takes the same list's write lock, so after + // `untrack_object` returns, no new list-based reference can appear and + // any previously minted one is visible to the count re-check below. + // This also keeps `__del__` side effects during `release_localsplus` + // (which can run arbitrary code, including `gc.get_objects`) from + // reaching the frame. + if frame_obj.strong_count() == 1 && frame_obj.is_gc_tracked() { + let ptr = NonNull::from(frame_obj); + // SAFETY: the frame is alive and currently tracked. + unsafe { crate::gc_state::gc_state().untrack_object(ptr) }; + if frame_obj.strong_count() == 1 { + // A reference minted and already released by another thread ends + // in a release-decref; the fence orders that thread's reads of + // localsplus before our drops below. + atomic::fence(Acquire); + // SAFETY: unique owner and no way to mint a new reference, so + // localsplus can never be observed again. The base pointer came + // from this thread's data stack. + unsafe { + if let Some(base) = frame.release_localsplus() { + vm.datastack_pop(base); + } + } + return; + } + // Lost the race: a reference was minted through the GC list before + // the untrack. Re-track and fall back to the copy path. + // SAFETY: the frame is alive and untracked. + unsafe { crate::gc_state::gc_state().track_object(ptr) }; + } + // SAFETY: the frame finished executing; the base pointer came from this + // thread's data stack. unsafe { if let Some(base) = frame.materialize_localsplus() { vm.datastack_pop(base); From 1888e7df28937083727feac8049a73ee4d5fb127 Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Mon, 6 Jul 2026 08:51:25 +0900 Subject: [PATCH 30/92] Stage exact-args call arguments in stack slot buffers CallPyExactArgs, CallBoundMethodExactArgs and CallAllocAndEnterInit no longer collect popped arguments into a per-call Vec (plus a second Vec for self-prepending). Arguments are popped into a fixed-size Option slot buffer (CallArgBuffer, 8 inline slots, heap fallback for larger arities) and moved into fastlocals via the new invoke_exact_args_slots / a take() iterator. - prepare_exact_args_frame now accepts an ExactSizeIterator of args - specialization_run_init_cleanup_shim takes the slot buffer and fills slot 0 with new_obj, dropping one redundant clone - LoadAttrGetattributeOverridden, LoadAttrProperty and BinaryOpSubscrGetitem pass inline slot arrays instead of vec![] Assisted-by: Claude --- crates/vm/src/builtins/function.rs | 62 +++++++++++------ crates/vm/src/frame.rs | 103 +++++++++++++++++++++-------- 2 files changed, 115 insertions(+), 50 deletions(-) diff --git a/crates/vm/src/builtins/function.rs b/crates/vm/src/builtins/function.rs index f3a8b719e23..9b20852a977 100644 --- a/crates/vm/src/builtins/function.rs +++ b/crates/vm/src/builtins/function.rs @@ -668,7 +668,7 @@ impl Py { pub(crate) fn prepare_exact_args_frame( &self, - mut args: Vec, + args: impl ExactSizeIterator, vm: &VirtualMachine, ) -> FrameRef { let code: PyRef = (*self.code).to_owned(); @@ -706,7 +706,7 @@ impl Py { { let fastlocals = unsafe { frame.fastlocals_mut() }; - for (slot, arg) in fastlocals.iter_mut().zip(args.drain(..)) { + for (slot, arg) in fastlocals.iter_mut().zip(args) { *slot = Some(arg); } } @@ -714,37 +714,55 @@ impl Py { frame } + fn invoke_prepared_exact_args( + &self, + args: impl ExactSizeIterator, + vm: &VirtualMachine, + ) -> PyResult { + let frame = self.prepare_exact_args_frame(args, vm); + + let result = vm.run_frame(frame.clone()); + crate::frame::release_datastack_frame(&frame, vm); + result + } + /// Fast path for calling a simple function with exact positional args. /// Skips FuncArgs allocation, prepend_arg, and fill_locals_from_args. /// Only valid when: CO_OPTIMIZED, no VARARGS, no VARKEYWORDS, no kwonlyargs, /// and nargs == co_argcount. pub fn invoke_exact_args(&self, args: Vec, vm: &VirtualMachine) -> PyResult { - let code: PyRef = (*self.code).to_owned(); - - debug_assert_eq!(args.len(), code.arg_count as usize); - debug_assert!(code.flags.contains(bytecode::CodeFlags::OPTIMIZED)); - debug_assert!( - !code - .flags - .intersects(bytecode::CodeFlags::VARARGS | bytecode::CodeFlags::VARKEYWORDS) - ); - debug_assert_eq!(code.kwonlyarg_count, 0); + debug_assert_eq!(args.len(), self.code.arg_count as usize); // Generator/coroutine code objects are SIMPLE_FUNCTION in call // specialization classification, but their call path must still // go through invoke() to produce generator/coroutine objects. - if code.flags.intersects( - bytecode::CodeFlags::GENERATOR - | bytecode::CodeFlags::COROUTINE - | bytecode::CodeFlags::ASYNC_GENERATOR, - ) { + if self.is_generator_like() { return self.invoke(FuncArgs::from(args), vm); } - let frame = self.prepare_exact_args_frame(args, vm); + self.invoke_prepared_exact_args(args.into_iter(), vm) + } - let result = vm.run_frame(frame.clone()); - crate::frame::release_datastack_frame(&frame, vm); - result + /// Like `invoke_exact_args`, but moves the args out of caller-provided + /// slots (all filled with `Some`), so callers can stage them in a + /// fixed-size stack buffer instead of allocating a Vec per call. + pub(crate) fn invoke_exact_args_slots( + &self, + args: &mut [Option], + vm: &VirtualMachine, + ) -> PyResult { + debug_assert_eq!(args.len(), self.code.arg_count as usize); + + let taken = args + .iter_mut() + .map(|slot| slot.take().expect("arg slot must be filled")); + // Generator/coroutine code objects are SIMPLE_FUNCTION in call + // specialization classification, but their call path must still + // go through invoke() to produce generator/coroutine objects. + if self.is_generator_like() { + let args: Vec = taken.collect(); + return self.invoke(FuncArgs::from(args), vm); + } + self.invoke_prepared_exact_args(taken, vm) } } @@ -1468,7 +1486,7 @@ pub(crate) fn vectorcall_function( // FAST PATH: simple positional-only call, exact arg count. // Move owned args directly into fastlocals — no clone needed. args.truncate(nargs); - let frame = zelf.prepare_exact_args_frame(args, vm); + let frame = zelf.prepare_exact_args_frame(args.into_iter(), vm); let result = vm.run_frame(frame.clone()); crate::frame::release_datastack_frame(&frame, vm); diff --git a/crates/vm/src/frame.rs b/crates/vm/src/frame.rs index 1ce80aaaf5a..7ef6257a361 100644 --- a/crates/vm/src/frame.rs +++ b/crates/vm/src/frame.rs @@ -1469,6 +1469,34 @@ struct BinaryOpExtendSpecializationDescr { const BINARY_OP_EXTEND_EXTERNAL_CACHE_OFFSET: usize = 1; +/// Max total args (including self) staged in a fixed-size stack buffer by the +/// exact-args call fast paths; larger arities fall back to a heap buffer. +const MAX_INLINE_CALL_ARGS: usize = 8; + +/// Staging buffer for exact-args call fast paths: fixed-size inline storage +/// for small arities, avoiding a per-call Vec allocation. +enum CallArgBuffer { + Inline(usize, [Option; MAX_INLINE_CALL_ARGS]), + Heap(Vec>), +} + +impl CallArgBuffer { + fn new(total_nargs: usize) -> Self { + if total_nargs <= MAX_INLINE_CALL_ARGS { + Self::Inline(total_nargs, [const { None }; MAX_INLINE_CALL_ARGS]) + } else { + Self::Heap(vec![None; total_nargs]) + } + } + + fn slots(&mut self) -> &mut [Option] { + match self { + Self::Inline(len, buf) => &mut buf[..*len], + Self::Heap(buf) => buf, + } + } +} + #[inline] fn compactlongs_guard(lhs: &PyObject, rhs: &PyObject, vm: &VirtualMachine) -> bool { compact_int_from_obj(lhs, vm).is_some() && compact_int_from_obj(rhs, vm).is_some() @@ -1647,22 +1675,25 @@ impl ExecutingFrame<'_> { .into_ref(&vm.ctx) } + /// `args` holds the `__init__` args with slot 0 left empty; it is filled + /// with `new_obj` here. fn specialization_run_init_cleanup_shim( &self, new_obj: PyObjectRef, init_func: &Py, - pos_args: Vec, + args: &mut [Option], vm: &VirtualMachine, ) -> PyResult { let shim = self.specialization_new_init_cleanup_frame(vm); let shim_result = vm.with_frame_untraced(shim.clone(), |shim| { shim.with_exec(vm, |mut exec| exec.push_value(new_obj.clone())); - let mut all_args = Vec::with_capacity(pos_args.len() + 1); - all_args.push(new_obj.clone()); - all_args.extend(pos_args); + args[0] = Some(new_obj); + let taken = args + .iter_mut() + .map(|slot| slot.take().expect("arg slot must be filled")); - let init_frame = init_func.prepare_exact_args_frame(all_args, vm); + let init_frame = init_func.prepare_exact_args_frame(taken, vm); let init_result = vm.run_frame(init_frame.clone()); release_datastack_frame(&init_frame, vm); let init_result = init_result?; @@ -4320,7 +4351,8 @@ impl ExecutingFrame<'_> { debug_assert!(func.has_exact_argcount(2)); let owner = self.pop_value(); let attr_name = self.code.names[oparg.name_idx() as usize].to_owned().into(); - let result = func.invoke_exact_args(vec![owner, attr_name], vm)?; + let result = + func.invoke_exact_args_slots(&mut [Some(owner), Some(attr_name)], vm)?; self.push_value(result); return Ok(None); } @@ -4367,7 +4399,7 @@ impl ExecutingFrame<'_> { && self.specialization_has_datastack_space_for_func(vm, func) { let owner = self.pop_value(); - let result = func.invoke_exact_args(vec![owner], vm)?; + let result = func.invoke_exact_args_slots(&mut [Some(owner)], vm)?; self.push_value(result); return Ok(None); } @@ -4508,7 +4540,7 @@ impl ExecutingFrame<'_> { debug_assert!(func.has_exact_argcount(2)); let sub = self.pop_value(); let owner = self.pop_value(); - let result = func.invoke_exact_args(vec![owner, sub], vm)?; + let result = func.invoke_exact_args_slots(&mut [Some(owner), Some(sub)], vm)?; self.push_value(result); return Ok(None); } @@ -4655,19 +4687,24 @@ impl ExecutingFrame<'_> { if self.specialization_call_recursion_guard(vm) { return self.execute_call_vectorcall(nargs, vm); } - let pos_args: Vec = self.pop_multiple(nargs as usize).collect(); + // Stage args without a per-call Vec: [self?, arg1, ..., argN] + let base = usize::from(self_or_null_is_some); + let mut arg_buf = CallArgBuffer::new(nargs as usize + base); + let args = arg_buf.slots(); + for (slot, arg) in args[base..] + .iter_mut() + .zip(self.pop_multiple(nargs as usize)) + { + *slot = Some(arg); + } let self_or_null = self.pop_value_opt(); + debug_assert_eq!(self_or_null.is_some(), self_or_null_is_some); + if self_or_null.is_some() { + args[0] = self_or_null; + } let callable = self.pop_value(); let func = callable.downcast_ref_if_exact::(vm).unwrap(); - let args = if let Some(self_val) = self_or_null { - let mut all_args = Vec::with_capacity(pos_args.len() + 1); - all_args.push(self_val); - all_args.extend(pos_args); - all_args - } else { - pos_args - }; - let result = func.invoke_exact_args(args, vm)?; + let result = func.invoke_exact_args_slots(args, vm)?; self.push_value(result); Ok(None) } else { @@ -4707,14 +4744,19 @@ impl ExecutingFrame<'_> { if self.specialization_call_recursion_guard(vm) { return self.execute_call_vectorcall(nargs, vm); } - let pos_args: Vec = - self.pop_multiple(nargs as usize).collect(); + // Stage args without a per-call Vec: + // [bound_self, arg1, ..., argN] + let mut arg_buf = CallArgBuffer::new(nargs as usize + 1); + let args = arg_buf.slots(); + for (slot, arg) in + args[1..].iter_mut().zip(self.pop_multiple(nargs as usize)) + { + *slot = Some(arg); + } self.pop_value_opt(); // null (self_or_null) self.pop_value(); // callable (bound method) - let mut all_args = Vec::with_capacity(pos_args.len() + 1); - all_args.push(bound_self); - all_args.extend(pos_args); - let result = func.invoke_exact_args(all_args, vm)?; + args[0] = Some(bound_self); + let result = func.invoke_exact_args_slots(args, vm)?; self.push_value(result); return Ok(None); } @@ -5241,12 +5283,17 @@ impl ExecutingFrame<'_> { let cls_ref = cls.to_owned(); let new_obj = cls_alloc(cls_ref, 0, vm)?; - // Build args: [new_obj, arg1, ..., argN] - let pos_args: Vec = self.pop_multiple(nargs as usize).collect(); + // Stage args as [new_obj, arg1, ..., argN]; slot 0 is + // filled by the shim runner. + let mut arg_buf = CallArgBuffer::new(nargs as usize + 1); + let args = arg_buf.slots(); + for (slot, arg) in args[1..].iter_mut().zip(self.pop_multiple(nargs as usize)) { + *slot = Some(arg); + } let _null = self.pop_value_opt(); // self_or_null (None) let _callable = self.pop_value(); // callable (type) - let result = self - .specialization_run_init_cleanup_shim(new_obj, &init_func, pos_args, vm)?; + let result = + self.specialization_run_init_cleanup_shim(new_obj, &init_func, args, vm)?; self.push_value(result); return Ok(None); } From 7517364e471ea07f18b3ff60a04c052876c0a84f Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Mon, 6 Jul 2026 09:18:56 +0900 Subject: [PATCH 31/92] Reduce allocations in generic call paths - FuncArgs::prepend_arg: use reserve instead of reserve_exact so exact-capacity vectors do not realloc on every prepend. - IntoFuncArgs::into_method_args: build the final args vec once with capacity len + 1 instead of prepending into a full vector. - PyType::call: skip cloning FuncArgs when no init slot can run after slot_new (no init slot, not `type`, and slot_new is not new_wrapper). Assisted-by: Claude --- crates/vm/src/builtins/type.rs | 19 ++++++++++++++++--- crates/vm/src/function/argument.rs | 10 ++++++++-- 2 files changed, 24 insertions(+), 5 deletions(-) diff --git a/crates/vm/src/builtins/type.rs b/crates/vm/src/builtins/type.rs index 7ac0b104d57..c78a16166fd 100644 --- a/crates/vm/src/builtins/type.rs +++ b/crates/vm/src/builtins/type.rs @@ -2702,12 +2702,25 @@ impl Callable for PyType { } } - let obj = if let Some(slot_new) = zelf.slots.new.load() { - slot_new(zelf.to_owned(), args.clone(), vm)? - } else { + let Some(slot_new) = zelf.slots.new.load() else { return Err(vm.new_type_error(format!("cannot create '{}' instances", zelf.slots.name))); }; + // `args` is cloned because both the new and init slots consume it. + // Skip the clone when no init call can follow: the class has no init + // slot, is not `type` itself, and its new slot is a native function. + // new_wrapper is excluded because a Python `__new__` can install an + // `__init__` on the class or return an instance of another class + // while it runs. + if zelf.slots.init.load().is_none() + && !zelf.is(vm.ctx.types.type_type) + && slot_new as usize != crate::types::new_wrapper as crate::types::NewFunc as usize + { + return slot_new(zelf.to_owned(), args, vm); + } + + let obj = slot_new(zelf.to_owned(), args.clone(), vm)?; + if !obj.class().fast_issubclass(zelf) { return Ok(obj); } diff --git a/crates/vm/src/function/argument.rs b/crates/vm/src/function/argument.rs index c37475ba48d..88ef5581bb2 100644 --- a/crates/vm/src/function/argument.rs +++ b/crates/vm/src/function/argument.rs @@ -12,7 +12,11 @@ pub trait IntoFuncArgs: Sized { fn into_args(self, vm: &VirtualMachine) -> FuncArgs; fn into_method_args(self, obj: PyObjectRef, vm: &VirtualMachine) -> FuncArgs { let mut args = self.into_args(vm); - args.prepend_arg(obj); + // Build the final vec once instead of prepending (realloc + memmove). + let mut with_obj = Vec::with_capacity(args.args.len() + 1); + with_obj.push(obj); + with_obj.append(&mut args.args); + args.args = with_obj; args } } @@ -202,7 +206,9 @@ impl FuncArgs { } pub fn prepend_arg(&mut self, item: PyObjectRef) { - self.args.reserve_exact(1); + // reserve (not reserve_exact): incoming vectors are usually built with + // exact capacity, so exact growth would realloc on every prepend. + self.args.reserve(1); self.args.insert(0, item) } From 71197eb9113336711046be8b5d698c10cb19ae44 Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Mon, 6 Jul 2026 09:22:33 +0900 Subject: [PATCH 32/92] Skip redundant exc_info restore on frame exit with_frame_impl saves the current exc_info slot value and restores it when the frame exits. When the slot still holds the same value, the restore rewrites the slot and recomputes the thread-exception mirror for no effect. Add restore_exception, which compares the slot against the saved value by pointer identity and skips the store when they match. The saved value is a strong reference held for the whole frame scope, so the pointed-to object cannot be freed and its address reused while the frame runs, making the pointer comparison free of ABA. Assisted-by: Claude --- crates/vm/src/vm/mod.rs | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/crates/vm/src/vm/mod.rs b/crates/vm/src/vm/mod.rs index 40a97e40862..569185612e0 100644 --- a/crates/vm/src/vm/mod.rs +++ b/crates/vm/src/vm/mod.rs @@ -1624,7 +1624,7 @@ impl VirtualMachine { // Ensure cleanup on panic: restore owner, exc_info, frame chain, and frames Vec. scopeguard::defer! { frame.owner.store(old_owner, core::sync::atomic::Ordering::Release); - self.set_exception(saved_exc); + self.restore_exception(saved_exc); crate::vm::thread::set_current_frame(old_frame); self.frames.borrow_mut().pop(); #[cfg(feature = "threading")] @@ -2122,6 +2122,25 @@ impl VirtualMachine { thread::update_thread_exception(self.topmost_exception()); } + /// Restore an exc_info slot value saved by `with_frame_impl`, skipping the + /// store when the slot is unchanged. `saved` is a strong reference taken + /// at save time, so the object it points to cannot have been freed and + /// its address reused while the frame ran; pointer identity therefore + /// proves the slot still holds the same value and both the store and the + /// thread-exception mirror update would be no-ops. + pub(crate) fn restore_exception(&self, saved: Option) { + let excs = self.exceptions.borrow(); + let unchanged = match (excs.stack.last(), &saved) { + (Some(Some(current)), Some(saved)) => current.is(saved), + (Some(None), None) => true, + _ => false, + }; + drop(excs); + if !unchanged { + self.set_exception(saved); + } + } + pub fn take_raised_exception(&self) -> Option { let mut excs = self.exceptions.borrow_mut(); if let Some(top) = excs.stack.last_mut() { From bc2927ce9a792c2b316b56005797758acf86c279 Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Mon, 6 Jul 2026 09:37:06 +0900 Subject: [PATCH 33/92] Apply rustfmt to gc_state.rs Assisted-by: Claude --- crates/vm/src/gc_state.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/crates/vm/src/gc_state.rs b/crates/vm/src/gc_state.rs index b8d0add67ba..072712901cb 100644 --- a/crates/vm/src/gc_state.rs +++ b/crates/vm/src/gc_state.rs @@ -773,7 +773,8 @@ impl GcState { let mut worklist: Vec = Vec::new(); for obj_ref in &truly_dead { let ptr = GcPtr(NonNull::from(obj_ref.as_ref())); - if obj_ref.strong_count() > expected_counts[&ptr] && late_resurrected.insert(ptr) + if obj_ref.strong_count() > expected_counts[&ptr] + && late_resurrected.insert(ptr) { worklist.push(ptr); } From de52aea08f4333b7a2825ef84f6721a8ef9a4166 Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Mon, 6 Jul 2026 10:00:53 +0900 Subject: [PATCH 34/92] Avoid empty FuncArgs clone in PyType::call Cloning args for the init call clones the kwargs map even when args is empty, which shows up in no-argument instantiation profiles. Prepare the init-call args before slot_new: a default FuncArgs when args is empty (indistinguishable from a clone of empty args), the existing clone otherwise. The empty check also keeps the init-slot load of the clone-elision path off the no-argument call path. Assisted-by: Claude --- crates/vm/src/builtins/type.rs | 36 +++++++++++++++++++++------------- 1 file changed, 22 insertions(+), 14 deletions(-) diff --git a/crates/vm/src/builtins/type.rs b/crates/vm/src/builtins/type.rs index c78a16166fd..23c581889d1 100644 --- a/crates/vm/src/builtins/type.rs +++ b/crates/vm/src/builtins/type.rs @@ -2706,27 +2706,35 @@ impl Callable for PyType { return Err(vm.new_type_error(format!("cannot create '{}' instances", zelf.slots.name))); }; - // `args` is cloned because both the new and init slots consume it. - // Skip the clone when no init call can follow: the class has no init - // slot, is not `type` itself, and its new slot is a native function. - // new_wrapper is excluded because a Python `__new__` can install an - // `__init__` on the class or return an instance of another class - // while it runs. - if zelf.slots.init.load().is_none() - && !zelf.is(vm.ctx.types.type_type) - && slot_new as usize != crate::types::new_wrapper as crate::types::NewFunc as usize - { - return slot_new(zelf.to_owned(), args, vm); - } + // Both the new and init slots consume args, so the init call gets a + // separate copy prepared before slot_new runs. + let init_args = if args.is_empty() { + // Even cloning empty args costs a kwargs map clone; a default + // FuncArgs is indistinguishable from such a clone. + FuncArgs::default() + } else { + // Skip the clone when no init call can follow: the class has no + // init slot, is not `type` itself, and its new slot is a native + // function. new_wrapper is excluded because a Python `__new__` + // can install an `__init__` on the class or return an instance + // of another class while it runs. + if zelf.slots.init.load().is_none() + && !zelf.is(vm.ctx.types.type_type) + && slot_new as usize != crate::types::new_wrapper as crate::types::NewFunc as usize + { + return slot_new(zelf.to_owned(), args, vm); + } + args.clone() + }; - let obj = slot_new(zelf.to_owned(), args.clone(), vm)?; + let obj = slot_new(zelf.to_owned(), args, vm)?; if !obj.class().fast_issubclass(zelf) { return Ok(obj); } if let Some(init_method) = obj.class().slots.init.load() { - init_method(obj.clone(), args, vm)?; + init_method(obj.clone(), init_args, vm)?; } Ok(obj) } From e10c44d89e152c91f8bd522a49e912646abe538e Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Mon, 6 Jul 2026 10:40:30 +0900 Subject: [PATCH 35/92] Push freelist husks only after tp_clear and child drops default_dealloc pushed the object onto the thread-local freelist before running clear_fn. Payloads that drop children inside clear (Frame) can run __del__ there, and a reentrant allocation could pop the husk and overwrite its payload while clear_fn still held a &mut borrow of it. Tuples had the same window through the extracted-edges drop, which ran after the push. Reorder default_dealloc to clear, drop the extracted children, and only then attempt the freelist push, so the husk becomes reusable only when no borrows into the payload remain. The tuple freelist bucketed husks by element count, which required reading the payload during push; after this reordering the elements are already cleared. PyInner is a fixed-size allocation (the elements box is dropped and replaced on reuse), so replace the per-size buckets with the shared single-list FreeList. Remove the now-unused pyinner_layout helper. Assisted-by: Claude --- crates/vm/src/builtins/tuple.rs | 63 ++++++--------------------------- crates/vm/src/object/core.rs | 34 +++++++++--------- crates/vm/src/object/payload.rs | 8 +++-- 3 files changed, 33 insertions(+), 72 deletions(-) diff --git a/crates/vm/src/builtins/tuple.rs b/crates/vm/src/builtins/tuple.rs index 7ed815ce24d..d48639b2c11 100644 --- a/crates/vm/src/builtins/tuple.rs +++ b/crates/vm/src/builtins/tuple.rs @@ -51,46 +51,14 @@ unsafe impl Traverse for PyTuple { } } -// spell-checker:ignore MAXSAVESIZE -/// Per-size freelist storage for tuples, matching tuples[PyTuple_MAXSAVESIZE]. -/// Each bucket caches tuples of a specific element count (index = len - 1). -struct TupleFreeList { - buckets: [Vec>; Self::MAX_SAVE_SIZE], -} - -impl TupleFreeList { - /// Largest tuple size to cache on the freelist (sizes 1..=20). - const MAX_SAVE_SIZE: usize = 20; - const fn new() -> Self { - Self { - buckets: [const { Vec::new() }; Self::MAX_SAVE_SIZE], - } - } -} - -impl Default for TupleFreeList { - fn default() -> Self { - Self::new() - } -} - -impl Drop for TupleFreeList { - fn drop(&mut self) { - // Same safety pattern as FreeList::drop — free raw allocation - // without running payload destructors to avoid TLS-after-destruction panics. - let layout = crate::object::pyinner_layout::(); - for bucket in &mut self.buckets { - for ptr in bucket.drain(..) { - unsafe { - alloc::alloc::dealloc(ptr.as_ptr() as *mut u8, layout); - } - } - } - } -} - thread_local! { - static TUPLE_FREELIST: Cell = const { Cell::new(TupleFreeList::new()) }; + // A single freelist for all tuple sizes: `PyInner` is a + // fixed-size allocation (elements are a separate boxed slice that is + // dropped and replaced on reuse), so husks are interchangeable. + // freelist_push must not read the payload — it runs after tp_clear, + // which has already emptied `elements`. + static TUPLE_FREELIST: Cell> = + const { Cell::new(crate::object::FreeList::new()) }; } impl PyPayload for PyTuple { @@ -104,16 +72,11 @@ impl PyPayload for PyTuple { #[inline] unsafe fn freelist_push(obj: *mut PyObject) -> bool { - let len = unsafe { &*(obj as *const crate::Py) }.elements.len(); - if len == 0 || len > TupleFreeList::MAX_SAVE_SIZE { - return false; - } TUPLE_FREELIST .try_with(|fl| { let mut list = fl.take(); - let bucket = &mut list.buckets[len - 1]; - let stored = if bucket.len() < Self::MAX_FREELIST { - bucket.push(unsafe { NonNull::new_unchecked(obj) }); + let stored = if list.len() < Self::MAX_FREELIST { + list.push(obj); true } else { false @@ -125,15 +88,11 @@ impl PyPayload for PyTuple { } #[inline] - unsafe fn freelist_pop(payload: &Self) -> Option> { - let len = payload.elements.len(); - if len == 0 || len > TupleFreeList::MAX_SAVE_SIZE { - return None; - } + unsafe fn freelist_pop(_payload: &Self) -> Option> { TUPLE_FREELIST .try_with(|fl| { let mut list = fl.take(); - let result = list.buckets[len - 1].pop(); + let result = list.pop().map(|p| unsafe { NonNull::new_unchecked(p) }); fl.set(list); result }) diff --git a/crates/vm/src/object/core.rs b/crates/vm/src/object/core.rs index 23ec2f752eb..43efac53a23 100644 --- a/crates/vm/src/object/core.rs +++ b/crates/vm/src/object/core.rs @@ -188,8 +188,23 @@ pub(super) unsafe fn default_dealloc(obj: *mut PyObject) { ); } - // Try to store in freelist for reuse BEFORE tp_clear, so that - // size-based freelists (e.g. PyTuple) can read the payload directly. + // Extract child references to break circular refs (tp_clear), then drop + // them. Some payloads (e.g. Frame) drop children in place inside clear_fn + // instead of extracting them, so user code (`__del__`) may run here. + let mut edges = Vec::new(); + if let Some(clear_fn) = vtable.clear { + unsafe { clear_fn(obj, &mut edges) }; + } + // Drop extracted child references - may trigger recursive destruction. + drop(edges); + + // Try to store in freelist for reuse. This must happen AFTER clear_fn and + // after the extracted-children drop: both can run user code (`__del__`) + // that allocates, and `PyRef::new_ref` pops from the same thread-local + // freelist. If the husk were already in the freelist, a reentrant + // allocation could pop it and write a fresh payload into it while clear_fn + // still holds a `&mut` borrow of that payload (aliasing UB). Pushing only + // once no borrows into the payload can be live closes that window. // Only exact base types (not heaptype or structseq subtypes) go into the freelist. // Published objects (e.g. a tuple stored as a type attribute) must skip the // freelist: `PyRef::new_ref` would reuse the slot and overwrite the refcount @@ -207,21 +222,11 @@ pub(super) unsafe fn default_dealloc(obj: *mut PyObject) { false }; - // Extract child references to break circular refs (tp_clear). - // This runs regardless of freelist push — the object's children must be released. - let mut edges = Vec::new(); - if let Some(clear_fn) = vtable.clear { - unsafe { clear_fn(obj, &mut edges) }; - } - if !pushed { // Deallocate the object memory (handles ObjExt prefix if present) unsafe { PyInner::dealloc(obj as *mut PyInner) }; } - // Drop child references - may trigger recursive destruction. - drop(edges); - // Trashcan: decrement depth and process deferred objects at outermost level unsafe { trashcan::end() }; } @@ -1158,11 +1163,6 @@ impl PyInner { } } -/// Returns the allocation layout for `PyInner`, for use in freelist Drop impls. -pub(crate) const fn pyinner_layout() -> core::alloc::Layout { - core::alloc::Layout::new::>() -} - /// Thread-local freelist storage for reusing object allocations. /// /// Wraps a `Vec<*mut PyObject>`. On thread teardown, `Drop` frees raw diff --git a/crates/vm/src/object/payload.rs b/crates/vm/src/object/payload.rs index 349b239f79f..772feaa4f4c 100644 --- a/crates/vm/src/object/payload.rs +++ b/crates/vm/src/object/payload.rs @@ -58,11 +58,13 @@ pub trait PyPayload: MaybeTraverse + PyThreadingConstraint + Sized + 'static { /// Try to push a dead object onto this type's freelist for reuse. /// Returns true if the object was stored (caller must NOT free the memory). - /// Called before tp_clear, so the payload is still intact. + /// Called after tp_clear, so the payload is a cleared husk; implementations + /// must not rely on its pre-clear contents. /// /// # Safety - /// `obj` must be a valid pointer to a `PyInner` with refcount 0. - /// The payload is still initialized and can be read for bucket selection. + /// `obj` must be a valid pointer to a `PyInner` with refcount 0 + /// whose tp_clear has already run, with no outstanding borrows into the + /// payload (`PyRef::new_ref` may pop and reuse the husk immediately). #[inline] unsafe fn freelist_push(_obj: *mut PyObject) -> bool { false From ebc56c91973a0c71b80904b27ca4a408dc2f7218 Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Mon, 6 Jul 2026 10:44:57 +0900 Subject: [PATCH 36/92] Resolve __set__ and __delete__ together for the descr_set slot The descr_set slot serves both __set__ and __delete__, but the TpDescrSet accessor resolved only the single modified name and fell back to MRO inheritance on delete. Deleting one of the pair could disable the surviving operation: after `del D.__set__`, an instance delete stopped calling the remaining __delete__. Resolve both names together like the TpSetattro accessor does: any Python-level definition selects the dispatching wrapper, matching native functions are stored directly, a single native function with the other name absent is stored directly, and the slot is inherited from the MRO only when neither name resolves. Assisted-by: Claude --- crates/vm/src/types/slot.rs | 47 +++++++++++++++++++++++-------------- 1 file changed, 30 insertions(+), 17 deletions(-) diff --git a/crates/vm/src/types/slot.rs b/crates/vm/src/types/slot.rs index 6e00c160975..f937c56e4a2 100644 --- a/crates/vm/src/types/slot.rs +++ b/crates/vm/src/types/slot.rs @@ -951,24 +951,37 @@ impl PyType { } SlotAccessor::TpDescrGet => update_main_slot!(descr_get, descr_get_wrapper, DescrGet), SlotAccessor::TpDescrSet => { - // __set__ and __delete__ share the same slot - if ADD { - match self.lookup_slot_in_mro(name, ctx, |sf| match sf { - SlotFunc::DescrSet(f) | SlotFunc::DescrDel(f) => Some(*f), - _ => None, - }) { - SlotLookupResult::NativeSlot(func) => { - self.slots.descr_set.store(Some(func)); - } - SlotLookupResult::PythonMethod => { - self.slots.descr_set.store(Some(descr_set_wrapper)); - } - SlotLookupResult::NotFound => { - accessor.inherit_from_mro(self); - } + // __set__ and __delete__ share the same slot, so both names + // must be resolved together: a Python-level definition of + // either one forces the dispatching wrapper, and the native + // slot is only usable when every resolved name agrees on it. + // This resolution reads the current attribute dicts, so it + // applies the same whether a name was just added or removed. + let extract = |sf: &SlotFunc| match sf { + SlotFunc::DescrSet(f) | SlotFunc::DescrDel(f) => Some(*f), + _ => None, + }; + let set = self.lookup_slot_in_mro(identifier!(ctx, __set__), ctx, extract); + let delete = self.lookup_slot_in_mro(identifier!(ctx, __delete__), ctx, extract); + use SlotLookupResult::{NativeSlot, NotFound, PythonMethod}; + match (set, delete) { + (PythonMethod, _) | (_, PythonMethod) => { + self.slots.descr_set.store(Some(descr_set_wrapper)); + } + (NativeSlot(set), NativeSlot(delete)) => { + let func = if set as usize == delete as usize { + set + } else { + descr_set_wrapper + }; + self.slots.descr_set.store(Some(func)); + } + (NativeSlot(func), NotFound) | (NotFound, NativeSlot(func)) => { + self.slots.descr_set.store(Some(func)); + } + (NotFound, NotFound) => { + accessor.inherit_from_mro(self); } - } else { - accessor.inherit_from_mro(self); } } From e902ad1942a920a94153c6c6ef92efd22dd874a4 Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Mon, 6 Jul 2026 11:04:26 +0900 Subject: [PATCH 37/92] Update type base on __bases__ assignment set_bases replaced bases and mro but never updated the stored base, so __base__ kept reporting the old value after reassignment and slot resolution (tp_new inheritance, set_new/set_alloc during init_slots) read the stale solid base. Change the base field to PyAtomicRef> so it can be swapped under the type lock while remaining lock-free for readers, and recompute it in set_bases with the same best_base validation type creation uses (BASETYPE flag, instance layout conflict among bases). The swapped-out base is parked in the frame's temporary refs; other references released inside the critical section are dropped after the lock is released. Also in set_bases: - remove this class from the old bases' subclass lists (pruning dead entries), so repeated assignment no longer accumulates duplicates - roll back bases, base, and all updated mros when the recursive mro update fails, instead of leaving the type half-reparented - fix the misformatted empty-tuple error message Remove the expectedFailure marker from test_unsubclassable_types, which now passes. Assisted-by: Claude --- Lib/test/test_descr.py | 1 - crates/vm/src/builtins/type.rs | 156 +++++++++++++++++++++++-------- crates/vm/src/object/core.rs | 8 +- crates/vm/src/types/slot.rs | 2 +- crates/vm/src/types/slot_defs.rs | 2 +- 5 files changed, 121 insertions(+), 48 deletions(-) diff --git a/Lib/test/test_descr.py b/Lib/test/test_descr.py index 92bf7998d75..ee85863bcca 100644 --- a/Lib/test/test_descr.py +++ b/Lib/test/test_descr.py @@ -4199,7 +4199,6 @@ class D(C): else: self.fail("best_base calculation found wanting") - @unittest.expectedFailure # TODO: RUSTPYTHON def test_unsubclassable_types(self): with self.assertRaises(TypeError): class X(type(None)): diff --git a/crates/vm/src/builtins/type.rs b/crates/vm/src/builtins/type.rs index 23c581889d1..cd242e0e328 100644 --- a/crates/vm/src/builtins/type.rs +++ b/crates/vm/src/builtins/type.rs @@ -44,7 +44,8 @@ use std::collections::HashSet; #[pyclass(module = false, name = "type", traverse = "manual")] pub struct PyType { - pub base: Option, + /// tp_base. Written under the type lock (see `set_bases`); read lock-free. + pub base: PyAtomicRef>, pub bases: PyRwLock>, pub mro: PyRwLock>, pub subclasses: PyRwLock>>, @@ -237,7 +238,9 @@ pub unsafe fn type_cache_after_fork() { unsafe impl crate::object::Traverse for PyType { fn traverse(&self, tracer_fn: &mut crate::object::TraverseFn<'_>) { - self.base.traverse(tracer_fn); + if let Some(base) = self.base.deref() { + tracer_fn(base.as_object()); + } self.bases.traverse(tracer_fn); self.mro.traverse(tracer_fn); self.subclasses.traverse(tracer_fn); @@ -253,7 +256,8 @@ unsafe impl crate::object::Traverse for PyType { /// type_clear: break reference cycles in type objects fn clear(&mut self, out: &mut Vec) { - if let Some(base) = self.base.take() { + // SAFETY: tp_clear runs with exclusive access to the type object. + if let Some(base) = unsafe { self.base.swap(None) } { out.push(base.into()); } if let Some(mut guard) = self.bases.try_write() { @@ -808,7 +812,7 @@ impl PyType { let inherited_abc_tpflags = Self::inherited_abc_tpflags(&bases); let new_type = PyRef::new_ref( Self { - base: Some(base), + base: Some(base).into(), bases: PyRwLock::new(bases), mro: PyRwLock::new(mro), subclasses: PyRwLock::default(), @@ -872,7 +876,7 @@ impl PyType { let new_type = PyRef::new_ref( Self { - base: Some(base), + base: Some(base).into(), bases, mro: PyRwLock::new(mro), subclasses: PyRwLock::default(), @@ -899,8 +903,8 @@ impl PyType { // Note: inherit_slots is called in PyClassImpl::init_class after // slots are fully initialized by make_slots() - Self::set_new(&new_type.slots, new_type.base.as_ref()); - Self::set_alloc(&new_type.slots, new_type.base.as_ref()); + Self::set_new(&new_type.slots, new_type.base.deref()); + Self::set_alloc(&new_type.slots, new_type.base.deref()); let weakref_type = super::PyWeak::static_type(); for base in new_type.bases.read().iter() { @@ -946,11 +950,11 @@ impl PyType { self.update_slot::(attr_name, ctx); } - Self::set_new(&self.slots, self.base.as_ref()); - Self::set_alloc(&self.slots, self.base.as_ref()); + Self::set_new(&self.slots, self.base.deref()); + Self::set_alloc(&self.slots, self.base.deref()); } - fn set_new(slots: &PyTypeSlots, base: Option<&PyTypeRef>) { + fn set_new(slots: &PyTypeSlots, base: Option<&Py>) { if slots.flags.contains(PyTypeFlags::DISALLOW_INSTANTIATION) { slots.new.store(None) } else if slots.new.load().is_none() { @@ -958,7 +962,7 @@ impl PyType { } } - fn set_alloc(slots: &PyTypeSlots, base: Option<&PyTypeRef>) { + fn set_alloc(slots: &PyTypeSlots, base: Option<&Py>) { if slots.alloc.load().is_none() { slots .alloc @@ -1411,7 +1415,7 @@ impl Py { } pub fn iter_base_chain(&self) -> impl Iterator { - core::iter::successors(Some(self), |cls| cls.base.as_deref()) + core::iter::successors(Some(self), |cls| cls.base.deref()) } pub fn extend_methods(&'static self, method_defs: &'static [PyMethodDef], ctx: &Context) { @@ -1458,36 +1462,114 @@ impl PyType { } if bases.is_empty() { return Err(vm.new_type_error(format!( - "can only assign non-empty tuple to %s.__bases__, not {}", + "can only assign non-empty tuple to {}.__bases__, not ()", zelf.name() ))); } // TODO: check for mro cycles + // TODO: check layout compatibility between the old and new solid base + + // Compute the new solid base before committing anything. This also + // validates the new bases (BASETYPE flag, no instance layout + // conflict), the same checks type creation performs. + let new_base = best_base(&bases, vm)?.to_owned(); + + // References released inside the critical section are collected here + // and dropped after the lock: dropping them inside can run arbitrary + // code that re-acquires the non-reentrant type mutex. + let mut retired: Vec = Vec::new(); + + // A base swapped out of `zelf.base` may still be observed by + // concurrent lock-free readers; keep it alive in the frame's + // temporary refs so they never see a dangling pointer. + let keep_alive = |type_ref: PyTypeRef, retired: &mut Vec| { + if let Some(frame) = vm.current_frame() { + frame.temporary_refs.lock().push(type_ref.into()); + } else { + retired.push(type_ref.into()); + } + }; - // TODO: Remove this class from all subclass lists - // for base in self.bases.read().iter() { - // let subclasses = base.subclasses.write(); - // // TODO: how to uniquely identify the subclasses to remove? - // } + // Register this type as a subclass of the given bases + let register_subclasses = |bases: &[PyTypeRef]| { + let weakref_type = super::PyWeak::static_type(); + for base in bases { + base.subclasses.write().push( + zelf.as_object() + .downgrade_with_weakref_typ_opt(None, weakref_type.to_owned()) + .unwrap(), + ); + } + }; - Self::with_type_lock(vm, || { - *zelf.bases.write() = bases; - // Recursively update the mros of this class and all subclasses - fn update_mro_recursively(cls: &PyType, vm: &VirtualMachine) -> PyResult<()> { + let result = Self::with_type_lock(vm, || { + // Remove this class from the old bases' subclass lists, pruning + // dead entries along the way. Upgraded refs are retired so the + // last strong reference is never dropped under the lock. + for base in zelf.bases.read().iter() { + let mut subclasses = base.subclasses.write(); + let mut kept = Vec::with_capacity(subclasses.len()); + for weak in subclasses.drain(..) { + match weak.upgrade() { + Some(obj) if obj.is(zelf.as_object()) => { + retired.push(obj); + retired.push(weak.into()); + } + Some(obj) => { + retired.push(obj); + kept.push(weak); + } + None => retired.push(weak.into()), + } + } + *subclasses = kept; + } + + let old_bases = core::mem::replace(&mut *zelf.bases.write(), bases); + let old_base = unsafe { zelf.base.swap(Some(new_base)) }; + + // Recursively update the mros of this class and all subclasses, + // recording the previous mros so a failure can be rolled back. + fn update_mro_recursively( + cls: &Py, + undo: &mut Vec<(PyTypeRef, Vec)>, + vm: &VirtualMachine, + ) -> PyResult<()> { let mut mro = PyType::resolve_mro(&cls.bases.read()).map_err(|msg| vm.new_type_error(msg))?; // Preserve self (mro[0]) when updating MRO mro.insert(0, cls.mro.read()[0].to_owned()); - *cls.mro.write() = mro; + let old_mro = core::mem::replace(&mut *cls.mro.write(), mro); + undo.push((cls.to_owned(), old_mro)); for subclass in cls.subclasses.read().iter() { let subclass = subclass.upgrade().unwrap(); let subclass: &Py = subclass.downcast_ref().unwrap(); - update_mro_recursively(subclass, vm)?; + update_mro_recursively(subclass, undo, vm)?; } Ok(()) } - update_mro_recursively(zelf, vm)?; + let mut undo = Vec::new(); + if let Err(err) = update_mro_recursively(zelf, &mut undo, vm) { + // Roll back to the previous state + for (cls, old_mro) in undo { + let failed_mro = core::mem::replace(&mut *cls.mro.write(), old_mro); + retired.extend(failed_mro.into_iter().map(Into::into)); + retired.push(cls.into()); + } + let failed_bases = core::mem::replace(&mut *zelf.bases.write(), old_bases); + if let Some(failed_base) = unsafe { zelf.base.swap(old_base) } { + keep_alive(failed_base, &mut retired); + } + register_subclasses(&zelf.bases.read()); + retired.extend(failed_bases.into_iter().map(Into::into)); + zelf.modified_inner(); + return Err(err); + } + retired.extend(old_bases.into_iter().map(Into::into)); + if let Some(old_base) = old_base { + keep_alive(old_base, &mut retired); + } // Invalidate inline caches zelf.modified_inner(); @@ -1495,24 +1577,16 @@ impl PyType { // TODO: do any old slots need to be cleaned up first? zelf.init_slots(&vm.ctx); - // Register this type as a subclass of its new bases - let weakref_type = super::PyWeak::static_type(); - for base in zelf.bases.read().iter() { - base.subclasses.write().push( - zelf.as_object() - .downgrade_with_weakref_typ_opt(None, weakref_type.to_owned()) - .unwrap(), - ); - } + register_subclasses(&zelf.bases.read()); Ok(()) - })?; - - Ok(()) + }); + drop(retired); + result } #[pygetset] fn __base__(&self) -> Option { - self.base.clone() + self.base.to_owned() } #[pygetset] @@ -2947,8 +3021,8 @@ pub(crate) fn call_slot_new( // that's not a heap type is this type. let mut staticbase = subtype.clone(); while staticbase.slots.flags.has_feature(PyTypeFlags::HEAPTYPE) { - if let Some(base) = staticbase.base.as_ref() { - staticbase = base.clone(); + if let Some(base) = staticbase.base.to_owned() { + staticbase = base; } else { break; } @@ -3080,7 +3154,7 @@ fn shape_differs(t1: &Py, t2: &Py) -> bool { } fn solid_base<'a>(typ: &'a Py, vm: &VirtualMachine) -> &'a Py { - let base = if let Some(base) = &typ.base { + let base = if let Some(base) = typ.base.deref() { solid_base(base, vm) } else { vm.ctx.types.object_type diff --git a/crates/vm/src/object/core.rs b/crates/vm/src/object/core.rs index 43efac53a23..d1638881ac3 100644 --- a/crates/vm/src/object/core.rs +++ b/crates/vm/src/object/core.rs @@ -2495,7 +2495,7 @@ pub(crate) fn init_type_hierarchy() -> (PyTypeRef, PyTypeRef, PyTypeRef) { static_assertions::assert_eq_align!(MaybeUninit>, PyInner); let type_payload = PyType { - base: None, + base: None.into(), bases: PyRwLock::default(), mro: PyRwLock::default(), subclasses: PyRwLock::default(), @@ -2506,7 +2506,7 @@ pub(crate) fn init_type_hierarchy() -> (PyTypeRef, PyTypeRef, PyTypeRef) { abc_tpflags: core::sync::atomic::AtomicU64::new(0), }; let object_payload = PyType { - base: None, + base: None.into(), bases: PyRwLock::default(), mro: PyRwLock::default(), subclasses: PyRwLock::default(), @@ -2591,7 +2591,7 @@ pub(crate) fn init_type_hierarchy() -> (PyTypeRef, PyTypeRef, PyTypeRef) { (*object_type_ptr).payload.mro = PyRwLock::new(vec![object_type.clone()]); (*type_type_ptr).payload.bases = PyRwLock::new(vec![object_type.clone()]); - (*type_type_ptr).payload.base = Some(object_type.clone()); + (*type_type_ptr).payload.base = Some(object_type.clone()).into(); let type_type = PyTypeRef::from_raw(type_type_ptr.cast()); // type's mro is [type, object] @@ -2603,7 +2603,7 @@ pub(crate) fn init_type_hierarchy() -> (PyTypeRef, PyTypeRef, PyTypeRef) { }; let weakref_type = PyType { - base: Some(object_type.clone()), + base: Some(object_type.clone()).into(), bases: PyRwLock::new(vec![object_type.clone()]), mro: PyRwLock::new(vec![object_type.clone()]), subclasses: PyRwLock::default(), diff --git a/crates/vm/src/types/slot.rs b/crates/vm/src/types/slot.rs index f937c56e4a2..089343b4270 100644 --- a/crates/vm/src/types/slot.rs +++ b/crates/vm/src/types/slot.rs @@ -866,7 +866,7 @@ impl PyType { self.slots.new.store(Some(new_wrapper)); self.slots.vectorcall.store(None); } else { - let inherited = self.base.as_ref().and_then(|base| base.slots.new.load()); + let inherited = self.base.deref().and_then(|base| base.slots.new.load()); self.slots.new.store(inherited); } } diff --git a/crates/vm/src/types/slot_defs.rs b/crates/vm/src/types/slot_defs.rs index 8637f811272..34d62147e0c 100644 --- a/crates/vm/src/types/slot_defs.rs +++ b/crates/vm/src/types/slot_defs.rs @@ -608,7 +608,7 @@ impl SlotAccessor { if typ.slots.init.load().is_none() && let Some(base_val) = base.slots.init.load() { - let slot_defined = base.base.as_ref().is_none_or(|bb| { + let slot_defined = base.base.deref().is_none_or(|bb| { bb.slots.init.load().map(|v| v as usize) != Some(base_val as usize) }); if slot_defined { From bc2aba2c19d4b157aedac82eab4032f2c2e0419a Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Mon, 6 Jul 2026 11:35:52 +0900 Subject: [PATCH 38/92] Fix rollback order, dead weakrefs, and lock-held drops in set_bases - Restore recorded mros in reverse on rollback so a class visited multiple times through diamond inheritance ends with its original mro instead of an intermediate one. - Skip dead weakrefs in the subclass list during the recursive mro update instead of panicking on upgrade. - Retire the replaced mros on the success path instead of dropping them while the type lock is held. Assisted-by: Claude --- crates/vm/src/builtins/type.rs | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/crates/vm/src/builtins/type.rs b/crates/vm/src/builtins/type.rs index cd242e0e328..ec11f0a0c24 100644 --- a/crates/vm/src/builtins/type.rs +++ b/crates/vm/src/builtins/type.rs @@ -1543,7 +1543,10 @@ impl PyType { let old_mro = core::mem::replace(&mut *cls.mro.write(), mro); undo.push((cls.to_owned(), old_mro)); for subclass in cls.subclasses.read().iter() { - let subclass = subclass.upgrade().unwrap(); + // Dead entries are pruned elsewhere; skip them here. + let Some(subclass) = subclass.upgrade() else { + continue; + }; let subclass: &Py = subclass.downcast_ref().unwrap(); update_mro_recursively(subclass, undo, vm)?; } @@ -1551,8 +1554,10 @@ impl PyType { } let mut undo = Vec::new(); if let Err(err) = update_mro_recursively(zelf, &mut undo, vm) { - // Roll back to the previous state - for (cls, old_mro) in undo { + // Roll back to the previous state. A class reachable through + // multiple bases is recorded once per visit, so restore in + // reverse to end with the first-recorded (original) mro. + for (cls, old_mro) in undo.into_iter().rev() { let failed_mro = core::mem::replace(&mut *cls.mro.write(), old_mro); retired.extend(failed_mro.into_iter().map(Into::into)); retired.push(cls.into()); @@ -1566,6 +1571,12 @@ impl PyType { zelf.modified_inner(); return Err(err); } + // Retire the replaced mros as well; dropping them here would + // release them while the lock is held. + for (cls, old_mro) in undo { + retired.extend(old_mro.into_iter().map(Into::into)); + retired.push(cls.into()); + } retired.extend(old_bases.into_iter().map(Into::into)); if let Some(old_base) = old_base { keep_alive(old_base, &mut retired); From b8d11f23e6171011a77736843bb35c2ce95f0891 Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Mon, 6 Jul 2026 16:32:31 +0900 Subject: [PATCH 39/92] Reify number sub-slot wrapper once in update_one_slot The update_sub_slot! macro expanded the Python-method wrapper closure at both the own and inherited store sites. Each expansion is a distinct fn item, so a base and a non-overriding subclass stored wrappers with distinct addresses in unmerged debug builds. binary_op1 compares slot fn addresses to detect whether a subclass overrides the operator, so the inherited slot was misread as an override and C() // E() dispatched to __rfloordiv__ instead of __floordiv__. Bind the wrapper store in a single closure and call it from both branches so the fn item is reified once and the slot value stays identical across a base and its subclasses. Document the conservative-on-mismatch nature of the new_wrapper address guard in call_wrapped. Assisted-by: Claude --- crates/vm/src/builtins/type.rs | 4 ++++ crates/vm/src/types/slot.rs | 11 +++++++++-- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/crates/vm/src/builtins/type.rs b/crates/vm/src/builtins/type.rs index ec11f0a0c24..db7fc490544 100644 --- a/crates/vm/src/builtins/type.rs +++ b/crates/vm/src/builtins/type.rs @@ -2803,6 +2803,10 @@ impl Callable for PyType { // function. new_wrapper is excluded because a Python `__new__` // can install an `__init__` on the class or return an instance // of another class while it runs. + // The address comparison is against the single new_wrapper fn item, + // so a mismatch is conservative: if it ever compared unequal for the + // wrapper it would only take the slower cloning path, never the fast + // path incorrectly. if zelf.slots.init.load().is_none() && !zelf.is(vm.ctx.types.type_type) && slot_new as usize != crate::types::new_wrapper as crate::types::NewFunc as usize diff --git a/crates/vm/src/types/slot.rs b/crates/vm/src/types/slot.rs index 089343b4270..a2aa62e0e71 100644 --- a/crates/vm/src/types/slot.rs +++ b/crates/vm/src/types/slot.rs @@ -760,8 +760,15 @@ impl PyType { } result }; + // Reify the wrapper at a single site so the own and inherited + // branches store the same fn item. binary_op1 compares slot + // fn addresses to decide whether a subclass overrides the op; + // duplicating the wrapper closure across branches yields + // distinct addresses in unmerged debug builds and breaks that + // comparison for an inherited slot. + let store_wrapper = || self.slots.$group.$slot.store(Some($wrapper)); if has_own { - self.slots.$group.$slot.store(Some($wrapper)); + store_wrapper(); } else { match self.lookup_slot_in_mro(name, ctx, |sf| { if let SlotFunc::$variant(f) = sf { @@ -774,7 +781,7 @@ impl PyType { self.slots.$group.$slot.store(Some(func)); } SlotLookupResult::PythonMethod => { - self.slots.$group.$slot.store(Some($wrapper)); + store_wrapper(); } SlotLookupResult::NotFound => { accessor.inherit_from_mro(self); From a7fc01a2a0064a669f76d6e31993d8b5deecb3d4 Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Mon, 6 Jul 2026 16:43:05 +0900 Subject: [PATCH 40/92] Restrict type_cache_after_fork visibility and guard type-cache reads type_cache_after_fork is only called from within the crate, so mark it pub(crate) to resolve the unreachable-pub warning. Add debug_assert_current_thread_attached at the has_name_in_mro lock-free type-cache read site, matching lookup_ref_and_version_interned, and gate the function on debug_assertions so it is not compiled unused in release builds where the call sites are elided. Assisted-by: Claude --- crates/vm/src/builtins/type.rs | 5 ++++- crates/vm/src/vm/thread.rs | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/crates/vm/src/builtins/type.rs b/crates/vm/src/builtins/type.rs index db7fc490544..bc9947c2c74 100644 --- a/crates/vm/src/builtins/type.rs +++ b/crates/vm/src/builtins/type.rs @@ -223,7 +223,7 @@ pub(crate) fn type_cache_clear() { /// If fork happens while a writer holds an entry SeqLock, the child inherits /// the odd sequence value with no surviving writer to release it. Clear only /// those in-progress entries, matching CPython's `_PyTypes_AfterFork()`. -pub unsafe fn type_cache_after_fork() { +pub(crate) unsafe fn type_cache_after_fork() { for entry in TYPE_CACHE.iter() { let seq = entry.sequence.load(Ordering::Relaxed); if (seq & 1) == 0 { @@ -1243,6 +1243,9 @@ impl PyType { /// Check if attribute exists in MRO, using method cache for fast check. /// Unlike find_name_in_mro, avoids cloning the value on cache hit. fn has_name_in_mro(&self, name: &'static PyStrInterned) -> bool { + #[cfg(all(unix, feature = "threading", debug_assertions))] + crate::vm::thread::debug_assert_current_thread_attached(); + let version = self.tp_version_tag.load(Ordering::Acquire); if version != 0 { let idx = type_cache_hash(version, name); diff --git a/crates/vm/src/vm/thread.rs b/crates/vm/src/vm/thread.rs index bb7980004e6..8a8ff17a1dc 100644 --- a/crates/vm/src/vm/thread.rs +++ b/crates/vm/src/vm/thread.rs @@ -545,7 +545,7 @@ pub(crate) fn qsbr_checkpoint() { /// Debug check: lock-free type-cache reads are only sound on threads that /// are registered with QSBR and currently ATTACHED. -#[cfg(all(unix, feature = "threading"))] +#[cfg(all(unix, feature = "threading", debug_assertions))] pub(crate) fn debug_assert_current_thread_attached() { CURRENT_THREAD_SLOT.with(|slot| { if let Some(s) = slot.borrow().as_ref() { From 4d7126a6c2c9df4ab1e70f43db2d2a07e6572929 Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Mon, 6 Jul 2026 16:51:08 +0900 Subject: [PATCH 41/92] Guard BinaryOpSubscrGetitem with the cached type version The specialized BINARY_OP_SUBSCR_GETITEM handler used the cached __getitem__ after checking only the function version, unlike the sibling specialized handlers which revalidate the type version tag first. Store the type version in the inline cache at specialization time and revalidate owner.class().tp_version_tag against it before using the cached function, deopting to the generic subscript path on mismatch. Assisted-by: Claude --- crates/vm/src/frame.rs | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/crates/vm/src/frame.rs b/crates/vm/src/frame.rs index 7ef6257a361..be89ce2c10c 100644 --- a/crates/vm/src/frame.rs +++ b/crates/vm/src/frame.rs @@ -4530,8 +4530,12 @@ impl ExecutingFrame<'_> { } } Instruction::BinaryOpSubscrGetitem => { + let cache_base = self.lasti() as usize; + let type_version = self.code.instructions.read_cache_u32(cache_base + 1); let owner = self.nth_value(1); if !self.specialization_eval_frame_active(vm) + && type_version != 0 + && owner.class().tp_version_tag.load(Acquire) == type_version && let Some((func, func_version)) = owner.class().get_cached_getitem_for_specialization() && func.func_version() == func_version @@ -8303,16 +8307,17 @@ impl ExecutingFrame<'_> { && let Some(func) = _getitem.downcast_ref_if_exact::(vm) && func.can_specialize_call(2) { - if type_version != 0 { - if cls.cache_getitem_for_specialization( - func.to_owned(), - type_version, - vm, - ) { - Some(Instruction::BinaryOpSubscrGetitem) - } else { - None + if type_version != 0 + && cls.cache_getitem_for_specialization(func.to_owned(), type_version, vm) + { + // Record the type version so the specialized handler + // can revalidate before using the cached __getitem__. + unsafe { + self.code + .instructions + .write_cache_u32(cache_base + 1, type_version); } + Some(Instruction::BinaryOpSubscrGetitem) } else { None } From 18e847480e683eb6fab388ddab1f9c8dc71aaa0b Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Mon, 6 Jul 2026 16:53:32 +0900 Subject: [PATCH 42/92] Call __abstractmethods__ __len__ once in object.__new__ The abstract-method check invoked the user-visible __len__ twice: once via length_opt for the count and again while materializing the method names. Derive the count from the materialized list instead, so __len__ runs once. Assisted-by: Claude --- crates/vm/src/builtins/object.rs | 44 ++++++++++++-------------------- 1 file changed, 17 insertions(+), 27 deletions(-) diff --git a/crates/vm/src/builtins/object.rs b/crates/vm/src/builtins/object.rs index fcdd0ffeb34..b1feb467397 100644 --- a/crates/vm/src/builtins/object.rs +++ b/crates/vm/src/builtins/object.rs @@ -65,34 +65,24 @@ impl Constructor for PyBaseObject { } // Ensure that all abstract methods are implemented before instantiating instance. - if let Some(abs_methods) = cls.get_attr(identifier!(vm, __abstractmethods__)) - && let Some(unimplemented_abstract_method_count) = abs_methods.length_opt(vm) - { + if let Some(abs_methods) = cls.get_attr(identifier!(vm, __abstractmethods__)) { let methods: Vec = abs_methods.try_to_value(vm)?; - let methods: String = Itertools::intersperse( - methods.iter().map(|name| name.as_str().to_owned()), - "', '".to_owned(), - ) - .collect(); - - let unimplemented_abstract_method_count = unimplemented_abstract_method_count?; - let name = cls.name().to_string(); - - match unimplemented_abstract_method_count { - 0 => {} - 1 => { - return Err(vm.new_type_error(format!( - "class {name} without an implementation for abstract method '{methods}'" - ))); - } - 2.. => { - return Err(vm.new_type_error(format!( - "class {name} without an implementation for abstract methods '{methods}'" - ))); - } - // TODO: remove `allow` when redox build doesn't complain about it - #[allow(unreachable_patterns)] - _ => unreachable!(), + let unimplemented_abstract_method_count = methods.len(); + if unimplemented_abstract_method_count > 0 { + let methods: String = Itertools::intersperse( + methods.iter().map(|name| name.as_str().to_owned()), + "', '".to_owned(), + ) + .collect(); + let name = cls.name().to_string(); + let noun = if unimplemented_abstract_method_count == 1 { + "method" + } else { + "methods" + }; + return Err(vm.new_type_error(format!( + "class {name} without an implementation for abstract {noun} '{methods}'" + ))); } } From 6133908d565ae2ef2844dedc887f51743d1d2e91 Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Mon, 6 Jul 2026 17:15:40 +0900 Subject: [PATCH 43/92] Use i64 fast paths for specialized int add/sub/mul Rewrite execute_binary_op_int to box results through new_int via i64 checked arithmetic instead of raw BigInt ops with new_bigint. Add an int_mul helper and a shared int_fast_op that computes the i64 result and falls back to the BigInt operation on to_i64 or checked-op failure. Wire int_mul into both the specialized BinaryOpMultiplyInt handler and the generic execute_bin_op Multiply/InplaceMultiply path, gated on exact int operands so subclasses keep dispatching through _mul/_imul. Assisted-by: Claude --- crates/vm/src/frame.rs | 70 +++++++++++++++++++++++++++++------------- 1 file changed, 48 insertions(+), 22 deletions(-) diff --git a/crates/vm/src/frame.rs b/crates/vm/src/frame.rs index be89ce2c10c..6746bab0d04 100644 --- a/crates/vm/src/frame.rs +++ b/crates/vm/src/frame.rs @@ -4496,13 +4496,13 @@ impl ExecutingFrame<'_> { } // Specialized BINARY_OP opcodes Instruction::BinaryOpAddInt => { - self.execute_binary_op_int(vm, |a, b| a + b, bytecode::BinaryOperator::Add) + self.execute_binary_op_int(vm, Self::int_add, bytecode::BinaryOperator::Add) } Instruction::BinaryOpSubtractInt => { - self.execute_binary_op_int(vm, |a, b| a - b, bytecode::BinaryOperator::Subtract) + self.execute_binary_op_int(vm, Self::int_sub, bytecode::BinaryOperator::Subtract) } Instruction::BinaryOpMultiplyInt => { - self.execute_binary_op_int(vm, |a, b| a * b, bytecode::BinaryOperator::Multiply) + self.execute_binary_op_int(vm, Self::int_mul, bytecode::BinaryOperator::Multiply) } Instruction::BinaryOpAddFloat => { self.execute_binary_op_float(vm, |a, b| a + b, bytecode::BinaryOperator::Add) @@ -7389,7 +7389,7 @@ impl ExecutingFrame<'_> { a_ref.downcast_ref_if_exact::(vm), b_ref.downcast_ref_if_exact::(vm), ) { - Ok(self.int_add(a.as_bigint(), b.as_bigint(), vm)) + Ok(Self::int_add(a.as_bigint(), b.as_bigint(), vm)) } else if matches!(op, bytecode::BinaryOperator::Add) { vm._add(a_ref, b_ref) } else { @@ -7401,14 +7401,25 @@ impl ExecutingFrame<'_> { a_ref.downcast_ref_if_exact::(vm), b_ref.downcast_ref_if_exact::(vm), ) { - Ok(self.int_sub(a.as_bigint(), b.as_bigint(), vm)) + Ok(Self::int_sub(a.as_bigint(), b.as_bigint(), vm)) } else if matches!(op, bytecode::BinaryOperator::Subtract) { vm._sub(a_ref, b_ref) } else { vm._isub(a_ref, b_ref) } } - bytecode::BinaryOperator::Multiply => vm._mul(a_ref, b_ref), + bytecode::BinaryOperator::Multiply | bytecode::BinaryOperator::InplaceMultiply => { + if let (Some(a), Some(b)) = ( + a_ref.downcast_ref_if_exact::(vm), + b_ref.downcast_ref_if_exact::(vm), + ) { + Ok(Self::int_mul(a.as_bigint(), b.as_bigint(), vm)) + } else if matches!(op, bytecode::BinaryOperator::Multiply) { + vm._mul(a_ref, b_ref) + } else { + vm._imul(a_ref, b_ref) + } + } bytecode::BinaryOperator::MatrixMultiply => vm._matmul(a_ref, b_ref), bytecode::BinaryOperator::Power => vm._pow(a_ref, b_ref, vm.ctx.none.as_object()), bytecode::BinaryOperator::TrueDivide => vm._truediv(a_ref, b_ref), @@ -7419,7 +7430,6 @@ impl ExecutingFrame<'_> { bytecode::BinaryOperator::Xor => vm._xor(a_ref, b_ref), bytecode::BinaryOperator::Or => vm._or(a_ref, b_ref), bytecode::BinaryOperator::And => vm._and(a_ref, b_ref), - bytecode::BinaryOperator::InplaceMultiply => vm._imul(a_ref, b_ref), bytecode::BinaryOperator::InplaceMatrixMultiply => vm._imatmul(a_ref, b_ref), bytecode::BinaryOperator::InplacePower => { vm._ipow(a_ref, b_ref, vm.ctx.none.as_object()) @@ -7439,28 +7449,44 @@ impl ExecutingFrame<'_> { Ok(None) } - /// Int addition with i64 fast path to avoid BigInt heap allocation. + /// Int binary op with an i64 fast path to avoid BigInt heap allocation. + /// `checked` computes the i64 result; on `None` (either operand does not + /// fit i64, or the op overflows i64) it falls through to `fallback` on the + /// full BigInt values. Result boxing always goes through `new_int` so the + /// small-int cache is consulted identically. #[inline] - fn int_add(&self, a: &BigInt, b: &BigInt, vm: &VirtualMachine) -> PyObjectRef { + fn int_fast_op( + a: &BigInt, + b: &BigInt, + vm: &VirtualMachine, + checked: fn(i64, i64) -> Option, + fallback: impl FnOnce(&BigInt, &BigInt) -> BigInt, + ) -> PyObjectRef { use num_traits::ToPrimitive; if let (Some(av), Some(bv)) = (a.to_i64(), b.to_i64()) - && let Some(result) = av.checked_add(bv) + && let Some(result) = checked(av, bv) { return vm.ctx.new_int(result).into(); } - vm.ctx.new_int(a + b).into() + vm.ctx.new_int(fallback(a, b)).into() + } + + /// Int addition with i64 fast path to avoid BigInt heap allocation. + #[inline] + fn int_add(a: &BigInt, b: &BigInt, vm: &VirtualMachine) -> PyObjectRef { + Self::int_fast_op(a, b, vm, i64::checked_add, |a, b| a + b) } /// Int subtraction with i64 fast path to avoid BigInt heap allocation. #[inline] - fn int_sub(&self, a: &BigInt, b: &BigInt, vm: &VirtualMachine) -> PyObjectRef { - use num_traits::ToPrimitive; - if let (Some(av), Some(bv)) = (a.to_i64(), b.to_i64()) - && let Some(result) = av.checked_sub(bv) - { - return vm.ctx.new_int(result).into(); - } - vm.ctx.new_int(a - b).into() + fn int_sub(a: &BigInt, b: &BigInt, vm: &VirtualMachine) -> PyObjectRef { + Self::int_fast_op(a, b, vm, i64::checked_sub, |a, b| a - b) + } + + /// Int multiplication with i64 fast path to avoid BigInt heap allocation. + #[inline] + fn int_mul(a: &BigInt, b: &BigInt, vm: &VirtualMachine) -> PyObjectRef { + Self::int_fast_op(a, b, vm, i64::checked_mul, |a, b| a * b) } #[cold] @@ -8487,7 +8513,7 @@ impl ExecutingFrame<'_> { fn execute_binary_op_int( &mut self, vm: &VirtualMachine, - op: impl FnOnce(&BigInt, &BigInt) -> BigInt, + op: impl FnOnce(&BigInt, &BigInt, &VirtualMachine) -> PyObjectRef, deopt_op: bytecode::BinaryOperator, ) -> FrameResult { let b = self.top_value(); @@ -8496,10 +8522,10 @@ impl ExecutingFrame<'_> { a.downcast_ref_if_exact::(vm), b.downcast_ref_if_exact::(vm), ) { - let result = op(a_int.as_bigint(), b_int.as_bigint()); + let result = op(a_int.as_bigint(), b_int.as_bigint(), vm); self.pop_value(); self.pop_value(); - self.push_value(vm.ctx.new_bigint(&result).into()); + self.push_value(result); Ok(None) } else { self.execute_bin_op(vm, deopt_op) From 1499b31022a205c99163473a6f0f18b5c93bbd4a Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Mon, 6 Jul 2026 17:32:08 +0900 Subject: [PATCH 44/92] Use i64 fast paths for exact int floordiv and remainder Add floordiv_i64/mod_i64 computing i64 floor-division and divisor-signed remainder, guarding zero divisor and i64::MIN overflow by returning None. Wire them through int_floordiv/int_mod (shared int_div_fast_op boxes via new_int) into the generic execute_bin_op FloorDivide/Remainder and their Inplace variants, gated on exact int operands so subclasses and zero divisors fall through to the existing _floordiv/_mod/_ifloordiv/_imod slow path. Assisted-by: Claude --- crates/vm/src/frame.rs | 100 ++++++++++++++++++++++++++++++++++++++--- 1 file changed, 93 insertions(+), 7 deletions(-) diff --git a/crates/vm/src/frame.rs b/crates/vm/src/frame.rs index 6746bab0d04..0d8714b26fc 100644 --- a/crates/vm/src/frame.rs +++ b/crates/vm/src/frame.rs @@ -7381,9 +7381,9 @@ impl ExecutingFrame<'_> { let b_ref = &self.pop_value(); let a_ref = &self.pop_value(); let value = match op { - // BINARY_OP_ADD_INT / BINARY_OP_SUBTRACT_INT fast paths: - // bypass binary_op1 dispatch for exact int types, use i64 arithmetic - // when possible to avoid BigInt heap allocation. + // Exact-int fast paths for +, -, *, //, %: bypass binary_op1 + // dispatch and use i64 arithmetic when possible to avoid BigInt + // heap allocation, falling back to the slow path otherwise. bytecode::BinaryOperator::Add | bytecode::BinaryOperator::InplaceAdd => { if let (Some(a), Some(b)) = ( a_ref.downcast_ref_if_exact::(vm), @@ -7423,8 +7423,33 @@ impl ExecutingFrame<'_> { bytecode::BinaryOperator::MatrixMultiply => vm._matmul(a_ref, b_ref), bytecode::BinaryOperator::Power => vm._pow(a_ref, b_ref, vm.ctx.none.as_object()), bytecode::BinaryOperator::TrueDivide => vm._truediv(a_ref, b_ref), - bytecode::BinaryOperator::FloorDivide => vm._floordiv(a_ref, b_ref), - bytecode::BinaryOperator::Remainder => vm._mod(a_ref, b_ref), + bytecode::BinaryOperator::FloorDivide + | bytecode::BinaryOperator::InplaceFloorDivide => { + if let (Some(a), Some(b)) = ( + a_ref.downcast_ref_if_exact::(vm), + b_ref.downcast_ref_if_exact::(vm), + ) && let Some(result) = Self::int_floordiv(a.as_bigint(), b.as_bigint(), vm) + { + Ok(result) + } else if matches!(op, bytecode::BinaryOperator::FloorDivide) { + vm._floordiv(a_ref, b_ref) + } else { + vm._ifloordiv(a_ref, b_ref) + } + } + bytecode::BinaryOperator::Remainder | bytecode::BinaryOperator::InplaceRemainder => { + if let (Some(a), Some(b)) = ( + a_ref.downcast_ref_if_exact::(vm), + b_ref.downcast_ref_if_exact::(vm), + ) && let Some(result) = Self::int_mod(a.as_bigint(), b.as_bigint(), vm) + { + Ok(result) + } else if matches!(op, bytecode::BinaryOperator::Remainder) { + vm._mod(a_ref, b_ref) + } else { + vm._imod(a_ref, b_ref) + } + } bytecode::BinaryOperator::Lshift => vm._lshift(a_ref, b_ref), bytecode::BinaryOperator::Rshift => vm._rshift(a_ref, b_ref), bytecode::BinaryOperator::Xor => vm._xor(a_ref, b_ref), @@ -7435,8 +7460,6 @@ impl ExecutingFrame<'_> { vm._ipow(a_ref, b_ref, vm.ctx.none.as_object()) } bytecode::BinaryOperator::InplaceTrueDivide => vm._itruediv(a_ref, b_ref), - bytecode::BinaryOperator::InplaceFloorDivide => vm._ifloordiv(a_ref, b_ref), - bytecode::BinaryOperator::InplaceRemainder => vm._imod(a_ref, b_ref), bytecode::BinaryOperator::InplaceLshift => vm._ilshift(a_ref, b_ref), bytecode::BinaryOperator::InplaceRshift => vm._irshift(a_ref, b_ref), bytecode::BinaryOperator::InplaceXor => vm._ixor(a_ref, b_ref), @@ -7489,6 +7512,69 @@ impl ExecutingFrame<'_> { Self::int_fast_op(a, b, vm, i64::checked_mul, |a, b| a * b) } + /// Int divide/remainder i64 fast path. Returns `None` to signal the caller + /// to fall through to the slow path when either operand does not fit i64 + /// or `compute` reports a case it cannot handle (zero divisor or i64 + /// overflow). Result boxing goes through `new_int` so the small-int cache + /// is consulted identically. + #[inline] + fn int_div_fast_op( + a: &BigInt, + b: &BigInt, + vm: &VirtualMachine, + compute: fn(i64, i64) -> Option, + ) -> Option { + use num_traits::ToPrimitive; + let (av, bv) = (a.to_i64()?, b.to_i64()?); + compute(av, bv).map(|r| vm.ctx.new_int(r).into()) + } + + /// Floor division of two i64 values with floor (toward negative infinity) + /// semantics. `None` when `b == 0` or the quotient overflows i64 + /// (`i64::MIN / -1`). + #[inline] + fn floordiv_i64(a: i64, b: i64) -> Option { + if b == 0 { + return None; + } + let q = a.checked_div(b)?; + let r = a % b; + Some(if r != 0 && (r < 0) != (b < 0) { + q - 1 + } else { + q + }) + } + + /// Remainder of two i64 values, taking the sign of the divisor. `None` + /// when `b == 0` or the operation overflows i64 (`i64::MIN % -1`). + #[inline] + fn mod_i64(a: i64, b: i64) -> Option { + if b == 0 { + return None; + } + let r = a.checked_rem(b)?; + Some(if r != 0 && (r < 0) != (b < 0) { + r + b + } else { + r + }) + } + + /// Int floor division with i64 fast path. `None` falls through to the + /// slow path (bigint operands, zero divisor, or i64 overflow). + #[inline] + fn int_floordiv(a: &BigInt, b: &BigInt, vm: &VirtualMachine) -> Option { + Self::int_div_fast_op(a, b, vm, Self::floordiv_i64) + } + + /// Int remainder with i64 fast path. `None` falls through to the slow + /// path (bigint operands, zero divisor, or i64 overflow). + #[inline] + fn int_mod(a: &BigInt, b: &BigInt, vm: &VirtualMachine) -> Option { + Self::int_div_fast_op(a, b, vm, Self::mod_i64) + } + #[cold] fn setup_annotations(&mut self, vm: &VirtualMachine) -> FrameResult { let __annotations__ = identifier!(vm, __annotations__); From 70409fab3309336d81179a64af2a80d8a41c1560 Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Mon, 6 Jul 2026 17:50:23 +0900 Subject: [PATCH 45/92] Skip trashcan and untrack for non-GC-tracked objects in dealloc default_dealloc read is_gc_tracked() only to decide untracking, and entered the trashcan recursion guard unconditionally. Non-GC objects (int, float, str, ...) own no child references that recurse during deallocation, so they need neither the trashcan nor untracking. Read is_gc_tracked() once and gate both trashcan begin/end and untrack on it, removing three thread-local accesses per non-GC object deallocation. Assisted-by: Claude --- crates/vm/src/object/core.rs | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/crates/vm/src/object/core.rs b/crates/vm/src/object/core.rs index d1638881ac3..4e68dc85e56 100644 --- a/crates/vm/src/object/core.rs +++ b/crates/vm/src/object/core.rs @@ -161,8 +161,14 @@ pub(super) unsafe fn default_dealloc(obj: *mut PyObject) { return; // resurrected by __del__ } + // Only GC-tracked objects own child references that can recurse during + // deallocation, so only they need the trashcan recursion guard. Non-GC + // objects (int, float, str, ...) have no reachable children and skip both + // the trashcan and the untrack path. Read once and reuse for both. + let tracked = obj_ref.is_gc_tracked(); + // Trashcan: limit recursive deallocation depth to prevent stack overflow - if !unsafe { trashcan::begin(obj, default_dealloc::) } { + if tracked && !unsafe { trashcan::begin(obj, default_dealloc::) } { return; // deferred to queue } @@ -171,7 +177,7 @@ pub(super) unsafe fn default_dealloc(obj: *mut PyObject) { // Untrack from GC BEFORE deallocation. // Must happen before memory is freed because intrusive list removal // reads the object's gc_pointers (prev/next). - if obj_ref.is_gc_tracked() { + if tracked { let ptr = unsafe { NonNull::new_unchecked(obj) }; unsafe { crate::gc_state::gc_state().untrack_object(ptr); @@ -228,7 +234,9 @@ pub(super) unsafe fn default_dealloc(obj: *mut PyObject) { } // Trashcan: decrement depth and process deferred objects at outermost level - unsafe { trashcan::end() }; + if tracked { + unsafe { trashcan::end() }; + } } pub(super) unsafe fn debug_obj( x: &PyObject, From 1b8683bf4a652ea0e4055e79c0ef9ac830ecbce0 Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Mon, 6 Jul 2026 17:56:35 +0900 Subject: [PATCH 46/92] Merge trashcan depth and defer queue into one thread-local struct trashcan::end accessed two separate thread-locals (DEALLOC_DEPTH and DEALLOC_QUEUE) at the outermost deallocation. Combine both into a single `Trashcan` thread-local holding Cell-based depth and queue fields, so begin and end each reach their state through one thread-local access. The queue is set back before each deferred dealloc call so reentrant begin/end during draining never holds an outstanding borrow. Assisted-by: Claude --- crates/vm/src/object/core.rs | 57 ++++++++++++++++++++++-------------- 1 file changed, 35 insertions(+), 22 deletions(-) diff --git a/crates/vm/src/object/core.rs b/crates/vm/src/object/core.rs index 4e68dc85e56..f2671d50ec7 100644 --- a/crates/vm/src/object/core.rs +++ b/crates/vm/src/object/core.rs @@ -95,9 +95,23 @@ mod trashcan { type DeallocFn = unsafe fn(*mut super::PyObject); type DeallocQueue = Vec<(*mut super::PyObject, DeallocFn)>; + /// Per-thread trashcan state. Depth and deferral queue live in one + /// thread-local so a single access reaches both fields (one `_tlv_get_addr` + /// on platforms where thread-local access is a function call). Both fields + /// are `Cell`-based so reentrant deallocation (nested `begin`/`end` triggered + /// by draining deferred objects) never holds an outstanding borrow. + struct Trashcan { + depth: Cell, + queue: Cell, + } + thread_local! { - static DEALLOC_DEPTH: Cell = const { Cell::new(0) }; - static DEALLOC_QUEUE: Cell = const { Cell::new(Vec::new()) }; + static TRASHCAN: Trashcan = const { + Trashcan { + depth: Cell::new(0), + queue: Cell::new(Vec::new()), + } + }; } /// Try to begin deallocation. Returns true if we should proceed, @@ -107,18 +121,16 @@ mod trashcan { obj: *mut super::PyObject, dealloc: unsafe fn(*mut super::PyObject), ) -> bool { - DEALLOC_DEPTH.with(|d| { - let depth = d.get(); + TRASHCAN.with(|t| { + let depth = t.depth.get(); if depth >= TRASHCAN_LIMIT { // Depth exceeded: defer this deallocation - DEALLOC_QUEUE.with(|q| { - let mut queue = q.take(); - queue.push((obj, dealloc)); - q.set(queue); - }); + let mut queue = t.queue.take(); + queue.push((obj, dealloc)); + t.queue.set(queue); false } else { - d.set(depth + 1); + t.depth.set(depth + 1); true } }) @@ -127,29 +139,30 @@ mod trashcan { /// End deallocation and process any deferred objects if at outermost level. #[inline] pub(super) unsafe fn end() { - let depth = DEALLOC_DEPTH.with(|d| { - let depth = d.get(); + TRASHCAN.with(|t| { + let depth = t.depth.get(); debug_assert!(depth > 0, "trashcan::end called without matching begin"); let depth = depth - 1; - d.set(depth); - depth - }); - if depth == 0 { - // Process deferred deallocations iteratively + t.depth.set(depth); + if depth != 0 { + return; + } + // Process deferred deallocations iteratively. The queue is set back + // before each `dealloc` call so a reentrant `begin` can push freely. loop { - let next = DEALLOC_QUEUE.with(|q| { - let mut queue = q.take(); + let next = { + let mut queue = t.queue.take(); let item = queue.pop(); - q.set(queue); + t.queue.set(queue); item - }); + }; if let Some((obj, dealloc)) = next { unsafe { dealloc(obj) }; } else { break; } } - } + }) } } From f85c30dec3b34eaa83323724f7db353f9d0e6506 Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Mon, 6 Jul 2026 18:27:00 +0900 Subject: [PATCH 47/92] Check object layout compatibility on __bases__ and __class__ assignment Add a shared compatible_for_assignment helper that walks each type to the base that fixes its instance layout and rejects the assignment when the old and new layouts differ (basicsize, itemsize, member count, __dict__, __weakref__, and __slots__ names). Wire it into set_bases, which previously performed no layout check, and replace the inline check in __class__ assignment, which compared the two types directly and over-rejected a subclass that adds no layout. Enable test_descr.test_builtin_bases, which the new set_bases check passes. Assisted-by: Claude --- Lib/test/test_descr.py | 1 - crates/vm/src/builtins/object.rs | 39 +++----------- crates/vm/src/builtins/type.rs | 91 +++++++++++++++++++++++++++++++- 3 files changed, 96 insertions(+), 35 deletions(-) diff --git a/Lib/test/test_descr.py b/Lib/test/test_descr.py index ee85863bcca..0b19496ec4b 100644 --- a/Lib/test/test_descr.py +++ b/Lib/test/test_descr.py @@ -4154,7 +4154,6 @@ class E(D): else: self.fail("shouldn't be able to create inheritance cycles") - @unittest.expectedFailure # TODO: RUSTPYTHON def test_builtin_bases(self): # Make sure all the builtin types can have their base queried without # segfaulting. See issue #5787. diff --git a/crates/vm/src/builtins/object.rs b/crates/vm/src/builtins/object.rs index b1feb467397..b86c3edd423 100644 --- a/crates/vm/src/builtins/object.rs +++ b/crates/vm/src/builtins/object.rs @@ -435,39 +435,12 @@ impl PyBaseObject { && !cls.slots.flags.has_feature(PyTypeFlags::IMMUTABLETYPE); // FIXME(#1979) cls instances might have a payload if both_mutable || both_module { - let has_dict = - |typ: &Py| typ.slots.flags.has_feature(PyTypeFlags::HAS_DICT); - let has_weakref = - |typ: &Py| typ.slots.flags.has_feature(PyTypeFlags::HAS_WEAKREF); - // Compare slots tuples - let slots_equal = match ( - current_cls - .heaptype_ext - .as_ref() - .and_then(|e| e.slots.as_ref()), - cls.heaptype_ext.as_ref().and_then(|e| e.slots.as_ref()), - ) { - (Some(a), Some(b)) => { - a.len() == b.len() - && a.iter() - .zip(b.iter()) - .all(|(x, y)| x.as_wtf8() == y.as_wtf8()) - } - (None, None) => true, - _ => false, - }; - if current_cls.slots.basicsize != cls.slots.basicsize - || !slots_equal - || has_dict(current_cls) != has_dict(&cls) - || has_weakref(current_cls) != has_weakref(&cls) - || current_cls.slots.member_count != cls.slots.member_count - { - return Err(vm.new_type_error(format!( - "__class__ assignment: '{}' object layout differs from '{}'", - cls.name(), - current_cls.name() - ))); - } + super::type_::compatible_for_assignment( + ¤t_cls, + &cls, + "__class__", + vm, + )?; instance.set_class(cls, vm); Ok(()) } else { diff --git a/crates/vm/src/builtins/type.rs b/crates/vm/src/builtins/type.rs index bc9947c2c74..61d49e79ad8 100644 --- a/crates/vm/src/builtins/type.rs +++ b/crates/vm/src/builtins/type.rs @@ -1471,13 +1471,17 @@ impl PyType { } // TODO: check for mro cycles - // TODO: check layout compatibility between the old and new solid base // Compute the new solid base before committing anything. This also // validates the new bases (BASETYPE flag, no instance layout // conflict), the same checks type creation performs. let new_base = best_base(&bases, vm)?.to_owned(); + // Reject reparenting onto a base whose instances have an incompatible + // object layout. + let old_base = zelf.base.deref().unwrap_or(vm.ctx.types.object_type); + compatible_for_assignment(old_base, &new_base, "__bases__", vm)?; + // References released inside the critical section are collected here // and dropped after the lock: dropping them inside can run arbitrary // code that re-acquires the non-reentrant type mutex. @@ -3216,6 +3220,91 @@ fn best_base<'a>(bases: &'a [PyTypeRef], vm: &VirtualMachine) -> PyResult<&'a Py Ok(base.unwrap()) } +fn type_has_dict(typ: &Py) -> bool { + typ.slots.flags.has_feature(PyTypeFlags::HAS_DICT) +} + +fn type_has_weakref(typ: &Py) -> bool { + typ.slots.flags.has_feature(PyTypeFlags::HAS_WEAKREF) +} + +/// Returns true if `child` adds no instance layout of its own beyond its base, +/// so the base can stand in for it when comparing object layouts. +fn compatible_with_base(child: &Py) -> bool { + let Some(parent) = child.base.deref() else { + return false; + }; + child.slots.basicsize == parent.slots.basicsize + && child.slots.itemsize == parent.slots.itemsize + && child.slots.member_count == parent.slots.member_count + && type_has_dict(child) == type_has_dict(parent) + && type_has_weakref(child) == type_has_weakref(parent) +} + +/// Walk up to the most derived base that actually fixes the instance layout. +fn layout_solid_base(mut typ: &Py) -> &Py { + while compatible_with_base(typ) { + typ = typ.base.deref().unwrap(); + } + typ +} + +/// Returns true if `a` and `b`, which share the same base, added the same +/// instance layout (`__dict__`, `__weakref__`, and `__slots__`). +fn same_slots_added(a: &Py, b: &Py) -> bool { + if a.slots.basicsize != b.slots.basicsize + || a.slots.itemsize != b.slots.itemsize + || a.slots.member_count != b.slots.member_count + || type_has_dict(a) != type_has_dict(b) + || type_has_weakref(a) != type_has_weakref(b) + { + return false; + } + match ( + a.heaptype_ext.as_ref().and_then(|e| e.slots.as_ref()), + b.heaptype_ext.as_ref().and_then(|e| e.slots.as_ref()), + ) { + (Some(x), Some(y)) => { + x.len() == y.len() + && x.iter() + .zip(y.iter()) + .all(|(p, q)| p.as_wtf8() == q.as_wtf8()) + } + (None, None) => true, + _ => false, + } +} + +/// Validates that instances of `oldto` and `newto` share an interchangeable +/// object layout, the check `__class__` and `__bases__` assignment perform. +/// +/// `attr` names the attribute being assigned for the error message; the +/// message reports `newto` first and `oldto` second. +pub(crate) fn compatible_for_assignment( + oldto: &Py, + newto: &Py, + attr: &str, + vm: &VirtualMachine, +) -> PyResult<()> { + let newbase = layout_solid_base(newto); + let oldbase = layout_solid_base(oldto); + let bases_equal = match (newbase.base.deref(), oldbase.base.deref()) { + (Some(x), Some(y)) => x.is(y), + (None, None) => true, + _ => false, + }; + let compatible = + newbase.is(oldbase) || (bases_equal && same_slots_added(newbase, oldbase)); + if compatible { + return Ok(()); + } + Err(vm.new_type_error(format!( + "{attr} assignment: '{}' object layout differs from '{}'", + newto.name(), + oldto.name() + ))) +} + /// Apply Python name mangling for private attributes. /// `__x` becomes `_ClassName__x` if inside a class. fn mangle_name(class_name: &str, name: &str) -> String { From f7dda037d700ca5f79540cd0f710fd1ce0363cd5 Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Mon, 6 Jul 2026 19:45:09 +0900 Subject: [PATCH 48/92] Stop the world around GC pointer-reading phases The cycle collector reads each tracked object's interpreter state during reference subtraction, the reachability walk and the strong-reference snapshot, including the localsplus of frames other threads are executing. Those slots are written without synchronization, so the reads are a data race under threading. collect_inner now stops the world before taking the generation read locks and restarts it after the strong-reference snapshot, before the finalizer/weakref/tp_clear phases. A debug assertion checks that no frame on any thread's call stack was classified unreachable. Automatic collections from maybe_collect are deferred to the next bytecode safepoint via a new eval-breaker GC bit instead of running synchronously: a synchronous collection can hold an internal lock (e.g. the lazy frame locals cell) that another thread is blocked on with no way to reach a safepoint, deadlocking the stop. Hardens the stop-the-world machinery for these frequent, concurrent GC stops: attach_thread now honors a pending stop after re-attaching so a thread doing rapid allow_threads calls cannot run past the requester; stop_the_world completion is level-triggered on all-threads-suspended rather than an edge-triggered countdown; and the thread-start started/ready handshakes detach while waiting so the waiter is parkable. Updates the Frame and PyRwLock/PyMutex traversal SAFETY comments to state the actual invariant. Assisted-by: Claude --- crates/vm/src/frame.rs | 74 ++++++++++--------- crates/vm/src/gc_state.rs | 118 ++++++++++++++++++++++++++++++- crates/vm/src/object/traverse.rs | 21 +++--- crates/vm/src/signal.rs | 17 +++++ crates/vm/src/stdlib/_thread.rs | 45 +++++++----- crates/vm/src/vm/mod.rs | 52 ++++++++++++-- crates/vm/src/vm/thread.rs | 8 +++ 7 files changed, 272 insertions(+), 63 deletions(-) diff --git a/crates/vm/src/frame.rs b/crates/vm/src/frame.rs index 0d8714b26fc..62b9fa39f7a 100644 --- a/crates/vm/src/frame.rs +++ b/crates/vm/src/frame.rs @@ -772,8 +772,13 @@ impl PyPayload for Frame { unsafe impl Traverse for Frame { fn traverse(&self, tracer_fn: &mut TraverseFn<'_>) { - // SAFETY: GC traversal does not run concurrently with frame execution. - // A cleared frame (iframe == None) has no children to visit. + // SAFETY: this traversal reads the frame's live interpreter state + // (`localsplus`), which the owning thread mutates without + // synchronization while executing bytecode. It is only sound when no + // other thread is executing this frame: in threading builds the + // collector stops the world around the traversal phases, and in + // single-threaded builds there is no other thread. A cleared frame + // (iframe == None) has no children to visit. let Some(iframe) = (unsafe { &*self.iframe.get() }) else { return; }; @@ -1846,39 +1851,44 @@ impl ExecutingFrame<'_> { } } - if vm.eval_breaker_tripped() - && let Err(exception) = vm.check_signals() - { - #[cold] - fn handle_signal_exception( - frame: &mut ExecutingFrame<'_>, - exception: PyBaseExceptionRef, - idx: usize, - vm: &VirtualMachine, - ) -> FrameResult { - if let Some((loc, _end_loc)) = frame.code.locations.get(idx) { - let next = exception.__traceback__(); - let new_traceback = PyTraceback::new( - next, - frame.object.to_owned(), - idx as u32 * 2, - loc.line, - ); - exception.set_traceback_typed(Some(new_traceback.into_ref(&vm.ctx))); - } - vm.contextualize_exception(&exception); - frame.unwind_blocks(vm, UnwindReason::Raising { exception }) - } - match handle_signal_exception(self, exception, idx, vm) { - Ok(None) => {} - Ok(Some(value)) => { - break Ok(value); + if vm.eval_breaker_tripped() { + if let Err(exception) = vm.check_signals() { + #[cold] + fn handle_signal_exception( + frame: &mut ExecutingFrame<'_>, + exception: PyBaseExceptionRef, + idx: usize, + vm: &VirtualMachine, + ) -> FrameResult { + if let Some((loc, _end_loc)) = frame.code.locations.get(idx) { + let next = exception.__traceback__(); + let new_traceback = PyTraceback::new( + next, + frame.object.to_owned(), + idx as u32 * 2, + loc.line, + ); + exception.set_traceback_typed(Some(new_traceback.into_ref(&vm.ctx))); + } + vm.contextualize_exception(&exception); + frame.unwind_blocks(vm, UnwindReason::Raising { exception }) } - Err(exception) => { - break Err(exception); + match handle_signal_exception(self, exception, idx, vm) { + Ok(None) => {} + Ok(Some(value)) => { + break Ok(value); + } + Err(exception) => { + break Err(exception); + } } + continue; } - continue; + // Run a scheduled automatic collection here — a safepoint with + // no interpreter locks held — instead of synchronously inside + // the allocation that tripped the threshold. + #[cfg(all(unix, feature = "threading"))] + vm.run_scheduled_gc(); } let lasti_before = self.lasti(); let result = self.execute_instruction(op, arg, &mut do_extend_arg, vm); diff --git a/crates/vm/src/gc_state.rs b/crates/vm/src/gc_state.rs index 072712901cb..c823dff366c 100644 --- a/crates/vm/src/gc_state.rs +++ b/crates/vm/src/gc_state.rs @@ -136,6 +136,61 @@ impl GcGeneration { #[derive(Clone, Copy, PartialEq, Eq, Hash)] struct GcPtr(NonNull); +/// RAII barrier that parks every other thread for the pointer-reading phases +/// of a collection and lets them run again before finalizers execute. +/// +/// Reference subtraction, the reachability walk and the strong-reference +/// snapshot dereference the interpreter state of every tracked object, +/// including the `localsplus` of frames that other threads are actively +/// executing. Those writes carry no synchronization, so the reads are only +/// well-defined while all other threads are parked at a safepoint. Restarting +/// happens explicitly once the snapshot has pinned every object; `Drop` is a +/// backstop that also restarts on the early-return paths. +#[cfg(all(unix, feature = "threading"))] +struct CollectStopTheWorld { + vm: *const crate::VirtualMachine, + stopped: bool, +} + +#[cfg(all(unix, feature = "threading"))] +impl CollectStopTheWorld { + /// Request stop-the-world when the current thread has an attached VM. + /// Falls back to no barrier when no VM is attached (the tracked-object + /// reads then run without other threads only if the caller guarantees it). + fn new() -> Self { + let vm = crate::vm::thread::try_with_current_vm(|vm| { + vm.state.stop_the_world.stop_the_world(vm); + vm as *const crate::VirtualMachine + }); + match vm { + Some(vm) => Self { vm, stopped: true }, + None => Self { + vm: core::ptr::null(), + stopped: false, + }, + } + } + + /// Restart the world. Idempotent. + fn restart(&mut self) { + if self.stopped { + // SAFETY: the current thread stays attached to this VM for the + // whole collection — the VM is never popped from the thread's VM + // stack while collecting — so the pointer is valid here. + let vm = unsafe { &*self.vm }; + vm.state.stop_the_world.start_the_world(vm); + self.stopped = false; + } + } +} + +#[cfg(all(unix, feature = "threading"))] +impl Drop for CollectStopTheWorld { + fn drop(&mut self) { + self.restart(); + } +} + /// Global GC state pub struct GcState { /// 3 generations (0 = youngest, 2 = oldest) @@ -366,8 +421,21 @@ impl GcState { let count0 = self.generations[0].count.load(Ordering::SeqCst) as u32; let threshold0 = self.generations[0].threshold(); if threshold0 > 0 && count0 >= threshold0 { - self.collect(0); - return true; + #[cfg(all(unix, feature = "threading"))] + { + // Defer to the next bytecode safepoint. Collecting here would + // stop the world while this thread may hold an internal lock + // (e.g. a lazily-initialized frame locals cell) that another + // thread is blocked on with no way to reach a safepoint — + // a deadlock. At a safepoint no such lock is held. + crate::signal::schedule_gc(); + return false; + } + #[cfg(not(all(unix, feature = "threading")))] + { + self.collect(0); + return true; + } } false @@ -413,6 +481,15 @@ impl GcState { #[cfg(feature = "threading")] crate::object::qsbr::QSBR.process(); + // Stop the world before reading any tracked object's interpreter + // state. Requested *before* the generation read locks are taken: a + // thread parking at a safepoint may still hold a generation write lock + // (track/untrack/promote) and must be able to release it to reach the + // safepoint. It could not do so if this thread already held a read + // lock it was waiting behind — hence the ordering. + #[cfg(all(unix, feature = "threading"))] + let mut stw = CollectStopTheWorld::new(); + // Step 1: Gather objects from generations 0..=generation // Hold read locks for the entire scan to prevent concurrent modifications. let gen_locks: Vec<_> = (0..=generation) @@ -536,6 +613,35 @@ impl GcState { // Step 5: Find unreachable objects let unreachable: Vec = collecting.difference(&reachable).copied().collect(); + // With the world stopped, every frame on any thread's call stack is a + // live root that is externally referenced and must have been + // classified reachable. A running frame appearing in `unreachable` + // would mean the reachability analysis observed its interpreter state + // as garbage — the exact hazard the barrier exists to prevent. + #[cfg(all(unix, feature = "threading", debug_assertions))] + if stw.stopped { + let unreachable_set: HashSet = unreachable.iter().copied().collect(); + crate::vm::thread::try_with_current_vm(|vm| { + let registry = vm.state.thread_frames.lock(); + #[expect( + clippy::iter_over_hash_type, + reason = "assertion over every registered thread slot" + )] + for slot in registry.values() { + for fp in slot.frames.lock().iter() { + // SAFETY: frames on a thread's active call stack are + // alive, and the world is stopped so none can be popped. + let obj = unsafe { fp.as_ref() }.as_object(); + let ptr = GcPtr(NonNull::from(obj)); + debug_assert!( + !unreachable_set.contains(&ptr), + "running frame {obj:p} classified unreachable during GC" + ); + } + } + }); + } + if debug.contains(GcDebugFlags::STATS) { eprintln!( "gc: {} reachable, {} unreachable", @@ -569,6 +675,14 @@ impl GcState { }) .collect(); + // The pointer-reading phases are done: strong references now pin every + // survivor and unreachable object, so the remaining phases can run with + // the world restarted. Finalizers and tp_clear must not run under + // stop-the-world — they execute arbitrary Python — and they only touch + // dead/husk objects, never a running frame. + #[cfg(all(unix, feature = "threading"))] + stw.restart(); + if unreachable.is_empty() { drop(gen_locks); self.promote_survivors(generation, &survivor_refs); diff --git a/crates/vm/src/object/traverse.rs b/crates/vm/src/object/traverse.rs index 9a5ae324baf..2e8927faa05 100644 --- a/crates/vm/src/object/traverse.rs +++ b/crates/vm/src/object/traverse.rs @@ -111,19 +111,22 @@ where unsafe impl Traverse for PyRwLock { #[inline] fn traverse(&self, traverse_fn: &mut TraverseFn<'_>) { - // if can't get a lock, this means something else is holding the lock, - // but since gc stopped the world, during gc the lock is always held - // so it is safe to ignore those in gc + // A failed try_read means another thread holds the write lock. In + // threading builds the collector stops the world around traversal, so + // any writer thread is parked at a safepoint and cannot be holding + // this lock; a failure therefore only reflects the current thread's + // own re-entrant read and is safely skipped. In single-threaded builds + // there is no other thread to contend. if let Some(inner) = self.try_read_recursive() { inner.traverse(traverse_fn) } } } -/// Safety: We can't hold lock during traverse it's child because it may cause deadlock. -/// TODO(discord9): check if this is thread-safe to do -/// (Outside of gc phase, only incref/decref will call trace, -/// and refcnt is atomic, so it should be fine?) +/// Safety: the lock is not held across visiting children to avoid a re-entrant +/// deadlock. In threading builds traversal runs under stop-the-world so no +/// other thread mutates the guarded value while we read it; in single-threaded +/// builds there is no other writer. unsafe impl Traverse for PyMutex { #[inline] fn traverse(&self, traverse_fn: &mut TraverseFn<'_>) { @@ -135,7 +138,9 @@ unsafe impl Traverse for PyMutex { } chs.iter() .map(|ch| { - // Safety: during gc, this should be fine, because nothing should write during gc's tracing? + // Safety: the world is stopped (threading builds) or the + // interpreter is single-threaded, so `ch` is not concurrently + // freed while we hand it to the tracer. let ch = unsafe { ch.as_ref() }; traverse_fn(ch); }) diff --git a/crates/vm/src/signal.rs b/crates/vm/src/signal.rs index 05e655680a5..044a40aa27e 100644 --- a/crates/vm/src/signal.rs +++ b/crates/vm/src/signal.rs @@ -23,6 +23,11 @@ const SIGNAL_BIT: u8 = 1 << 0; /// QSBR has retired allocations pending reclamation. #[cfg(feature = "threading")] const QSBR_BIT: u8 = 1 << 1; +/// An automatic collection was scheduled by `maybe_collect` and must run at +/// the next bytecode safepoint rather than synchronously inside the +/// allocation that tripped the threshold. +#[cfg(all(unix, feature = "threading"))] +const GC_BIT: u8 = 1 << 2; #[expect( clippy::declare_interior_mutable_const, @@ -136,6 +141,18 @@ pub(crate) fn qsbr_bit_set() -> bool { EVAL_BREAKER.load(Ordering::Relaxed) & QSBR_BIT != 0 } +/// Schedule an automatic collection to run at the next bytecode safepoint. +#[cfg(all(unix, feature = "threading"))] +pub(crate) fn schedule_gc() { + EVAL_BREAKER.fetch_or(GC_BIT, Ordering::Release); +} + +/// Clear the scheduled-GC bit, returning whether it had been set. +#[cfg(all(unix, feature = "threading"))] +pub(crate) fn take_gc_scheduled() -> bool { + EVAL_BREAKER.fetch_and(!GC_BIT, Ordering::Acquire) & GC_BIT != 0 +} + /// Reset all signal trigger state after fork in child process. /// Stale triggers from the parent must not fire in the child. #[cfg(all(unix, feature = "host_env"))] diff --git a/crates/vm/src/stdlib/_thread.rs b/crates/vm/src/stdlib/_thread.rs index f3e6bec898f..73c044cbb9f 100644 --- a/crates/vm/src/stdlib/_thread.rs +++ b/crates/vm/src/stdlib/_thread.rs @@ -1659,17 +1659,22 @@ pub(crate) mod _thread { started_cvar.notify_all(); } // Don't execute the target function until parent marks the - // handle as running. + // handle as running. Detach while blocked so a concurrent + // stop-the-world (e.g. a GC on another thread) can park this + // thread instead of stalling waiting for it to reach a + // safepoint it will not reach until released. { let (ready_lock, ready_cvar) = &*handle_ready_event_clone; - let mut ready = ready_lock.lock().unwrap(); - while !*ready { - // Short timeout so we stay responsive to STW requests. - let (guard, _) = ready_cvar - .wait_timeout(ready, core::time::Duration::from_millis(1)) - .unwrap(); - ready = guard; - } + vm.allow_threads(|| { + let mut ready = ready_lock.lock().unwrap(); + while !*ready { + // Short timeout so we stay responsive to STW requests. + let (guard, _) = ready_cvar + .wait_timeout(ready, core::time::Duration::from_millis(1)) + .unwrap(); + ready = guard; + } + }); } // Ensure cleanup happens even if the function panics @@ -1750,16 +1755,22 @@ pub(crate) mod _thread { vm.new_runtime_error("can't start new thread") })?; - // Wait until the new thread has reported its ident. + // Wait until the new thread has reported its ident. Detach while + // waiting so a concurrent stop-the-world (e.g. a GC on another thread) + // can park this thread instead of stalling on it: the child may park + // itself at startup while the world is stopped and cannot report until + // released, so the waiter must be parkable too. { let (started_lock, started_cvar) = &*started_event; - let mut started = started_lock.lock().unwrap(); - while !*started { - let (guard, _) = started_cvar - .wait_timeout(started, core::time::Duration::from_millis(1)) - .unwrap(); - started = guard; - } + vm.allow_threads(|| { + let mut started = started_lock.lock().unwrap(); + while !*started { + let (guard, _) = started_cvar + .wait_timeout(started, core::time::Duration::from_millis(1)) + .unwrap(); + started = guard; + } + }); } // Mark the handle running in the parent thread (like CPython's diff --git a/crates/vm/src/vm/mod.rs b/crates/vm/src/vm/mod.rs index 569185612e0..3e1cc8f893b 100644 --- a/crates/vm/src/vm/mod.rs +++ b/crates/vm/src/vm/mod.rs @@ -349,7 +349,12 @@ impl StopTheWorldState { self.stats_stop_calls.fetch_add(1, Ordering::Relaxed); let initial_countdown = self.init_thread_countdown(vm); stw_trace(format_args!("stop begin requester={requester_ident}")); - if initial_countdown == 0 { + // Park detached threads and set stop bits, then confirm every other + // thread is SUSPENDED. The completion condition is level-triggered + // (`all_non_requester_suspended`) so an already-suspended thread that + // was counted but will not notify again cannot stall the stop. + self.park_detached_threads(vm); + if initial_countdown == 0 || self.all_non_requester_suspended(vm) { self.world_stopped.store(true, Ordering::Release); #[cfg(debug_assertions)] self.debug_assert_all_non_requester_suspended(vm); @@ -361,7 +366,8 @@ impl StopTheWorldState { let mut polls = 0u64; loop { - if self.park_detached_threads(vm) { + self.park_detached_threads(vm); + if self.all_non_requester_suspended(vm) { break; } polls = polls.saturating_add(1); @@ -369,8 +375,7 @@ impl StopTheWorldState { // Re-check under the wait mutex first to avoid a lost-wake race: // a thread may have suspended and notified right before we enter wait. let guard = self.notify_mutex.lock().unwrap(); - if self.thread_countdown.load(Ordering::Acquire) == 0 || self.park_detached_threads(vm) - { + if self.all_non_requester_suspended(vm) { drop(guard); break; } @@ -514,6 +519,33 @@ impl StopTheWorldState { } } + /// Whether every non-requester registered thread is currently SUSPENDED. + /// + /// Level-triggered stop-the-world completion check. Relying on this rather + /// than solely on the edge-triggered `thread_countdown` avoids a + /// lost-decrement race under rapid back-to-back stops: a thread that is + /// already SUSPENDED when a new stop counts it neither notifies nor is + /// force-parked again, so an edge-based countdown could never reach zero. + fn all_non_requester_suspended(&self, vm: &VirtualMachine) -> bool { + use thread::THREAD_SUSPENDED; + let requester = self.requester.load(Ordering::Relaxed); + let registry = vm.state.thread_frames.lock(); + + #[expect( + clippy::iter_over_hash_type, + reason = "Iteration order doesn't matter here" + )] + for (&id, slot) in registry.iter() { + if id == requester { + continue; + } + if slot.state.load(Ordering::Acquire) != THREAD_SUSPENDED { + return false; + } + } + true + } + #[cfg(debug_assertions)] fn debug_assert_all_non_requester_suspended(&self, vm: &VirtualMachine) { use thread::THREAD_SUSPENDED; @@ -2078,6 +2110,18 @@ impl VirtualMachine { Ok(()) } + /// Run an automatic collection scheduled by `maybe_collect`, if any. + /// + /// Called only from the bytecode-loop safepoint, where no interpreter + /// locks are held, so the stop-the-world it performs cannot deadlock + /// against a thread blocked on a lock this thread would otherwise hold. + #[cfg(all(unix, feature = "threading"))] + pub(crate) fn run_scheduled_gc(&self) { + if crate::signal::take_gc_scheduled() { + crate::gc_state::gc_state().collect(0); + } + } + /// Push a new exc_info slot (for generator/coroutine resume). pub(crate) fn push_exception(&self, exc: Option) { self.exceptions.borrow_mut().stack.push(exc); diff --git a/crates/vm/src/vm/thread.rs b/crates/vm/src/vm/thread.rs index 8a8ff17a1dc..7990faf97ab 100644 --- a/crates/vm/src/vm/thread.rs +++ b/crates/vm/src/vm/thread.rs @@ -339,6 +339,14 @@ fn attach_thread(vm: &VirtualMachine) { } } }); + // A stop-the-world may have been requested while this thread was detached. + // Honoring it here (rather than only at the next bytecode safepoint) keeps + // a thread doing rapid allow_threads calls from re-attaching and running + // past the requester forever, which would stall stop-the-world. Done + // outside the CURRENT_THREAD_SLOT borrow above because suspend re-borrows + // it. Safe against a concurrent start_the_world: suspend_if_needed only + // parks while the request is still live and self-recovers otherwise. + suspend_if_needed(&vm.state.stop_the_world); } /// Transition ATTACHED → DETACHED (like `_PyThreadState_Detach`). From bca1ad9674e29e1199bbd1587e34a5cbd805fe6c Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Mon, 6 Jul 2026 19:45:14 +0900 Subject: [PATCH 49/92] Add threaded GC vs executing-frame stress snippet Workers churn frame state (recursion, generators, frame cycles) while a collector loops gc.collect() and an introspector walks live frame objects. Exercises the stop-the-world barrier around GC traversal. Assisted-by: Claude --- .../stdlib_threading_gc_frame_race.py | 100 ++++++++++++++++++ 1 file changed, 100 insertions(+) create mode 100644 extra_tests/snippets/stdlib_threading_gc_frame_race.py diff --git a/extra_tests/snippets/stdlib_threading_gc_frame_race.py b/extra_tests/snippets/stdlib_threading_gc_frame_race.py new file mode 100644 index 00000000000..e6777330a26 --- /dev/null +++ b/extra_tests/snippets/stdlib_threading_gc_frame_race.py @@ -0,0 +1,100 @@ +"""Stress GC traversal against concurrently executing frames. + +The cycle collector reads each tracked object's interpreter state, including +the data stack and fast locals of frames that other threads are actively +executing. Those slots are written without synchronization by the running +thread, so the collector must only read them while the world is stopped. + +Workers churn frame state hard: deep recursion (many nested frames), heavy +local rebind / stack traffic, and generators repeatedly resumed. Meanwhile a +collector thread loops gc.collect() and an introspector walks live frame +objects via gc.get_objects(). A regression (torn read of a running frame) +shows up as a crash, a use-after-free, or a hang. +""" + +import gc +import sys +import threading +import time + +DURATION = 1.5 + + +def deep(n): + # Deep recursion + local rebind churns fast locals and the data stack. + a = n + b = [n, n + 1] + c = {"k": a} + if n <= 0: + return a + len(b) + len(c) + a = a - 1 + b.append(a) + return deep(n - 1) + a + + +def gen_worker(): + def counter(limit): + acc = 0 + i = 0 + while i < limit: + box = {"i": i} + box["self"] = box # a cycle held by the running generator frame + acc += i + yield acc + i += 1 + + g = counter(200) + total = 0 + for v in g: + total += v + return total + + +def make_frame_cycles(n): + for _ in range(n): + def inner(): + fr = sys._getframe() + box = {"fr": fr} + box["self"] = box + return None + + inner() + + +def worker(stop): + # deep() nesting is kept modest so the recursion also fits the smaller + # worker-thread stack of unoptimized (debug) builds; the generators and + # frame cycles supply the rest of the frame churn. + while not stop.is_set(): + deep(12) + gen_worker() + make_frame_cycles(20) + + +def collector(stop): + while not stop.is_set(): + gc.collect() + + +def introspector(stop): + while not stop.is_set(): + for o in gc.get_objects(): + if type(o).__name__ == "frame": + try: + _ = o.f_lineno + _ = o.f_code.co_name + except Exception: + pass + + +stop = threading.Event() +threads = [threading.Thread(target=worker, args=(stop,)) for _ in range(4)] +threads.append(threading.Thread(target=collector, args=(stop,))) +threads.append(threading.Thread(target=introspector, args=(stop,))) +for t in threads: + t.start() +time.sleep(DURATION) +stop.set() +for t in threads: + t.join() +print("ok") From 2769945e3ca9d9f85d67e22e454c3090cfd44307 Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Mon, 6 Jul 2026 20:34:36 +0900 Subject: [PATCH 50/92] Serialize fork and GC stop-the-world requesters fork() (posix before/after-fork) and the cycle collector both drive the single StopTheWorldState. With no mutual exclusion their requester word and suspension countdown could interleave and be clobbered, so the completion check never converged and a requester waited on itself forever (reproducible parent-side hang when forking with GC enabled). Add one exclusion held for the whole stop->start span of either requester: stop_the_world acquires it before any stop bookkeeping and start_the_world/reset_after_fork release it. The acquire is park-friendly (poll try-lock and honor a pending suspend between tries) so a requester blocked behind an active stop can still be force-parked instead of deadlocking the active requester. The acquirer holds no other lock while spinning. Also correct the PyRwLock traverse SAFETY note: a failed try_read may mean a force-parked thread holds the write lock; skipping is safe because under-traversal only over-approximates liveness. Document the residual gc.collect()-under-a-non-generation-lock exposure at the collect_inner barrier. Assisted-by: Claude --- crates/vm/src/gc_state.rs | 13 +++++++ crates/vm/src/object/traverse.rs | 15 ++++---- crates/vm/src/vm/mod.rs | 60 ++++++++++++++++++++++++++++++++ 3 files changed, 82 insertions(+), 6 deletions(-) diff --git a/crates/vm/src/gc_state.rs b/crates/vm/src/gc_state.rs index c823dff366c..6b71ba7c7dc 100644 --- a/crates/vm/src/gc_state.rs +++ b/crates/vm/src/gc_state.rs @@ -487,6 +487,19 @@ impl GcState { // (track/untrack/promote) and must be able to release it to reach the // safepoint. It could not do so if this thread already held a read // lock it was waiting behind — hence the ordering. + // + // Auto-collection is deferred to a bytecode safepoint (see + // `maybe_collect`), where no internal lock is held, so it never stops + // the world under a lock. Explicit `gc.collect()` runs synchronously + // here; a re-entrant call from a finalizer during an in-progress + // collection is turned into a no-op by the `collecting` try_lock above. + // The one residual is an explicit `gc.collect()` reached from a + // finalizer/`__del__` that runs inline while a non-generation internal + // lock is still held (e.g. a container write lock during element + // replacement) with another thread blocked on that same lock: stopping + // the world then waits for a thread that cannot reach a safepoint. + // Closing it fully would require making those locks stop-the-world + // aware; the exclusion above only serializes the fork/GC requesters. #[cfg(all(unix, feature = "threading"))] let mut stw = CollectStopTheWorld::new(); diff --git a/crates/vm/src/object/traverse.rs b/crates/vm/src/object/traverse.rs index 2e8927faa05..d0a20d2afa7 100644 --- a/crates/vm/src/object/traverse.rs +++ b/crates/vm/src/object/traverse.rs @@ -111,12 +111,15 @@ where unsafe impl Traverse for PyRwLock { #[inline] fn traverse(&self, traverse_fn: &mut TraverseFn<'_>) { - // A failed try_read means another thread holds the write lock. In - // threading builds the collector stops the world around traversal, so - // any writer thread is parked at a safepoint and cannot be holding - // this lock; a failure therefore only reflects the current thread's - // own re-entrant read and is safely skipped. In single-threaded builds - // there is no other thread to contend. + // A failed try_read means a writer holds the lock. Traversal runs with + // the world stopped, but a thread force-parked while DETACHED (CAS'd + // straight to SUSPENDED from native code) may still hold the write lock + // it was in the middle of taking. Skipping such an object is safe: the + // collector then does not see its outgoing edges, which only + // under-traverses and thus over-approximates liveness (a conservative + // keep-alive), never freeing a reachable object. In single-threaded + // builds a failure only reflects the current thread's own re-entrant + // read, likewise safely skipped. if let Some(inner) = self.try_read_recursive() { inner.traverse(traverse_fn) } diff --git a/crates/vm/src/vm/mod.rs b/crates/vm/src/vm/mod.rs index 3e1cc8f893b..c79f26a2669 100644 --- a/crates/vm/src/vm/mod.rs +++ b/crates/vm/src/vm/mod.rs @@ -151,6 +151,11 @@ pub struct StopTheWorldState { world_stopped: AtomicBool, /// Ident of the thread that requested the stop (like `stw->requester`) requester: AtomicU64, + /// Single exclusion held for the whole stop→start span. Fork and GC are + /// both stop-the-world requesters driving this shared state; only one may + /// hold it at a time. Acquired before any stop bookkeeping (see + /// `acquire_exclusion`) and released by `start_the_world`/`reset_after_fork`. + exclusion: AtomicBool, /// Signaled by suspending threads when their state transitions to SUSPENDED notify_mutex: std::sync::Mutex<()>, notify_cv: std::sync::Condvar, @@ -209,6 +214,7 @@ impl StopTheWorldState { requested: AtomicBool::new(false), world_stopped: AtomicBool::new(false), requester: AtomicU64::new(0), + exclusion: AtomicBool::new(false), notify_mutex: std::sync::Mutex::new(()), notify_cv: std::sync::Condvar::new(), thread_countdown: AtomicI64::new(0), @@ -336,13 +342,61 @@ impl StopTheWorldState { forced_parks != 0 && self.thread_countdown.load(Ordering::Acquire) == 0 } + /// Acquire the single stop-the-world exclusion in a park-friendly way. + /// + /// Fork and GC both request stop-the-world through the same shared state; + /// without this exclusion their `requester`/`requested`/countdown words + /// could be clobbered by an interleaving requester, so the completion + /// check could never converge and a requester would wait on itself forever. + /// + /// The acquire must be park-friendly. While another requester's stop is in + /// progress it sets this thread's stop bit and waits for it to suspend; + /// blocking on a plain lock here would keep this thread from ever reaching + /// that safepoint, so the active requester would wait for this thread while + /// this thread waits for the lock — a deadlock swap. Instead we poll and + /// honor the suspend request between tries. This thread holds no other lock + /// while spinning (a requester always enters from a clean call site), so + /// suspending here is safe: the active requester force-parks this thread, + /// finishes its whole stop→start span, releases the exclusion, and only + /// then does this thread resume and acquire it. + fn acquire_exclusion(&self) { + if self + .exclusion + .compare_exchange(false, true, Ordering::AcqRel, Ordering::Relaxed) + .is_ok() + { + return; + } + loop { + crate::vm::thread::suspend_if_needed(self); + std::thread::yield_now(); + if self + .exclusion + .compare_exchange(false, true, Ordering::AcqRel, Ordering::Relaxed) + .is_ok() + { + return; + } + } + } + + /// Release the stop-the-world exclusion taken by `acquire_exclusion`. + fn release_exclusion(&self) { + self.exclusion.store(false, Ordering::Release); + } + /// Stop all non-requester threads (`stop_the_world`). /// /// 1. Sets `requested`, marking the requester thread. /// 2. CAS detached threads to SUSPENDED. /// 3. Waits (polling with 1 ms condvar timeout) for attached threads /// to self-suspend in `check_signals`. + /// + /// Takes the shared exclusion first so at most one requester (fork or GC) + /// drives the stop→start span at a time; it is released by + /// `start_the_world`/`reset_after_fork`. pub fn stop_the_world(&self, vm: &VirtualMachine) { + self.acquire_exclusion(); let start = std::time::Instant::now(); let requester_ident = crate::stdlib::_thread::get_ident(); self.requester.store(requester_ident, Ordering::Relaxed); @@ -450,6 +504,9 @@ impl StopTheWorldState { self.requester.store(0, Ordering::Relaxed); #[cfg(debug_assertions)] self.debug_assert_all_non_requester_detached(vm); + // Release the exclusion last, ending the stop→start span so the next + // requester (fork or GC) can proceed. + self.release_exclusion(); stw_trace(format_args!("start end requester={requester}")); } @@ -459,6 +516,9 @@ impl StopTheWorldState { self.world_stopped.store(false, Ordering::Relaxed); self.requester.store(0, Ordering::Relaxed); self.thread_countdown.store(0, Ordering::Relaxed); + // The surviving child thread inherited the exclusion taken by the + // pre-fork `stop_the_world`; release it (no start_the_world runs here). + self.release_exclusion(); stw_trace(format_args!("reset-after-fork")); } From a33d16f955897d35f0a8e0a330cf204c84df25b4 Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Mon, 6 Jul 2026 20:34:45 +0900 Subject: [PATCH 51/92] Add fork under concurrent GC stop-the-world snippet Worker threads allocate cyclic garbage with GC enabled while the main thread forks repeatedly and each child collects. Exercises the fork/GC stop-the-world exclusion; the allocation rate is throttled so a collection stays cheap in unoptimized builds. Assisted-by: Claude --- .../snippets/stdlib_threading_gc_fork.py | 62 +++++++++++++++++++ 1 file changed, 62 insertions(+) create mode 100644 extra_tests/snippets/stdlib_threading_gc_fork.py diff --git a/extra_tests/snippets/stdlib_threading_gc_fork.py b/extra_tests/snippets/stdlib_threading_gc_fork.py new file mode 100644 index 00000000000..cd00cf00983 --- /dev/null +++ b/extra_tests/snippets/stdlib_threading_gc_fork.py @@ -0,0 +1,62 @@ +"""Fork while other threads drive concurrent GC stop-the-world. + +fork() and the cycle collector both stop the world through the same shared +state. Without a single exclusion around each stop->start span, an interleaving +of the fork requester and a GC requester clobbers that state (requester word, +suspension countdown) so the completion check never converges and a requester +waits on itself forever. + +Worker threads allocate cyclic garbage with GC enabled while the main thread +forks repeatedly; each child collects and exits. A regression shows up as a +hang in the parent (never finishing the fork loop). The allocation rate is kept +light so the collection stays cheap even in unoptimized builds. +""" + +import gc +import os +import threading +import time + +if not hasattr(os, "fork"): + print("skipped (no fork)") + raise SystemExit(0) + +gc.enable() +stop = threading.Event() + + +def churn(): + while not stop.is_set(): + a = {} + b = {"a": a} + a["b"] = b # cycle collectable only by the cycle collector + lst = [a, b] + lst.append(lst) + del a, b, lst + # Throttle so the collector keeps the heap small; the point is to + # interleave fork with concurrent collections, not to grow the heap. + time.sleep(0.001) + + +workers = [threading.Thread(target=churn) for _ in range(4)] +for w in workers: + w.start() + +# Let the workers get going before forking. +time.sleep(0.05) + +N = 25 +for _ in range(N): + pid = os.fork() + if pid == 0: + # Child: run its own stop-the-world collection, then exit. + gc.collect() + os._exit(0) + _, status = os.waitpid(pid, 0) + assert status == 0, status + +stop.set() +for w in workers: + w.join() + +print("ok") From 1b0fa99bf4a4426d9f50f1cfddfdf60dc85aa5a8 Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Mon, 6 Jul 2026 21:12:12 +0900 Subject: [PATCH 52/92] Detach while acquiring the import lock The global import lock is held across bytecode by the importlib bootstrap, so its holder can be parked at a safepoint mid-hold. Acquiring it while attached let another thread block attached on the lock, so a stop-the-world requester could wait forever for that attached thread to suspend while the holder stayed parked. Wrap IMP_LOCK acquisition in allow_threads in both _imp.acquire_lock and the pre-fork acquire_imp_lock_for_fork so the wait honors stop-the-world requests. Correct the acquire_exclusion comment: a spinning requester may hold IMP_LOCK (fork) or the collecting mutex (GC); safety relies on those never being acquired attached-blocking by another thread. Assisted-by: Claude --- crates/vm/src/stdlib/_imp.rs | 19 +++++++++++++++---- crates/vm/src/stdlib/posix.rs | 2 +- crates/vm/src/vm/mod.rs | 13 ++++++++----- 3 files changed, 24 insertions(+), 10 deletions(-) diff --git a/crates/vm/src/stdlib/_imp.rs b/crates/vm/src/stdlib/_imp.rs index 7021895c9f7..322eaedd7d0 100644 --- a/crates/vm/src/stdlib/_imp.rs +++ b/crates/vm/src/stdlib/_imp.rs @@ -15,8 +15,14 @@ mod lock { static IMP_LOCK: RawRMutex = RawRMutex::INIT; #[pyfunction] - fn acquire_lock(_vm: &VirtualMachine) { - acquire_lock_for_fork() + fn acquire_lock(vm: &VirtualMachine) { + // Detach while blocking on IMP_LOCK. The import lock is held across + // bytecode by the importlib bootstrap, so its holder can be parked at a + // safepoint mid-hold. Blocking here while attached would keep this + // thread from honoring a stop-the-world request, so a requester could + // wait for this thread while this thread waits for the parked holder. + // Detaching makes the wait park-friendly. + vm.allow_threads(acquire_lock_for_fork); } #[pyfunction] @@ -76,9 +82,14 @@ mod lock { } /// Re-export for fork safety code in posix.rs +/// +/// Runs pre-fork on a normal attached VM thread. Detach while blocking so the +/// wait honors a concurrent stop-the-world request instead of pinning this +/// thread attached on IMP_LOCK; re-attach completes before `stop_the_world`, so +/// the fork requester protocol is unaffected. #[cfg(all(unix, feature = "threading", feature = "host_env"))] -pub(crate) fn acquire_imp_lock_for_fork() { - lock::acquire_lock_for_fork(); +pub(crate) fn acquire_imp_lock_for_fork(vm: &VirtualMachine) { + vm.allow_threads(lock::acquire_lock_for_fork); } #[cfg(all(unix, feature = "threading", feature = "host_env"))] diff --git a/crates/vm/src/stdlib/posix.rs b/crates/vm/src/stdlib/posix.rs index e9b44a3786c..80245aed08f 100644 --- a/crates/vm/src/stdlib/posix.rs +++ b/crates/vm/src/stdlib/posix.rs @@ -623,7 +623,7 @@ pub mod module { run_at_forkers(before_forkers, true, vm); #[cfg(feature = "threading")] - crate::stdlib::_imp::acquire_imp_lock_for_fork(); + crate::stdlib::_imp::acquire_imp_lock_for_fork(vm); #[cfg(feature = "threading")] vm.state.stop_the_world.stop_the_world(vm); diff --git a/crates/vm/src/vm/mod.rs b/crates/vm/src/vm/mod.rs index c79f26a2669..37fd370d71b 100644 --- a/crates/vm/src/vm/mod.rs +++ b/crates/vm/src/vm/mod.rs @@ -354,11 +354,14 @@ impl StopTheWorldState { /// blocking on a plain lock here would keep this thread from ever reaching /// that safepoint, so the active requester would wait for this thread while /// this thread waits for the lock — a deadlock swap. Instead we poll and - /// honor the suspend request between tries. This thread holds no other lock - /// while spinning (a requester always enters from a clean call site), so - /// suspending here is safe: the active requester force-parks this thread, - /// finishes its whole stop→start span, releases the exclusion, and only - /// then does this thread resume and acquire it. + /// honor the suspend request between tries. Suspending here is safe as long + /// as any lock a spinning requester still holds is never acquired + /// attached-blocking by another thread. The fork requester holds IMP_LOCK, + /// but its acquisition detaches (`allow_threads`), so no attached thread + /// blocks on it; the GC requester holds only the `collecting` mutex, which + /// is only ever `try_lock`'d. The active requester therefore force-parks + /// this thread, finishes its whole stop→start span, releases the exclusion, + /// and only then does this thread resume and acquire it. fn acquire_exclusion(&self) { if self .exclusion From 92a4d0001512a147d3e057ddce30c571c7e8c750 Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Mon, 6 Jul 2026 21:12:15 +0900 Subject: [PATCH 53/92] Add concurrent-import vs GC deadlock snippet Two threads re-import modules (contending the import lock) while a third storms the cycle collector and a fourth allocates cyclic garbage. Regressed as a hang before the import lock acquisition was made park-friendly. Assisted-by: Claude --- .../snippets/stdlib_threading_gc_import.py | 58 +++++++++++++++++++ 1 file changed, 58 insertions(+) create mode 100644 extra_tests/snippets/stdlib_threading_gc_import.py diff --git a/extra_tests/snippets/stdlib_threading_gc_import.py b/extra_tests/snippets/stdlib_threading_gc_import.py new file mode 100644 index 00000000000..061ff04a799 --- /dev/null +++ b/extra_tests/snippets/stdlib_threading_gc_import.py @@ -0,0 +1,58 @@ +"""Concurrent imports plus GC stop-the-world must not deadlock (no fork). + +The global import lock is held across bytecode by the importlib bootstrap, so +its holder can be parked at a safepoint mid-hold. If another thread blocks on +that lock while attached, a GC stop-the-world requester waits forever for that +attached thread to suspend while the lock holder stays parked -- a three-party +deadlock. Acquiring the import lock must therefore detach so the wait honors a +stop-the-world request. + +Two threads repeatedly re-import modules (contending the import lock) while a +third storms the cycle collector and a fourth allocates cyclic garbage. A +regression shows up as a hang (the importer threads never finishing). +""" + +import gc +import sys +import threading +import importlib + +gc.enable() +stop = threading.Event() + +# Modules cheap to import and safe to drop/re-import repeatedly. +MODS = ("colorsys", "stringprep") +ITERS = 2000 + + +def importer(mod): + for _ in range(ITERS): + if stop.is_set(): + break + sys.modules.pop(mod, None) + importlib.import_module(mod) + + +def collector(): + while not stop.is_set(): + gc.collect() + + +def allocator(): + while not stop.is_set(): + y = [{"i": i} for i in range(50)] + y[0]["self"] = y # cycle collectable only by the cycle collector + + +importers = [threading.Thread(target=importer, args=(m,)) for m in MODS] +helpers = [threading.Thread(target=collector), threading.Thread(target=allocator)] + +for t in importers + helpers: + t.start() +for t in importers: + t.join() +stop.set() +for t in helpers: + t.join() + +print("ok") From d1e5e97531cdf53264b61a27857e3c029fc51f55 Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Mon, 6 Jul 2026 21:26:10 +0900 Subject: [PATCH 54/92] Create call frames untracked and track generators explicitly Add PyPayload::NEW_REF_UNTRACKED (default false, true for Frame) so PyRef::new_ref skips auto-tracking ordinary call frames in the GC. Generator/coroutine/async-generator frames are tracked explicitly in invoke_with_locals before their generator back-reference is installed. run_frame debug-asserts that a datastack frame is untracked on entry. Assisted-by: Claude --- crates/vm/src/builtins/function.rs | 40 ++++++++++++++++++------------ crates/vm/src/frame.rs | 4 +++ crates/vm/src/object/core.rs | 6 ++++- crates/vm/src/object/payload.rs | 7 ++++++ crates/vm/src/vm/mod.rs | 9 +++++++ 5 files changed, 49 insertions(+), 17 deletions(-) diff --git a/crates/vm/src/builtins/function.rs b/crates/vm/src/builtins/function.rs index 9b20852a977..bbfca0c93d2 100644 --- a/crates/vm/src/builtins/function.rs +++ b/crates/vm/src/builtins/function.rs @@ -580,26 +580,34 @@ impl Py { .into_ref(&vm.ctx); self.fill_locals_from_args(&frame, func_args, vm)?; - if is_async_gen { - let obj = PyAsyncGen::new(frame.clone(), self.__name__(), self.__qualname__()) - .into_pyobject(vm); - frame.set_generator(&obj); - Ok(obj) - } else if is_gen { - let obj = PyGenerator::new(frame.clone(), self.__name__(), self.__qualname__()) - .into_pyobject(vm); - frame.set_generator(&obj); - Ok(obj) - } else if is_coro { - let obj = PyCoroutine::new(frame.clone(), self.__name__(), self.__qualname__()) - .into_pyobject(vm); - frame.set_generator(&obj); - Ok(obj) - } else { + if use_datastack { let result = vm.run_frame(frame.clone()); // Release data stack memory after frame execution completes. crate::frame::release_datastack_frame(&frame, vm); result + } else { + let obj = if is_async_gen { + PyAsyncGen::new(frame.clone(), self.__name__(), self.__qualname__()) + .into_pyobject(vm) + } else if is_gen { + PyGenerator::new(frame.clone(), self.__name__(), self.__qualname__()) + .into_pyobject(vm) + } else { + PyCoroutine::new(frame.clone(), self.__name__(), self.__qualname__()) + .into_pyobject(vm) + }; + // Generator/coroutine frames outlive this call and can join a + // reference cycle through their owning generator, so they must + // participate in the GC. They were created untracked + // (NEW_REF_UNTRACKED); track them now, before the back-reference + // is installed. + // SAFETY: the frame is alive (held by `frame`) and untracked. + unsafe { + crate::gc_state::gc_state() + .track_object(core::ptr::NonNull::from(frame.as_object())); + } + frame.set_generator(&obj); + Ok(obj) } } diff --git a/crates/vm/src/frame.rs b/crates/vm/src/frame.rs index 62b9fa39f7a..f7f171e5a03 100644 --- a/crates/vm/src/frame.rs +++ b/crates/vm/src/frame.rs @@ -733,6 +733,10 @@ thread_local! { impl PyPayload for Frame { const MAX_FREELIST: usize = 200; const HAS_FREELIST: bool = true; + // Ordinary call frames are created untracked and only enter the GC when + // they escape (see `release_datastack_frame`); generator/coroutine frames + // are tracked explicitly at creation in `invoke_with_locals`. + const NEW_REF_UNTRACKED: bool = true; #[inline] fn class(ctx: &Context) -> &'static Py { diff --git a/crates/vm/src/object/core.rs b/crates/vm/src/object/core.rs index f2671d50ec7..11419bec237 100644 --- a/crates/vm/src/object/core.rs +++ b/crates/vm/src/object/core.rs @@ -2320,7 +2320,11 @@ impl PyRef { // - HAS_TRAVERSE is true (Rust payload implements Traverse), OR // - has instance dict (user-defined class instances), OR // - heap type (all heap type instances are GC-tracked, like Py_TPFLAGS_HAVE_GC) - if ::HAS_TRAVERSE || has_dict || is_heaptype { + // unless the payload opts out via NEW_REF_UNTRACKED (e.g. call frames, + // which are tracked lazily only on escape). + if (::HAS_TRAVERSE || has_dict || is_heaptype) + && !T::NEW_REF_UNTRACKED + { let gc = crate::gc_state::gc_state(); unsafe { gc.track_object(ptr.cast()); diff --git a/crates/vm/src/object/payload.rs b/crates/vm/src/object/payload.rs index 772feaa4f4c..7925f656de5 100644 --- a/crates/vm/src/object/payload.rs +++ b/crates/vm/src/object/payload.rs @@ -48,6 +48,13 @@ pub trait PyPayload: MaybeTraverse + PyThreadingConstraint + Sized + 'static { fn class(ctx: &Context) -> &'static Py; + /// Whether `PyRef::new_ref` skips auto-tracking this type in the GC even + /// when it would otherwise qualify (has traverse, dict, or heap type). + /// Such objects are created untracked and must be tracked explicitly if + /// and when they can become part of a reference cycle. Used by `Frame`, + /// which is created untracked and tracked lazily only on escape. + const NEW_REF_UNTRACKED: bool = false; + /// Whether this type has a freelist. Types with freelists require /// immediate (non-deferred) GC untracking during dealloc to prevent /// race conditions when the object is reused. diff --git a/crates/vm/src/vm/mod.rs b/crates/vm/src/vm/mod.rs index 37fd370d71b..f22d08d2809 100644 --- a/crates/vm/src/vm/mod.rs +++ b/crates/vm/src/vm/mod.rs @@ -1316,6 +1316,15 @@ impl VirtualMachine { #[inline(always)] pub fn run_frame(&self, frame: FrameRef) -> PyResult { + // Only ordinary (datastack) call frames reach `run_frame`; generator + // and coroutine frames are resumed through `resume_gen_frame`. A + // datastack frame is created untracked and is tracked lazily only when + // it escapes, which happens no earlier than `release_datastack_frame` + // after this call returns. So it must be untracked on entry. + debug_assert!( + !frame.as_object().is_gc_tracked(), + "datastack frame is GC-tracked before execution" + ); match self.with_frame(frame, |f| f.run(self))? { ExecutionResult::Return(value) => Ok(value), _ => panic!("Got unexpected result from function"), From ee948401f1cbe85c5b2a14557bd13bdd6299c1f0 Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Mon, 6 Jul 2026 21:28:38 +0900 Subject: [PATCH 55/92] Track escaped call frames lazily at datastack release Rewrite release_datastack_frame around the invariant that a datastack frame is never GC-tracked while it runs: strong_count() == 1 means the frame never escaped, so drop localsplus in place without touching the GC; strong_count() > 1 means it escaped, so materialize localsplus onto the heap and then track the frame. This removes the untrack and the untrack-recheck dance from the common non-escaping path. Assisted-by: Claude --- crates/vm/src/frame.rs | 63 +++++++++++++++++++++--------------------- 1 file changed, 32 insertions(+), 31 deletions(-) diff --git a/crates/vm/src/frame.rs b/crates/vm/src/frame.rs index f7f171e5a03..23dd2c8ed88 100644 --- a/crates/vm/src/frame.rs +++ b/crates/vm/src/frame.rs @@ -1424,40 +1424,38 @@ pub(crate) fn release_datastack_frame(frame: &Py, vm: &VirtualMachine) { // Uniqueness argument: at this point the frame is already out of // `vm.frames`, the thread-frames registry and the current-frame chain // (all unlinked inside `with_frame_impl` before it returned), and the - // frame type has no weakref support, so with one exception no thread can - // mint a new reference without already holding one. The exception is the - // GC generation lists (`gc.get_objects`, `gc.get_referrers`, collection - // survivor refs), whose readers incref under the generation-list read - // lock. Untracking the frame takes the same list's write lock, so after - // `untrack_object` returns, no new list-based reference can appear and - // any previously minted one is visible to the count re-check below. - // This also keeps `__del__` side effects during `release_localsplus` - // (which can run arbitrary code, including `gc.get_objects`) from - // reaching the frame. - if frame_obj.strong_count() == 1 && frame_obj.is_gc_tracked() { - let ptr = NonNull::from(frame_obj); - // SAFETY: the frame is alive and currently tracked. - unsafe { crate::gc_state::gc_state().untrack_object(ptr) }; - if frame_obj.strong_count() == 1 { - // A reference minted and already released by another thread ends - // in a release-decref; the fence orders that thread's reads of - // localsplus before our drops below. - atomic::fence(Acquire); - // SAFETY: unique owner and no way to mint a new reference, so - // localsplus can never be observed again. The base pointer came - // from this thread's data stack. - unsafe { - if let Some(base) = frame.release_localsplus() { - vm.datastack_pop(base); - } + // frame type has no weakref support. A datastack frame is created + // untracked and stays untracked while it runs, so it is in no GC + // generation list and no collector can observe or incref it. Hence no + // thread can mint a new reference without already holding one, and every + // escape (traceback, `sys._getframe`, `f_back`, a stored trace-hook arg) + // is a heap reference created on this thread while the frame ran. + // Therefore `strong_count() == 1` here means nothing escaped, and + // `strong_count() > 1` means the frame escaped. + debug_assert!( + !frame_obj.is_gc_tracked(), + "datastack frame is GC-tracked at release" + ); + if frame_obj.strong_count() == 1 { + // A reference minted and already released by another thread (through a + // heap escape carried across threads) ends in a release-decref; the + // fence orders that thread's memory before our drops below. + atomic::fence(Acquire); + // SAFETY: unique owner and no way to mint a new reference, so + // localsplus can never be observed again. The base pointer came + // from this thread's data stack. + unsafe { + if let Some(base) = frame.release_localsplus() { + vm.datastack_pop(base); } - return; } - // Lost the race: a reference was minted through the GC list before - // the untrack. Re-track and fall back to the copy path. - // SAFETY: the frame is alive and untracked. - unsafe { crate::gc_state::gc_state().track_object(ptr) }; + return; } + // Escaped. Stabilise localsplus on the heap FIRST, then join the GC. This + // order guarantees a concurrent (stop-the-world) collector only ever sees a + // tracked frame whose localsplus is heap-resident and no longer mutating: + // the frame has stopped executing before it becomes a candidate, so its + // outgoing edges are stable while a collector traverses them. // SAFETY: the frame finished executing; the base pointer came from this // thread's data stack. unsafe { @@ -1465,6 +1463,9 @@ pub(crate) fn release_datastack_frame(frame: &Py, vm: &VirtualMachine) { vm.datastack_pop(base); } } + // SAFETY: the frame is alive (held by `frame` and the escaped reference) + // and untracked. + unsafe { crate::gc_state::gc_state().track_object(NonNull::from(frame_obj)) }; } type BinaryOpExtendGuard = fn(&PyObject, &PyObject, &VirtualMachine) -> bool; From 2982b66397b00454b7f1d4f12769e77822501727 Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Mon, 6 Jul 2026 21:31:31 +0900 Subject: [PATCH 56/92] Guard the tracked-frame localsplus invariant Document at Frame::traverse that references to a frame are always recorded as graph edges, so the collector reads a frame's localsplus only when the frame is itself a tracked candidate. Add debug assertions at both frame track sites (escaped datastack frames and generator frames) that a frame is heap-backed before it is tracked, so a collector never reads data-stack-resident, still-mutating storage. Assisted-by: Claude --- crates/vm/src/builtins/function.rs | 8 ++++++- crates/vm/src/frame.rs | 38 ++++++++++++++++++++++++++++++ 2 files changed, 45 insertions(+), 1 deletion(-) diff --git a/crates/vm/src/builtins/function.rs b/crates/vm/src/builtins/function.rs index bbfca0c93d2..c522815e2aa 100644 --- a/crates/vm/src/builtins/function.rs +++ b/crates/vm/src/builtins/function.rs @@ -600,7 +600,13 @@ impl Py { // reference cycle through their owning generator, so they must // participate in the GC. They were created untracked // (NEW_REF_UNTRACKED); track them now, before the back-reference - // is installed. + // is installed. Their localsplus is heap-backed by construction + // (use_datastack == false), so a collector never reads data-stack + // storage when it traverses them. + debug_assert!( + !frame.localsplus_is_datastack_backed(), + "generator frame is data-stack-backed" + ); // SAFETY: the frame is alive (held by `frame`) and untracked. unsafe { crate::gc_state::gc_state() diff --git a/crates/vm/src/frame.rs b/crates/vm/src/frame.rs index 23dd2c8ed88..c0d8eae3f3b 100644 --- a/crates/vm/src/frame.rs +++ b/crates/vm/src/frame.rs @@ -282,6 +282,12 @@ impl LocalsPlus { } } + /// Whether the backing storage still lives on the thread data stack (a + /// running call frame that has not been materialized onto the heap). + fn is_datastack_backed(&self) -> bool { + matches!(self.data, LocalsPlusData::DataStack { .. }) + } + /// Stack capacity (max stack depth). #[inline(always)] fn stack_capacity(&self) -> usize { @@ -783,6 +789,19 @@ unsafe impl Traverse for Frame { // collector stops the world around the traversal phases, and in // single-threaded builds there is no other thread. A cleared frame // (iframe == None) has no children to visit. + // + // Invariant (load-bearing for the untracked-frame optimization): every + // reference *to* a frame is recorded as a graph edge by + // `PyRef::traverse` — no other type's `traverse` recurses into a + // frame's payload. So the collector reads a frame's `localsplus` only + // when the frame is itself a tracked candidate. Tracked datastack + // frames are tracked only at `release_datastack_frame`, after they stop + // executing and their localsplus is materialized onto the heap; tracked + // generator frames are heap-backed by construction and their execution + // is stopped-the-world. Hence a running, data-stack-resident frame is + // never traversed by a concurrent collector. If a future change makes + // some type's `traverse` recurse into a frame payload, this invariant + // (and the debug asserts at the track sites) breaks. let Some(iframe) = (unsafe { &*self.iframe.get() }) else { return; }; @@ -939,6 +958,15 @@ impl Frame { unsafe { self.iframe_mut().localsplus.release_datastack() } } + /// Whether this frame's localsplus is still data-stack-backed. A frame + /// must have heap-backed localsplus before it is GC-tracked so that a + /// concurrent collector never reads data-stack-resident, still-mutating + /// storage. Used only in debug assertions at the track sites. + pub(crate) fn localsplus_is_datastack_backed(&self) -> bool { + // SAFETY: called at a track site where the frame is not executing. + unsafe { self.iframe_ref().localsplus.is_datastack_backed() } + } + /// Clear evaluation stack and state-owned cell/free references. /// For full local/cell cleanup, call `clear_locals_and_stack()`. pub(crate) fn clear_stack_and_cells(&self) { @@ -1463,6 +1491,16 @@ pub(crate) fn release_datastack_frame(frame: &Py, vm: &VirtualMachine) { vm.datastack_pop(base); } } + // Invariant guarding the K5 race: a tracked frame must always have + // heap-backed localsplus (proven here for escaped datastack frames and by + // construction for generator frames, which are born heap-backed). The + // collector reads a frame's localsplus only when the frame is a tracked + // candidate, so this keeps it from ever reading data-stack-resident, + // still-mutating storage. + debug_assert!( + !frame.localsplus_is_datastack_backed(), + "escaped frame tracked before its localsplus was materialized" + ); // SAFETY: the frame is alive (held by `frame` and the escaped reference) // and untracked. unsafe { crate::gc_state::gc_state().track_object(NonNull::from(frame_obj)) }; From 1fa2d5f5b8c58ed94ab3702904c74394c2d8361b Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Mon, 6 Jul 2026 22:38:16 +0900 Subject: [PATCH 57/92] Reword frame/dealloc invariant comments and drop needless borrow - frame.rs: replace the task codename in the tracked-frame localsplus invariant comment with a description of the collector-vs-executing-frame behavior. - object.rs: remove needless borrow on current_cls in the __class__ assignment compatibility check. - core.rs: rewrite the default_dealloc trashcan/untrack-skip comment to state the actual invariant, which now covers untracked non-escaped frames releasing at interpreter depth with bounded recursion. Assisted-by: Claude --- crates/vm/src/builtins/object.rs | 2 +- crates/vm/src/frame.rs | 8 ++++---- crates/vm/src/object/core.rs | 11 +++++++---- 3 files changed, 12 insertions(+), 9 deletions(-) diff --git a/crates/vm/src/builtins/object.rs b/crates/vm/src/builtins/object.rs index b86c3edd423..b6ec1d5c829 100644 --- a/crates/vm/src/builtins/object.rs +++ b/crates/vm/src/builtins/object.rs @@ -436,7 +436,7 @@ impl PyBaseObject { // FIXME(#1979) cls instances might have a payload if both_mutable || both_module { super::type_::compatible_for_assignment( - ¤t_cls, + current_cls, &cls, "__class__", vm, diff --git a/crates/vm/src/frame.rs b/crates/vm/src/frame.rs index c0d8eae3f3b..aab62def97a 100644 --- a/crates/vm/src/frame.rs +++ b/crates/vm/src/frame.rs @@ -1491,12 +1491,12 @@ pub(crate) fn release_datastack_frame(frame: &Py, vm: &VirtualMachine) { vm.datastack_pop(base); } } - // Invariant guarding the K5 race: a tracked frame must always have - // heap-backed localsplus (proven here for escaped datastack frames and by - // construction for generator frames, which are born heap-backed). The + // Invariant: a tracked frame must always have heap-backed localsplus + // (proven here for escaped datastack frames and by construction for + // generator frames, which are born heap-backed). A stop-the-world // collector reads a frame's localsplus only when the frame is a tracked // candidate, so this keeps it from ever reading data-stack-resident, - // still-mutating storage. + // still-mutating storage of an executing frame. debug_assert!( !frame.localsplus_is_datastack_backed(), "escaped frame tracked before its localsplus was materialized" diff --git a/crates/vm/src/object/core.rs b/crates/vm/src/object/core.rs index 11419bec237..80a44d037aa 100644 --- a/crates/vm/src/object/core.rs +++ b/crates/vm/src/object/core.rs @@ -174,10 +174,13 @@ pub(super) unsafe fn default_dealloc(obj: *mut PyObject) { return; // resurrected by __del__ } - // Only GC-tracked objects own child references that can recurse during - // deallocation, so only they need the trashcan recursion guard. Non-GC - // objects (int, float, str, ...) have no reachable children and skip both - // the trashcan and the untrack path. Read once and reuse for both. + // Only tracked objects take the trashcan recursion guard and untrack path. + // Untracked objects either own no children (int, float, str, ...) or, like + // non-escaped frames, are released at interpreter depth with at most one + // unguarded link before their tracked children (dicts, functions, code) + // re-enter guarded deallocation, so recursion stays bounded. A frame stored + // in an object graph is forced to escape, becoming tracked and guarded here. + // Read once and reuse for both gates below. let tracked = obj_ref.is_gc_tracked(); // Trashcan: limit recursive deallocation depth to prevent stack overflow From bd4b3718bfbfb780cc3e192ba55c8a54247d6a96 Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Mon, 6 Jul 2026 23:14:29 +0900 Subject: [PATCH 58/92] Move type SetAttr slot rewrite inside the type lock and tighten cache reads Run update_slot inside the same with_type_lock transaction as modified_inner and the attributes dict mutation, so the version invalidation, dict change, and slot-table rewrite are published together. Load init_version and getitem_version with Acquire in the specialization cache readers to pair with the Release stores in the writers. Assisted-by: Claude --- crates/vm/src/builtins/type.rs | 29 ++++++++++++++++------------- 1 file changed, 16 insertions(+), 13 deletions(-) diff --git a/crates/vm/src/builtins/type.rs b/crates/vm/src/builtins/type.rs index 61d49e79ad8..3ba483fea82 100644 --- a/crates/vm/src/builtins/type.rs +++ b/crates/vm/src/builtins/type.rs @@ -1142,7 +1142,7 @@ impl PyType { let cached_version = ext .specialization_cache .init_version - .load(Ordering::Relaxed); + .load(Ordering::Acquire); if cached_version == 0 { return None; } @@ -1190,7 +1190,7 @@ impl PyType { let cached_version = ext .specialization_cache .getitem_version - .load(Ordering::Relaxed); + .load(Ordering::Acquire); if cached_version == 0 { return None; } @@ -2753,8 +2753,8 @@ impl SetAttr for PyType { // any attribute changes, preventing use-after-free of cached descriptors. zelf.modified_inner(); - if let PySetterValue::Assign(value) = value { - Ok(zelf.attributes.write().insert(attr_name, value)) + let prev_value = if let PySetterValue::Assign(value) = value { + zelf.attributes.write().insert(attr_name, value) } else { let prev_value = zelf.attributes.write().shift_remove(attr_name); // TODO: swap_remove applicable? if prev_value.is_none() { @@ -2764,17 +2764,20 @@ impl SetAttr for PyType { attr_name, ))); } - Ok(prev_value) - } - })?; + prev_value + }; - if attr_name.as_wtf8().starts_with("__") && attr_name.as_wtf8().ends_with("__") { - if assign { - zelf.update_slot::(attr_name, &vm.ctx); - } else { - zelf.update_slot::(attr_name, &vm.ctx); + // Keep the slot-table rewrite inside the same transaction as the + // dict mutation and version invalidation. + if attr_name.as_wtf8().starts_with("__") && attr_name.as_wtf8().ends_with("__") { + if assign { + zelf.update_slot::(attr_name, &vm.ctx); + } else { + zelf.update_slot::(attr_name, &vm.ctx); + } } - } + Ok(prev_value) + })?; Ok(()) } } From 2ae9fb83dffa738082a1a248d4b958062f6a70f8 Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Mon, 6 Jul 2026 23:14:37 +0900 Subject: [PATCH 59/92] Capture the type version before reading mutable slots in specializers specialize_load_attr, specialize_store_attr, specialize_to_bool, and the CallAllocAndEnterInit path of specialize_call read the version tag with version_for_specialization before inspecting getattro, setattro, the bool/len slots, and tp_new/tp_alloc, so a concurrent install of the corresponding dunder invalidates the version the specialization is cached against. In the BinaryOpSubscr __getitem__ path, check the HEAPTYPE and eval-frame gates before lookup_ref_and_version_interned, which takes the global type lock and may allocate a version tag. Assisted-by: Claude --- crates/vm/src/frame.rs | 145 +++++++++++++++++++++-------------------- 1 file changed, 75 insertions(+), 70 deletions(-) diff --git a/crates/vm/src/frame.rs b/crates/vm/src/frame.rs index aab62def97a..45ed78fcae2 100644 --- a/crates/vm/src/frame.rs +++ b/crates/vm/src/frame.rs @@ -7974,6 +7974,22 @@ impl ExecutingFrame<'_> { return; } + // Capture the version before inspecting getattro and the MRO so a + // concurrently installed __getattribute__/__getattr__ invalidates the + // version this specialization is cached against. + let type_version = cls.version_for_specialization(_vm); + if type_version == 0 { + unsafe { + self.code.instructions.write_adaptive_counter( + cache_base, + bytecode::adaptive_counter_backoff( + self.code.instructions.read_adaptive_counter(cache_base), + ), + ); + } + return; + } + // Only specialize if getattro is the default (PyBaseObject::getattro) let is_default_getattro = cls .slots @@ -7981,10 +7997,8 @@ impl ExecutingFrame<'_> { .load() .is_some_and(|f| f as usize == PyBaseObject::getattro as *const () as usize); if !is_default_getattro { - let (getattribute, type_version) = - cls.lookup_ref_and_version_interned(identifier!(_vm, __getattribute__), _vm); - if type_version != 0 - && !oparg.is_method() + let getattribute = cls.get_attr(identifier!(_vm, __getattribute__)); + if !oparg.is_method() && !self.specialization_eval_frame_active(_vm) && cls.get_attr(identifier!(_vm, __getattr__)).is_none() && let Some(getattribute) = getattribute @@ -8025,18 +8039,6 @@ impl ExecutingFrame<'_> { // current module dict has no __getattr__ override and the attribute is // already present. if let Some(module) = obj.downcast_ref_if_exact::(_vm) { - let type_version = cls.version_for_specialization(_vm); - if type_version == 0 { - unsafe { - self.code.instructions.write_adaptive_counter( - cache_base, - bytecode::adaptive_counter_backoff( - self.code.instructions.read_adaptive_counter(cache_base), - ), - ); - } - return; - } let module_dict = module.dict(); match ( module_dict.get_item_opt(identifier!(_vm, __getattr__), _vm), @@ -8063,18 +8065,7 @@ impl ExecutingFrame<'_> { return; } - let (cls_attr, type_version) = cls.lookup_ref_and_version_interned(attr_name, _vm); - if type_version == 0 { - unsafe { - self.code.instructions.write_adaptive_counter( - cache_base, - bytecode::adaptive_counter_backoff( - self.code.instructions.read_adaptive_counter(cache_base), - ), - ); - } - return; - } + let cls_attr = cls.get_attr(attr_name); let class_has_dict = cls.slots.flags.has_feature(PyTypeFlags::HAS_DICT); if oparg.is_method() { @@ -8464,15 +8455,17 @@ impl ExecutingFrame<'_> { Some(Instruction::BinaryOpSubscrListSlice) } else { let cls = a.class(); - let (getitem, type_version) = - cls.lookup_ref_and_version_interned(identifier!(vm, __getitem__), vm); + // Check the cheap gates before the __getitem__ lookup, which + // takes the global type lock and may allocate a version tag. if cls.slots.flags.has_feature(PyTypeFlags::HEAPTYPE) && !self.specialization_eval_frame_active(vm) - && let Some(_getitem) = getitem - && let Some(func) = _getitem.downcast_ref_if_exact::(vm) - && func.can_specialize_call(2) { + let (getitem, type_version) = + cls.lookup_ref_and_version_interned(identifier!(vm, __getitem__), vm); if type_version != 0 + && let Some(getitem) = getitem + && let Some(func) = getitem.downcast_ref_if_exact::(vm) + && func.can_specialize_call(2) && cls.cache_getitem_for_specialization(func.to_owned(), type_version, vm) { // Record the type version so the specialized handler @@ -9002,6 +8995,10 @@ impl ExecutingFrame<'_> { // CallAllocAndEnterInit: heap type with default __new__ if !self_or_null_is_some && cls.slots.flags.has_feature(PyTypeFlags::HEAPTYPE) { + // Capture the version before inspecting tp_new/tp_alloc so a + // concurrently installed __new__ invalidates the version this + // specialization is cached against. + let type_version = cls.version_for_specialization(vm); let object_new = vm.ctx.types.object_type.slots.new.load(); let cls_new = cls.slots.new.load(); let object_alloc = vm.ctx.types.object_type.slots.alloc.load(); @@ -9011,9 +9008,7 @@ impl ExecutingFrame<'_> { && cls_new_fn as usize == obj_new_fn as usize && cls_alloc_fn as usize == obj_alloc_fn as usize { - let (init, version) = - cls.lookup_ref_and_version_interned(identifier!(vm, __init__), vm); - if version == 0 { + if type_version == 0 { unsafe { self.code.instructions.write_adaptive_counter( cache_base, @@ -9024,16 +9019,17 @@ impl ExecutingFrame<'_> { } return; } + let init = cls.get_attr(identifier!(vm, __init__)); if let Some(init) = init && let Some(init_func) = init.downcast_ref_if_exact::(vm) && init_func.can_specialize_call(nargs + 1) && !init_func.is_generator_like() - && cls.cache_init_for_specialization(init_func.to_owned(), version, vm) + && cls.cache_init_for_specialization(init_func.to_owned(), type_version, vm) { unsafe { self.code .instructions - .write_cache_u32(cache_base + 1, version); + .write_cache_u32(cache_base + 1, type_version); } self.specialize_at( instr_idx, @@ -9327,31 +9323,35 @@ impl ExecutingFrame<'_> { Some(Instruction::ToBoolList) } else if cls.is(PyStr::class(&vm.ctx)) { Some(Instruction::ToBoolStr) - } else if cls.slots.flags.has_feature(PyTypeFlags::HEAPTYPE) - && cls.slots.as_number.boolean.load().is_none() - && cls.slots.as_mapping.length.load().is_none() - && cls.slots.as_sequence.length.load().is_none() - { - // Cache type version for ToBoolAlwaysTrue guard + } else if cls.slots.flags.has_feature(PyTypeFlags::HEAPTYPE) { + // Capture the version before inspecting the bool/len slots so a + // concurrently installed __bool__/__len__ invalidates the version + // the ToBoolAlwaysTrue guard is cached against. let type_version = cls.version_for_specialization(vm); - if type_version != 0 { - unsafe { - self.code - .instructions - .write_cache_u32(cache_base + 1, type_version); - } - self.specialize_at(instr_idx, cache_base, Instruction::ToBoolAlwaysTrue); - } else { - unsafe { - self.code.instructions.write_adaptive_counter( - cache_base, - bytecode::adaptive_counter_backoff( - self.code.instructions.read_adaptive_counter(cache_base), - ), - ); + let has_bool_or_len = cls.slots.as_number.boolean.load().is_some() + || cls.slots.as_mapping.length.load().is_some() + || cls.slots.as_sequence.length.load().is_some(); + if !has_bool_or_len { + if type_version != 0 { + unsafe { + self.code + .instructions + .write_cache_u32(cache_base + 1, type_version); + } + self.specialize_at(instr_idx, cache_base, Instruction::ToBoolAlwaysTrue); + } else { + unsafe { + self.code.instructions.write_adaptive_counter( + cache_base, + bytecode::adaptive_counter_backoff( + self.code.instructions.read_adaptive_counter(cache_base), + ), + ); + } } + return; } - return; + None } else { None }; @@ -9649,13 +9649,11 @@ impl ExecutingFrame<'_> { let owner = self.top_value(); let cls = owner.class(); - // Only specialize if setattr is the default (generic_setattr) - let is_default_setattr = cls - .slots - .setattro - .load() - .is_some_and(|f| f as usize == PyBaseObject::slot_setattro as *const () as usize); - if !is_default_setattr { + // Capture the version before inspecting the setattro slot so a + // concurrently installed __setattr__ invalidates the version this + // specialization is cached against. + let type_version = cls.version_for_specialization(vm); + if type_version == 0 { unsafe { self.code.instructions.write_adaptive_counter( cache_base, @@ -9667,9 +9665,13 @@ impl ExecutingFrame<'_> { return; } - let attr_name = self.code.names[attr_idx as usize]; - let (cls_attr, type_version) = cls.lookup_ref_and_version_interned(attr_name, vm); - if type_version == 0 { + // Only specialize if setattr is the default (generic_setattr) + let is_default_setattr = cls + .slots + .setattro + .load() + .is_some_and(|f| f as usize == PyBaseObject::slot_setattro as *const () as usize); + if !is_default_setattr { unsafe { self.code.instructions.write_adaptive_counter( cache_base, @@ -9680,6 +9682,9 @@ impl ExecutingFrame<'_> { } return; } + + let attr_name = self.code.names[attr_idx as usize]; + let cls_attr = cls.get_attr(attr_name); let has_data_descr = cls_attr.as_ref().is_some_and(|descr| { let descr_cls = descr.class(); descr_cls.slots.descr_get.load().is_some() && descr_cls.slots.descr_set.load().is_some() From 68e8b91d11859e273f2388d465ca695ff4fd311f Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Mon, 6 Jul 2026 23:14:43 +0900 Subject: [PATCH 60/92] Use _get_method_dict for the test_type check in the test runner Replace the direct __func__.__dict__ access with the _get_method_dict helper so plain functions without __func__ do not raise AttributeError. Assisted-by: Claude --- extra_tests/custom_text_test_runner.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/extra_tests/custom_text_test_runner.py b/extra_tests/custom_text_test_runner.py index 3457bdfd0e4..865265750f4 100644 --- a/extra_tests/custom_text_test_runner.py +++ b/extra_tests/custom_text_test_runner.py @@ -389,11 +389,9 @@ def startTest(self, test): } self.start_time = time.time() if self.test_types: - if "test_type" in getattr( - test, test._testMethodName - ).__func__.__dict__ and set([s.lower() for s in self.test_types]) == set( - [s.lower() for s in _get_method_dict(test)["test_type"]] - ): + if "test_type" in _get_method_dict(test) and set( + [s.lower() for s in self.test_types] + ) == set([s.lower() for s in _get_method_dict(test)["test_type"]]): pass else: _get_method_dict(test)["__unittest_skip_why__"] = ( From ec8b128c74e5c6bc2d7659a8dd8af3f63b21a294 Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Tue, 7 Jul 2026 00:21:24 +0900 Subject: [PATCH 61/92] Fix lint hooks: cspell dictionary entries, spelling fix, formatting Add qsbr to the rustpython dictionary and reborrows/reparenting to the top-level word list. Rename oldto/newto locals to old_to/new_to. Fix Stabilise -> Stabilize typo in a frame.rs comment. Apply cargo fmt to object.rs, type.rs, and frame.rs, and ruff format/check fixes to two extra_tests snippet files. Assisted-by: Claude --- .cspell.dict/rustpython.txt | 1 + .cspell.json | 2 ++ crates/vm/src/builtins/object.rs | 7 +------ crates/vm/src/builtins/type.rs | 19 +++++++++---------- crates/vm/src/frame.rs | 8 ++++++-- .../stdlib_threading_gc_frame_race.py | 1 + .../snippets/stdlib_threading_gc_import.py | 2 +- 7 files changed, 21 insertions(+), 19 deletions(-) diff --git a/.cspell.dict/rustpython.txt b/.cspell.dict/rustpython.txt index 8cd08358019..07099bbb171 100644 --- a/.cspell.dict/rustpython.txt +++ b/.cspell.dict/rustpython.txt @@ -27,6 +27,7 @@ pystr pystruct pystructseq pytype +qsbr rustix struc zelf diff --git a/.cspell.json b/.cspell.json index af2f1401d95..45ec8b60d39 100644 --- a/.cspell.json +++ b/.cspell.json @@ -81,6 +81,8 @@ "opargs", "pyc", "reborrow", + "reborrows", + "reparenting", "reraises", "reraising", "significand", diff --git a/crates/vm/src/builtins/object.rs b/crates/vm/src/builtins/object.rs index b6ec1d5c829..633eaf48a44 100644 --- a/crates/vm/src/builtins/object.rs +++ b/crates/vm/src/builtins/object.rs @@ -435,12 +435,7 @@ impl PyBaseObject { && !cls.slots.flags.has_feature(PyTypeFlags::IMMUTABLETYPE); // FIXME(#1979) cls instances might have a payload if both_mutable || both_module { - super::type_::compatible_for_assignment( - current_cls, - &cls, - "__class__", - vm, - )?; + super::type_::compatible_for_assignment(current_cls, &cls, "__class__", vm)?; instance.set_class(cls, vm); Ok(()) } else { diff --git a/crates/vm/src/builtins/type.rs b/crates/vm/src/builtins/type.rs index 3ba483fea82..9a7b7a3ace7 100644 --- a/crates/vm/src/builtins/type.rs +++ b/crates/vm/src/builtins/type.rs @@ -3278,33 +3278,32 @@ fn same_slots_added(a: &Py, b: &Py) -> bool { } } -/// Validates that instances of `oldto` and `newto` share an interchangeable +/// Validates that instances of `old_to` and `new_to` share an interchangeable /// object layout, the check `__class__` and `__bases__` assignment perform. /// /// `attr` names the attribute being assigned for the error message; the -/// message reports `newto` first and `oldto` second. +/// message reports `new_to` first and `old_to` second. pub(crate) fn compatible_for_assignment( - oldto: &Py, - newto: &Py, + old_to: &Py, + new_to: &Py, attr: &str, vm: &VirtualMachine, ) -> PyResult<()> { - let newbase = layout_solid_base(newto); - let oldbase = layout_solid_base(oldto); + let newbase = layout_solid_base(new_to); + let oldbase = layout_solid_base(old_to); let bases_equal = match (newbase.base.deref(), oldbase.base.deref()) { (Some(x), Some(y)) => x.is(y), (None, None) => true, _ => false, }; - let compatible = - newbase.is(oldbase) || (bases_equal && same_slots_added(newbase, oldbase)); + let compatible = newbase.is(oldbase) || (bases_equal && same_slots_added(newbase, oldbase)); if compatible { return Ok(()); } Err(vm.new_type_error(format!( "{attr} assignment: '{}' object layout differs from '{}'", - newto.name(), - oldto.name() + new_to.name(), + old_to.name() ))) } diff --git a/crates/vm/src/frame.rs b/crates/vm/src/frame.rs index 45ed78fcae2..ed7d696b357 100644 --- a/crates/vm/src/frame.rs +++ b/crates/vm/src/frame.rs @@ -1479,7 +1479,7 @@ pub(crate) fn release_datastack_frame(frame: &Py, vm: &VirtualMachine) { } return; } - // Escaped. Stabilise localsplus on the heap FIRST, then join the GC. This + // Escaped. Stabilize localsplus on the heap FIRST, then join the GC. This // order guarantees a concurrent (stop-the-world) collector only ever sees a // tracked frame whose localsplus is heap-resident and no longer mutating: // the frame has stopped executing before it becomes a candidate, so its @@ -8466,7 +8466,11 @@ impl ExecutingFrame<'_> { && let Some(getitem) = getitem && let Some(func) = getitem.downcast_ref_if_exact::(vm) && func.can_specialize_call(2) - && cls.cache_getitem_for_specialization(func.to_owned(), type_version, vm) + && cls.cache_getitem_for_specialization( + func.to_owned(), + type_version, + vm, + ) { // Record the type version so the specialized handler // can revalidate before using the cached __getitem__. diff --git a/extra_tests/snippets/stdlib_threading_gc_frame_race.py b/extra_tests/snippets/stdlib_threading_gc_frame_race.py index e6777330a26..37cdbbf122c 100644 --- a/extra_tests/snippets/stdlib_threading_gc_frame_race.py +++ b/extra_tests/snippets/stdlib_threading_gc_frame_race.py @@ -52,6 +52,7 @@ def counter(limit): def make_frame_cycles(n): for _ in range(n): + def inner(): fr = sys._getframe() box = {"fr": fr} diff --git a/extra_tests/snippets/stdlib_threading_gc_import.py b/extra_tests/snippets/stdlib_threading_gc_import.py index 061ff04a799..340184093f3 100644 --- a/extra_tests/snippets/stdlib_threading_gc_import.py +++ b/extra_tests/snippets/stdlib_threading_gc_import.py @@ -13,9 +13,9 @@ """ import gc +import importlib import sys import threading -import importlib gc.enable() stop = threading.Event() From 9d44aaf62ffda29d644b6aff5bb7ea81e81306e9 Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Tue, 7 Jul 2026 00:46:34 +0900 Subject: [PATCH 62/92] Rebuild all slots for type and descendants on __bases__ reassignment Add PyType::update_all_slots, which invalidates version tags and iterates the full SLOT_DEFS name table calling update_slot:: for each distinct name. Unlike init_slots, which is additive and driven only by dunder names present in the current MRO, this resets a slot whose method left the MRO instead of leaving a stale dispatcher pointer. Call it from set_bases in place of the previous modified_inner + init_slots pair, so reassigning __bases__ rebuilds slots for the type and every descendant. Add extra_tests/snippets/type_bases_slot_rebuild.py covering removed-method resets on zelf and deep descendants, the wrong-target switch, __getattr__, the added-method mirror, and a swap-away-and-back round trip. Assisted-by: Claude --- crates/vm/src/builtins/type.rs | 28 ++- .../snippets/type_bases_slot_rebuild.py | 202 ++++++++++++++++++ 2 files changed, 225 insertions(+), 5 deletions(-) create mode 100644 extra_tests/snippets/type_bases_slot_rebuild.py diff --git a/crates/vm/src/builtins/type.rs b/crates/vm/src/builtins/type.rs index 9a7b7a3ace7..0aeed2db17f 100644 --- a/crates/vm/src/builtins/type.rs +++ b/crates/vm/src/builtins/type.rs @@ -954,6 +954,26 @@ impl PyType { Self::set_alloc(&self.slots, self.base.deref()); } + /// Recompute every slot for this type and all its descendants. update_all_slots + /// + /// Unlike `init_slots`, which is additive and driven only by the dunder names + /// present in the current MRO, this iterates the full `SLOT_DEFS` name table so + /// a slot whose method left the MRO is reset instead of left stale. Must be + /// called under the type lock after MROs have been recomputed. + pub(crate) fn update_all_slots(&self, ctx: &Context) { + // Invalidate version tags first; cascades to subclasses. + self.modified_inner(); + // Distinct names only; update_slot fans out to every SLOT_DEFS entry + // sharing the name and recurses into subclasses on its own. + let mut seen = std::collections::HashSet::new(); + for def in SLOT_DEFS { + if seen.insert(def.name) { + let name = ctx.intern_str(def.name); + self.update_slot::(name, ctx); + } + } + } + fn set_new(slots: &PyTypeSlots, base: Option<&Py>) { if slots.flags.contains(PyTypeFlags::DISALLOW_INSTANTIATION) { slots.new.store(None) @@ -1589,11 +1609,9 @@ impl PyType { keep_alive(old_base, &mut retired); } - // Invalidate inline caches - zelf.modified_inner(); - - // TODO: do any old slots need to be cleaned up first? - zelf.init_slots(&vm.ctx); + // Invalidate inline caches and rebuild every slot for this type and + // all descendants so slots whose methods left the MRO are reset. + zelf.update_all_slots(&vm.ctx); register_subclasses(&zelf.bases.read()); Ok(()) diff --git a/extra_tests/snippets/type_bases_slot_rebuild.py b/extra_tests/snippets/type_bases_slot_rebuild.py new file mode 100644 index 00000000000..a70e433b50a --- /dev/null +++ b/extra_tests/snippets/type_bases_slot_rebuild.py @@ -0,0 +1,202 @@ +from testutils import assert_raises + + +# Reassigning __bases__ must rebuild slot dispatchers for the type and all its +# descendants: a slot whose method left the new MRO must be reset, not left stale. + +# --- zelf itself loses __add__ (nb_add) --- +class OldAdd: + def __add__(self, other): + return "OLD" + + +class Bare: + pass + + +class C(OldAdd): + pass + + +c = C() +assert c + 1 == "OLD" +C.__bases__ = (Bare,) +with assert_raises(TypeError): + c + 1 + + +# --- 3-level descendant loses __iter__ (tp_iter) --- +class Itr: + def __iter__(self): + return iter([1, 2, 3]) + + +class New: + pass + + +class C2(Itr): + pass + + +class D2(C2): + pass + + +class E2(D2): + pass + + +e = E2() +assert list(e) == [1, 2, 3] +C2.__bases__ = (New,) +with assert_raises(TypeError): + list(e) + + +# --- descendant loses __len__ (sq_length), sibling slot untouched --- +class Sized: + def __len__(self): + return 7 + + +class C3(Sized): + pass + + +class D3(C3): + pass + + +d3 = D3() +assert len(d3) == 7 +C3.__bases__ = (Bare,) +with assert_raises(TypeError): + len(d3) + + +# --- descendant loses __getitem__ (mp_subscript) --- +class Subscriptable: + def __getitem__(self, key): + return key * 2 + + +class C4(Subscriptable): + pass + + +class D4(C4): + pass + + +d4 = D4() +assert d4[3] == 6 +C4.__bases__ = (Bare,) +with assert_raises(TypeError): + d4[3] + + +# --- descendant loses __call__ (tp_call) --- +class Callable: + def __call__(self): + return "called" + + +class C5(Callable): + pass + + +class D5(C5): + pass + + +d5 = D5() +assert d5() == "called" +C5.__bases__ = (Bare,) +with assert_raises(TypeError): + d5() + + +# --- guard: stale-wrong-target, name present in both bases must switch --- +class OldTarget: + def __add__(self, other): + return "OLD.__add__" + + +class NewTarget: + def __add__(self, other): + return "NEW.__add__" + + +class C6(OldTarget): + pass + + +class D6(C6): + pass + + +d6 = D6() +assert d6 + 1 == "OLD.__add__" +C6.__bases__ = (NewTarget,) +assert d6 + 1 == "NEW.__add__" + + +# --- guard: __getattr__ resolves at call time, stays correct --- +class OldGetattr: + def __getattr__(self, name): + return "OLD:" + name + + +class C7(OldGetattr): + pass + + +class D7(C7): + pass + + +d7 = D7() +assert d7.missing == "OLD:missing" +C7.__bases__ = (Bare,) +with assert_raises(AttributeError): + d7.missing + + +# --- mirror: new base ADDS a dunder the old chain lacked --- +class Adder: + def __add__(self, other): + return "ADDED" + + +class C8(Bare): + pass + + +class D8(C8): + pass + + +d8 = D8() +with assert_raises(TypeError): + d8 + 1 +C8.__bases__ = (Adder,) +assert d8 + 1 == "ADDED" + + +# --- round trip: swap away then back restores the slot --- +class C9(OldAdd): + pass + + +class D9(C9): + pass + + +d9 = D9() +assert d9 + 1 == "OLD" +C9.__bases__ = (Bare,) +with assert_raises(TypeError): + d9 + 1 +C9.__bases__ = (OldAdd,) +assert d9 + 1 == "OLD" From ab0cddb3f6402b95b2761098ecb663b5e733abb2 Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Tue, 7 Jul 2026 01:24:56 +0900 Subject: [PATCH 63/92] Derive the thread-local frame stack from the frame chain Remove the per-VM `frames` Vec. current_frame, sys._getframe, sys._getframemodulename, frame.f_back, sys.monitoring re-instrumentation, gc.get_referrers, faulthandler stack dumps and the post-fork slot rebuild now walk the signal-safe CURRENT_FRAME/`previous` chain instead. Add `Py::from_payload_ptr` to recover a frame object from a chain pointer. The cross-thread `ThreadSlot::frames` registry (sys._current_frames, cross-thread f_back, GC stop-the-world assertion) is unchanged. Assisted-by: Claude --- crates/stdlib/src/faulthandler.rs | 27 +++++------ crates/vm/src/builtins/frame.rs | 17 ++----- crates/vm/src/frame.rs | 65 ++++++++++++++++++++++++++ crates/vm/src/object/core.rs | 13 ++++++ crates/vm/src/stdlib/gc.rs | 14 ++---- crates/vm/src/stdlib/sys.rs | 27 +++-------- crates/vm/src/stdlib/sys/monitoring.rs | 6 +-- crates/vm/src/vm/mod.rs | 28 ++++------- crates/vm/src/vm/thread.rs | 15 ++++-- 9 files changed, 130 insertions(+), 82 deletions(-) diff --git a/crates/stdlib/src/faulthandler.rs b/crates/stdlib/src/faulthandler.rs index 3717c18b78f..35c6f07fad0 100644 --- a/crates/stdlib/src/faulthandler.rs +++ b/crates/stdlib/src/faulthandler.rs @@ -278,11 +278,9 @@ mod decl { dump_all_threads(fd, vm); } else { puts(fd, "Stack (most recent call first):\n"); - let frames = vm.frames.borrow(); - for fp in frames.iter().rev() { - // SAFETY: the frame is alive while it's in the Vec - dump_frame_from_ref(fd, unsafe { fp.as_ref() }); - } + crate::vm::frame::for_each_current_frame(|frame| { + dump_frame_from_ref(fd, frame); + }); } } @@ -318,25 +316,24 @@ mod decl { puts(fd, "\n"); } - // Now dump current thread (use vm.frames for most up-to-date data) + // Now dump current thread from its live frame chain. write_thread_id(fd, current_tid, true); - let frames = vm.frames.borrow(); - if frames.is_empty() { + if crate::vm::vm::thread::get_current_frame().is_null() { puts(fd, " \n"); } else { - for fp in frames.iter().rev() { - dump_frame_from_ref(fd, unsafe { fp.as_ref() }); - } + crate::vm::frame::for_each_current_frame(|frame| { + dump_frame_from_ref(fd, frame); + }); } } #[cfg(not(feature = "threading"))] { + let _ = vm; write_thread_id(fd, current_thread_id(), true); - let frames = vm.frames.borrow(); - for fp in frames.iter().rev() { - dump_frame_from_ref(fd, unsafe { fp.as_ref() }); - } + crate::vm::frame::for_each_current_frame(|frame| { + dump_frame_from_ref(fd, frame); + }); } } diff --git a/crates/vm/src/builtins/frame.rs b/crates/vm/src/builtins/frame.rs index ab45f68673c..8068f4b807a 100644 --- a/crates/vm/src/builtins/frame.rs +++ b/crates/vm/src/builtins/frame.rs @@ -715,23 +715,16 @@ impl Py { #[pygetset] pub fn f_back(&self, vm: &VirtualMachine) -> Option> { + #[cfg(not(feature = "threading"))] + let _ = vm; let previous = self.previous_frame(); if previous.is_null() { return None; } - if let Some(frame) = vm - .frames - .borrow() - .iter() - .find(|fp| { - // SAFETY: the caller keeps the FrameRef alive while it's in the Vec - let py: &Self = unsafe { fp.as_ref() }; - let ptr: *const Frame = &**py; - core::ptr::eq(ptr, previous) - }) - .map(|fp| unsafe { fp.as_ref() }.to_owned()) - { + // Look for the caller on the current thread's signal-safe frame chain. + // Finding it there proves it is still live on this thread. + if let Some(frame) = crate::frame::find_owned_chain_frame(previous) { return Some(frame); } diff --git a/crates/vm/src/frame.rs b/crates/vm/src/frame.rs index ed7d696b357..ac62e41ac8b 100644 --- a/crates/vm/src/frame.rs +++ b/crates/vm/src/frame.rs @@ -55,6 +55,71 @@ use rustpython_compiler_core::SourceLocation; pub type FrameRef = PyRef; +/// Recover an owned reference to a live chain frame, or `None` for null. +/// +/// # Safety +/// A non-null `frame` must reference a frame that is live on the current +/// thread's execution chain, so the object outlives this call. +unsafe fn owned_chain_frame(frame: *const Frame) -> Option { + if frame.is_null() { + return None; + } + // SAFETY: caller guarantees the frame is live; from_payload_ptr recovers + // the enclosing object from the payload address. + let py = unsafe { &*Py::::from_payload_ptr(frame) }; + Some(py.to_owned()) +} + +/// The current thread's topmost frame object, if any. +#[must_use] +pub fn current_thread_frame() -> Option { + // SAFETY: the chain top executes on this thread, hence is alive. + unsafe { owned_chain_frame(crate::vm::thread::get_current_frame()) } +} + +/// The frame `offset` positions below the current thread's top frame (offset 0 +/// is the top), or `None` if the stack is not that deep. +#[must_use] +pub fn frame_at_offset(offset: usize) -> Option { + let mut cur = crate::vm::thread::get_current_frame(); + for _ in 0..offset { + if cur.is_null() { + return None; + } + // SAFETY: chain frames are alive on the current thread's stack. + cur = unsafe { (*cur).previous_frame() }; + } + // SAFETY: same as above. + unsafe { owned_chain_frame(cur) } +} + +/// If `target` is a frame on the current thread's chain, return an owned +/// reference to it; otherwise `None`. Presence on the chain proves liveness. +#[must_use] +pub fn find_owned_chain_frame(target: *const Frame) -> Option { + let mut cur = crate::vm::thread::get_current_frame(); + while !cur.is_null() { + if core::ptr::eq(cur, target) { + // SAFETY: a frame on the current thread's chain is alive. + return unsafe { owned_chain_frame(cur) }; + } + // SAFETY: chain frames are alive on the current thread's stack. + cur = unsafe { (*cur).previous_frame() }; + } + None +} + +/// Invoke `f` for each frame on the current thread's chain, from the topmost +/// frame down to the bottom. +pub fn for_each_current_frame(mut f: impl FnMut(&Py)) { + let mut cur = crate::vm::thread::get_current_frame(); + while !cur.is_null() { + // SAFETY: chain frames are alive on the current thread's stack. + f(unsafe { &*Py::::from_payload_ptr(cur) }); + cur = unsafe { (*cur).previous_frame() }; + } +} + /// The reason why we might be unwinding a block. /// This could be return of function, exception being /// raised, a break or continue being hit, etc.. diff --git a/crates/vm/src/object/core.rs b/crates/vm/src/object/core.rs index 80a44d037aa..f643be7e1fa 100644 --- a/crates/vm/src/object/core.rs +++ b/crates/vm/src/object/core.rs @@ -2133,6 +2133,19 @@ impl Py { pub fn payload(&self) -> &T { &self.0.payload } + + /// Recover the object pointer from a pointer to its `payload` field. + /// + /// # Safety + /// `payload` must point to the `payload` of a live `Py` (e.g. a `&T` + /// obtained by dereferencing a `Py`), and the object must outlive the + /// returned pointer's use. + #[inline] + pub(crate) unsafe fn from_payload_ptr(payload: *const T) -> *const Self { + let offset = core::mem::offset_of!(PyInner, payload); + // `Py` is a newtype over `PyInner`, so their addresses coincide. + unsafe { (payload as *const u8).sub(offset) as *const Self } + } } impl ToOwned for Py { diff --git a/crates/vm/src/stdlib/gc.rs b/crates/vm/src/stdlib/gc.rs index 00eea1b39d5..b0007b4c867 100644 --- a/crates/vm/src/stdlib/gc.rs +++ b/crates/vm/src/stdlib/gc.rs @@ -199,15 +199,11 @@ mod gc { // PyObjects, so they never appear in get_referrers results. Since // RustPython materializes every frame as a PyObject, we must exclude // them manually to match the expected behavior. - let stack_frames: HashSet = vm - .frames - .borrow() - .iter() - .map(|fp| { - let frame: &crate::PyObject = unsafe { fp.as_ref() }.as_ref(); - frame as *const crate::PyObject as usize - }) - .collect(); + let mut stack_frames: HashSet = HashSet::new(); + crate::frame::for_each_current_frame(|frame| { + let obj: &crate::PyObject = frame.as_ref(); + stack_frames.insert(obj as *const crate::PyObject as usize); + }); let mut result = Vec::new(); diff --git a/crates/vm/src/stdlib/sys.rs b/crates/vm/src/stdlib/sys.rs index b8fe578f238..e5ae1157fa1 100644 --- a/crates/vm/src/stdlib/sys.rs +++ b/crates/vm/src/stdlib/sys.rs @@ -43,7 +43,7 @@ pub mod sys { hash::{PyHash, PyUHash}, }, convert::ToPyObject, - frame::{Frame, FrameRef}, + frame::FrameRef, function::{FuncArgs, KwArgs, OptionalArg, PosArgs}, stdlib::{_warnings::warn, builtins}, types::PyStructSequence, @@ -968,17 +968,8 @@ pub mod sys { #[pyfunction] fn _getframe(offset: OptionalArg, vm: &VirtualMachine) -> PyResult { let offset = offset.into_option().unwrap_or(0); - let frame_ref = { - let frames = vm.frames.borrow(); - if offset >= frames.len() { - return Err(vm.new_value_error("call stack is not deep enough")); - } - - let idx = frames.len() - offset - 1; - // SAFETY: the FrameRef is alive on the call stack while it's in the Vec - let py: &crate::Py = unsafe { frames[idx].as_ref() }; - py.to_owned() - }; + let frame_ref = crate::frame::frame_at_offset(offset) + .ok_or_else(|| vm.new_value_error("call stack is not deep enough"))?; if let Ok(audit) = vm.sys_module.get_attr("audit", vm) { audit.call((vm.ctx.new_str("sys._getframe"), frame_ref.to_owned()), vm)?; @@ -998,15 +989,9 @@ pub mod sys { } // Get the frame at the specified depth - let func_obj = { - let frames = vm.frames.borrow(); - if depth >= frames.len() { - return Ok(vm.ctx.none()); - } - let idx = frames.len() - depth - 1; - // SAFETY: the FrameRef is alive on the call stack while it's in the Vec - let frame: &crate::Py = unsafe { frames[idx].as_ref() }; - frame.func_obj.clone() + let func_obj = match crate::frame::frame_at_offset(depth) { + Some(frame) => frame.func_obj.clone(), + None => return Ok(vm.ctx.none()), }; // If the frame has a function object, return its __module__ attribute diff --git a/crates/vm/src/stdlib/sys/monitoring.rs b/crates/vm/src/stdlib/sys/monitoring.rs index accf2001675..56a68cea619 100644 --- a/crates/vm/src/stdlib/sys/monitoring.rs +++ b/crates/vm/src/stdlib/sys/monitoring.rs @@ -528,9 +528,7 @@ fn update_events_mask(vm: &VirtualMachine, state: &MonitoringState) { // Each code object gets only the events that apply to it (global + its // own local events), preventing e.g. INSTRUCTION from being applied to // unrelated code objects. - for fp in vm.frames.borrow().iter() { - // SAFETY: frames in the Vec are alive while their FrameRef is on the call stack. - let frame = unsafe { fp.as_ref() }; + crate::frame::for_each_current_frame(|frame| { let code = &frame.code; let code_ver = code.instrumentation_version.load(Ordering::Acquire); if code_ver != new_ver { @@ -539,7 +537,7 @@ fn update_events_mask(vm: &VirtualMachine, state: &MonitoringState) { code.instrumentation_version .store(new_ver, Ordering::Release); } - } + }); } fn use_tool_id(tool_id: i32, name: &str, vm: &VirtualMachine) -> PyResult<()> { diff --git a/crates/vm/src/vm/mod.rs b/crates/vm/src/vm/mod.rs index f22d08d2809..06f67313af2 100644 --- a/crates/vm/src/vm/mod.rs +++ b/crates/vm/src/vm/mod.rs @@ -74,7 +74,6 @@ pub struct VirtualMachine { pub builtins: PyRef, pub sys_module: PyRef, pub ctx: PyRc, - pub frames: RefCell>, /// Thread-local data stack for bump-allocating frame-local data /// (localsplus arrays for non-generator frames). datastack: core::cell::UnsafeCell, @@ -122,8 +121,9 @@ impl FramePtr { } } -// SAFETY: FramePtr is only stored in the VM's frames Vec while the corresponding -// FrameRef is alive on the call stack. The Vec is always empty when the VM moves between threads. +// SAFETY: FramePtr is only stored in a thread's shared frame stack +// (`ThreadSlot::frames`) while the corresponding FrameRef is alive on that +// thread's call stack; readers dereference it under the slot mutex. unsafe impl Send for FramePtr {} #[derive(Debug)] @@ -856,7 +856,6 @@ impl VirtualMachine { builtins, sys_module, ctx, - frames: RefCell::new(vec![]), datastack: core::cell::UnsafeCell::new(crate::datastack::DataStack::new()), wasm_id: None, exceptions: RefCell::default(), @@ -1702,12 +1701,12 @@ impl VirtualMachine { // SAFETY: `frame` (FrameRef) stays alive for the entire closure scope, // keeping the FramePtr valid. We pass a clone to `f` so that `f` // consuming its FrameRef doesn't invalidate our pointer. - let fp = FramePtr(NonNull::from(&*frame)); - self.frames.borrow_mut().push(fp); - // Update the shared frame stack for sys._current_frames() and faulthandler + // Publish the frame for sys._current_frames() and faulthandler. #[cfg(feature = "threading")] - crate::vm::thread::push_thread_frame(fp); - // Link frame into the signal-safe frame chain (previous pointer) + crate::vm::thread::push_thread_frame(FramePtr(NonNull::from(&*frame))); + // Link frame into the signal-safe frame chain (previous pointer). + // This chain is the single source for the current thread's frame + // stack (current_frame, sys._getframe, f_back, monitoring). let old_frame = crate::vm::thread::set_current_frame((&**frame) as *const Frame); frame.previous.store( old_frame as *mut Frame, @@ -1730,7 +1729,6 @@ impl VirtualMachine { frame.owner.store(old_owner, core::sync::atomic::Ordering::Release); self.restore_exception(saved_exc); crate::vm::thread::set_current_frame(old_frame); - self.frames.borrow_mut().pop(); #[cfg(feature = "threading")] crate::vm::thread::pop_thread_frame(); } @@ -1759,10 +1757,8 @@ impl VirtualMachine { self.recursion_depth.update(|d| d + 1); // SAFETY: frame (&FrameRef) stays alive for the duration, so NonNull is valid until pop. - let fp = FramePtr(NonNull::from(&**frame)); - self.frames.borrow_mut().push(fp); #[cfg(feature = "threading")] - crate::vm::thread::push_thread_frame(fp); + crate::vm::thread::push_thread_frame(FramePtr(NonNull::from(&**frame))); let old_frame = crate::vm::thread::set_current_frame((&***frame) as *const Frame); frame.previous.store( old_frame as *mut Frame, @@ -1783,7 +1779,6 @@ impl VirtualMachine { frame.owner.store(old_owner, core::sync::atomic::Ordering::Release); self.pop_exception(); crate::vm::thread::set_current_frame(old_frame); - self.frames.borrow_mut().pop(); #[cfg(feature = "threading")] crate::vm::thread::pop_thread_frame(); @@ -1874,10 +1869,7 @@ impl VirtualMachine { } pub fn current_frame(&self) -> Option { - self.frames.borrow().last().map(|fp| { - // SAFETY: the caller keeps the FrameRef alive while it's in the Vec - unsafe { fp.as_ref() }.to_owned() - }) + crate::frame::current_thread_frame() } pub fn current_locals(&self) -> PyResult { diff --git a/crates/vm/src/vm/thread.rs b/crates/vm/src/vm/thread.rs index 7990faf97ab..eabfbc043ef 100644 --- a/crates/vm/src/vm/thread.rs +++ b/crates/vm/src/vm/thread.rs @@ -682,14 +682,24 @@ pub fn cleanup_current_thread_frames(vm: &VirtualMachine) { /// Reinitialize thread slot after fork. Called in child process. /// Creates a fresh slot and registers it for the current thread, -/// preserving the current thread's frames from `vm.frames`. +/// preserving the current thread's frames from the signal-safe frame chain. /// /// Precondition: `reinit_locks_after_fork()` has already reset all /// VmState locks to unlocked. #[cfg(feature = "threading")] pub fn reinit_frame_slot_after_fork(vm: &VirtualMachine) { let current_ident = crate::stdlib::_thread::get_ident(); - let current_frames: Vec = vm.frames.borrow().clone(); + // Rebuild the shared frame stack (bottom-to-top) from the current thread's + // frame chain, which walks top-to-bottom via `previous`. + let mut current_frames: Vec = Vec::new(); + let mut cur = get_current_frame(); + while !cur.is_null() { + // SAFETY: the forking thread's chain frames are alive. + let py = unsafe { crate::Py::::from_payload_ptr(cur) }; + current_frames.push(FramePtr(unsafe { NonNull::new_unchecked(py as *mut _) })); + cur = unsafe { (*cur).previous_frame() }; + } + current_frames.reverse(); let new_slot = Arc::new(ThreadSlot { frames: parking_lot::Mutex::new(current_frames), exception: crate::PyAtomicRef::from(vm.topmost_exception()), @@ -835,7 +845,6 @@ impl VirtualMachine { builtins: self.builtins.clone(), sys_module: self.sys_module.clone(), ctx: self.ctx.clone(), - frames: RefCell::new(vec![]), datastack: core::cell::UnsafeCell::new(crate::datastack::DataStack::new()), wasm_id: self.wasm_id.clone(), exceptions: RefCell::default(), From ae106abb4ad03b911bac4de4f74326a47cd04b16 Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Tue, 7 Jul 2026 01:40:50 +0900 Subject: [PATCH 64/92] Inherit sub-slot fallback into the field being resolved update_one_slot's number/sequence/mapping fallback inherited via the accessor's default field. Left and right binary ops (add/right_add) share one accessor but occupy distinct fields, so resolving an absent right op or deleting it reset the left op's field, dropping a still-defined __add__ dispatcher. Inherit the exact field under resolution instead. Assisted-by: Claude --- crates/vm/src/types/slot.rs | 16 +++- .../snippets/type_bases_slot_rebuild.py | 91 +++++++++++++++++++ 2 files changed, 105 insertions(+), 2 deletions(-) diff --git a/crates/vm/src/types/slot.rs b/crates/vm/src/types/slot.rs index a2aa62e0e71..a5d7a41d3fd 100644 --- a/crates/vm/src/types/slot.rs +++ b/crates/vm/src/types/slot.rs @@ -739,6 +739,18 @@ impl PyType { // Helper macro for number/sequence/mapping sub-slots macro_rules! update_sub_slot { ($group:ident, $slot:ident, $wrapper:expr, $variant:ident) => {{ + // Fall back to the value inherited for this exact field. Left and + // right binary ops (e.g. add / right_add) share one accessor but + // occupy distinct fields, so the fallback must target this field + // rather than the accessor's default field, otherwise resolving + // an absent right op would overwrite the left op's dispatcher. + let inherit_this_field = || { + let mro = self.mro.read(); + let inherited = mro[1..] + .iter() + .find_map(|cls| cls.slots.$group.$slot.load()); + self.slots.$group.$slot.store(inherited); + }; if ADD { // Check if this type defines any method that maps to this slot. // Some slots like SqAssItem/MpAssSubscript are shared by multiple @@ -784,12 +796,12 @@ impl PyType { store_wrapper(); } SlotLookupResult::NotFound => { - accessor.inherit_from_mro(self); + inherit_this_field(); } } } } else { - accessor.inherit_from_mro(self); + inherit_this_field(); } }}; } diff --git a/extra_tests/snippets/type_bases_slot_rebuild.py b/extra_tests/snippets/type_bases_slot_rebuild.py index a70e433b50a..f40343dfb7c 100644 --- a/extra_tests/snippets/type_bases_slot_rebuild.py +++ b/extra_tests/snippets/type_bases_slot_rebuild.py @@ -200,3 +200,94 @@ class D9(C9): d9 + 1 C9.__bases__ = (OldAdd,) assert d9 + 1 == "OLD" + + +# --- left-only __add__ defined on the type itself survives a base swap --- +# __add__ and __radd__ share one accessor but occupy distinct fields; resolving +# the absent __radd__ must not overwrite the __add__ dispatcher. +class Mixin: + pass + + +class Other: + pass + + +class C10(Mixin): + def __add__(self, o): + return "C10" + + +c10 = C10() +assert c10 + 1 == "C10" +C10.__bases__ = (Other,) +assert c10 + 1 == "C10" + + +# --- right-only __radd__ survives a base swap --- +class C11(Mixin): + def __radd__(self, o): + return "C11" + + +c11 = C11() +assert 1 + c11 == "C11" +C11.__bases__ = (Other,) +assert 1 + c11 == "C11" + + +# --- subclass/grandchild shadowing __add__ keeps it when an ancestor swaps bases --- +class AddBase: + def __add__(self, o): + return "AddBase" + + +class Ancestor(AddBase): + pass + + +class Shadow(Ancestor): + def __add__(self, o): + return "Shadow" + + +class GrandShadow(Shadow): + pass + + +sh = Shadow() +gsh = GrandShadow() +assert sh + 1 == "Shadow" +assert gsh + 1 == "Shadow" +Ancestor.__bases__ = (Mixin,) +assert sh + 1 == "Shadow" +assert gsh + 1 == "Shadow" + + +# --- another Nb* pair: left-only __sub__ survives a base swap --- +class C12(Mixin): + def __sub__(self, o): + return "C12" + + +c12 = C12() +assert c12 - 1 == "C12" +C12.__bases__ = (Other,) +assert c12 - 1 == "C12" + + +# --- setattr/delattr-driven right-op updates keep the left op intact --- +class C13: + def __add__(self, o): + return "C13.add" + + +c13 = C13() +assert c13 + 1 == "C13.add" +C13.__radd__ = lambda self, o: "C13.radd" +assert c13 + 1 == "C13.add" +assert 1 + c13 == "C13.radd" +del C13.__radd__ +assert c13 + 1 == "C13.add" +with assert_raises(TypeError): + 1 + c13 From 8f4f7a5e5449269c7f229d16bb7f80164d190cb3 Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Tue, 7 Jul 2026 02:13:11 +0900 Subject: [PATCH 65/92] Gate type_cache_after_fork to its fork caller and use Self in downcast type_cache_after_fork is only called from the unix fork path, so gate it with all(feature = "host_env", unix) to match the caller and avoid a dead-code error on non-unix targets. Replace the explicit PyType with Self in the modified_inner downcast. Assisted-by: Claude --- crates/vm/src/builtins/type.rs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/crates/vm/src/builtins/type.rs b/crates/vm/src/builtins/type.rs index 0aeed2db17f..372fa3ba4f8 100644 --- a/crates/vm/src/builtins/type.rs +++ b/crates/vm/src/builtins/type.rs @@ -222,7 +222,8 @@ pub(crate) fn type_cache_clear() { /// /// If fork happens while a writer holds an entry SeqLock, the child inherits /// the odd sequence value with no surviving writer to release it. Clear only -/// those in-progress entries, matching CPython's `_PyTypes_AfterFork()`. +/// those in-progress entries, matching `_PyTypes_AfterFork()`. +#[cfg(all(feature = "host_env", unix))] pub(crate) unsafe fn type_cache_after_fork() { for entry in TYPE_CACHE.iter() { let seq = entry.sequence.load(Ordering::Relaxed); @@ -528,7 +529,7 @@ impl PyType { let subclasses = self.subclasses.read(); for weak_ref in subclasses.iter() { if let Some(sub) = weak_ref.upgrade() { - sub.downcast_ref::().unwrap().modified_inner(); + sub.downcast_ref::().unwrap().modified_inner(); } } self.tp_version_tag.store(0, Ordering::SeqCst); From 5f43c79422b6a8811925d387cc0cdaab1870591b Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Tue, 7 Jul 2026 02:13:19 +0900 Subject: [PATCH 66/92] Unmark test_attr and test_method_call_error in test_monitoring Both TestLoadSuperAttr tests now pass; remove their expectedFailure markers. Assisted-by: Claude --- Lib/test/test_monitoring.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/Lib/test/test_monitoring.py b/Lib/test/test_monitoring.py index 1f72b552c6c..30eee65dc12 100644 --- a/Lib/test/test_monitoring.py +++ b/Lib/test/test_monitoring.py @@ -1984,7 +1984,6 @@ def f(): ] return d["f"], expected - @unittest.expectedFailure # TODO: RUSTPYTHON; line number differences in multi-line super() calls def test_method_call_error(self): nonopt_func, nonopt_expected = self._super_method_call_error(optimized=False) opt_func, opt_expected = self._super_method_call_error(optimized=True) @@ -2022,7 +2021,6 @@ def f(): ] return d["f"], expected - @unittest.expectedFailure # TODO: RUSTPYTHON; line number differences in multi-line super() calls def test_attr(self): nonopt_func, nonopt_expected = self._super_attr(optimized=False) opt_func, opt_expected = self._super_attr(optimized=True) From a3f92432e59367bf1a03c8cc7a02686b5fa8d331 Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Tue, 7 Jul 2026 02:37:43 +0900 Subject: [PATCH 67/92] Run tp_new specialization __init__ without a trampoline frame CallAllocAndEnterInit ran __init__ inside a synthetic init-cleanup shim frame whose code carried the __init__ name. Since the thread frame stack is derived from the frame chain, that shim frame was visible to sys._getframe, f_back walks, traceback construction and inspect.stack / inspect.trace. Its code object has empty co_positions, so a stack walk that reached it raised StopIteration inside inspect._get_code_position, cascading into unrelated failures (test_inspect trace/stack/frame, asyncio source traceback). Call __init__ directly via run_frame, enforce the __init__() should return None contract inline, and drop the shim: the init-cleanup code object, its builder, with_frame_untraced, monitoring_disabled_for_code, and the extra-frame datastack/recursion budget. Removing the second frame per construction also cuts the specialization's per-call cost. Assisted-by: Claude --- crates/vm/src/frame.rs | 120 +++++++++--------------------------- crates/vm/src/vm/context.rs | 49 --------------- crates/vm/src/vm/mod.rs | 23 +------ 3 files changed, 31 insertions(+), 161 deletions(-) diff --git a/crates/vm/src/frame.rs b/crates/vm/src/frame.rs index ac62e41ac8b..f9bc1b992fd 100644 --- a/crates/vm/src/frame.rs +++ b/crates/vm/src/frame.rs @@ -12,10 +12,7 @@ use crate::{ builtin_func::PyNativeFunction, descriptor::{MemberGetter, PyMemberDescriptor, PyMethodDescriptor}, frame::stack_analysis, - function::{ - PyBoundMethod, PyCell, PyCellRef, PyFunction, datastack_frame_size_bytes_for_code, - vectorcall_function, - }, + function::{PyBoundMethod, PyCell, PyCellRef, PyFunction, vectorcall_function}, list::PyListIterator, range::PyRangeIterator, tuple::{PyTuple, PyTupleIterator, PyTupleRef}, @@ -1516,7 +1513,7 @@ pub(crate) fn release_datastack_frame(frame: &Py, vm: &VirtualMachine) { let frame_obj = frame.as_object(); // Uniqueness argument: at this point the frame is already out of // `vm.frames`, the thread-frames registry and the current-frame chain - // (all unlinked inside `with_frame_impl` before it returned), and the + // (all unlinked inside `with_frame` before it returned), and the // frame type has no weakref support. A datastack frame is created // untracked and stays untracked while it runs, so it is in no GC // generation list and no collector can observe or incref it. Hence no @@ -1767,58 +1764,34 @@ impl fmt::Debug for ExecutingFrame<'_> { } impl ExecutingFrame<'_> { - #[inline] - fn monitoring_disabled_for_code(&self, vm: &VirtualMachine) -> bool { - self.code.is(&vm.ctx.init_cleanup_code) - } - - fn specialization_new_init_cleanup_frame(&self, vm: &VirtualMachine) -> FrameRef { - // The shim code has NEWLOCALS, so passing no locals selects - // FrameLocals::lazy() and no locals dict is allocated; the shim - // never touches locals. - Frame::new( - vm.ctx.init_cleanup_code.clone(), - Scope::new(None, self.globals.clone()), - self.builtins.clone(), - &[], - None, - true, - vm, - ) - .into_ref(&vm.ctx) - } - - /// `args` holds the `__init__` args with slot 0 left empty; it is filled - /// with `new_obj` here. - fn specialization_run_init_cleanup_shim( + /// Run `__init__` for the tp_new specialization. `args` holds the + /// `__init__` args with slot 0 left empty; it is filled with `new_obj` + /// here. Enforces the `__init__() should return None` contract and + /// returns the constructed object. + fn specialization_run_init( &self, new_obj: PyObjectRef, init_func: &Py, args: &mut [Option], vm: &VirtualMachine, ) -> PyResult { - let shim = self.specialization_new_init_cleanup_frame(vm); - let shim_result = vm.with_frame_untraced(shim.clone(), |shim| { - shim.with_exec(vm, |mut exec| exec.push_value(new_obj.clone())); - - args[0] = Some(new_obj); - let taken = args - .iter_mut() - .map(|slot| slot.take().expect("arg slot must be filled")); + args[0] = Some(new_obj.clone()); + let taken = args + .iter_mut() + .map(|slot| slot.take().expect("arg slot must be filled")); - let init_frame = init_func.prepare_exact_args_frame(taken, vm); - let init_result = vm.run_frame(init_frame.clone()); - release_datastack_frame(&init_frame, vm); - let init_result = init_result?; + let init_frame = init_func.prepare_exact_args_frame(taken, vm); + let init_result = vm.run_frame(init_frame.clone()); + release_datastack_frame(&init_frame, vm); + let init_result = init_result?; - shim.with_exec(vm, |mut exec| exec.push_value(init_result)); - match shim.run(vm)? { - ExecutionResult::Return(value) => Ok(value), - ExecutionResult::Yield(_) => unreachable!("_Py_InitCleanup shim cannot yield"), - } - }); - release_datastack_frame(&shim, vm); - shim_result + if !vm.is_none(&init_result) { + return Err(vm.new_type_error(format!( + "__init__() should return None, not '{}'", + init_result.class().name() + ))); + } + Ok(new_obj) } #[inline(always)] @@ -3748,17 +3721,6 @@ impl ExecutingFrame<'_> { self.code.instructions.quicken(); atomic::fence(atomic::Ordering::Release); } - if self.monitoring_disabled_for_code(vm) { - let global_ver = vm - .state - .instrumentation_version - .load(atomic::Ordering::Acquire); - monitoring::instrument_code(self.code, 0); - self.code - .instrumentation_version - .store(global_ver, atomic::Ordering::Release); - return Ok(None); - } // Check if bytecode needs re-instrumentation let global_ver = vm .state @@ -5382,23 +5344,14 @@ impl ExecutingFrame<'_> { && init_func.has_exact_argcount(nargs + 1) && let Some(cls_alloc) = cls.slots.alloc.load() { - // Match CPython's `code->co_framesize + _Py_InitCleanup.co_framesize` - // shape, using RustPython's datastack-backed frame size - // equivalent for the extra shim frame. - let init_cleanup_stack_bytes = - datastack_frame_size_bytes_for_code(&vm.ctx.init_cleanup_code) - .expect("_Py_InitCleanup shim is not a generator/coroutine"); - if !self.specialization_has_datastack_space_for_func_with_extra( - vm, - &init_func, - init_cleanup_stack_bytes, - ) { + // The specialization runs `__init__` directly with no + // interpreter-visible trampoline frame. Deopt when the + // datastack or recursion budget for the `__init__` frame is + // unavailable. + if !self.specialization_has_datastack_space_for_func(vm, &init_func) { return self.execute_call_vectorcall(nargs, vm); } - // CPython creates `_Py_InitCleanup` + `__init__` frames here. - // Keep the guard conservative and deopt when the effective - // recursion budget for those two frames is not available. - if self.specialization_call_recursion_guard_with_extra_frames(vm, 1) { + if self.specialization_call_recursion_guard(vm) { return self.execute_call_vectorcall(nargs, vm); } // Allocate object directly (tp_new == object.__new__, tp_alloc == generic). @@ -5406,7 +5359,7 @@ impl ExecutingFrame<'_> { let new_obj = cls_alloc(cls_ref, 0, vm)?; // Stage args as [new_obj, arg1, ..., argN]; slot 0 is - // filled by the shim runner. + // filled by the init runner. let mut arg_buf = CallArgBuffer::new(nargs as usize + 1); let args = arg_buf.slots(); for (slot, arg) in args[1..].iter_mut().zip(self.pop_multiple(nargs as usize)) { @@ -5414,8 +5367,7 @@ impl ExecutingFrame<'_> { } let _null = self.pop_value_opt(); // self_or_null (None) let _callable = self.pop_value(); // callable (type) - let result = - self.specialization_run_init_cleanup_shim(new_obj, &init_func, args, vm)?; + let result = self.specialization_run_init(new_obj, &init_func, args, vm)?; self.push_value(result); return Ok(None); } @@ -6202,18 +6154,6 @@ impl ExecutingFrame<'_> { instruction.is_instrumented(), "execute_instrumented called with non-instrumented opcode {instruction:?}" ); - if self.monitoring_disabled_for_code(vm) { - let global_ver = vm - .state - .instrumentation_version - .load(atomic::Ordering::Acquire); - monitoring::instrument_code(self.code, 0); - self.code - .instrumentation_version - .store(global_ver, atomic::Ordering::Release); - self.update_lasti(|i| *i -= 1); - return Ok(None); - } self.monitoring_mask = vm.state.monitoring_events.load(); match instruction { Instruction::InstrumentedResume => { diff --git a/crates/vm/src/vm/context.rs b/crates/vm/src/vm/context.rs index a64d8410b61..9a545663576 100644 --- a/crates/vm/src/vm/context.rs +++ b/crates/vm/src/vm/context.rs @@ -14,7 +14,6 @@ use crate::{ object, pystr, type_::PyAttributes, }, - bytecode::{self, CodeFlags, CodeUnit, Instruction, Opcode}, class::StaticType, common::rc::PyRc, exceptions, @@ -31,7 +30,6 @@ use malachite_bigint::BigInt; use num_complex::Complex64; use num_traits::ToPrimitive; use rustpython_common::lock::PyRwLock; -use rustpython_compiler_core::{OneIndexed, SourceLocation}; #[derive(Debug)] pub struct Context { @@ -52,7 +50,6 @@ pub struct Context { pub int_cache_pool: Vec, pub(crate) latin1_char_cache: Vec>, pub(crate) ascii_char_cache: Vec>, - pub(crate) init_cleanup_code: PyRef, // there should only be exact objects of str in here, no non-str objects and no subclasses pub(crate) string_pool: StringPool, pub(crate) slot_new_wrapper: PyMethodDef, @@ -362,8 +359,6 @@ impl Context { PyMethodFlags::METHOD, None, ); - let init_cleanup_code = Self::new_init_cleanup_code(&types, &names); - let empty_str = unsafe { string_pool.intern("", types.str_type.to_owned()) }; let empty_bytes = create_object(PyBytes::from(Vec::new()), types.bytes_type); @@ -389,7 +384,6 @@ impl Context { int_cache_pool, latin1_char_cache, ascii_char_cache, - init_cleanup_code, string_pool, slot_new_wrapper, names, @@ -399,49 +393,6 @@ impl Context { } } - fn new_init_cleanup_code(types: &TypeZoo, names: &ConstName) -> PyRef { - let loc = SourceLocation { - line: OneIndexed::MIN, - character_offset: OneIndexed::from_zero_indexed(0), - }; - let instructions = [ - CodeUnit { - op: Instruction::ExitInitCheck, - arg: 0.into(), - }, - CodeUnit { - op: Instruction::ReturnValue, - arg: 0.into(), - }, - CodeUnit { - op: Opcode::Resume.into(), - arg: 0.into(), - }, - ]; - let code = bytecode::CodeObject { - instructions: instructions.into(), - locations: vec![(loc, loc); instructions.len()].into_boxed_slice(), - flags: CodeFlags::OPTIMIZED | CodeFlags::NEWLOCALS, - posonlyarg_count: 0, - arg_count: 0, - kwonlyarg_count: 0, - source_path: names.__init__, - first_line_number: None, - max_stackdepth: 2, - obj_name: names.__init__, - qualname: names.__init__, - constants: core::iter::empty().collect(), - names: Vec::new().into_boxed_slice(), - varnames: Vec::new().into_boxed_slice(), - cellvars: Vec::new().into_boxed_slice(), - freevars: Vec::new().into_boxed_slice(), - localspluskinds: Vec::new().into_boxed_slice(), - linetable: Vec::new().into_boxed_slice(), - exceptiontable: Vec::new().into_boxed_slice(), - }; - PyRef::new_ref(PyCode::new(code), types.code_type.to_owned(), None) - } - pub fn intern_str(&self, s: S) -> &'static PyStrInterned { unsafe { self.string_pool.intern(s, self.types.str_type.to_owned()) } } diff --git a/crates/vm/src/vm/mod.rs b/crates/vm/src/vm/mod.rs index 06f67313af2..dfae400b116 100644 --- a/crates/vm/src/vm/mod.rs +++ b/crates/vm/src/vm/mod.rs @@ -1679,23 +1679,6 @@ impl VirtualMachine { &self, frame: FrameRef, f: F, - ) -> PyResult { - self.with_frame_impl(frame, true, f) - } - - pub(crate) fn with_frame_untraced PyResult>( - &self, - frame: FrameRef, - f: F, - ) -> PyResult { - self.with_frame_impl(frame, false, f) - } - - fn with_frame_impl PyResult>( - &self, - frame: FrameRef, - traced: bool, - f: F, ) -> PyResult { self.with_recursion("", || { // SAFETY: `frame` (FrameRef) stays alive for the entire closure scope, @@ -1733,11 +1716,7 @@ impl VirtualMachine { crate::vm::thread::pop_thread_frame(); } - if traced { - self.dispatch_traced_frame(&frame, |frame| f(frame.to_owned())) - } else { - f(frame.to_owned()) - } + self.dispatch_traced_frame(&frame, |frame| f(frame.to_owned())) }) } From df702842532b3856a8fd2c94ec1bd9e620073e45 Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Tue, 7 Jul 2026 03:51:20 +0900 Subject: [PATCH 68/92] Replace per-call thread-frame mutex with an atomic top-of-stack (unix) On unix threading builds, publish each thread's top Python frame in a single relaxed AtomicPtr store from set_current_frame instead of pushing onto a parking_lot::Mutex> per call. Cross-thread readers (sys._current_frames, cross-thread f_back, faulthandler.dump_traceback, the GC unreachable debug-assert) run under stop-the-world and walk the published top frame down the Frame::previous chain; the owning thread is then parked at a safepoint, so the pointer and the frames it reaches are quiescent and alive. The faulthandler watchdog is a plain OS thread that cannot stop-the-world, so it walks the chain lock-free and best-effort. Non-unix threading builds have no stop-the-world and keep the existing mutex-guarded frame stack unchanged. Assisted-by: Claude --- crates/stdlib/src/faulthandler.rs | 84 +++++++++++++++++++++++++++++-- crates/vm/src/builtins/frame.rs | 31 +++++++++++- crates/vm/src/gc_state.rs | 10 +++- crates/vm/src/stdlib/_thread.rs | 52 ++++++++++++++----- crates/vm/src/vm/mod.rs | 22 +++++--- crates/vm/src/vm/thread.rs | 82 ++++++++++++++++++++++++------ 6 files changed, 242 insertions(+), 39 deletions(-) diff --git a/crates/stdlib/src/faulthandler.rs b/crates/stdlib/src/faulthandler.rs index 35c6f07fad0..cb559eb7a9d 100644 --- a/crates/stdlib/src/faulthandler.rs +++ b/crates/stdlib/src/faulthandler.rs @@ -241,7 +241,7 @@ mod decl { /// Dump traceback for a thread given its frame stack (for cross-thread dumping). /// # Safety /// Each `FramePtr` must point to a live frame (caller holds the Mutex). - #[cfg(all(any(unix, windows), feature = "threading"))] + #[cfg(all(windows, feature = "threading"))] fn dump_traceback_thread_frames( fd: i32, thread_id: u64, @@ -260,6 +260,35 @@ mod decl { } } + /// Dump a thread's traceback by walking its published top frame down the + /// `previous` chain (most recent first). Signal-safe: only atomic pointer + /// loads, no locks. Callers guarantee frame liveness — under stop-the-world + /// for `faulthandler.dump_traceback`, or best-effort for the watchdog (like + /// `_Py_DumpTracebackThreads`, which walks lock-free while other threads + /// may still run). + #[cfg(all(unix, feature = "threading"))] + fn dump_traceback_thread_chain(fd: i32, thread_id: u64, is_current: bool, top: *const Frame) { + const MAX_FRAME_DEPTH: usize = 100; + write_thread_id(fd, thread_id, is_current); + + if top.is_null() { + puts(fd, " \n"); + return; + } + let mut frame_ptr = top; + let mut depth = 0; + while !frame_ptr.is_null() && depth < MAX_FRAME_DEPTH { + // SAFETY: the frame is alive per the caller's liveness guarantee. + let frame = unsafe { &*frame_ptr }; + dump_frame_from_raw(fd, frame); + frame_ptr = frame.previous_frame(); + depth += 1; + } + if depth >= MAX_FRAME_DEPTH && !frame_ptr.is_null() { + puts(fd, " ...\n"); + } + } + #[derive(FromArgs)] struct DumpTracebackArgs { #[pyarg(any, default)] @@ -296,7 +325,43 @@ mod decl { #[cfg(any(unix, windows))] fn dump_all_threads(fd: i32, vm: &VirtualMachine) { // Get all threads' frame stacks from the shared registry - #[cfg(feature = "threading")] + // unix: stop-the-world so every other thread is parked at a safepoint + // and its frame chain is quiescent and alive while we walk it (matches + // faulthandler.dump_traceback running with the GIL held in CPython). + #[cfg(all(unix, feature = "threading"))] + { + use core::sync::atomic::Ordering; + let current_tid = rustpython_vm::stdlib::_thread::get_ident(); + vm.state.stop_the_world.stop_the_world(vm); + { + let registry = vm.state.thread_frames.lock(); + #[expect( + clippy::iter_over_hash_type, + reason = "Iteration order doesn't matter here" + )] + for (&tid, slot) in registry.iter() { + if tid == current_tid { + continue; + } + let top = slot.top_frame.load(Ordering::Relaxed) as *const Frame; + dump_traceback_thread_chain(fd, tid, false, top); + puts(fd, "\n"); + } + } + vm.state.stop_the_world.start_the_world(vm); + + // Now dump current thread from its live frame chain. + write_thread_id(fd, current_tid, true); + if crate::vm::vm::thread::get_current_frame().is_null() { + puts(fd, " \n"); + } else { + crate::vm::frame::for_each_current_frame(|frame| { + dump_frame_from_ref(fd, frame); + }); + } + } + + #[cfg(all(not(unix), feature = "threading"))] { let current_tid = rustpython_vm::stdlib::_thread::get_ident(); let registry = vm.state.thread_frames.lock(); @@ -654,7 +719,20 @@ mod decl { // Use thread frame slots when threading is enabled (includes all threads). // Fall back to live frame walking for non-threaded builds. cfg_select! { - feature = "threading" => { + all(unix, feature = "threading") => { + // The watchdog is a plain OS thread, not attached to + // the VM, so it cannot stop-the-world. Walk each + // published top frame lock-free and best-effort, like + // CPython's faulthandler watchdog. + for (tid, slot) in &thread_frame_slots { + let top = slot + .top_frame + .load(core::sync::atomic::Ordering::Relaxed) + as *const Frame; + dump_traceback_thread_chain(fd, *tid, false, top); + } + } + all(not(unix), feature = "threading") => { for (tid, slot) in &thread_frame_slots { let frames = slot.frames.lock(); dump_traceback_thread_frames(fd, *tid, false, &frames); diff --git a/crates/vm/src/builtins/frame.rs b/crates/vm/src/builtins/frame.rs index 8068f4b807a..158ee85f5f2 100644 --- a/crates/vm/src/builtins/frame.rs +++ b/crates/vm/src/builtins/frame.rs @@ -728,7 +728,36 @@ impl Py { return Some(frame); } - #[cfg(feature = "threading")] + // The caller lives on another thread. unix: park every thread under + // stop-the-world so their frame chains are quiescent and alive, then + // walk each published top frame down its `previous` chain looking for + // the caller. Request stop-the-world before the registry lock. + #[cfg(all(unix, feature = "threading"))] + { + use core::sync::atomic::Ordering; + vm.state.stop_the_world.stop_the_world(vm); + scopeguard::defer! { vm.state.stop_the_world.start_the_world(vm); } + let registry = vm.state.thread_frames.lock(); + #[expect( + clippy::iter_over_hash_type, + reason = "Iteration order doesn't matter here" + )] + for slot in registry.values() { + let mut cur = slot.top_frame.load(Ordering::Relaxed) as *const Frame; + while !cur.is_null() { + if core::ptr::eq(cur, previous) { + // SAFETY: world stopped -> this frame is alive on its + // owning thread's parked call stack. + let f = unsafe { &*Self::from_payload_ptr(cur) }; + return Some(f.to_owned()); + } + // SAFETY: chain frames on a parked thread are alive. + cur = unsafe { (*cur).previous_frame() }; + } + } + } + + #[cfg(all(not(unix), feature = "threading"))] { let registry = vm.state.thread_frames.lock(); #[expect( diff --git a/crates/vm/src/gc_state.rs b/crates/vm/src/gc_state.rs index 6b71ba7c7dc..670f3dd2741 100644 --- a/crates/vm/src/gc_state.rs +++ b/crates/vm/src/gc_state.rs @@ -641,15 +641,21 @@ impl GcState { reason = "assertion over every registered thread slot" )] for slot in registry.values() { - for fp in slot.frames.lock().iter() { + let mut cur = slot + .top_frame + .load(core::sync::atomic::Ordering::Relaxed) + as *const crate::frame::Frame; + while !cur.is_null() { // SAFETY: frames on a thread's active call stack are // alive, and the world is stopped so none can be popped. - let obj = unsafe { fp.as_ref() }.as_object(); + let obj = unsafe { &*crate::Py::::from_payload_ptr(cur) } + .as_object(); let ptr = GcPtr(NonNull::from(obj)); debug_assert!( !unreachable_set.contains(&ptr), "running frame {obj:p} classified unreachable during GC" ); + cur = unsafe { (*cur).previous_frame() }; } } }); diff --git a/crates/vm/src/stdlib/_thread.rs b/crates/vm/src/stdlib/_thread.rs index 73c044cbb9f..99f9b1787ed 100644 --- a/crates/vm/src/stdlib/_thread.rs +++ b/crates/vm/src/stdlib/_thread.rs @@ -1053,18 +1053,46 @@ pub(crate) mod _thread { /// Get all threads' current (top) frames. Used by sys._current_frames(). pub(crate) fn get_all_current_frames(vm: &VirtualMachine) -> Vec<(u64, FrameRef)> { - let registry = vm.state.thread_frames.lock(); - registry - .iter() - .filter_map(|(id, slot)| { - let frames = slot.frames.lock(); - // SAFETY: the owning thread can't pop while we hold the Mutex, - // so the FramePtr is valid for the duration of the lock. - frames - .last() - .map(|fp| (*id, unsafe { fp.as_ref() }.to_owned())) - }) - .collect() + // unix: read each thread's published top frame under stop-the-world so + // the owning thread is parked at a safepoint and cannot pop or free the + // frame while we take a strong reference. Request stop-the-world before + // the registry lock to avoid deadlocking a thread parking mid-registry. + #[cfg(unix)] + { + use core::sync::atomic::Ordering; + vm.state.stop_the_world.stop_the_world(vm); + scopeguard::defer! { vm.state.stop_the_world.start_the_world(vm); } + let registry = vm.state.thread_frames.lock(); + registry + .iter() + .filter_map(|(id, slot)| { + let top = slot.top_frame.load(Ordering::Relaxed); + core::ptr::NonNull::new(top).map(|p| { + // SAFETY: world stopped -> the owning thread is parked + // and cannot pop or free this frame; it is alive on + // that thread's call stack. + let py = + unsafe { &*Py::::from_payload_ptr(p.as_ptr()) }; + (*id, py.to_owned()) + }) + }) + .collect() + } + #[cfg(not(unix))] + { + let registry = vm.state.thread_frames.lock(); + registry + .iter() + .filter_map(|(id, slot)| { + let frames = slot.frames.lock(); + // SAFETY: the owning thread can't pop while we hold the Mutex, + // so the FramePtr is valid for the duration of the lock. + frames + .last() + .map(|fp| (*id, unsafe { fp.as_ref() }.to_owned())) + }) + .collect() + } } /// Called after fork() in child process to mark all other threads as done. diff --git a/crates/vm/src/vm/mod.rs b/crates/vm/src/vm/mod.rs index dfae400b116..b765fec1338 100644 --- a/crates/vm/src/vm/mod.rs +++ b/crates/vm/src/vm/mod.rs @@ -46,9 +46,10 @@ use crate::{ use alloc::{borrow::Cow, collections::BTreeMap}; #[cfg(all(unix, feature = "threading"))] use core::sync::atomic::AtomicI64; +#[cfg(all(not(unix), feature = "threading"))] +use core::ptr::NonNull; use core::{ cell::{Cell, OnceCell, RefCell}, - ptr::NonNull, sync::atomic::{AtomicBool, AtomicU64, Ordering}, }; use crossbeam_utils::atomic::AtomicCell; @@ -107,11 +108,15 @@ pub struct VirtualMachine { pub(crate) audit_hooks: RefCell>, } -/// Non-owning frame pointer for the frames stack. +/// Non-owning frame pointer for the non-unix threading frames stack. /// The pointed-to frame is kept alive by the caller of with_frame/resume_gen_frame. +/// Unix threading builds publish the top frame through `ThreadSlot::top_frame` +/// and walk the rest via `Frame::previous`, so they do not use this type. +#[cfg(all(not(unix), feature = "threading"))] #[derive(Copy, Clone)] pub struct FramePtr(NonNull>); +#[cfg(all(not(unix), feature = "threading"))] impl FramePtr { /// # Safety /// The pointed-to frame must still be alive. @@ -124,6 +129,7 @@ impl FramePtr { // SAFETY: FramePtr is only stored in a thread's shared frame stack // (`ThreadSlot::frames`) while the corresponding FrameRef is alive on that // thread's call stack; readers dereference it under the slot mutex. +#[cfg(all(not(unix), feature = "threading"))] unsafe impl Send for FramePtr {} #[derive(Debug)] @@ -1685,7 +1691,9 @@ impl VirtualMachine { // keeping the FramePtr valid. We pass a clone to `f` so that `f` // consuming its FrameRef doesn't invalidate our pointer. // Publish the frame for sys._current_frames() and faulthandler. - #[cfg(feature = "threading")] + // On unix, set_current_frame below publishes the top frame into the + // thread slot; only non-unix builds maintain the mutex-guarded Vec. + #[cfg(all(not(unix), feature = "threading"))] crate::vm::thread::push_thread_frame(FramePtr(NonNull::from(&*frame))); // Link frame into the signal-safe frame chain (previous pointer). // This chain is the single source for the current thread's frame @@ -1707,12 +1715,12 @@ impl VirtualMachine { core::sync::atomic::Ordering::AcqRel, ); - // Ensure cleanup on panic: restore owner, exc_info, frame chain, and frames Vec. + // Ensure cleanup on panic: restore owner, exc_info, and frame chain. scopeguard::defer! { frame.owner.store(old_owner, core::sync::atomic::Ordering::Release); self.restore_exception(saved_exc); crate::vm::thread::set_current_frame(old_frame); - #[cfg(feature = "threading")] + #[cfg(all(not(unix), feature = "threading"))] crate::vm::thread::pop_thread_frame(); } @@ -1736,7 +1744,7 @@ impl VirtualMachine { self.recursion_depth.update(|d| d + 1); // SAFETY: frame (&FrameRef) stays alive for the duration, so NonNull is valid until pop. - #[cfg(feature = "threading")] + #[cfg(all(not(unix), feature = "threading"))] crate::vm::thread::push_thread_frame(FramePtr(NonNull::from(&**frame))); let old_frame = crate::vm::thread::set_current_frame((&***frame) as *const Frame); frame.previous.store( @@ -1758,7 +1766,7 @@ impl VirtualMachine { frame.owner.store(old_owner, core::sync::atomic::Ordering::Release); self.pop_exception(); crate::vm::thread::set_current_frame(old_frame); - #[cfg(feature = "threading")] + #[cfg(all(not(unix), feature = "threading"))] crate::vm::thread::pop_thread_frame(); self.recursion_depth.update(|d| d - 1); diff --git a/crates/vm/src/vm/thread.rs b/crates/vm/src/vm/thread.rs index eabfbc043ef..009de4f4da8 100644 --- a/crates/vm/src/vm/thread.rs +++ b/crates/vm/src/vm/thread.rs @@ -1,4 +1,4 @@ -#[cfg(feature = "threading")] +#[cfg(all(not(unix), feature = "threading"))] use super::FramePtr; #[cfg(feature = "threading")] use crate::builtins::PyBaseExceptionRef; @@ -30,8 +30,19 @@ pub const THREAD_SUSPENDED: i32 = 2; /// The exception field uses atomic operations for lock-free cross-thread reads. #[cfg(feature = "threading")] pub struct ThreadSlot { + /// Top of the owning thread's Python call stack, published for + /// cross-thread readers (`sys._current_frames`, cross-thread `f_back`). + /// The rest of the stack is reachable via each frame's `previous` pointer. + /// Written lock-free on the hot push/pop path with relaxed ordering; every + /// cross-thread read runs under stop-the-world, which parks the owning + /// thread at a safepoint and supplies the happens-before edge, so the + /// pointer and the frames it reaches are quiescent and alive at read time. + #[cfg(unix)] + pub top_frame: AtomicPtr, /// Raw frame pointers, valid while the owning thread's call stack is active. /// Readers must hold the Mutex and convert to FrameRef inside the lock. + /// Used on non-unix threading builds, which have no stop-the-world. + #[cfg(not(unix))] pub frames: parking_lot::Mutex>, pub exception: crate::PyAtomicRef>, /// Thread state for stop-the-world: DETACHED / ATTACHED / SUSPENDED @@ -84,6 +95,15 @@ thread_local! { pub(crate) static CURRENT_FRAME: AtomicPtr = const { AtomicPtr::new(core::ptr::null_mut()) }; + /// Cached pointer to this thread's `ThreadSlot::top_frame`, so the hot + /// push/pop path can publish the top frame with a single relaxed store and + /// no `CURRENT_THREAD_SLOT` RefCell borrow. Null until the slot is + /// initialized; the `Arc` in `CURRENT_THREAD_SLOT` keeps the + /// pointee alive until `cleanup_current_thread_frames` clears this. + #[cfg(all(unix, feature = "threading"))] + static CURRENT_TOP_FRAME_SLOT: Cell<*const AtomicPtr> = + const { Cell::new(core::ptr::null()) }; + } #[must_use] @@ -270,6 +290,9 @@ fn init_thread_slot_if_needed(vm: &VirtualMachine) { let thread_id = crate::stdlib::_thread::get_ident(); let mut registry = vm.state.thread_frames.lock(); let new_slot = Arc::new(ThreadSlot { + #[cfg(unix)] + top_frame: AtomicPtr::new(core::ptr::null_mut()), + #[cfg(not(unix))] frames: parking_lot::Mutex::new(Vec::new()), exception: crate::PyAtomicRef::from(None::), #[cfg(unix)] @@ -290,6 +313,8 @@ fn init_thread_slot_if_needed(vm: &VirtualMachine) { }); registry.insert(thread_id, new_slot.clone()); drop(registry); + #[cfg(all(unix, feature = "threading"))] + CURRENT_TOP_FRAME_SLOT.with(|c| c.set(&new_slot.top_frame)); *slot.borrow_mut() = Some(new_slot); } }); @@ -568,7 +593,10 @@ pub(crate) fn debug_assert_current_thread_attached() { /// Push a frame pointer onto the current thread's shared frame stack. /// The pointed-to frame must remain alive until the matching pop. -#[cfg(feature = "threading")] +/// +/// Only used on non-unix threading builds; unix builds publish the top frame +/// through `set_current_frame` writing `ThreadSlot::top_frame`. +#[cfg(all(not(unix), feature = "threading"))] pub fn push_thread_frame(fp: FramePtr) { CURRENT_THREAD_SLOT.with(|slot| { if let Some(s) = slot.borrow().as_ref() { @@ -584,7 +612,7 @@ pub fn push_thread_frame(fp: FramePtr) { /// Pop a frame from the current thread's shared frame stack. /// Called when a frame is exited. -#[cfg(feature = "threading")] +#[cfg(all(not(unix), feature = "threading"))] pub fn pop_thread_frame() { CURRENT_THREAD_SLOT.with(|slot| { if let Some(s) = slot.borrow().as_ref() { @@ -601,6 +629,17 @@ pub fn pop_thread_frame() { /// Set the current thread's top frame pointer for signal-safe traceback walking. /// Returns the previous frame pointer so it can be restored on pop. pub fn set_current_frame(frame: *const Frame) -> *const Frame { + // Publish the top frame for cross-thread readers. The relaxed store is + // ordered by stop-the-world at read time (see `ThreadSlot::top_frame`). + #[cfg(all(unix, feature = "threading"))] + { + let slot_top = CURRENT_TOP_FRAME_SLOT.with(Cell::get); + if !slot_top.is_null() { + // SAFETY: points to this thread's `ThreadSlot::top_frame`, kept + // alive by the Arc in `CURRENT_THREAD_SLOT` for the thread's life. + unsafe { (*slot_top).store(frame as *mut Frame, Ordering::Relaxed) }; + } + } CURRENT_FRAME.with(|c| c.swap(frame as *mut Frame, Ordering::Relaxed) as *const Frame) } @@ -675,6 +714,10 @@ pub fn cleanup_current_thread_frames(vm: &VirtualMachine) { // Unblock requester countdown progress. vm.state.stop_the_world.notify_thread_gone(); } + // Clear the cached top-frame pointer before dropping the slot Arc so no + // later `set_current_frame` dereferences freed slot memory. + #[cfg(all(unix, feature = "threading"))] + CURRENT_TOP_FRAME_SLOT.with(|c| c.set(core::ptr::null())); CURRENT_THREAD_SLOT.with(|s| { *s.borrow_mut() = None; }); @@ -689,18 +732,27 @@ pub fn cleanup_current_thread_frames(vm: &VirtualMachine) { #[cfg(feature = "threading")] pub fn reinit_frame_slot_after_fork(vm: &VirtualMachine) { let current_ident = crate::stdlib::_thread::get_ident(); - // Rebuild the shared frame stack (bottom-to-top) from the current thread's - // frame chain, which walks top-to-bottom via `previous`. - let mut current_frames: Vec = Vec::new(); - let mut cur = get_current_frame(); - while !cur.is_null() { - // SAFETY: the forking thread's chain frames are alive. - let py = unsafe { crate::Py::::from_payload_ptr(cur) }; - current_frames.push(FramePtr(unsafe { NonNull::new_unchecked(py as *mut _) })); - cur = unsafe { (*cur).previous_frame() }; - } - current_frames.reverse(); + // On non-unix, rebuild the shared frame stack (bottom-to-top) from the + // current thread's frame chain, which walks top-to-bottom via `previous`. + #[cfg(not(unix))] + let current_frames: Vec = { + let mut current_frames = Vec::new(); + let mut cur = get_current_frame(); + while !cur.is_null() { + // SAFETY: the forking thread's chain frames are alive. + let py = unsafe { crate::Py::::from_payload_ptr(cur) }; + current_frames.push(FramePtr(unsafe { NonNull::new_unchecked(py as *mut _) })); + cur = unsafe { (*cur).previous_frame() }; + } + current_frames.reverse(); + current_frames + }; let new_slot = Arc::new(ThreadSlot { + // The surviving child thread keeps executing its current frame chain, + // whose top is the signal-safe `get_current_frame()`. + #[cfg(unix)] + top_frame: AtomicPtr::new(get_current_frame() as *mut Frame), + #[cfg(not(unix))] frames: parking_lot::Mutex::new(current_frames), exception: crate::PyAtomicRef::from(vm.topmost_exception()), #[cfg(unix)] @@ -711,6 +763,8 @@ pub fn reinit_frame_slot_after_fork(vm: &VirtualMachine) { thread: std::thread::current(), qsbr: crate::object::qsbr::QSBR.register(), }); + #[cfg(all(unix, feature = "threading"))] + CURRENT_TOP_FRAME_SLOT.with(|c| c.set(&new_slot.top_frame)); // Lock is safe: reinit_locks_after_fork() already reset it to unlocked. let mut registry = vm.state.thread_frames.lock(); From f525b380de794dc9b709c1c79f19646f34cf6a13 Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Tue, 7 Jul 2026 03:51:42 +0900 Subject: [PATCH 69/92] Skip exc_info save/restore for callees that never touch the slot with_frame saves and restores the shared exc_info slot around every Python call to contain frames that leave it unbalanced. Cache a has_exc_handling bit on PyCode at creation, set when the bytecode contains any opcode that calls vm.set_exception (PushExcInfo, PopExcept, CheckEgMatch, EndAsyncFor, InstrumentedEndAsyncFor). A callee whose code has none of these cannot mutate the slot, so the save and restore are skipped for it. Generators go through resume_gen_frame and are unaffected. Assisted-by: Claude --- crates/vm/src/builtins/code.rs | 19 +++++++++++++++++++ crates/vm/src/vm/mod.rs | 13 +++++++++++-- 2 files changed, 30 insertions(+), 2 deletions(-) diff --git a/crates/vm/src/builtins/code.rs b/crates/vm/src/builtins/code.rs index e30813bf87d..b1132a55a20 100644 --- a/crates/vm/src/builtins/code.rs +++ b/crates/vm/src/builtins/code.rs @@ -471,6 +471,11 @@ pub struct PyCode { pub monitoring_data: PyMutex>, /// Whether adaptive counters have been initialized (lazy quickening). pub quickened: core::sync::atomic::AtomicBool, + /// Whether the bytecode contains any instruction that mutates the current + /// exc_info slot (`vm.set_exception`). When false, a normal frame call for + /// this code cannot leave the slot unbalanced, so `with_frame` skips the + /// exc_info save/restore. Computed once by scanning the instruction stream. + pub has_exc_handling: bool, } impl Deref for PyCode { @@ -483,12 +488,26 @@ impl Deref for PyCode { impl PyCode { pub fn new(code: CodeObject) -> Self { let sp = code.source_path as *const PyStrInterned as *mut PyStrInterned; + // The only opcodes that call `vm.set_exception` (mutating the shared + // exc_info slot); instrumented variants only replace these base opcodes + // in place, so scanning the freshly-built stream is a sound predicate. + let has_exc_handling = code.instructions.iter().any(|u| { + matches!( + u.op, + Instruction::PushExcInfo + | Instruction::PopExcept + | Instruction::CheckEgMatch + | Instruction::EndAsyncFor + | Instruction::InstrumentedEndAsyncFor + ) + }); Self { code, source_path: AtomicPtr::new(sp), instrumentation_version: AtomicU64::new(0), monitoring_data: PyMutex::new(None), quickened: core::sync::atomic::AtomicBool::new(false), + has_exc_handling, } } diff --git a/crates/vm/src/vm/mod.rs b/crates/vm/src/vm/mod.rs index b765fec1338..becfa99b08b 100644 --- a/crates/vm/src/vm/mod.rs +++ b/crates/vm/src/vm/mod.rs @@ -1709,7 +1709,14 @@ impl VirtualMachine { // exc_info pollution from frames with unbalanced // PUSH_EXC_INFO/POP_EXCEPT (e.g., exception escaping an except block // whose cleanup entry is missing from the exception table). - let saved_exc = self.current_exception(); + // A callee whose bytecode never mutates the slot cannot pollute it, + // so the save/restore is skipped for it. + let save_exc = frame.code.has_exc_handling; + let saved_exc = if save_exc { + self.current_exception() + } else { + None + }; let old_owner = frame.owner.swap( crate::frame::FrameOwner::Thread as i8, core::sync::atomic::Ordering::AcqRel, @@ -1718,7 +1725,9 @@ impl VirtualMachine { // Ensure cleanup on panic: restore owner, exc_info, and frame chain. scopeguard::defer! { frame.owner.store(old_owner, core::sync::atomic::Ordering::Release); - self.restore_exception(saved_exc); + if save_exc { + self.restore_exception(saved_exc); + } crate::vm::thread::set_current_frame(old_frame); #[cfg(all(not(unix), feature = "threading"))] crate::vm::thread::pop_thread_frame(); From 98351a58d1fe688b327b2ce287f0f690d249763c Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Tue, 7 Jul 2026 05:15:55 +0900 Subject: [PATCH 70/92] Return a write-through FrameLocalsProxy from frame.f_locals Optimized (function) frames now expose `f_locals` as a `FrameLocalsProxy` implementing PEP 667 semantics instead of a cached snapshot dict: - reads go live through the fast-local slots; each access mints a fresh proxy; keys that do not name a fast local are stored in a per-frame `f_extra_locals` side dict and folded into `locals()`. - writes to a fast-local key store into the slot (or its cell) in place; deleting a fast local raises ValueError; extra keys delete normally. - full mapping protocol: keys/values/items (lists), get/pop/setdefault, update (dict or FrameLocalsProxy only), __or__/__ior__/__ror__ (dict result), copy (plain dict), __reduce__ blocks pickling/copy, repr with recursion guard, mapping-pattern and Mapping ABC support. Class/module/exec frames keep returning their namespace mapping directly. Cross-thread access to a frame running on another thread still raises RuntimeError. A closed generator now keeps its frame locals when a durable frame reference escaped (f_locals proxy, sys._getframe, f_back), matching take_ownership; the escape is tracked with a per-frame flag. The snapshot-then-fold locals_to_fast/locals_dirty write-back is retired since proxy writes reach the slots directly. Assisted-by: Claude --- Lib/test/test_frame.py | 10 - Lib/test/test_generators.py | 1 - Lib/test/test_pdb.py | 2 +- crates/vm/src/builtins/frame.rs | 29 +- crates/vm/src/builtins/frame_locals_proxy.rs | 315 ++++++++++++++++ crates/vm/src/builtins/mod.rs | 2 + crates/vm/src/coroutine.rs | 20 +- crates/vm/src/frame.rs | 366 +++++++++++++++---- crates/vm/src/stdlib/sys.rs | 1 + crates/vm/src/types/zoo.rs | 6 +- 10 files changed, 645 insertions(+), 107 deletions(-) create mode 100644 crates/vm/src/builtins/frame_locals_proxy.rs diff --git a/Lib/test/test_frame.py b/Lib/test/test_frame.py index ae02e2a59f9..53d42a595b7 100644 --- a/Lib/test/test_frame.py +++ b/Lib/test/test_frame.py @@ -315,7 +315,6 @@ def inner(): % (file_repr, offset + 5)) class TestFrameLocals(unittest.TestCase): - @unittest.expectedFailure # TODO: RUSTPYTHON def test_scope(self): class A: x = 1 @@ -333,7 +332,6 @@ def f(): self.assertEqual(locals()['y'], 2) f() - @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: 1 != 2 def test_closure(self): x = 1 y = 2 @@ -356,7 +354,6 @@ def test_closure_with_inline_comprehension(self): lst = [locals() for k in [0]] self.assertEqual(lst[0]['k'], 0) - @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: 3 != 4 def test_as_dict(self): x = 1 y = 2 @@ -414,7 +411,6 @@ def test_non_string_key(self): d[1] = 2 self.assertEqual(d[1], 2) - @unittest.expectedFailure # TODO: RUSTPYTHON; UnboundLocalError: local variable 'b' referenced before assignment def test_write_with_hidden(self): def f(): f_locals = [sys._getframe().f_locals for b in [0]][0] @@ -426,7 +422,6 @@ def f(): c = 0 f() - @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: != 'a.b.c' def test_local_objects(self): o = object() k = '.'.join(['a', 'b', 'c']) @@ -457,7 +452,6 @@ def test_repr(self): frame = sys._getframe() self.assertEqual(repr(frame.f_locals), repr(dict(frame.f_locals))) - @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: ValueError not raised def test_delete(self): x = 1 d = sys._getframe().f_locals @@ -501,7 +495,6 @@ def test_sizeof(self): proxy = sys._getframe().f_locals support.check_sizeof(self, proxy, support.calcobjsize("P")) - @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: TypeError not raised def test_unsupport(self): x = 1 d = sys._getframe().f_locals @@ -536,7 +529,6 @@ def __eq__(self, other): return StringSubclass('x'), ImpostorX(), 'x' - @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: dict_keys(['obj', 'x']) != ['obj', 'x', 'proxy'] def test_proxy_key_stringlikes_overwrite(self): def f(obj): x = 1 @@ -559,7 +551,6 @@ def f(obj): self.assertEqual(keys_snapshot, expected_keys) self.assertEqual(proxy_snapshot, expected_dict) - @unittest.expectedFailure # TODO: RUSTPYTHON; UnboundLocalError: local variable 'b' referenced before assignment def test_proxy_key_stringlikes_ftrst_write(self): def f(obj): proxy = sys._getframe().f_locals @@ -587,7 +578,6 @@ class ObjectSubclass: with self.assertRaises(TypeError): proxy[obj] = 0 - @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: 'dict' != 'FrameLocalsProxy' def test_constructor(self): FrameLocalsProxy = type([sys._getframe().f_locals for x in range(1)][0]) diff --git a/Lib/test/test_generators.py b/Lib/test/test_generators.py index 8ede6e22fab..b3826f4229d 100644 --- a/Lib/test/test_generators.py +++ b/Lib/test/test_generators.py @@ -762,7 +762,6 @@ def get_frame(index): self.assertIn('a', frame_locals) self.assertEqual(frame_locals['a'], 42) - @unittest.expectedFailure # TODO: RUSTPYTHON; frame locals don't survive generator deallocation def test_frame_locals_outlive_generator(self): frame_locals1 = None diff --git a/Lib/test/test_pdb.py b/Lib/test/test_pdb.py index 97f084088c5..cd1e88f5475 100644 --- a/Lib/test/test_pdb.py +++ b/Lib/test/test_pdb.py @@ -3122,7 +3122,7 @@ def test_pdb_issue_gh_101673(): ... a = 1 ... import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() - >>> with PdbTestInput([ # TODO: RUSTPYTHON # doctest: +NORMALIZE_WHITESPACE +EXPECTED_FAILURE + >>> with PdbTestInput([ # doctest: +NORMALIZE_WHITESPACE ... '!a = 2', ... 'll', ... 'p a', diff --git a/crates/vm/src/builtins/frame.rs b/crates/vm/src/builtins/frame.rs index 158ee85f5f2..50721eecd11 100644 --- a/crates/vm/src/builtins/frame.rs +++ b/crates/vm/src/builtins/frame.rs @@ -4,7 +4,7 @@ use super::{PyCode, PyDictRef, PyIntRef, PyStrRef}; use crate::{ - Context, Py, PyObjectRef, PyRef, PyResult, VirtualMachine, + Context, Py, PyObjectRef, PyPayload, PyRef, PyResult, VirtualMachine, class::PyClassImpl, frame::{Frame, FrameOwner, FrameRef}, function::PySetterValue, @@ -454,14 +454,6 @@ impl Frame { self.builtins.clone() } - #[pygetset] - fn f_locals(&self, vm: &VirtualMachine) -> PyResult { - let result = self.f_locals_mapping(vm).map(Into::into); - self.locals_dirty - .store(true, core::sync::atomic::Ordering::Release); - result - } - #[pygetset] pub fn f_code(&self) -> PyRef { self.code.clone() @@ -704,10 +696,26 @@ impl Py { // Clear temporary refs self.temporary_refs.lock().clear(); self.f_locals_hidden_overlay.lock().take(); + self.f_extra_locals.lock().take(); Ok(()) } + #[pygetset] + fn f_locals(&self, vm: &VirtualMachine) -> PyResult { + // Optimized (function) frames expose a live write-through + // FrameLocalsProxy; class/module/exec frames expose their namespace + // mapping directly. + if self.code.flags.contains(bytecode::CodeFlags::OPTIMIZED) { + self.check_locals_access(vm)?; + self.mark_escaped(); + let proxy = crate::builtins::FrameLocalsProxy::new(self.to_owned()); + Ok(proxy.into_ref(&vm.ctx).into()) + } else { + self.f_locals_mapping(vm).map(Into::into) + } + } + #[pygetset] fn f_generator(&self) -> Option { self.generator.to_owned() @@ -725,6 +733,7 @@ impl Py { // Look for the caller on the current thread's signal-safe frame chain. // Finding it there proves it is still live on this thread. if let Some(frame) = crate::frame::find_owned_chain_frame(previous) { + frame.mark_escaped(); return Some(frame); } @@ -749,6 +758,7 @@ impl Py { // SAFETY: world stopped -> this frame is alive on its // owning thread's parked call stack. let f = unsafe { &*Self::from_payload_ptr(cur) }; + f.mark_escaped(); return Some(f.to_owned()); } // SAFETY: chain frames on a parked thread are alive. @@ -773,6 +783,7 @@ impl Py { let ptr: *const Frame = &**f; core::ptr::eq(ptr, previous).then(|| f.to_owned()) }) { + frame.mark_escaped(); return Some(frame); } } diff --git a/crates/vm/src/builtins/frame_locals_proxy.rs b/crates/vm/src/builtins/frame_locals_proxy.rs new file mode 100644 index 00000000000..b94a6495263 --- /dev/null +++ b/crates/vm/src/builtins/frame_locals_proxy.rs @@ -0,0 +1,315 @@ +//! The `FrameLocalsProxy` type returned by `frame.f_locals` for optimized +//! (function) frames. Implements PEP 667 write-through semantics on top of the +//! frame's fast-local slots and an extra-locals side dict. + +use super::{PyDict, PyDictRef, PyType}; +use crate::{ + AsObject, Context, Py, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, VirtualMachine, + atomic_func, + class::PyClassImpl, + frame::FrameRef, + function::{FuncArgs, OptionalArg, PyArithmeticValue, PyComparisonValue}, + object::{Traverse, TraverseFn}, + protocol::{PyMappingMethods, PyNumberMethods, PySequenceMethods}, + recursion::ReprGuard, + types::{ + AsMapping, AsNumber, AsSequence, Comparable, Constructor, Iterable, PyComparisonOp, + Representable, + }, +}; +use rustpython_common::lock::LazyLock; +use rustpython_common::wtf8::Wtf8Buf; + +#[pyclass(module = false, name = "FrameLocalsProxy", traverse = "manual")] +#[derive(Debug)] +pub struct FrameLocalsProxy { + frame: FrameRef, +} + +unsafe impl Traverse for FrameLocalsProxy { + fn traverse(&self, tracer_fn: &mut TraverseFn<'_>) { + self.frame.traverse(tracer_fn); + } +} + +impl PyPayload for FrameLocalsProxy { + #[inline] + fn class(ctx: &Context) -> &'static Py { + ctx.types.frame_locals_proxy_type + } +} + +impl FrameLocalsProxy { + pub(crate) fn new(frame: FrameRef) -> Self { + Self { frame } + } + + fn snapshot(&self, vm: &VirtualMachine) -> PyResult { + self.frame.framelocalsproxy_snapshot(vm) + } + + fn keys_vec(&self, vm: &VirtualMachine) -> PyResult> { + Ok(self.snapshot(vm)?.into_iter().map(|(k, _)| k).collect()) + } +} + +impl Constructor for FrameLocalsProxy { + type Args = FuncArgs; + + fn py_new(_cls: &Py, args: Self::Args, vm: &VirtualMachine) -> PyResult { + if !args.kwargs.is_empty() { + return Err(vm.new_type_error("FrameLocalsProxy() takes no keyword arguments")); + } + let mut args = args.args; + if args.len() != 1 { + return Err(vm.new_type_error(format!( + "FrameLocalsProxy expected 1 argument, got {}", + args.len() + ))); + } + let frame: FrameRef = args + .pop() + .unwrap() + .downcast() + .map_err(|_| vm.new_type_error("FrameLocalsProxy expected a frame"))?; + Ok(Self::new(frame)) + } +} + +#[pyclass(with( + Constructor, + AsMapping, + AsSequence, + AsNumber, + Iterable, + Comparable, + Representable +))] +impl FrameLocalsProxy { + fn __getitem__(&self, key: PyObjectRef, vm: &VirtualMachine) -> PyResult { + self.frame.framelocalsproxy_getitem(key, vm) + } + + fn __setitem__(&self, key: PyObjectRef, value: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { + self.frame.framelocalsproxy_setitem(key, value, vm) + } + + fn __delitem__(&self, key: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { + self.frame.framelocalsproxy_delitem(key, vm) + } + + fn __contains__(&self, key: PyObjectRef, vm: &VirtualMachine) -> PyResult { + self.frame.framelocalsproxy_contains(key, vm) + } + + fn __len__(&self, vm: &VirtualMachine) -> PyResult { + Ok(self.snapshot(vm)?.__len__()) + } + + #[pymethod] + fn keys(&self, vm: &VirtualMachine) -> PyResult { + Ok(vm.ctx.new_list(self.keys_vec(vm)?).into()) + } + + #[pymethod] + fn values(&self, vm: &VirtualMachine) -> PyResult { + let values = self.snapshot(vm)?.into_iter().map(|(_, v)| v).collect(); + Ok(vm.ctx.new_list(values).into()) + } + + #[pymethod] + fn items(&self, vm: &VirtualMachine) -> PyResult { + let items = self + .snapshot(vm)? + .into_iter() + .map(|(k, v)| vm.ctx.new_tuple(vec![k, v]).into()) + .collect(); + Ok(vm.ctx.new_list(items).into()) + } + + #[pymethod] + fn get(&self, key: PyObjectRef, default: OptionalArg, vm: &VirtualMachine) -> PyResult { + match self.frame.framelocalsproxy_getitem(key, vm) { + Ok(value) => Ok(value), + Err(e) if e.fast_isinstance(vm.ctx.exceptions.key_error) => { + Ok(default.unwrap_or_none(vm)) + } + Err(e) => Err(e), + } + } + + #[pymethod] + fn pop(&self, key: PyObjectRef, default: OptionalArg, vm: &VirtualMachine) -> PyResult { + self.frame + .framelocalsproxy_pop(key, default.into_option(), vm) + } + + #[pymethod] + fn setdefault( + &self, + key: PyObjectRef, + default: OptionalArg, + vm: &VirtualMachine, + ) -> PyResult { + self.frame + .framelocalsproxy_setdefault(key, default.unwrap_or_none(vm), vm) + } + + #[pymethod] + fn copy(&self, vm: &VirtualMachine) -> PyResult { + Ok(self.snapshot(vm)?.into()) + } + + #[pymethod] + fn update(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { + self.update_from(&other, vm) + } + + fn update_from(&self, other: &PyObject, vm: &VirtualMachine) -> PyResult<()> { + let items: Vec<(PyObjectRef, PyObjectRef)> = + if let Some(dict) = other.downcast_ref::() { + dict.into_iter().collect() + } else if let Some(proxy) = other.downcast_ref::() { + proxy.snapshot(vm)?.into_iter().collect() + } else { + return Err(vm.new_type_error( + "update() argument must be dict or another FrameLocalsProxy", + )); + }; + for (key, value) in items { + self.frame.framelocalsproxy_setitem(key, value, vm)?; + } + Ok(()) + } + + #[pymethod] + fn __reversed__(&self, vm: &VirtualMachine) -> PyResult { + let mut keys = self.keys_vec(vm)?; + keys.reverse(); + Ok(vm.ctx.new_list(keys).into()) + } + + fn __ior__(zelf: PyRef, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + zelf.update_from(&other, vm)?; + Ok(zelf.into()) + } + + fn __or__(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + let base = self.snapshot(vm)?; + vm._or(base.as_object(), &other) + } + + fn __ror__(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + let base = self.snapshot(vm)?; + vm._or(&other, base.as_object()) + } + + #[pymethod] + fn __reduce__(&self, vm: &VirtualMachine) -> PyResult { + Err(vm.new_type_error("cannot pickle 'FrameLocalsProxy' object")) + } + + #[pymethod] + fn __reduce_ex__(&self, _protocol: OptionalArg, vm: &VirtualMachine) -> PyResult { + Err(vm.new_type_error("cannot pickle 'FrameLocalsProxy' object")) + } +} + +impl AsMapping for FrameLocalsProxy { + fn as_mapping() -> &'static PyMappingMethods { + static AS_MAPPING: LazyLock = LazyLock::new(|| PyMappingMethods { + length: atomic_func!(|mapping, vm| FrameLocalsProxy::mapping_downcast(mapping).__len__(vm)), + subscript: atomic_func!(|mapping, needle, vm| { + FrameLocalsProxy::mapping_downcast(mapping).__getitem__(needle.to_owned(), vm) + }), + ass_subscript: atomic_func!(|mapping, needle, value, vm| { + let zelf = FrameLocalsProxy::mapping_downcast(mapping); + match value { + Some(value) => zelf.__setitem__(needle.to_owned(), value, vm), + None => zelf.__delitem__(needle.to_owned(), vm), + } + }), + }); + &AS_MAPPING + } +} + +impl AsSequence for FrameLocalsProxy { + fn as_sequence() -> &'static PySequenceMethods { + static AS_SEQUENCE: LazyLock = LazyLock::new(|| PySequenceMethods { + contains: atomic_func!(|seq, target, vm| { + FrameLocalsProxy::sequence_downcast(seq).__contains__(target.to_owned(), vm) + }), + ..PySequenceMethods::NOT_IMPLEMENTED + }); + &AS_SEQUENCE + } +} + +impl AsNumber for FrameLocalsProxy { + fn as_number() -> &'static PyNumberMethods { + static AS_NUMBER: PyNumberMethods = PyNumberMethods { + or: Some(|a, b, vm| { + if let Some(proxy) = a.downcast_ref::() { + proxy.__or__(b.to_owned(), vm) + } else if let Some(proxy) = b.downcast_ref::() { + proxy.__ror__(a.to_owned(), vm) + } else { + Ok(vm.ctx.not_implemented()) + } + }), + inplace_or: Some(|a, b, vm| { + let proxy = a + .to_owned() + .downcast::() + .map_err(|_| vm.new_type_error("expected FrameLocalsProxy"))?; + FrameLocalsProxy::__ior__(proxy, b.to_owned(), vm) + }), + ..PyNumberMethods::NOT_IMPLEMENTED + }; + &AS_NUMBER + } +} + +impl Iterable for FrameLocalsProxy { + fn iter(zelf: PyRef, vm: &VirtualMachine) -> PyResult { + let keys = vm.ctx.new_list(zelf.keys_vec(vm)?); + keys.as_object().to_owned().get_iter(vm).map(Into::into) + } +} + +impl Comparable for FrameLocalsProxy { + fn cmp( + zelf: &Py, + other: &PyObject, + op: PyComparisonOp, + vm: &VirtualMachine, + ) -> PyResult { + op.eq_only(|| { + let self_dict: PyObjectRef = zelf.snapshot(vm)?.into(); + let other_obj = match other.downcast_ref::() { + Some(proxy) => proxy.snapshot(vm)?.into(), + None => other.to_owned(), + }; + let res = self_dict.rich_compare(other_obj, PyComparisonOp::Eq, vm)?; + PyArithmeticValue::from_object(vm, res) + .map(|o| o.try_to_bool(vm)) + .transpose() + }) + } +} + +impl Representable for FrameLocalsProxy { + fn repr_wtf8(zelf: &Py, vm: &VirtualMachine) -> PyResult { + if let Some(_guard) = ReprGuard::enter(vm, zelf.as_object()) { + let dict = zelf.snapshot(vm)?; + Ok(dict.as_object().repr(vm)?.as_wtf8().to_owned()) + } else { + Ok(Wtf8Buf::from("{...}")) + } + } +} + +pub(crate) fn init(context: &'static Context) { + FrameLocalsProxy::extend_class(context, context.types.frame_locals_proxy_type); +} diff --git a/crates/vm/src/builtins/mod.rs b/crates/vm/src/builtins/mod.rs index eba5af36686..ffc01b00f29 100644 --- a/crates/vm/src/builtins/mod.rs +++ b/crates/vm/src/builtins/mod.rs @@ -28,6 +28,8 @@ pub use filter::PyFilter; pub(crate) mod float; pub use float::PyFloat; pub(crate) mod frame; +pub(crate) mod frame_locals_proxy; +pub use frame_locals_proxy::FrameLocalsProxy; pub(crate) mod function; pub use function::{PyBoundMethod, PyFunction}; pub(crate) mod generator; diff --git a/crates/vm/src/coroutine.rs b/crates/vm/src/coroutine.rs index 34d280acdca..c6586ab3046 100644 --- a/crates/vm/src/coroutine.rs +++ b/crates/vm/src/coroutine.rs @@ -73,6 +73,20 @@ impl Coro { } } + /// Free the finished frame's locals and stack, unless a frame object has + /// escaped (e.g. through an `f_locals` proxy or `sys._getframe`). An + /// escaped frame husk owns its heap-resident locals and must keep them + /// readable after the generator closes. + fn clear_frame_locals_on_close(&self) { + // Keep locals alive if a durable frame reference escaped (e.g. through + // an `f_locals` proxy or `sys._getframe`): that reference now owns the + // heap-resident locals and must keep them readable after close, + // matching `take_ownership`. + if !self.frame.has_escaped() { + self.frame.clear_locals_and_stack(); + } + } + fn maybe_close(&self, res: &PyResult, entered_frame: bool) { if !entered_frame { return; @@ -87,7 +101,7 @@ impl Coro { ); // Completed generators/coroutines should not keep their locals // alive while the wrapper object itself remains referenced. - self.frame.clear_locals_and_stack(); + self.clear_frame_locals_on_close(); } Ok(ExecutionResult::Yield(_)) => {} } @@ -170,7 +184,6 @@ impl Coro { None }; let (result, entered_frame) = self.run_with_context(jen, vm, |f| { - self.frame.locals_to_fast(vm)?; f.resume(value, vm) }); self.finalize_send_result(result, entered_frame, jen, vm) @@ -199,7 +212,6 @@ impl Coro { None }; let (result, entered_frame) = self.run_with_context(jen, vm, |f| { - self.frame.locals_to_fast(vm)?; f.resume(value, vm) }); self.finalize_send_result(result, entered_frame, jen, vm) @@ -259,7 +271,7 @@ impl Coro { self.closed.store(true); // Release frame locals and stack to free references held by the // closed generator, matching gen_send_ex2 with close_on_completion. - self.frame.clear_locals_and_stack(); + self.clear_frame_locals_on_close(); match result { Ok(ExecutionResult::Yield(_)) => { Err(vm.new_runtime_error(format!("{} ignored GeneratorExit", gen_name(jen, vm)))) diff --git a/crates/vm/src/frame.rs b/crates/vm/src/frame.rs index f9bc1b992fd..068dbe1a84d 100644 --- a/crates/vm/src/frame.rs +++ b/crates/vm/src/frame.rs @@ -704,11 +704,16 @@ pub struct InterpreterFrame { /// Used by `frame.clear()` to reject clearing an executing frame, /// even when called from a different thread. pub(crate) owner: atomic::AtomicI8, - /// Set when f_locals is accessed. Cleared after locals_to_fast() sync. - pub(crate) locals_dirty: atomic::AtomicBool, /// Persistent overlay for `frame.f_locals` when hidden locals need a /// snapshot separate from the backing locals mapping. pub(crate) f_locals_hidden_overlay: PyMutex>, + /// Side storage for `f_locals` proxy keys that do not name a fast local. + /// Lazily created on first non-fast-key write. Mirrors `f_extra_locals`. + pub(crate) f_extra_locals: PyMutex>, + /// Set once a durable Python-level reference to this frame is handed out + /// (`f_locals` proxy, `sys._getframe`, `f_back`). A closed generator keeps + /// its locals alive while this is set, mirroring `frame_obj` ownership. + pub(crate) escaped: atomic::AtomicBool, /// Number of stack entries to pop after set_f_lineno returns to the /// execution loop. set_f_lineno cannot pop directly because the /// execution loop holds the state mutex. @@ -876,6 +881,7 @@ unsafe impl Traverse for Frame { iframe.trace.traverse(tracer_fn); iframe.temporary_refs.traverse(tracer_fn); iframe.f_locals_hidden_overlay.traverse(tracer_fn); + iframe.f_extra_locals.traverse(tracer_fn); } fn clear(&mut self, _out: &mut Vec) { @@ -966,8 +972,9 @@ impl Frame { generator: PyAtomicBorrow::new(), previous: AtomicPtr::new(core::ptr::null_mut()), owner: atomic::AtomicI8::new(FrameOwner::FrameObject as i8), - locals_dirty: atomic::AtomicBool::new(false), f_locals_hidden_overlay: PyMutex::new(None), + f_extra_locals: PyMutex::new(None), + escaped: atomic::AtomicBool::new(false), pending_stack_pops: Default::default(), pending_unwind_from_stack: Default::default(), }; @@ -1049,6 +1056,7 @@ impl Frame { *slot = None; } self.f_locals_hidden_overlay.lock().take(); + self.f_extra_locals.lock().take(); } /// Get cell contents by localsplus index. @@ -1095,6 +1103,16 @@ impl Frame { self.previous.load(atomic::Ordering::Relaxed) } + /// Record that a durable Python-level reference to this frame escaped. + pub(crate) fn mark_escaped(&self) { + self.escaped.store(true, atomic::Ordering::Release); + } + + /// Whether a durable reference to this frame has escaped. + pub(crate) fn has_escaped(&self) -> bool { + self.escaped.load(atomic::Ordering::Acquire) + } + pub fn lasti(&self) -> u32 { self.lasti.load(Relaxed) } @@ -1119,37 +1137,6 @@ impl Frame { self.pending_unwind_from_stack.store(val, Relaxed); } - /// Sync locals dict back to fastlocals. Called before generator/coroutine resume - /// to apply any modifications made via f_locals. - pub fn locals_to_fast(&self, vm: &VirtualMachine) -> PyResult<()> { - if !self.locals_dirty.load(atomic::Ordering::Acquire) { - return Ok(()); - } - let code = &**self.code; - let overlay_locals = self - .has_active_hidden_locals() - .then(|| self.f_locals_hidden_overlay.lock().clone()) - .flatten() - .map(ArgMapping::from_dict_exact); - let locals_map = overlay_locals - .as_ref() - .map_or_else(|| self.locals.mapping(vm), ArgMapping::mapping); - // SAFETY: Called before generator resume; no concurrent access. - let fastlocals = unsafe { self.iframe_mut().localsplus.fastlocals_mut() }; - for (i, &varname) in code.varnames.iter().enumerate() { - if i >= fastlocals.len() { - break; - } - match locals_map.subscript(varname, vm) { - Ok(value) => fastlocals[i] = Some(value), - Err(e) if e.fast_isinstance(vm.ctx.exceptions.key_error) => {} - Err(e) => return Err(e), - } - } - self.locals_dirty.store(false, atomic::Ordering::Release); - Ok(()) - } - fn has_active_hidden_locals(&self) -> bool { use rustpython_compiler_core::bytecode::{CO_FAST_CELL, CO_FAST_FREE, CO_FAST_HIDDEN}; let code = &**self.code; @@ -1287,7 +1274,7 @@ impl Frame { /// reader clones them). Access from the executing thread itself (locals() /// builtin, trace callbacks) is fine: the frame sits on the current /// thread's frame chain and is at a bytecode boundary. - fn check_locals_access(&self, vm: &VirtualMachine) -> PyResult<()> { + pub(crate) fn check_locals_access(&self, vm: &VirtualMachine) -> PyResult<()> { let owner = FrameOwner::from_i8(self.owner.load(atomic::Ordering::Acquire)); if owner != FrameOwner::Thread { return Ok(()); @@ -1311,7 +1298,6 @@ impl Frame { return self.locals(vm); } - let needs_refresh = !self.locals_dirty.load(atomic::Ordering::Acquire); let overlay_dict = { let mut overlay = self.f_locals_hidden_overlay.lock(); match overlay.as_ref() { @@ -1323,25 +1309,238 @@ impl Frame { } } }; - if needs_refresh { - PyDict::clear(&overlay_dict); - let overlay = ArgMapping::from_dict_exact(overlay_dict.clone()); - self.sync_visible_locals_to_mapping(overlay.mapping(), vm)?; - } + PyDict::clear(&overlay_dict); + let overlay = ArgMapping::from_dict_exact(overlay_dict.clone()); + self.sync_visible_locals_to_mapping(overlay.mapping(), vm)?; Ok(ArgMapping::from_dict_exact(overlay_dict)) } pub fn locals(&self, vm: &VirtualMachine) -> PyResult { - if self.has_active_hidden_locals() { + let mapping = if self.has_active_hidden_locals() { // Match CPython's locals() behavior for frames with PEP 709 hidden // locals: return a fresh snapshot instead of the backing mapping. let overlay = ArgMapping::from_dict_exact(vm.ctx.new_dict()); self.sync_visible_locals_to_mapping(overlay.mapping(), vm)?; - Ok(overlay) + overlay } else { self.sync_visible_locals_to_mapping(self.locals.mapping(vm), vm)?; - Ok(self.locals.clone_mapping(vm)) + self.locals.clone_mapping(vm) + }; + self.fold_extra_locals(&mapping, vm)?; + Ok(mapping) + } + + /// Copy the frame's extra-locals side storage (proxy keys that are not + /// fast locals) into `mapping`. No-op when nothing was ever stored. + fn fold_extra_locals(&self, mapping: &ArgMapping, vm: &VirtualMachine) -> PyResult<()> { + let extra = self.f_extra_locals.lock().clone(); + if let Some(extra) = extra { + for (key, value) in &extra { + mapping.mapping().ass_subscript(&key, Some(value), vm)?; + } + } + Ok(()) + } + + /// Read a fast-local slot's visible value, dereferencing cells. `None` if + /// the slot is empty or its cell holds no value. + fn framelocalsproxy_getval(&self, i: usize) -> Option { + use rustpython_compiler_core::bytecode::{CO_FAST_CELL, CO_FAST_FREE}; + // SAFETY: callers first pass through `check_locals_access`, so the + // frame is not executing on another thread. + let fastlocals = unsafe { self.iframe_ref().localsplus.fastlocals() }; + let obj = fastlocals.get(i)?.as_ref()?; + let kind = self.code.localspluskinds.get(i).copied().unwrap_or(0); + if kind & (CO_FAST_CELL | CO_FAST_FREE) != 0 { + if let Some(cell) = obj.downcast_ref::() { + cell.get() + } else { + Some(obj.clone()) + } + } else { + Some(obj.clone()) + } + } + + /// Write `value` into fast-local slot `i`, routing through the cell when + /// the slot holds one so closures keep sharing the same cell. + fn framelocalsproxy_setval(&self, i: usize, value: PyObjectRef) { + use rustpython_compiler_core::bytecode::{CO_FAST_CELL, CO_FAST_FREE}; + let kind = self.code.localspluskinds.get(i).copied().unwrap_or(0); + // SAFETY: callers first pass through `check_locals_access`. + let fastlocals = unsafe { self.iframe_mut().localsplus.fastlocals_mut() }; + if kind & (CO_FAST_CELL | CO_FAST_FREE) != 0 + && let Some(obj) = fastlocals[i].as_ref() + && let Some(cell) = obj.downcast_ref::() + { + cell.set(Some(value)); + return; + } + fastlocals[i] = Some(value); + } + + /// Resolve `key` to a fast-local slot index, or `None` if it names no fast + /// local. `read` selects read semantics (only bound slots match) versus + /// write semantics (hidden slots are skipped, unbound slots still match). + /// Raises `TypeError` for an unhashable key. + fn framelocalsproxy_getkeyindex( + &self, + key: &PyObject, + read: bool, + vm: &VirtualMachine, + ) -> PyResult> { + use rustpython_compiler_core::bytecode::CO_FAST_HIDDEN; + // The proxy hashes the key first; an unhashable key raises TypeError. + key.hash(vm)?; + for (i, &kind) in self.code.localspluskinds.iter().enumerate() { + let name = localsplus_name(&self.code, i); + if !name + .as_object() + .rich_compare_bool(key, PyComparisonOp::Eq, vm)? + { + continue; + } + if read { + if self.framelocalsproxy_getval(i).is_some() { + return Ok(Some(i)); + } + } else if kind & CO_FAST_HIDDEN == 0 { + return Ok(Some(i)); + } + } + Ok(None) + } + + /// Build a fresh ordered snapshot dict of the proxy's visible contents: + /// bound fast locals in localsplus order followed by extra locals. + pub(crate) fn framelocalsproxy_snapshot(&self, vm: &VirtualMachine) -> PyResult { + self.check_locals_access(vm)?; + let dict = vm.ctx.new_dict(); + let mapping = ArgMapping::from_dict_exact(dict.clone()); + self.sync_visible_locals_to_mapping(mapping.mapping(), vm)?; + self.fold_extra_locals(&mapping, vm)?; + Ok(dict) + } + + /// `proxy[key]`: read a fast local live, else fall back to extra locals. + pub(crate) fn framelocalsproxy_getitem( + &self, + key: PyObjectRef, + vm: &VirtualMachine, + ) -> PyResult { + self.check_locals_access(vm)?; + if let Some(i) = self.framelocalsproxy_getkeyindex(&key, true, vm)? + && let Some(value) = self.framelocalsproxy_getval(i) + { + return Ok(value); + } + let extra = self.f_extra_locals.lock().clone(); + if let Some(extra) = extra + && let Some(value) = extra.get_item_opt(&*key, vm)? + { + return Ok(value); + } + Err(vm.new_key_error(key)) + } + + /// `key in proxy`. + pub(crate) fn framelocalsproxy_contains( + &self, + key: PyObjectRef, + vm: &VirtualMachine, + ) -> PyResult { + self.check_locals_access(vm)?; + if self.framelocalsproxy_getkeyindex(&key, true, vm)?.is_some() { + return Ok(true); + } + let extra = self.f_extra_locals.lock().clone(); + if let Some(extra) = extra { + return Ok(extra.get_item_opt(&*key, vm)?.is_some()); + } + Ok(false) + } + + /// `proxy[key] = value`: fast-key writes the slot in place, other keys go + /// to the extra-locals side dict. + pub(crate) fn framelocalsproxy_setitem( + &self, + key: PyObjectRef, + value: PyObjectRef, + vm: &VirtualMachine, + ) -> PyResult<()> { + self.check_locals_access(vm)?; + if let Some(i) = self.framelocalsproxy_getkeyindex(&key, false, vm)? { + self.framelocalsproxy_setval(i, value); + return Ok(()); + } + let extra = self.extra_locals_get_or_create(vm); + extra.set_item(&*key, value, vm) + } + + /// `del proxy[key]`: deleting a fast local raises ValueError; extra keys are + /// removed (KeyError if absent). + pub(crate) fn framelocalsproxy_delitem( + &self, + key: PyObjectRef, + vm: &VirtualMachine, + ) -> PyResult<()> { + self.check_locals_access(vm)?; + if self.framelocalsproxy_getkeyindex(&key, false, vm)?.is_some() { + return Err( + vm.new_value_error("cannot remove local variables from FrameLocalsProxy") + ); + } + let extra = self.f_extra_locals.lock().clone(); + if let Some(extra) = extra + && extra.get_item_opt(&*key, vm)?.is_some() + { + return extra.del_item(&*key, vm); } + Err(vm.new_key_error(key)) + } + + /// `proxy.pop(key[, default])`. + pub(crate) fn framelocalsproxy_pop( + &self, + key: PyObjectRef, + default: Option, + vm: &VirtualMachine, + ) -> PyResult { + self.check_locals_access(vm)?; + if self.framelocalsproxy_getkeyindex(&key, false, vm)?.is_some() { + return Err( + vm.new_value_error("cannot remove local variables from FrameLocalsProxy") + ); + } + let extra = self.f_extra_locals.lock().clone(); + if let Some(extra) = extra + && let Some(value) = extra.pop_item(&*key, vm)? + { + return Ok(value); + } + default.ok_or_else(|| vm.new_key_error(key)) + } + + /// `proxy.setdefault(key, default)`. + pub(crate) fn framelocalsproxy_setdefault( + &self, + key: PyObjectRef, + default: PyObjectRef, + vm: &VirtualMachine, + ) -> PyResult { + match self.framelocalsproxy_getitem(key.clone(), vm) { + Ok(value) => Ok(value), + Err(e) if e.fast_isinstance(vm.ctx.exceptions.key_error) => { + self.framelocalsproxy_setitem(key, default.clone(), vm)?; + Ok(default) + } + Err(e) => Err(e), + } + } + + fn extra_locals_get_or_create(&self, vm: &VirtualMachine) -> PyDictRef { + let mut extra = self.f_extra_locals.lock(); + extra.get_or_insert_with(|| vm.ctx.new_dict()).clone() } } @@ -1502,6 +1701,47 @@ fn specialization_nonnegative_compact_index(i: &PyInt, vm: &VirtualMachine) -> O } } +/// Get the variable name for a localsplus index of `code`. +fn localsplus_name(code: &PyCode, idx: usize) -> &'static PyStrInterned { + use rustpython_compiler_core::bytecode::{CO_FAST_CELL, CO_FAST_FREE, CO_FAST_LOCAL}; + let nlocals = code.varnames.len(); + let kind = code.localspluskinds.get(idx).copied().unwrap_or(0); + if kind & CO_FAST_LOCAL != 0 { + // Merged cell or regular local: name is in varnames + code.varnames[idx] + } else if kind & CO_FAST_FREE != 0 { + // Free var: slots are at the end of localsplus + let nlocalsplus = code.localspluskinds.len(); + let nfrees = code.freevars.len(); + let free_start = nlocalsplus - nfrees; + code.freevars[idx - free_start] + } else if kind & CO_FAST_CELL != 0 { + // Non-merged cell: count how many non-merged cell slots are before + // this index to find the corresponding cellvars entry. + // Non-merged cellvars appear in their original order (skipping merged ones). + let nonmerged_pos = code.localspluskinds[nlocals..idx] + .iter() + .filter(|&&k| k == CO_FAST_CELL) + .count(); + // Skip merged cellvars to find the right one + let mut cv_idx = 0; + let mut nonmerged_count = 0; + for (i, name) in code.cellvars.iter().enumerate() { + let is_merged = code.varnames.contains(name); + if !is_merged { + if nonmerged_count == nonmerged_pos { + cv_idx = i; + break; + } + nonmerged_count += 1; + } + } + code.cellvars[cv_idx] + } else { + code.varnames[idx] + } +} + /// Free a finished call frame's data stack storage. /// /// When the caller holds the only reference to the frame, the locals and @@ -2397,43 +2637,7 @@ impl ExecutingFrame<'_> { /// Get the variable name for a localsplus index. fn localsplus_name(&self, idx: usize) -> &'static PyStrInterned { - use rustpython_compiler_core::bytecode::{CO_FAST_CELL, CO_FAST_FREE, CO_FAST_LOCAL}; - let nlocals = self.code.varnames.len(); - let kind = self.code.localspluskinds.get(idx).copied().unwrap_or(0); - if kind & CO_FAST_LOCAL != 0 { - // Merged cell or regular local: name is in varnames - self.code.varnames[idx] - } else if kind & CO_FAST_FREE != 0 { - // Free var: slots are at the end of localsplus - let nlocalsplus = self.code.localspluskinds.len(); - let nfrees = self.code.freevars.len(); - let free_start = nlocalsplus - nfrees; - self.code.freevars[idx - free_start] - } else if kind & CO_FAST_CELL != 0 { - // Non-merged cell: count how many non-merged cell slots are before - // this index to find the corresponding cellvars entry. - // Non-merged cellvars appear in their original order (skipping merged ones). - let nonmerged_pos = self.code.localspluskinds[nlocals..idx] - .iter() - .filter(|&&k| k == CO_FAST_CELL) - .count(); - // Skip merged cellvars to find the right one - let mut cv_idx = 0; - let mut nonmerged_count = 0; - for (i, name) in self.code.cellvars.iter().enumerate() { - let is_merged = self.code.varnames.contains(name); - if !is_merged { - if nonmerged_count == nonmerged_pos { - cv_idx = i; - break; - } - nonmerged_count += 1; - } - } - self.code.cellvars[cv_idx] - } else { - self.code.varnames[idx] - } + localsplus_name(self.code, idx) } /// Execute a single instruction. diff --git a/crates/vm/src/stdlib/sys.rs b/crates/vm/src/stdlib/sys.rs index e5ae1157fa1..4e31075da45 100644 --- a/crates/vm/src/stdlib/sys.rs +++ b/crates/vm/src/stdlib/sys.rs @@ -970,6 +970,7 @@ pub mod sys { let offset = offset.into_option().unwrap_or(0); let frame_ref = crate::frame::frame_at_offset(offset) .ok_or_else(|| vm.new_value_error("call stack is not deep enough"))?; + frame_ref.mark_escaped(); if let Ok(audit) = vm.sys_module.get_attr("audit", vm) { audit.call((vm.ctx.new_str("sys._getframe"), frame_ref.to_owned()), vm)?; diff --git a/crates/vm/src/types/zoo.rs b/crates/vm/src/types/zoo.rs index 13d439345f7..8fe148ac2d2 100644 --- a/crates/vm/src/types/zoo.rs +++ b/crates/vm/src/types/zoo.rs @@ -2,7 +2,8 @@ use crate::{ Py, builtins::{ asyncgenerator, bool_, builtin_func, bytearray, bytes, capsule, classmethod, code, complex, - coroutine, descriptor, dict, enumerate, filter, float, frame, function, generator, + coroutine, descriptor, dict, enumerate, filter, float, frame, frame_locals_proxy, function, + generator, genericalias, getset, int, interpolation, iter, list, map, mappingproxy, memory, module, namespace, object, property, pystr, range, set, singletons, slice, staticmethod, super_, template, traceback, tuple, @@ -39,6 +40,7 @@ pub struct TypeZoo { pub filter_type: &'static Py, pub float_type: &'static Py, pub frame_type: &'static Py, + pub frame_locals_proxy_type: &'static Py, pub frozenset_type: &'static Py, pub generator_type: &'static Py, pub int_type: &'static Py, @@ -178,6 +180,7 @@ impl TypeZoo { dict_reverseitemiterator_type: dict::PyDictReverseItemIterator::init_builtin_type(), ellipsis_type: slice::PyEllipsis::init_builtin_type(), frame_type: crate::frame::Frame::init_builtin_type(), + frame_locals_proxy_type: frame_locals_proxy::FrameLocalsProxy::init_builtin_type(), function_type: function::PyFunction::init_builtin_type(), generator_type: generator::PyGenerator::init_builtin_type(), getset_type: getset::PyGetSet::init_builtin_type(), @@ -253,6 +256,7 @@ impl TypeZoo { bool_::init(context); code::init(context); frame::init(context); + frame_locals_proxy::init(context); weakref::init(context); weakproxy::init(context); singletons::init(context); From a9f8b61e7949f3e023ce7ce9ae1e79d723ee743a Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Tue, 7 Jul 2026 05:25:30 +0900 Subject: [PATCH 71/92] Retain the caller frame so f_back resolves after it returns When a frame escapes its execution (referenced through a traceback, `sys._getframe`, `f_locals`, ...), capture a strong reference to its caller at release time. `f_back` consults it once the caller has left the live frame chain, so the Python-visible frame chain survives return. The retained reference is a GC-traversed edge and is cleared by `frame.clear()`, so ancestor chains stay collectable. Assisted-by: Claude --- Lib/test/test_inspect/test_inspect.py | 1 - Lib/test/test_traceback.py | 1 - crates/vm/src/builtins/frame.rs | 9 +++++++++ crates/vm/src/frame.rs | 12 ++++++++++++ 4 files changed, 21 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_inspect/test_inspect.py b/Lib/test/test_inspect/test_inspect.py index f7a7c0cc825..74126751835 100644 --- a/Lib/test/test_inspect/test_inspect.py +++ b/Lib/test/test_inspect/test_inspect.py @@ -588,7 +588,6 @@ def test_frame(self): self.assertEqual(inspect.formatargvalues(args, varargs, varkw, locals), '(x=11, y=14)') - @unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError: 'NoneType' object has no attribute 'f_code' def test_previous_frame(self): args, varargs, varkw, locals = inspect.getargvalues(mod.fr.f_back) self.assertEqual(args, ['a', 'b', 'c', 'd', 'e', 'f']) diff --git a/Lib/test/test_traceback.py b/Lib/test/test_traceback.py index ec56f26a735..42f066a6239 100644 --- a/Lib/test/test_traceback.py +++ b/Lib/test/test_traceback.py @@ -3141,7 +3141,6 @@ def last_returns_frame4(self): def last_returns_frame5(self): return self.last_returns_frame4() - @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: 1 not greater than 5 def test_extract_stack(self): frame = self.last_returns_frame5() def extract(**kwargs): diff --git a/crates/vm/src/builtins/frame.rs b/crates/vm/src/builtins/frame.rs index 50721eecd11..710d23b7c2f 100644 --- a/crates/vm/src/builtins/frame.rs +++ b/crates/vm/src/builtins/frame.rs @@ -697,6 +697,7 @@ impl Py { self.temporary_refs.lock().clear(); self.f_locals_hidden_overlay.lock().take(); self.f_extra_locals.lock().take(); + self.retained_back.lock().take(); Ok(()) } @@ -737,6 +738,14 @@ impl Py { return Some(frame); } + // The caller already returned and left the live chain, but this frame + // escaped and retained a strong reference to it at release time. + let retained = self.retained_back.lock().clone(); + if let Some(frame) = retained { + frame.mark_escaped(); + return Some(frame); + } + // The caller lives on another thread. unix: park every thread under // stop-the-world so their frame chains are quiescent and alive, then // walk each published top frame down its `previous` chain looking for diff --git a/crates/vm/src/frame.rs b/crates/vm/src/frame.rs index 068dbe1a84d..97ae1784b32 100644 --- a/crates/vm/src/frame.rs +++ b/crates/vm/src/frame.rs @@ -714,6 +714,10 @@ pub struct InterpreterFrame { /// (`f_locals` proxy, `sys._getframe`, `f_back`). A closed generator keeps /// its locals alive while this is set, mirroring `frame_obj` ownership. pub(crate) escaped: atomic::AtomicBool, + /// Strong reference to the caller frame, captured when this frame escapes + /// its execution so `f_back` still resolves after the caller returns and + /// leaves the live frame chain. + pub(crate) retained_back: PyMutex>, /// Number of stack entries to pop after set_f_lineno returns to the /// execution loop. set_f_lineno cannot pop directly because the /// execution loop holds the state mutex. @@ -882,6 +886,7 @@ unsafe impl Traverse for Frame { iframe.temporary_refs.traverse(tracer_fn); iframe.f_locals_hidden_overlay.traverse(tracer_fn); iframe.f_extra_locals.traverse(tracer_fn); + iframe.retained_back.traverse(tracer_fn); } fn clear(&mut self, _out: &mut Vec) { @@ -975,6 +980,7 @@ impl Frame { f_locals_hidden_overlay: PyMutex::new(None), f_extra_locals: PyMutex::new(None), escaped: atomic::AtomicBool::new(false), + retained_back: PyMutex::new(None), pending_stack_pops: Default::default(), pending_unwind_from_stack: Default::default(), }; @@ -1793,6 +1799,12 @@ pub(crate) fn release_datastack_frame(frame: &Py, vm: &VirtualMachine) { vm.datastack_pop(base); } } + // Retain a strong reference to the caller so `f_back` keeps resolving once + // the caller returns and leaves the live frame chain. The caller is still + // executing here (this frame is unwinding back into it), so its payload + // pointer is live. + // SAFETY: `previous` points at the live caller on this thread's stack. + *frame.retained_back.lock() = unsafe { owned_chain_frame(frame.previous_frame()) }; // Invariant: a tracked frame must always have heap-backed localsplus // (proven here for escaped datastack frames and by construction for // generator frames, which are born heap-backed). A stop-the-world From f05926a280ef2f9bdc6633002fca4af3a302249a Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Tue, 7 Jul 2026 05:49:31 +0900 Subject: [PATCH 72/92] Fix debug-build native stack overflow on deep recursion Make check_c_stack_overflow one-sided (trip whenever the stack pointer is below the soft limit) so a single native frame larger than the margin cannot step past the danger band undetected. Raise the debug STACK_MARGIN_BYTES from 4096 to 16384 words so the margin exceeds a single debug interpreter frame, leaving headroom to raise RecursionError. Release margin unchanged. Clamp the soft-limit margin to half the stack so small explicit thread stacks do not get a soft limit above their stack top. Assisted-by: Claude --- crates/vm/src/vm/mod.rs | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/crates/vm/src/vm/mod.rs b/crates/vm/src/vm/mod.rs index becfa99b08b..bb05c5d0fbe 100644 --- a/crates/vm/src/vm/mod.rs +++ b/crates/vm/src/vm/mod.rs @@ -1569,7 +1569,7 @@ impl VirtualMachine { /// evaluation consumes more native stack in those configurations. #[cfg_attr(any(miri, target_env = "musl"), allow(dead_code))] const STACK_MARGIN_BYTES: usize = - (if cfg!(debug_assertions) { 4096 } else { 2048 }) * core::mem::size_of::(); + (if cfg!(debug_assertions) { 16384 } else { 2048 }) * core::mem::size_of::(); /// Get the stack boundaries using platform-specific APIs. /// Returns (base, top) where base is the lowest address and top is the highest. @@ -1630,11 +1630,16 @@ impl VirtualMachine { } /// Calculate the C stack soft limit based on actual stack boundaries. - /// soft_limit = base + 2 * margin (for downward-growing stacks) + /// soft_limit = base + 2 * margin (for downward-growing stacks). + /// The margin is clamped to half the stack so threads created with a stack + /// smaller than 2 * (2 * margin) still get usable headroom instead of a + /// soft limit above their stack top (which would trip on entry). #[cfg(all(not(miri), not(target_env = "musl")))] fn calculate_c_stack_soft_limit() -> usize { - let (base, _top) = Self::get_stack_bounds(); - base + Self::STACK_MARGIN_BYTES * 2 + let (base, top) = Self::get_stack_bounds(); + let stack_size = top.saturating_sub(base); + let margin = (Self::STACK_MARGIN_BYTES * 2).min(stack_size / 2); + base + margin } /// Musl currently reports stack bounds in a way that trips the VM's @@ -1646,15 +1651,14 @@ impl VirtualMachine { } /// Check if we're near the C stack limit (like _Py_MakeRecCheck). - /// Returns true only when stack pointer is in the "danger zone" between - /// soft_limit and hard_limit (soft_limit - 2*margin). + /// One-sided: any stack pointer below the soft limit is in danger, since a + /// single native frame can exceed the margin and step past it. #[cfg(all(not(miri), not(target_env = "musl")))] #[inline(always)] fn check_c_stack_overflow(&self) -> bool { let current_sp = psm::stack_pointer() as usize; let soft_limit = self.c_stack_soft_limit.get(); current_sp < soft_limit - && current_sp >= soft_limit.saturating_sub(Self::STACK_MARGIN_BYTES * 2) } /// Miri does not support the native stack probe, and musl currently trips From 4a92aadaed7e6f728ba1468cb17c75f9bd6ce0cc Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Tue, 7 Jul 2026 06:04:55 +0900 Subject: [PATCH 73/92] Allocate exception instance __dict__ lazily Exception construction went through into_ref_with_type, which eagerly allocated an empty instance dict for every HAS_DICT type. Add into_ref_with_type_lazy_dict, which builds the instance with an unallocated dict slot, and route the four exception construction sites (PyBaseException, PyOSError, OSErrorBuilder, PyBaseExceptionGroup) through it. The dict now materializes on first attribute write or __dict__ access via the existing get_or_insert path. add_note and PyImportError::slot_init now obtain the dict through object_get_dict so they materialize it instead of assuming it exists. A freshly constructed exception no longer reports an empty dict in gc.get_referents, matching the reference interpreter. Assisted-by: Claude --- crates/vm/src/exception_group.rs | 2 +- crates/vm/src/exceptions.rs | 14 ++++++------- crates/vm/src/object/payload.rs | 35 +++++++++++++++++++++++++++++++- 3 files changed, 41 insertions(+), 10 deletions(-) diff --git a/crates/vm/src/exception_group.rs b/crates/vm/src/exception_group.rs index 198273a6914..a2c76378ab3 100644 --- a/crates/vm/src/exception_group.rs +++ b/crates/vm/src/exception_group.rs @@ -334,7 +334,7 @@ pub(super) mod types { let exceptions_tuple = vm.ctx.new_tuple(exceptions); let init_args = vec![message, exceptions_tuple.into()]; PyBaseException::new(init_args, vm) - .into_ref_with_type(vm, actual_cls) + .into_ref_with_type_lazy_dict(vm, actual_cls) .map(Into::into) } diff --git a/crates/vm/src/exceptions.rs b/crates/vm/src/exceptions.rs index 1ad86a35aed..2568afca6c0 100644 --- a/crates/vm/src/exceptions.rs +++ b/crates/vm/src/exceptions.rs @@ -690,10 +690,8 @@ impl PyRef { #[pymethod] fn add_note(self, note: PyStrRef, vm: &VirtualMachine) -> PyResult<()> { - let dict = self - .as_object() - .dict() - .ok_or_else(|| vm.new_attribute_error("Exception object has no __dict__"))?; + let dict = crate::builtins::object::object_get_dict(self.as_object().to_owned(), vm) + .map_err(|_| vm.new_attribute_error("Exception object has no __dict__"))?; let notes = if let Ok(notes) = dict.get_item("__notes__", vm) { notes @@ -747,7 +745,7 @@ impl Constructor for PyBaseException { return Err(vm.new_type_error("BaseException() takes no keyword arguments")); } Self::new(args.args, vm) - .into_ref_with_type(vm, cls) + .into_ref_with_type_lazy_dict(vm, cls) .map(Into::into) } @@ -1364,7 +1362,7 @@ impl OSErrorBuilder { let payload = PyOSError::py_new(&exc_type, args.clone().into(), vm) .expect("new_os_error usage error"); let os_error = payload - .into_ref_with_type(vm, exc_type) + .into_ref_with_type_lazy_dict(vm, exc_type) .expect("new_os_error usage error"); PyOSError::slot_init(os_error.as_object().to_owned(), args.into(), vm) .expect("new_os_error usage error"); @@ -1805,7 +1803,7 @@ pub(super) mod types { ))); } - let dict = zelf.dict().unwrap(); + let dict = crate::builtins::object::object_get_dict(zelf.clone(), vm)?; dict.set_item("name", vm.unwrap_or_none(name), vm)?; dict.set_item("path", vm.unwrap_or_none(path), vm)?; dict.set_item("name_from", vm.unwrap_or_none(name_from), vm)?; @@ -2013,7 +2011,7 @@ pub(super) mod types { } } let payload = Self::py_new(&cls, args, vm)?; - payload.into_ref_with_type(vm, cls).map(Into::into) + payload.into_ref_with_type_lazy_dict(vm, cls).map(Into::into) } } diff --git a/crates/vm/src/object/payload.rs b/crates/vm/src/object/payload.rs index 7925f656de5..36262607a1a 100644 --- a/crates/vm/src/object/payload.rs +++ b/crates/vm/src/object/payload.rs @@ -133,6 +133,34 @@ pub trait PyPayload: MaybeTraverse + PyThreadingConstraint + Sized + 'static { #[inline] fn into_ref_with_type(self, vm: &VirtualMachine, cls: PyTypeRef) -> PyResult> + where + Self: core::fmt::Debug, + { + self.into_ref_with_type_and_dict(vm, cls, true) + } + + /// Like `into_ref_with_type`, but leaves the instance `__dict__` unallocated + /// until the first attribute write or `__dict__` access. Only valid for types + /// whose attribute protocol materializes the dict lazily via `get_or_insert`. + #[inline] + fn into_ref_with_type_lazy_dict( + self, + vm: &VirtualMachine, + cls: PyTypeRef, + ) -> PyResult> + where + Self: core::fmt::Debug, + { + self.into_ref_with_type_and_dict(vm, cls, false) + } + + #[inline] + fn into_ref_with_type_and_dict( + self, + vm: &VirtualMachine, + cls: PyTypeRef, + eager_dict: bool, + ) -> PyResult> where Self: core::fmt::Debug, { @@ -154,7 +182,12 @@ pub trait PyPayload: MaybeTraverse + PyThreadingConstraint + Sized + 'static { } return Err(_into_ref_size_error(vm, &cls, exact_class)); } - Ok(self._into_ref(cls, &vm.ctx)) + let dict = if eager_dict && cls.slots.flags.has_feature(PyTypeFlags::HAS_DICT) { + Some(vm.ctx.new_dict()) + } else { + None + }; + Ok(PyRef::new_ref(self, cls, dict)) } else { #[cold] #[inline(never)] From 072bc2ac5dcd3585a2f945094b4dcfca6dfc90bb Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Tue, 7 Jul 2026 06:21:01 +0900 Subject: [PATCH 74/92] Allocate exception __dict__ lazily in vm.new_exception Route vm.new_exception() through into_ref_with_type_lazy_dict so internally raised exceptions (new_type_error, new_value_error, etc.) start without an instance dict, matching the slot_new path. The dict is materialized on the first attribute write or __dict__ access. Assisted-by: Claude --- crates/vm/src/vm/vm_new.rs | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/crates/vm/src/vm/vm_new.rs b/crates/vm/src/vm/vm_new.rs index c71cf842520..ae728aaba67 100644 --- a/crates/vm/src/vm/vm_new.rs +++ b/crates/vm/src/vm/vm_new.rs @@ -10,7 +10,7 @@ use rustpython_compiler_core::SourceLocation; use rustpython_compiler::{CompileError, ParseError}; use crate::{ - AsObject, Py, PyObject, PyObjectRef, PyRef, PyResult, + AsObject, Py, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, builtins::{ PyBaseException, PyBaseExceptionRef, PyBytesRef, PyDictRef, PyModule, PyOSError, PyStrRef, PyType, PyTypeRef, @@ -348,11 +348,9 @@ impl VirtualMachine { exc_type.name() ); - PyRef::new_ref( - PyBaseException::new(args, self), - exc_type, - Some(self.ctx.new_dict()), - ) + PyBaseException::new(args, self) + .into_ref_with_type_lazy_dict(self, exc_type) + .expect("vm.new_exception() called with an invalid exception type") } pub fn new_os_error(&self, msg: impl ToPyObject) -> PyRef { From c172c74c2e421cff69ca4b0e0241d7fafe62e379 Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Tue, 7 Jul 2026 06:21:06 +0900 Subject: [PATCH 75/92] Validate FrameLocalsProxy.update() arguments Reject keyword arguments and require exactly one positional argument, raising TypeError with the "takes no keyword arguments" and "takes exactly one argument (N given)" messages. Assisted-by: Claude --- crates/vm/src/builtins/frame_locals_proxy.rs | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/crates/vm/src/builtins/frame_locals_proxy.rs b/crates/vm/src/builtins/frame_locals_proxy.rs index b94a6495263..1335cf59497 100644 --- a/crates/vm/src/builtins/frame_locals_proxy.rs +++ b/crates/vm/src/builtins/frame_locals_proxy.rs @@ -161,8 +161,19 @@ impl FrameLocalsProxy { } #[pymethod] - fn update(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { - self.update_from(&other, vm) + fn update(&self, args: FuncArgs, vm: &VirtualMachine) -> PyResult<()> { + if !args.kwargs.is_empty() { + return Err( + vm.new_type_error("FrameLocalsProxy.update() takes no keyword arguments") + ); + } + if args.args.len() != 1 { + return Err(vm.new_type_error(format!( + "FrameLocalsProxy.update() takes exactly one argument ({} given)", + args.args.len() + ))); + } + self.update_from(&args.args[0], vm) } fn update_from(&self, other: &PyObject, vm: &VirtualMachine) -> PyResult<()> { From 2652f4d29e1173cdc5cf609ec6d865c9e4fb5aed Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Tue, 7 Jul 2026 06:21:15 +0900 Subject: [PATCH 76/92] Address review nits in frame and faulthandler - Drop stale vm.frames reference from the release_datastack_frame uniqueness argument. - Assert has_exc_handling when unwinding an Except-typed stack slot, documenting the invariant that guards the shared exc_info write. - Truncate the specialized __init__ return-type name to 200 chars, matching the unspecialized wrapper. - Reword two comments to describe behavior without prose references to CPython. Assisted-by: Claude --- crates/stdlib/src/faulthandler.rs | 4 ++-- crates/vm/src/frame.rs | 17 +++++++++++++---- 2 files changed, 15 insertions(+), 6 deletions(-) diff --git a/crates/stdlib/src/faulthandler.rs b/crates/stdlib/src/faulthandler.rs index cb559eb7a9d..fb779f232aa 100644 --- a/crates/stdlib/src/faulthandler.rs +++ b/crates/stdlib/src/faulthandler.rs @@ -327,7 +327,7 @@ mod decl { // Get all threads' frame stacks from the shared registry // unix: stop-the-world so every other thread is parked at a safepoint // and its frame chain is quiescent and alive while we walk it (matches - // faulthandler.dump_traceback running with the GIL held in CPython). + // faulthandler.dump_traceback running with the GIL held). #[cfg(all(unix, feature = "threading"))] { use core::sync::atomic::Ordering; @@ -723,7 +723,7 @@ mod decl { // The watchdog is a plain OS thread, not attached to // the VM, so it cannot stop-the-world. Walk each // published top frame lock-free and best-effort, like - // CPython's faulthandler watchdog. + // the faulthandler watchdog thread. for (tid, slot) in &thread_frame_slots { let top = slot .top_frame diff --git a/crates/vm/src/frame.rs b/crates/vm/src/frame.rs index 97ae1784b32..d83022424ae 100644 --- a/crates/vm/src/frame.rs +++ b/crates/vm/src/frame.rs @@ -1758,8 +1758,8 @@ fn localsplus_name(code: &PyCode, idx: usize) -> &'static PyStrInterned { pub(crate) fn release_datastack_frame(frame: &Py, vm: &VirtualMachine) { let frame_obj = frame.as_object(); // Uniqueness argument: at this point the frame is already out of - // `vm.frames`, the thread-frames registry and the current-frame chain - // (all unlinked inside `with_frame` before it returned), and the + // the thread-frames registry and the current-frame chain + // (both unlinked inside `with_frame` before it returned), and the // frame type has no weakref support. A datastack frame is created // untracked and stays untracked while it runs, so it is in no GC // generation list and no collector can observe or incref it. Hence no @@ -2039,7 +2039,7 @@ impl ExecutingFrame<'_> { if !vm.is_none(&init_result) { return Err(vm.new_type_error(format!( - "__init__() should return None, not '{}'", + "__init__() should return None, not '{:.200}'", init_result.class().name() ))); } @@ -2082,6 +2082,15 @@ impl ExecutingFrame<'_> { if stack_analysis::top_of_stack(cur_stack) == stack_analysis::Kind::Except as i64 && let Some(exc_obj) = val { + // An Except-typed stack slot is only produced by bytecode that + // also carries one of the opcodes scanned by + // `PyCode::has_exc_handling`; otherwise the save/restore that + // brackets this frame's exc_info is elided and this write would + // corrupt the shared exc_info slot. + debug_assert!( + self.code.has_exc_handling, + "unwinding an Except slot in a frame without exc-handling opcodes" + ); if vm.is_none(&exc_obj) { vm.set_exception(None); } else { @@ -4309,7 +4318,7 @@ impl ExecutingFrame<'_> { let should_be_none = self.pop_value(); if !vm.is_none(&should_be_none) { return Err(vm.new_type_error(format!( - "__init__() should return None, not '{}'", + "__init__() should return None, not '{:.200}'", should_be_none.class().name() ))); } From 0499c0a1f39c9bd0bc0772155fca0736de494f5a Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Tue, 7 Jul 2026 06:41:23 +0900 Subject: [PATCH 77/92] Use scopeguard for start_the_world in faulthandler dump_all_threads Wrap the unix stop-the-world registry walk in a scope with scopeguard::defer! so start_the_world runs on panic, matching the f_back and get_all_current_frames sites. Add scopeguard to the stdlib dependencies. Also correct the restore_exception doc comment to name with_frame after the rename. Assisted-by: Claude --- Cargo.lock | 1 + crates/stdlib/Cargo.toml | 1 + crates/stdlib/src/faulthandler.rs | 4 ++-- crates/vm/src/vm/mod.rs | 2 +- 4 files changed, 5 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index aeb2426e52a..73afeda8a18 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3700,6 +3700,7 @@ dependencies = [ "rustpython-ruff_text_size", "rustpython-unicode", "rustpython-vm", + "scopeguard", "sha1 0.11.0", "sha2", "sha3", diff --git a/crates/stdlib/Cargo.toml b/crates/stdlib/Cargo.toml index 1e071869549..d32e7ce8b35 100644 --- a/crates/stdlib/Cargo.toml +++ b/crates/stdlib/Cargo.toml @@ -53,6 +53,7 @@ num_enum = { workspace = true } parking_lot = { workspace = true } phf = { workspace = true, default-features = true, features = ["macros"] } rapidhash = { workspace = true } +scopeguard = { workspace = true } memchr = { workspace = true } base64 = { workspace = true } diff --git a/crates/stdlib/src/faulthandler.rs b/crates/stdlib/src/faulthandler.rs index fb779f232aa..900d66b76e6 100644 --- a/crates/stdlib/src/faulthandler.rs +++ b/crates/stdlib/src/faulthandler.rs @@ -332,8 +332,9 @@ mod decl { { use core::sync::atomic::Ordering; let current_tid = rustpython_vm::stdlib::_thread::get_ident(); - vm.state.stop_the_world.stop_the_world(vm); { + vm.state.stop_the_world.stop_the_world(vm); + scopeguard::defer! { vm.state.stop_the_world.start_the_world(vm); } let registry = vm.state.thread_frames.lock(); #[expect( clippy::iter_over_hash_type, @@ -348,7 +349,6 @@ mod decl { puts(fd, "\n"); } } - vm.state.stop_the_world.start_the_world(vm); // Now dump current thread from its live frame chain. write_thread_id(fd, current_tid, true); diff --git a/crates/vm/src/vm/mod.rs b/crates/vm/src/vm/mod.rs index bb05c5d0fbe..74fe0d7cea9 100644 --- a/crates/vm/src/vm/mod.rs +++ b/crates/vm/src/vm/mod.rs @@ -2230,7 +2230,7 @@ impl VirtualMachine { thread::update_thread_exception(self.topmost_exception()); } - /// Restore an exc_info slot value saved by `with_frame_impl`, skipping the + /// Restore an exc_info slot value saved by `with_frame`, skipping the /// store when the slot is unchanged. `saved` is a strong reference taken /// at save time, so the object it points to cannot have been freed and /// its address reused while the frame ran; pointer identity therefore From 811f23040e922bc8b42dae3776141dc10d5f3cf9 Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Tue, 7 Jul 2026 09:05:15 +0900 Subject: [PATCH 78/92] Gitignore docs/superpowers Assisted-by: Claude --- .gitignore | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index b5887be53b5..09e1b97b9f8 100644 --- a/.gitignore +++ b/.gitignore @@ -27,4 +27,5 @@ Lib/site-packages/* Lib/test/data/* !Lib/test/data/README cpython/ -.claude/scheduled_tasks.lock \ No newline at end of file +.claude/scheduled_tasks.lock +docs/superpowers/ \ No newline at end of file From 7982da8cb52c64bea6c28327c0da55ec622e7cd1 Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Tue, 7 Jul 2026 09:06:58 +0900 Subject: [PATCH 79/92] Rename type_bases_slot_rebuild.py to builtin_type_bases.py Match the builtin_* naming convention of extra_tests/snippets. Assisted-by: Claude --- .../{type_bases_slot_rebuild.py => builtin_type_bases.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename extra_tests/snippets/{type_bases_slot_rebuild.py => builtin_type_bases.py} (100%) diff --git a/extra_tests/snippets/type_bases_slot_rebuild.py b/extra_tests/snippets/builtin_type_bases.py similarity index 100% rename from extra_tests/snippets/type_bases_slot_rebuild.py rename to extra_tests/snippets/builtin_type_bases.py From 683f16490d269cff6367e857f03704bb368781e9 Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Tue, 7 Jul 2026 13:23:46 +0900 Subject: [PATCH 80/92] Apply rustfmt, ruff, and cspell lint fixes Reformat with rustfmt, fix import spacing in builtin_type_bases.py with ruff, and add "pointee" to the cspell word list. Assisted-by: Claude --- .cspell.json | 1 + crates/vm/src/builtins/frame_locals_proxy.rs | 28 ++++++++++---------- crates/vm/src/coroutine.rs | 8 ++---- crates/vm/src/exceptions.rs | 4 ++- crates/vm/src/frame.rs | 18 +++++++------ crates/vm/src/gc_state.rs | 9 +++---- crates/vm/src/types/zoo.rs | 7 +++-- crates/vm/src/vm/mod.rs | 4 +-- extra_tests/snippets/builtin_type_bases.py | 1 - 9 files changed, 39 insertions(+), 41 deletions(-) diff --git a/.cspell.json b/.cspell.json index 45ec8b60d39..6f0ac213672 100644 --- a/.cspell.json +++ b/.cspell.json @@ -79,6 +79,7 @@ "mcache", "oparg", "opargs", + "pointee", "pyc", "reborrow", "reborrows", diff --git a/crates/vm/src/builtins/frame_locals_proxy.rs b/crates/vm/src/builtins/frame_locals_proxy.rs index 1335cf59497..fbbc7f5d9cd 100644 --- a/crates/vm/src/builtins/frame_locals_proxy.rs +++ b/crates/vm/src/builtins/frame_locals_proxy.rs @@ -90,7 +90,12 @@ impl FrameLocalsProxy { self.frame.framelocalsproxy_getitem(key, vm) } - fn __setitem__(&self, key: PyObjectRef, value: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { + fn __setitem__( + &self, + key: PyObjectRef, + value: PyObjectRef, + vm: &VirtualMachine, + ) -> PyResult<()> { self.frame.framelocalsproxy_setitem(key, value, vm) } @@ -145,12 +150,7 @@ impl FrameLocalsProxy { } #[pymethod] - fn setdefault( - &self, - key: PyObjectRef, - default: OptionalArg, - vm: &VirtualMachine, - ) -> PyResult { + fn setdefault(&self, key: PyObjectRef, default: OptionalArg, vm: &VirtualMachine) -> PyResult { self.frame .framelocalsproxy_setdefault(key, default.unwrap_or_none(vm), vm) } @@ -163,9 +163,7 @@ impl FrameLocalsProxy { #[pymethod] fn update(&self, args: FuncArgs, vm: &VirtualMachine) -> PyResult<()> { if !args.kwargs.is_empty() { - return Err( - vm.new_type_error("FrameLocalsProxy.update() takes no keyword arguments") - ); + return Err(vm.new_type_error("FrameLocalsProxy.update() takes no keyword arguments")); } if args.args.len() != 1 { return Err(vm.new_type_error(format!( @@ -183,9 +181,9 @@ impl FrameLocalsProxy { } else if let Some(proxy) = other.downcast_ref::() { proxy.snapshot(vm)?.into_iter().collect() } else { - return Err(vm.new_type_error( - "update() argument must be dict or another FrameLocalsProxy", - )); + return Err( + vm.new_type_error("update() argument must be dict or another FrameLocalsProxy") + ); }; for (key, value) in items { self.frame.framelocalsproxy_setitem(key, value, vm)?; @@ -229,7 +227,9 @@ impl FrameLocalsProxy { impl AsMapping for FrameLocalsProxy { fn as_mapping() -> &'static PyMappingMethods { static AS_MAPPING: LazyLock = LazyLock::new(|| PyMappingMethods { - length: atomic_func!(|mapping, vm| FrameLocalsProxy::mapping_downcast(mapping).__len__(vm)), + length: atomic_func!( + |mapping, vm| FrameLocalsProxy::mapping_downcast(mapping).__len__(vm) + ), subscript: atomic_func!(|mapping, needle, vm| { FrameLocalsProxy::mapping_downcast(mapping).__getitem__(needle.to_owned(), vm) }), diff --git a/crates/vm/src/coroutine.rs b/crates/vm/src/coroutine.rs index c6586ab3046..844887f8520 100644 --- a/crates/vm/src/coroutine.rs +++ b/crates/vm/src/coroutine.rs @@ -183,9 +183,7 @@ impl Coro { } else { None }; - let (result, entered_frame) = self.run_with_context(jen, vm, |f| { - f.resume(value, vm) - }); + let (result, entered_frame) = self.run_with_context(jen, vm, |f| f.resume(value, vm)); self.finalize_send_result(result, entered_frame, jen, vm) } @@ -211,9 +209,7 @@ impl Coro { } else { None }; - let (result, entered_frame) = self.run_with_context(jen, vm, |f| { - f.resume(value, vm) - }); + let (result, entered_frame) = self.run_with_context(jen, vm, |f| f.resume(value, vm)); self.finalize_send_result(result, entered_frame, jen, vm) } diff --git a/crates/vm/src/exceptions.rs b/crates/vm/src/exceptions.rs index 2568afca6c0..67f27a72dc2 100644 --- a/crates/vm/src/exceptions.rs +++ b/crates/vm/src/exceptions.rs @@ -2011,7 +2011,9 @@ pub(super) mod types { } } let payload = Self::py_new(&cls, args, vm)?; - payload.into_ref_with_type_lazy_dict(vm, cls).map(Into::into) + payload + .into_ref_with_type_lazy_dict(vm, cls) + .map(Into::into) } } diff --git a/crates/vm/src/frame.rs b/crates/vm/src/frame.rs index d83022424ae..1d39c9edb38 100644 --- a/crates/vm/src/frame.rs +++ b/crates/vm/src/frame.rs @@ -1491,10 +1491,11 @@ impl Frame { vm: &VirtualMachine, ) -> PyResult<()> { self.check_locals_access(vm)?; - if self.framelocalsproxy_getkeyindex(&key, false, vm)?.is_some() { - return Err( - vm.new_value_error("cannot remove local variables from FrameLocalsProxy") - ); + if self + .framelocalsproxy_getkeyindex(&key, false, vm)? + .is_some() + { + return Err(vm.new_value_error("cannot remove local variables from FrameLocalsProxy")); } let extra = self.f_extra_locals.lock().clone(); if let Some(extra) = extra @@ -1513,10 +1514,11 @@ impl Frame { vm: &VirtualMachine, ) -> PyResult { self.check_locals_access(vm)?; - if self.framelocalsproxy_getkeyindex(&key, false, vm)?.is_some() { - return Err( - vm.new_value_error("cannot remove local variables from FrameLocalsProxy") - ); + if self + .framelocalsproxy_getkeyindex(&key, false, vm)? + .is_some() + { + return Err(vm.new_value_error("cannot remove local variables from FrameLocalsProxy")); } let extra = self.f_extra_locals.lock().clone(); if let Some(extra) = extra diff --git a/crates/vm/src/gc_state.rs b/crates/vm/src/gc_state.rs index 670f3dd2741..5189352b8c9 100644 --- a/crates/vm/src/gc_state.rs +++ b/crates/vm/src/gc_state.rs @@ -641,15 +641,14 @@ impl GcState { reason = "assertion over every registered thread slot" )] for slot in registry.values() { - let mut cur = slot - .top_frame - .load(core::sync::atomic::Ordering::Relaxed) + let mut cur = slot.top_frame.load(core::sync::atomic::Ordering::Relaxed) as *const crate::frame::Frame; while !cur.is_null() { // SAFETY: frames on a thread's active call stack are // alive, and the world is stopped so none can be popped. - let obj = unsafe { &*crate::Py::::from_payload_ptr(cur) } - .as_object(); + let obj = + unsafe { &*crate::Py::::from_payload_ptr(cur) } + .as_object(); let ptr = GcPtr(NonNull::from(obj)); debug_assert!( !unreachable_set.contains(&ptr), diff --git a/crates/vm/src/types/zoo.rs b/crates/vm/src/types/zoo.rs index 8fe148ac2d2..64807fc0973 100644 --- a/crates/vm/src/types/zoo.rs +++ b/crates/vm/src/types/zoo.rs @@ -3,10 +3,9 @@ use crate::{ builtins::{ asyncgenerator, bool_, builtin_func, bytearray, bytes, capsule, classmethod, code, complex, coroutine, descriptor, dict, enumerate, filter, float, frame, frame_locals_proxy, function, - generator, - genericalias, getset, int, interpolation, iter, list, map, mappingproxy, memory, module, - namespace, object, property, pystr, range, set, singletons, slice, staticmethod, super_, - template, traceback, tuple, + generator, genericalias, getset, int, interpolation, iter, list, map, mappingproxy, memory, + module, namespace, object, property, pystr, range, set, singletons, slice, staticmethod, + super_, template, traceback, tuple, type_::{self, PyType}, union_, weakproxy, weakref, zip, }, diff --git a/crates/vm/src/vm/mod.rs b/crates/vm/src/vm/mod.rs index 74fe0d7cea9..2db7dc5fbee 100644 --- a/crates/vm/src/vm/mod.rs +++ b/crates/vm/src/vm/mod.rs @@ -44,10 +44,10 @@ use crate::{ warn::WarningsState, }; use alloc::{borrow::Cow, collections::BTreeMap}; -#[cfg(all(unix, feature = "threading"))] -use core::sync::atomic::AtomicI64; #[cfg(all(not(unix), feature = "threading"))] use core::ptr::NonNull; +#[cfg(all(unix, feature = "threading"))] +use core::sync::atomic::AtomicI64; use core::{ cell::{Cell, OnceCell, RefCell}, sync::atomic::{AtomicBool, AtomicU64, Ordering}, diff --git a/extra_tests/snippets/builtin_type_bases.py b/extra_tests/snippets/builtin_type_bases.py index f40343dfb7c..c517ef2119c 100644 --- a/extra_tests/snippets/builtin_type_bases.py +++ b/extra_tests/snippets/builtin_type_bases.py @@ -1,6 +1,5 @@ from testutils import assert_raises - # Reassigning __bases__ must rebuild slot dispatchers for the type and all its # descendants: a slot whose method left the new MRO must be reset, not left stale. From ac2ce2bf3ef48e00c4856aca1467f4f1a349096c Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Tue, 7 Jul 2026 13:23:51 +0900 Subject: [PATCH 81/92] Gate unix-only QSBR methods to their call sites `online`, `drain_all`, and `reset_after_fork` are called only from unix code (thread attach/detach, post-fork reset), and `offline` from unix code plus a unit test. Gate them with matching cfg so `-D dead_code` does not fire on non-unix targets. Assisted-by: Claude --- crates/vm/src/object/qsbr.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/crates/vm/src/object/qsbr.rs b/crates/vm/src/object/qsbr.rs index c4e84208dba..9f5fca76727 100644 --- a/crates/vm/src/object/qsbr.rs +++ b/crates/vm/src/object/qsbr.rs @@ -126,11 +126,13 @@ mod threading { /// Mark a thread offline (detached); it no longer delays grace /// periods (_Py_qsbr_detach). The thread must not perform lock-free /// cache reads while offline. + #[cfg(any(unix, test))] pub(crate) fn offline(&self, slot: &QsbrSlot) { slot.seq.store(QSBR_OFFLINE, Ordering::Release); } /// Mark a thread online again (_Py_qsbr_attach). + #[cfg(unix)] pub(crate) fn online(&self, slot: &QsbrSlot) { self.quiescent_state(slot); } @@ -223,6 +225,7 @@ mod threading { /// # Safety /// Only sound when no other thread can be mid-read: the post-fork /// child, or teardown after all threads exited. + #[cfg(unix)] pub(crate) unsafe fn drain_all(&self) { let mut queue = self.queue.lock().unwrap(); for item in queue.drain(..) { @@ -241,6 +244,7 @@ mod threading { /// # Safety /// Only sound in the single-threaded post-fork child, before the /// surviving thread re-registers. + #[cfg(unix)] pub(crate) unsafe fn reset_after_fork(&self) { self.threads.lock().unwrap().clear(); // SAFETY: single-threaded child, no concurrent reader exists. From f843473f99efe965def5e0a6cd21de5641a05e51 Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Tue, 7 Jul 2026 13:23:56 +0900 Subject: [PATCH 82/92] Visit the function closure tuple as a GC edge PyFunction::traverse visited the cells inside the closure tuple instead of the tuple object itself, so the tuple's reference from the function was never subtracted during cycle collection. A closure tuple that reached back to its function (or, through a frame retained by f_back, to a Thread) was stranded as a false GC root and never collected, leaking the whole cycle. Visit the tuple itself, matching clear(). Assisted-by: Claude --- crates/vm/src/builtins/function.rs | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/crates/vm/src/builtins/function.rs b/crates/vm/src/builtins/function.rs index c522815e2aa..47fda299455 100644 --- a/crates/vm/src/builtins/function.rs +++ b/crates/vm/src/builtins/function.rs @@ -95,7 +95,12 @@ unsafe impl Traverse for PyFunction { fn traverse(&self, tracer_fn: &mut TraverseFn<'_>) { self.globals.traverse(tracer_fn); if let Some(closure) = self.closure.as_ref() { - closure.as_untyped().traverse(tracer_fn); + // Visit the closure tuple itself as an edge, not its cells: the + // tuple is a tracked object that can join a reference cycle, and + // `clear` releases the whole tuple. Visiting only the cells would + // leave the tuple's reference unaccounted, stranding it as a false + // GC root. + tracer_fn(closure.as_untyped().as_object()); } self.defaults_and_kwdefaults.traverse(tracer_fn); // Traverse additional fields that may contain references From 7ab5f768ea2bfdbfa87d4bc0d1afb85d2f19eccc Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Tue, 7 Jul 2026 16:37:49 +0900 Subject: [PATCH 83/92] Apply formatting hook fix to builtin_type_bases.py Assisted-by: Claude --- extra_tests/snippets/builtin_type_bases.py | 1 + 1 file changed, 1 insertion(+) diff --git a/extra_tests/snippets/builtin_type_bases.py b/extra_tests/snippets/builtin_type_bases.py index c517ef2119c..1d413e48e5c 100644 --- a/extra_tests/snippets/builtin_type_bases.py +++ b/extra_tests/snippets/builtin_type_bases.py @@ -3,6 +3,7 @@ # Reassigning __bases__ must rebuild slot dispatchers for the type and all its # descendants: a slot whose method left the new MRO must be reset, not left stale. + # --- zelf itself loses __add__ (nb_add) --- class OldAdd: def __add__(self, other): From 6626177711182c7334706672a604411aa6a06132 Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Tue, 7 Jul 2026 17:08:54 +0900 Subject: [PATCH 84/92] Unmark asyncgen finalization-by-gc tests in test_base_events test_asyncgen_finalization_by_gc and test_asyncgen_finalization_by_gc_in_other_thread now pass; GC finalizes the async generators. Assisted-by: Claude --- Lib/test/test_asyncio/test_base_events.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/Lib/test/test_asyncio/test_base_events.py b/Lib/test/test_asyncio/test_base_events.py index 92895bbb420..1b727f3b1fe 100644 --- a/Lib/test/test_asyncio/test_base_events.py +++ b/Lib/test/test_asyncio/test_base_events.py @@ -1019,7 +1019,6 @@ async def iter_one(): asyncio.create_task(iter_one()) return status - @unittest.expectedFailure # TODO: RUSTPYTHON; - GC doesn't finalize async generators def test_asyncgen_finalization_by_gc(self): # Async generators should be finalized when garbage collected. self.loop._process_events = mock.Mock() @@ -1035,7 +1034,6 @@ def test_asyncgen_finalization_by_gc(self): test_utils.run_briefly(self.loop) self.assertTrue(status['finalized']) - @unittest.expectedFailure # TODO: RUSTPYTHON; - GC doesn't finalize async generators def test_asyncgen_finalization_by_gc_in_other_thread(self): # Python issue 34769: If garbage collector runs in another # thread, async generators will not finalize in debug From 77e6d31fedb09286808c7bf7a669c0f8262582c1 Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Tue, 7 Jul 2026 18:18:52 +0900 Subject: [PATCH 85/92] Unmark test_sni_callback_refcycle in test_ssl The servername-callback reference cycle is now collected by GC. Assisted-by: Claude --- Lib/test/test_ssl.py | 1 - 1 file changed, 1 deletion(-) diff --git a/Lib/test/test_ssl.py b/Lib/test/test_ssl.py index 10fb5c80b9b..4808da82f20 100644 --- a/Lib/test/test_ssl.py +++ b/Lib/test/test_ssl.py @@ -1526,7 +1526,6 @@ def sni_callback(sock, servername, ctx): pass self.assertIn(libssl_error_reason, str(cm.exception)) self.assertEqual(cm.exception.errno, ssl.SSL_ERROR_SSL) - @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: is not None def test_sni_callback_refcycle(self): # Reference cycles through the servername callback are detected # and cleared. From d2720037c2704d3c5a7bfa3187d33e8c293c47f1 Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Tue, 7 Jul 2026 19:59:07 +0900 Subject: [PATCH 86/92] Widen GC stop-the-world gates from unix-only to all threading builds Change `cfg(all(unix, feature = "threading"))` to `cfg(feature = "threading")` on the stop-the-world machinery so it also compiles and runs on non-unix threading builds: - StopTheWorldState, its stats, stw_trace, and the stop_the_world field - ThreadSlot state/stop_requested/thread fields and their initializers - wait_while_suspended/attach_thread/detach_thread/suspend_if_needed/do_suspend, allow_threads, stop_requested_for_current_thread, and the enter_vm / VmBootstrapGuard / attach_current_thread / release_current_thread / cleanup attach-state wiring - eval_breaker_tripped, check_signals, run_scheduled_gc, signal GC_BIT / schedule_gc / take_gc_scheduled, and the frame.rs safepoint call - CollectStopTheWorld and its use in collect_inner - QSBR::online/offline, now called from attach/detach on all threading builds - debug_assert_current_thread_attached and its type-cache call sites maybe_collect defers auto-collection to the bytecode safepoint on every threading build instead of only unix; non-threading builds keep the inline collect. stw_trace writes to std stderr on non-unix. top_frame publishing (CURRENT_TOP_FRAME_SLOT, set_current_frame), the frame-walk debug assert in collect_inner, and the fork reinit helpers remain unix-only; non-unix keeps ThreadSlot::frames for introspection. Assisted-by: Claude --- crates/vm/src/builtins/type.rs | 4 +-- crates/vm/src/frame.rs | 2 +- crates/vm/src/gc_state.rs | 16 +++++---- crates/vm/src/object/qsbr.rs | 2 -- crates/vm/src/signal.rs | 6 ++-- crates/vm/src/vm/interpreter.rs | 4 +-- crates/vm/src/vm/mod.rs | 28 +++++++++------ crates/vm/src/vm/thread.rs | 62 +++++++++++++-------------------- 8 files changed, 58 insertions(+), 66 deletions(-) diff --git a/crates/vm/src/builtins/type.rs b/crates/vm/src/builtins/type.rs index 372fa3ba4f8..1c98e6861bc 100644 --- a/crates/vm/src/builtins/type.rs +++ b/crates/vm/src/builtins/type.rs @@ -1046,7 +1046,7 @@ impl PyType { name: &'static PyStrInterned, vm: &VirtualMachine, ) -> (Option, u32) { - #[cfg(all(unix, feature = "threading", debug_assertions))] + #[cfg(all(feature = "threading", debug_assertions))] crate::vm::thread::debug_assert_current_thread_attached(); let version = self.tp_version_tag.load(Ordering::Acquire); @@ -1264,7 +1264,7 @@ impl PyType { /// Check if attribute exists in MRO, using method cache for fast check. /// Unlike find_name_in_mro, avoids cloning the value on cache hit. fn has_name_in_mro(&self, name: &'static PyStrInterned) -> bool { - #[cfg(all(unix, feature = "threading", debug_assertions))] + #[cfg(all(feature = "threading", debug_assertions))] crate::vm::thread::debug_assert_current_thread_attached(); let version = self.tp_version_tag.load(Ordering::Acquire); diff --git a/crates/vm/src/frame.rs b/crates/vm/src/frame.rs index 1d39c9edb38..40499efb110 100644 --- a/crates/vm/src/frame.rs +++ b/crates/vm/src/frame.rs @@ -2231,7 +2231,7 @@ impl ExecutingFrame<'_> { // Run a scheduled automatic collection here — a safepoint with // no interpreter locks held — instead of synchronously inside // the allocation that tripped the threshold. - #[cfg(all(unix, feature = "threading"))] + #[cfg(feature = "threading")] vm.run_scheduled_gc(); } let lasti_before = self.lasti(); diff --git a/crates/vm/src/gc_state.rs b/crates/vm/src/gc_state.rs index 5189352b8c9..ebae58f96cb 100644 --- a/crates/vm/src/gc_state.rs +++ b/crates/vm/src/gc_state.rs @@ -146,13 +146,13 @@ struct GcPtr(NonNull); /// well-defined while all other threads are parked at a safepoint. Restarting /// happens explicitly once the snapshot has pinned every object; `Drop` is a /// backstop that also restarts on the early-return paths. -#[cfg(all(unix, feature = "threading"))] +#[cfg(feature = "threading")] struct CollectStopTheWorld { vm: *const crate::VirtualMachine, stopped: bool, } -#[cfg(all(unix, feature = "threading"))] +#[cfg(feature = "threading")] impl CollectStopTheWorld { /// Request stop-the-world when the current thread has an attached VM. /// Falls back to no barrier when no VM is attached (the tracked-object @@ -184,7 +184,7 @@ impl CollectStopTheWorld { } } -#[cfg(all(unix, feature = "threading"))] +#[cfg(feature = "threading")] impl Drop for CollectStopTheWorld { fn drop(&mut self) { self.restart(); @@ -421,7 +421,7 @@ impl GcState { let count0 = self.generations[0].count.load(Ordering::SeqCst) as u32; let threshold0 = self.generations[0].threshold(); if threshold0 > 0 && count0 >= threshold0 { - #[cfg(all(unix, feature = "threading"))] + #[cfg(feature = "threading")] { // Defer to the next bytecode safepoint. Collecting here would // stop the world while this thread may hold an internal lock @@ -431,7 +431,9 @@ impl GcState { crate::signal::schedule_gc(); return false; } - #[cfg(not(all(unix, feature = "threading")))] + // Without threading there is no safepoint to defer to and no other + // thread whose frames could be read mid-mutation, so collect inline. + #[cfg(not(feature = "threading"))] { self.collect(0); return true; @@ -500,7 +502,7 @@ impl GcState { // the world then waits for a thread that cannot reach a safepoint. // Closing it fully would require making those locks stop-the-world // aware; the exclusion above only serializes the fork/GC requesters. - #[cfg(all(unix, feature = "threading"))] + #[cfg(feature = "threading")] let mut stw = CollectStopTheWorld::new(); // Step 1: Gather objects from generations 0..=generation @@ -698,7 +700,7 @@ impl GcState { // the world restarted. Finalizers and tp_clear must not run under // stop-the-world — they execute arbitrary Python — and they only touch // dead/husk objects, never a running frame. - #[cfg(all(unix, feature = "threading"))] + #[cfg(feature = "threading")] stw.restart(); if unreachable.is_empty() { diff --git a/crates/vm/src/object/qsbr.rs b/crates/vm/src/object/qsbr.rs index 9f5fca76727..576cadaeaed 100644 --- a/crates/vm/src/object/qsbr.rs +++ b/crates/vm/src/object/qsbr.rs @@ -126,13 +126,11 @@ mod threading { /// Mark a thread offline (detached); it no longer delays grace /// periods (_Py_qsbr_detach). The thread must not perform lock-free /// cache reads while offline. - #[cfg(any(unix, test))] pub(crate) fn offline(&self, slot: &QsbrSlot) { slot.seq.store(QSBR_OFFLINE, Ordering::Release); } /// Mark a thread online again (_Py_qsbr_attach). - #[cfg(unix)] pub(crate) fn online(&self, slot: &QsbrSlot) { self.quiescent_state(slot); } diff --git a/crates/vm/src/signal.rs b/crates/vm/src/signal.rs index 044a40aa27e..16a097ea62b 100644 --- a/crates/vm/src/signal.rs +++ b/crates/vm/src/signal.rs @@ -26,7 +26,7 @@ const QSBR_BIT: u8 = 1 << 1; /// An automatic collection was scheduled by `maybe_collect` and must run at /// the next bytecode safepoint rather than synchronously inside the /// allocation that tripped the threshold. -#[cfg(all(unix, feature = "threading"))] +#[cfg(feature = "threading")] const GC_BIT: u8 = 1 << 2; #[expect( @@ -142,13 +142,13 @@ pub(crate) fn qsbr_bit_set() -> bool { } /// Schedule an automatic collection to run at the next bytecode safepoint. -#[cfg(all(unix, feature = "threading"))] +#[cfg(feature = "threading")] pub(crate) fn schedule_gc() { EVAL_BREAKER.fetch_or(GC_BIT, Ordering::Release); } /// Clear the scheduled-GC bit, returning whether it had been set. -#[cfg(all(unix, feature = "threading"))] +#[cfg(feature = "threading")] pub(crate) fn take_gc_scheduled() -> bool { EVAL_BREAKER.fetch_and(!GC_BIT, Ordering::Acquire) & GC_BIT != 0 } diff --git a/crates/vm/src/vm/interpreter.rs b/crates/vm/src/vm/interpreter.rs index 43d9560244e..f456e8587ea 100644 --- a/crates/vm/src/vm/interpreter.rs +++ b/crates/vm/src/vm/interpreter.rs @@ -1,4 +1,4 @@ -#[cfg(all(unix, feature = "threading"))] +#[cfg(feature = "threading")] use super::StopTheWorldState; use super::{Context, PyConfig, PyGlobalState, VirtualMachine, setting::Settings, thread}; use crate::{ @@ -134,7 +134,7 @@ where monitoring: PyMutex::default(), monitoring_events: AtomicCell::new(0), instrumentation_version: AtomicU64::new(0), - #[cfg(all(unix, feature = "threading"))] + #[cfg(feature = "threading")] stop_the_world: StopTheWorldState::new(), }); diff --git a/crates/vm/src/vm/mod.rs b/crates/vm/src/vm/mod.rs index 2db7dc5fbee..852e57cc0b4 100644 --- a/crates/vm/src/vm/mod.rs +++ b/crates/vm/src/vm/mod.rs @@ -46,7 +46,7 @@ use crate::{ use alloc::{borrow::Cow, collections::BTreeMap}; #[cfg(all(not(unix), feature = "threading"))] use core::ptr::NonNull; -#[cfg(all(unix, feature = "threading"))] +#[cfg(feature = "threading")] use core::sync::atomic::AtomicI64; use core::{ cell::{Cell, OnceCell, RefCell}, @@ -149,7 +149,7 @@ impl Default for ExceptionStack { /// Stop-the-world state for fork safety. Before `fork()`, the requester /// stops all other Python threads so they are not holding internal locks. -#[cfg(all(unix, feature = "threading"))] +#[cfg(feature = "threading")] pub struct StopTheWorldState { /// Fast-path flag checked in the bytecode loop (like `_PY_EVAL_PLEASE_STOP_BIT`) pub(crate) requested: AtomicBool, @@ -189,7 +189,7 @@ pub struct StopTheWorldState { stats_suspend_wait_yields: AtomicU64, } -#[cfg(all(unix, feature = "threading"))] +#[cfg(feature = "threading")] #[derive(Debug, Clone, Copy)] pub struct StopTheWorldStats { pub stop_calls: u64, @@ -205,14 +205,14 @@ pub struct StopTheWorldStats { pub world_stopped: bool, } -#[cfg(all(unix, feature = "threading"))] +#[cfg(feature = "threading")] impl Default for StopTheWorldState { fn default() -> Self { Self::new() } } -#[cfg(all(unix, feature = "threading"))] +#[cfg(feature = "threading")] impl StopTheWorldState { #[must_use] pub const fn new() -> Self { @@ -662,13 +662,13 @@ impl StopTheWorldState { } } -#[cfg(all(unix, feature = "threading"))] +#[cfg(feature = "threading")] pub(super) fn stw_trace_enabled() -> bool { static ENABLED: std::sync::OnceLock = std::sync::OnceLock::new(); *ENABLED.get_or_init(|| crate::host_env::os::var_os("RUSTPYTHON_STW_TRACE").is_some()) } -#[cfg(all(unix, feature = "threading"))] +#[cfg(feature = "threading")] pub(super) fn stw_trace(msg: core::fmt::Arguments<'_>) { if stw_trace_enabled() { use core::fmt::Write as _; @@ -704,7 +704,13 @@ pub(super) fn stw_trace(msg: core::fmt::Arguments<'_>) { crate::stdlib::_thread::get_ident(), msg ); + #[cfg(unix)] crate::host_env::io::write_stderr_raw(&out.buf[..out.len]); + #[cfg(not(unix))] + { + use std::io::Write as _; + let _ = std::io::stderr().write_all(&out.buf[..out.len]); + } } } @@ -760,7 +766,7 @@ pub struct PyGlobalState { /// local version against this to decide whether re-instrumentation is needed. pub instrumentation_version: AtomicU64, /// Stop-the-world state for pre-fork thread suspension - #[cfg(all(unix, feature = "threading"))] + #[cfg(feature = "threading")] pub stop_the_world: StopTheWorldState, } @@ -2132,7 +2138,7 @@ impl VirtualMachine { return true; } - #[cfg(all(unix, feature = "threading"))] + #[cfg(feature = "threading")] if thread::stop_requested_for_current_thread() { return true; } @@ -2159,7 +2165,7 @@ impl VirtualMachine { } // Suspend this thread if stop-the-world is in progress - #[cfg(all(unix, feature = "threading"))] + #[cfg(feature = "threading")] thread::suspend_if_needed(&self.state.stop_the_world); // Pass a QSBR checkpoint if requested (deferred memory reclamation). @@ -2179,7 +2185,7 @@ impl VirtualMachine { /// Called only from the bytecode-loop safepoint, where no interpreter /// locks are held, so the stop-the-world it performs cannot deadlock /// against a thread blocked on a lock this thread would otherwise hold. - #[cfg(all(unix, feature = "threading"))] + #[cfg(feature = "threading")] pub(crate) fn run_scheduled_gc(&self) { if crate::signal::take_gc_scheduled() { crate::gc_state::gc_state().collect(0); diff --git a/crates/vm/src/vm/thread.rs b/crates/vm/src/vm/thread.rs index 009de4f4da8..5009cb695c6 100644 --- a/crates/vm/src/vm/thread.rs +++ b/crates/vm/src/vm/thread.rs @@ -19,11 +19,11 @@ use std::thread_local; // DETACHED: not executing Python bytecode (in native code, or idle) // ATTACHED: actively executing Python bytecode // SUSPENDED: parked by a stop-the-world request -#[cfg(all(unix, feature = "threading"))] +#[cfg(feature = "threading")] pub const THREAD_DETACHED: i32 = 0; -#[cfg(all(unix, feature = "threading"))] +#[cfg(feature = "threading")] pub const THREAD_ATTACHED: i32 = 1; -#[cfg(all(unix, feature = "threading"))] +#[cfg(feature = "threading")] pub const THREAD_SUSPENDED: i32 = 2; /// Per-thread shared state for sys._current_frames() and sys._current_exceptions(). @@ -46,18 +46,12 @@ pub struct ThreadSlot { pub frames: parking_lot::Mutex>, pub exception: crate::PyAtomicRef>, /// Thread state for stop-the-world: DETACHED / ATTACHED / SUSPENDED - #[cfg(unix)] pub state: core::sync::atomic::AtomicI32, /// Per-thread stop request bit (eval breaker equivalent). - #[cfg(unix)] pub stop_requested: core::sync::atomic::AtomicBool, /// Handle for waking this thread from park in stop-the-world paths. - #[cfg(unix)] pub thread: std::thread::Thread, - /// QSBR state for deferred memory reclamation. On non-unix threading - /// builds there is no attach/detach state machine, so the slot stays - /// online from registration to thread exit; a thread blocked in native - /// code simply delays reclamation until it runs again. + /// QSBR state for deferred memory reclamation. pub(crate) qsbr: Arc, } @@ -145,21 +139,21 @@ pub fn try_with_current_vm(f: impl FnOnce(&VirtualMachine) -> R) -> Option pub fn enter_vm(vm: &VirtualMachine, f: impl FnOnce() -> R) -> R { // Outermost enter_vm: transition DETACHED → ATTACHED - #[cfg(all(unix, feature = "threading"))] + #[cfg(feature = "threading")] let was_outermost = !current_vm_is_set(); // Initialize thread slot for this thread if not already done #[cfg(feature = "threading")] init_thread_slot_if_needed(vm); - #[cfg(all(unix, feature = "threading"))] + #[cfg(feature = "threading")] if was_outermost { attach_thread(vm); } scopeguard::defer! { // Outermost exit: transition ATTACHED → DETACHED - #[cfg(all(unix, feature = "threading"))] + #[cfg(feature = "threading")] if was_outermost { detach_thread(); } @@ -181,21 +175,21 @@ pub fn enter_vm(vm: &VirtualMachine, f: impl FnOnce() -> R) -> R { /// reads unsound. #[must_use] pub(crate) struct VmBootstrapGuard { - #[cfg(all(unix, feature = "threading"))] + #[cfg(feature = "threading")] was_outermost: bool, } impl VmBootstrapGuard { pub(crate) fn new(vm: &VirtualMachine) -> Self { // Outermost: transition DETACHED → ATTACHED - #[cfg(all(unix, feature = "threading"))] + #[cfg(feature = "threading")] let was_outermost = !current_vm_is_set(); // Initialize thread slot for this thread if not already done #[cfg(feature = "threading")] init_thread_slot_if_needed(vm); - #[cfg(all(unix, feature = "threading"))] + #[cfg(feature = "threading")] if was_outermost { attach_thread(vm); } @@ -203,7 +197,7 @@ impl VmBootstrapGuard { VM_STACK.with(|vms| vms.borrow_mut().push(vm.into())); Self { - #[cfg(all(unix, feature = "threading"))] + #[cfg(feature = "threading")] was_outermost, } } @@ -216,7 +210,7 @@ impl Drop for VmBootstrapGuard { }); // Outermost exit: transition ATTACHED → DETACHED - #[cfg(all(unix, feature = "threading"))] + #[cfg(feature = "threading")] if self.was_outermost { detach_thread(); } @@ -250,7 +244,6 @@ pub fn attach_current_thread( init_thread_slot_if_needed(vm); - #[cfg(unix)] attach_thread(vm); VM_STACK.with(|vms| { @@ -277,7 +270,6 @@ pub fn release_current_thread(state: CurrentVmAttachState) { .expect("release_current_thread() called without an attached VM"); }); - #[cfg(unix)] detach_thread(); } @@ -295,7 +287,6 @@ fn init_thread_slot_if_needed(vm: &VirtualMachine) { #[cfg(not(unix))] frames: parking_lot::Mutex::new(Vec::new()), exception: crate::PyAtomicRef::from(None::), - #[cfg(unix)] state: core::sync::atomic::AtomicI32::new( if vm.state.stop_the_world.requested.load(Ordering::Acquire) { // Match init_threadstate(): new thread-state starts @@ -305,9 +296,7 @@ fn init_thread_slot_if_needed(vm: &VirtualMachine) { THREAD_DETACHED }, ), - #[cfg(unix)] stop_requested: core::sync::atomic::AtomicBool::new(false), - #[cfg(unix)] thread: std::thread::current(), qsbr: crate::object::qsbr::QSBR.register(), }); @@ -322,7 +311,7 @@ fn init_thread_slot_if_needed(vm: &VirtualMachine) { /// Transition DETACHED → ATTACHED. Blocks if the thread was SUSPENDED by /// a stop-the-world request (like `_PyThreadState_Attach` + `tstate_wait_attach`). -#[cfg(all(unix, feature = "threading"))] +#[cfg(feature = "threading")] fn wait_while_suspended(slot: &ThreadSlot) -> u64 { let mut wait_yields = 0u64; while slot.state.load(Ordering::Acquire) == THREAD_SUSPENDED { @@ -332,7 +321,7 @@ fn wait_while_suspended(slot: &ThreadSlot) -> u64 { wait_yields } -#[cfg(all(unix, feature = "threading"))] +#[cfg(feature = "threading")] fn attach_thread(vm: &VirtualMachine) { CURRENT_THREAD_SLOT.with(|slot| { if let Some(s) = slot.borrow().as_ref() { @@ -375,7 +364,7 @@ fn attach_thread(vm: &VirtualMachine) { } /// Transition ATTACHED → DETACHED (like `_PyThreadState_Detach`). -#[cfg(all(unix, feature = "threading"))] +#[cfg(feature = "threading")] fn detach_thread() { CURRENT_THREAD_SLOT.with(|slot| { if let Some(s) = slot.borrow().as_ref() { @@ -407,7 +396,7 @@ fn detach_thread() { /// to park this thread during blocking operations. /// /// `Py_BEGIN_ALLOW_THREADS` / `Py_END_ALLOW_THREADS` equivalent. -#[cfg(all(unix, feature = "threading"))] +#[cfg(feature = "threading")] pub fn allow_threads(vm: &VirtualMachine, f: impl FnOnce() -> R) -> R { // Preserve save/restore semantics: // only detach if this call observed ATTACHED at entry, and always restore @@ -428,8 +417,8 @@ pub fn allow_threads(vm: &VirtualMachine, f: impl FnOnce() -> R) -> R { result } -/// No-op on non-unix or non-threading builds. -#[cfg(not(all(unix, feature = "threading")))] +/// No-op on non-threading builds. +#[cfg(not(feature = "threading"))] pub fn allow_threads(_vm: &VirtualMachine, f: impl FnOnce() -> R) -> R { f() } @@ -437,7 +426,7 @@ pub fn allow_threads(_vm: &VirtualMachine, f: impl FnOnce() -> R) -> R { /// Called from check_signals when stop-the-world is requested. /// Transitions ATTACHED → SUSPENDED and waits until released /// (like `_PyThreadState_Suspend` + `_PyThreadState_Attach`). -#[cfg(all(unix, feature = "threading"))] +#[cfg(feature = "threading")] pub fn suspend_if_needed(stw: &super::StopTheWorldState) { let should_suspend = CURRENT_THREAD_SLOT.with(|slot| { slot.borrow() @@ -460,7 +449,7 @@ pub fn suspend_if_needed(stw: &super::StopTheWorldState) { do_suspend(stw); } -#[cfg(all(unix, feature = "threading"))] +#[cfg(feature = "threading")] #[cold] fn do_suspend(stw: &super::StopTheWorldState) { CURRENT_THREAD_SLOT.with(|slot| { @@ -537,7 +526,7 @@ fn do_suspend(stw: &super::StopTheWorldState) { }); } -#[cfg(all(unix, feature = "threading"))] +#[cfg(feature = "threading")] #[inline] #[must_use] pub fn stop_requested_for_current_thread() -> bool { @@ -578,7 +567,7 @@ pub(crate) fn qsbr_checkpoint() { /// Debug check: lock-free type-cache reads are only sound on threads that /// are registered with QSBR and currently ATTACHED. -#[cfg(all(unix, feature = "threading", debug_assertions))] +#[cfg(all(feature = "threading", debug_assertions))] pub(crate) fn debug_assert_current_thread_attached() { CURRENT_THREAD_SLOT.with(|slot| { if let Some(s) = slot.borrow().as_ref() { @@ -682,7 +671,7 @@ pub fn cleanup_current_thread_frames(vm: &VirtualMachine) { // A dying thread should not remain logically ATTACHED while its // thread-state slot is being removed. - #[cfg(all(unix, feature = "threading"))] + #[cfg(feature = "threading")] if let Some(slot) = ¤t_slot { let _ = slot.state.compare_exchange( THREAD_ATTACHED, @@ -704,7 +693,7 @@ pub fn cleanup_current_thread_frames(vm: &VirtualMachine) { None }; - #[cfg(all(unix, feature = "threading"))] + #[cfg(feature = "threading")] if let Some(slot) = &_removed && vm.state.stop_the_world.requested.load(Ordering::Acquire) && thread_id != vm.state.stop_the_world.requester_ident() @@ -755,11 +744,8 @@ pub fn reinit_frame_slot_after_fork(vm: &VirtualMachine) { #[cfg(not(unix))] frames: parking_lot::Mutex::new(current_frames), exception: crate::PyAtomicRef::from(vm.topmost_exception()), - #[cfg(unix)] state: core::sync::atomic::AtomicI32::new(THREAD_ATTACHED), - #[cfg(unix)] stop_requested: core::sync::atomic::AtomicBool::new(false), - #[cfg(unix)] thread: std::thread::current(), qsbr: crate::object::qsbr::QSBR.register(), }); From e590e3b40a4cb41f24405f55ef6049e59f04f032 Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Tue, 7 Jul 2026 23:18:58 +0900 Subject: [PATCH 87/92] Detach current thread around blocking _winapi/_overlapped waits Wrap the blocking Windows wait calls in `vm.allow_threads` so the calling thread transitions ATTACHED -> DETACHED for the duration of the wait: - _winapi: WaitForSingleObject, WaitForMultipleObjects, BatchedWaitForMultipleObjects, ConnectNamedPipe, ReadFile, Overlapped.GetOverlappedResult - _overlapped: Overlapped.getresult These previously blocked while ATTACHED, so a stop-the-world requester could never suspend the thread and spun in its wait loop indefinitely. --- crates/stdlib/src/overlapped.rs | 5 +++-- crates/vm/src/stdlib/_winapi.rs | 30 ++++++++++++++++++------------ 2 files changed, 21 insertions(+), 14 deletions(-) diff --git a/crates/stdlib/src/overlapped.rs b/crates/stdlib/src/overlapped.rs index 4ce3d3ba830..8610fadb3bf 100644 --- a/crates/stdlib/src/overlapped.rs +++ b/crates/stdlib/src/overlapped.rs @@ -309,8 +309,9 @@ mod _overlapped { return Err(vm.new_value_error("operation failed to start")); } - let result = - host_overlapped::get_overlapped_result(inner.handle, &inner.overlapped, wait); + let result = vm.allow_threads(|| { + host_overlapped::get_overlapped_result(inner.handle, &inner.overlapped, wait) + }); let transferred = result.transferred; let err = result.error; inner.error = err; diff --git a/crates/vm/src/stdlib/_winapi.rs b/crates/vm/src/stdlib/_winapi.rs index f6faf6a8a95..7d4880c2f82 100644 --- a/crates/vm/src/stdlib/_winapi.rs +++ b/crates/vm/src/stdlib/_winapi.rs @@ -357,7 +357,8 @@ mod _winapi { } else { ms as u32 }; - host_winapi::wait_for_single_object(h.0, ms).map_err(|e| e.to_pyexception(vm)) + vm.allow_threads(|| host_winapi::wait_for_single_object(h.0, ms)) + .map_err(|e| e.to_pyexception(vm)) } #[pyfunction] @@ -381,7 +382,7 @@ mod _winapi { return Err(vm.new_value_error("WaitForMultipleObjects supports at most 64 handles")); } - host_winapi::wait_for_multiple_objects(&handles, wait_all, milliseconds) + vm.allow_threads(|| host_winapi::wait_for_multiple_objects(&handles, wait_all, milliseconds)) .map_err(|e| e.to_pyexception(vm)) } @@ -566,8 +567,7 @@ mod _winapi { #[pymethod] fn GetOverlappedResult(&self, wait: bool, vm: &VirtualMachine) -> PyResult<(u32, u32)> { let mut inner = self.inner.lock(); - inner - .get_result(wait) + vm.allow_threads(|| inner.get_result(wait)) .map(|result| (result.transferred, result.error)) .map_err(|e| e.to_pyexception(vm)) } @@ -634,7 +634,8 @@ mod _winapi { } Ok(ov.into_pyobject(vm)) } else { - host_winapi::connect_named_pipe(handle.0).map_err(|e| e.to_pyexception(vm))?; + vm.allow_threads(|| host_winapi::connect_named_pipe(handle.0)) + .map_err(|e| e.to_pyexception(vm))?; Ok(vm.ctx.none()) } } @@ -802,7 +803,9 @@ mod _winapi { return Ok(result.into()); } - let result = host_winapi::read_file(handle.0, size).map_err(|e| e.to_pyexception(vm))?; + let result = vm + .allow_threads(|| host_winapi::read_file(handle.0, size)) + .map_err(|e| e.to_pyexception(vm))?; Ok(vm .ctx .new_tuple(vec![ @@ -926,12 +929,15 @@ mod _winapi { #[cfg(not(feature = "threading"))] let sigint_event: Option = None; - match host_winapi::batched_wait_for_multiple_objects( - &handles, - wait_all, - milliseconds, - sigint_event, - ) { + let batched_result = vm.allow_threads(|| { + host_winapi::batched_wait_for_multiple_objects( + &handles, + wait_all, + milliseconds, + sigint_event, + ) + }); + match batched_result { Ok(host_winapi::BatchedWaitResult::All) => Ok(vm.ctx.none()), Ok(host_winapi::BatchedWaitResult::Indices(indices)) => Ok(vm .ctx From cdd89083fdf7aa272b47a1b6e0cb11fbedef9cfa Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Wed, 8 Jul 2026 00:22:51 +0900 Subject: [PATCH 88/92] Format WaitForMultipleObjects allow_threads closure --- crates/vm/src/stdlib/_winapi.rs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/crates/vm/src/stdlib/_winapi.rs b/crates/vm/src/stdlib/_winapi.rs index 7d4880c2f82..e1b86f2ef54 100644 --- a/crates/vm/src/stdlib/_winapi.rs +++ b/crates/vm/src/stdlib/_winapi.rs @@ -382,8 +382,10 @@ mod _winapi { return Err(vm.new_value_error("WaitForMultipleObjects supports at most 64 handles")); } - vm.allow_threads(|| host_winapi::wait_for_multiple_objects(&handles, wait_all, milliseconds)) - .map_err(|e| e.to_pyexception(vm)) + vm.allow_threads(|| { + host_winapi::wait_for_multiple_objects(&handles, wait_all, milliseconds) + }) + .map_err(|e| e.to_pyexception(vm)) } #[pyfunction] From b6e8cbf678013c24f3b830800c973d749bf3a6e9 Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Wed, 8 Jul 2026 00:53:35 +0900 Subject: [PATCH 89/92] Treat concurrent sni_callback removal as no-op in invoke_sni_callback --- crates/stdlib/src/ssl.rs | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/crates/stdlib/src/ssl.rs b/crates/stdlib/src/ssl.rs index edcdf18fa09..81d69b8c64e 100644 --- a/crates/stdlib/src/ssl.rs +++ b/crates/stdlib/src/ssl.rs @@ -2709,13 +2709,13 @@ mod _ssl { sni_name: Option<&str>, vm: &VirtualMachine, ) -> PyResult<()> { - let callback = self - .context - .read() - .sni_callback - .read() - .clone() - .ok_or_else(|| vm.new_value_error("SNI callback not set"))?; + // The callback may have been cleared (sni_callback = None) between the + // handshake deciding to invoke it and this point. A concurrent removal + // is not an error: there is simply nothing to run. + let callback = self.context.read().sni_callback.read().clone(); + let Some(callback) = callback else { + return Ok(()); + }; let ssl_sock = self.owner.read().clone().unwrap_or_else(|| vm.ctx.none()); let server_name_py: PyObjectRef = match sni_name { From 5e2d09506ee35800f9ce6e5a3de27cba2e753b11 Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Wed, 8 Jul 2026 00:45:44 +0900 Subject: [PATCH 90/92] Assert capi refcount on a fresh mortal list instead of the int type object The refcount test asserted exact incref/decref deltas on PyInt's shared type object. That object's reference count is perturbed by the other capi tests running in parallel, and is immortal under some interpreter configurations, so the deltas were not reliably +1/-1. Assert them on a freshly created, uniquely owned list whose reference count is private to the test and mortal. Assisted-by: Claude --- crates/capi/src/refcount.rs | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/crates/capi/src/refcount.rs b/crates/capi/src/refcount.rs index 849b130e292..48c7132b0f1 100644 --- a/crates/capi/src/refcount.rs +++ b/crates/capi/src/refcount.rs @@ -26,14 +26,18 @@ pub unsafe extern "C" fn Py_REFCNT(op: *mut PyObject) -> isize { #[cfg(test)] mod tests { + use pyo3::ffi; use pyo3::prelude::*; - use pyo3::types::PyInt; - use pyo3::{PyTypeInfo, ffi}; + use pyo3::types::PyList; #[test] fn refcount() { Python::attach(|py| unsafe { - let obj = PyInt::type_object(py); + // A freshly created, non-empty list is uniquely owned here: its + // reference count is private to this test (so parallel tests cannot + // perturb it) and it is mortal (not interned), so incref then decref + // must move the count by exactly one and back. + let obj = PyList::new(py, [1, 2, 3]).unwrap(); let ref_count = ffi::Py_REFCNT(obj.as_ptr()); let obj_clone = obj.clone(); assert_eq!(ffi::Py_REFCNT(obj.as_ptr()), ref_count + 1); From 4f8116d3d2f2d7d9ff5a191aa288e118ac70a06f Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Wed, 8 Jul 2026 10:57:13 +0900 Subject: [PATCH 91/92] Traverse and GC-track PyOSError instances The `#[pyexception]` struct macro now forwards a `traverse` option to the generated `#[pyclass]`, and `ExceptionItemMeta` accepts the `traverse` key. `PyOSError` is marked `traverse = "manual"`, so `HAS_TRAVERSE` is true and OSError-family instances are tracked at creation and traversed by the collector. `PyOSError::traverse` now visits the underlying `PyBaseException` (traceback, cause, context, args) instead of `PyException::try_traverse`, which was a no-op because `PyException` has `HAS_TRAVERSE = false`. --- crates/derive-impl/src/pyclass.rs | 9 ++++++++- crates/derive-impl/src/util.rs | 11 +++++++++-- crates/vm/src/exceptions.rs | 9 ++++++--- 3 files changed, 23 insertions(+), 6 deletions(-) diff --git a/crates/derive-impl/src/pyclass.rs b/crates/derive-impl/src/pyclass.rs index aed2c5d8c5f..809d3164b4a 100644 --- a/crates/derive-impl/src/pyclass.rs +++ b/crates/derive-impl/src/pyclass.rs @@ -796,8 +796,15 @@ pub(crate) fn impl_pyexception(attr: PunctuatedNestedMeta, item: &Item) -> Resul quote! {} }; + // Forward a `traverse` option to the generated `#[pyclass]` so exception + // payloads with a manual `Traverse` impl are GC-tracked and traversed. + let traverse_attr = match class_meta.inner()._optional_str("traverse").ok().flatten() { + Some(value) => quote! { , traverse = #value }, + None => quote! {}, + }; + let ret = quote! { - #[pyclass(module = false, name = #class_name, base = #base_class_name)] + #[pyclass(module = false, name = #class_name, base = #base_class_name #traverse_attr)] #item #impl_pyclass }; diff --git a/crates/derive-impl/src/util.rs b/crates/derive-impl/src/util.rs index 60b2296cea7..a0708444691 100644 --- a/crates/derive-impl/src/util.rs +++ b/crates/derive-impl/src/util.rs @@ -465,8 +465,15 @@ impl ClassItemMeta { pub(crate) struct ExceptionItemMeta(ClassItemMeta); impl ItemMeta for ExceptionItemMeta { - const ALLOWED_NAMES: &'static [&'static str] = - &["module", "name", "base", "unhashable", "ctx", "impl"]; + const ALLOWED_NAMES: &'static [&'static str] = &[ + "module", + "name", + "base", + "unhashable", + "ctx", + "impl", + "traverse", + ]; fn from_inner(inner: ItemMetaInner) -> Self { Self(ClassItemMeta(inner)) diff --git a/crates/vm/src/exceptions.rs b/crates/vm/src/exceptions.rs index 67f27a72dc2..845b01c3816 100644 --- a/crates/vm/src/exceptions.rs +++ b/crates/vm/src/exceptions.rs @@ -1583,7 +1583,7 @@ impl ToPyException for rustpython_host_env::multiprocessing::SemError { pub(super) mod types { use crate::common::lock::PyRwLock; - use crate::object::{MaybeTraverse, Traverse, TraverseFn}; + use crate::object::{Traverse, TraverseFn}; #[cfg_attr(target_arch = "wasm32", allow(unused_imports))] use crate::{ AsObject, Py, PyAtomicRef, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, @@ -1900,7 +1900,7 @@ pub(super) mod types { #[repr(transparent)] pub struct PyUnboundLocalError(PyNameError); - #[pyexception(name, base = PyException, ctx = "os_error")] + #[pyexception(name, base = PyException, ctx = "os_error", traverse = "manual")] #[repr(C)] pub struct PyOSError { base: PyException, @@ -1930,7 +1930,10 @@ pub(super) mod types { unsafe impl Traverse for PyOSError { fn traverse(&self, tracer_fn: &mut TraverseFn<'_>) { - self.base.try_traverse(tracer_fn); + // `self.base` is a `PyException` newtype whose `MaybeTraverse` is a + // no-op; reach the underlying `PyBaseException` so its traceback, + // cause, context and args are visited by the collector. + self.base.0.traverse(tracer_fn); if let Some(obj) = self.errno.deref() { tracer_fn(obj); } From db737e1865bf74473fde73591c8dd6cfcd0a1173 Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Wed, 8 Jul 2026 14:10:11 +0900 Subject: [PATCH 92/92] Unmark test_blockingioerror in test_io The BlockingIOError reference cycle is now collected by GC. Assisted-by: Claude --- Lib/test/test_io.py | 1 - 1 file changed, 1 deletion(-) diff --git a/Lib/test/test_io.py b/Lib/test/test_io.py index c51b547f31a..99ab6c7ba90 100644 --- a/Lib/test/test_io.py +++ b/Lib/test/test_io.py @@ -4473,7 +4473,6 @@ def test_io_after_close(self): self.assertRaises(ValueError, f.writelines, []) self.assertRaises(ValueError, next, f) - @unittest.expectedFailure # TODO: RUSTPYTHON; cyclic gc def test_blockingioerror(self): # Various BlockingIOError issues class C(str):