diff --git a/.cspell.dict/rustpython.txt b/.cspell.dict/rustpython.txt index 8cd08358019..07099bbb171 100644 --- a/.cspell.dict/rustpython.txt +++ b/.cspell.dict/rustpython.txt @@ -27,6 +27,7 @@ pystr pystruct pystructseq pytype +qsbr rustix struc zelf diff --git a/.cspell.json b/.cspell.json index af2f1401d95..6f0ac213672 100644 --- a/.cspell.json +++ b/.cspell.json @@ -79,8 +79,11 @@ "mcache", "oparg", "opargs", + "pointee", "pyc", "reborrow", + "reborrows", + "reparenting", "reraises", "reraising", "significand", diff --git a/.gitignore b/.gitignore index b5887be53b5..09e1b97b9f8 100644 --- a/.gitignore +++ b/.gitignore @@ -27,4 +27,5 @@ Lib/site-packages/* Lib/test/data/* !Lib/test/data/README cpython/ -.claude/scheduled_tasks.lock \ No newline at end of file +.claude/scheduled_tasks.lock +docs/superpowers/ \ No newline at end of file diff --git a/Cargo.lock b/Cargo.lock index 5eb95bea29c..73afeda8a18 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3700,6 +3700,7 @@ dependencies = [ "rustpython-ruff_text_size", "rustpython-unicode", "rustpython-vm", + "scopeguard", "sha1 0.11.0", "sha2", "sha3", @@ -3754,6 +3755,7 @@ dependencies = [ "indexmap", "is-macro", "itertools 0.15.0", + "itoa", "libc", "log", "malachite-bigint", diff --git a/Cargo.toml b/Cargo.toml index d8081f50166..540324a4aaf 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -229,6 +229,7 @@ hmac = "0.13" indexmap = { version = "2.14.0", features = ["std"] } insta = "1.47" itertools = { version = "0.15.0", default-features = false, features = ["use_alloc"] } +itoa = "1" is-macro = "0.3.7" js-sys = "0.3" junction = "2.0.0" diff --git a/Lib/test/test_asyncio/test_base_events.py b/Lib/test/test_asyncio/test_base_events.py index 92895bbb420..1b727f3b1fe 100644 --- a/Lib/test/test_asyncio/test_base_events.py +++ b/Lib/test/test_asyncio/test_base_events.py @@ -1019,7 +1019,6 @@ async def iter_one(): asyncio.create_task(iter_one()) return status - @unittest.expectedFailure # TODO: RUSTPYTHON; - GC doesn't finalize async generators def test_asyncgen_finalization_by_gc(self): # Async generators should be finalized when garbage collected. self.loop._process_events = mock.Mock() @@ -1035,7 +1034,6 @@ def test_asyncgen_finalization_by_gc(self): test_utils.run_briefly(self.loop) self.assertTrue(status['finalized']) - @unittest.expectedFailure # TODO: RUSTPYTHON; - GC doesn't finalize async generators def test_asyncgen_finalization_by_gc_in_other_thread(self): # Python issue 34769: If garbage collector runs in another # thread, async generators will not finalize in debug diff --git a/Lib/test/test_descr.py b/Lib/test/test_descr.py index 92bf7998d75..0b19496ec4b 100644 --- a/Lib/test/test_descr.py +++ b/Lib/test/test_descr.py @@ -4154,7 +4154,6 @@ class E(D): else: self.fail("shouldn't be able to create inheritance cycles") - @unittest.expectedFailure # TODO: RUSTPYTHON def test_builtin_bases(self): # Make sure all the builtin types can have their base queried without # segfaulting. See issue #5787. @@ -4199,7 +4198,6 @@ class D(C): else: self.fail("best_base calculation found wanting") - @unittest.expectedFailure # TODO: RUSTPYTHON def test_unsubclassable_types(self): with self.assertRaises(TypeError): class X(type(None)): diff --git a/Lib/test/test_frame.py b/Lib/test/test_frame.py index ae02e2a59f9..53d42a595b7 100644 --- a/Lib/test/test_frame.py +++ b/Lib/test/test_frame.py @@ -315,7 +315,6 @@ def inner(): % (file_repr, offset + 5)) class TestFrameLocals(unittest.TestCase): - @unittest.expectedFailure # TODO: RUSTPYTHON def test_scope(self): class A: x = 1 @@ -333,7 +332,6 @@ def f(): self.assertEqual(locals()['y'], 2) f() - @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: 1 != 2 def test_closure(self): x = 1 y = 2 @@ -356,7 +354,6 @@ def test_closure_with_inline_comprehension(self): lst = [locals() for k in [0]] self.assertEqual(lst[0]['k'], 0) - @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: 3 != 4 def test_as_dict(self): x = 1 y = 2 @@ -414,7 +411,6 @@ def test_non_string_key(self): d[1] = 2 self.assertEqual(d[1], 2) - @unittest.expectedFailure # TODO: RUSTPYTHON; UnboundLocalError: local variable 'b' referenced before assignment def test_write_with_hidden(self): def f(): f_locals = [sys._getframe().f_locals for b in [0]][0] @@ -426,7 +422,6 @@ def f(): c = 0 f() - @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: != 'a.b.c' def test_local_objects(self): o = object() k = '.'.join(['a', 'b', 'c']) @@ -457,7 +452,6 @@ def test_repr(self): frame = sys._getframe() self.assertEqual(repr(frame.f_locals), repr(dict(frame.f_locals))) - @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: ValueError not raised def test_delete(self): x = 1 d = sys._getframe().f_locals @@ -501,7 +495,6 @@ def test_sizeof(self): proxy = sys._getframe().f_locals support.check_sizeof(self, proxy, support.calcobjsize("P")) - @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: TypeError not raised def test_unsupport(self): x = 1 d = sys._getframe().f_locals @@ -536,7 +529,6 @@ def __eq__(self, other): return StringSubclass('x'), ImpostorX(), 'x' - @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: dict_keys(['obj', 'x']) != ['obj', 'x', 'proxy'] def test_proxy_key_stringlikes_overwrite(self): def f(obj): x = 1 @@ -559,7 +551,6 @@ def f(obj): self.assertEqual(keys_snapshot, expected_keys) self.assertEqual(proxy_snapshot, expected_dict) - @unittest.expectedFailure # TODO: RUSTPYTHON; UnboundLocalError: local variable 'b' referenced before assignment def test_proxy_key_stringlikes_ftrst_write(self): def f(obj): proxy = sys._getframe().f_locals @@ -587,7 +578,6 @@ class ObjectSubclass: with self.assertRaises(TypeError): proxy[obj] = 0 - @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: 'dict' != 'FrameLocalsProxy' def test_constructor(self): FrameLocalsProxy = type([sys._getframe().f_locals for x in range(1)][0]) diff --git a/Lib/test/test_generators.py b/Lib/test/test_generators.py index 8ede6e22fab..b3826f4229d 100644 --- a/Lib/test/test_generators.py +++ b/Lib/test/test_generators.py @@ -762,7 +762,6 @@ def get_frame(index): self.assertIn('a', frame_locals) self.assertEqual(frame_locals['a'], 42) - @unittest.expectedFailure # TODO: RUSTPYTHON; frame locals don't survive generator deallocation def test_frame_locals_outlive_generator(self): frame_locals1 = None diff --git a/Lib/test/test_inspect/test_inspect.py b/Lib/test/test_inspect/test_inspect.py index f7a7c0cc825..74126751835 100644 --- a/Lib/test/test_inspect/test_inspect.py +++ b/Lib/test/test_inspect/test_inspect.py @@ -588,7 +588,6 @@ def test_frame(self): self.assertEqual(inspect.formatargvalues(args, varargs, varkw, locals), '(x=11, y=14)') - @unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError: 'NoneType' object has no attribute 'f_code' def test_previous_frame(self): args, varargs, varkw, locals = inspect.getargvalues(mod.fr.f_back) self.assertEqual(args, ['a', 'b', 'c', 'd', 'e', 'f']) diff --git a/Lib/test/test_io.py b/Lib/test/test_io.py index c51b547f31a..99ab6c7ba90 100644 --- a/Lib/test/test_io.py +++ b/Lib/test/test_io.py @@ -4473,7 +4473,6 @@ def test_io_after_close(self): self.assertRaises(ValueError, f.writelines, []) self.assertRaises(ValueError, next, f) - @unittest.expectedFailure # TODO: RUSTPYTHON; cyclic gc def test_blockingioerror(self): # Various BlockingIOError issues class C(str): diff --git a/Lib/test/test_monitoring.py b/Lib/test/test_monitoring.py index 1f72b552c6c..30eee65dc12 100644 --- a/Lib/test/test_monitoring.py +++ b/Lib/test/test_monitoring.py @@ -1984,7 +1984,6 @@ def f(): ] return d["f"], expected - @unittest.expectedFailure # TODO: RUSTPYTHON; line number differences in multi-line super() calls def test_method_call_error(self): nonopt_func, nonopt_expected = self._super_method_call_error(optimized=False) opt_func, opt_expected = self._super_method_call_error(optimized=True) @@ -2022,7 +2021,6 @@ def f(): ] return d["f"], expected - @unittest.expectedFailure # TODO: RUSTPYTHON; line number differences in multi-line super() calls def test_attr(self): nonopt_func, nonopt_expected = self._super_attr(optimized=False) opt_func, opt_expected = self._super_attr(optimized=True) diff --git a/Lib/test/test_pdb.py b/Lib/test/test_pdb.py index 97f084088c5..cd1e88f5475 100644 --- a/Lib/test/test_pdb.py +++ b/Lib/test/test_pdb.py @@ -3122,7 +3122,7 @@ def test_pdb_issue_gh_101673(): ... a = 1 ... import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() - >>> with PdbTestInput([ # TODO: RUSTPYTHON # doctest: +NORMALIZE_WHITESPACE +EXPECTED_FAILURE + >>> with PdbTestInput([ # doctest: +NORMALIZE_WHITESPACE ... '!a = 2', ... 'll', ... 'p a', diff --git a/Lib/test/test_ssl.py b/Lib/test/test_ssl.py index 10fb5c80b9b..4808da82f20 100644 --- a/Lib/test/test_ssl.py +++ b/Lib/test/test_ssl.py @@ -1526,7 +1526,6 @@ def sni_callback(sock, servername, ctx): pass self.assertIn(libssl_error_reason, str(cm.exception)) self.assertEqual(cm.exception.errno, ssl.SSL_ERROR_SSL) - @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: is not None def test_sni_callback_refcycle(self): # Reference cycles through the servername callback are detected # and cleared. diff --git a/Lib/test/test_traceback.py b/Lib/test/test_traceback.py index ec56f26a735..42f066a6239 100644 --- a/Lib/test/test_traceback.py +++ b/Lib/test/test_traceback.py @@ -3141,7 +3141,6 @@ def last_returns_frame4(self): def last_returns_frame5(self): return self.last_returns_frame4() - @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: 1 not greater than 5 def test_extract_stack(self): frame = self.last_returns_frame5() def extract(**kwargs): diff --git a/crates/capi/src/refcount.rs b/crates/capi/src/refcount.rs index 849b130e292..48c7132b0f1 100644 --- a/crates/capi/src/refcount.rs +++ b/crates/capi/src/refcount.rs @@ -26,14 +26,18 @@ pub unsafe extern "C" fn Py_REFCNT(op: *mut PyObject) -> isize { #[cfg(test)] mod tests { + use pyo3::ffi; use pyo3::prelude::*; - use pyo3::types::PyInt; - use pyo3::{PyTypeInfo, ffi}; + use pyo3::types::PyList; #[test] fn refcount() { Python::attach(|py| unsafe { - let obj = PyInt::type_object(py); + // A freshly created, non-empty list is uniquely owned here: its + // reference count is private to this test (so parallel tests cannot + // perturb it) and it is mortal (not interned), so incref then decref + // must move the count by exactly one and back. + let obj = PyList::new(py, [1, 2, 3]).unwrap(); let ref_count = ffi::Py_REFCNT(obj.as_ptr()); let obj_clone = obj.clone(); assert_eq!(ffi::Py_REFCNT(obj.as_ptr()), ref_count + 1); diff --git a/crates/common/src/refcount.rs b/crates/common/src/refcount.rs index c589ead40f6..4d52e1382e6 100644 --- a/crates/common/src/refcount.rs +++ b/crates/common/src/refcount.rs @@ -1,10 +1,14 @@ use crate::atomic::{Ordering, PyAtomic, Radium}; // State layout (usize): -// [1 bit: destructed] [1 bit: reserved] [1 bit: leaked] [N bits: weak_count] [M bits: strong_count] +// [1 bit: destructed] [1 bit: published] [1 bit: leaked] [N bits: weak_count] [M bits: strong_count] // 64-bit: N=30, M=31. 32-bit: N=14, M=15. const FLAG_BITS: u32 = 3; const DESTRUCTED: usize = 1 << (usize::BITS - 1); +/// Object was published to a lock-free cache; memory reclamation is +/// deferred through QSBR so concurrent try-incref readers never touch +/// freed memory. Sticky once set. +const PUBLISHED: usize = 1 << (usize::BITS - 2); const LEAKED: usize = 1 << (usize::BITS - 3); const TOTAL_COUNT_WIDTH: u32 = usize::BITS - FLAG_BITS; const WEAK_WIDTH: u32 = TOTAL_COUNT_WIDTH / 2; @@ -72,8 +76,8 @@ impl State { /// Reference count using state layout with LEAKED support. /// /// State layout (usize): -/// 64-bit: [1 bit: destructed] [1 bit: reserved] [1 bit: leaked] [30 bits: weak_count] [31 bits: strong_count] -/// 32-bit: [1 bit: destructed] [1 bit: reserved] [1 bit: leaked] [14 bits: weak_count] [15 bits: strong_count] +/// 64-bit: [1 bit: destructed] [1 bit: published] [1 bit: leaked] [30 bits: weak_count] [31 bits: strong_count] +/// 32-bit: [1 bit: destructed] [1 bit: published] [1 bit: leaked] [14 bits: weak_count] [15 bits: strong_count] pub struct RefCount { state: PyAtomic, } @@ -187,6 +191,17 @@ impl RefCount { pub fn is_leaked(&self) -> bool { State::from_raw(self.state.load(Ordering::Acquire)).leaked() } + + /// Mark the object as published to a lock-free cache (sticky). + #[inline] + pub fn mark_published(&self) { + self.state.fetch_or(PUBLISHED, Ordering::Release); + } + + #[inline] + pub fn is_published(&self) -> bool { + (self.state.load(Ordering::Acquire) & PUBLISHED) != 0 + } } // Deferred Drop Infrastructure @@ -279,3 +294,24 @@ pub fn flush_deferred_drops() { } }); } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn published_bit_survives_refcount_traffic() { + let rc = RefCount::new(); // strong = 1 + assert!(!rc.is_published()); + rc.mark_published(); + assert!(rc.is_published()); + rc.inc(); // strong = 2 + assert!(rc.is_published()); + assert!(!rc.dec()); // strong = 1 + assert!(rc.is_published()); + assert!(rc.safe_inc()); // strong = 2 + assert!(!rc.dec()); // strong = 1 + assert!(rc.dec()); // strong = 0 -> true + assert!(rc.is_published()); + } +} diff --git a/crates/compiler-core/src/bytecode/instruction.rs b/crates/compiler-core/src/bytecode/instruction.rs index 76871b2c97e..63e29222570 100644 --- a/crates/compiler-core/src/bytecode/instruction.rs +++ b/crates/compiler-core/src/bytecode/instruction.rs @@ -18,23 +18,29 @@ macro_rules! define_opcodes { } ) => { #[derive(Clone, Copy, Debug, Eq, PartialEq)] + #[repr($typ)] $opcode_vis enum $opcode_name { - $($op_name),* + $($op_name = $op_id),* } impl $opcode_name { #[doc = concat!("Converts this opcode to [`", stringify!($instr_name), "`].")] #[must_use] + #[inline] $opcode_vis const fn as_instruction(&self) -> $instr_name { - match self { - $( - Self::$op_name => $instr_name::$op_name $({ $arg_name: Arg::marker() })?, - )* - } + // SAFETY: `$opcode_name` and `$instr_name` are both `#[repr($typ)]` + // enums sharing identical explicit discriminants, and every + // `$instr_name` payload field is the zero-sized `Arg` marker + // (see the `size_of` assertion near its definition), so both + // enums have the same one-`$typ`-wide representation: just the + // discriminant. Converting a live `$opcode_name` value therefore + // yields the `$instr_name` variant with the matching discriminant. + unsafe { core::mem::transmute(*self) } } /// Map a specialized or instrumented opcode back to its adaptive (base) variant. #[must_use] + #[inline] $opcode_vis const fn deoptimize(self) -> Self { match self.deopt() { Some(v) => v, @@ -49,6 +55,9 @@ macro_rules! define_opcodes { } // NOTE: Keep private. Will be exposed under `try_from_u8/try_from_u16`. + // Kept as a match rather than a range check + transmute: `$op_id` + // values are not contiguous (specialized/instrumented opcodes leave + // gaps), so validity can't be expressed as a simple bound. pub(super) const fn try_from_numeric(value: $typ) -> Result { match value { $($op_id => Ok(Self::$op_name),)* @@ -58,10 +67,11 @@ macro_rules! define_opcodes { // NOTE: Keep private. Will be exposed under `as_u8/as_u16`. #[must_use] + #[inline] pub(super) const fn as_numeric(self) -> $typ { - match self { - $(Self::$op_name => $op_id,)* - } + // `$opcode_name` is `#[repr($typ)]` with an explicit `$op_id` + // discriminant on every variant, so this is a plain identity cast. + self as $typ } } @@ -95,15 +105,24 @@ macro_rules! define_opcodes { ),* } + // Every `$instr_name` payload field is the zero-sized `Arg` marker, so + // (combined with the `#[repr($typ)]` above) each variant's representation + // is exactly its `$typ` discriminant with no padding. `as_opcode` and + // `$opcode_name::as_instruction` rely on this to convert via + // `mem::transmute` instead of a per-variant match. + const _: () = assert!(core::mem::size_of::<$instr_name>() == core::mem::size_of::<$typ>()); + impl $instr_name { #[doc = concat!("Get the corresponding [`", stringify!($opcode_name), "`].")] #[must_use] + #[inline] $instr_vis const fn as_opcode(&self) -> $opcode_name { - match self { - $( - Self::$op_name $({ $arg_name: _ })? => $opcode_name::$op_name, - )* - } + // SAFETY: symmetric to `$opcode_name::as_instruction` above: + // `*self`'s representation is exactly its `$typ` discriminant + // (checked by the `size_of` assertion above this impl), and that + // discriminant is always a valid `$opcode_name` discriminant + // because both enums share the same explicit `$op_id` list. + unsafe { core::mem::transmute(*self) } } #[must_use] @@ -1450,4 +1469,228 @@ mod tests { assert!(AnyInstruction::from(PseudoOpcode::Jump).is_no_fallthrough()); } + + /// Snapshot of the chained `match` implementations that `Opcode::deopt` + /// and `Opcode::cache_entries` used before they were rewritten as table + /// lookups. Exists only to pin the observable behavior of the table + /// lookups against the logic they replaced. + mod reference { + use super::Opcode; + + pub(super) const fn deopt(op: Opcode) -> Option { + Some(match op { + Opcode::ResumeCheck => Opcode::Resume, + Opcode::LoadConstMortal | Opcode::LoadConstImmortal => Opcode::LoadConst, + Opcode::ToBoolAlwaysTrue + | Opcode::ToBoolBool + | Opcode::ToBoolInt + | Opcode::ToBoolList + | Opcode::ToBoolNone + | Opcode::ToBoolStr => Opcode::ToBool, + Opcode::BinaryOpMultiplyInt + | Opcode::BinaryOpAddInt + | Opcode::BinaryOpSubtractInt + | Opcode::BinaryOpMultiplyFloat + | Opcode::BinaryOpAddFloat + | Opcode::BinaryOpSubtractFloat + | Opcode::BinaryOpAddUnicode + | Opcode::BinaryOpSubscrListInt + | Opcode::BinaryOpSubscrListSlice + | Opcode::BinaryOpSubscrTupleInt + | Opcode::BinaryOpSubscrStrInt + | Opcode::BinaryOpSubscrDict + | Opcode::BinaryOpSubscrGetitem + | Opcode::BinaryOpExtend + | Opcode::BinaryOpInplaceAddUnicode => Opcode::BinaryOp, + Opcode::StoreSubscrDict | Opcode::StoreSubscrListInt => Opcode::StoreSubscr, + Opcode::SendGen => Opcode::Send, + Opcode::UnpackSequenceTwoTuple + | Opcode::UnpackSequenceTuple + | Opcode::UnpackSequenceList => Opcode::UnpackSequence, + Opcode::StoreAttrInstanceValue + | Opcode::StoreAttrSlot + | Opcode::StoreAttrWithHint => Opcode::StoreAttr, + Opcode::LoadGlobalModule | Opcode::LoadGlobalBuiltin => Opcode::LoadGlobal, + Opcode::LoadSuperAttrAttr | Opcode::LoadSuperAttrMethod => Opcode::LoadSuperAttr, + Opcode::LoadAttrInstanceValue + | Opcode::LoadAttrModule + | Opcode::LoadAttrWithHint + | Opcode::LoadAttrSlot + | Opcode::LoadAttrClass + | Opcode::LoadAttrClassWithMetaclassCheck + | Opcode::LoadAttrProperty + | Opcode::LoadAttrGetattributeOverridden + | Opcode::LoadAttrMethodWithValues + | Opcode::LoadAttrMethodNoDict + | Opcode::LoadAttrMethodLazyDict + | Opcode::LoadAttrNondescriptorWithValues + | Opcode::LoadAttrNondescriptorNoDict => Opcode::LoadAttr, + Opcode::CompareOpFloat | Opcode::CompareOpInt | Opcode::CompareOpStr => { + Opcode::CompareOp + } + Opcode::ContainsOpSet | Opcode::ContainsOpDict => Opcode::ContainsOp, + Opcode::JumpBackwardNoJit | Opcode::JumpBackwardJit => Opcode::JumpBackward, + Opcode::ForIterList + | Opcode::ForIterTuple + | Opcode::ForIterRange + | Opcode::ForIterGen => Opcode::ForIter, + Opcode::CallBoundMethodExactArgs + | Opcode::CallPyExactArgs + | Opcode::CallType1 + | Opcode::CallStr1 + | Opcode::CallTuple1 + | Opcode::CallBuiltinClass + | Opcode::CallBuiltinO + | Opcode::CallBuiltinFast + | Opcode::CallBuiltinFastWithKeywords + | Opcode::CallLen + | Opcode::CallIsinstance + | Opcode::CallListAppend + | Opcode::CallMethodDescriptorO + | Opcode::CallMethodDescriptorFastWithKeywords + | Opcode::CallMethodDescriptorNoargs + | Opcode::CallMethodDescriptorFast + | Opcode::CallAllocAndEnterInit + | Opcode::CallPyGeneral + | Opcode::CallBoundMethodGeneral + | Opcode::CallNonPyGeneral => Opcode::Call, + Opcode::CallKwBoundMethod | Opcode::CallKwPy | Opcode::CallKwNonPy => { + Opcode::CallKw + } + _ => return None, + }) + } + + pub(super) const fn deoptimize(op: Opcode) -> Opcode { + match deopt(op) { + Some(v) => v, + None => match op.to_base() { + Some(v) => v, + None => op, + }, + } + } + + pub(super) const fn cache_entries(op: Opcode) -> usize { + match deoptimize(op) { + Opcode::StoreSubscr => 1, + Opcode::ToBool => 3, + Opcode::BinaryOp => 5, + Opcode::Call => 3, + Opcode::CallKw => 3, + Opcode::CompareOp => 1, + Opcode::ContainsOp => 1, + Opcode::ForIter => 1, + Opcode::JumpBackward => 1, + Opcode::LoadAttr => 9, + Opcode::LoadGlobal => 4, + Opcode::LoadSuperAttr => 1, + Opcode::PopJumpIfFalse => 1, + Opcode::PopJumpIfNone => 1, + Opcode::PopJumpIfNotNone => 1, + Opcode::PopJumpIfTrue => 1, + Opcode::Send => 1, + Opcode::StoreAttr => 4, + Opcode::UnpackSequence => 1, + _ => 0, + } + } + } + + #[test] + fn cache_entries_and_deopt_tables_match_reference_impl() { + let mut checked = 0; + for byte in 0u8..=255 { + let Ok(op) = Opcode::try_from_u8(byte) else { + continue; + }; + + assert_eq!( + op.deopt(), + reference::deopt(op), + "deopt() mismatch for {op:?}" + ); + assert_eq!( + op.cache_entries(), + reference::cache_entries(op), + "cache_entries() mismatch for {op:?}" + ); + checked += 1; + } + + // Sanity check that the loop actually exercised opcodes rather than + // silently skipping all of them. + assert!(checked > 200); + } + + /// `Opcode::as_numeric`, `Opcode::as_instruction` and + /// `Instruction::as_opcode` used to be per-variant matches; they are now + /// an identity cast and two `mem::transmute`s respectively. `byte` (an + /// input independent of any of those three functions) together with the + /// untouched `try_from_u8`/`TryFrom` conversions serve as the + /// reference: every opcode reachable from a byte must convert back to + /// that exact byte and round-trip through `Instruction`. + #[test] + fn opcode_instruction_numeric_conversions_match_try_from_numeric() { + let mut checked = 0; + for byte in 0u8..=255 { + let Ok(op) = Opcode::try_from_u8(byte) else { + continue; + }; + + assert_eq!(op.as_numeric(), byte, "as_numeric() mismatch for {op:?}"); + + let instr = op.as_instruction(); + assert_eq!( + instr.as_opcode(), + op, + "as_instruction()/as_opcode() round trip mismatch for {op:?}" + ); + + let instr_via_try_from = Instruction::try_from(byte).unwrap(); + assert_eq!( + instr_via_try_from.as_opcode(), + op, + "Instruction::try_from({byte}) mismatch" + ); + + checked += 1; + } + + assert!(checked > 200); + } + + /// Same as [`opcode_instruction_numeric_conversions_match_try_from_numeric`] + /// but for the `u16`-discriminant pseudo-opcode instantiation of + /// `define_opcodes!`. + #[test] + fn pseudo_opcode_instruction_numeric_conversions_match_try_from_numeric() { + let mut checked = 0; + for value in 0u16..=u16::MAX { + let Ok(op) = PseudoOpcode::try_from_u16(value) else { + continue; + }; + + assert_eq!(op.as_numeric(), value, "as_numeric() mismatch for {op:?}"); + + let instr = op.as_instruction(); + assert_eq!( + instr.as_opcode(), + op, + "as_instruction()/as_opcode() round trip mismatch for {op:?}" + ); + + let instr_via_try_from = PseudoInstruction::try_from(value).unwrap(); + assert_eq!( + instr_via_try_from.as_opcode(), + op, + "PseudoInstruction::try_from({value}) mismatch" + ); + + checked += 1; + } + + // All 11 `PseudoInstruction` variants should have been exercised. + assert_eq!(checked, 11); + } } diff --git a/crates/compiler-core/src/bytecode/opcode_metadata.rs b/crates/compiler-core/src/bytecode/opcode_metadata.rs index 64c8d3c5330..16db49bea0d 100644 --- a/crates/compiler-core/src/bytecode/opcode_metadata.rs +++ b/crates/compiler-core/src/bytecode/opcode_metadata.rs @@ -6,114 +6,292 @@ use crate::{bytecode::instruction::StackEffect, marshal::MarshalError}; impl super::Opcode { /// Returns [`Self`] as [`u8`]. #[must_use] + #[inline] pub const fn as_u8(self) -> u8 { self.as_numeric() } #[must_use] + #[inline] pub const fn cache_entries(self) -> usize { - match self.deoptimize() { - Self::StoreSubscr => 1, - Self::ToBool => 3, - Self::BinaryOp => 5, - Self::Call => 3, - Self::CallKw => 3, - Self::CompareOp => 1, - Self::ContainsOp => 1, - Self::ForIter => 1, - Self::JumpBackward => 1, - Self::LoadAttr => 9, - Self::LoadGlobal => 4, - Self::LoadSuperAttr => 1, - Self::PopJumpIfFalse => 1, - Self::PopJumpIfNone => 1, - Self::PopJumpIfNotNone => 1, - Self::PopJumpIfTrue => 1, - Self::Send => 1, - Self::StoreAttr => 4, - Self::UnpackSequence => 1, - _ => 0, - } + const CACHE_ENTRIES: [u8; 256] = [ + 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 3, 1, 1, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 4, 0, 0, 0, 1, 0, 0, 0, 1, 1, 1, 1, 0, 0, 1, 0, 0, 0, 4, 0, 0, 0, 0, 0, + 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 3, 3, + 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 4, 4, 1, 1, 0, 1, 4, 4, 4, 1, 1, + 3, 3, 3, 3, 3, 3, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 1, 1, 1, 0, 0, 0, 0, 1, 3, 3, 0, 1, 0, 0, + ]; + + CACHE_ENTRIES[self.as_numeric() as usize] as usize } #[must_use] + #[inline] pub const fn deopt(self) -> Option { - Some(match self { - Self::ResumeCheck => Self::Resume, - Self::LoadConstMortal | Self::LoadConstImmortal => Self::LoadConst, - Self::ToBoolAlwaysTrue - | Self::ToBoolBool - | Self::ToBoolInt - | Self::ToBoolList - | Self::ToBoolNone - | Self::ToBoolStr => Self::ToBool, - Self::BinaryOpMultiplyInt - | Self::BinaryOpAddInt - | Self::BinaryOpSubtractInt - | Self::BinaryOpMultiplyFloat - | Self::BinaryOpAddFloat - | Self::BinaryOpSubtractFloat - | Self::BinaryOpAddUnicode - | Self::BinaryOpSubscrListInt - | Self::BinaryOpSubscrListSlice - | Self::BinaryOpSubscrTupleInt - | Self::BinaryOpSubscrStrInt - | Self::BinaryOpSubscrDict - | Self::BinaryOpSubscrGetitem - | Self::BinaryOpExtend - | Self::BinaryOpInplaceAddUnicode => Self::BinaryOp, - Self::StoreSubscrDict | Self::StoreSubscrListInt => Self::StoreSubscr, - Self::SendGen => Self::Send, - Self::UnpackSequenceTwoTuple | Self::UnpackSequenceTuple | Self::UnpackSequenceList => { - Self::UnpackSequence - } - Self::StoreAttrInstanceValue | Self::StoreAttrSlot | Self::StoreAttrWithHint => { - Self::StoreAttr - } - Self::LoadGlobalModule | Self::LoadGlobalBuiltin => Self::LoadGlobal, - Self::LoadSuperAttrAttr | Self::LoadSuperAttrMethod => Self::LoadSuperAttr, - Self::LoadAttrInstanceValue - | Self::LoadAttrModule - | Self::LoadAttrWithHint - | Self::LoadAttrSlot - | Self::LoadAttrClass - | Self::LoadAttrClassWithMetaclassCheck - | Self::LoadAttrProperty - | Self::LoadAttrGetattributeOverridden - | Self::LoadAttrMethodWithValues - | Self::LoadAttrMethodNoDict - | Self::LoadAttrMethodLazyDict - | Self::LoadAttrNondescriptorWithValues - | Self::LoadAttrNondescriptorNoDict => Self::LoadAttr, - Self::CompareOpFloat | Self::CompareOpInt | Self::CompareOpStr => Self::CompareOp, - Self::ContainsOpSet | Self::ContainsOpDict => Self::ContainsOp, - Self::JumpBackwardNoJit | Self::JumpBackwardJit => Self::JumpBackward, - Self::ForIterList | Self::ForIterTuple | Self::ForIterRange | Self::ForIterGen => { - Self::ForIter - } - Self::CallBoundMethodExactArgs - | Self::CallPyExactArgs - | Self::CallType1 - | Self::CallStr1 - | Self::CallTuple1 - | Self::CallBuiltinClass - | Self::CallBuiltinO - | Self::CallBuiltinFast - | Self::CallBuiltinFastWithKeywords - | Self::CallLen - | Self::CallIsinstance - | Self::CallListAppend - | Self::CallMethodDescriptorO - | Self::CallMethodDescriptorFastWithKeywords - | Self::CallMethodDescriptorNoargs - | Self::CallMethodDescriptorFast - | Self::CallAllocAndEnterInit - | Self::CallPyGeneral - | Self::CallBoundMethodGeneral - | Self::CallNonPyGeneral => Self::Call, - Self::CallKwBoundMethod | Self::CallKwPy | Self::CallKwNonPy => Self::CallKw, - _ => return None, - }) + const DEOPT: [Option; 256] = [ + None, + None, + None, + Some(super::Opcode::BinaryOp), + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + Some(super::Opcode::BinaryOp), + Some(super::Opcode::BinaryOp), + Some(super::Opcode::BinaryOp), + Some(super::Opcode::BinaryOp), + Some(super::Opcode::BinaryOp), + Some(super::Opcode::BinaryOp), + Some(super::Opcode::BinaryOp), + Some(super::Opcode::BinaryOp), + Some(super::Opcode::BinaryOp), + Some(super::Opcode::BinaryOp), + Some(super::Opcode::BinaryOp), + Some(super::Opcode::BinaryOp), + Some(super::Opcode::BinaryOp), + Some(super::Opcode::BinaryOp), + Some(super::Opcode::Call), + Some(super::Opcode::Call), + Some(super::Opcode::Call), + Some(super::Opcode::Call), + Some(super::Opcode::Call), + Some(super::Opcode::Call), + Some(super::Opcode::Call), + Some(super::Opcode::Call), + Some(super::Opcode::CallKw), + Some(super::Opcode::CallKw), + Some(super::Opcode::CallKw), + Some(super::Opcode::Call), + Some(super::Opcode::Call), + Some(super::Opcode::Call), + Some(super::Opcode::Call), + Some(super::Opcode::Call), + Some(super::Opcode::Call), + Some(super::Opcode::Call), + Some(super::Opcode::Call), + Some(super::Opcode::Call), + Some(super::Opcode::Call), + Some(super::Opcode::Call), + Some(super::Opcode::Call), + Some(super::Opcode::CompareOp), + Some(super::Opcode::CompareOp), + Some(super::Opcode::CompareOp), + Some(super::Opcode::ContainsOp), + Some(super::Opcode::ContainsOp), + Some(super::Opcode::ForIter), + Some(super::Opcode::ForIter), + Some(super::Opcode::ForIter), + Some(super::Opcode::ForIter), + Some(super::Opcode::JumpBackward), + Some(super::Opcode::JumpBackward), + Some(super::Opcode::LoadAttr), + Some(super::Opcode::LoadAttr), + Some(super::Opcode::LoadAttr), + Some(super::Opcode::LoadAttr), + Some(super::Opcode::LoadAttr), + Some(super::Opcode::LoadAttr), + Some(super::Opcode::LoadAttr), + Some(super::Opcode::LoadAttr), + Some(super::Opcode::LoadAttr), + Some(super::Opcode::LoadAttr), + Some(super::Opcode::LoadAttr), + Some(super::Opcode::LoadAttr), + Some(super::Opcode::LoadAttr), + Some(super::Opcode::LoadConst), + Some(super::Opcode::LoadConst), + Some(super::Opcode::LoadGlobal), + Some(super::Opcode::LoadGlobal), + Some(super::Opcode::LoadSuperAttr), + Some(super::Opcode::LoadSuperAttr), + Some(super::Opcode::Resume), + Some(super::Opcode::Send), + Some(super::Opcode::StoreAttr), + Some(super::Opcode::StoreAttr), + Some(super::Opcode::StoreAttr), + Some(super::Opcode::StoreSubscr), + Some(super::Opcode::StoreSubscr), + Some(super::Opcode::ToBool), + Some(super::Opcode::ToBool), + Some(super::Opcode::ToBool), + Some(super::Opcode::ToBool), + Some(super::Opcode::ToBool), + Some(super::Opcode::ToBool), + Some(super::Opcode::UnpackSequence), + Some(super::Opcode::UnpackSequence), + Some(super::Opcode::UnpackSequence), + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + ]; + + DEOPT[self.as_numeric() as usize] } /// Does this opcode have 'HAS_ARG_FLAG' set. @@ -664,6 +842,7 @@ impl super::Opcode { } #[must_use] + #[inline] pub const fn to_base(self) -> Option { Some(match self { Self::InstrumentedCall => Self::Call, @@ -723,16 +902,19 @@ impl super::Opcode { impl super::PseudoOpcode { /// Returns [`Self`] as [`u16`]. #[must_use] + #[inline] pub const fn as_u16(self) -> u16 { self.as_numeric() } #[must_use] + #[inline] pub const fn cache_entries(self) -> usize { 0 } #[must_use] + #[inline] pub const fn deopt(self) -> Option { None } @@ -818,6 +1000,7 @@ impl super::PseudoOpcode { } #[must_use] + #[inline] pub const fn to_base(self) -> Option { None } diff --git a/crates/derive-impl/src/pyclass.rs b/crates/derive-impl/src/pyclass.rs index aed2c5d8c5f..809d3164b4a 100644 --- a/crates/derive-impl/src/pyclass.rs +++ b/crates/derive-impl/src/pyclass.rs @@ -796,8 +796,15 @@ pub(crate) fn impl_pyexception(attr: PunctuatedNestedMeta, item: &Item) -> Resul quote! {} }; + // Forward a `traverse` option to the generated `#[pyclass]` so exception + // payloads with a manual `Traverse` impl are GC-tracked and traversed. + let traverse_attr = match class_meta.inner()._optional_str("traverse").ok().flatten() { + Some(value) => quote! { , traverse = #value }, + None => quote! {}, + }; + let ret = quote! { - #[pyclass(module = false, name = #class_name, base = #base_class_name)] + #[pyclass(module = false, name = #class_name, base = #base_class_name #traverse_attr)] #item #impl_pyclass }; diff --git a/crates/derive-impl/src/util.rs b/crates/derive-impl/src/util.rs index 60b2296cea7..a0708444691 100644 --- a/crates/derive-impl/src/util.rs +++ b/crates/derive-impl/src/util.rs @@ -465,8 +465,15 @@ impl ClassItemMeta { pub(crate) struct ExceptionItemMeta(ClassItemMeta); impl ItemMeta for ExceptionItemMeta { - const ALLOWED_NAMES: &'static [&'static str] = - &["module", "name", "base", "unhashable", "ctx", "impl"]; + const ALLOWED_NAMES: &'static [&'static str] = &[ + "module", + "name", + "base", + "unhashable", + "ctx", + "impl", + "traverse", + ]; fn from_inner(inner: ItemMetaInner) -> Self { Self(ClassItemMeta(inner)) diff --git a/crates/stdlib/Cargo.toml b/crates/stdlib/Cargo.toml index 1e071869549..d32e7ce8b35 100644 --- a/crates/stdlib/Cargo.toml +++ b/crates/stdlib/Cargo.toml @@ -53,6 +53,7 @@ num_enum = { workspace = true } parking_lot = { workspace = true } phf = { workspace = true, default-features = true, features = ["macros"] } rapidhash = { workspace = true } +scopeguard = { workspace = true } memchr = { workspace = true } base64 = { workspace = true } diff --git a/crates/stdlib/src/faulthandler.rs b/crates/stdlib/src/faulthandler.rs index 3717c18b78f..900d66b76e6 100644 --- a/crates/stdlib/src/faulthandler.rs +++ b/crates/stdlib/src/faulthandler.rs @@ -241,7 +241,7 @@ mod decl { /// Dump traceback for a thread given its frame stack (for cross-thread dumping). /// # Safety /// Each `FramePtr` must point to a live frame (caller holds the Mutex). - #[cfg(all(any(unix, windows), feature = "threading"))] + #[cfg(all(windows, feature = "threading"))] fn dump_traceback_thread_frames( fd: i32, thread_id: u64, @@ -260,6 +260,35 @@ mod decl { } } + /// Dump a thread's traceback by walking its published top frame down the + /// `previous` chain (most recent first). Signal-safe: only atomic pointer + /// loads, no locks. Callers guarantee frame liveness — under stop-the-world + /// for `faulthandler.dump_traceback`, or best-effort for the watchdog (like + /// `_Py_DumpTracebackThreads`, which walks lock-free while other threads + /// may still run). + #[cfg(all(unix, feature = "threading"))] + fn dump_traceback_thread_chain(fd: i32, thread_id: u64, is_current: bool, top: *const Frame) { + const MAX_FRAME_DEPTH: usize = 100; + write_thread_id(fd, thread_id, is_current); + + if top.is_null() { + puts(fd, " \n"); + return; + } + let mut frame_ptr = top; + let mut depth = 0; + while !frame_ptr.is_null() && depth < MAX_FRAME_DEPTH { + // SAFETY: the frame is alive per the caller's liveness guarantee. + let frame = unsafe { &*frame_ptr }; + dump_frame_from_raw(fd, frame); + frame_ptr = frame.previous_frame(); + depth += 1; + } + if depth >= MAX_FRAME_DEPTH && !frame_ptr.is_null() { + puts(fd, " ...\n"); + } + } + #[derive(FromArgs)] struct DumpTracebackArgs { #[pyarg(any, default)] @@ -278,11 +307,9 @@ mod decl { dump_all_threads(fd, vm); } else { puts(fd, "Stack (most recent call first):\n"); - let frames = vm.frames.borrow(); - for fp in frames.iter().rev() { - // SAFETY: the frame is alive while it's in the Vec - dump_frame_from_ref(fd, unsafe { fp.as_ref() }); - } + crate::vm::frame::for_each_current_frame(|frame| { + dump_frame_from_ref(fd, frame); + }); } } @@ -298,7 +325,43 @@ mod decl { #[cfg(any(unix, windows))] fn dump_all_threads(fd: i32, vm: &VirtualMachine) { // Get all threads' frame stacks from the shared registry - #[cfg(feature = "threading")] + // unix: stop-the-world so every other thread is parked at a safepoint + // and its frame chain is quiescent and alive while we walk it (matches + // faulthandler.dump_traceback running with the GIL held). + #[cfg(all(unix, feature = "threading"))] + { + use core::sync::atomic::Ordering; + let current_tid = rustpython_vm::stdlib::_thread::get_ident(); + { + vm.state.stop_the_world.stop_the_world(vm); + scopeguard::defer! { vm.state.stop_the_world.start_the_world(vm); } + let registry = vm.state.thread_frames.lock(); + #[expect( + clippy::iter_over_hash_type, + reason = "Iteration order doesn't matter here" + )] + for (&tid, slot) in registry.iter() { + if tid == current_tid { + continue; + } + let top = slot.top_frame.load(Ordering::Relaxed) as *const Frame; + dump_traceback_thread_chain(fd, tid, false, top); + puts(fd, "\n"); + } + } + + // Now dump current thread from its live frame chain. + write_thread_id(fd, current_tid, true); + if crate::vm::vm::thread::get_current_frame().is_null() { + puts(fd, " \n"); + } else { + crate::vm::frame::for_each_current_frame(|frame| { + dump_frame_from_ref(fd, frame); + }); + } + } + + #[cfg(all(not(unix), feature = "threading"))] { let current_tid = rustpython_vm::stdlib::_thread::get_ident(); let registry = vm.state.thread_frames.lock(); @@ -318,25 +381,24 @@ mod decl { puts(fd, "\n"); } - // Now dump current thread (use vm.frames for most up-to-date data) + // Now dump current thread from its live frame chain. write_thread_id(fd, current_tid, true); - let frames = vm.frames.borrow(); - if frames.is_empty() { + if crate::vm::vm::thread::get_current_frame().is_null() { puts(fd, " \n"); } else { - for fp in frames.iter().rev() { - dump_frame_from_ref(fd, unsafe { fp.as_ref() }); - } + crate::vm::frame::for_each_current_frame(|frame| { + dump_frame_from_ref(fd, frame); + }); } } #[cfg(not(feature = "threading"))] { + let _ = vm; write_thread_id(fd, current_thread_id(), true); - let frames = vm.frames.borrow(); - for fp in frames.iter().rev() { - dump_frame_from_ref(fd, unsafe { fp.as_ref() }); - } + crate::vm::frame::for_each_current_frame(|frame| { + dump_frame_from_ref(fd, frame); + }); } } @@ -657,7 +719,20 @@ mod decl { // Use thread frame slots when threading is enabled (includes all threads). // Fall back to live frame walking for non-threaded builds. cfg_select! { - feature = "threading" => { + all(unix, feature = "threading") => { + // The watchdog is a plain OS thread, not attached to + // the VM, so it cannot stop-the-world. Walk each + // published top frame lock-free and best-effort, like + // the faulthandler watchdog thread. + for (tid, slot) in &thread_frame_slots { + let top = slot + .top_frame + .load(core::sync::atomic::Ordering::Relaxed) + as *const Frame; + dump_traceback_thread_chain(fd, *tid, false, top); + } + } + all(not(unix), feature = "threading") => { for (tid, slot) in &thread_frame_slots { let frames = slot.frames.lock(); dump_traceback_thread_frames(fd, *tid, false, &frames); diff --git a/crates/stdlib/src/overlapped.rs b/crates/stdlib/src/overlapped.rs index 4ce3d3ba830..8610fadb3bf 100644 --- a/crates/stdlib/src/overlapped.rs +++ b/crates/stdlib/src/overlapped.rs @@ -309,8 +309,9 @@ mod _overlapped { return Err(vm.new_value_error("operation failed to start")); } - let result = - host_overlapped::get_overlapped_result(inner.handle, &inner.overlapped, wait); + let result = vm.allow_threads(|| { + host_overlapped::get_overlapped_result(inner.handle, &inner.overlapped, wait) + }); let transferred = result.transferred; let err = result.error; inner.error = err; diff --git a/crates/stdlib/src/ssl.rs b/crates/stdlib/src/ssl.rs index edcdf18fa09..81d69b8c64e 100644 --- a/crates/stdlib/src/ssl.rs +++ b/crates/stdlib/src/ssl.rs @@ -2709,13 +2709,13 @@ mod _ssl { sni_name: Option<&str>, vm: &VirtualMachine, ) -> PyResult<()> { - let callback = self - .context - .read() - .sni_callback - .read() - .clone() - .ok_or_else(|| vm.new_value_error("SNI callback not set"))?; + // The callback may have been cleared (sni_callback = None) between the + // handshake deciding to invoke it and this point. A concurrent removal + // is not an error: there is simply nothing to run. + let callback = self.context.read().sni_callback.read().clone(); + let Some(callback) = callback else { + return Ok(()); + }; let ssl_sock = self.owner.read().clone().unwrap_or_else(|| vm.ctx.none()); let server_name_py: PyObjectRef = match sni_name { diff --git a/crates/vm/Cargo.toml b/crates/vm/Cargo.toml index 0b2feed50a3..24006c8b3b9 100644 --- a/crates/vm/Cargo.toml +++ b/crates/vm/Cargo.toml @@ -54,6 +54,7 @@ flame = { workspace = true, optional = true } hex = { workspace = true } indexmap = { workspace = true } itertools = { workspace = true } +itoa = { workspace = true } is-macro = { workspace = true } libc = { workspace = true } log = { workspace = true } diff --git a/crates/vm/src/builtins/code.rs b/crates/vm/src/builtins/code.rs index e30813bf87d..b1132a55a20 100644 --- a/crates/vm/src/builtins/code.rs +++ b/crates/vm/src/builtins/code.rs @@ -471,6 +471,11 @@ pub struct PyCode { pub monitoring_data: PyMutex>, /// Whether adaptive counters have been initialized (lazy quickening). pub quickened: core::sync::atomic::AtomicBool, + /// Whether the bytecode contains any instruction that mutates the current + /// exc_info slot (`vm.set_exception`). When false, a normal frame call for + /// this code cannot leave the slot unbalanced, so `with_frame` skips the + /// exc_info save/restore. Computed once by scanning the instruction stream. + pub has_exc_handling: bool, } impl Deref for PyCode { @@ -483,12 +488,26 @@ impl Deref for PyCode { impl PyCode { pub fn new(code: CodeObject) -> Self { let sp = code.source_path as *const PyStrInterned as *mut PyStrInterned; + // The only opcodes that call `vm.set_exception` (mutating the shared + // exc_info slot); instrumented variants only replace these base opcodes + // in place, so scanning the freshly-built stream is a sound predicate. + let has_exc_handling = code.instructions.iter().any(|u| { + matches!( + u.op, + Instruction::PushExcInfo + | Instruction::PopExcept + | Instruction::CheckEgMatch + | Instruction::EndAsyncFor + | Instruction::InstrumentedEndAsyncFor + ) + }); Self { code, source_path: AtomicPtr::new(sp), instrumentation_version: AtomicU64::new(0), monitoring_data: PyMutex::new(None), quickened: core::sync::atomic::AtomicBool::new(false), + has_exc_handling, } } diff --git a/crates/vm/src/builtins/frame.rs b/crates/vm/src/builtins/frame.rs index ab45f68673c..710d23b7c2f 100644 --- a/crates/vm/src/builtins/frame.rs +++ b/crates/vm/src/builtins/frame.rs @@ -4,7 +4,7 @@ use super::{PyCode, PyDictRef, PyIntRef, PyStrRef}; use crate::{ - Context, Py, PyObjectRef, PyRef, PyResult, VirtualMachine, + Context, Py, PyObjectRef, PyPayload, PyRef, PyResult, VirtualMachine, class::PyClassImpl, frame::{Frame, FrameOwner, FrameRef}, function::PySetterValue, @@ -454,14 +454,6 @@ impl Frame { self.builtins.clone() } - #[pygetset] - fn f_locals(&self, vm: &VirtualMachine) -> PyResult { - let result = self.f_locals_mapping(vm).map(Into::into); - self.locals_dirty - .store(true, core::sync::atomic::Ordering::Release); - result - } - #[pygetset] pub fn f_code(&self) -> PyRef { self.code.clone() @@ -704,10 +696,27 @@ impl Py { // Clear temporary refs self.temporary_refs.lock().clear(); self.f_locals_hidden_overlay.lock().take(); + self.f_extra_locals.lock().take(); + self.retained_back.lock().take(); Ok(()) } + #[pygetset] + fn f_locals(&self, vm: &VirtualMachine) -> PyResult { + // Optimized (function) frames expose a live write-through + // FrameLocalsProxy; class/module/exec frames expose their namespace + // mapping directly. + if self.code.flags.contains(bytecode::CodeFlags::OPTIMIZED) { + self.check_locals_access(vm)?; + self.mark_escaped(); + let proxy = crate::builtins::FrameLocalsProxy::new(self.to_owned()); + Ok(proxy.into_ref(&vm.ctx).into()) + } else { + self.f_locals_mapping(vm).map(Into::into) + } + } + #[pygetset] fn f_generator(&self) -> Option { self.generator.to_owned() @@ -715,27 +724,59 @@ impl Py { #[pygetset] pub fn f_back(&self, vm: &VirtualMachine) -> Option> { + #[cfg(not(feature = "threading"))] + let _ = vm; let previous = self.previous_frame(); if previous.is_null() { return None; } - if let Some(frame) = vm - .frames - .borrow() - .iter() - .find(|fp| { - // SAFETY: the caller keeps the FrameRef alive while it's in the Vec - let py: &Self = unsafe { fp.as_ref() }; - let ptr: *const Frame = &**py; - core::ptr::eq(ptr, previous) - }) - .map(|fp| unsafe { fp.as_ref() }.to_owned()) - { + // Look for the caller on the current thread's signal-safe frame chain. + // Finding it there proves it is still live on this thread. + if let Some(frame) = crate::frame::find_owned_chain_frame(previous) { + frame.mark_escaped(); return Some(frame); } - #[cfg(feature = "threading")] + // The caller already returned and left the live chain, but this frame + // escaped and retained a strong reference to it at release time. + let retained = self.retained_back.lock().clone(); + if let Some(frame) = retained { + frame.mark_escaped(); + return Some(frame); + } + + // The caller lives on another thread. unix: park every thread under + // stop-the-world so their frame chains are quiescent and alive, then + // walk each published top frame down its `previous` chain looking for + // the caller. Request stop-the-world before the registry lock. + #[cfg(all(unix, feature = "threading"))] + { + use core::sync::atomic::Ordering; + vm.state.stop_the_world.stop_the_world(vm); + scopeguard::defer! { vm.state.stop_the_world.start_the_world(vm); } + let registry = vm.state.thread_frames.lock(); + #[expect( + clippy::iter_over_hash_type, + reason = "Iteration order doesn't matter here" + )] + for slot in registry.values() { + let mut cur = slot.top_frame.load(Ordering::Relaxed) as *const Frame; + while !cur.is_null() { + if core::ptr::eq(cur, previous) { + // SAFETY: world stopped -> this frame is alive on its + // owning thread's parked call stack. + let f = unsafe { &*Self::from_payload_ptr(cur) }; + f.mark_escaped(); + return Some(f.to_owned()); + } + // SAFETY: chain frames on a parked thread are alive. + cur = unsafe { (*cur).previous_frame() }; + } + } + } + + #[cfg(all(not(unix), feature = "threading"))] { let registry = vm.state.thread_frames.lock(); #[expect( @@ -751,6 +792,7 @@ impl Py { let ptr: *const Frame = &**f; core::ptr::eq(ptr, previous).then(|| f.to_owned()) }) { + frame.mark_escaped(); return Some(frame); } } diff --git a/crates/vm/src/builtins/frame_locals_proxy.rs b/crates/vm/src/builtins/frame_locals_proxy.rs new file mode 100644 index 00000000000..fbbc7f5d9cd --- /dev/null +++ b/crates/vm/src/builtins/frame_locals_proxy.rs @@ -0,0 +1,326 @@ +//! The `FrameLocalsProxy` type returned by `frame.f_locals` for optimized +//! (function) frames. Implements PEP 667 write-through semantics on top of the +//! frame's fast-local slots and an extra-locals side dict. + +use super::{PyDict, PyDictRef, PyType}; +use crate::{ + AsObject, Context, Py, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, VirtualMachine, + atomic_func, + class::PyClassImpl, + frame::FrameRef, + function::{FuncArgs, OptionalArg, PyArithmeticValue, PyComparisonValue}, + object::{Traverse, TraverseFn}, + protocol::{PyMappingMethods, PyNumberMethods, PySequenceMethods}, + recursion::ReprGuard, + types::{ + AsMapping, AsNumber, AsSequence, Comparable, Constructor, Iterable, PyComparisonOp, + Representable, + }, +}; +use rustpython_common::lock::LazyLock; +use rustpython_common::wtf8::Wtf8Buf; + +#[pyclass(module = false, name = "FrameLocalsProxy", traverse = "manual")] +#[derive(Debug)] +pub struct FrameLocalsProxy { + frame: FrameRef, +} + +unsafe impl Traverse for FrameLocalsProxy { + fn traverse(&self, tracer_fn: &mut TraverseFn<'_>) { + self.frame.traverse(tracer_fn); + } +} + +impl PyPayload for FrameLocalsProxy { + #[inline] + fn class(ctx: &Context) -> &'static Py { + ctx.types.frame_locals_proxy_type + } +} + +impl FrameLocalsProxy { + pub(crate) fn new(frame: FrameRef) -> Self { + Self { frame } + } + + fn snapshot(&self, vm: &VirtualMachine) -> PyResult { + self.frame.framelocalsproxy_snapshot(vm) + } + + fn keys_vec(&self, vm: &VirtualMachine) -> PyResult> { + Ok(self.snapshot(vm)?.into_iter().map(|(k, _)| k).collect()) + } +} + +impl Constructor for FrameLocalsProxy { + type Args = FuncArgs; + + fn py_new(_cls: &Py, args: Self::Args, vm: &VirtualMachine) -> PyResult { + if !args.kwargs.is_empty() { + return Err(vm.new_type_error("FrameLocalsProxy() takes no keyword arguments")); + } + let mut args = args.args; + if args.len() != 1 { + return Err(vm.new_type_error(format!( + "FrameLocalsProxy expected 1 argument, got {}", + args.len() + ))); + } + let frame: FrameRef = args + .pop() + .unwrap() + .downcast() + .map_err(|_| vm.new_type_error("FrameLocalsProxy expected a frame"))?; + Ok(Self::new(frame)) + } +} + +#[pyclass(with( + Constructor, + AsMapping, + AsSequence, + AsNumber, + Iterable, + Comparable, + Representable +))] +impl FrameLocalsProxy { + fn __getitem__(&self, key: PyObjectRef, vm: &VirtualMachine) -> PyResult { + self.frame.framelocalsproxy_getitem(key, vm) + } + + fn __setitem__( + &self, + key: PyObjectRef, + value: PyObjectRef, + vm: &VirtualMachine, + ) -> PyResult<()> { + self.frame.framelocalsproxy_setitem(key, value, vm) + } + + fn __delitem__(&self, key: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { + self.frame.framelocalsproxy_delitem(key, vm) + } + + fn __contains__(&self, key: PyObjectRef, vm: &VirtualMachine) -> PyResult { + self.frame.framelocalsproxy_contains(key, vm) + } + + fn __len__(&self, vm: &VirtualMachine) -> PyResult { + Ok(self.snapshot(vm)?.__len__()) + } + + #[pymethod] + fn keys(&self, vm: &VirtualMachine) -> PyResult { + Ok(vm.ctx.new_list(self.keys_vec(vm)?).into()) + } + + #[pymethod] + fn values(&self, vm: &VirtualMachine) -> PyResult { + let values = self.snapshot(vm)?.into_iter().map(|(_, v)| v).collect(); + Ok(vm.ctx.new_list(values).into()) + } + + #[pymethod] + fn items(&self, vm: &VirtualMachine) -> PyResult { + let items = self + .snapshot(vm)? + .into_iter() + .map(|(k, v)| vm.ctx.new_tuple(vec![k, v]).into()) + .collect(); + Ok(vm.ctx.new_list(items).into()) + } + + #[pymethod] + fn get(&self, key: PyObjectRef, default: OptionalArg, vm: &VirtualMachine) -> PyResult { + match self.frame.framelocalsproxy_getitem(key, vm) { + Ok(value) => Ok(value), + Err(e) if e.fast_isinstance(vm.ctx.exceptions.key_error) => { + Ok(default.unwrap_or_none(vm)) + } + Err(e) => Err(e), + } + } + + #[pymethod] + fn pop(&self, key: PyObjectRef, default: OptionalArg, vm: &VirtualMachine) -> PyResult { + self.frame + .framelocalsproxy_pop(key, default.into_option(), vm) + } + + #[pymethod] + fn setdefault(&self, key: PyObjectRef, default: OptionalArg, vm: &VirtualMachine) -> PyResult { + self.frame + .framelocalsproxy_setdefault(key, default.unwrap_or_none(vm), vm) + } + + #[pymethod] + fn copy(&self, vm: &VirtualMachine) -> PyResult { + Ok(self.snapshot(vm)?.into()) + } + + #[pymethod] + fn update(&self, args: FuncArgs, vm: &VirtualMachine) -> PyResult<()> { + if !args.kwargs.is_empty() { + return Err(vm.new_type_error("FrameLocalsProxy.update() takes no keyword arguments")); + } + if args.args.len() != 1 { + return Err(vm.new_type_error(format!( + "FrameLocalsProxy.update() takes exactly one argument ({} given)", + args.args.len() + ))); + } + self.update_from(&args.args[0], vm) + } + + fn update_from(&self, other: &PyObject, vm: &VirtualMachine) -> PyResult<()> { + let items: Vec<(PyObjectRef, PyObjectRef)> = + if let Some(dict) = other.downcast_ref::() { + dict.into_iter().collect() + } else if let Some(proxy) = other.downcast_ref::() { + proxy.snapshot(vm)?.into_iter().collect() + } else { + return Err( + vm.new_type_error("update() argument must be dict or another FrameLocalsProxy") + ); + }; + for (key, value) in items { + self.frame.framelocalsproxy_setitem(key, value, vm)?; + } + Ok(()) + } + + #[pymethod] + fn __reversed__(&self, vm: &VirtualMachine) -> PyResult { + let mut keys = self.keys_vec(vm)?; + keys.reverse(); + Ok(vm.ctx.new_list(keys).into()) + } + + fn __ior__(zelf: PyRef, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + zelf.update_from(&other, vm)?; + Ok(zelf.into()) + } + + fn __or__(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + let base = self.snapshot(vm)?; + vm._or(base.as_object(), &other) + } + + fn __ror__(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + let base = self.snapshot(vm)?; + vm._or(&other, base.as_object()) + } + + #[pymethod] + fn __reduce__(&self, vm: &VirtualMachine) -> PyResult { + Err(vm.new_type_error("cannot pickle 'FrameLocalsProxy' object")) + } + + #[pymethod] + fn __reduce_ex__(&self, _protocol: OptionalArg, vm: &VirtualMachine) -> PyResult { + Err(vm.new_type_error("cannot pickle 'FrameLocalsProxy' object")) + } +} + +impl AsMapping for FrameLocalsProxy { + fn as_mapping() -> &'static PyMappingMethods { + static AS_MAPPING: LazyLock = LazyLock::new(|| PyMappingMethods { + length: atomic_func!( + |mapping, vm| FrameLocalsProxy::mapping_downcast(mapping).__len__(vm) + ), + subscript: atomic_func!(|mapping, needle, vm| { + FrameLocalsProxy::mapping_downcast(mapping).__getitem__(needle.to_owned(), vm) + }), + ass_subscript: atomic_func!(|mapping, needle, value, vm| { + let zelf = FrameLocalsProxy::mapping_downcast(mapping); + match value { + Some(value) => zelf.__setitem__(needle.to_owned(), value, vm), + None => zelf.__delitem__(needle.to_owned(), vm), + } + }), + }); + &AS_MAPPING + } +} + +impl AsSequence for FrameLocalsProxy { + fn as_sequence() -> &'static PySequenceMethods { + static AS_SEQUENCE: LazyLock = LazyLock::new(|| PySequenceMethods { + contains: atomic_func!(|seq, target, vm| { + FrameLocalsProxy::sequence_downcast(seq).__contains__(target.to_owned(), vm) + }), + ..PySequenceMethods::NOT_IMPLEMENTED + }); + &AS_SEQUENCE + } +} + +impl AsNumber for FrameLocalsProxy { + fn as_number() -> &'static PyNumberMethods { + static AS_NUMBER: PyNumberMethods = PyNumberMethods { + or: Some(|a, b, vm| { + if let Some(proxy) = a.downcast_ref::() { + proxy.__or__(b.to_owned(), vm) + } else if let Some(proxy) = b.downcast_ref::() { + proxy.__ror__(a.to_owned(), vm) + } else { + Ok(vm.ctx.not_implemented()) + } + }), + inplace_or: Some(|a, b, vm| { + let proxy = a + .to_owned() + .downcast::() + .map_err(|_| vm.new_type_error("expected FrameLocalsProxy"))?; + FrameLocalsProxy::__ior__(proxy, b.to_owned(), vm) + }), + ..PyNumberMethods::NOT_IMPLEMENTED + }; + &AS_NUMBER + } +} + +impl Iterable for FrameLocalsProxy { + fn iter(zelf: PyRef, vm: &VirtualMachine) -> PyResult { + let keys = vm.ctx.new_list(zelf.keys_vec(vm)?); + keys.as_object().to_owned().get_iter(vm).map(Into::into) + } +} + +impl Comparable for FrameLocalsProxy { + fn cmp( + zelf: &Py, + other: &PyObject, + op: PyComparisonOp, + vm: &VirtualMachine, + ) -> PyResult { + op.eq_only(|| { + let self_dict: PyObjectRef = zelf.snapshot(vm)?.into(); + let other_obj = match other.downcast_ref::() { + Some(proxy) => proxy.snapshot(vm)?.into(), + None => other.to_owned(), + }; + let res = self_dict.rich_compare(other_obj, PyComparisonOp::Eq, vm)?; + PyArithmeticValue::from_object(vm, res) + .map(|o| o.try_to_bool(vm)) + .transpose() + }) + } +} + +impl Representable for FrameLocalsProxy { + fn repr_wtf8(zelf: &Py, vm: &VirtualMachine) -> PyResult { + if let Some(_guard) = ReprGuard::enter(vm, zelf.as_object()) { + let dict = zelf.snapshot(vm)?; + Ok(dict.as_object().repr(vm)?.as_wtf8().to_owned()) + } else { + Ok(Wtf8Buf::from("{...}")) + } + } +} + +pub(crate) fn init(context: &'static Context) { + FrameLocalsProxy::extend_class(context, context.types.frame_locals_proxy_type); +} diff --git a/crates/vm/src/builtins/function.rs b/crates/vm/src/builtins/function.rs index 4125ef4c4c6..47fda299455 100644 --- a/crates/vm/src/builtins/function.rs +++ b/crates/vm/src/builtins/function.rs @@ -95,7 +95,12 @@ unsafe impl Traverse for PyFunction { fn traverse(&self, tracer_fn: &mut TraverseFn<'_>) { self.globals.traverse(tracer_fn); if let Some(closure) = self.closure.as_ref() { - closure.as_untyped().traverse(tracer_fn); + // Visit the closure tuple itself as an edge, not its cells: the + // tuple is a tracked object that can join a reference cycle, and + // `clear` releases the whole tuple. Visiting only the cells would + // leave the tuple's reference unaccounted, stranding it as a false + // GC root. + tracer_fn(closure.as_untyped().as_object()); } self.defaults_and_kwdefaults.traverse(tracer_fn); // Traverse additional fields that may contain references @@ -580,30 +585,40 @@ impl Py { .into_ref(&vm.ctx); self.fill_locals_from_args(&frame, func_args, vm)?; - if is_async_gen { - let obj = PyAsyncGen::new(frame.clone(), self.__name__(), self.__qualname__()) - .into_pyobject(vm); - frame.set_generator(&obj); - Ok(obj) - } else if is_gen { - let obj = PyGenerator::new(frame.clone(), self.__name__(), self.__qualname__()) - .into_pyobject(vm); - frame.set_generator(&obj); - Ok(obj) - } else if is_coro { - let obj = PyCoroutine::new(frame.clone(), self.__name__(), self.__qualname__()) - .into_pyobject(vm); - frame.set_generator(&obj); - Ok(obj) - } else { + if use_datastack { let result = vm.run_frame(frame.clone()); // Release data stack memory after frame execution completes. + crate::frame::release_datastack_frame(&frame, vm); + result + } else { + let obj = if is_async_gen { + PyAsyncGen::new(frame.clone(), self.__name__(), self.__qualname__()) + .into_pyobject(vm) + } else if is_gen { + PyGenerator::new(frame.clone(), self.__name__(), self.__qualname__()) + .into_pyobject(vm) + } else { + PyCoroutine::new(frame.clone(), self.__name__(), self.__qualname__()) + .into_pyobject(vm) + }; + // Generator/coroutine frames outlive this call and can join a + // reference cycle through their owning generator, so they must + // participate in the GC. They were created untracked + // (NEW_REF_UNTRACKED); track them now, before the back-reference + // is installed. Their localsplus is heap-backed by construction + // (use_datastack == false), so a collector never reads data-stack + // storage when it traverses them. + debug_assert!( + !frame.localsplus_is_datastack_backed(), + "generator frame is data-stack-backed" + ); + // SAFETY: the frame is alive (held by `frame`) and untracked. unsafe { - if let Some(base) = frame.materialize_localsplus() { - vm.datastack_pop(base); - } + crate::gc_state::gc_state() + .track_object(core::ptr::NonNull::from(frame.as_object())); } - result + frame.set_generator(&obj); + Ok(obj) } } @@ -634,16 +649,6 @@ impl Py { new_v } - /// function_kind(SIMPLE_FUNCTION) equivalent for CALL specialization. - /// Returns true if: CO_OPTIMIZED, no VARARGS, no VARKEYWORDS, no kwonly args. - pub(crate) fn is_simple_for_call_specialization(&self) -> bool { - let code: &Py = &self.code; - let flags = code.flags; - flags.contains(bytecode::CodeFlags::OPTIMIZED) - && !flags.intersects(bytecode::CodeFlags::VARARGS | bytecode::CodeFlags::VARKEYWORDS) - && code.kwonlyarg_count == 0 - } - /// Check if this function is eligible for exact-args call specialization. /// Returns true if: CO_OPTIMIZED, no VARARGS, no VARKEYWORDS, no kwonly args, /// and effective_nargs matches co_argcount. @@ -656,6 +661,16 @@ impl Py { && code.arg_count == effective_nargs } + /// True if the code object is a generator, coroutine or async generator. + #[inline] + pub(crate) fn is_generator_like(&self) -> bool { + self.code.flags.intersects( + bytecode::CodeFlags::GENERATOR + | bytecode::CodeFlags::COROUTINE + | bytecode::CodeFlags::ASYNC_GENERATOR, + ) + } + /// Runtime guard for CALL_*_EXACT_ARGS specialization: check only argcount. /// Other invariants are guaranteed by function versioning and specialization-time checks. #[inline] @@ -672,7 +687,7 @@ impl Py { pub(crate) fn prepare_exact_args_frame( &self, - mut args: Vec, + args: impl ExactSizeIterator, vm: &VirtualMachine, ) -> FrameRef { let code: PyRef = (*self.code).to_owned(); @@ -710,7 +725,7 @@ impl Py { { let fastlocals = unsafe { frame.fastlocals_mut() }; - for (slot, arg) in fastlocals.iter_mut().zip(args.drain(..)) { + for (slot, arg) in fastlocals.iter_mut().zip(args) { *slot = Some(arg); } } @@ -718,41 +733,55 @@ impl Py { frame } + fn invoke_prepared_exact_args( + &self, + args: impl ExactSizeIterator, + vm: &VirtualMachine, + ) -> PyResult { + let frame = self.prepare_exact_args_frame(args, vm); + + let result = vm.run_frame(frame.clone()); + crate::frame::release_datastack_frame(&frame, vm); + result + } + /// Fast path for calling a simple function with exact positional args. /// Skips FuncArgs allocation, prepend_arg, and fill_locals_from_args. /// Only valid when: CO_OPTIMIZED, no VARARGS, no VARKEYWORDS, no kwonlyargs, /// and nargs == co_argcount. pub fn invoke_exact_args(&self, args: Vec, vm: &VirtualMachine) -> PyResult { - let code: PyRef = (*self.code).to_owned(); - - debug_assert_eq!(args.len(), code.arg_count as usize); - debug_assert!(code.flags.contains(bytecode::CodeFlags::OPTIMIZED)); - debug_assert!( - !code - .flags - .intersects(bytecode::CodeFlags::VARARGS | bytecode::CodeFlags::VARKEYWORDS) - ); - debug_assert_eq!(code.kwonlyarg_count, 0); + debug_assert_eq!(args.len(), self.code.arg_count as usize); // Generator/coroutine code objects are SIMPLE_FUNCTION in call // specialization classification, but their call path must still // go through invoke() to produce generator/coroutine objects. - if code.flags.intersects( - bytecode::CodeFlags::GENERATOR - | bytecode::CodeFlags::COROUTINE - | bytecode::CodeFlags::ASYNC_GENERATOR, - ) { + if self.is_generator_like() { return self.invoke(FuncArgs::from(args), vm); } - let frame = self.prepare_exact_args_frame(args, vm); + self.invoke_prepared_exact_args(args.into_iter(), vm) + } - let result = vm.run_frame(frame.clone()); - unsafe { - if let Some(base) = frame.materialize_localsplus() { - vm.datastack_pop(base); - } + /// Like `invoke_exact_args`, but moves the args out of caller-provided + /// slots (all filled with `Some`), so callers can stage them in a + /// fixed-size stack buffer instead of allocating a Vec per call. + pub(crate) fn invoke_exact_args_slots( + &self, + args: &mut [Option], + vm: &VirtualMachine, + ) -> PyResult { + debug_assert_eq!(args.len(), self.code.arg_count as usize); + + let taken = args + .iter_mut() + .map(|slot| slot.take().expect("arg slot must be filled")); + // Generator/coroutine code objects are SIMPLE_FUNCTION in call + // specialization classification, but their call path must still + // go through invoke() to produce generator/coroutine objects. + if self.is_generator_like() { + let args: Vec = taken.collect(); + return self.invoke(FuncArgs::from(args), vm); } - result + self.invoke_prepared_exact_args(taken, vm) } } @@ -1476,14 +1505,10 @@ pub(crate) fn vectorcall_function( // FAST PATH: simple positional-only call, exact arg count. // Move owned args directly into fastlocals — no clone needed. args.truncate(nargs); - let frame = zelf.prepare_exact_args_frame(args, vm); + let frame = zelf.prepare_exact_args_frame(args.into_iter(), vm); let result = vm.run_frame(frame.clone()); - unsafe { - if let Some(base) = frame.materialize_localsplus() { - vm.datastack_pop(base); - } - } + crate::frame::release_datastack_frame(&frame, vm); return result; } diff --git a/crates/vm/src/builtins/int.rs b/crates/vm/src/builtins/int.rs index b9246149731..c5aa607d023 100644 --- a/crates/vm/src/builtins/int.rs +++ b/crates/vm/src/builtins/int.rs @@ -310,7 +310,7 @@ impl PyInt { #[must_use] pub fn to_str_radix_10(&self) -> String { match self.value.to_i64() { - Some(i) => i.to_string(), + Some(i) => itoa::Buffer::new().format(i).to_owned(), None => self.value.to_string(), } } diff --git a/crates/vm/src/builtins/mod.rs b/crates/vm/src/builtins/mod.rs index eba5af36686..ffc01b00f29 100644 --- a/crates/vm/src/builtins/mod.rs +++ b/crates/vm/src/builtins/mod.rs @@ -28,6 +28,8 @@ pub use filter::PyFilter; pub(crate) mod float; pub use float::PyFloat; pub(crate) mod frame; +pub(crate) mod frame_locals_proxy; +pub use frame_locals_proxy::FrameLocalsProxy; pub(crate) mod function; pub use function::{PyBoundMethod, PyFunction}; pub(crate) mod generator; diff --git a/crates/vm/src/builtins/object.rs b/crates/vm/src/builtins/object.rs index 147e215a0cb..633eaf48a44 100644 --- a/crates/vm/src/builtins/object.rs +++ b/crates/vm/src/builtins/object.rs @@ -65,34 +65,24 @@ impl Constructor for PyBaseObject { } // Ensure that all abstract methods are implemented before instantiating instance. - if let Some(abs_methods) = cls.get_attr(identifier!(vm, __abstractmethods__)) - && let Some(unimplemented_abstract_method_count) = abs_methods.length_opt(vm) - { + if let Some(abs_methods) = cls.get_attr(identifier!(vm, __abstractmethods__)) { let methods: Vec = abs_methods.try_to_value(vm)?; - let methods: String = Itertools::intersperse( - methods.iter().map(|name| name.as_str().to_owned()), - "', '".to_owned(), - ) - .collect(); - - let unimplemented_abstract_method_count = unimplemented_abstract_method_count?; - let name = cls.name().to_string(); - - match unimplemented_abstract_method_count { - 0 => {} - 1 => { - return Err(vm.new_type_error(format!( - "class {name} without an implementation for abstract method '{methods}'" - ))); - } - 2.. => { - return Err(vm.new_type_error(format!( - "class {name} without an implementation for abstract methods '{methods}'" - ))); - } - // TODO: remove `allow` when redox build doesn't complain about it - #[allow(unreachable_patterns)] - _ => unreachable!(), + let unimplemented_abstract_method_count = methods.len(); + if unimplemented_abstract_method_count > 0 { + let methods: String = Itertools::intersperse( + methods.iter().map(|name| name.as_str().to_owned()), + "', '".to_owned(), + ) + .collect(); + let name = cls.name().to_string(); + let noun = if unimplemented_abstract_method_count == 1 { + "method" + } else { + "methods" + }; + return Err(vm.new_type_error(format!( + "class {name} without an implementation for abstract {noun} '{methods}'" + ))); } } @@ -346,23 +336,7 @@ impl PyBaseObject { Ok(res) } - /// Implement setattr(self, name, value). - #[pymethod] - fn __setattr__( - obj: PyObjectRef, - name: PyStrRef, - value: PyObjectRef, - vm: &VirtualMachine, - ) -> PyResult<()> { - obj.generic_setattr(&name, PySetterValue::Assign(value), vm) - } - - /// Implement delattr(self, name). - #[pymethod] - fn __delattr__(obj: PyObjectRef, name: PyStrRef, vm: &VirtualMachine) -> PyResult<()> { - obj.generic_setattr(&name, PySetterValue::Delete, vm) - } - + // __setattr__ and __delattr__ are added as slot wrappers by add_operators. #[pyslot] pub(crate) fn slot_setattro( obj: &PyObject, @@ -461,39 +435,7 @@ impl PyBaseObject { && !cls.slots.flags.has_feature(PyTypeFlags::IMMUTABLETYPE); // FIXME(#1979) cls instances might have a payload if both_mutable || both_module { - let has_dict = - |typ: &Py| typ.slots.flags.has_feature(PyTypeFlags::HAS_DICT); - let has_weakref = - |typ: &Py| typ.slots.flags.has_feature(PyTypeFlags::HAS_WEAKREF); - // Compare slots tuples - let slots_equal = match ( - current_cls - .heaptype_ext - .as_ref() - .and_then(|e| e.slots.as_ref()), - cls.heaptype_ext.as_ref().and_then(|e| e.slots.as_ref()), - ) { - (Some(a), Some(b)) => { - a.len() == b.len() - && a.iter() - .zip(b.iter()) - .all(|(x, y)| x.as_wtf8() == y.as_wtf8()) - } - (None, None) => true, - _ => false, - }; - if current_cls.slots.basicsize != cls.slots.basicsize - || !slots_equal - || has_dict(current_cls) != has_dict(&cls) - || has_weakref(current_cls) != has_weakref(&cls) - || current_cls.slots.member_count != cls.slots.member_count - { - return Err(vm.new_type_error(format!( - "__class__ assignment: '{}' object layout differs from '{}'", - cls.name(), - current_cls.name() - ))); - } + super::type_::compatible_for_assignment(current_cls, &cls, "__class__", vm)?; instance.set_class(cls, vm); Ok(()) } else { @@ -513,17 +455,14 @@ impl PyBaseObject { } /// Return getattr(self, name). + /// + /// __getattribute__ is added as a slot wrapper by add_operators. #[pyslot] pub(crate) fn getattro(obj: &PyObject, name: &Py, vm: &VirtualMachine) -> PyResult { vm_trace!("object.__getattribute__({:?}, {:?})", obj, name); obj.as_object().generic_getattr(name, vm) } - #[pymethod] - fn __getattribute__(obj: PyObjectRef, name: PyStrRef, vm: &VirtualMachine) -> PyResult { - Self::getattro(&obj, &name, vm) - } - #[pymethod] fn __reduce__(obj: PyObjectRef, vm: &VirtualMachine) -> PyResult { common_reduce(obj, 0, vm) diff --git a/crates/vm/src/builtins/tuple.rs b/crates/vm/src/builtins/tuple.rs index 7ed815ce24d..d48639b2c11 100644 --- a/crates/vm/src/builtins/tuple.rs +++ b/crates/vm/src/builtins/tuple.rs @@ -51,46 +51,14 @@ unsafe impl Traverse for PyTuple { } } -// spell-checker:ignore MAXSAVESIZE -/// Per-size freelist storage for tuples, matching tuples[PyTuple_MAXSAVESIZE]. -/// Each bucket caches tuples of a specific element count (index = len - 1). -struct TupleFreeList { - buckets: [Vec>; Self::MAX_SAVE_SIZE], -} - -impl TupleFreeList { - /// Largest tuple size to cache on the freelist (sizes 1..=20). - const MAX_SAVE_SIZE: usize = 20; - const fn new() -> Self { - Self { - buckets: [const { Vec::new() }; Self::MAX_SAVE_SIZE], - } - } -} - -impl Default for TupleFreeList { - fn default() -> Self { - Self::new() - } -} - -impl Drop for TupleFreeList { - fn drop(&mut self) { - // Same safety pattern as FreeList::drop — free raw allocation - // without running payload destructors to avoid TLS-after-destruction panics. - let layout = crate::object::pyinner_layout::(); - for bucket in &mut self.buckets { - for ptr in bucket.drain(..) { - unsafe { - alloc::alloc::dealloc(ptr.as_ptr() as *mut u8, layout); - } - } - } - } -} - thread_local! { - static TUPLE_FREELIST: Cell = const { Cell::new(TupleFreeList::new()) }; + // A single freelist for all tuple sizes: `PyInner` is a + // fixed-size allocation (elements are a separate boxed slice that is + // dropped and replaced on reuse), so husks are interchangeable. + // freelist_push must not read the payload — it runs after tp_clear, + // which has already emptied `elements`. + static TUPLE_FREELIST: Cell> = + const { Cell::new(crate::object::FreeList::new()) }; } impl PyPayload for PyTuple { @@ -104,16 +72,11 @@ impl PyPayload for PyTuple { #[inline] unsafe fn freelist_push(obj: *mut PyObject) -> bool { - let len = unsafe { &*(obj as *const crate::Py) }.elements.len(); - if len == 0 || len > TupleFreeList::MAX_SAVE_SIZE { - return false; - } TUPLE_FREELIST .try_with(|fl| { let mut list = fl.take(); - let bucket = &mut list.buckets[len - 1]; - let stored = if bucket.len() < Self::MAX_FREELIST { - bucket.push(unsafe { NonNull::new_unchecked(obj) }); + let stored = if list.len() < Self::MAX_FREELIST { + list.push(obj); true } else { false @@ -125,15 +88,11 @@ impl PyPayload for PyTuple { } #[inline] - unsafe fn freelist_pop(payload: &Self) -> Option> { - let len = payload.elements.len(); - if len == 0 || len > TupleFreeList::MAX_SAVE_SIZE { - return None; - } + unsafe fn freelist_pop(_payload: &Self) -> Option> { TUPLE_FREELIST .try_with(|fl| { let mut list = fl.take(); - let result = list.buckets[len - 1].pop(); + let result = list.pop().map(|p| unsafe { NonNull::new_unchecked(p) }); fl.set(list); result }) diff --git a/crates/vm/src/builtins/type.rs b/crates/vm/src/builtins/type.rs index 19fca5cf473..1c98e6861bc 100644 --- a/crates/vm/src/builtins/type.rs +++ b/crates/vm/src/builtins/type.rs @@ -18,7 +18,7 @@ use crate::{ common::{ ascii, borrow::BorrowedValue, - lock::{PyMutex, PyRwLock, PyRwLockReadGuard}, + lock::{PyRwLock, PyRwLockReadGuard}, }, function::{FuncArgs, KwArgs, OptionalArg, PyMethodDef, PySetterValue}, object::{Traverse, TraverseFn}, @@ -44,7 +44,8 @@ use std::collections::HashSet; #[pyclass(module = false, name = "type", traverse = "manual")] pub struct PyType { - pub base: Option, + /// tp_base. Written under the type lock (see `set_bases`); read lock-free. + pub base: PyAtomicRef>, pub bases: PyRwLock>, pub mro: PyRwLock>, pub subclasses: PyRwLock>>, @@ -57,7 +58,7 @@ pub struct PyType { } /// Monotonic counter for type version tags. Once it reaches `u32::MAX`, -/// `assign_version_tag()` returns 0 permanently, disabling new inline-cache +/// version assignment returns 0 permanently, disabling new inline-cache /// entries but not invalidating correctness (cache misses fall back to the /// generic path). static NEXT_TYPE_VERSION: AtomicU32 = AtomicU32::new(1); @@ -217,9 +218,30 @@ pub(crate) fn type_cache_clear() { TYPE_CACHE_CLEARING.store(false, Ordering::Release); } +/// Repair type-cache SeqLock state in the post-fork child. +/// +/// If fork happens while a writer holds an entry SeqLock, the child inherits +/// the odd sequence value with no surviving writer to release it. Clear only +/// those in-progress entries, matching `_PyTypes_AfterFork()`. +#[cfg(all(feature = "host_env", unix))] +pub(crate) unsafe fn type_cache_after_fork() { + for entry in TYPE_CACHE.iter() { + let seq = entry.sequence.load(Ordering::Relaxed); + if (seq & 1) == 0 { + continue; + } + entry.value.store(core::ptr::null_mut(), Ordering::Relaxed); + entry.name.store(core::ptr::null_mut(), Ordering::Relaxed); + entry.version.store(0, Ordering::Relaxed); + entry.sequence.store(0, Ordering::Relaxed); + } +} + unsafe impl crate::object::Traverse for PyType { fn traverse(&self, tracer_fn: &mut crate::object::TraverseFn<'_>) { - self.base.traverse(tracer_fn); + if let Some(base) = self.base.deref() { + tracer_fn(base.as_object()); + } self.bases.traverse(tracer_fn); self.mro.traverse(tracer_fn); self.subclasses.traverse(tracer_fn); @@ -235,7 +257,8 @@ unsafe impl crate::object::Traverse for PyType { /// type_clear: break reference cycles in type objects fn clear(&mut self, out: &mut Vec) { - if let Some(base) = self.base.take() { + // SAFETY: tp_clear runs with exclusive access to the type object. + if let Some(base) = unsafe { self.base.swap(None) } { out.push(base.into()); } if let Some(mut guard) = self.bases.try_write() { @@ -275,65 +298,53 @@ pub struct HeapTypeExt { pub struct TypeSpecializationCache { pub init: PyAtomicRef>, + pub init_version: AtomicU32, pub getitem: PyAtomicRef>, pub getitem_version: AtomicU32, - // Serialize cache writes/invalidation similar to CPython's BEGIN_TYPE_LOCK. - write_lock: PyMutex<()>, - retired: PyRwLock>, } impl TypeSpecializationCache { fn new() -> Self { Self { init: PyAtomicRef::from(None::>), + init_version: AtomicU32::new(0), getitem: PyAtomicRef::from(None::>), getitem_version: AtomicU32::new(0), - write_lock: PyMutex::new(()), - retired: PyRwLock::new(Vec::new()), - } - } - - #[inline] - fn retire_old_function(&self, old: Option>) { - if let Some(old) = old { - self.retired.write().push(old.into()); } } #[inline] - fn swap_init(&self, new_init: Option>, vm: Option<&VirtualMachine>) { - if let Some(vm) = vm { - // Keep replaced refs alive for the currently executing frame, matching - // CPython-style "old pointer remains valid during ongoing execution" - // without accumulating global retired refs. - self.init.swap_to_temporary_refs(new_init, vm); - return; + fn swap_init(&self, new_init: Option>) { + if let Some(new) = &new_init { + new.as_object().mark_cache_published(); } - // SAFETY: old value is moved to `retired`, so it stays alive while - // concurrent readers may still hold borrowed references. + // SAFETY: reclamation of published objects is deferred via QSBR; + // racing try_to_owned readers never touch freed memory. let old = unsafe { self.init.swap(new_init) }; - self.retire_old_function(old); + if let Some(old) = old { + // Dropping may run arbitrary Python; defer past the type lock. + rustpython_common::refcount::try_defer_drop(move || drop(old)); + } } #[inline] - fn swap_getitem(&self, new_getitem: Option>, vm: Option<&VirtualMachine>) { - if let Some(vm) = vm { - self.getitem.swap_to_temporary_refs(new_getitem, vm); - return; + fn swap_getitem(&self, new_getitem: Option>) { + if let Some(new) = &new_getitem { + new.as_object().mark_cache_published(); } - // SAFETY: old value is moved to `retired`, so it stays alive while - // concurrent readers may still hold borrowed references. + // SAFETY: as in swap_init. let old = unsafe { self.getitem.swap(new_getitem) }; - self.retire_old_function(old); + if let Some(old) = old { + rustpython_common::refcount::try_defer_drop(move || drop(old)); + } } #[inline] fn invalidate_for_type_modified(&self) { - let _guard = self.write_lock.lock(); - // _spec_cache contract: type modification invalidates all cached - // specialization functions. - self.swap_init(None, None); - self.swap_getitem(None, None); + self.swap_init(None); + self.init_version.store(0, Ordering::Release); + self.swap_getitem(None); + self.getitem_version.store(0, Ordering::Release); } fn traverse(&self, tracer_fn: &mut TraverseFn<'_>) { @@ -343,25 +354,19 @@ impl TypeSpecializationCache { if let Some(getitem) = self.getitem.deref() { tracer_fn(getitem.as_object()); } - self.retired - .read() - .iter() - .map(|obj| obj.traverse(tracer_fn)) - .count(); } fn clear_into(&self, out: &mut Vec) { - let _guard = self.write_lock.lock(); let old_init = unsafe { self.init.swap(None) }; if let Some(old_init) = old_init { out.push(old_init.into()); } + self.init_version.store(0, Ordering::Release); let old_getitem = unsafe { self.getitem.swap(None) }; if let Some(old_getitem) = old_getitem { out.push(old_getitem.into()); } self.getitem_version.store(0, Ordering::Release); - out.extend(self.retired.write().drain(..)); } } @@ -460,9 +465,19 @@ fn is_subtype_with_mro(a_mro: &[PyTypeRef], a: &Py, b: &Py) -> b } impl PyType { + #[inline] + fn with_type_lock(vm: &VirtualMachine, f: impl FnOnce() -> R) -> R { + // Drops deferred via try_defer_drop inside the critical section run + // after the guard is released, outside the lock. + rustpython_common::refcount::with_deferred_drops(|| { + let _guard = vm.state.type_mutex.lock(); + f() + }) + } + /// Assign a fresh version tag. Returns 0 if the version counter has been /// exhausted, in which case no new cache entries can be created. - pub fn assign_version_tag(&self) -> u32 { + fn assign_version_tag_inner(&self) -> u32 { let v = self.tp_version_tag.load(Ordering::Acquire); if v != 0 { return v; @@ -470,7 +485,7 @@ impl PyType { // Assign versions to all direct bases first (MRO invariant). for base in self.bases.read().iter() { - if base.assign_version_tag() == 0 { + if base.assign_version_tag_inner() == 0 { return 0; } } @@ -490,27 +505,52 @@ impl PyType { } } - /// Invalidate this type's version tag and cascade to all subclasses. - pub fn modified(&self) { - if let Some(ext) = self.heaptype_ext.as_ref() { - ext.specialization_cache.invalidate_for_type_modified(); + pub(crate) fn version_for_specialization(&self, vm: &VirtualMachine) -> u32 { + let version = self.tp_version_tag.load(Ordering::Acquire); + if version != 0 { + return version; } - // If already invalidated, all subclasses must also be invalidated - // (guaranteed by the MRO invariant in assign_version_tag). + Self::with_type_lock(vm, || { + let version = self.tp_version_tag.load(Ordering::Acquire); + if version == 0 { + self.assign_version_tag_inner() + } else { + version + } + }) + } + + /// Invalidate this type's version tag and cascade to all subclasses. + fn modified_inner(&self) { let old_version = self.tp_version_tag.load(Ordering::Acquire); if old_version == 0 { return; } - self.tp_version_tag.store(0, Ordering::SeqCst); - // Nullify borrowed pointers in cache entries for this version - // so they don't dangle after the dict is modified. - type_cache_clear_version(old_version); let subclasses = self.subclasses.read(); for weak_ref in subclasses.iter() { if let Some(sub) = weak_ref.upgrade() { - sub.downcast_ref::().unwrap().modified(); + sub.downcast_ref::().unwrap().modified_inner(); } } + self.tp_version_tag.store(0, Ordering::SeqCst); + // Nullify borrowed pointers in cache entries for this version + // so they don't dangle after the dict is modified. + type_cache_clear_version(old_version); + if let Some(ext) = self.heaptype_ext.as_ref() { + ext.specialization_cache.invalidate_for_type_modified(); + } + } + + pub fn modified(&self) { + if self.tp_version_tag.load(Ordering::Acquire) == 0 { + return; + } + if let Some(()) = crate::vm::thread::try_with_current_vm(|vm| { + Self::with_type_lock(vm, || self.modified_inner()); + }) { + return; + } + self.modified_inner(); } pub fn new_simple_heap( @@ -773,7 +813,7 @@ impl PyType { let inherited_abc_tpflags = Self::inherited_abc_tpflags(&bases); let new_type = PyRef::new_ref( Self { - base: Some(base), + base: Some(base).into(), bases: PyRwLock::new(bases), mro: PyRwLock::new(mro), subclasses: PyRwLock::default(), @@ -837,7 +877,7 @@ impl PyType { let new_type = PyRef::new_ref( Self { - base: Some(base), + base: Some(base).into(), bases, mro: PyRwLock::new(mro), subclasses: PyRwLock::default(), @@ -864,8 +904,8 @@ impl PyType { // Note: inherit_slots is called in PyClassImpl::init_class after // slots are fully initialized by make_slots() - Self::set_new(&new_type.slots, new_type.base.as_ref()); - Self::set_alloc(&new_type.slots, new_type.base.as_ref()); + Self::set_new(&new_type.slots, new_type.base.deref()); + Self::set_alloc(&new_type.slots, new_type.base.deref()); let weakref_type = super::PyWeak::static_type(); for base in new_type.bases.read().iter() { @@ -911,11 +951,31 @@ impl PyType { self.update_slot::(attr_name, ctx); } - Self::set_new(&self.slots, self.base.as_ref()); - Self::set_alloc(&self.slots, self.base.as_ref()); + Self::set_new(&self.slots, self.base.deref()); + Self::set_alloc(&self.slots, self.base.deref()); + } + + /// Recompute every slot for this type and all its descendants. update_all_slots + /// + /// Unlike `init_slots`, which is additive and driven only by the dunder names + /// present in the current MRO, this iterates the full `SLOT_DEFS` name table so + /// a slot whose method left the MRO is reset instead of left stale. Must be + /// called under the type lock after MROs have been recomputed. + pub(crate) fn update_all_slots(&self, ctx: &Context) { + // Invalidate version tags first; cascades to subclasses. + self.modified_inner(); + // Distinct names only; update_slot fans out to every SLOT_DEFS entry + // sharing the name and recurses into subclasses on its own. + let mut seen = std::collections::HashSet::new(); + for def in SLOT_DEFS { + if seen.insert(def.name) { + let name = ctx.intern_str(def.name); + self.update_slot::(name, ctx); + } + } } - fn set_new(slots: &PyTypeSlots, base: Option<&PyTypeRef>) { + fn set_new(slots: &PyTypeSlots, base: Option<&Py>) { if slots.flags.contains(PyTypeFlags::DISALLOW_INSTANTIATION) { slots.new.store(None) } else if slots.new.load().is_none() { @@ -923,7 +983,7 @@ impl PyType { } } - fn set_alloc(slots: &PyTypeSlots, base: Option<&PyTypeRef>) { + fn set_alloc(slots: &PyTypeSlots, base: Option<&Py>) { if slots.alloc.load().is_none() { slots .alloc @@ -974,6 +1034,85 @@ impl PyType { self.find_name_in_mro(attr_name) } + /// `_PyType_LookupRefAndVersion` equivalent for interned names. + /// Returns the observed lookup result and the type version used for the lookup. + /// + /// Uses a lock-free SeqLock-style pattern: + /// Read: load sequence/version/name → load value + try_to_owned → + /// validate value pointer + sequence + /// Write: sequence(begin) → version=0 → swap value/name → version=assigned → sequence(end) + pub(crate) fn lookup_ref_and_version_interned( + &self, + name: &'static PyStrInterned, + vm: &VirtualMachine, + ) -> (Option, u32) { + #[cfg(all(feature = "threading", debug_assertions))] + crate::vm::thread::debug_assert_current_thread_attached(); + + let version = self.tp_version_tag.load(Ordering::Acquire); + if version != 0 { + let idx = type_cache_hash(version, name); + let entry = &TYPE_CACHE[idx]; + let name_ptr = name as *const _ as *mut _; + loop { + let seq1 = entry.begin_read(); + let entry_version = entry.version.load(Ordering::Acquire); + let type_version = self.tp_version_tag.load(Ordering::Acquire); + if entry_version != type_version + || !core::ptr::eq(entry.name.load(Ordering::Relaxed), name_ptr) + { + break; + } + let ptr = entry.value.load(Ordering::Acquire); + if ptr.is_null() { + if entry.end_read(seq1) { + return (None, entry_version); + } + continue; + } + if let Some(cloned) = unsafe { PyObject::try_to_owned_from_ptr(ptr) } { + let same_ptr = core::ptr::eq(entry.value.load(Ordering::Relaxed), ptr); + if same_ptr && entry.end_read(seq1) { + return (Some(cloned), entry_version); + } + drop(cloned); + continue; + } + break; + } + } + + Self::with_type_lock(vm, || { + let assigned = if self.tp_version_tag.load(Ordering::Acquire) == 0 { + self.assign_version_tag_inner() + } else { + self.tp_version_tag.load(Ordering::Acquire) + }; + let result = self.find_name_in_mro_uncached(name); + if assigned != 0 + && !TYPE_CACHE_CLEARING.load(Ordering::Acquire) + && self.tp_version_tag.load(Ordering::Acquire) == assigned + { + let idx = type_cache_hash(assigned, name); + let entry = &TYPE_CACHE[idx]; + let name_ptr = name as *const _ as *mut _; + entry.begin_write(); + entry.version.store(0, Ordering::Release); + let new_ptr = result.as_ref().map_or(core::ptr::null_mut(), |found| { + // Defer memory reclamation of cached values via QSBR so + // racing readers never try-incref freed memory. + found.mark_cache_published(); + &**found as *const PyObject as *mut _ + }); + entry.value.store(new_ptr, Ordering::Relaxed); + entry.name.store(name_ptr, Ordering::Relaxed); + entry.version.store(assigned, Ordering::Release); + entry.end_write(); + } + (result, assigned) + }) + } + /// Cache __init__ for CALL_ALLOC_AND_ENTER_INIT specialization. /// The cache is valid only when guarded by the type version check. pub(crate) fn cache_init_for_specialization( @@ -988,22 +1127,27 @@ impl PyType { if tp_version == 0 { return false; } - if self.tp_version_tag.load(Ordering::Acquire) != tp_version { - return false; - } - let _guard = ext.specialization_cache.write_lock.lock(); - if self.tp_version_tag.load(Ordering::Acquire) != tp_version { - return false; - } - ext.specialization_cache.swap_init(Some(init), Some(vm)); - true + Self::with_type_lock(vm, || { + if self.tp_version_tag.load(Ordering::Acquire) != tp_version { + return false; + } + let func_version = init.get_version_for_current_state(); + if func_version == 0 { + return false; + } + ext.specialization_cache.swap_init(Some(init)); + ext.specialization_cache + .init_version + .store(func_version, Ordering::Release); + true + }) } /// Read cached __init__ for CALL_ALLOC_AND_ENTER_INIT specialization. pub(crate) fn get_cached_init_for_specialization( &self, tp_version: u32, - ) -> Option> { + ) -> Option<(PyRef, u32)> { let ext = self.heaptype_ext.as_ref()?; if tp_version == 0 { return None; @@ -1011,9 +1155,19 @@ impl PyType { if self.tp_version_tag.load(Ordering::Acquire) != tp_version { return None; } - ext.specialization_cache + // Check order: pointer (Acquire) then function version. + let init = ext + .specialization_cache .init - .to_owned_ordering(Ordering::Acquire) + .try_to_owned(Ordering::Acquire)?; + let cached_version = ext + .specialization_cache + .init_version + .load(Ordering::Acquire); + if cached_version == 0 { + return None; + } + Some((init, cached_version)) } /// Cache __getitem__ for BINARY_OP_SUBSCR_GETITEM specialization. @@ -1030,34 +1184,34 @@ impl PyType { if tp_version == 0 { return false; } - let _guard = ext.specialization_cache.write_lock.lock(); - if self.tp_version_tag.load(Ordering::Acquire) != tp_version { - return false; - } - let func_version = getitem.get_version_for_current_state(); - if func_version == 0 { - return false; - } - ext.specialization_cache - .swap_getitem(Some(getitem), Some(vm)); - ext.specialization_cache - .getitem_version - .store(func_version, Ordering::Relaxed); - true + Self::with_type_lock(vm, || { + if self.tp_version_tag.load(Ordering::Acquire) != tp_version { + return false; + } + let func_version = getitem.get_version_for_current_state(); + if func_version == 0 { + return false; + } + ext.specialization_cache.swap_getitem(Some(getitem)); + ext.specialization_cache + .getitem_version + .store(func_version, Ordering::Release); + true + }) } /// Read cached __getitem__ for BINARY_OP_SUBSCR_GETITEM specialization. pub(crate) fn get_cached_getitem_for_specialization(&self) -> Option<(PyRef, u32)> { let ext = self.heaptype_ext.as_ref()?; - // Match CPython check order: pointer (Acquire) then function version. + // Check order: pointer (Acquire) then function version. let getitem = ext .specialization_cache .getitem - .to_owned_ordering(Ordering::Acquire)?; + .try_to_owned(Ordering::Acquire)?; let cached_version = ext .specialization_cache .getitem_version - .load(Ordering::Relaxed); + .load(Ordering::Acquire); if cached_version == 0 { return None; } @@ -1070,82 +1224,14 @@ impl PyType { /// find_name_in_mro with method cache (MCACHE). /// Looks in tp_dict of types in MRO, bypasses descriptors. - /// - /// Uses a lock-free SeqLock-style pattern: - /// Read: load sequence/version/name → load value + try_to_owned → - /// validate value pointer + sequence - /// Write: sequence(begin) → version=0 → swap value/name → version=assigned → sequence(end) fn find_name_in_mro(&self, name: &'static PyStrInterned) -> Option { - let version = self.tp_version_tag.load(Ordering::Acquire); - if version != 0 { - let idx = type_cache_hash(version, name); - let entry = &TYPE_CACHE[idx]; - let name_ptr = name as *const _ as *mut _; - loop { - let seq1 = entry.begin_read(); - let v1 = entry.version.load(Ordering::Acquire); - let type_version = self.tp_version_tag.load(Ordering::Acquire); - if v1 != type_version - || !core::ptr::eq(entry.name.load(Ordering::Relaxed), name_ptr) - { - break; - } - let ptr = entry.value.load(Ordering::Acquire); - if ptr.is_null() { - if entry.end_read(seq1) { - break; - } - continue; - } - // _Py_TryIncrefCompare-style validation: - // safe_inc via raw pointer, then ensure source is unchanged. - if let Some(cloned) = unsafe { PyObject::try_to_owned_from_ptr(ptr) } { - let same_ptr = core::ptr::eq(entry.value.load(Ordering::Relaxed), ptr); - if same_ptr && entry.end_read(seq1) { - return Some(cloned); - } - drop(cloned); - continue; - } - break; - } - } - - // Assign version BEFORE the MRO walk so that any concurrent - // modified() call during the walk invalidates this version. - let assigned = if version == 0 { - self.assign_version_tag() - } else { - version - }; - - // MRO walk - let result = self.find_name_in_mro_uncached(name); - - // Only cache positive results. Negative results are not cached to - // avoid stale entries from transient MRO walk failures during - // concurrent type modifications. - if let Some(ref found) = result - && assigned != 0 - && !TYPE_CACHE_CLEARING.load(Ordering::Acquire) - && self.tp_version_tag.load(Ordering::Acquire) == assigned - { - let idx = type_cache_hash(assigned, name); - let entry = &TYPE_CACHE[idx]; - let name_ptr = name as *const _ as *mut _; - entry.begin_write(); - // Invalidate first to prevent readers from seeing partial state - entry.version.store(0, Ordering::Release); - // Store borrowed pointer (no refcount increment). - let new_ptr = &**found as *const PyObject as *mut PyObject; - entry.value.store(new_ptr, Ordering::Relaxed); - entry.name.store(name_ptr, Ordering::Relaxed); - // Activate entry — Release ensures value/name writes are visible - entry.version.store(assigned, Ordering::Release); - entry.end_write(); - } - - result + crate::vm::thread::try_with_current_vm(|vm| { + self.lookup_ref_and_version_interned(name, vm).0 + }) + // No current VM: this thread is not registered for QSBR, so the + // lock-free cache read protocol is not sound here. Walk the MRO + // under the attributes locks instead (the dicts hold strong refs). + .unwrap_or_else(|| self.find_name_in_mro_uncached(name)) } /// Raw MRO walk without cache. @@ -1161,7 +1247,7 @@ impl PyType { /// _PyType_LookupRef: look up a name through the MRO without setting an exception. pub fn lookup_ref(&self, name: &Py, vm: &VirtualMachine) -> Option { let interned_name = vm.ctx.interned_str(name)?; - self.find_name_in_mro(interned_name) + self.lookup_ref_and_version_interned(interned_name, vm).0 } pub fn get_super_attr(&self, attr_name: &'static PyStrInterned) -> Option { @@ -1178,6 +1264,9 @@ impl PyType { /// Check if attribute exists in MRO, using method cache for fast check. /// Unlike find_name_in_mro, avoids cloning the value on cache hit. fn has_name_in_mro(&self, name: &'static PyStrInterned) -> bool { + #[cfg(all(feature = "threading", debug_assertions))] + crate::vm::thread::debug_assert_current_thread_attached(); + let version = self.tp_version_tag.load(Ordering::Acquire); if version != 0 { let idx = type_cache_hash(version, name); @@ -1350,7 +1439,7 @@ impl Py { } pub fn iter_base_chain(&self) -> impl Iterator { - core::iter::successors(Some(self), |cls| cls.base.as_deref()) + core::iter::successors(Some(self), |cls| cls.base.deref()) } pub fn extend_methods(&'static self, method_defs: &'static [PyMethodDef], ctx: &Context) { @@ -1397,58 +1486,144 @@ impl PyType { } if bases.is_empty() { return Err(vm.new_type_error(format!( - "can only assign non-empty tuple to %s.__bases__, not {}", + "can only assign non-empty tuple to {}.__bases__, not ()", zelf.name() ))); } // TODO: check for mro cycles - // TODO: Remove this class from all subclass lists - // for base in self.bases.read().iter() { - // let subclasses = base.subclasses.write(); - // // TODO: how to uniquely identify the subclasses to remove? - // } + // Compute the new solid base before committing anything. This also + // validates the new bases (BASETYPE flag, no instance layout + // conflict), the same checks type creation performs. + let new_base = best_base(&bases, vm)?.to_owned(); + + // Reject reparenting onto a base whose instances have an incompatible + // object layout. + let old_base = zelf.base.deref().unwrap_or(vm.ctx.types.object_type); + compatible_for_assignment(old_base, &new_base, "__bases__", vm)?; + + // References released inside the critical section are collected here + // and dropped after the lock: dropping them inside can run arbitrary + // code that re-acquires the non-reentrant type mutex. + let mut retired: Vec = Vec::new(); + + // A base swapped out of `zelf.base` may still be observed by + // concurrent lock-free readers; keep it alive in the frame's + // temporary refs so they never see a dangling pointer. + let keep_alive = |type_ref: PyTypeRef, retired: &mut Vec| { + if let Some(frame) = vm.current_frame() { + frame.temporary_refs.lock().push(type_ref.into()); + } else { + retired.push(type_ref.into()); + } + }; - *zelf.bases.write() = bases; - // Recursively update the mros of this class and all subclasses - fn update_mro_recursively(cls: &PyType, vm: &VirtualMachine) -> PyResult<()> { - let mut mro = - PyType::resolve_mro(&cls.bases.read()).map_err(|msg| vm.new_type_error(msg))?; - // Preserve self (mro[0]) when updating MRO - mro.insert(0, cls.mro.read()[0].to_owned()); - *cls.mro.write() = mro; - for subclass in cls.subclasses.write().iter() { - let subclass = subclass.upgrade().unwrap(); - let subclass: &Py = subclass.downcast_ref().unwrap(); - update_mro_recursively(subclass, vm)?; + // Register this type as a subclass of the given bases + let register_subclasses = |bases: &[PyTypeRef]| { + let weakref_type = super::PyWeak::static_type(); + for base in bases { + base.subclasses.write().push( + zelf.as_object() + .downgrade_with_weakref_typ_opt(None, weakref_type.to_owned()) + .unwrap(), + ); } - Ok(()) - } - update_mro_recursively(zelf, vm)?; + }; - // Invalidate inline caches - zelf.modified(); + let result = Self::with_type_lock(vm, || { + // Remove this class from the old bases' subclass lists, pruning + // dead entries along the way. Upgraded refs are retired so the + // last strong reference is never dropped under the lock. + for base in zelf.bases.read().iter() { + let mut subclasses = base.subclasses.write(); + let mut kept = Vec::with_capacity(subclasses.len()); + for weak in subclasses.drain(..) { + match weak.upgrade() { + Some(obj) if obj.is(zelf.as_object()) => { + retired.push(obj); + retired.push(weak.into()); + } + Some(obj) => { + retired.push(obj); + kept.push(weak); + } + None => retired.push(weak.into()), + } + } + *subclasses = kept; + } - // TODO: do any old slots need to be cleaned up first? - zelf.init_slots(&vm.ctx); + let old_bases = core::mem::replace(&mut *zelf.bases.write(), bases); + let old_base = unsafe { zelf.base.swap(Some(new_base)) }; + + // Recursively update the mros of this class and all subclasses, + // recording the previous mros so a failure can be rolled back. + fn update_mro_recursively( + cls: &Py, + undo: &mut Vec<(PyTypeRef, Vec)>, + vm: &VirtualMachine, + ) -> PyResult<()> { + let mut mro = + PyType::resolve_mro(&cls.bases.read()).map_err(|msg| vm.new_type_error(msg))?; + // Preserve self (mro[0]) when updating MRO + mro.insert(0, cls.mro.read()[0].to_owned()); + let old_mro = core::mem::replace(&mut *cls.mro.write(), mro); + undo.push((cls.to_owned(), old_mro)); + for subclass in cls.subclasses.read().iter() { + // Dead entries are pruned elsewhere; skip them here. + let Some(subclass) = subclass.upgrade() else { + continue; + }; + let subclass: &Py = subclass.downcast_ref().unwrap(); + update_mro_recursively(subclass, undo, vm)?; + } + Ok(()) + } + let mut undo = Vec::new(); + if let Err(err) = update_mro_recursively(zelf, &mut undo, vm) { + // Roll back to the previous state. A class reachable through + // multiple bases is recorded once per visit, so restore in + // reverse to end with the first-recorded (original) mro. + for (cls, old_mro) in undo.into_iter().rev() { + let failed_mro = core::mem::replace(&mut *cls.mro.write(), old_mro); + retired.extend(failed_mro.into_iter().map(Into::into)); + retired.push(cls.into()); + } + let failed_bases = core::mem::replace(&mut *zelf.bases.write(), old_bases); + if let Some(failed_base) = unsafe { zelf.base.swap(old_base) } { + keep_alive(failed_base, &mut retired); + } + register_subclasses(&zelf.bases.read()); + retired.extend(failed_bases.into_iter().map(Into::into)); + zelf.modified_inner(); + return Err(err); + } + // Retire the replaced mros as well; dropping them here would + // release them while the lock is held. + for (cls, old_mro) in undo { + retired.extend(old_mro.into_iter().map(Into::into)); + retired.push(cls.into()); + } + retired.extend(old_bases.into_iter().map(Into::into)); + if let Some(old_base) = old_base { + keep_alive(old_base, &mut retired); + } - // Register this type as a subclass of its new bases - let weakref_type = super::PyWeak::static_type(); - for base in zelf.bases.read().iter() { - base.subclasses.write().push( - zelf.as_object() - .downgrade_with_weakref_typ_opt(None, weakref_type.to_owned()) - .unwrap(), - ); - } + // Invalidate inline caches and rebuild every slot for this type and + // all descendants so slots whose methods left the MRO are reset. + zelf.update_all_slots(&vm.ctx); - Ok(()) + register_subclasses(&zelf.bases.read()); + Ok(()) + }); + drop(retired); + result } #[pygetset] fn __base__(&self) -> Option { - self.base.clone() + self.base.to_owned() } #[pygetset] @@ -1533,20 +1708,31 @@ impl PyType { ))); } - let mut attrs = self.attributes.write(); - // First try __annotate__, in case that's been set explicitly - if let Some(annotate) = attrs.get(identifier!(vm, __annotate__)).cloned() { + let annotate_key = identifier!(vm, __annotate__); + let annotate_func_key = identifier!(vm, __annotate_func__); + let attrs = self.attributes.read(); + if let Some(annotate) = attrs.get(annotate_key).cloned() { return Ok(annotate); } - // Then try __annotate_func__ - if let Some(annotate) = attrs.get(identifier!(vm, __annotate_func__)).cloned() { - // TODO: Apply descriptor tp_descr_get if needed + if let Some(annotate) = attrs.get(annotate_func_key).cloned() { return Ok(annotate); } - // Set __annotate_func__ = None and return None + drop(attrs); + let none = vm.ctx.none(); - attrs.insert(identifier!(vm, __annotate_func__), none.clone()); - Ok(none) + let (result, _prev) = Self::with_type_lock(vm, || { + let mut attrs = self.attributes.write(); + if let Some(annotate) = attrs.get(annotate_key).cloned() { + return (annotate, None); + } + if let Some(annotate) = attrs.get(annotate_func_key).cloned() { + return (annotate, None); + } + self.modified_inner(); + let prev = attrs.insert(annotate_func_key, none.clone()); + (none, prev) + }); + Ok(result) } #[pygetset(setter)] @@ -1569,20 +1755,28 @@ impl PyType { return Err(vm.new_type_error("__annotate__ must be callable or None")); } - let mut attrs = self.attributes.write(); - // Clear cached annotations only when setting to a new callable - if !vm.is_none(&value) { - attrs.swap_remove(identifier!(vm, __annotations_cache__)); - } - attrs.insert(identifier!(vm, __annotate_func__), value); + let _prev_values = Self::with_type_lock(vm, || { + self.modified_inner(); + let mut attrs = self.attributes.write(); + // Clear cached annotations only when setting to a new callable + let removed = if !vm.is_none(&value) { + attrs.swap_remove(identifier!(vm, __annotations_cache__)) + } else { + None + }; + let prev = attrs.insert(identifier!(vm, __annotate_func__), value); + (removed, prev) + }); Ok(()) } #[pygetset] fn __annotations__(&self, vm: &VirtualMachine) -> PyResult { + let annotations_key = identifier!(vm, __annotations__); + let annotations_cache_key = identifier!(vm, __annotations_cache__); let attrs = self.attributes.read(); - if let Some(annotations) = attrs.get(identifier!(vm, __annotations__)).cloned() { + if let Some(annotations) = attrs.get(annotations_key).cloned() { // Ignore the __annotations__ descriptor stored on type itself. if !annotations.class().is(vm.ctx.types.getset_type) { if vm.is_none(&annotations) @@ -1597,8 +1791,7 @@ impl PyType { ))); } } - // Then try __annotations_cache__ - if let Some(annotations) = attrs.get(identifier!(vm, __annotations_cache__)).cloned() { + if let Some(annotations) = attrs.get(annotations_cache_key).cloned() { if vm.is_none(&annotations) || annotations.class().is(vm.ctx.types.dict_type) || self.slots.flags.has_feature(PyTypeFlags::HEAPTYPE) @@ -1635,11 +1828,21 @@ impl PyType { vm.ctx.new_dict().into() }; - // Cache the result in __annotations_cache__ - self.attributes - .write() - .insert(identifier!(vm, __annotations_cache__), annotations.clone()); - Ok(annotations) + let (result, _prev) = Self::with_type_lock(vm, || { + let mut attrs = self.attributes.write(); + if let Some(existing) = attrs.get(annotations_key).cloned() + && !existing.class().is(vm.ctx.types.getset_type) + { + return (existing, None); + } + if let Some(existing) = attrs.get(annotations_cache_key).cloned() { + return (existing, None); + } + self.modified_inner(); + let prev = attrs.insert(annotations_cache_key, annotations.clone()); + (annotations, prev) + }); + Ok(result) } #[pygetset(setter)] @@ -1655,43 +1858,43 @@ impl PyType { ))); } - let mut attrs = self.attributes.write(); - let has_annotations = attrs.contains_key(identifier!(vm, __annotations__)); - - match value { - crate::function::PySetterValue::Assign(value) => { - // SET path: store the value (including None) - let key = if has_annotations { - identifier!(vm, __annotations__) - } else { - identifier!(vm, __annotations_cache__) - }; - attrs.insert(key, value); - if has_annotations { - attrs.swap_remove(identifier!(vm, __annotations_cache__)); - } - } - crate::function::PySetterValue::Delete => { - // DELETE path: remove the key - let removed = if has_annotations { - attrs - .swap_remove(identifier!(vm, __annotations__)) - .is_some() - } else { - attrs - .swap_remove(identifier!(vm, __annotations_cache__)) - .is_some() - }; - if !removed { - return Err(vm.new_attribute_error("__annotations__")); + let _prev_values = Self::with_type_lock(vm, || { + self.modified_inner(); + let mut attrs = self.attributes.write(); + let has_annotations = attrs.contains_key(identifier!(vm, __annotations__)); + + let mut prev = Vec::new(); + match value { + crate::function::PySetterValue::Assign(value) => { + let key = if has_annotations { + identifier!(vm, __annotations__) + } else { + identifier!(vm, __annotations_cache__) + }; + prev.extend(attrs.insert(key, value)); + if has_annotations { + prev.extend(attrs.swap_remove(identifier!(vm, __annotations_cache__))); + } } - if has_annotations { - attrs.swap_remove(identifier!(vm, __annotations_cache__)); + crate::function::PySetterValue::Delete => { + let removed = if has_annotations { + attrs.swap_remove(identifier!(vm, __annotations__)) + } else { + attrs.swap_remove(identifier!(vm, __annotations_cache__)) + }; + if removed.is_none() { + return Err(vm.new_attribute_error("__annotations__")); + } + prev.extend(removed); + if has_annotations { + prev.extend(attrs.swap_remove(identifier!(vm, __annotations_cache__))); + } } } - } - attrs.swap_remove(identifier!(vm, __annotate_func__)); - attrs.swap_remove(identifier!(vm, __annotate__)); + prev.extend(attrs.swap_remove(identifier!(vm, __annotate_func__))); + prev.extend(attrs.swap_remove(identifier!(vm, __annotate__))); + Ok(prev) + })?; Ok(()) } @@ -1724,9 +1927,13 @@ impl PyType { #[pygetset(setter)] fn set___module__(&self, value: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { self.check_set_special_type_attr(identifier!(vm, __module__), vm)?; - let mut attributes = self.attributes.write(); - attributes.swap_remove(identifier!(vm, __firstlineno__)); - attributes.insert(identifier!(vm, __module__), value); + let _prev_values = Self::with_type_lock(vm, || { + self.modified_inner(); + let mut attributes = self.attributes.write(); + let removed = attributes.swap_remove(identifier!(vm, __firstlineno__)); + let prev = attributes.insert(identifier!(vm, __module__), value); + (removed, prev) + }); Ok(()) } @@ -1848,24 +2055,26 @@ impl PyType { value: PySetterValue, vm: &VirtualMachine, ) -> PyResult<()> { + let key = identifier!(vm, __type_params__); match value { - PySetterValue::Assign(ref val) => { - let key = identifier!(vm, __type_params__); + PySetterValue::Assign(val) => { self.check_set_special_type_attr(key, vm)?; - self.modified(); - self.attributes.write().insert(key, val.clone().into()); + let _prev_value = Self::with_type_lock(vm, || { + self.modified_inner(); + self.attributes.write().insert(key, val.into()) + }); } PySetterValue::Delete => { - // For delete, we still need to check if the type is immutable if self.slots.flags.has_feature(PyTypeFlags::IMMUTABLETYPE) { return Err(vm.new_type_error(format!( "cannot delete '__type_params__' attribute of immutable type '{}'", self.slot_name() ))); } - let key = identifier!(vm, __type_params__); - self.modified(); - self.attributes.write().shift_remove(&key); + let _prev_value = Self::with_type_lock(vm, || { + self.modified_inner(); + self.attributes.write().shift_remove(&key) + }); } } Ok(()) @@ -2487,10 +2696,12 @@ impl Py { // Check if we can set this special type attribute self.check_set_special_type_attr(identifier!(vm, __doc__), vm)?; - // Set the __doc__ in the type's dict - self.attributes - .write() - .insert(identifier!(vm, __doc__), value); + let _prev_value = PyType::with_type_lock(vm, || { + self.modified_inner(); + self.attributes + .write() + .insert(identifier!(vm, __doc__), value) + }); Ok(()) } @@ -2552,31 +2763,40 @@ impl SetAttr for PyType { } let assign = value.is_assign(); - // Invalidate inline caches before modifying attributes. - // This ensures other threads see the version invalidation before - // any attribute changes, preventing use-after-free of cached descriptors. - zelf.modified(); - - if let PySetterValue::Assign(value) = value { - zelf.attributes.write().insert(attr_name, value); - } else { - let prev_value = zelf.attributes.write().shift_remove(attr_name); // TODO: swap_remove applicable? - if prev_value.is_none() { - return Err(vm.new_attribute_error(format!( - "type object '{}' has no attribute '{}'", - zelf.name(), - attr_name, - ))); - } - } - - if attr_name.as_wtf8().starts_with("__") && attr_name.as_wtf8().ends_with("__") { - if assign { - zelf.update_slot::(attr_name, &vm.ctx); + // Drop old value OUTSIDE the type lock to avoid deadlock: + // dropping may trigger weakref callbacks → method calls → + // LOAD_ATTR specialization → version_for_specialization → type lock. + let _prev_value = Self::with_type_lock(vm, || { + // Invalidate inline caches before modifying attributes. + // This ensures other threads see the version invalidation before + // any attribute changes, preventing use-after-free of cached descriptors. + zelf.modified_inner(); + + let prev_value = if let PySetterValue::Assign(value) = value { + zelf.attributes.write().insert(attr_name, value) } else { - zelf.update_slot::(attr_name, &vm.ctx); + let prev_value = zelf.attributes.write().shift_remove(attr_name); // TODO: swap_remove applicable? + if prev_value.is_none() { + return Err(vm.new_attribute_error(format!( + "type object '{}' has no attribute '{}'", + zelf.name(), + attr_name, + ))); + } + prev_value + }; + + // Keep the slot-table rewrite inside the same transaction as the + // dict mutation and version invalidation. + if attr_name.as_wtf8().starts_with("__") && attr_name.as_wtf8().ends_with("__") { + if assign { + zelf.update_slot::(attr_name, &vm.ctx); + } else { + zelf.update_slot::(attr_name, &vm.ctx); + } } - } + Ok(prev_value) + })?; Ok(()) } } @@ -2596,18 +2816,43 @@ impl Callable for PyType { } } - let obj = if let Some(slot_new) = zelf.slots.new.load() { - slot_new(zelf.to_owned(), args.clone(), vm)? - } else { + let Some(slot_new) = zelf.slots.new.load() else { return Err(vm.new_type_error(format!("cannot create '{}' instances", zelf.slots.name))); }; + // Both the new and init slots consume args, so the init call gets a + // separate copy prepared before slot_new runs. + let init_args = if args.is_empty() { + // Even cloning empty args costs a kwargs map clone; a default + // FuncArgs is indistinguishable from such a clone. + FuncArgs::default() + } else { + // Skip the clone when no init call can follow: the class has no + // init slot, is not `type` itself, and its new slot is a native + // function. new_wrapper is excluded because a Python `__new__` + // can install an `__init__` on the class or return an instance + // of another class while it runs. + // The address comparison is against the single new_wrapper fn item, + // so a mismatch is conservative: if it ever compared unequal for the + // wrapper it would only take the slower cloning path, never the fast + // path incorrectly. + if zelf.slots.init.load().is_none() + && !zelf.is(vm.ctx.types.type_type) + && slot_new as usize != crate::types::new_wrapper as crate::types::NewFunc as usize + { + return slot_new(zelf.to_owned(), args, vm); + } + args.clone() + }; + + let obj = slot_new(zelf.to_owned(), args, vm)?; + if !obj.class().fast_issubclass(zelf) { return Ok(obj); } if let Some(init_method) = obj.class().slots.init.load() { - init_method(obj.clone(), args, vm)?; + init_method(obj.clone(), init_args, vm)?; } Ok(obj) } @@ -2820,8 +3065,8 @@ pub(crate) fn call_slot_new( // that's not a heap type is this type. let mut staticbase = subtype.clone(); while staticbase.slots.flags.has_feature(PyTypeFlags::HEAPTYPE) { - if let Some(base) = staticbase.base.as_ref() { - staticbase = base.clone(); + if let Some(base) = staticbase.base.to_owned() { + staticbase = base; } else { break; } @@ -2953,7 +3198,7 @@ fn shape_differs(t1: &Py, t2: &Py) -> bool { } fn solid_base<'a>(typ: &'a Py, vm: &VirtualMachine) -> &'a Py { - let base = if let Some(base) = &typ.base { + let base = if let Some(base) = typ.base.deref() { solid_base(base, vm) } else { vm.ctx.types.object_type @@ -2997,6 +3242,90 @@ fn best_base<'a>(bases: &'a [PyTypeRef], vm: &VirtualMachine) -> PyResult<&'a Py Ok(base.unwrap()) } +fn type_has_dict(typ: &Py) -> bool { + typ.slots.flags.has_feature(PyTypeFlags::HAS_DICT) +} + +fn type_has_weakref(typ: &Py) -> bool { + typ.slots.flags.has_feature(PyTypeFlags::HAS_WEAKREF) +} + +/// Returns true if `child` adds no instance layout of its own beyond its base, +/// so the base can stand in for it when comparing object layouts. +fn compatible_with_base(child: &Py) -> bool { + let Some(parent) = child.base.deref() else { + return false; + }; + child.slots.basicsize == parent.slots.basicsize + && child.slots.itemsize == parent.slots.itemsize + && child.slots.member_count == parent.slots.member_count + && type_has_dict(child) == type_has_dict(parent) + && type_has_weakref(child) == type_has_weakref(parent) +} + +/// Walk up to the most derived base that actually fixes the instance layout. +fn layout_solid_base(mut typ: &Py) -> &Py { + while compatible_with_base(typ) { + typ = typ.base.deref().unwrap(); + } + typ +} + +/// Returns true if `a` and `b`, which share the same base, added the same +/// instance layout (`__dict__`, `__weakref__`, and `__slots__`). +fn same_slots_added(a: &Py, b: &Py) -> bool { + if a.slots.basicsize != b.slots.basicsize + || a.slots.itemsize != b.slots.itemsize + || a.slots.member_count != b.slots.member_count + || type_has_dict(a) != type_has_dict(b) + || type_has_weakref(a) != type_has_weakref(b) + { + return false; + } + match ( + a.heaptype_ext.as_ref().and_then(|e| e.slots.as_ref()), + b.heaptype_ext.as_ref().and_then(|e| e.slots.as_ref()), + ) { + (Some(x), Some(y)) => { + x.len() == y.len() + && x.iter() + .zip(y.iter()) + .all(|(p, q)| p.as_wtf8() == q.as_wtf8()) + } + (None, None) => true, + _ => false, + } +} + +/// Validates that instances of `old_to` and `new_to` share an interchangeable +/// object layout, the check `__class__` and `__bases__` assignment perform. +/// +/// `attr` names the attribute being assigned for the error message; the +/// message reports `new_to` first and `old_to` second. +pub(crate) fn compatible_for_assignment( + old_to: &Py, + new_to: &Py, + attr: &str, + vm: &VirtualMachine, +) -> PyResult<()> { + let newbase = layout_solid_base(new_to); + let oldbase = layout_solid_base(old_to); + let bases_equal = match (newbase.base.deref(), oldbase.base.deref()) { + (Some(x), Some(y)) => x.is(y), + (None, None) => true, + _ => false, + }; + let compatible = newbase.is(oldbase) || (bases_equal && same_slots_added(newbase, oldbase)); + if compatible { + return Ok(()); + } + Err(vm.new_type_error(format!( + "{attr} assignment: '{}' object layout differs from '{}'", + new_to.name(), + old_to.name() + ))) +} + /// Apply Python name mangling for private attributes. /// `__x` becomes `_ClassName__x` if inside a class. fn mangle_name(class_name: &str, name: &str) -> String { diff --git a/crates/vm/src/coroutine.rs b/crates/vm/src/coroutine.rs index 34d280acdca..844887f8520 100644 --- a/crates/vm/src/coroutine.rs +++ b/crates/vm/src/coroutine.rs @@ -73,6 +73,20 @@ impl Coro { } } + /// Free the finished frame's locals and stack, unless a frame object has + /// escaped (e.g. through an `f_locals` proxy or `sys._getframe`). An + /// escaped frame husk owns its heap-resident locals and must keep them + /// readable after the generator closes. + fn clear_frame_locals_on_close(&self) { + // Keep locals alive if a durable frame reference escaped (e.g. through + // an `f_locals` proxy or `sys._getframe`): that reference now owns the + // heap-resident locals and must keep them readable after close, + // matching `take_ownership`. + if !self.frame.has_escaped() { + self.frame.clear_locals_and_stack(); + } + } + fn maybe_close(&self, res: &PyResult, entered_frame: bool) { if !entered_frame { return; @@ -87,7 +101,7 @@ impl Coro { ); // Completed generators/coroutines should not keep their locals // alive while the wrapper object itself remains referenced. - self.frame.clear_locals_and_stack(); + self.clear_frame_locals_on_close(); } Ok(ExecutionResult::Yield(_)) => {} } @@ -169,10 +183,7 @@ impl Coro { } else { None }; - let (result, entered_frame) = self.run_with_context(jen, vm, |f| { - self.frame.locals_to_fast(vm)?; - f.resume(value, vm) - }); + let (result, entered_frame) = self.run_with_context(jen, vm, |f| f.resume(value, vm)); self.finalize_send_result(result, entered_frame, jen, vm) } @@ -198,10 +209,7 @@ impl Coro { } else { None }; - let (result, entered_frame) = self.run_with_context(jen, vm, |f| { - self.frame.locals_to_fast(vm)?; - f.resume(value, vm) - }); + let (result, entered_frame) = self.run_with_context(jen, vm, |f| f.resume(value, vm)); self.finalize_send_result(result, entered_frame, jen, vm) } @@ -259,7 +267,7 @@ impl Coro { self.closed.store(true); // Release frame locals and stack to free references held by the // closed generator, matching gen_send_ex2 with close_on_completion. - self.frame.clear_locals_and_stack(); + self.clear_frame_locals_on_close(); match result { Ok(ExecutionResult::Yield(_)) => { Err(vm.new_runtime_error(format!("{} ignored GeneratorExit", gen_name(jen, vm)))) diff --git a/crates/vm/src/exception_group.rs b/crates/vm/src/exception_group.rs index 198273a6914..a2c76378ab3 100644 --- a/crates/vm/src/exception_group.rs +++ b/crates/vm/src/exception_group.rs @@ -334,7 +334,7 @@ pub(super) mod types { let exceptions_tuple = vm.ctx.new_tuple(exceptions); let init_args = vec![message, exceptions_tuple.into()]; PyBaseException::new(init_args, vm) - .into_ref_with_type(vm, actual_cls) + .into_ref_with_type_lazy_dict(vm, actual_cls) .map(Into::into) } diff --git a/crates/vm/src/exceptions.rs b/crates/vm/src/exceptions.rs index 1ad86a35aed..845b01c3816 100644 --- a/crates/vm/src/exceptions.rs +++ b/crates/vm/src/exceptions.rs @@ -690,10 +690,8 @@ impl PyRef { #[pymethod] fn add_note(self, note: PyStrRef, vm: &VirtualMachine) -> PyResult<()> { - let dict = self - .as_object() - .dict() - .ok_or_else(|| vm.new_attribute_error("Exception object has no __dict__"))?; + let dict = crate::builtins::object::object_get_dict(self.as_object().to_owned(), vm) + .map_err(|_| vm.new_attribute_error("Exception object has no __dict__"))?; let notes = if let Ok(notes) = dict.get_item("__notes__", vm) { notes @@ -747,7 +745,7 @@ impl Constructor for PyBaseException { return Err(vm.new_type_error("BaseException() takes no keyword arguments")); } Self::new(args.args, vm) - .into_ref_with_type(vm, cls) + .into_ref_with_type_lazy_dict(vm, cls) .map(Into::into) } @@ -1364,7 +1362,7 @@ impl OSErrorBuilder { let payload = PyOSError::py_new(&exc_type, args.clone().into(), vm) .expect("new_os_error usage error"); let os_error = payload - .into_ref_with_type(vm, exc_type) + .into_ref_with_type_lazy_dict(vm, exc_type) .expect("new_os_error usage error"); PyOSError::slot_init(os_error.as_object().to_owned(), args.into(), vm) .expect("new_os_error usage error"); @@ -1585,7 +1583,7 @@ impl ToPyException for rustpython_host_env::multiprocessing::SemError { pub(super) mod types { use crate::common::lock::PyRwLock; - use crate::object::{MaybeTraverse, Traverse, TraverseFn}; + use crate::object::{Traverse, TraverseFn}; #[cfg_attr(target_arch = "wasm32", allow(unused_imports))] use crate::{ AsObject, Py, PyAtomicRef, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, @@ -1805,7 +1803,7 @@ pub(super) mod types { ))); } - let dict = zelf.dict().unwrap(); + let dict = crate::builtins::object::object_get_dict(zelf.clone(), vm)?; dict.set_item("name", vm.unwrap_or_none(name), vm)?; dict.set_item("path", vm.unwrap_or_none(path), vm)?; dict.set_item("name_from", vm.unwrap_or_none(name_from), vm)?; @@ -1902,7 +1900,7 @@ pub(super) mod types { #[repr(transparent)] pub struct PyUnboundLocalError(PyNameError); - #[pyexception(name, base = PyException, ctx = "os_error")] + #[pyexception(name, base = PyException, ctx = "os_error", traverse = "manual")] #[repr(C)] pub struct PyOSError { base: PyException, @@ -1932,7 +1930,10 @@ pub(super) mod types { unsafe impl Traverse for PyOSError { fn traverse(&self, tracer_fn: &mut TraverseFn<'_>) { - self.base.try_traverse(tracer_fn); + // `self.base` is a `PyException` newtype whose `MaybeTraverse` is a + // no-op; reach the underlying `PyBaseException` so its traceback, + // cause, context and args are visited by the collector. + self.base.0.traverse(tracer_fn); if let Some(obj) = self.errno.deref() { tracer_fn(obj); } @@ -2013,7 +2014,9 @@ pub(super) mod types { } } let payload = Self::py_new(&cls, args, vm)?; - payload.into_ref_with_type(vm, cls).map(Into::into) + payload + .into_ref_with_type_lazy_dict(vm, cls) + .map(Into::into) } } diff --git a/crates/vm/src/frame.rs b/crates/vm/src/frame.rs index 85b15aaac49..40499efb110 100644 --- a/crates/vm/src/frame.rs +++ b/crates/vm/src/frame.rs @@ -12,10 +12,7 @@ use crate::{ builtin_func::PyNativeFunction, descriptor::{MemberGetter, PyMemberDescriptor, PyMethodDescriptor}, frame::stack_analysis, - function::{ - PyBoundMethod, PyCell, PyCellRef, PyFunction, datastack_frame_size_bytes_for_code, - vectorcall_function, - }, + function::{PyBoundMethod, PyCell, PyCellRef, PyFunction, vectorcall_function}, list::PyListIterator, range::PyRangeIterator, tuple::{PyTuple, PyTupleIterator, PyTupleRef}, @@ -39,6 +36,7 @@ use crate::{ use alloc::fmt; use bstr::ByteSlice; use core::cell::UnsafeCell; +use core::ptr::NonNull; use core::sync::atomic; use core::sync::atomic::AtomicPtr; use core::sync::atomic::Ordering::{Acquire, Relaxed}; @@ -54,6 +52,71 @@ use rustpython_compiler_core::SourceLocation; pub type FrameRef = PyRef; +/// Recover an owned reference to a live chain frame, or `None` for null. +/// +/// # Safety +/// A non-null `frame` must reference a frame that is live on the current +/// thread's execution chain, so the object outlives this call. +unsafe fn owned_chain_frame(frame: *const Frame) -> Option { + if frame.is_null() { + return None; + } + // SAFETY: caller guarantees the frame is live; from_payload_ptr recovers + // the enclosing object from the payload address. + let py = unsafe { &*Py::::from_payload_ptr(frame) }; + Some(py.to_owned()) +} + +/// The current thread's topmost frame object, if any. +#[must_use] +pub fn current_thread_frame() -> Option { + // SAFETY: the chain top executes on this thread, hence is alive. + unsafe { owned_chain_frame(crate::vm::thread::get_current_frame()) } +} + +/// The frame `offset` positions below the current thread's top frame (offset 0 +/// is the top), or `None` if the stack is not that deep. +#[must_use] +pub fn frame_at_offset(offset: usize) -> Option { + let mut cur = crate::vm::thread::get_current_frame(); + for _ in 0..offset { + if cur.is_null() { + return None; + } + // SAFETY: chain frames are alive on the current thread's stack. + cur = unsafe { (*cur).previous_frame() }; + } + // SAFETY: same as above. + unsafe { owned_chain_frame(cur) } +} + +/// If `target` is a frame on the current thread's chain, return an owned +/// reference to it; otherwise `None`. Presence on the chain proves liveness. +#[must_use] +pub fn find_owned_chain_frame(target: *const Frame) -> Option { + let mut cur = crate::vm::thread::get_current_frame(); + while !cur.is_null() { + if core::ptr::eq(cur, target) { + // SAFETY: a frame on the current thread's chain is alive. + return unsafe { owned_chain_frame(cur) }; + } + // SAFETY: chain frames are alive on the current thread's stack. + cur = unsafe { (*cur).previous_frame() }; + } + None +} + +/// Invoke `f` for each frame on the current thread's chain, from the topmost +/// frame down to the bottom. +pub fn for_each_current_frame(mut f: impl FnMut(&Py)) { + let mut cur = crate::vm::thread::get_current_frame(); + while !cur.is_null() { + // SAFETY: chain frames are alive on the current thread's stack. + f(unsafe { &*Py::::from_payload_ptr(cur) }); + cur = unsafe { (*cur).previous_frame() }; + } +} + /// The reason why we might be unwinding a block. /// This could be return of function, exception being /// raised, a break or continue being hit, etc.. @@ -111,6 +174,12 @@ impl FrameUnsafeCell { unsafe fn get(&self) -> *mut T { self.0.get() } + + /// Safe exclusive access through `&mut self`. + #[inline(always)] + fn get_mut(&mut self) -> &mut T { + self.0.get_mut() + } } // SAFETY: Frame execution is single-threaded. See FrameUnsafeCell doc. @@ -175,8 +244,9 @@ impl LocalsPlus { /// Create a new LocalsPlus backed by the thread data stack. /// All slots are zero-initialized. /// - /// The caller must call `materialize_localsplus()` when the frame finishes - /// to migrate data to the heap, then `datastack_pop()` to free the memory. + /// When the frame finishes, the caller must migrate data to the heap with + /// `materialize_localsplus()` (or drop it in place with + /// `release_localsplus()`), then `datastack_pop()` to free the memory. fn new_on_datastack(nlocalsplus: usize, stacksize: usize, vm: &VirtualMachine) -> Self { let capacity = nlocalsplus .checked_add(stacksize) @@ -211,6 +281,29 @@ impl LocalsPlus { } } + /// Drop all contained values and detach the data stack backing without + /// copying to the heap, leaving an empty heap-backed husk. + /// Returns the data stack base pointer for `DataStack::pop()`. + /// Returns `None` if already heap-backed. + /// + /// Only valid when the values can never be observed again (the enclosing + /// frame is uniquely referenced): the locals are gone afterwards. + fn release_datastack(&mut self) -> Option<*mut u8> { + let LocalsPlusData::DataStack { ptr, .. } = &self.data else { + return None; + }; + let base = *ptr as *mut u8; + // Drop values while the backing store is still valid. Value drops may + // run `__del__`, which can push nested data stack frames above `base`; + // those are popped before the caller pops `base` (LIFO preserved). + self.drop_values(); + self.data = LocalsPlusData::Heap(Box::default()); + // Keep the accessors consistent with the empty backing store. + // stack_top is already 0 after drop_values(). + self.nlocalsplus = 0; + Some(base) + } + /// Drop all contained values without freeing the backing storage. fn drop_values(&mut self) { self.stack_clear(); @@ -251,6 +344,12 @@ impl LocalsPlus { } } + /// Whether the backing storage still lives on the thread data stack (a + /// running call frame that has not been materialized onto the heap). + fn is_datastack_backed(&self) -> bool { + matches!(self.data, LocalsPlusData::DataStack { .. }) + } + /// Stack capacity (max stack depth). #[inline(always)] fn stack_capacity(&self) -> usize { @@ -605,11 +704,20 @@ pub struct InterpreterFrame { /// Used by `frame.clear()` to reject clearing an executing frame, /// even when called from a different thread. pub(crate) owner: atomic::AtomicI8, - /// Set when f_locals is accessed. Cleared after locals_to_fast() sync. - pub(crate) locals_dirty: atomic::AtomicBool, /// Persistent overlay for `frame.f_locals` when hidden locals need a /// snapshot separate from the backing locals mapping. pub(crate) f_locals_hidden_overlay: PyMutex>, + /// Side storage for `f_locals` proxy keys that do not name a fast local. + /// Lazily created on first non-fast-key write. Mirrors `f_extra_locals`. + pub(crate) f_extra_locals: PyMutex>, + /// Set once a durable Python-level reference to this frame is handed out + /// (`f_locals` proxy, `sys._getframe`, `f_back`). A closed generator keeps + /// its locals alive while this is set, mirroring `frame_obj` ownership. + pub(crate) escaped: atomic::AtomicBool, + /// Strong reference to the caller frame, captured when this frame escapes + /// its execution so `f_back` still resolves after the caller returns and + /// leaves the live frame chain. + pub(crate) retained_back: PyMutex>, /// Number of stack entries to pop after set_f_lineno returns to the /// execution loop. set_f_lineno cannot pop directly because the /// execution loop holds the state mutex. @@ -624,7 +732,55 @@ pub struct InterpreterFrame { /// Analogous to CPython's `PyFrameObject`. #[pyclass(module = false, name = "frame", traverse = "manual")] pub struct Frame { - pub(crate) iframe: FrameUnsafeCell, + /// Always `Some` while the frame is reachable from Python. Emptied only + /// by `Traverse::clear` during deallocation, leaving a trivially-droppable + /// husk that the freelist can cache. + pub(crate) iframe: FrameUnsafeCell>, +} + +impl Frame { + /// Shared access to the embedded interpreter frame. + /// + /// # Safety + /// Caller must ensure no concurrent mutable access (see `FrameUnsafeCell`) + /// and that the frame has not been cleared (i.e. it is still reachable + /// from Python; `Traverse::clear` only runs during deallocation). + #[inline(always)] + unsafe fn iframe_ref(&self) -> &InterpreterFrame { + let opt = unsafe { &*self.iframe.get() }; + #[cfg(debug_assertions)] + if opt.is_none() { + cleared_frame_access(); + } + // SAFETY: iframe is always Some while the frame is reachable (see above). + unsafe { opt.as_ref().unwrap_unchecked() } + } + + /// Exclusive access to the embedded interpreter frame. + /// + /// # Safety + /// Caller must ensure exclusive access (see `FrameUnsafeCell`) and that + /// the frame has not been cleared. + #[inline(always)] + #[allow(clippy::mut_from_ref)] + unsafe fn iframe_mut(&self) -> &mut InterpreterFrame { + let opt = unsafe { &mut *self.iframe.get() }; + #[cfg(debug_assertions)] + if opt.is_none() { + cleared_frame_access(); + } + // SAFETY: iframe is always Some while the frame is reachable (see above). + unsafe { opt.as_mut().unwrap_unchecked() } + } +} + +/// Out-of-line panic for the debug-only cleared-frame check, keeping the +/// inlined accessors' stack frames minimal. +#[cfg(debug_assertions)] +#[cold] +#[inline(never)] +fn cleared_frame_access() -> ! { + panic!("frame accessed after clear"); } impl core::ops::Deref for Frame { @@ -638,21 +794,88 @@ impl core::ops::Deref for Frame { /// are only mutated during single-threaded execution via `with_exec`. #[inline(always)] fn deref(&self) -> &InterpreterFrame { - unsafe { &*self.iframe.get() } + unsafe { self.iframe_ref() } } } +thread_local! { + /// Free list of dead frame objects for reuse. Entries are cleared husks + /// (`iframe == None`) whose child references were already released. + /// PyInner is fixed-size (localsplus storage is out-of-line), + /// so a single bucket suffices. + static FRAME_FREELIST: core::cell::Cell> = + const { core::cell::Cell::new(crate::object::FreeList::new()) }; +} + impl PyPayload for Frame { + const MAX_FREELIST: usize = 200; + const HAS_FREELIST: bool = true; + // Ordinary call frames are created untracked and only enter the GC when + // they escape (see `release_datastack_frame`); generator/coroutine frames + // are tracked explicitly at creation in `invoke_with_locals`. + const NEW_REF_UNTRACKED: bool = true; + #[inline] fn class(ctx: &Context) -> &'static Py { ctx.types.frame_type } + + #[inline] + unsafe fn freelist_push(obj: *mut PyObject) -> bool { + FRAME_FREELIST + .try_with(|fl| { + let mut list = fl.take(); + let stored = if list.len() < Self::MAX_FREELIST { + list.push(obj); + true + } else { + false + }; + fl.set(list); + stored + }) + .unwrap_or(false) + } + + #[inline] + unsafe fn freelist_pop(_payload: &Self) -> Option> { + FRAME_FREELIST + .try_with(|fl| { + let mut list = fl.take(); + let result = list.pop().map(|p| unsafe { NonNull::new_unchecked(p) }); + fl.set(list); + result + }) + .ok() + .flatten() + } } unsafe impl Traverse for Frame { fn traverse(&self, tracer_fn: &mut TraverseFn<'_>) { - // SAFETY: GC traversal does not run concurrently with frame execution. - let iframe = unsafe { &*self.iframe.get() }; + // SAFETY: this traversal reads the frame's live interpreter state + // (`localsplus`), which the owning thread mutates without + // synchronization while executing bytecode. It is only sound when no + // other thread is executing this frame: in threading builds the + // collector stops the world around the traversal phases, and in + // single-threaded builds there is no other thread. A cleared frame + // (iframe == None) has no children to visit. + // + // Invariant (load-bearing for the untracked-frame optimization): every + // reference *to* a frame is recorded as a graph edge by + // `PyRef::traverse` — no other type's `traverse` recurses into a + // frame's payload. So the collector reads a frame's `localsplus` only + // when the frame is itself a tracked candidate. Tracked datastack + // frames are tracked only at `release_datastack_frame`, after they stop + // executing and their localsplus is materialized onto the heap; tracked + // generator frames are heap-backed by construction and their execution + // is stopped-the-world. Hence a running, data-stack-resident frame is + // never traversed by a concurrent collector. If a future change makes + // some type's `traverse` recurse into a frame payload, this invariant + // (and the debug asserts at the track sites) breaks. + let Some(iframe) = (unsafe { &*self.iframe.get() }) else { + return; + }; iframe.code.traverse(tracer_fn); iframe.func_obj.traverse(tracer_fn); iframe.localsplus.traverse(tracer_fn); @@ -662,6 +885,19 @@ unsafe impl Traverse for Frame { iframe.trace.traverse(tracer_fn); iframe.temporary_refs.traverse(tracer_fn); iframe.f_locals_hidden_overlay.traverse(tracer_fn); + iframe.f_extra_locals.traverse(tracer_fn); + iframe.retained_back.traverse(tracer_fn); + } + + fn clear(&mut self, _out: &mut Vec) { + // Drop the interpreter frame in place instead of extracting children + // into `_out`: pushing ~10 refs per frame would grow the buffer, a + // heap allocation on the hot dealloc path. Direct drops release the + // same references under the same recursion protection (trashcan in + // dealloc, deferred-drop context in cycle collection) as the payload + // drop did before the freelist existed. The payload is left as a + // trivially-droppable husk for the freelist. + drop(self.iframe.get_mut().take()); } } @@ -741,13 +977,15 @@ impl Frame { generator: PyAtomicBorrow::new(), previous: AtomicPtr::new(core::ptr::null_mut()), owner: atomic::AtomicI8::new(FrameOwner::FrameObject as i8), - locals_dirty: atomic::AtomicBool::new(false), f_locals_hidden_overlay: PyMutex::new(None), + f_extra_locals: PyMutex::new(None), + escaped: atomic::AtomicBool::new(false), + retained_back: PyMutex::new(None), pending_stack_pops: Default::default(), pending_unwind_from_stack: Default::default(), }; Self { - iframe: FrameUnsafeCell::new(iframe), + iframe: FrameUnsafeCell::new(Some(iframe)), } } @@ -758,7 +996,7 @@ impl Frame { /// or called from the same thread during trace callback). #[inline(always)] pub unsafe fn fastlocals(&self) -> &[Option] { - unsafe { (*self.iframe.get()).localsplus.fastlocals() } + unsafe { self.iframe_ref().localsplus.fastlocals() } } /// Access fastlocals mutably. @@ -768,7 +1006,7 @@ impl Frame { #[inline(always)] #[allow(clippy::mut_from_ref)] pub unsafe fn fastlocals_mut(&self) -> &mut [Option] { - unsafe { (*self.iframe.get()).localsplus.fastlocals_mut() } + unsafe { self.iframe_mut().localsplus.fastlocals_mut() } } /// Migrate data-stack-backed storage to the heap, preserving all values, @@ -779,7 +1017,29 @@ impl Frame { /// Caller must ensure the frame is not executing and the returned /// pointer is passed to `VirtualMachine::datastack_pop()`. pub(crate) unsafe fn materialize_localsplus(&self) -> Option<*mut u8> { - unsafe { (*self.iframe.get()).localsplus.materialize_to_heap() } + unsafe { self.iframe_mut().localsplus.materialize_to_heap() } + } + + /// Drop all localsplus values in place and detach the data stack backing + /// without the heap copy. Returns the data stack base pointer for + /// `VirtualMachine::datastack_pop()`, or `None` if heap-backed. + /// + /// # Safety + /// Caller must ensure the frame is not executing, that no other reference + /// to the frame exists or can be created (localsplus is unobservable + /// afterwards), and that the returned pointer is passed to + /// `VirtualMachine::datastack_pop()`. + pub(crate) unsafe fn release_localsplus(&self) -> Option<*mut u8> { + unsafe { self.iframe_mut().localsplus.release_datastack() } + } + + /// Whether this frame's localsplus is still data-stack-backed. A frame + /// must have heap-backed localsplus before it is GC-tracked so that a + /// concurrent collector never reads data-stack-resident, still-mutating + /// storage. Used only in debug assertions at the track sites. + pub(crate) fn localsplus_is_datastack_backed(&self) -> bool { + // SAFETY: called at a track site where the frame is not executing. + unsafe { self.iframe_ref().localsplus.is_datastack_backed() } } /// Clear evaluation stack and state-owned cell/free references. @@ -788,7 +1048,7 @@ impl Frame { // SAFETY: Called when frame is not executing (generator closed). // Cell refs in fastlocals[nlocals..] are cleared by clear_locals_and_stack(). unsafe { - (*self.iframe.get()).localsplus.stack_clear(); + self.iframe_mut().localsplus.stack_clear(); } } @@ -797,17 +1057,18 @@ impl Frame { pub(crate) fn clear_locals_and_stack(&self) { self.clear_stack_and_cells(); // SAFETY: Frame is not executing (generator closed). - let fastlocals = unsafe { (*self.iframe.get()).localsplus.fastlocals_mut() }; + let fastlocals = unsafe { self.iframe_mut().localsplus.fastlocals_mut() }; for slot in fastlocals.iter_mut() { *slot = None; } self.f_locals_hidden_overlay.lock().take(); + self.f_extra_locals.lock().take(); } /// Get cell contents by localsplus index. pub(crate) fn get_cell_contents(&self, localsplus_idx: usize) -> Option { // SAFETY: Frame not executing; no concurrent mutation. - let fastlocals = unsafe { (*self.iframe.get()).localsplus.fastlocals() }; + let fastlocals = unsafe { self.iframe_ref().localsplus.fastlocals() }; fastlocals .get(localsplus_idx) .and_then(|slot| slot.as_ref()) @@ -825,8 +1086,17 @@ impl Frame { /// Clear the generator back-reference. Called when the generator is finalized. pub fn clear_generator(&self) { - self.generator.clear(); - self.owner + // The generator's drop may run after this frame was already cleared + // by cycle collection (both were garbage and the frame was cleared + // first); nothing to unlink then. + // SAFETY: shared access; the finalizing generator owns the frame, + // which is not executing. + let Some(iframe) = (unsafe { &*self.iframe.get() }) else { + return; + }; + iframe.generator.clear(); + iframe + .owner .store(FrameOwner::FrameObject as i8, atomic::Ordering::Release); } @@ -839,6 +1109,16 @@ impl Frame { self.previous.load(atomic::Ordering::Relaxed) } + /// Record that a durable Python-level reference to this frame escaped. + pub(crate) fn mark_escaped(&self) { + self.escaped.store(true, atomic::Ordering::Release); + } + + /// Whether a durable reference to this frame has escaped. + pub(crate) fn has_escaped(&self) -> bool { + self.escaped.load(atomic::Ordering::Acquire) + } + pub fn lasti(&self) -> u32 { self.lasti.load(Relaxed) } @@ -863,41 +1143,10 @@ impl Frame { self.pending_unwind_from_stack.store(val, Relaxed); } - /// Sync locals dict back to fastlocals. Called before generator/coroutine resume - /// to apply any modifications made via f_locals. - pub fn locals_to_fast(&self, vm: &VirtualMachine) -> PyResult<()> { - if !self.locals_dirty.load(atomic::Ordering::Acquire) { - return Ok(()); - } - let code = &**self.code; - let overlay_locals = self - .has_active_hidden_locals() - .then(|| self.f_locals_hidden_overlay.lock().clone()) - .flatten() - .map(ArgMapping::from_dict_exact); - let locals_map = overlay_locals - .as_ref() - .map_or_else(|| self.locals.mapping(vm), ArgMapping::mapping); - // SAFETY: Called before generator resume; no concurrent access. - let fastlocals = unsafe { (*self.iframe.get()).localsplus.fastlocals_mut() }; - for (i, &varname) in code.varnames.iter().enumerate() { - if i >= fastlocals.len() { - break; - } - match locals_map.subscript(varname, vm) { - Ok(value) => fastlocals[i] = Some(value), - Err(e) if e.fast_isinstance(vm.ctx.exceptions.key_error) => {} - Err(e) => return Err(e), - } - } - self.locals_dirty.store(false, atomic::Ordering::Release); - Ok(()) - } - fn has_active_hidden_locals(&self) -> bool { use rustpython_compiler_core::bytecode::{CO_FAST_CELL, CO_FAST_FREE, CO_FAST_HIDDEN}; let code = &**self.code; - let fastlocals = unsafe { (*self.iframe.get()).localsplus.fastlocals() }; + let fastlocals = unsafe { self.iframe_ref().localsplus.fastlocals() }; let is_optimized = code.flags.contains(bytecode::CodeFlags::OPTIMIZED); !is_optimized && code.localspluskinds.iter().enumerate().any(|(i, &kind)| { @@ -929,7 +1178,7 @@ impl Frame { // SAFETY: Either the frame is not executing (caller checked owner), // or we're in a trace callback on the same thread that's executing. let code = &**self.code; - let fastlocals = unsafe { (*self.iframe.get()).localsplus.fastlocals() }; + let fastlocals = unsafe { self.iframe_ref().localsplus.fastlocals() }; // Iterate through all localsplus slots using localspluskinds let nlocalsplus = code.localspluskinds.len(); @@ -1023,13 +1272,38 @@ impl Frame { Ok(()) } + /// Reject locals access for a frame that is executing on another thread. + /// + /// A thread-owned frame mutates `localsplus` without synchronization, so + /// reading fastlocals from a different thread would be a data race (the + /// executing thread overwrites slots and drops the old values while the + /// reader clones them). Access from the executing thread itself (locals() + /// builtin, trace callbacks) is fine: the frame sits on the current + /// thread's frame chain and is at a bytecode boundary. + pub(crate) fn check_locals_access(&self, vm: &VirtualMachine) -> PyResult<()> { + let owner = FrameOwner::from_i8(self.owner.load(atomic::Ordering::Acquire)); + if owner != FrameOwner::Thread { + return Ok(()); + } + let mut cur = crate::vm::thread::get_current_frame(); + while !cur.is_null() { + if core::ptr::eq(cur, self) { + return Ok(()); + } + cur = unsafe { (*cur).previous_frame() }; + } + Err(vm.new_runtime_error( + "cannot access frame locals while the frame is executing in another thread", + )) + } + pub fn f_locals_mapping(&self, vm: &VirtualMachine) -> PyResult { + self.check_locals_access(vm)?; if !self.has_active_hidden_locals() { self.f_locals_hidden_overlay.lock().take(); return self.locals(vm); } - let needs_refresh = !self.locals_dirty.load(atomic::Ordering::Acquire); let overlay_dict = { let mut overlay = self.f_locals_hidden_overlay.lock(); match overlay.as_ref() { @@ -1041,25 +1315,240 @@ impl Frame { } } }; - if needs_refresh { - PyDict::clear(&overlay_dict); - let overlay = ArgMapping::from_dict_exact(overlay_dict.clone()); - self.sync_visible_locals_to_mapping(overlay.mapping(), vm)?; - } + PyDict::clear(&overlay_dict); + let overlay = ArgMapping::from_dict_exact(overlay_dict.clone()); + self.sync_visible_locals_to_mapping(overlay.mapping(), vm)?; Ok(ArgMapping::from_dict_exact(overlay_dict)) } pub fn locals(&self, vm: &VirtualMachine) -> PyResult { - if self.has_active_hidden_locals() { + let mapping = if self.has_active_hidden_locals() { // Match CPython's locals() behavior for frames with PEP 709 hidden // locals: return a fresh snapshot instead of the backing mapping. let overlay = ArgMapping::from_dict_exact(vm.ctx.new_dict()); self.sync_visible_locals_to_mapping(overlay.mapping(), vm)?; - Ok(overlay) + overlay } else { self.sync_visible_locals_to_mapping(self.locals.mapping(vm), vm)?; - Ok(self.locals.clone_mapping(vm)) + self.locals.clone_mapping(vm) + }; + self.fold_extra_locals(&mapping, vm)?; + Ok(mapping) + } + + /// Copy the frame's extra-locals side storage (proxy keys that are not + /// fast locals) into `mapping`. No-op when nothing was ever stored. + fn fold_extra_locals(&self, mapping: &ArgMapping, vm: &VirtualMachine) -> PyResult<()> { + let extra = self.f_extra_locals.lock().clone(); + if let Some(extra) = extra { + for (key, value) in &extra { + mapping.mapping().ass_subscript(&key, Some(value), vm)?; + } } + Ok(()) + } + + /// Read a fast-local slot's visible value, dereferencing cells. `None` if + /// the slot is empty or its cell holds no value. + fn framelocalsproxy_getval(&self, i: usize) -> Option { + use rustpython_compiler_core::bytecode::{CO_FAST_CELL, CO_FAST_FREE}; + // SAFETY: callers first pass through `check_locals_access`, so the + // frame is not executing on another thread. + let fastlocals = unsafe { self.iframe_ref().localsplus.fastlocals() }; + let obj = fastlocals.get(i)?.as_ref()?; + let kind = self.code.localspluskinds.get(i).copied().unwrap_or(0); + if kind & (CO_FAST_CELL | CO_FAST_FREE) != 0 { + if let Some(cell) = obj.downcast_ref::() { + cell.get() + } else { + Some(obj.clone()) + } + } else { + Some(obj.clone()) + } + } + + /// Write `value` into fast-local slot `i`, routing through the cell when + /// the slot holds one so closures keep sharing the same cell. + fn framelocalsproxy_setval(&self, i: usize, value: PyObjectRef) { + use rustpython_compiler_core::bytecode::{CO_FAST_CELL, CO_FAST_FREE}; + let kind = self.code.localspluskinds.get(i).copied().unwrap_or(0); + // SAFETY: callers first pass through `check_locals_access`. + let fastlocals = unsafe { self.iframe_mut().localsplus.fastlocals_mut() }; + if kind & (CO_FAST_CELL | CO_FAST_FREE) != 0 + && let Some(obj) = fastlocals[i].as_ref() + && let Some(cell) = obj.downcast_ref::() + { + cell.set(Some(value)); + return; + } + fastlocals[i] = Some(value); + } + + /// Resolve `key` to a fast-local slot index, or `None` if it names no fast + /// local. `read` selects read semantics (only bound slots match) versus + /// write semantics (hidden slots are skipped, unbound slots still match). + /// Raises `TypeError` for an unhashable key. + fn framelocalsproxy_getkeyindex( + &self, + key: &PyObject, + read: bool, + vm: &VirtualMachine, + ) -> PyResult> { + use rustpython_compiler_core::bytecode::CO_FAST_HIDDEN; + // The proxy hashes the key first; an unhashable key raises TypeError. + key.hash(vm)?; + for (i, &kind) in self.code.localspluskinds.iter().enumerate() { + let name = localsplus_name(&self.code, i); + if !name + .as_object() + .rich_compare_bool(key, PyComparisonOp::Eq, vm)? + { + continue; + } + if read { + if self.framelocalsproxy_getval(i).is_some() { + return Ok(Some(i)); + } + } else if kind & CO_FAST_HIDDEN == 0 { + return Ok(Some(i)); + } + } + Ok(None) + } + + /// Build a fresh ordered snapshot dict of the proxy's visible contents: + /// bound fast locals in localsplus order followed by extra locals. + pub(crate) fn framelocalsproxy_snapshot(&self, vm: &VirtualMachine) -> PyResult { + self.check_locals_access(vm)?; + let dict = vm.ctx.new_dict(); + let mapping = ArgMapping::from_dict_exact(dict.clone()); + self.sync_visible_locals_to_mapping(mapping.mapping(), vm)?; + self.fold_extra_locals(&mapping, vm)?; + Ok(dict) + } + + /// `proxy[key]`: read a fast local live, else fall back to extra locals. + pub(crate) fn framelocalsproxy_getitem( + &self, + key: PyObjectRef, + vm: &VirtualMachine, + ) -> PyResult { + self.check_locals_access(vm)?; + if let Some(i) = self.framelocalsproxy_getkeyindex(&key, true, vm)? + && let Some(value) = self.framelocalsproxy_getval(i) + { + return Ok(value); + } + let extra = self.f_extra_locals.lock().clone(); + if let Some(extra) = extra + && let Some(value) = extra.get_item_opt(&*key, vm)? + { + return Ok(value); + } + Err(vm.new_key_error(key)) + } + + /// `key in proxy`. + pub(crate) fn framelocalsproxy_contains( + &self, + key: PyObjectRef, + vm: &VirtualMachine, + ) -> PyResult { + self.check_locals_access(vm)?; + if self.framelocalsproxy_getkeyindex(&key, true, vm)?.is_some() { + return Ok(true); + } + let extra = self.f_extra_locals.lock().clone(); + if let Some(extra) = extra { + return Ok(extra.get_item_opt(&*key, vm)?.is_some()); + } + Ok(false) + } + + /// `proxy[key] = value`: fast-key writes the slot in place, other keys go + /// to the extra-locals side dict. + pub(crate) fn framelocalsproxy_setitem( + &self, + key: PyObjectRef, + value: PyObjectRef, + vm: &VirtualMachine, + ) -> PyResult<()> { + self.check_locals_access(vm)?; + if let Some(i) = self.framelocalsproxy_getkeyindex(&key, false, vm)? { + self.framelocalsproxy_setval(i, value); + return Ok(()); + } + let extra = self.extra_locals_get_or_create(vm); + extra.set_item(&*key, value, vm) + } + + /// `del proxy[key]`: deleting a fast local raises ValueError; extra keys are + /// removed (KeyError if absent). + pub(crate) fn framelocalsproxy_delitem( + &self, + key: PyObjectRef, + vm: &VirtualMachine, + ) -> PyResult<()> { + self.check_locals_access(vm)?; + if self + .framelocalsproxy_getkeyindex(&key, false, vm)? + .is_some() + { + return Err(vm.new_value_error("cannot remove local variables from FrameLocalsProxy")); + } + let extra = self.f_extra_locals.lock().clone(); + if let Some(extra) = extra + && extra.get_item_opt(&*key, vm)?.is_some() + { + return extra.del_item(&*key, vm); + } + Err(vm.new_key_error(key)) + } + + /// `proxy.pop(key[, default])`. + pub(crate) fn framelocalsproxy_pop( + &self, + key: PyObjectRef, + default: Option, + vm: &VirtualMachine, + ) -> PyResult { + self.check_locals_access(vm)?; + if self + .framelocalsproxy_getkeyindex(&key, false, vm)? + .is_some() + { + return Err(vm.new_value_error("cannot remove local variables from FrameLocalsProxy")); + } + let extra = self.f_extra_locals.lock().clone(); + if let Some(extra) = extra + && let Some(value) = extra.pop_item(&*key, vm)? + { + return Ok(value); + } + default.ok_or_else(|| vm.new_key_error(key)) + } + + /// `proxy.setdefault(key, default)`. + pub(crate) fn framelocalsproxy_setdefault( + &self, + key: PyObjectRef, + default: PyObjectRef, + vm: &VirtualMachine, + ) -> PyResult { + match self.framelocalsproxy_getitem(key.clone(), vm) { + Ok(value) => Ok(value), + Err(e) if e.fast_isinstance(vm.ctx.exceptions.key_error) => { + self.framelocalsproxy_setitem(key, default.clone(), vm)?; + Ok(default) + } + Err(e) => Err(e), + } + } + + fn extra_locals_get_or_create(&self, vm: &VirtualMachine) -> PyDictRef { + let mut extra = self.f_extra_locals.lock(); + extra.get_or_insert_with(|| vm.ctx.new_dict()).clone() } } @@ -1069,7 +1558,7 @@ impl Py { // SAFETY: Frame execution is single-threaded. Only one thread at a time // executes a given frame (enforced by the owner field and generator // running flag). Same safety argument as FastLocals (UnsafeCell). - let iframe = unsafe { &mut *self.iframe.get() }; + let iframe = unsafe { self.iframe_mut() }; let exec = ExecutingFrame { code: &iframe.code, localsplus: &mut iframe.localsplus, @@ -1129,7 +1618,7 @@ impl Py { return None; } // SAFETY: Frame is not executing, so UnsafeCell access is safe. - let iframe = unsafe { &mut *self.iframe.get() }; + let iframe = unsafe { self.iframe_mut() }; let exec = ExecutingFrame { code: &iframe.code, localsplus: &mut iframe.localsplus, @@ -1220,12 +1709,117 @@ fn specialization_nonnegative_compact_index(i: &PyInt, vm: &VirtualMachine) -> O } } -fn release_datastack_frame(frame: &Py, vm: &VirtualMachine) { +/// Get the variable name for a localsplus index of `code`. +fn localsplus_name(code: &PyCode, idx: usize) -> &'static PyStrInterned { + use rustpython_compiler_core::bytecode::{CO_FAST_CELL, CO_FAST_FREE, CO_FAST_LOCAL}; + let nlocals = code.varnames.len(); + let kind = code.localspluskinds.get(idx).copied().unwrap_or(0); + if kind & CO_FAST_LOCAL != 0 { + // Merged cell or regular local: name is in varnames + code.varnames[idx] + } else if kind & CO_FAST_FREE != 0 { + // Free var: slots are at the end of localsplus + let nlocalsplus = code.localspluskinds.len(); + let nfrees = code.freevars.len(); + let free_start = nlocalsplus - nfrees; + code.freevars[idx - free_start] + } else if kind & CO_FAST_CELL != 0 { + // Non-merged cell: count how many non-merged cell slots are before + // this index to find the corresponding cellvars entry. + // Non-merged cellvars appear in their original order (skipping merged ones). + let nonmerged_pos = code.localspluskinds[nlocals..idx] + .iter() + .filter(|&&k| k == CO_FAST_CELL) + .count(); + // Skip merged cellvars to find the right one + let mut cv_idx = 0; + let mut nonmerged_count = 0; + for (i, name) in code.cellvars.iter().enumerate() { + let is_merged = code.varnames.contains(name); + if !is_merged { + if nonmerged_count == nonmerged_pos { + cv_idx = i; + break; + } + nonmerged_count += 1; + } + } + code.cellvars[cv_idx] + } else { + code.varnames[idx] + } +} + +/// Free a finished call frame's data stack storage. +/// +/// When the caller holds the only reference to the frame, the locals and +/// stack values are dropped in place and the storage is released without a +/// heap copy. Otherwise (the frame escaped through a traceback, +/// `sys._getframe`, a trace callback, ...) the values are copied to the heap +/// first so they stay readable through the escaped reference. +pub(crate) fn release_datastack_frame(frame: &Py, vm: &VirtualMachine) { + let frame_obj = frame.as_object(); + // Uniqueness argument: at this point the frame is already out of + // the thread-frames registry and the current-frame chain + // (both unlinked inside `with_frame` before it returned), and the + // frame type has no weakref support. A datastack frame is created + // untracked and stays untracked while it runs, so it is in no GC + // generation list and no collector can observe or incref it. Hence no + // thread can mint a new reference without already holding one, and every + // escape (traceback, `sys._getframe`, `f_back`, a stored trace-hook arg) + // is a heap reference created on this thread while the frame ran. + // Therefore `strong_count() == 1` here means nothing escaped, and + // `strong_count() > 1` means the frame escaped. + debug_assert!( + !frame_obj.is_gc_tracked(), + "datastack frame is GC-tracked at release" + ); + if frame_obj.strong_count() == 1 { + // A reference minted and already released by another thread (through a + // heap escape carried across threads) ends in a release-decref; the + // fence orders that thread's memory before our drops below. + atomic::fence(Acquire); + // SAFETY: unique owner and no way to mint a new reference, so + // localsplus can never be observed again. The base pointer came + // from this thread's data stack. + unsafe { + if let Some(base) = frame.release_localsplus() { + vm.datastack_pop(base); + } + } + return; + } + // Escaped. Stabilize localsplus on the heap FIRST, then join the GC. This + // order guarantees a concurrent (stop-the-world) collector only ever sees a + // tracked frame whose localsplus is heap-resident and no longer mutating: + // the frame has stopped executing before it becomes a candidate, so its + // outgoing edges are stable while a collector traverses them. + // SAFETY: the frame finished executing; the base pointer came from this + // thread's data stack. unsafe { if let Some(base) = frame.materialize_localsplus() { vm.datastack_pop(base); } } + // Retain a strong reference to the caller so `f_back` keeps resolving once + // the caller returns and leaves the live frame chain. The caller is still + // executing here (this frame is unwinding back into it), so its payload + // pointer is live. + // SAFETY: `previous` points at the live caller on this thread's stack. + *frame.retained_back.lock() = unsafe { owned_chain_frame(frame.previous_frame()) }; + // Invariant: a tracked frame must always have heap-backed localsplus + // (proven here for escaped datastack frames and by construction for + // generator frames, which are born heap-backed). A stop-the-world + // collector reads a frame's localsplus only when the frame is a tracked + // candidate, so this keeps it from ever reading data-stack-resident, + // still-mutating storage of an executing frame. + debug_assert!( + !frame.localsplus_is_datastack_backed(), + "escaped frame tracked before its localsplus was materialized" + ); + // SAFETY: the frame is alive (held by `frame` and the escaped reference) + // and untracked. + unsafe { crate::gc_state::gc_state().track_object(NonNull::from(frame_obj)) }; } type BinaryOpExtendGuard = fn(&PyObject, &PyObject, &VirtualMachine) -> bool; @@ -1239,6 +1833,34 @@ struct BinaryOpExtendSpecializationDescr { const BINARY_OP_EXTEND_EXTERNAL_CACHE_OFFSET: usize = 1; +/// Max total args (including self) staged in a fixed-size stack buffer by the +/// exact-args call fast paths; larger arities fall back to a heap buffer. +const MAX_INLINE_CALL_ARGS: usize = 8; + +/// Staging buffer for exact-args call fast paths: fixed-size inline storage +/// for small arities, avoiding a per-call Vec allocation. +enum CallArgBuffer { + Inline(usize, [Option; MAX_INLINE_CALL_ARGS]), + Heap(Vec>), +} + +impl CallArgBuffer { + fn new(total_nargs: usize) -> Self { + if total_nargs <= MAX_INLINE_CALL_ARGS { + Self::Inline(total_nargs, [const { None }; MAX_INLINE_CALL_ARGS]) + } else { + Self::Heap(vec![None; total_nargs]) + } + } + + fn slots(&mut self) -> &mut [Option] { + match self { + Self::Inline(len, buf) => &mut buf[..*len], + Self::Heap(buf) => buf, + } + } +} + #[inline] fn compactlongs_guard(lhs: &PyObject, rhs: &PyObject, vm: &VirtualMachine) -> bool { compact_int_from_obj(lhs, vm).is_some() && compact_int_from_obj(rhs, vm).is_some() @@ -1396,55 +2018,34 @@ impl fmt::Debug for ExecutingFrame<'_> { } impl ExecutingFrame<'_> { - #[inline] - fn monitoring_disabled_for_code(&self, vm: &VirtualMachine) -> bool { - self.code.is(&vm.ctx.init_cleanup_code) - } - - fn specialization_new_init_cleanup_frame(&self, vm: &VirtualMachine) -> FrameRef { - Frame::new( - vm.ctx.init_cleanup_code.clone(), - Scope::new( - Some(ArgMapping::from_dict_exact(vm.ctx.new_dict())), - self.globals.clone(), - ), - self.builtins.clone(), - &[], - None, - true, - vm, - ) - .into_ref(&vm.ctx) - } - - fn specialization_run_init_cleanup_shim( + /// Run `__init__` for the tp_new specialization. `args` holds the + /// `__init__` args with slot 0 left empty; it is filled with `new_obj` + /// here. Enforces the `__init__() should return None` contract and + /// returns the constructed object. + fn specialization_run_init( &self, new_obj: PyObjectRef, init_func: &Py, - pos_args: Vec, + args: &mut [Option], vm: &VirtualMachine, ) -> PyResult { - let shim = self.specialization_new_init_cleanup_frame(vm); - let shim_result = vm.with_frame_untraced(shim.clone(), |shim| { - shim.with_exec(vm, |mut exec| exec.push_value(new_obj.clone())); - - let mut all_args = Vec::with_capacity(pos_args.len() + 1); - all_args.push(new_obj.clone()); - all_args.extend(pos_args); + args[0] = Some(new_obj.clone()); + let taken = args + .iter_mut() + .map(|slot| slot.take().expect("arg slot must be filled")); - let init_frame = init_func.prepare_exact_args_frame(all_args, vm); - let init_result = vm.run_frame(init_frame.clone()); - release_datastack_frame(&init_frame, vm); - let init_result = init_result?; + let init_frame = init_func.prepare_exact_args_frame(taken, vm); + let init_result = vm.run_frame(init_frame.clone()); + release_datastack_frame(&init_frame, vm); + let init_result = init_result?; - shim.with_exec(vm, |mut exec| exec.push_value(init_result)); - match shim.run(vm)? { - ExecutionResult::Return(value) => Ok(value), - ExecutionResult::Yield(_) => unreachable!("_Py_InitCleanup shim cannot yield"), - } - }); - release_datastack_frame(&shim, vm); - shim_result + if !vm.is_none(&init_result) { + return Err(vm.new_type_error(format!( + "__init__() should return None, not '{:.200}'", + init_result.class().name() + ))); + } + Ok(new_obj) } #[inline(always)] @@ -1483,6 +2084,15 @@ impl ExecutingFrame<'_> { if stack_analysis::top_of_stack(cur_stack) == stack_analysis::Kind::Except as i64 && let Some(exc_obj) = val { + // An Except-typed stack slot is only produced by bytecode that + // also carries one of the opcodes scanned by + // `PyCode::has_exc_handling`; otherwise the save/restore that + // brackets this frame's exc_info is elided and this write would + // corrupt the shared exc_info slot. + debug_assert!( + self.code.has_exc_handling, + "unwinding an Except slot in a frame without exc-handling opcodes" + ); if vm.is_none(&exc_obj) { vm.set_exception(None); } else { @@ -1585,39 +2195,44 @@ impl ExecutingFrame<'_> { } } - if vm.eval_breaker_tripped() - && let Err(exception) = vm.check_signals() - { - #[cold] - fn handle_signal_exception( - frame: &mut ExecutingFrame<'_>, - exception: PyBaseExceptionRef, - idx: usize, - vm: &VirtualMachine, - ) -> FrameResult { - if let Some((loc, _end_loc)) = frame.code.locations.get(idx) { - let next = exception.__traceback__(); - let new_traceback = PyTraceback::new( - next, - frame.object.to_owned(), - idx as u32 * 2, - loc.line, - ); - exception.set_traceback_typed(Some(new_traceback.into_ref(&vm.ctx))); - } - vm.contextualize_exception(&exception); - frame.unwind_blocks(vm, UnwindReason::Raising { exception }) - } - match handle_signal_exception(self, exception, idx, vm) { - Ok(None) => {} - Ok(Some(value)) => { - break Ok(value); + if vm.eval_breaker_tripped() { + if let Err(exception) = vm.check_signals() { + #[cold] + fn handle_signal_exception( + frame: &mut ExecutingFrame<'_>, + exception: PyBaseExceptionRef, + idx: usize, + vm: &VirtualMachine, + ) -> FrameResult { + if let Some((loc, _end_loc)) = frame.code.locations.get(idx) { + let next = exception.__traceback__(); + let new_traceback = PyTraceback::new( + next, + frame.object.to_owned(), + idx as u32 * 2, + loc.line, + ); + exception.set_traceback_typed(Some(new_traceback.into_ref(&vm.ctx))); + } + vm.contextualize_exception(&exception); + frame.unwind_blocks(vm, UnwindReason::Raising { exception }) } - Err(exception) => { - break Err(exception); + match handle_signal_exception(self, exception, idx, vm) { + Ok(None) => {} + Ok(Some(value)) => { + break Ok(value); + } + Err(exception) => { + break Err(exception); + } } + continue; } - continue; + // Run a scheduled automatic collection here — a safepoint with + // no interpreter locks held — instead of synchronously inside + // the allocation that tripped the threshold. + #[cfg(feature = "threading")] + vm.run_scheduled_gc(); } let lasti_before = self.lasti(); let result = self.execute_instruction(op, arg, &mut do_extend_arg, vm); @@ -2045,43 +2660,7 @@ impl ExecutingFrame<'_> { /// Get the variable name for a localsplus index. fn localsplus_name(&self, idx: usize) -> &'static PyStrInterned { - use rustpython_compiler_core::bytecode::{CO_FAST_CELL, CO_FAST_FREE, CO_FAST_LOCAL}; - let nlocals = self.code.varnames.len(); - let kind = self.code.localspluskinds.get(idx).copied().unwrap_or(0); - if kind & CO_FAST_LOCAL != 0 { - // Merged cell or regular local: name is in varnames - self.code.varnames[idx] - } else if kind & CO_FAST_FREE != 0 { - // Free var: slots are at the end of localsplus - let nlocalsplus = self.code.localspluskinds.len(); - let nfrees = self.code.freevars.len(); - let free_start = nlocalsplus - nfrees; - self.code.freevars[idx - free_start] - } else if kind & CO_FAST_CELL != 0 { - // Non-merged cell: count how many non-merged cell slots are before - // this index to find the corresponding cellvars entry. - // Non-merged cellvars appear in their original order (skipping merged ones). - let nonmerged_pos = self.code.localspluskinds[nlocals..idx] - .iter() - .filter(|&&k| k == CO_FAST_CELL) - .count(); - // Skip merged cellvars to find the right one - let mut cv_idx = 0; - let mut nonmerged_count = 0; - for (i, name) in self.code.cellvars.iter().enumerate() { - let is_merged = self.code.varnames.contains(name); - if !is_merged { - if nonmerged_count == nonmerged_pos { - cv_idx = i; - break; - } - nonmerged_count += 1; - } - } - self.code.cellvars[cv_idx] - } else { - self.code.varnames[idx] - } + localsplus_name(self.code, idx) } /// Execute a single instruction. @@ -3369,17 +3948,6 @@ impl ExecutingFrame<'_> { self.code.instructions.quicken(); atomic::fence(atomic::Ordering::Release); } - if self.monitoring_disabled_for_code(vm) { - let global_ver = vm - .state - .instrumentation_version - .load(atomic::Ordering::Acquire); - monitoring::instrument_code(self.code, 0); - self.code - .instrumentation_version - .store(global_ver, atomic::Ordering::Release); - return Ok(None); - } // Check if bytecode needs re-instrumentation let global_ver = vm .state @@ -3752,7 +4320,7 @@ impl ExecutingFrame<'_> { let should_be_none = self.pop_value(); if !vm.is_none(&should_be_none) { return Err(vm.new_type_error(format!( - "__init__() should return None, not '{}'", + "__init__() should return None, not '{:.200}'", should_be_none.class().name() ))); } @@ -4090,7 +4658,8 @@ impl ExecutingFrame<'_> { debug_assert!(func.has_exact_argcount(2)); let owner = self.pop_value(); let attr_name = self.code.names[oparg.name_idx() as usize].to_owned().into(); - let result = func.invoke_exact_args(vec![owner, attr_name], vm)?; + let result = + func.invoke_exact_args_slots(&mut [Some(owner), Some(attr_name)], vm)?; self.push_value(result); return Ok(None); } @@ -4137,7 +4706,7 @@ impl ExecutingFrame<'_> { && self.specialization_has_datastack_space_for_func(vm, func) { let owner = self.pop_value(); - let result = func.invoke_exact_args(vec![owner], vm)?; + let result = func.invoke_exact_args_slots(&mut [Some(owner)], vm)?; self.push_value(result); return Ok(None); } @@ -4234,13 +4803,13 @@ impl ExecutingFrame<'_> { } // Specialized BINARY_OP opcodes Instruction::BinaryOpAddInt => { - self.execute_binary_op_int(vm, |a, b| a + b, bytecode::BinaryOperator::Add) + self.execute_binary_op_int(vm, Self::int_add, bytecode::BinaryOperator::Add) } Instruction::BinaryOpSubtractInt => { - self.execute_binary_op_int(vm, |a, b| a - b, bytecode::BinaryOperator::Subtract) + self.execute_binary_op_int(vm, Self::int_sub, bytecode::BinaryOperator::Subtract) } Instruction::BinaryOpMultiplyInt => { - self.execute_binary_op_int(vm, |a, b| a * b, bytecode::BinaryOperator::Multiply) + self.execute_binary_op_int(vm, Self::int_mul, bytecode::BinaryOperator::Multiply) } Instruction::BinaryOpAddFloat => { self.execute_binary_op_float(vm, |a, b| a + b, bytecode::BinaryOperator::Add) @@ -4268,8 +4837,12 @@ impl ExecutingFrame<'_> { } } Instruction::BinaryOpSubscrGetitem => { + let cache_base = self.lasti() as usize; + let type_version = self.code.instructions.read_cache_u32(cache_base + 1); let owner = self.nth_value(1); if !self.specialization_eval_frame_active(vm) + && type_version != 0 + && owner.class().tp_version_tag.load(Acquire) == type_version && let Some((func, func_version)) = owner.class().get_cached_getitem_for_specialization() && func.func_version() == func_version @@ -4278,7 +4851,7 @@ impl ExecutingFrame<'_> { debug_assert!(func.has_exact_argcount(2)); let sub = self.pop_value(); let owner = self.pop_value(); - let result = func.invoke_exact_args(vec![owner, sub], vm)?; + let result = func.invoke_exact_args_slots(&mut [Some(owner), Some(sub)], vm)?; self.push_value(result); return Ok(None); } @@ -4425,19 +4998,24 @@ impl ExecutingFrame<'_> { if self.specialization_call_recursion_guard(vm) { return self.execute_call_vectorcall(nargs, vm); } - let pos_args: Vec = self.pop_multiple(nargs as usize).collect(); + // Stage args without a per-call Vec: [self?, arg1, ..., argN] + let base = usize::from(self_or_null_is_some); + let mut arg_buf = CallArgBuffer::new(nargs as usize + base); + let args = arg_buf.slots(); + for (slot, arg) in args[base..] + .iter_mut() + .zip(self.pop_multiple(nargs as usize)) + { + *slot = Some(arg); + } let self_or_null = self.pop_value_opt(); + debug_assert_eq!(self_or_null.is_some(), self_or_null_is_some); + if self_or_null.is_some() { + args[0] = self_or_null; + } let callable = self.pop_value(); let func = callable.downcast_ref_if_exact::(vm).unwrap(); - let args = if let Some(self_val) = self_or_null { - let mut all_args = Vec::with_capacity(pos_args.len() + 1); - all_args.push(self_val); - all_args.extend(pos_args); - all_args - } else { - pos_args - }; - let result = func.invoke_exact_args(args, vm)?; + let result = func.invoke_exact_args_slots(args, vm)?; self.push_value(result); Ok(None) } else { @@ -4477,14 +5055,19 @@ impl ExecutingFrame<'_> { if self.specialization_call_recursion_guard(vm) { return self.execute_call_vectorcall(nargs, vm); } - let pos_args: Vec = - self.pop_multiple(nargs as usize).collect(); + // Stage args without a per-call Vec: + // [bound_self, arg1, ..., argN] + let mut arg_buf = CallArgBuffer::new(nargs as usize + 1); + let args = arg_buf.slots(); + for (slot, arg) in + args[1..].iter_mut().zip(self.pop_multiple(nargs as usize)) + { + *slot = Some(arg); + } self.pop_value_opt(); // null (self_or_null) self.pop_value(); // callable (bound method) - let mut all_args = Vec::with_capacity(pos_args.len() + 1); - all_args.push(bound_self); - all_args.extend(pos_args); - let result = func.invoke_exact_args(all_args, vm)?; + args[0] = Some(bound_self); + let result = func.invoke_exact_args_slots(args, vm)?; self.push_value(result); return Ok(None); } @@ -4532,16 +5115,18 @@ impl ExecutingFrame<'_> { .as_ref() .is_some_and(|isinstance_callable| callable.is(isinstance_callable)) { - let nargs_usize = nargs as usize; - let pos_args: Vec = self.pop_multiple(nargs_usize).collect(); - let self_or_null = self.pop_value_opt(); + // Stack: [callable, self_or_null, args...]; effective_nargs == 2, + // so the instance is either the first positional arg or self_or_null. + let cls = self.pop_value(); + let inst = if nargs == 2 { + let inst = self.pop_value(); + self.pop_value_opt(); // null + inst + } else { + self.pop_value() // self_or_null holds the instance + }; self.pop_value(); // callable - let mut all_args = Vec::with_capacity(2); - if let Some(self_val) = self_or_null { - all_args.push(self_val); - } - all_args.extend(pos_args); - let result = all_args[0].is_instance(&all_args[1], vm)?; + let result = inst.is_instance(&cls, vm)?; self.push_value(vm.ctx.new_bool(result).into()); return Ok(None); } @@ -4980,38 +5565,36 @@ impl ExecutingFrame<'_> { && cached_version != 0 && let Some(cls) = callable.downcast_ref::() && cls.tp_version_tag.load(Acquire) == cached_version - && let Some(init_func) = cls.get_cached_init_for_specialization(cached_version) + && let Some((init_func, init_func_version)) = + cls.get_cached_init_for_specialization(cached_version) + && init_func.func_version() == init_func_version + && init_func.has_exact_argcount(nargs + 1) && let Some(cls_alloc) = cls.slots.alloc.load() { - // Match CPython's `code->co_framesize + _Py_InitCleanup.co_framesize` - // shape, using RustPython's datastack-backed frame size - // equivalent for the extra shim frame. - let init_cleanup_stack_bytes = - datastack_frame_size_bytes_for_code(&vm.ctx.init_cleanup_code) - .expect("_Py_InitCleanup shim is not a generator/coroutine"); - if !self.specialization_has_datastack_space_for_func_with_extra( - vm, - &init_func, - init_cleanup_stack_bytes, - ) { + // The specialization runs `__init__` directly with no + // interpreter-visible trampoline frame. Deopt when the + // datastack or recursion budget for the `__init__` frame is + // unavailable. + if !self.specialization_has_datastack_space_for_func(vm, &init_func) { return self.execute_call_vectorcall(nargs, vm); } - // CPython creates `_Py_InitCleanup` + `__init__` frames here. - // Keep the guard conservative and deopt when the effective - // recursion budget for those two frames is not available. - if self.specialization_call_recursion_guard_with_extra_frames(vm, 1) { + if self.specialization_call_recursion_guard(vm) { return self.execute_call_vectorcall(nargs, vm); } // Allocate object directly (tp_new == object.__new__, tp_alloc == generic). let cls_ref = cls.to_owned(); let new_obj = cls_alloc(cls_ref, 0, vm)?; - // Build args: [new_obj, arg1, ..., argN] - let pos_args: Vec = self.pop_multiple(nargs as usize).collect(); + // Stage args as [new_obj, arg1, ..., argN]; slot 0 is + // filled by the init runner. + let mut arg_buf = CallArgBuffer::new(nargs as usize + 1); + let args = arg_buf.slots(); + for (slot, arg) in args[1..].iter_mut().zip(self.pop_multiple(nargs as usize)) { + *slot = Some(arg); + } let _null = self.pop_value_opt(); // self_or_null (None) let _callable = self.pop_value(); // callable (type) - let result = self - .specialization_run_init_cleanup_shim(new_obj, &init_func, pos_args, vm)?; + let result = self.specialization_run_init(new_obj, &init_func, args, vm)?; self.push_value(result); return Ok(None); } @@ -5798,18 +6381,6 @@ impl ExecutingFrame<'_> { instruction.is_instrumented(), "execute_instrumented called with non-instrumented opcode {instruction:?}" ); - if self.monitoring_disabled_for_code(vm) { - let global_ver = vm - .state - .instrumentation_version - .load(atomic::Ordering::Acquire); - monitoring::instrument_code(self.code, 0); - self.code - .instrumentation_version - .store(global_ver, atomic::Ordering::Release); - self.update_lasti(|i| *i -= 1); - return Ok(None); - } self.monitoring_mask = vm.state.monitoring_events.load(); match instruction { Instruction::InstrumentedResume => { @@ -7095,15 +7666,15 @@ impl ExecutingFrame<'_> { let b_ref = &self.pop_value(); let a_ref = &self.pop_value(); let value = match op { - // BINARY_OP_ADD_INT / BINARY_OP_SUBTRACT_INT fast paths: - // bypass binary_op1 dispatch for exact int types, use i64 arithmetic - // when possible to avoid BigInt heap allocation. + // Exact-int fast paths for +, -, *, //, %: bypass binary_op1 + // dispatch and use i64 arithmetic when possible to avoid BigInt + // heap allocation, falling back to the slow path otherwise. bytecode::BinaryOperator::Add | bytecode::BinaryOperator::InplaceAdd => { if let (Some(a), Some(b)) = ( a_ref.downcast_ref_if_exact::(vm), b_ref.downcast_ref_if_exact::(vm), ) { - Ok(self.int_add(a.as_bigint(), b.as_bigint(), vm)) + Ok(Self::int_add(a.as_bigint(), b.as_bigint(), vm)) } else if matches!(op, bytecode::BinaryOperator::Add) { vm._add(a_ref, b_ref) } else { @@ -7115,32 +7686,65 @@ impl ExecutingFrame<'_> { a_ref.downcast_ref_if_exact::(vm), b_ref.downcast_ref_if_exact::(vm), ) { - Ok(self.int_sub(a.as_bigint(), b.as_bigint(), vm)) + Ok(Self::int_sub(a.as_bigint(), b.as_bigint(), vm)) } else if matches!(op, bytecode::BinaryOperator::Subtract) { vm._sub(a_ref, b_ref) } else { vm._isub(a_ref, b_ref) } } - bytecode::BinaryOperator::Multiply => vm._mul(a_ref, b_ref), + bytecode::BinaryOperator::Multiply | bytecode::BinaryOperator::InplaceMultiply => { + if let (Some(a), Some(b)) = ( + a_ref.downcast_ref_if_exact::(vm), + b_ref.downcast_ref_if_exact::(vm), + ) { + Ok(Self::int_mul(a.as_bigint(), b.as_bigint(), vm)) + } else if matches!(op, bytecode::BinaryOperator::Multiply) { + vm._mul(a_ref, b_ref) + } else { + vm._imul(a_ref, b_ref) + } + } bytecode::BinaryOperator::MatrixMultiply => vm._matmul(a_ref, b_ref), bytecode::BinaryOperator::Power => vm._pow(a_ref, b_ref, vm.ctx.none.as_object()), bytecode::BinaryOperator::TrueDivide => vm._truediv(a_ref, b_ref), - bytecode::BinaryOperator::FloorDivide => vm._floordiv(a_ref, b_ref), - bytecode::BinaryOperator::Remainder => vm._mod(a_ref, b_ref), + bytecode::BinaryOperator::FloorDivide + | bytecode::BinaryOperator::InplaceFloorDivide => { + if let (Some(a), Some(b)) = ( + a_ref.downcast_ref_if_exact::(vm), + b_ref.downcast_ref_if_exact::(vm), + ) && let Some(result) = Self::int_floordiv(a.as_bigint(), b.as_bigint(), vm) + { + Ok(result) + } else if matches!(op, bytecode::BinaryOperator::FloorDivide) { + vm._floordiv(a_ref, b_ref) + } else { + vm._ifloordiv(a_ref, b_ref) + } + } + bytecode::BinaryOperator::Remainder | bytecode::BinaryOperator::InplaceRemainder => { + if let (Some(a), Some(b)) = ( + a_ref.downcast_ref_if_exact::(vm), + b_ref.downcast_ref_if_exact::(vm), + ) && let Some(result) = Self::int_mod(a.as_bigint(), b.as_bigint(), vm) + { + Ok(result) + } else if matches!(op, bytecode::BinaryOperator::Remainder) { + vm._mod(a_ref, b_ref) + } else { + vm._imod(a_ref, b_ref) + } + } bytecode::BinaryOperator::Lshift => vm._lshift(a_ref, b_ref), bytecode::BinaryOperator::Rshift => vm._rshift(a_ref, b_ref), bytecode::BinaryOperator::Xor => vm._xor(a_ref, b_ref), bytecode::BinaryOperator::Or => vm._or(a_ref, b_ref), bytecode::BinaryOperator::And => vm._and(a_ref, b_ref), - bytecode::BinaryOperator::InplaceMultiply => vm._imul(a_ref, b_ref), bytecode::BinaryOperator::InplaceMatrixMultiply => vm._imatmul(a_ref, b_ref), bytecode::BinaryOperator::InplacePower => { vm._ipow(a_ref, b_ref, vm.ctx.none.as_object()) } bytecode::BinaryOperator::InplaceTrueDivide => vm._itruediv(a_ref, b_ref), - bytecode::BinaryOperator::InplaceFloorDivide => vm._ifloordiv(a_ref, b_ref), - bytecode::BinaryOperator::InplaceRemainder => vm._imod(a_ref, b_ref), bytecode::BinaryOperator::InplaceLshift => vm._ilshift(a_ref, b_ref), bytecode::BinaryOperator::InplaceRshift => vm._irshift(a_ref, b_ref), bytecode::BinaryOperator::InplaceXor => vm._ixor(a_ref, b_ref), @@ -7153,28 +7757,107 @@ impl ExecutingFrame<'_> { Ok(None) } - /// Int addition with i64 fast path to avoid BigInt heap allocation. + /// Int binary op with an i64 fast path to avoid BigInt heap allocation. + /// `checked` computes the i64 result; on `None` (either operand does not + /// fit i64, or the op overflows i64) it falls through to `fallback` on the + /// full BigInt values. Result boxing always goes through `new_int` so the + /// small-int cache is consulted identically. #[inline] - fn int_add(&self, a: &BigInt, b: &BigInt, vm: &VirtualMachine) -> PyObjectRef { + fn int_fast_op( + a: &BigInt, + b: &BigInt, + vm: &VirtualMachine, + checked: fn(i64, i64) -> Option, + fallback: impl FnOnce(&BigInt, &BigInt) -> BigInt, + ) -> PyObjectRef { use num_traits::ToPrimitive; if let (Some(av), Some(bv)) = (a.to_i64(), b.to_i64()) - && let Some(result) = av.checked_add(bv) + && let Some(result) = checked(av, bv) { return vm.ctx.new_int(result).into(); } - vm.ctx.new_int(a + b).into() + vm.ctx.new_int(fallback(a, b)).into() + } + + /// Int addition with i64 fast path to avoid BigInt heap allocation. + #[inline] + fn int_add(a: &BigInt, b: &BigInt, vm: &VirtualMachine) -> PyObjectRef { + Self::int_fast_op(a, b, vm, i64::checked_add, |a, b| a + b) } /// Int subtraction with i64 fast path to avoid BigInt heap allocation. #[inline] - fn int_sub(&self, a: &BigInt, b: &BigInt, vm: &VirtualMachine) -> PyObjectRef { + fn int_sub(a: &BigInt, b: &BigInt, vm: &VirtualMachine) -> PyObjectRef { + Self::int_fast_op(a, b, vm, i64::checked_sub, |a, b| a - b) + } + + /// Int multiplication with i64 fast path to avoid BigInt heap allocation. + #[inline] + fn int_mul(a: &BigInt, b: &BigInt, vm: &VirtualMachine) -> PyObjectRef { + Self::int_fast_op(a, b, vm, i64::checked_mul, |a, b| a * b) + } + + /// Int divide/remainder i64 fast path. Returns `None` to signal the caller + /// to fall through to the slow path when either operand does not fit i64 + /// or `compute` reports a case it cannot handle (zero divisor or i64 + /// overflow). Result boxing goes through `new_int` so the small-int cache + /// is consulted identically. + #[inline] + fn int_div_fast_op( + a: &BigInt, + b: &BigInt, + vm: &VirtualMachine, + compute: fn(i64, i64) -> Option, + ) -> Option { use num_traits::ToPrimitive; - if let (Some(av), Some(bv)) = (a.to_i64(), b.to_i64()) - && let Some(result) = av.checked_sub(bv) - { - return vm.ctx.new_int(result).into(); + let (av, bv) = (a.to_i64()?, b.to_i64()?); + compute(av, bv).map(|r| vm.ctx.new_int(r).into()) + } + + /// Floor division of two i64 values with floor (toward negative infinity) + /// semantics. `None` when `b == 0` or the quotient overflows i64 + /// (`i64::MIN / -1`). + #[inline] + fn floordiv_i64(a: i64, b: i64) -> Option { + if b == 0 { + return None; } - vm.ctx.new_int(a - b).into() + let q = a.checked_div(b)?; + let r = a % b; + Some(if r != 0 && (r < 0) != (b < 0) { + q - 1 + } else { + q + }) + } + + /// Remainder of two i64 values, taking the sign of the divisor. `None` + /// when `b == 0` or the operation overflows i64 (`i64::MIN % -1`). + #[inline] + fn mod_i64(a: i64, b: i64) -> Option { + if b == 0 { + return None; + } + let r = a.checked_rem(b)?; + Some(if r != 0 && (r < 0) != (b < 0) { + r + b + } else { + r + }) + } + + /// Int floor division with i64 fast path. `None` falls through to the + /// slow path (bigint operands, zero divisor, or i64 overflow). + #[inline] + fn int_floordiv(a: &BigInt, b: &BigInt, vm: &VirtualMachine) -> Option { + Self::int_div_fast_op(a, b, vm, Self::floordiv_i64) + } + + /// Int remainder with i64 fast path. `None` falls through to the slow + /// path (bigint operands, zero divisor, or i64 overflow). + #[inline] + fn int_mod(a: &BigInt, b: &BigInt, vm: &VirtualMachine) -> Option { + Self::int_div_fast_op(a, b, vm, Self::mod_i64) } #[cold] @@ -7523,6 +8206,22 @@ impl ExecutingFrame<'_> { return; } + // Capture the version before inspecting getattro and the MRO so a + // concurrently installed __getattribute__/__getattr__ invalidates the + // version this specialization is cached against. + let type_version = cls.version_for_specialization(_vm); + if type_version == 0 { + unsafe { + self.code.instructions.write_adaptive_counter( + cache_base, + bytecode::adaptive_counter_backoff( + self.code.instructions.read_adaptive_counter(cache_base), + ), + ); + } + return; + } + // Only specialize if getattro is the default (PyBaseObject::getattro) let is_default_getattro = cls .slots @@ -7530,15 +8229,11 @@ impl ExecutingFrame<'_> { .load() .is_some_and(|f| f as usize == PyBaseObject::getattro as *const () as usize); if !is_default_getattro { - let mut type_version = cls.tp_version_tag.load(Acquire); - if type_version == 0 { - type_version = cls.assign_version_tag(); - } - if type_version != 0 - && !oparg.is_method() + let getattribute = cls.get_attr(identifier!(_vm, __getattribute__)); + if !oparg.is_method() && !self.specialization_eval_frame_active(_vm) && cls.get_attr(identifier!(_vm, __getattr__)).is_none() - && let Some(getattribute) = cls.get_attr(identifier!(_vm, __getattribute__)) + && let Some(getattribute) = getattribute && let Some(func) = getattribute.downcast_ref_if_exact::(_vm) && func.can_specialize_call(2) { @@ -7570,24 +8265,6 @@ impl ExecutingFrame<'_> { return; } - // Get or assign type version - let mut type_version = cls.tp_version_tag.load(Acquire); - if type_version == 0 { - type_version = cls.assign_version_tag(); - } - if type_version == 0 { - // Version counter overflow — backoff to avoid re-attempting every execution - unsafe { - self.code.instructions.write_adaptive_counter( - cache_base, - bytecode::adaptive_counter_backoff( - self.code.instructions.read_adaptive_counter(cache_base), - ), - ); - } - return; - } - let attr_name = self.code.names[oparg.name_idx() as usize]; // Match CPython: only specialize module attribute loads when the @@ -7620,7 +8297,6 @@ impl ExecutingFrame<'_> { return; } - // Look up attr in class via MRO let cls_attr = cls.get_attr(attr_name); let class_has_dict = cls.slots.flags.has_feature(PyTypeFlags::HAS_DICT); @@ -7792,29 +8468,11 @@ impl ExecutingFrame<'_> { ) { let obj = self.top_value(); let owner_type = obj.downcast_ref::().unwrap(); - - // Get or assign type version for the type object itself - let mut type_version = owner_type.tp_version_tag.load(Acquire); - if type_version == 0 { - type_version = owner_type.assign_version_tag(); - } - if type_version == 0 { - unsafe { - self.code.instructions.write_adaptive_counter( - cache_base, - bytecode::adaptive_counter_backoff( - self.code.instructions.read_adaptive_counter(cache_base), - ), - ); - } - return; - } - let attr_name = self.code.names[oparg.name_idx() as usize]; // Check metaclass: ensure no data descriptor on metaclass for this name let mcl = obj.class(); - let mcl_attr = mcl.get_attr(attr_name); + let (mcl_attr, mut metaclass_version) = mcl.lookup_ref_and_version_interned(attr_name, _vm); if let Some(ref attr) = mcl_attr { let attr_class = attr.class(); if attr_class.slots.descr_set.load().is_some() { @@ -7830,12 +8488,7 @@ impl ExecutingFrame<'_> { return; } } - let mut metaclass_version = 0; if !mcl.slots.flags.has_feature(PyTypeFlags::IMMUTABLETYPE) { - metaclass_version = mcl.tp_version_tag.load(Acquire); - if metaclass_version == 0 { - metaclass_version = mcl.assign_version_tag(); - } if metaclass_version == 0 { unsafe { self.code.instructions.write_adaptive_counter( @@ -7847,10 +8500,22 @@ impl ExecutingFrame<'_> { } return; } + } else { + metaclass_version = 0; } - // Look up attr in the type's own MRO - let cls_attr = owner_type.get_attr(attr_name); + let (cls_attr, type_version) = owner_type.lookup_ref_and_version_interned(attr_name, _vm); + if type_version == 0 { + unsafe { + self.code.instructions.write_adaptive_counter( + cache_base, + bytecode::adaptive_counter_backoff( + self.code.instructions.read_adaptive_counter(cache_base), + ), + ); + } + return; + } if let Some(ref descr) = cls_attr { let descr_class = descr.class(); let has_descr_get = descr_class.slots.descr_get.load().is_some(); @@ -8022,26 +8687,31 @@ impl ExecutingFrame<'_> { Some(Instruction::BinaryOpSubscrListSlice) } else { let cls = a.class(); + // Check the cheap gates before the __getitem__ lookup, which + // takes the global type lock and may allocate a version tag. if cls.slots.flags.has_feature(PyTypeFlags::HEAPTYPE) && !self.specialization_eval_frame_active(vm) - && let Some(_getitem) = cls.get_attr(identifier!(vm, __getitem__)) - && let Some(func) = _getitem.downcast_ref_if_exact::(vm) - && func.can_specialize_call(2) { - let mut type_version = cls.tp_version_tag.load(Acquire); - if type_version == 0 { - type_version = cls.assign_version_tag(); - } - if type_version != 0 { - if cls.cache_getitem_for_specialization( + let (getitem, type_version) = + cls.lookup_ref_and_version_interned(identifier!(vm, __getitem__), vm); + if type_version != 0 + && let Some(getitem) = getitem + && let Some(func) = getitem.downcast_ref_if_exact::(vm) + && func.can_specialize_call(2) + && cls.cache_getitem_for_specialization( func.to_owned(), type_version, vm, - ) { - Some(Instruction::BinaryOpSubscrGetitem) - } else { - None + ) + { + // Record the type version so the specialized handler + // can revalidate before using the cached __getitem__. + unsafe { + self.code + .instructions + .write_cache_u32(cache_base + 1, type_version); } + Some(Instruction::BinaryOpSubscrGetitem) } else { None } @@ -8211,7 +8881,7 @@ impl ExecutingFrame<'_> { fn execute_binary_op_int( &mut self, vm: &VirtualMachine, - op: impl FnOnce(&BigInt, &BigInt) -> BigInt, + op: impl FnOnce(&BigInt, &BigInt, &VirtualMachine) -> PyObjectRef, deopt_op: bytecode::BinaryOperator, ) -> FrameResult { let b = self.top_value(); @@ -8220,10 +8890,10 @@ impl ExecutingFrame<'_> { a.downcast_ref_if_exact::(vm), b.downcast_ref_if_exact::(vm), ) { - let result = op(a_int.as_bigint(), b_int.as_bigint()); + let result = op(a_int.as_bigint(), b_int.as_bigint(), vm); self.pop_value(); self.pop_value(); - self.push_value(vm.ctx.new_bigint(&result).into()); + self.push_value(result); Ok(None) } else { self.execute_bin_op(vm, deopt_op) @@ -8561,6 +9231,10 @@ impl ExecutingFrame<'_> { // CallAllocAndEnterInit: heap type with default __new__ if !self_or_null_is_some && cls.slots.flags.has_feature(PyTypeFlags::HEAPTYPE) { + // Capture the version before inspecting tp_new/tp_alloc so a + // concurrently installed __new__ invalidates the version this + // specialization is cached against. + let type_version = cls.version_for_specialization(vm); let object_new = vm.ctx.types.object_type.slots.new.load(); let cls_new = cls.slots.new.load(); let object_alloc = vm.ctx.types.object_type.slots.alloc.load(); @@ -8570,12 +9244,7 @@ impl ExecutingFrame<'_> { && cls_new_fn as usize == obj_new_fn as usize && cls_alloc_fn as usize == obj_alloc_fn as usize { - let init = cls.get_attr(identifier!(vm, __init__)); - let mut version = cls.tp_version_tag.load(Acquire); - if version == 0 { - version = cls.assign_version_tag(); - } - if version == 0 { + if type_version == 0 { unsafe { self.code.instructions.write_adaptive_counter( cache_base, @@ -8586,15 +9255,17 @@ impl ExecutingFrame<'_> { } return; } + let init = cls.get_attr(identifier!(vm, __init__)); if let Some(init) = init && let Some(init_func) = init.downcast_ref_if_exact::(vm) - && init_func.is_simple_for_call_specialization() - && cls.cache_init_for_specialization(init_func.to_owned(), version, vm) + && init_func.can_specialize_call(nargs + 1) + && !init_func.is_generator_like() + && cls.cache_init_for_specialization(init_func.to_owned(), type_version, vm) { unsafe { self.code .instructions - .write_cache_u32(cache_base + 1, version); + .write_cache_u32(cache_base + 1, type_version); } self.specialize_at( instr_idx, @@ -8888,34 +9559,35 @@ impl ExecutingFrame<'_> { Some(Instruction::ToBoolList) } else if cls.is(PyStr::class(&vm.ctx)) { Some(Instruction::ToBoolStr) - } else if cls.slots.flags.has_feature(PyTypeFlags::HEAPTYPE) - && cls.slots.as_number.boolean.load().is_none() - && cls.slots.as_mapping.length.load().is_none() - && cls.slots.as_sequence.length.load().is_none() - { - // Cache type version for ToBoolAlwaysTrue guard - let mut type_version = cls.tp_version_tag.load(Acquire); - if type_version == 0 { - type_version = cls.assign_version_tag(); - } - if type_version != 0 { - unsafe { - self.code - .instructions - .write_cache_u32(cache_base + 1, type_version); - } - self.specialize_at(instr_idx, cache_base, Instruction::ToBoolAlwaysTrue); - } else { - unsafe { - self.code.instructions.write_adaptive_counter( - cache_base, - bytecode::adaptive_counter_backoff( - self.code.instructions.read_adaptive_counter(cache_base), - ), - ); + } else if cls.slots.flags.has_feature(PyTypeFlags::HEAPTYPE) { + // Capture the version before inspecting the bool/len slots so a + // concurrently installed __bool__/__len__ invalidates the version + // the ToBoolAlwaysTrue guard is cached against. + let type_version = cls.version_for_specialization(vm); + let has_bool_or_len = cls.slots.as_number.boolean.load().is_some() + || cls.slots.as_mapping.length.load().is_some() + || cls.slots.as_sequence.length.load().is_some(); + if !has_bool_or_len { + if type_version != 0 { + unsafe { + self.code + .instructions + .write_cache_u32(cache_base + 1, type_version); + } + self.specialize_at(instr_idx, cache_base, Instruction::ToBoolAlwaysTrue); + } else { + unsafe { + self.code.instructions.write_adaptive_counter( + cache_base, + bytecode::adaptive_counter_backoff( + self.code.instructions.read_adaptive_counter(cache_base), + ), + ); + } } + return; } - return; + None } else { None }; @@ -9213,13 +9885,11 @@ impl ExecutingFrame<'_> { let owner = self.top_value(); let cls = owner.class(); - // Only specialize if setattr is the default (generic_setattr) - let is_default_setattr = cls - .slots - .setattro - .load() - .is_some_and(|f| f as usize == PyBaseObject::slot_setattro as *const () as usize); - if !is_default_setattr { + // Capture the version before inspecting the setattro slot so a + // concurrently installed __setattr__ invalidates the version this + // specialization is cached against. + let type_version = cls.version_for_specialization(vm); + if type_version == 0 { unsafe { self.code.instructions.write_adaptive_counter( cache_base, @@ -9231,12 +9901,13 @@ impl ExecutingFrame<'_> { return; } - // Get or assign type version - let mut type_version = cls.tp_version_tag.load(Acquire); - if type_version == 0 { - type_version = cls.assign_version_tag(); - } - if type_version == 0 { + // Only specialize if setattr is the default (generic_setattr) + let is_default_setattr = cls + .slots + .setattro + .load() + .is_some_and(|f| f as usize == PyBaseObject::slot_setattro as *const () as usize); + if !is_default_setattr { unsafe { self.code.instructions.write_adaptive_counter( cache_base, @@ -9248,7 +9919,6 @@ impl ExecutingFrame<'_> { return; } - // Check for data descriptor let attr_name = self.code.names[attr_idx as usize]; let cls_attr = cls.get_attr(attr_name); let has_data_descr = cls_attr.as_ref().is_some_and(|descr| { @@ -9670,7 +10340,9 @@ impl fmt::Debug for Frame { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { // SAFETY: Debug is best-effort; concurrent mutation is unlikely // and would only affect debug output. - let iframe = unsafe { &*self.iframe.get() }; + let Some(iframe) = (unsafe { &*self.iframe.get() }) else { + return f.write_str("Frame Object { cleared }"); + }; let stack_str = iframe .localsplus diff --git a/crates/vm/src/function/argument.rs b/crates/vm/src/function/argument.rs index c37475ba48d..88ef5581bb2 100644 --- a/crates/vm/src/function/argument.rs +++ b/crates/vm/src/function/argument.rs @@ -12,7 +12,11 @@ pub trait IntoFuncArgs: Sized { fn into_args(self, vm: &VirtualMachine) -> FuncArgs; fn into_method_args(self, obj: PyObjectRef, vm: &VirtualMachine) -> FuncArgs { let mut args = self.into_args(vm); - args.prepend_arg(obj); + // Build the final vec once instead of prepending (realloc + memmove). + let mut with_obj = Vec::with_capacity(args.args.len() + 1); + with_obj.push(obj); + with_obj.append(&mut args.args); + args.args = with_obj; args } } @@ -202,7 +206,9 @@ impl FuncArgs { } pub fn prepend_arg(&mut self, item: PyObjectRef) { - self.args.reserve_exact(1); + // reserve (not reserve_exact): incoming vectors are usually built with + // exact capacity, so exact growth would realloc on every prepend. + self.args.reserve(1); self.args.insert(0, item) } diff --git a/crates/vm/src/gc_state.rs b/crates/vm/src/gc_state.rs index e20cbcb8ecf..ebae58f96cb 100644 --- a/crates/vm/src/gc_state.rs +++ b/crates/vm/src/gc_state.rs @@ -136,6 +136,61 @@ impl GcGeneration { #[derive(Clone, Copy, PartialEq, Eq, Hash)] struct GcPtr(NonNull); +/// RAII barrier that parks every other thread for the pointer-reading phases +/// of a collection and lets them run again before finalizers execute. +/// +/// Reference subtraction, the reachability walk and the strong-reference +/// snapshot dereference the interpreter state of every tracked object, +/// including the `localsplus` of frames that other threads are actively +/// executing. Those writes carry no synchronization, so the reads are only +/// well-defined while all other threads are parked at a safepoint. Restarting +/// happens explicitly once the snapshot has pinned every object; `Drop` is a +/// backstop that also restarts on the early-return paths. +#[cfg(feature = "threading")] +struct CollectStopTheWorld { + vm: *const crate::VirtualMachine, + stopped: bool, +} + +#[cfg(feature = "threading")] +impl CollectStopTheWorld { + /// Request stop-the-world when the current thread has an attached VM. + /// Falls back to no barrier when no VM is attached (the tracked-object + /// reads then run without other threads only if the caller guarantees it). + fn new() -> Self { + let vm = crate::vm::thread::try_with_current_vm(|vm| { + vm.state.stop_the_world.stop_the_world(vm); + vm as *const crate::VirtualMachine + }); + match vm { + Some(vm) => Self { vm, stopped: true }, + None => Self { + vm: core::ptr::null(), + stopped: false, + }, + } + } + + /// Restart the world. Idempotent. + fn restart(&mut self) { + if self.stopped { + // SAFETY: the current thread stays attached to this VM for the + // whole collection — the VM is never popped from the thread's VM + // stack while collecting — so the pointer is valid here. + let vm = unsafe { &*self.vm }; + vm.state.stop_the_world.start_the_world(vm); + self.stopped = false; + } + } +} + +#[cfg(feature = "threading")] +impl Drop for CollectStopTheWorld { + fn drop(&mut self) { + self.restart(); + } +} + /// Global GC state pub struct GcState { /// 3 generations (0 = youngest, 2 = oldest) @@ -366,8 +421,23 @@ impl GcState { let count0 = self.generations[0].count.load(Ordering::SeqCst) as u32; let threshold0 = self.generations[0].threshold(); if threshold0 > 0 && count0 >= threshold0 { - self.collect(0); - return true; + #[cfg(feature = "threading")] + { + // Defer to the next bytecode safepoint. Collecting here would + // stop the world while this thread may hold an internal lock + // (e.g. a lazily-initialized frame locals cell) that another + // thread is blocked on with no way to reach a safepoint — + // a deadlock. At a safepoint no such lock is held. + crate::signal::schedule_gc(); + return false; + } + // Without threading there is no safepoint to defer to and no other + // thread whose frames could be read mid-mutation, so collect inline. + #[cfg(not(feature = "threading"))] + { + self.collect(0); + return true; + } } false @@ -409,6 +479,32 @@ impl GcState { // might prevent cycle collection (_PyType_ClearCache). crate::builtins::type_::type_cache_clear(); + // Backstop for QSBR reclamation (threads may have missed requests). + #[cfg(feature = "threading")] + crate::object::qsbr::QSBR.process(); + + // Stop the world before reading any tracked object's interpreter + // state. Requested *before* the generation read locks are taken: a + // thread parking at a safepoint may still hold a generation write lock + // (track/untrack/promote) and must be able to release it to reach the + // safepoint. It could not do so if this thread already held a read + // lock it was waiting behind — hence the ordering. + // + // Auto-collection is deferred to a bytecode safepoint (see + // `maybe_collect`), where no internal lock is held, so it never stops + // the world under a lock. Explicit `gc.collect()` runs synchronously + // here; a re-entrant call from a finalizer during an in-progress + // collection is turned into a no-op by the `collecting` try_lock above. + // The one residual is an explicit `gc.collect()` reached from a + // finalizer/`__del__` that runs inline while a non-generation internal + // lock is still held (e.g. a container write lock during element + // replacement) with another thread blocked on that same lock: stopping + // the world then waits for a thread that cannot reach a safepoint. + // Closing it fully would require making those locks stop-the-world + // aware; the exclusion above only serializes the fork/GC requesters. + #[cfg(feature = "threading")] + let mut stw = CollectStopTheWorld::new(); + // Step 1: Gather objects from generations 0..=generation // Hold read locks for the entire scan to prevent concurrent modifications. let gen_locks: Vec<_> = (0..=generation) @@ -532,6 +628,40 @@ impl GcState { // Step 5: Find unreachable objects let unreachable: Vec = collecting.difference(&reachable).copied().collect(); + // With the world stopped, every frame on any thread's call stack is a + // live root that is externally referenced and must have been + // classified reachable. A running frame appearing in `unreachable` + // would mean the reachability analysis observed its interpreter state + // as garbage — the exact hazard the barrier exists to prevent. + #[cfg(all(unix, feature = "threading", debug_assertions))] + if stw.stopped { + let unreachable_set: HashSet = unreachable.iter().copied().collect(); + crate::vm::thread::try_with_current_vm(|vm| { + let registry = vm.state.thread_frames.lock(); + #[expect( + clippy::iter_over_hash_type, + reason = "assertion over every registered thread slot" + )] + for slot in registry.values() { + let mut cur = slot.top_frame.load(core::sync::atomic::Ordering::Relaxed) + as *const crate::frame::Frame; + while !cur.is_null() { + // SAFETY: frames on a thread's active call stack are + // alive, and the world is stopped so none can be popped. + let obj = + unsafe { &*crate::Py::::from_payload_ptr(cur) } + .as_object(); + let ptr = GcPtr(NonNull::from(obj)); + debug_assert!( + !unreachable_set.contains(&ptr), + "running frame {obj:p} classified unreachable during GC" + ); + cur = unsafe { (*cur).previous_frame() }; + } + } + }); + } + if debug.contains(GcDebugFlags::STATS) { eprintln!( "gc: {} reachable, {} unreachable", @@ -565,6 +695,14 @@ impl GcState { }) .collect(); + // The pointer-reading phases are done: strong references now pin every + // survivor and unreachable object, so the remaining phases can run with + // the world restarted. Finalizers and tp_clear must not run under + // stop-the-world — they execute arbitrary Python — and they only touch + // dead/husk objects, never a running frame. + #[cfg(feature = "threading")] + stw.restart(); + if unreachable.is_empty() { drop(gen_locks); self.promote_survivors(generation, &survivor_refs); @@ -727,11 +865,88 @@ impl GcState { if !truly_dead.is_empty() { // Break cycles by clearing references (tp_clear) // Use deferred drop context to prevent stack overflow. - rustpython_common::refcount::with_deferred_drops(|| { + // With DEBUG_SAVEALL the objects stay reachable through + // gc.garbage, so they must not be cleared (delete_garbage + // skips tp_clear for saved objects). + let save_all = debug.contains(GcDebugFlags::SAVEALL); + + // Untrack dead objects BEFORE clearing them, mirroring the + // untrack-then-clear ordering of the refcount dealloc path. + // A cleared object (e.g. a frame husk with iframe == None) must + // never be observable through the generation lists, or another + // thread could obtain a strong reference via gc.get_objects() + // and access the cleared payload. + let mut late_resurrected: HashSet = HashSet::new(); + if !save_all { + let mut expected_counts: std::collections::HashMap = + std::collections::HashMap::new(); + for obj_ref in &truly_dead { + let obj = obj_ref.as_ref(); + if obj.is_gc_tracked() { + unsafe { self.untrack_object(NonNull::from(obj)) }; + } + // One strong reference held by the `truly_dead` vec itself. + expected_counts.insert(GcPtr(NonNull::from(obj)), 1); + } + // With the objects out of the generation lists, no new external + // reference can appear. Count the references coming from within + // the dead set; any surplus in strong_count means another thread + // grabbed a reference before untracking (late resurrection) and + // the object must not be cleared. + let mut referents: std::collections::HashMap>> = + std::collections::HashMap::new(); for obj_ref in &truly_dead { - if obj_ref.gc_has_clear() { - let edges = unsafe { obj_ref.gc_clear() }; - drop(edges); + let referent_ptrs = unsafe { obj_ref.gc_get_referent_ptrs() }; + for child_ptr in &referent_ptrs { + if let Some(n) = expected_counts.get_mut(&GcPtr(*child_ptr)) { + *n += 1; + } + } + referents.insert(GcPtr(NonNull::from(obj_ref.as_ref())), referent_ptrs); + } + let mut worklist: Vec = Vec::new(); + for obj_ref in &truly_dead { + let ptr = GcPtr(NonNull::from(obj_ref.as_ref())); + if obj_ref.strong_count() > expected_counts[&ptr] + && late_resurrected.insert(ptr) + { + worklist.push(ptr); + } + } + // A holder of a late-resurrected object can reach its referents, + // so everything reachable from it must stay intact as well. + while let Some(ptr) = worklist.pop() { + let Some(referent_ptrs) = referents.get(&ptr) else { + continue; + }; + for child_ptr in referent_ptrs { + let child = GcPtr(*child_ptr); + if expected_counts.contains_key(&child) && late_resurrected.insert(child) { + worklist.push(child); + } + } + } + // Re-track late-resurrected objects so a future collection can + // retry once the external references are released. + #[expect( + clippy::iter_over_hash_type, + reason = "Iteration order doesn't matter here" + )] + for &ptr in &late_resurrected { + unsafe { self.track_object(ptr.0) }; + } + } + rustpython_common::refcount::with_deferred_drops(|| { + if !save_all { + for obj_ref in &truly_dead { + let obj = obj_ref.as_ref(); + if late_resurrected.contains(&GcPtr(NonNull::from(obj))) { + continue; + } + if obj.gc_has_clear() { + let edges = unsafe { obj.gc_clear() }; + drop(edges); + } } } drop(truly_dead); diff --git a/crates/vm/src/object/core.rs b/crates/vm/src/object/core.rs index 88ca646a4f1..f643be7e1fa 100644 --- a/crates/vm/src/object/core.rs +++ b/crates/vm/src/object/core.rs @@ -95,9 +95,23 @@ mod trashcan { type DeallocFn = unsafe fn(*mut super::PyObject); type DeallocQueue = Vec<(*mut super::PyObject, DeallocFn)>; + /// Per-thread trashcan state. Depth and deferral queue live in one + /// thread-local so a single access reaches both fields (one `_tlv_get_addr` + /// on platforms where thread-local access is a function call). Both fields + /// are `Cell`-based so reentrant deallocation (nested `begin`/`end` triggered + /// by draining deferred objects) never holds an outstanding borrow. + struct Trashcan { + depth: Cell, + queue: Cell, + } + thread_local! { - static DEALLOC_DEPTH: Cell = const { Cell::new(0) }; - static DEALLOC_QUEUE: Cell = const { Cell::new(Vec::new()) }; + static TRASHCAN: Trashcan = const { + Trashcan { + depth: Cell::new(0), + queue: Cell::new(Vec::new()), + } + }; } /// Try to begin deallocation. Returns true if we should proceed, @@ -107,18 +121,16 @@ mod trashcan { obj: *mut super::PyObject, dealloc: unsafe fn(*mut super::PyObject), ) -> bool { - DEALLOC_DEPTH.with(|d| { - let depth = d.get(); + TRASHCAN.with(|t| { + let depth = t.depth.get(); if depth >= TRASHCAN_LIMIT { // Depth exceeded: defer this deallocation - DEALLOC_QUEUE.with(|q| { - let mut queue = q.take(); - queue.push((obj, dealloc)); - q.set(queue); - }); + let mut queue = t.queue.take(); + queue.push((obj, dealloc)); + t.queue.set(queue); false } else { - d.set(depth + 1); + t.depth.set(depth + 1); true } }) @@ -127,29 +139,30 @@ mod trashcan { /// End deallocation and process any deferred objects if at outermost level. #[inline] pub(super) unsafe fn end() { - let depth = DEALLOC_DEPTH.with(|d| { - let depth = d.get(); + TRASHCAN.with(|t| { + let depth = t.depth.get(); debug_assert!(depth > 0, "trashcan::end called without matching begin"); let depth = depth - 1; - d.set(depth); - depth - }); - if depth == 0 { - // Process deferred deallocations iteratively + t.depth.set(depth); + if depth != 0 { + return; + } + // Process deferred deallocations iteratively. The queue is set back + // before each `dealloc` call so a reentrant `begin` can push freely. loop { - let next = DEALLOC_QUEUE.with(|q| { - let mut queue = q.take(); + let next = { + let mut queue = t.queue.take(); let item = queue.pop(); - q.set(queue); + t.queue.set(queue); item - }); + }; if let Some((obj, dealloc)) = next { unsafe { dealloc(obj) }; } else { break; } } - } + }) } } @@ -161,8 +174,17 @@ pub(super) unsafe fn default_dealloc(obj: *mut PyObject) { return; // resurrected by __del__ } + // Only tracked objects take the trashcan recursion guard and untrack path. + // Untracked objects either own no children (int, float, str, ...) or, like + // non-escaped frames, are released at interpreter depth with at most one + // unguarded link before their tracked children (dicts, functions, code) + // re-enter guarded deallocation, so recursion stays bounded. A frame stored + // in an object graph is forced to escape, becoming tracked and guarded here. + // Read once and reuse for both gates below. + let tracked = obj_ref.is_gc_tracked(); + // Trashcan: limit recursive deallocation depth to prevent stack overflow - if !unsafe { trashcan::begin(obj, default_dealloc::) } { + if tracked && !unsafe { trashcan::begin(obj, default_dealloc::) } { return; // deferred to queue } @@ -171,7 +193,7 @@ pub(super) unsafe fn default_dealloc(obj: *mut PyObject) { // Untrack from GC BEFORE deallocation. // Must happen before memory is freed because intrusive list removal // reads the object's gc_pointers (prev/next). - if obj_ref.is_gc_tracked() { + if tracked { let ptr = unsafe { NonNull::new_unchecked(obj) }; unsafe { crate::gc_state::gc_state().untrack_object(ptr); @@ -188,36 +210,49 @@ pub(super) unsafe fn default_dealloc(obj: *mut PyObject) { ); } - // Try to store in freelist for reuse BEFORE tp_clear, so that - // size-based freelists (e.g. PyTuple) can read the payload directly. + // Extract child references to break circular refs (tp_clear), then drop + // them. Some payloads (e.g. Frame) drop children in place inside clear_fn + // instead of extracting them, so user code (`__del__`) may run here. + let mut edges = Vec::new(); + if let Some(clear_fn) = vtable.clear { + unsafe { clear_fn(obj, &mut edges) }; + } + // Drop extracted child references - may trigger recursive destruction. + drop(edges); + + // Try to store in freelist for reuse. This must happen AFTER clear_fn and + // after the extracted-children drop: both can run user code (`__del__`) + // that allocates, and `PyRef::new_ref` pops from the same thread-local + // freelist. If the husk were already in the freelist, a reentrant + // allocation could pop it and write a fresh payload into it while clear_fn + // still holds a `&mut` borrow of that payload (aliasing UB). Pushing only + // once no borrows into the payload can be live closes that window. // Only exact base types (not heaptype or structseq subtypes) go into the freelist. + // Published objects (e.g. a tuple stored as a type attribute) must skip the + // freelist: `PyRef::new_ref` would reuse the slot and overwrite the refcount + // word with a non-atomic write, racing a reader's atomic try-incref. Route + // them through `PyInner::dealloc` instead, whose QSBR hook defers the actual + // memory free until readers can no longer observe it. let typ = obj_ref.class(); let pushed = if T::HAS_FREELIST && typ.heaptype_ext.is_none() && core::ptr::eq(typ, T::class(crate::vm::Context::genesis())) + && !obj_ref.0.ref_count.is_published() { unsafe { T::freelist_push(obj) } } else { false }; - // Extract child references to break circular refs (tp_clear). - // This runs regardless of freelist push — the object's children must be released. - let mut edges = Vec::new(); - if let Some(clear_fn) = vtable.clear { - unsafe { clear_fn(obj, &mut edges) }; - } - if !pushed { // Deallocate the object memory (handles ObjExt prefix if present) unsafe { PyInner::dealloc(obj as *mut PyInner) }; } - // Drop child references - may trigger recursive destruction. - drop(edges); - // Trashcan: decrement depth and process deferred objects at outermost level - unsafe { trashcan::end() }; + if tracked { + unsafe { trashcan::end() }; + } } pub(super) unsafe fn debug_obj( x: &PyObject, @@ -1005,6 +1040,9 @@ impl PyInner { let has_ext = flags.has_feature(crate::types::PyTypeFlags::HAS_DICT) || member_count > 0; let has_weakref = flags.has_feature(crate::types::PyTypeFlags::HAS_WEAKREF); + // Objects published to lock-free caches keep their memory mapped + // until a QSBR grace period passes; destructors still run now. + let published = (*ptr).ref_count.is_published(); if has_ext || has_weakref { // Reconstruct the same layout used in new() @@ -1037,7 +1075,15 @@ impl PyInner { } // WeakRefList has no Drop (just raw pointers), no drop_in_place needed - alloc::alloc::dealloc(alloc_ptr, combined); + if published { + crate::object::qsbr::free_delayed(alloc_ptr, combined); + } else { + alloc::alloc::dealloc(alloc_ptr, combined); + } + } else if published { + let layout = core::alloc::Layout::new::(); + core::ptr::drop_in_place(ptr); + crate::object::qsbr::free_delayed(ptr as *mut u8, layout); } else { drop(Box::from_raw(ptr)); } @@ -1141,11 +1187,6 @@ impl PyInner { } } -/// Returns the allocation layout for `PyInner`, for use in freelist Drop impls. -pub(crate) const fn pyinner_layout() -> core::alloc::Layout { - core::alloc::Layout::new::>() -} - /// Thread-local freelist storage for reusing object allocations. /// /// Wraps a `Vec<*mut PyObject>`. On thread teardown, `Drop` frees raw @@ -1287,6 +1328,13 @@ impl PyObject { None } } + + /// Mark this object as published to a lock-free cache. Its memory + /// reclamation is deferred through QSBR (see `object::qsbr`) so that + /// concurrent try-incref readers never touch freed memory. + pub(crate) fn mark_cache_published(&self) { + self.0.ref_count.mark_published(); + } } impl PyObjectRef { @@ -2085,6 +2133,19 @@ impl Py { pub fn payload(&self) -> &T { &self.0.payload } + + /// Recover the object pointer from a pointer to its `payload` field. + /// + /// # Safety + /// `payload` must point to the `payload` of a live `Py` (e.g. a `&T` + /// obtained by dereferencing a `Py`), and the object must outlive the + /// returned pointer's use. + #[inline] + pub(crate) unsafe fn from_payload_ptr(payload: *const T) -> *const Self { + let offset = core::mem::offset_of!(PyInner, payload); + // `Py` is a newtype over `PyInner`, so their addresses coincide. + unsafe { (payload as *const u8).sub(offset) as *const Self } + } } impl ToOwned for Py { @@ -2275,7 +2336,11 @@ impl PyRef { // - HAS_TRAVERSE is true (Rust payload implements Traverse), OR // - has instance dict (user-defined class instances), OR // - heap type (all heap type instances are GC-tracked, like Py_TPFLAGS_HAVE_GC) - if ::HAS_TRAVERSE || has_dict || is_heaptype { + // unless the payload opts out via NEW_REF_UNTRACKED (e.g. call frames, + // which are tracked lazily only on escape). + if (::HAS_TRAVERSE || has_dict || is_heaptype) + && !T::NEW_REF_UNTRACKED + { let gc = crate::gc_state::gc_state(); unsafe { gc.track_object(ptr.cast()); @@ -2471,7 +2536,7 @@ pub(crate) fn init_type_hierarchy() -> (PyTypeRef, PyTypeRef, PyTypeRef) { static_assertions::assert_eq_align!(MaybeUninit>, PyInner); let type_payload = PyType { - base: None, + base: None.into(), bases: PyRwLock::default(), mro: PyRwLock::default(), subclasses: PyRwLock::default(), @@ -2482,7 +2547,7 @@ pub(crate) fn init_type_hierarchy() -> (PyTypeRef, PyTypeRef, PyTypeRef) { abc_tpflags: core::sync::atomic::AtomicU64::new(0), }; let object_payload = PyType { - base: None, + base: None.into(), bases: PyRwLock::default(), mro: PyRwLock::default(), subclasses: PyRwLock::default(), @@ -2567,7 +2632,7 @@ pub(crate) fn init_type_hierarchy() -> (PyTypeRef, PyTypeRef, PyTypeRef) { (*object_type_ptr).payload.mro = PyRwLock::new(vec![object_type.clone()]); (*type_type_ptr).payload.bases = PyRwLock::new(vec![object_type.clone()]); - (*type_type_ptr).payload.base = Some(object_type.clone()); + (*type_type_ptr).payload.base = Some(object_type.clone()).into(); let type_type = PyTypeRef::from_raw(type_type_ptr.cast()); // type's mro is [type, object] @@ -2579,7 +2644,7 @@ pub(crate) fn init_type_hierarchy() -> (PyTypeRef, PyTypeRef, PyTypeRef) { }; let weakref_type = PyType { - base: Some(object_type.clone()), + base: Some(object_type.clone()).into(), bases: PyRwLock::new(vec![object_type.clone()]), mro: PyRwLock::new(vec![object_type.clone()]), subclasses: PyRwLock::default(), diff --git a/crates/vm/src/object/ext.rs b/crates/vm/src/object/ext.rs index d400de29c38..e576ac2c191 100644 --- a/crates/vm/src/object/ext.rs +++ b/crates/vm/src/object/ext.rs @@ -365,6 +365,35 @@ impl PyAtomicRef> { self.deref_ordering(ordering).map(|x| x.to_owned()) } + /// Try-incref read of the current value. + /// + /// Unlike [`Self::to_owned`], this never increfs a destructed object: + /// it uses a conditional incref and revalidates that the slot still + /// holds the same pointer. Returns `None` when the slot is empty. + /// + /// Soundness relies on published-object memory being reclaimed only + /// after a QSBR grace period (see `object::qsbr`), so the refcount + /// word of a concurrently swapped-out value stays readable. + pub fn try_to_owned(&self, ordering: Ordering) -> Option> { + loop { + let ptr = self.inner.load(ordering); + if ptr.is_null() { + return None; + } + if let Some(obj) = unsafe { PyObject::try_to_owned_from_ptr(ptr.cast::()) } { + if core::ptr::eq(self.inner.load(Ordering::Acquire), ptr) { + // SAFETY: the slot only ever stores `PyRef` values. + return Some(unsafe { obj.downcast_unchecked::() }); + } + drop(obj); + } + // Slot changed or the value was torn down mid-read; a failed + // incref with an unchanged slot is impossible (the slot's own + // strong ref keeps the value alive), so this loop progresses. + core::hint::spin_loop(); + } + } + /// # Safety /// The caller is responsible to keep the returned PyRef alive /// until no more reference can be used via PyAtomicRef::deref() diff --git a/crates/vm/src/object/mod.rs b/crates/vm/src/object/mod.rs index 56db97aef1d..b06957e1bc6 100644 --- a/crates/vm/src/object/mod.rs +++ b/crates/vm/src/object/mod.rs @@ -1,6 +1,7 @@ mod core; mod ext; mod payload; +pub(crate) mod qsbr; mod traverse; mod traverse_object; diff --git a/crates/vm/src/object/payload.rs b/crates/vm/src/object/payload.rs index 349b239f79f..36262607a1a 100644 --- a/crates/vm/src/object/payload.rs +++ b/crates/vm/src/object/payload.rs @@ -48,6 +48,13 @@ pub trait PyPayload: MaybeTraverse + PyThreadingConstraint + Sized + 'static { fn class(ctx: &Context) -> &'static Py; + /// Whether `PyRef::new_ref` skips auto-tracking this type in the GC even + /// when it would otherwise qualify (has traverse, dict, or heap type). + /// Such objects are created untracked and must be tracked explicitly if + /// and when they can become part of a reference cycle. Used by `Frame`, + /// which is created untracked and tracked lazily only on escape. + const NEW_REF_UNTRACKED: bool = false; + /// Whether this type has a freelist. Types with freelists require /// immediate (non-deferred) GC untracking during dealloc to prevent /// race conditions when the object is reused. @@ -58,11 +65,13 @@ pub trait PyPayload: MaybeTraverse + PyThreadingConstraint + Sized + 'static { /// Try to push a dead object onto this type's freelist for reuse. /// Returns true if the object was stored (caller must NOT free the memory). - /// Called before tp_clear, so the payload is still intact. + /// Called after tp_clear, so the payload is a cleared husk; implementations + /// must not rely on its pre-clear contents. /// /// # Safety - /// `obj` must be a valid pointer to a `PyInner` with refcount 0. - /// The payload is still initialized and can be read for bucket selection. + /// `obj` must be a valid pointer to a `PyInner` with refcount 0 + /// whose tp_clear has already run, with no outstanding borrows into the + /// payload (`PyRef::new_ref` may pop and reuse the husk immediately). #[inline] unsafe fn freelist_push(_obj: *mut PyObject) -> bool { false @@ -124,6 +133,34 @@ pub trait PyPayload: MaybeTraverse + PyThreadingConstraint + Sized + 'static { #[inline] fn into_ref_with_type(self, vm: &VirtualMachine, cls: PyTypeRef) -> PyResult> + where + Self: core::fmt::Debug, + { + self.into_ref_with_type_and_dict(vm, cls, true) + } + + /// Like `into_ref_with_type`, but leaves the instance `__dict__` unallocated + /// until the first attribute write or `__dict__` access. Only valid for types + /// whose attribute protocol materializes the dict lazily via `get_or_insert`. + #[inline] + fn into_ref_with_type_lazy_dict( + self, + vm: &VirtualMachine, + cls: PyTypeRef, + ) -> PyResult> + where + Self: core::fmt::Debug, + { + self.into_ref_with_type_and_dict(vm, cls, false) + } + + #[inline] + fn into_ref_with_type_and_dict( + self, + vm: &VirtualMachine, + cls: PyTypeRef, + eager_dict: bool, + ) -> PyResult> where Self: core::fmt::Debug, { @@ -145,7 +182,12 @@ pub trait PyPayload: MaybeTraverse + PyThreadingConstraint + Sized + 'static { } return Err(_into_ref_size_error(vm, &cls, exact_class)); } - Ok(self._into_ref(cls, &vm.ctx)) + let dict = if eager_dict && cls.slots.flags.has_feature(PyTypeFlags::HAS_DICT) { + Some(vm.ctx.new_dict()) + } else { + None + }; + Ok(PyRef::new_ref(self, cls, dict)) } else { #[cold] #[inline(never)] diff --git a/crates/vm/src/object/qsbr.rs b/crates/vm/src/object/qsbr.rs new file mode 100644 index 00000000000..576cadaeaed --- /dev/null +++ b/crates/vm/src/object/qsbr.rs @@ -0,0 +1,333 @@ +//! Quiescent-state-based reclamation (QSBR) for lock-free caches. +//! +//! Objects published to lock-free caches (type method cache, type +//! specialization caches) are read via borrowed pointers plus try-incref. +//! Their memory must stay mapped until every thread that could hold such a +//! borrowed pointer has passed a quiescent state. Destructors run at the +//! normal drop point; only the final deallocation is deferred. +//! +//! Mirrors _Py_qsbr (Python/qsbr.c): a global write sequence advances on +//! each retirement; each thread records the last sequence it observed at a +//! quiescent point (eval-breaker checkpoint, attach/detach). A retired +//! allocation is freed once every online thread's sequence passes its goal. + +use core::alloc::Layout; + +/// Sequence value of an offline (detached) thread. +#[cfg(feature = "threading")] +const QSBR_OFFLINE: u64 = 0; +/// Initial write sequence value. +#[cfg(feature = "threading")] +const QSBR_INITIAL: u64 = 1; +/// Write sequence increment. +#[cfg(feature = "threading")] +const QSBR_INCR: u64 = 2; + +#[cfg(feature = "threading")] +pub(crate) use threading::*; + +#[cfg(feature = "threading")] +mod threading { + use super::*; + use alloc::sync::{Arc, Weak}; + use core::sync::atomic::{AtomicBool, AtomicU64, Ordering}; + use std::sync::Mutex; + + /// Per-thread QSBR state, owned by the thread's `ThreadSlot`. + pub(crate) struct QsbrSlot { + /// Last write sequence observed at a quiescent point; + /// `QSBR_OFFLINE` while the thread is detached. + seq: AtomicU64, + /// Set when this thread should pass a checkpoint (eval-breaker bit). + pub(crate) requested: AtomicBool, + } + + struct Retired { + ptr: *mut u8, + layout: Layout, + goal: u64, + } + // SAFETY: `ptr` is an exclusively owned dead allocation; only the + // processing thread touches it. + unsafe impl Send for Retired {} + + pub(crate) struct Qsbr { + /// Global write sequence (_Py_qsbr wr_seq). + wr_seq: AtomicU64, + /// Cached minimum observed read sequence (rd_seq). + rd_seq: AtomicU64, + threads: Mutex>>, + queue: Mutex>, + /// Set while the retire queue is non-empty; gates the per-instruction + /// eval-breaker check so the hot path pays only one relaxed static + /// load when nothing is pending. + pending: AtomicBool, + } + + pub(crate) static QSBR: Qsbr = Qsbr::new(); + + impl Qsbr { + const fn new() -> Self { + Self { + wr_seq: AtomicU64::new(QSBR_INITIAL), + rd_seq: AtomicU64::new(QSBR_INITIAL), + threads: Mutex::new(Vec::new()), + queue: Mutex::new(Vec::new()), + pending: AtomicBool::new(false), + } + } + + /// Whether retired allocations are pending. The hot path now reads + /// the mirrored bit in the eval-breaker word instead; this stays + /// only for unit tests that exercise local, non-global instances. + #[inline] + #[cfg_attr(not(test), allow(dead_code))] + pub(crate) fn break_pending(&self) -> bool { + self.pending.load(Ordering::Relaxed) + } + + /// Mirror `pending` into the global eval-breaker word — only for + /// the global QSBR instance, so unit-test instances never touch + /// process-global state. + fn update_breaker_bit(&self, on: bool) { + if core::ptr::eq(self, &QSBR) { + if on { + crate::signal::set_qsbr_bit(); + } else { + crate::signal::clear_qsbr_bit(); + } + } + } + + /// Register the calling thread. The returned slot is stored in the + /// thread's `ThreadSlot`; dropping it unregisters the thread. + pub(crate) fn register(&self) -> Arc { + let slot = Arc::new(QsbrSlot { + seq: AtomicU64::new(self.wr_seq.load(Ordering::Acquire)), + requested: AtomicBool::new(false), + }); + self.threads.lock().unwrap().push(Arc::downgrade(&slot)); + slot + } + + /// Advance the write sequence; returns the goal a retirement must + /// wait for (_Py_qsbr_advance). + fn advance(&self) -> u64 { + self.wr_seq.fetch_add(QSBR_INCR, Ordering::AcqRel) + QSBR_INCR + } + + /// Record that the calling thread is at a quiescent point: it holds + /// no borrowed cache pointers (_Py_qsbr_quiescent_state). + pub(crate) fn quiescent_state(&self, slot: &QsbrSlot) { + slot.seq + .store(self.wr_seq.load(Ordering::Acquire), Ordering::Release); + } + + /// Mark a thread offline (detached); it no longer delays grace + /// periods (_Py_qsbr_detach). The thread must not perform lock-free + /// cache reads while offline. + pub(crate) fn offline(&self, slot: &QsbrSlot) { + slot.seq.store(QSBR_OFFLINE, Ordering::Release); + } + + /// Mark a thread online again (_Py_qsbr_attach). + pub(crate) fn online(&self, slot: &QsbrSlot) { + self.quiescent_state(slot); + } + + /// Whether every online thread has passed `goal` (_Py_qsbr_poll). + fn poll(&self, goal: u64) -> bool { + if self.rd_seq.load(Ordering::Acquire) >= goal { + return true; + } + self.poll_scan() >= goal + } + + /// Recompute the minimum sequence over all live online threads, + /// pruning dead ones. + fn poll_scan(&self) -> u64 { + let mut min_seq = self.wr_seq.load(Ordering::Acquire); + let mut threads = self.threads.lock().unwrap(); + threads.retain(|weak| match weak.upgrade() { + Some(slot) => { + let seq = slot.seq.load(Ordering::Acquire); + if seq != QSBR_OFFLINE { + min_seq = min_seq.min(seq); + } + true + } + None => false, + }); + drop(threads); + self.rd_seq.fetch_max(min_seq, Ordering::AcqRel); + min_seq + } + + /// Defer deallocation of a dead object's memory until a grace + /// period passes (_PyMem_FreeDelayed). + /// + /// # Safety + /// `ptr`/`layout` must describe an allocation whose contents have + /// been dropped and which nothing accesses afterwards except the + /// racing try-incref reads this mechanism protects against. + pub(crate) unsafe fn free_delayed(&self, ptr: *mut u8, layout: Layout) { + let goal = self.advance(); + { + let mut queue = self.queue.lock().unwrap(); + queue.push(Retired { ptr, layout, goal }); + // Set while still holding the queue lock, so this pairs with + // `process` clearing the flag under the same lock and no + // push can be left behind with the flag cleared. + self.pending.store(true, Ordering::Release); + self.update_breaker_bit(true); + } + // Ask every registered thread to pass a checkpoint. + for weak in self.threads.lock().unwrap().iter() { + if let Some(slot) = weak.upgrade() { + slot.requested.store(true, Ordering::Release); + } + } + } + + /// Free retired allocations whose grace period has passed + /// (_PyMem_ProcessDelayed). + pub(crate) fn process(&self) { + let Ok(mut queue) = self.queue.try_lock() else { + // Another thread is already processing. + return; + }; + // Goals are usually increasing in push order, but concurrent + // `free_delayed` calls can interleave their `advance()` and + // queue push, so a smaller goal can occasionally land behind a + // larger one. Free the longest prefix whose grace period has + // passed; each drained item individually passed `poll`, so this + // is sound regardless of ordering. A goal stuck behind an + // out-of-order neighbor just waits for the next checkpoint or + // GC pass, not a correctness issue. + let safe_prefix = queue + .iter() + .position(|item| !self.poll(item.goal)) + .unwrap_or(queue.len()); + for item in queue.drain(..safe_prefix) { + // SAFETY: grace period passed; no reader can hold `ptr`. + unsafe { alloc::alloc::dealloc(item.ptr, item.layout) }; + } + if queue.is_empty() { + self.pending.store(false, Ordering::Release); + self.update_breaker_bit(false); + } + } + + /// Free all retired allocations immediately. + /// + /// # Safety + /// Only sound when no other thread can be mid-read: the post-fork + /// child, or teardown after all threads exited. + #[cfg(unix)] + pub(crate) unsafe fn drain_all(&self) { + let mut queue = self.queue.lock().unwrap(); + for item in queue.drain(..) { + // SAFETY: guaranteed single-threaded by the caller. + unsafe { alloc::alloc::dealloc(item.ptr, item.layout) }; + } + self.pending.store(false, Ordering::Release); + self.update_breaker_bit(false); + } + + /// Reset after fork: drop all registered thread entries (dead + /// parent threads' slots would otherwise stay online forever and + /// stall every future grace period) and free all retired + /// allocations. + /// + /// # Safety + /// Only sound in the single-threaded post-fork child, before the + /// surviving thread re-registers. + #[cfg(unix)] + pub(crate) unsafe fn reset_after_fork(&self) { + self.threads.lock().unwrap().clear(); + // SAFETY: single-threaded child, no concurrent reader exists. + unsafe { self.drain_all() }; + } + + #[cfg(test)] + fn pending(&self) -> usize { + self.queue.lock().unwrap().len() + } + } + + #[cfg(test)] + mod tests { + use super::*; + + #[test] + fn poll_requires_all_online_threads() { + let q = Qsbr::new(); + let a = q.register(); + let b = q.register(); + let goal = q.advance(); + assert!(!q.poll(goal)); + q.quiescent_state(&a); + assert!(!q.poll(goal)); + q.quiescent_state(&b); + assert!(q.poll(goal)); + } + + #[test] + fn offline_thread_does_not_delay_grace() { + let q = Qsbr::new(); + let a = q.register(); + let b = q.register(); + let goal = q.advance(); + q.quiescent_state(&a); + q.offline(&b); + assert!(q.poll(goal)); + } + + #[test] + fn dead_thread_is_pruned() { + let q = Qsbr::new(); + let a = q.register(); + let b = q.register(); + drop(b); + let goal = q.advance(); + q.quiescent_state(&a); + assert!(q.poll(goal)); + } + + #[test] + fn process_frees_only_after_grace() { + let q = Qsbr::new(); + let a = q.register(); + let layout = Layout::new::(); + let ptr = unsafe { alloc::alloc::alloc(layout) }; + unsafe { q.free_delayed(ptr, layout) }; + assert!(a.requested.load(Ordering::Acquire)); + assert!(q.break_pending()); + q.process(); + assert_eq!(q.pending(), 1); // grace period not passed yet + assert!(q.break_pending()); + q.quiescent_state(&a); + q.process(); + assert_eq!(q.pending(), 0); + assert!(!q.break_pending()); + } + } +} + +/// Defer (threading) or immediately perform (non-threading) deallocation +/// of a dead published object's memory. +/// +/// # Safety +/// Same contract as [`Qsbr::free_delayed`]. +#[inline] +pub(crate) unsafe fn free_delayed(ptr: *mut u8, layout: Layout) { + #[cfg(feature = "threading")] + unsafe { + QSBR.free_delayed(ptr, layout) + }; + #[cfg(not(feature = "threading"))] + // No concurrent readers can exist without threads. + unsafe { + alloc::alloc::dealloc(ptr, layout) + }; +} diff --git a/crates/vm/src/object/traverse.rs b/crates/vm/src/object/traverse.rs index 9a5ae324baf..d0a20d2afa7 100644 --- a/crates/vm/src/object/traverse.rs +++ b/crates/vm/src/object/traverse.rs @@ -111,19 +111,25 @@ where unsafe impl Traverse for PyRwLock { #[inline] fn traverse(&self, traverse_fn: &mut TraverseFn<'_>) { - // if can't get a lock, this means something else is holding the lock, - // but since gc stopped the world, during gc the lock is always held - // so it is safe to ignore those in gc + // A failed try_read means a writer holds the lock. Traversal runs with + // the world stopped, but a thread force-parked while DETACHED (CAS'd + // straight to SUSPENDED from native code) may still hold the write lock + // it was in the middle of taking. Skipping such an object is safe: the + // collector then does not see its outgoing edges, which only + // under-traverses and thus over-approximates liveness (a conservative + // keep-alive), never freeing a reachable object. In single-threaded + // builds a failure only reflects the current thread's own re-entrant + // read, likewise safely skipped. if let Some(inner) = self.try_read_recursive() { inner.traverse(traverse_fn) } } } -/// Safety: We can't hold lock during traverse it's child because it may cause deadlock. -/// TODO(discord9): check if this is thread-safe to do -/// (Outside of gc phase, only incref/decref will call trace, -/// and refcnt is atomic, so it should be fine?) +/// Safety: the lock is not held across visiting children to avoid a re-entrant +/// deadlock. In threading builds traversal runs under stop-the-world so no +/// other thread mutates the guarded value while we read it; in single-threaded +/// builds there is no other writer. unsafe impl Traverse for PyMutex { #[inline] fn traverse(&self, traverse_fn: &mut TraverseFn<'_>) { @@ -135,7 +141,9 @@ unsafe impl Traverse for PyMutex { } chs.iter() .map(|ch| { - // Safety: during gc, this should be fine, because nothing should write during gc's tracing? + // Safety: the world is stopped (threading builds) or the + // interpreter is single-threaded, so `ch` is not concurrently + // freed while we hand it to the tracer. let ch = unsafe { ch.as_ref() }; traverse_fn(ch); }) diff --git a/crates/vm/src/signal.rs b/crates/vm/src/signal.rs index eea42f4a87e..16a097ea62b 100644 --- a/crates/vm/src/signal.rs +++ b/crates/vm/src/signal.rs @@ -2,7 +2,7 @@ use core::{ cell::{Cell, RefCell}, fmt, ops::{Deref, DerefMut, Index, IndexMut, Range}, - sync::atomic::{AtomicBool, Ordering}, + sync::atomic::{AtomicBool, AtomicU8, Ordering}, }; use std::sync::mpsc; @@ -13,7 +13,21 @@ use crate::{PyObjectRef, PyResult, TryFromBorrowedObject, TryFromObject, Virtual pub(crate) const NSIG: usize = 64; -static ANY_TRIGGERED: AtomicBool = AtomicBool::new(false); +/// Eval-breaker word: bit flags checked once per bytecode instruction. +/// Signal handlers and QSBR set bits with fetch_or (async-signal-safe, +/// lock-free); consumers clear only their own bit with fetch_and. +static EVAL_BREAKER: AtomicU8 = AtomicU8::new(0); + +/// A signal handler recorded a pending signal. +const SIGNAL_BIT: u8 = 1 << 0; +/// QSBR has retired allocations pending reclamation. +#[cfg(feature = "threading")] +const QSBR_BIT: u8 = 1 << 1; +/// An automatic collection was scheduled by `maybe_collect` and must run at +/// the next bytecode safepoint rather than synchronously inside the +/// allocation that tripped the threshold. +#[cfg(feature = "threading")] +const GC_BIT: u8 = 1 << 2; #[expect( clippy::declare_interior_mutable_const, @@ -49,12 +63,12 @@ pub fn check_signals(vm: &VirtualMachine) -> PyResult<()> { // Read-only check first: avoids cache-line invalidation on every // instruction when no signal is pending (the common case). - if !ANY_TRIGGERED.load(Ordering::Relaxed) { + if EVAL_BREAKER.load(Ordering::Relaxed) & SIGNAL_BIT == 0 { return Ok(()); } // Atomic RMW only when a signal is actually pending. - if !ANY_TRIGGERED.swap(false, Ordering::Acquire) { + if EVAL_BREAKER.fetch_and(!SIGNAL_BIT, Ordering::Acquire) & SIGNAL_BIT == 0 { return Ok(()); } @@ -101,20 +115,49 @@ fn trigger_signals(vm: &VirtualMachine) -> PyResult<()> { } pub(crate) fn set_triggered() { - ANY_TRIGGERED.store(true, Ordering::Release); + // fetch_or (not store) so a signal handler never clobbers the QSBR bit; + // this compiles to a lock-free RMW, safe to call from a signal handler. + EVAL_BREAKER.fetch_or(SIGNAL_BIT, Ordering::Release); } +/// Any eval-breaker bit pending? One relaxed load; checked per instruction. #[inline(always)] -#[cfg(not(target_arch = "wasm32"))] -pub(crate) fn is_triggered() -> bool { - ANY_TRIGGERED.load(Ordering::Relaxed) +pub(crate) fn eval_breaker_pending() -> bool { + EVAL_BREAKER.load(Ordering::Relaxed) != 0 +} + +#[cfg(feature = "threading")] +pub(crate) fn set_qsbr_bit() { + EVAL_BREAKER.fetch_or(QSBR_BIT, Ordering::Release); +} + +#[cfg(feature = "threading")] +pub(crate) fn clear_qsbr_bit() { + EVAL_BREAKER.fetch_and(!QSBR_BIT, Ordering::Release); +} + +#[cfg(feature = "threading")] +pub(crate) fn qsbr_bit_set() -> bool { + EVAL_BREAKER.load(Ordering::Relaxed) & QSBR_BIT != 0 +} + +/// Schedule an automatic collection to run at the next bytecode safepoint. +#[cfg(feature = "threading")] +pub(crate) fn schedule_gc() { + EVAL_BREAKER.fetch_or(GC_BIT, Ordering::Release); +} + +/// Clear the scheduled-GC bit, returning whether it had been set. +#[cfg(feature = "threading")] +pub(crate) fn take_gc_scheduled() -> bool { + EVAL_BREAKER.fetch_and(!GC_BIT, Ordering::Acquire) & GC_BIT != 0 } /// Reset all signal trigger state after fork in child process. /// Stale triggers from the parent must not fire in the child. #[cfg(all(unix, feature = "host_env"))] pub(crate) fn clear_after_fork() { - ANY_TRIGGERED.store(false, Ordering::Release); + EVAL_BREAKER.fetch_and(!SIGNAL_BIT, Ordering::Release); for trigger in &TRIGGERS { trigger.store(false, Ordering::Relaxed); } diff --git a/crates/vm/src/stdlib/_imp.rs b/crates/vm/src/stdlib/_imp.rs index 7021895c9f7..322eaedd7d0 100644 --- a/crates/vm/src/stdlib/_imp.rs +++ b/crates/vm/src/stdlib/_imp.rs @@ -15,8 +15,14 @@ mod lock { static IMP_LOCK: RawRMutex = RawRMutex::INIT; #[pyfunction] - fn acquire_lock(_vm: &VirtualMachine) { - acquire_lock_for_fork() + fn acquire_lock(vm: &VirtualMachine) { + // Detach while blocking on IMP_LOCK. The import lock is held across + // bytecode by the importlib bootstrap, so its holder can be parked at a + // safepoint mid-hold. Blocking here while attached would keep this + // thread from honoring a stop-the-world request, so a requester could + // wait for this thread while this thread waits for the parked holder. + // Detaching makes the wait park-friendly. + vm.allow_threads(acquire_lock_for_fork); } #[pyfunction] @@ -76,9 +82,14 @@ mod lock { } /// Re-export for fork safety code in posix.rs +/// +/// Runs pre-fork on a normal attached VM thread. Detach while blocking so the +/// wait honors a concurrent stop-the-world request instead of pinning this +/// thread attached on IMP_LOCK; re-attach completes before `stop_the_world`, so +/// the fork requester protocol is unaffected. #[cfg(all(unix, feature = "threading", feature = "host_env"))] -pub(crate) fn acquire_imp_lock_for_fork() { - lock::acquire_lock_for_fork(); +pub(crate) fn acquire_imp_lock_for_fork(vm: &VirtualMachine) { + vm.allow_threads(lock::acquire_lock_for_fork); } #[cfg(all(unix, feature = "threading", feature = "host_env"))] diff --git a/crates/vm/src/stdlib/_thread.rs b/crates/vm/src/stdlib/_thread.rs index f3e6bec898f..99f9b1787ed 100644 --- a/crates/vm/src/stdlib/_thread.rs +++ b/crates/vm/src/stdlib/_thread.rs @@ -1053,18 +1053,46 @@ pub(crate) mod _thread { /// Get all threads' current (top) frames. Used by sys._current_frames(). pub(crate) fn get_all_current_frames(vm: &VirtualMachine) -> Vec<(u64, FrameRef)> { - let registry = vm.state.thread_frames.lock(); - registry - .iter() - .filter_map(|(id, slot)| { - let frames = slot.frames.lock(); - // SAFETY: the owning thread can't pop while we hold the Mutex, - // so the FramePtr is valid for the duration of the lock. - frames - .last() - .map(|fp| (*id, unsafe { fp.as_ref() }.to_owned())) - }) - .collect() + // unix: read each thread's published top frame under stop-the-world so + // the owning thread is parked at a safepoint and cannot pop or free the + // frame while we take a strong reference. Request stop-the-world before + // the registry lock to avoid deadlocking a thread parking mid-registry. + #[cfg(unix)] + { + use core::sync::atomic::Ordering; + vm.state.stop_the_world.stop_the_world(vm); + scopeguard::defer! { vm.state.stop_the_world.start_the_world(vm); } + let registry = vm.state.thread_frames.lock(); + registry + .iter() + .filter_map(|(id, slot)| { + let top = slot.top_frame.load(Ordering::Relaxed); + core::ptr::NonNull::new(top).map(|p| { + // SAFETY: world stopped -> the owning thread is parked + // and cannot pop or free this frame; it is alive on + // that thread's call stack. + let py = + unsafe { &*Py::::from_payload_ptr(p.as_ptr()) }; + (*id, py.to_owned()) + }) + }) + .collect() + } + #[cfg(not(unix))] + { + let registry = vm.state.thread_frames.lock(); + registry + .iter() + .filter_map(|(id, slot)| { + let frames = slot.frames.lock(); + // SAFETY: the owning thread can't pop while we hold the Mutex, + // so the FramePtr is valid for the duration of the lock. + frames + .last() + .map(|fp| (*id, unsafe { fp.as_ref() }.to_owned())) + }) + .collect() + } } /// Called after fork() in child process to mark all other threads as done. @@ -1659,17 +1687,22 @@ pub(crate) mod _thread { started_cvar.notify_all(); } // Don't execute the target function until parent marks the - // handle as running. + // handle as running. Detach while blocked so a concurrent + // stop-the-world (e.g. a GC on another thread) can park this + // thread instead of stalling waiting for it to reach a + // safepoint it will not reach until released. { let (ready_lock, ready_cvar) = &*handle_ready_event_clone; - let mut ready = ready_lock.lock().unwrap(); - while !*ready { - // Short timeout so we stay responsive to STW requests. - let (guard, _) = ready_cvar - .wait_timeout(ready, core::time::Duration::from_millis(1)) - .unwrap(); - ready = guard; - } + vm.allow_threads(|| { + let mut ready = ready_lock.lock().unwrap(); + while !*ready { + // Short timeout so we stay responsive to STW requests. + let (guard, _) = ready_cvar + .wait_timeout(ready, core::time::Duration::from_millis(1)) + .unwrap(); + ready = guard; + } + }); } // Ensure cleanup happens even if the function panics @@ -1750,16 +1783,22 @@ pub(crate) mod _thread { vm.new_runtime_error("can't start new thread") })?; - // Wait until the new thread has reported its ident. + // Wait until the new thread has reported its ident. Detach while + // waiting so a concurrent stop-the-world (e.g. a GC on another thread) + // can park this thread instead of stalling on it: the child may park + // itself at startup while the world is stopped and cannot report until + // released, so the waiter must be parkable too. { let (started_lock, started_cvar) = &*started_event; - let mut started = started_lock.lock().unwrap(); - while !*started { - let (guard, _) = started_cvar - .wait_timeout(started, core::time::Duration::from_millis(1)) - .unwrap(); - started = guard; - } + vm.allow_threads(|| { + let mut started = started_lock.lock().unwrap(); + while !*started { + let (guard, _) = started_cvar + .wait_timeout(started, core::time::Duration::from_millis(1)) + .unwrap(); + started = guard; + } + }); } // Mark the handle running in the parent thread (like CPython's diff --git a/crates/vm/src/stdlib/_winapi.rs b/crates/vm/src/stdlib/_winapi.rs index f6faf6a8a95..e1b86f2ef54 100644 --- a/crates/vm/src/stdlib/_winapi.rs +++ b/crates/vm/src/stdlib/_winapi.rs @@ -357,7 +357,8 @@ mod _winapi { } else { ms as u32 }; - host_winapi::wait_for_single_object(h.0, ms).map_err(|e| e.to_pyexception(vm)) + vm.allow_threads(|| host_winapi::wait_for_single_object(h.0, ms)) + .map_err(|e| e.to_pyexception(vm)) } #[pyfunction] @@ -381,8 +382,10 @@ mod _winapi { return Err(vm.new_value_error("WaitForMultipleObjects supports at most 64 handles")); } - host_winapi::wait_for_multiple_objects(&handles, wait_all, milliseconds) - .map_err(|e| e.to_pyexception(vm)) + vm.allow_threads(|| { + host_winapi::wait_for_multiple_objects(&handles, wait_all, milliseconds) + }) + .map_err(|e| e.to_pyexception(vm)) } #[pyfunction] @@ -566,8 +569,7 @@ mod _winapi { #[pymethod] fn GetOverlappedResult(&self, wait: bool, vm: &VirtualMachine) -> PyResult<(u32, u32)> { let mut inner = self.inner.lock(); - inner - .get_result(wait) + vm.allow_threads(|| inner.get_result(wait)) .map(|result| (result.transferred, result.error)) .map_err(|e| e.to_pyexception(vm)) } @@ -634,7 +636,8 @@ mod _winapi { } Ok(ov.into_pyobject(vm)) } else { - host_winapi::connect_named_pipe(handle.0).map_err(|e| e.to_pyexception(vm))?; + vm.allow_threads(|| host_winapi::connect_named_pipe(handle.0)) + .map_err(|e| e.to_pyexception(vm))?; Ok(vm.ctx.none()) } } @@ -802,7 +805,9 @@ mod _winapi { return Ok(result.into()); } - let result = host_winapi::read_file(handle.0, size).map_err(|e| e.to_pyexception(vm))?; + let result = vm + .allow_threads(|| host_winapi::read_file(handle.0, size)) + .map_err(|e| e.to_pyexception(vm))?; Ok(vm .ctx .new_tuple(vec![ @@ -926,12 +931,15 @@ mod _winapi { #[cfg(not(feature = "threading"))] let sigint_event: Option = None; - match host_winapi::batched_wait_for_multiple_objects( - &handles, - wait_all, - milliseconds, - sigint_event, - ) { + let batched_result = vm.allow_threads(|| { + host_winapi::batched_wait_for_multiple_objects( + &handles, + wait_all, + milliseconds, + sigint_event, + ) + }); + match batched_result { Ok(host_winapi::BatchedWaitResult::All) => Ok(vm.ctx.none()), Ok(host_winapi::BatchedWaitResult::Indices(indices)) => Ok(vm .ctx diff --git a/crates/vm/src/stdlib/gc.rs b/crates/vm/src/stdlib/gc.rs index 00eea1b39d5..b0007b4c867 100644 --- a/crates/vm/src/stdlib/gc.rs +++ b/crates/vm/src/stdlib/gc.rs @@ -199,15 +199,11 @@ mod gc { // PyObjects, so they never appear in get_referrers results. Since // RustPython materializes every frame as a PyObject, we must exclude // them manually to match the expected behavior. - let stack_frames: HashSet = vm - .frames - .borrow() - .iter() - .map(|fp| { - let frame: &crate::PyObject = unsafe { fp.as_ref() }.as_ref(); - frame as *const crate::PyObject as usize - }) - .collect(); + let mut stack_frames: HashSet = HashSet::new(); + crate::frame::for_each_current_frame(|frame| { + let obj: &crate::PyObject = frame.as_ref(); + stack_frames.insert(obj as *const crate::PyObject as usize); + }); let mut result = Vec::new(); diff --git a/crates/vm/src/stdlib/posix.rs b/crates/vm/src/stdlib/posix.rs index 21576e8da2d..80245aed08f 100644 --- a/crates/vm/src/stdlib/posix.rs +++ b/crates/vm/src/stdlib/posix.rs @@ -623,7 +623,7 @@ pub mod module { run_at_forkers(before_forkers, true, vm); #[cfg(feature = "threading")] - crate::stdlib::_imp::acquire_imp_lock_for_fork(); + crate::stdlib::_imp::acquire_imp_lock_for_fork(vm); #[cfg(feature = "threading")] vm.state.stop_the_world.stop_the_world(vm); @@ -655,6 +655,17 @@ pub mod module { #[cfg(feature = "threading")] crate::object::reset_weakref_locks_after_fork(); + // Repair any type-cache entries left mid-update at fork time. + unsafe { crate::builtins::type_::type_cache_after_fork() }; + + // Reset QSBR: dead parent threads' slots would stall reclamation + // forever, and retired memory can be freed immediately in the + // single-threaded child. + #[cfg(feature = "threading")] + unsafe { + crate::object::qsbr::QSBR.reset_after_fork() + }; + // Phase 3: Clean up thread state. Locks are now reinit'd so we can // acquire them normally instead of using try_lock(). #[cfg(feature = "threading")] @@ -694,6 +705,7 @@ pub mod module { reinit_mutex_after_fork(&vm.state.atexit_funcs); reinit_mutex_after_fork(&vm.state.global_trace_func); reinit_mutex_after_fork(&vm.state.global_profile_func); + reinit_mutex_after_fork(&vm.state.type_mutex); reinit_mutex_after_fork(&vm.state.monitoring); // PyGlobalState parking_lot::Mutex locks diff --git a/crates/vm/src/stdlib/sys.rs b/crates/vm/src/stdlib/sys.rs index b8fe578f238..4e31075da45 100644 --- a/crates/vm/src/stdlib/sys.rs +++ b/crates/vm/src/stdlib/sys.rs @@ -43,7 +43,7 @@ pub mod sys { hash::{PyHash, PyUHash}, }, convert::ToPyObject, - frame::{Frame, FrameRef}, + frame::FrameRef, function::{FuncArgs, KwArgs, OptionalArg, PosArgs}, stdlib::{_warnings::warn, builtins}, types::PyStructSequence, @@ -968,17 +968,9 @@ pub mod sys { #[pyfunction] fn _getframe(offset: OptionalArg, vm: &VirtualMachine) -> PyResult { let offset = offset.into_option().unwrap_or(0); - let frame_ref = { - let frames = vm.frames.borrow(); - if offset >= frames.len() { - return Err(vm.new_value_error("call stack is not deep enough")); - } - - let idx = frames.len() - offset - 1; - // SAFETY: the FrameRef is alive on the call stack while it's in the Vec - let py: &crate::Py = unsafe { frames[idx].as_ref() }; - py.to_owned() - }; + let frame_ref = crate::frame::frame_at_offset(offset) + .ok_or_else(|| vm.new_value_error("call stack is not deep enough"))?; + frame_ref.mark_escaped(); if let Ok(audit) = vm.sys_module.get_attr("audit", vm) { audit.call((vm.ctx.new_str("sys._getframe"), frame_ref.to_owned()), vm)?; @@ -998,15 +990,9 @@ pub mod sys { } // Get the frame at the specified depth - let func_obj = { - let frames = vm.frames.borrow(); - if depth >= frames.len() { - return Ok(vm.ctx.none()); - } - let idx = frames.len() - depth - 1; - // SAFETY: the FrameRef is alive on the call stack while it's in the Vec - let frame: &crate::Py = unsafe { frames[idx].as_ref() }; - frame.func_obj.clone() + let func_obj = match crate::frame::frame_at_offset(depth) { + Some(frame) => frame.func_obj.clone(), + None => return Ok(vm.ctx.none()), }; // If the frame has a function object, return its __module__ attribute diff --git a/crates/vm/src/stdlib/sys/monitoring.rs b/crates/vm/src/stdlib/sys/monitoring.rs index accf2001675..56a68cea619 100644 --- a/crates/vm/src/stdlib/sys/monitoring.rs +++ b/crates/vm/src/stdlib/sys/monitoring.rs @@ -528,9 +528,7 @@ fn update_events_mask(vm: &VirtualMachine, state: &MonitoringState) { // Each code object gets only the events that apply to it (global + its // own local events), preventing e.g. INSTRUCTION from being applied to // unrelated code objects. - for fp in vm.frames.borrow().iter() { - // SAFETY: frames in the Vec are alive while their FrameRef is on the call stack. - let frame = unsafe { fp.as_ref() }; + crate::frame::for_each_current_frame(|frame| { let code = &frame.code; let code_ver = code.instrumentation_version.load(Ordering::Acquire); if code_ver != new_ver { @@ -539,7 +537,7 @@ fn update_events_mask(vm: &VirtualMachine, state: &MonitoringState) { code.instrumentation_version .store(new_ver, Ordering::Release); } - } + }); } fn use_tool_id(tool_id: i32, name: &str, vm: &VirtualMachine) -> PyResult<()> { diff --git a/crates/vm/src/types/slot.rs b/crates/vm/src/types/slot.rs index 83d9706cb33..a5d7a41d3fd 100644 --- a/crates/vm/src/types/slot.rs +++ b/crates/vm/src/types/slot.rs @@ -739,6 +739,18 @@ impl PyType { // Helper macro for number/sequence/mapping sub-slots macro_rules! update_sub_slot { ($group:ident, $slot:ident, $wrapper:expr, $variant:ident) => {{ + // Fall back to the value inherited for this exact field. Left and + // right binary ops (e.g. add / right_add) share one accessor but + // occupy distinct fields, so the fallback must target this field + // rather than the accessor's default field, otherwise resolving + // an absent right op would overwrite the left op's dispatcher. + let inherit_this_field = || { + let mro = self.mro.read(); + let inherited = mro[1..] + .iter() + .find_map(|cls| cls.slots.$group.$slot.load()); + self.slots.$group.$slot.store(inherited); + }; if ADD { // Check if this type defines any method that maps to this slot. // Some slots like SqAssItem/MpAssSubscript are shared by multiple @@ -760,8 +772,15 @@ impl PyType { } result }; + // Reify the wrapper at a single site so the own and inherited + // branches store the same fn item. binary_op1 compares slot + // fn addresses to decide whether a subclass overrides the op; + // duplicating the wrapper closure across branches yields + // distinct addresses in unmerged debug builds and breaks that + // comparison for an inherited slot. + let store_wrapper = || self.slots.$group.$slot.store(Some($wrapper)); if has_own { - self.slots.$group.$slot.store(Some($wrapper)); + store_wrapper(); } else { match self.lookup_slot_in_mro(name, ctx, |sf| { if let SlotFunc::$variant(f) = sf { @@ -774,15 +793,15 @@ impl PyType { self.slots.$group.$slot.store(Some(func)); } SlotLookupResult::PythonMethod => { - self.slots.$group.$slot.store(Some($wrapper)); + store_wrapper(); } SlotLookupResult::NotFound => { - accessor.inherit_from_mro(self); + inherit_this_field(); } } } } else { - accessor.inherit_from_mro(self); + inherit_this_field(); } }}; } @@ -843,12 +862,31 @@ impl PyType { } } SlotAccessor::TpNew => { - // __new__ is not wrapped via PyWrapper - if ADD { + // __new__ is a staticmethod, not a PyWrapper descriptor, so + // lookup_slot_in_mro cannot classify it. Resolve __new__ + // through the MRO dicts instead: a Python-level definition + // needs the dynamic new_wrapper, while a native type's + // builtin __new__ entry (or no entry at all) means the slot + // is inherited from the solid base, matching update_one_slot's + // tp_new special case over the tp_base-inherited value. + let needs_wrapper = if ADD && self.attributes.read().contains_key(name) { + true + } else { + // mro[0] is self, so skip it + self.mro.read()[1..] + .iter() + .find(|cls| cls.attributes.read().contains_key(name)) + .is_some_and(|cls| { + cls.slots.new.load().map(|f| f as usize) + == Some(new_wrapper as NewFunc as usize) + }) + }; + if needs_wrapper { self.slots.new.store(Some(new_wrapper)); self.slots.vectorcall.store(None); } else { - accessor.inherit_from_mro(self); + let inherited = self.base.deref().and_then(|base| base.slots.new.load()); + self.slots.new.store(inherited); } } SlotAccessor::TpDel => update_main_slot!(del, del_wrapper, Del), @@ -897,46 +935,72 @@ impl PyType { } } SlotAccessor::TpSetattro => { - // __setattr__ and __delattr__ share the same slot - if ADD { - match self.lookup_slot_in_mro(name, ctx, |sf| match sf { - SlotFunc::SetAttro(f) | SlotFunc::DelAttro(f) => Some(*f), - _ => None, - }) { - SlotLookupResult::NativeSlot(func) => { - self.slots.setattro.store(Some(func)); - } - SlotLookupResult::PythonMethod => { - self.slots.setattro.store(Some(setattro_wrapper)); - } - SlotLookupResult::NotFound => { - accessor.inherit_from_mro(self); - } + // __setattr__ and __delattr__ share the same slot, so both + // names must be resolved together: a Python-level override of + // either one forces the dispatching wrapper, and the native + // slot is only usable when every resolved name agrees on it. + // This resolution reads the current attribute dicts, so it + // applies the same whether a name was just added or removed. + let extract = |sf: &SlotFunc| match sf { + SlotFunc::SetAttro(f) | SlotFunc::DelAttro(f) => Some(*f), + _ => None, + }; + let setattr = self.lookup_slot_in_mro(identifier!(ctx, __setattr__), ctx, extract); + let delattr = self.lookup_slot_in_mro(identifier!(ctx, __delattr__), ctx, extract); + use SlotLookupResult::{NativeSlot, NotFound, PythonMethod}; + match (setattr, delattr) { + (PythonMethod, _) | (_, PythonMethod) => { + self.slots.setattro.store(Some(setattro_wrapper)); + } + (NativeSlot(set), NativeSlot(del)) => { + let func = if set as usize == del as usize { + set + } else { + setattro_wrapper + }; + self.slots.setattro.store(Some(func)); + } + (NativeSlot(func), NotFound) | (NotFound, NativeSlot(func)) => { + self.slots.setattro.store(Some(func)); + } + (NotFound, NotFound) => { + accessor.inherit_from_mro(self); } - } else { - accessor.inherit_from_mro(self); } } SlotAccessor::TpDescrGet => update_main_slot!(descr_get, descr_get_wrapper, DescrGet), SlotAccessor::TpDescrSet => { - // __set__ and __delete__ share the same slot - if ADD { - match self.lookup_slot_in_mro(name, ctx, |sf| match sf { - SlotFunc::DescrSet(f) | SlotFunc::DescrDel(f) => Some(*f), - _ => None, - }) { - SlotLookupResult::NativeSlot(func) => { - self.slots.descr_set.store(Some(func)); - } - SlotLookupResult::PythonMethod => { - self.slots.descr_set.store(Some(descr_set_wrapper)); - } - SlotLookupResult::NotFound => { - accessor.inherit_from_mro(self); - } + // __set__ and __delete__ share the same slot, so both names + // must be resolved together: a Python-level definition of + // either one forces the dispatching wrapper, and the native + // slot is only usable when every resolved name agrees on it. + // This resolution reads the current attribute dicts, so it + // applies the same whether a name was just added or removed. + let extract = |sf: &SlotFunc| match sf { + SlotFunc::DescrSet(f) | SlotFunc::DescrDel(f) => Some(*f), + _ => None, + }; + let set = self.lookup_slot_in_mro(identifier!(ctx, __set__), ctx, extract); + let delete = self.lookup_slot_in_mro(identifier!(ctx, __delete__), ctx, extract); + use SlotLookupResult::{NativeSlot, NotFound, PythonMethod}; + match (set, delete) { + (PythonMethod, _) | (_, PythonMethod) => { + self.slots.descr_set.store(Some(descr_set_wrapper)); + } + (NativeSlot(set), NativeSlot(delete)) => { + let func = if set as usize == delete as usize { + set + } else { + descr_set_wrapper + }; + self.slots.descr_set.store(Some(func)); + } + (NativeSlot(func), NotFound) | (NotFound, NativeSlot(func)) => { + self.slots.descr_set.store(Some(func)); + } + (NotFound, NotFound) => { + accessor.inherit_from_mro(self); } - } else { - accessor.inherit_from_mro(self); } } diff --git a/crates/vm/src/types/slot_defs.rs b/crates/vm/src/types/slot_defs.rs index 8637f811272..34d62147e0c 100644 --- a/crates/vm/src/types/slot_defs.rs +++ b/crates/vm/src/types/slot_defs.rs @@ -608,7 +608,7 @@ impl SlotAccessor { if typ.slots.init.load().is_none() && let Some(base_val) = base.slots.init.load() { - let slot_defined = base.base.as_ref().is_none_or(|bb| { + let slot_defined = base.base.deref().is_none_or(|bb| { bb.slots.init.load().map(|v| v as usize) != Some(base_val as usize) }); if slot_defined { diff --git a/crates/vm/src/types/zoo.rs b/crates/vm/src/types/zoo.rs index 13d439345f7..64807fc0973 100644 --- a/crates/vm/src/types/zoo.rs +++ b/crates/vm/src/types/zoo.rs @@ -2,10 +2,10 @@ use crate::{ Py, builtins::{ asyncgenerator, bool_, builtin_func, bytearray, bytes, capsule, classmethod, code, complex, - coroutine, descriptor, dict, enumerate, filter, float, frame, function, generator, - genericalias, getset, int, interpolation, iter, list, map, mappingproxy, memory, module, - namespace, object, property, pystr, range, set, singletons, slice, staticmethod, super_, - template, traceback, tuple, + coroutine, descriptor, dict, enumerate, filter, float, frame, frame_locals_proxy, function, + generator, genericalias, getset, int, interpolation, iter, list, map, mappingproxy, memory, + module, namespace, object, property, pystr, range, set, singletons, slice, staticmethod, + super_, template, traceback, tuple, type_::{self, PyType}, union_, weakproxy, weakref, zip, }, @@ -39,6 +39,7 @@ pub struct TypeZoo { pub filter_type: &'static Py, pub float_type: &'static Py, pub frame_type: &'static Py, + pub frame_locals_proxy_type: &'static Py, pub frozenset_type: &'static Py, pub generator_type: &'static Py, pub int_type: &'static Py, @@ -178,6 +179,7 @@ impl TypeZoo { dict_reverseitemiterator_type: dict::PyDictReverseItemIterator::init_builtin_type(), ellipsis_type: slice::PyEllipsis::init_builtin_type(), frame_type: crate::frame::Frame::init_builtin_type(), + frame_locals_proxy_type: frame_locals_proxy::FrameLocalsProxy::init_builtin_type(), function_type: function::PyFunction::init_builtin_type(), generator_type: generator::PyGenerator::init_builtin_type(), getset_type: getset::PyGetSet::init_builtin_type(), @@ -253,6 +255,7 @@ impl TypeZoo { bool_::init(context); code::init(context); frame::init(context); + frame_locals_proxy::init(context); weakref::init(context); weakproxy::init(context); singletons::init(context); diff --git a/crates/vm/src/vm/context.rs b/crates/vm/src/vm/context.rs index 226d5f1a1a7..9a545663576 100644 --- a/crates/vm/src/vm/context.rs +++ b/crates/vm/src/vm/context.rs @@ -14,7 +14,6 @@ use crate::{ object, pystr, type_::PyAttributes, }, - bytecode::{self, CodeFlags, CodeUnit, Instruction, Opcode}, class::StaticType, common::rc::PyRc, exceptions, @@ -31,7 +30,6 @@ use malachite_bigint::BigInt; use num_complex::Complex64; use num_traits::ToPrimitive; use rustpython_common::lock::PyRwLock; -use rustpython_compiler_core::{OneIndexed, SourceLocation}; #[derive(Debug)] pub struct Context { @@ -52,7 +50,6 @@ pub struct Context { pub int_cache_pool: Vec, pub(crate) latin1_char_cache: Vec>, pub(crate) ascii_char_cache: Vec>, - pub(crate) init_cleanup_code: PyRef, // there should only be exact objects of str in here, no non-str objects and no subclasses pub(crate) string_pool: StringPool, pub(crate) slot_new_wrapper: PyMethodDef, @@ -362,8 +359,6 @@ impl Context { PyMethodFlags::METHOD, None, ); - let init_cleanup_code = Self::new_init_cleanup_code(&types, &names); - let empty_str = unsafe { string_pool.intern("", types.str_type.to_owned()) }; let empty_bytes = create_object(PyBytes::from(Vec::new()), types.bytes_type); @@ -389,7 +384,6 @@ impl Context { int_cache_pool, latin1_char_cache, ascii_char_cache, - init_cleanup_code, string_pool, slot_new_wrapper, names, @@ -399,49 +393,6 @@ impl Context { } } - fn new_init_cleanup_code(types: &TypeZoo, names: &ConstName) -> PyRef { - let loc = SourceLocation { - line: OneIndexed::MIN, - character_offset: OneIndexed::from_zero_indexed(0), - }; - let instructions = [ - CodeUnit { - op: Instruction::ExitInitCheck, - arg: 0.into(), - }, - CodeUnit { - op: Instruction::ReturnValue, - arg: 0.into(), - }, - CodeUnit { - op: Opcode::Resume.into(), - arg: 0.into(), - }, - ]; - let code = bytecode::CodeObject { - instructions: instructions.into(), - locations: vec![(loc, loc); instructions.len()].into_boxed_slice(), - flags: CodeFlags::OPTIMIZED, - posonlyarg_count: 0, - arg_count: 0, - kwonlyarg_count: 0, - source_path: names.__init__, - first_line_number: None, - max_stackdepth: 2, - obj_name: names.__init__, - qualname: names.__init__, - constants: core::iter::empty().collect(), - names: Vec::new().into_boxed_slice(), - varnames: Vec::new().into_boxed_slice(), - cellvars: Vec::new().into_boxed_slice(), - freevars: Vec::new().into_boxed_slice(), - localspluskinds: Vec::new().into_boxed_slice(), - linetable: Vec::new().into_boxed_slice(), - exceptiontable: Vec::new().into_boxed_slice(), - }; - PyRef::new_ref(PyCode::new(code), types.code_type.to_owned(), None) - } - pub fn intern_str(&self, s: S) -> &'static PyStrInterned { unsafe { self.string_pool.intern(s, self.types.str_type.to_owned()) } } diff --git a/crates/vm/src/vm/interpreter.rs b/crates/vm/src/vm/interpreter.rs index 56c9606f7de..f456e8587ea 100644 --- a/crates/vm/src/vm/interpreter.rs +++ b/crates/vm/src/vm/interpreter.rs @@ -1,4 +1,4 @@ -#[cfg(all(unix, feature = "threading"))] +#[cfg(feature = "threading")] use super::StopTheWorldState; use super::{Context, PyConfig, PyGlobalState, VirtualMachine, setting::Settings, thread}; use crate::{ @@ -122,6 +122,7 @@ where switch_interval: AtomicCell::new(0.005), global_trace_func: PyMutex::default(), global_profile_func: PyMutex::default(), + type_mutex: PyMutex::default(), #[cfg(feature = "threading")] main_thread_ident: AtomicCell::new(0), #[cfg(feature = "threading")] @@ -133,7 +134,7 @@ where monitoring: PyMutex::default(), monitoring_events: AtomicCell::new(0), instrumentation_version: AtomicU64::new(0), - #[cfg(all(unix, feature = "threading"))] + #[cfg(feature = "threading")] stop_the_world: StopTheWorldState::new(), }); @@ -149,7 +150,12 @@ where // Call custom init function (can mutate vm.state) init(&mut vm); + // `initialize()` runs Python bytecode directly (e.g. importing `codecs` + // and `encodings`) before any `enter_vm` scope exists, so attach this + // thread for the duration so type cache reads see it as ATTACHED. + let vm_guard = thread::VmBootstrapGuard::new(&vm); vm.initialize(); + drop(vm_guard); // Clone global_state for Interpreter after all initialization is done let global_state = vm.state.clone(); @@ -461,11 +467,18 @@ impl Interpreter { } // Match CPython: if exit_code is 0 and stdout flush failed, exit 120 - if exit_code == 0 && flush_status < 0 { + let exit_code = if exit_code == 0 && flush_status < 0 { EXITCODE_FLUSH_FAILURE } else { exit_code - } + }; + + // Daemon threads may still exist, so use the safe `process()`, + // not `drain_all()`. + #[cfg(feature = "threading")] + crate::object::qsbr::QSBR.process(); + + exit_code }) } } diff --git a/crates/vm/src/vm/mod.rs b/crates/vm/src/vm/mod.rs index 8ad8a0d0bca..852e57cc0b4 100644 --- a/crates/vm/src/vm/mod.rs +++ b/crates/vm/src/vm/mod.rs @@ -44,11 +44,12 @@ use crate::{ warn::WarningsState, }; use alloc::{borrow::Cow, collections::BTreeMap}; -#[cfg(all(unix, feature = "threading"))] +#[cfg(all(not(unix), feature = "threading"))] +use core::ptr::NonNull; +#[cfg(feature = "threading")] use core::sync::atomic::AtomicI64; use core::{ cell::{Cell, OnceCell, RefCell}, - ptr::NonNull, sync::atomic::{AtomicBool, AtomicU64, Ordering}, }; use crossbeam_utils::atomic::AtomicCell; @@ -74,7 +75,6 @@ pub struct VirtualMachine { pub builtins: PyRef, pub sys_module: PyRef, pub ctx: PyRc, - pub frames: RefCell>, /// Thread-local data stack for bump-allocating frame-local data /// (localsplus arrays for non-generator frames). datastack: core::cell::UnsafeCell, @@ -108,11 +108,15 @@ pub struct VirtualMachine { pub(crate) audit_hooks: RefCell>, } -/// Non-owning frame pointer for the frames stack. +/// Non-owning frame pointer for the non-unix threading frames stack. /// The pointed-to frame is kept alive by the caller of with_frame/resume_gen_frame. +/// Unix threading builds publish the top frame through `ThreadSlot::top_frame` +/// and walk the rest via `Frame::previous`, so they do not use this type. +#[cfg(all(not(unix), feature = "threading"))] #[derive(Copy, Clone)] pub struct FramePtr(NonNull>); +#[cfg(all(not(unix), feature = "threading"))] impl FramePtr { /// # Safety /// The pointed-to frame must still be alive. @@ -122,8 +126,10 @@ impl FramePtr { } } -// SAFETY: FramePtr is only stored in the VM's frames Vec while the corresponding -// FrameRef is alive on the call stack. The Vec is always empty when the VM moves between threads. +// SAFETY: FramePtr is only stored in a thread's shared frame stack +// (`ThreadSlot::frames`) while the corresponding FrameRef is alive on that +// thread's call stack; readers dereference it under the slot mutex. +#[cfg(all(not(unix), feature = "threading"))] unsafe impl Send for FramePtr {} #[derive(Debug)] @@ -143,7 +149,7 @@ impl Default for ExceptionStack { /// Stop-the-world state for fork safety. Before `fork()`, the requester /// stops all other Python threads so they are not holding internal locks. -#[cfg(all(unix, feature = "threading"))] +#[cfg(feature = "threading")] pub struct StopTheWorldState { /// Fast-path flag checked in the bytecode loop (like `_PY_EVAL_PLEASE_STOP_BIT`) pub(crate) requested: AtomicBool, @@ -151,6 +157,11 @@ pub struct StopTheWorldState { world_stopped: AtomicBool, /// Ident of the thread that requested the stop (like `stw->requester`) requester: AtomicU64, + /// Single exclusion held for the whole stop→start span. Fork and GC are + /// both stop-the-world requesters driving this shared state; only one may + /// hold it at a time. Acquired before any stop bookkeeping (see + /// `acquire_exclusion`) and released by `start_the_world`/`reset_after_fork`. + exclusion: AtomicBool, /// Signaled by suspending threads when their state transitions to SUSPENDED notify_mutex: std::sync::Mutex<()>, notify_cv: std::sync::Condvar, @@ -178,7 +189,7 @@ pub struct StopTheWorldState { stats_suspend_wait_yields: AtomicU64, } -#[cfg(all(unix, feature = "threading"))] +#[cfg(feature = "threading")] #[derive(Debug, Clone, Copy)] pub struct StopTheWorldStats { pub stop_calls: u64, @@ -194,14 +205,14 @@ pub struct StopTheWorldStats { pub world_stopped: bool, } -#[cfg(all(unix, feature = "threading"))] +#[cfg(feature = "threading")] impl Default for StopTheWorldState { fn default() -> Self { Self::new() } } -#[cfg(all(unix, feature = "threading"))] +#[cfg(feature = "threading")] impl StopTheWorldState { #[must_use] pub const fn new() -> Self { @@ -209,6 +220,7 @@ impl StopTheWorldState { requested: AtomicBool::new(false), world_stopped: AtomicBool::new(false), requester: AtomicU64::new(0), + exclusion: AtomicBool::new(false), notify_mutex: std::sync::Mutex::new(()), notify_cv: std::sync::Condvar::new(), thread_countdown: AtomicI64::new(0), @@ -336,20 +348,76 @@ impl StopTheWorldState { forced_parks != 0 && self.thread_countdown.load(Ordering::Acquire) == 0 } + /// Acquire the single stop-the-world exclusion in a park-friendly way. + /// + /// Fork and GC both request stop-the-world through the same shared state; + /// without this exclusion their `requester`/`requested`/countdown words + /// could be clobbered by an interleaving requester, so the completion + /// check could never converge and a requester would wait on itself forever. + /// + /// The acquire must be park-friendly. While another requester's stop is in + /// progress it sets this thread's stop bit and waits for it to suspend; + /// blocking on a plain lock here would keep this thread from ever reaching + /// that safepoint, so the active requester would wait for this thread while + /// this thread waits for the lock — a deadlock swap. Instead we poll and + /// honor the suspend request between tries. Suspending here is safe as long + /// as any lock a spinning requester still holds is never acquired + /// attached-blocking by another thread. The fork requester holds IMP_LOCK, + /// but its acquisition detaches (`allow_threads`), so no attached thread + /// blocks on it; the GC requester holds only the `collecting` mutex, which + /// is only ever `try_lock`'d. The active requester therefore force-parks + /// this thread, finishes its whole stop→start span, releases the exclusion, + /// and only then does this thread resume and acquire it. + fn acquire_exclusion(&self) { + if self + .exclusion + .compare_exchange(false, true, Ordering::AcqRel, Ordering::Relaxed) + .is_ok() + { + return; + } + loop { + crate::vm::thread::suspend_if_needed(self); + std::thread::yield_now(); + if self + .exclusion + .compare_exchange(false, true, Ordering::AcqRel, Ordering::Relaxed) + .is_ok() + { + return; + } + } + } + + /// Release the stop-the-world exclusion taken by `acquire_exclusion`. + fn release_exclusion(&self) { + self.exclusion.store(false, Ordering::Release); + } + /// Stop all non-requester threads (`stop_the_world`). /// /// 1. Sets `requested`, marking the requester thread. /// 2. CAS detached threads to SUSPENDED. /// 3. Waits (polling with 1 ms condvar timeout) for attached threads /// to self-suspend in `check_signals`. + /// + /// Takes the shared exclusion first so at most one requester (fork or GC) + /// drives the stop→start span at a time; it is released by + /// `start_the_world`/`reset_after_fork`. pub fn stop_the_world(&self, vm: &VirtualMachine) { + self.acquire_exclusion(); let start = std::time::Instant::now(); let requester_ident = crate::stdlib::_thread::get_ident(); self.requester.store(requester_ident, Ordering::Relaxed); self.stats_stop_calls.fetch_add(1, Ordering::Relaxed); let initial_countdown = self.init_thread_countdown(vm); stw_trace(format_args!("stop begin requester={requester_ident}")); - if initial_countdown == 0 { + // Park detached threads and set stop bits, then confirm every other + // thread is SUSPENDED. The completion condition is level-triggered + // (`all_non_requester_suspended`) so an already-suspended thread that + // was counted but will not notify again cannot stall the stop. + self.park_detached_threads(vm); + if initial_countdown == 0 || self.all_non_requester_suspended(vm) { self.world_stopped.store(true, Ordering::Release); #[cfg(debug_assertions)] self.debug_assert_all_non_requester_suspended(vm); @@ -361,7 +429,8 @@ impl StopTheWorldState { let mut polls = 0u64; loop { - if self.park_detached_threads(vm) { + self.park_detached_threads(vm); + if self.all_non_requester_suspended(vm) { break; } polls = polls.saturating_add(1); @@ -369,8 +438,7 @@ impl StopTheWorldState { // Re-check under the wait mutex first to avoid a lost-wake race: // a thread may have suspended and notified right before we enter wait. let guard = self.notify_mutex.lock().unwrap(); - if self.thread_countdown.load(Ordering::Acquire) == 0 || self.park_detached_threads(vm) - { + if self.all_non_requester_suspended(vm) { drop(guard); break; } @@ -445,6 +513,9 @@ impl StopTheWorldState { self.requester.store(0, Ordering::Relaxed); #[cfg(debug_assertions)] self.debug_assert_all_non_requester_detached(vm); + // Release the exclusion last, ending the stop→start span so the next + // requester (fork or GC) can proceed. + self.release_exclusion(); stw_trace(format_args!("start end requester={requester}")); } @@ -454,6 +525,9 @@ impl StopTheWorldState { self.world_stopped.store(false, Ordering::Relaxed); self.requester.store(0, Ordering::Relaxed); self.thread_countdown.store(0, Ordering::Relaxed); + // The surviving child thread inherited the exclusion taken by the + // pre-fork `stop_the_world`; release it (no start_the_world runs here). + self.release_exclusion(); stw_trace(format_args!("reset-after-fork")); } @@ -514,6 +588,33 @@ impl StopTheWorldState { } } + /// Whether every non-requester registered thread is currently SUSPENDED. + /// + /// Level-triggered stop-the-world completion check. Relying on this rather + /// than solely on the edge-triggered `thread_countdown` avoids a + /// lost-decrement race under rapid back-to-back stops: a thread that is + /// already SUSPENDED when a new stop counts it neither notifies nor is + /// force-parked again, so an edge-based countdown could never reach zero. + fn all_non_requester_suspended(&self, vm: &VirtualMachine) -> bool { + use thread::THREAD_SUSPENDED; + let requester = self.requester.load(Ordering::Relaxed); + let registry = vm.state.thread_frames.lock(); + + #[expect( + clippy::iter_over_hash_type, + reason = "Iteration order doesn't matter here" + )] + for (&id, slot) in registry.iter() { + if id == requester { + continue; + } + if slot.state.load(Ordering::Acquire) != THREAD_SUSPENDED { + return false; + } + } + true + } + #[cfg(debug_assertions)] fn debug_assert_all_non_requester_suspended(&self, vm: &VirtualMachine) { use thread::THREAD_SUSPENDED; @@ -561,13 +662,13 @@ impl StopTheWorldState { } } -#[cfg(all(unix, feature = "threading"))] +#[cfg(feature = "threading")] pub(super) fn stw_trace_enabled() -> bool { static ENABLED: std::sync::OnceLock = std::sync::OnceLock::new(); *ENABLED.get_or_init(|| crate::host_env::os::var_os("RUSTPYTHON_STW_TRACE").is_some()) } -#[cfg(all(unix, feature = "threading"))] +#[cfg(feature = "threading")] pub(super) fn stw_trace(msg: core::fmt::Arguments<'_>) { if stw_trace_enabled() { use core::fmt::Write as _; @@ -603,7 +704,13 @@ pub(super) fn stw_trace(msg: core::fmt::Arguments<'_>) { crate::stdlib::_thread::get_ident(), msg ); + #[cfg(unix)] crate::host_env::io::write_stderr_raw(&out.buf[..out.len]); + #[cfg(not(unix))] + { + use std::io::Write as _; + let _ = std::io::stderr().write_all(&out.buf[..out.len]); + } } } @@ -637,6 +744,8 @@ pub struct PyGlobalState { pub global_trace_func: PyMutex>, /// Global profile function for all threads (set by sys._setprofileallthreads) pub global_profile_func: PyMutex>, + /// Global type mutation/versioning mutex for CPython-style FT type operations. + pub type_mutex: PyMutex<()>, /// Main thread identifier (pthread_self on Unix) #[cfg(feature = "threading")] pub main_thread_ident: AtomicCell, @@ -657,7 +766,7 @@ pub struct PyGlobalState { /// local version against this to decide whether re-instrumentation is needed. pub instrumentation_version: AtomicU64, /// Stop-the-world state for pre-fork thread suspension - #[cfg(all(unix, feature = "threading"))] + #[cfg(feature = "threading")] pub stop_the_world: StopTheWorldState, } @@ -759,7 +868,6 @@ impl VirtualMachine { builtins, sys_module, ctx, - frames: RefCell::new(vec![]), datastack: core::cell::UnsafeCell::new(crate::datastack::DataStack::new()), wasm_id: None, exceptions: RefCell::default(), @@ -1219,6 +1327,15 @@ impl VirtualMachine { #[inline(always)] pub fn run_frame(&self, frame: FrameRef) -> PyResult { + // Only ordinary (datastack) call frames reach `run_frame`; generator + // and coroutine frames are resumed through `resume_gen_frame`. A + // datastack frame is created untracked and is tracked lazily only when + // it escapes, which happens no earlier than `release_datastack_frame` + // after this call returns. So it must be untracked on entry. + debug_assert!( + !frame.as_object().is_gc_tracked(), + "datastack frame is GC-tracked before execution" + ); match self.with_frame(frame, |f| f.run(self))? { ExecutionResult::Return(value) => Ok(value), _ => panic!("Got unexpected result from function"), @@ -1458,7 +1575,7 @@ impl VirtualMachine { /// evaluation consumes more native stack in those configurations. #[cfg_attr(any(miri, target_env = "musl"), allow(dead_code))] const STACK_MARGIN_BYTES: usize = - (if cfg!(debug_assertions) { 4096 } else { 2048 }) * core::mem::size_of::(); + (if cfg!(debug_assertions) { 16384 } else { 2048 }) * core::mem::size_of::(); /// Get the stack boundaries using platform-specific APIs. /// Returns (base, top) where base is the lowest address and top is the highest. @@ -1519,11 +1636,16 @@ impl VirtualMachine { } /// Calculate the C stack soft limit based on actual stack boundaries. - /// soft_limit = base + 2 * margin (for downward-growing stacks) + /// soft_limit = base + 2 * margin (for downward-growing stacks). + /// The margin is clamped to half the stack so threads created with a stack + /// smaller than 2 * (2 * margin) still get usable headroom instead of a + /// soft limit above their stack top (which would trip on entry). #[cfg(all(not(miri), not(target_env = "musl")))] fn calculate_c_stack_soft_limit() -> usize { - let (base, _top) = Self::get_stack_bounds(); - base + Self::STACK_MARGIN_BYTES * 2 + let (base, top) = Self::get_stack_bounds(); + let stack_size = top.saturating_sub(base); + let margin = (Self::STACK_MARGIN_BYTES * 2).min(stack_size / 2); + base + margin } /// Musl currently reports stack bounds in a way that trips the VM's @@ -1535,15 +1657,14 @@ impl VirtualMachine { } /// Check if we're near the C stack limit (like _Py_MakeRecCheck). - /// Returns true only when stack pointer is in the "danger zone" between - /// soft_limit and hard_limit (soft_limit - 2*margin). + /// One-sided: any stack pointer below the soft limit is in danger, since a + /// single native frame can exceed the margin and step past it. #[cfg(all(not(miri), not(target_env = "musl")))] #[inline(always)] fn check_c_stack_overflow(&self) -> bool { let current_sp = psm::stack_pointer() as usize; let soft_limit = self.c_stack_soft_limit.get(); current_sp < soft_limit - && current_sp >= soft_limit.saturating_sub(Self::STACK_MARGIN_BYTES * 2) } /// Miri does not support the native stack probe, and musl currently trips @@ -1574,34 +1695,19 @@ impl VirtualMachine { &self, frame: FrameRef, f: F, - ) -> PyResult { - self.with_frame_impl(frame, true, f) - } - - pub(crate) fn with_frame_untraced PyResult>( - &self, - frame: FrameRef, - f: F, - ) -> PyResult { - self.with_frame_impl(frame, false, f) - } - - fn with_frame_impl PyResult>( - &self, - frame: FrameRef, - traced: bool, - f: F, ) -> PyResult { self.with_recursion("", || { // SAFETY: `frame` (FrameRef) stays alive for the entire closure scope, // keeping the FramePtr valid. We pass a clone to `f` so that `f` // consuming its FrameRef doesn't invalidate our pointer. - let fp = FramePtr(NonNull::from(&*frame)); - self.frames.borrow_mut().push(fp); - // Update the shared frame stack for sys._current_frames() and faulthandler - #[cfg(feature = "threading")] - crate::vm::thread::push_thread_frame(fp); - // Link frame into the signal-safe frame chain (previous pointer) + // Publish the frame for sys._current_frames() and faulthandler. + // On unix, set_current_frame below publishes the top frame into the + // thread slot; only non-unix builds maintain the mutex-guarded Vec. + #[cfg(all(not(unix), feature = "threading"))] + crate::vm::thread::push_thread_frame(FramePtr(NonNull::from(&*frame))); + // Link frame into the signal-safe frame chain (previous pointer). + // This chain is the single source for the current thread's frame + // stack (current_frame, sys._getframe, f_back, monitoring). let old_frame = crate::vm::thread::set_current_frame((&**frame) as *const Frame); frame.previous.store( old_frame as *mut Frame, @@ -1613,27 +1719,31 @@ impl VirtualMachine { // exc_info pollution from frames with unbalanced // PUSH_EXC_INFO/POP_EXCEPT (e.g., exception escaping an except block // whose cleanup entry is missing from the exception table). - let saved_exc = self.current_exception(); + // A callee whose bytecode never mutates the slot cannot pollute it, + // so the save/restore is skipped for it. + let save_exc = frame.code.has_exc_handling; + let saved_exc = if save_exc { + self.current_exception() + } else { + None + }; let old_owner = frame.owner.swap( crate::frame::FrameOwner::Thread as i8, core::sync::atomic::Ordering::AcqRel, ); - // Ensure cleanup on panic: restore owner, exc_info, frame chain, and frames Vec. + // Ensure cleanup on panic: restore owner, exc_info, and frame chain. scopeguard::defer! { frame.owner.store(old_owner, core::sync::atomic::Ordering::Release); - self.set_exception(saved_exc); + if save_exc { + self.restore_exception(saved_exc); + } crate::vm::thread::set_current_frame(old_frame); - self.frames.borrow_mut().pop(); - #[cfg(feature = "threading")] + #[cfg(all(not(unix), feature = "threading"))] crate::vm::thread::pop_thread_frame(); } - if traced { - self.dispatch_traced_frame(&frame, |frame| f(frame.to_owned())) - } else { - f(frame.to_owned()) - } + self.dispatch_traced_frame(&frame, |frame| f(frame.to_owned())) }) } @@ -1653,10 +1763,8 @@ impl VirtualMachine { self.recursion_depth.update(|d| d + 1); // SAFETY: frame (&FrameRef) stays alive for the duration, so NonNull is valid until pop. - let fp = FramePtr(NonNull::from(&**frame)); - self.frames.borrow_mut().push(fp); - #[cfg(feature = "threading")] - crate::vm::thread::push_thread_frame(fp); + #[cfg(all(not(unix), feature = "threading"))] + crate::vm::thread::push_thread_frame(FramePtr(NonNull::from(&**frame))); let old_frame = crate::vm::thread::set_current_frame((&***frame) as *const Frame); frame.previous.store( old_frame as *mut Frame, @@ -1677,8 +1785,7 @@ impl VirtualMachine { frame.owner.store(old_owner, core::sync::atomic::Ordering::Release); self.pop_exception(); crate::vm::thread::set_current_frame(old_frame); - self.frames.borrow_mut().pop(); - #[cfg(feature = "threading")] + #[cfg(all(not(unix), feature = "threading"))] crate::vm::thread::pop_thread_frame(); self.recursion_depth.update(|d| d - 1); @@ -1768,10 +1875,7 @@ impl VirtualMachine { } pub fn current_frame(&self) -> Option { - self.frames.borrow().last().map(|fp| { - // SAFETY: the caller keeps the FrameRef alive while it's in the Vec - unsafe { fp.as_ref() }.to_owned() - }) + crate::frame::current_thread_frame() } pub fn current_locals(&self) -> PyResult { @@ -2034,13 +2138,15 @@ impl VirtualMachine { return true; } - #[cfg(all(unix, feature = "threading"))] + #[cfg(feature = "threading")] if thread::stop_requested_for_current_thread() { return true; } + // Signal and QSBR bits share one word: a single relaxed load per + // instruction covers both. #[cfg(not(target_arch = "wasm32"))] - if crate::signal::is_triggered() { + if crate::signal::eval_breaker_pending() { return true; } @@ -2059,15 +2165,33 @@ impl VirtualMachine { } // Suspend this thread if stop-the-world is in progress - #[cfg(all(unix, feature = "threading"))] + #[cfg(feature = "threading")] thread::suspend_if_needed(&self.state.stop_the_world); + // Pass a QSBR checkpoint if requested (deferred memory reclamation). + #[cfg(feature = "threading")] + if crate::signal::qsbr_bit_set() && thread::qsbr_break_requested() { + thread::qsbr_checkpoint(); + } + #[cfg(not(target_arch = "wasm32"))] crate::signal::check_signals(self)?; Ok(()) } + /// Run an automatic collection scheduled by `maybe_collect`, if any. + /// + /// Called only from the bytecode-loop safepoint, where no interpreter + /// locks are held, so the stop-the-world it performs cannot deadlock + /// against a thread blocked on a lock this thread would otherwise hold. + #[cfg(feature = "threading")] + pub(crate) fn run_scheduled_gc(&self) { + if crate::signal::take_gc_scheduled() { + crate::gc_state::gc_state().collect(0); + } + } + /// Push a new exc_info slot (for generator/coroutine resume). pub(crate) fn push_exception(&self, exc: Option) { self.exceptions.borrow_mut().stack.push(exc); @@ -2112,6 +2236,25 @@ impl VirtualMachine { thread::update_thread_exception(self.topmost_exception()); } + /// Restore an exc_info slot value saved by `with_frame`, skipping the + /// store when the slot is unchanged. `saved` is a strong reference taken + /// at save time, so the object it points to cannot have been freed and + /// its address reused while the frame ran; pointer identity therefore + /// proves the slot still holds the same value and both the store and the + /// thread-exception mirror update would be no-ops. + pub(crate) fn restore_exception(&self, saved: Option) { + let excs = self.exceptions.borrow(); + let unchanged = match (excs.stack.last(), &saved) { + (Some(Some(current)), Some(saved)) => current.is(saved), + (Some(None), None) => true, + _ => false, + }; + drop(excs); + if !unchanged { + self.set_exception(saved); + } + } + pub fn take_raised_exception(&self) -> Option { let mut excs = self.exceptions.borrow_mut(); if let Some(top) = excs.stack.last_mut() { diff --git a/crates/vm/src/vm/thread.rs b/crates/vm/src/vm/thread.rs index 5e73cc5f618..5009cb695c6 100644 --- a/crates/vm/src/vm/thread.rs +++ b/crates/vm/src/vm/thread.rs @@ -1,4 +1,4 @@ -#[cfg(feature = "threading")] +#[cfg(all(not(unix), feature = "threading"))] use super::FramePtr; #[cfg(feature = "threading")] use crate::builtins::PyBaseExceptionRef; @@ -19,30 +19,40 @@ use std::thread_local; // DETACHED: not executing Python bytecode (in native code, or idle) // ATTACHED: actively executing Python bytecode // SUSPENDED: parked by a stop-the-world request -#[cfg(all(unix, feature = "threading"))] +#[cfg(feature = "threading")] pub const THREAD_DETACHED: i32 = 0; -#[cfg(all(unix, feature = "threading"))] +#[cfg(feature = "threading")] pub const THREAD_ATTACHED: i32 = 1; -#[cfg(all(unix, feature = "threading"))] +#[cfg(feature = "threading")] pub const THREAD_SUSPENDED: i32 = 2; /// Per-thread shared state for sys._current_frames() and sys._current_exceptions(). /// The exception field uses atomic operations for lock-free cross-thread reads. #[cfg(feature = "threading")] pub struct ThreadSlot { + /// Top of the owning thread's Python call stack, published for + /// cross-thread readers (`sys._current_frames`, cross-thread `f_back`). + /// The rest of the stack is reachable via each frame's `previous` pointer. + /// Written lock-free on the hot push/pop path with relaxed ordering; every + /// cross-thread read runs under stop-the-world, which parks the owning + /// thread at a safepoint and supplies the happens-before edge, so the + /// pointer and the frames it reaches are quiescent and alive at read time. + #[cfg(unix)] + pub top_frame: AtomicPtr, /// Raw frame pointers, valid while the owning thread's call stack is active. /// Readers must hold the Mutex and convert to FrameRef inside the lock. + /// Used on non-unix threading builds, which have no stop-the-world. + #[cfg(not(unix))] pub frames: parking_lot::Mutex>, pub exception: crate::PyAtomicRef>, /// Thread state for stop-the-world: DETACHED / ATTACHED / SUSPENDED - #[cfg(unix)] pub state: core::sync::atomic::AtomicI32, /// Per-thread stop request bit (eval breaker equivalent). - #[cfg(unix)] pub stop_requested: core::sync::atomic::AtomicBool, /// Handle for waking this thread from park in stop-the-world paths. - #[cfg(unix)] pub thread: std::thread::Thread, + /// QSBR state for deferred memory reclamation. + pub(crate) qsbr: Arc, } #[cfg(feature = "threading")] @@ -79,6 +89,15 @@ thread_local! { pub(crate) static CURRENT_FRAME: AtomicPtr = const { AtomicPtr::new(core::ptr::null_mut()) }; + /// Cached pointer to this thread's `ThreadSlot::top_frame`, so the hot + /// push/pop path can publish the top frame with a single relaxed store and + /// no `CURRENT_THREAD_SLOT` RefCell borrow. Null until the slot is + /// initialized; the `Arc` in `CURRENT_THREAD_SLOT` keeps the + /// pointee alive until `cleanup_current_thread_frames` clears this. + #[cfg(all(unix, feature = "threading"))] + static CURRENT_TOP_FRAME_SLOT: Cell<*const AtomicPtr> = + const { Cell::new(core::ptr::null()) }; + } #[must_use] @@ -109,23 +128,32 @@ fn set_current_vm(vm: &VirtualMachine, f: impl FnOnce() -> R) -> R { }) } +pub fn try_with_current_vm(f: impl FnOnce(&VirtualMachine) -> R) -> Option { + VM_STACK.with(|vms| { + let vm = vms.borrow().last().copied()?; + // SAFETY: entries in VM_STACK either borrow a VM for the dynamic + // scope of a set_current_vm()/enter_vm() call or point at GILSTATE_VM. + Some(f(unsafe { vm.as_ref() })) + }) +} + pub fn enter_vm(vm: &VirtualMachine, f: impl FnOnce() -> R) -> R { // Outermost enter_vm: transition DETACHED → ATTACHED - #[cfg(all(unix, feature = "threading"))] + #[cfg(feature = "threading")] let was_outermost = !current_vm_is_set(); // Initialize thread slot for this thread if not already done #[cfg(feature = "threading")] init_thread_slot_if_needed(vm); - #[cfg(all(unix, feature = "threading"))] + #[cfg(feature = "threading")] if was_outermost { attach_thread(vm); } scopeguard::defer! { // Outermost exit: transition ATTACHED → DETACHED - #[cfg(all(unix, feature = "threading"))] + #[cfg(feature = "threading")] if was_outermost { detach_thread(); } @@ -134,6 +162,61 @@ pub fn enter_vm(vm: &VirtualMachine, f: impl FnOnce() -> R) -> R { set_current_vm(vm, f) } +/// RAII counterpart to `enter_vm`, for code that runs Python bytecode across +/// several statements interspersed with `&mut VirtualMachine` calls +/// (`VirtualMachine::initialize`), where a single closure-based `enter_vm` +/// scope cannot be expressed because the borrow checker won't let a closure +/// hold `&mut VirtualMachine` at the same time `enter_vm` reborrows it as +/// `&VirtualMachine`. Construction only needs a transient `&VirtualMachine` +/// borrow, so it can be dropped before subsequent `&mut` use. +/// +/// Without this, code that runs Python bytecode before any `enter_vm` scope +/// exists would leave the thread not ATTACHED, making lock-free type cache +/// reads unsound. +#[must_use] +pub(crate) struct VmBootstrapGuard { + #[cfg(feature = "threading")] + was_outermost: bool, +} + +impl VmBootstrapGuard { + pub(crate) fn new(vm: &VirtualMachine) -> Self { + // Outermost: transition DETACHED → ATTACHED + #[cfg(feature = "threading")] + let was_outermost = !current_vm_is_set(); + + // Initialize thread slot for this thread if not already done + #[cfg(feature = "threading")] + init_thread_slot_if_needed(vm); + + #[cfg(feature = "threading")] + if was_outermost { + attach_thread(vm); + } + + VM_STACK.with(|vms| vms.borrow_mut().push(vm.into())); + + Self { + #[cfg(feature = "threading")] + was_outermost, + } + } +} + +impl Drop for VmBootstrapGuard { + fn drop(&mut self) { + VM_STACK.with(|vms| { + vms.borrow_mut().pop(); + }); + + // Outermost exit: transition ATTACHED → DETACHED + #[cfg(feature = "threading")] + if self.was_outermost { + detach_thread(); + } + } +} + #[cfg(feature = "threading")] #[derive(Clone, Copy, Debug, Eq, PartialEq)] pub enum CurrentVmAttachState { @@ -161,7 +244,6 @@ pub fn attach_current_thread( init_thread_slot_if_needed(vm); - #[cfg(unix)] attach_thread(vm); VM_STACK.with(|vms| { @@ -188,7 +270,6 @@ pub fn release_current_thread(state: CurrentVmAttachState) { .expect("release_current_thread() called without an attached VM"); }); - #[cfg(unix)] detach_thread(); } @@ -201,9 +282,11 @@ fn init_thread_slot_if_needed(vm: &VirtualMachine) { let thread_id = crate::stdlib::_thread::get_ident(); let mut registry = vm.state.thread_frames.lock(); let new_slot = Arc::new(ThreadSlot { + #[cfg(unix)] + top_frame: AtomicPtr::new(core::ptr::null_mut()), + #[cfg(not(unix))] frames: parking_lot::Mutex::new(Vec::new()), exception: crate::PyAtomicRef::from(None::), - #[cfg(unix)] state: core::sync::atomic::AtomicI32::new( if vm.state.stop_the_world.requested.load(Ordering::Acquire) { // Match init_threadstate(): new thread-state starts @@ -213,13 +296,14 @@ fn init_thread_slot_if_needed(vm: &VirtualMachine) { THREAD_DETACHED }, ), - #[cfg(unix)] stop_requested: core::sync::atomic::AtomicBool::new(false), - #[cfg(unix)] thread: std::thread::current(), + qsbr: crate::object::qsbr::QSBR.register(), }); registry.insert(thread_id, new_slot.clone()); drop(registry); + #[cfg(all(unix, feature = "threading"))] + CURRENT_TOP_FRAME_SLOT.with(|c| c.set(&new_slot.top_frame)); *slot.borrow_mut() = Some(new_slot); } }); @@ -227,7 +311,7 @@ fn init_thread_slot_if_needed(vm: &VirtualMachine) { /// Transition DETACHED → ATTACHED. Blocks if the thread was SUSPENDED by /// a stop-the-world request (like `_PyThreadState_Attach` + `tstate_wait_attach`). -#[cfg(all(unix, feature = "threading"))] +#[cfg(feature = "threading")] fn wait_while_suspended(slot: &ThreadSlot) -> u64 { let mut wait_yields = 0u64; while slot.state.load(Ordering::Acquire) == THREAD_SUSPENDED { @@ -237,7 +321,7 @@ fn wait_while_suspended(slot: &ThreadSlot) -> u64 { wait_yields } -#[cfg(all(unix, feature = "threading"))] +#[cfg(feature = "threading")] fn attach_thread(vm: &VirtualMachine) { CURRENT_THREAD_SLOT.with(|slot| { if let Some(s) = slot.borrow().as_ref() { @@ -250,6 +334,7 @@ fn attach_thread(vm: &VirtualMachine) { Ordering::Relaxed, ) { Ok(_) => { + crate::object::qsbr::QSBR.online(&s.qsbr); super::stw_trace(format_args!("attach DETACHED->ATTACHED")); break; } @@ -268,10 +353,18 @@ fn attach_thread(vm: &VirtualMachine) { } } }); + // A stop-the-world may have been requested while this thread was detached. + // Honoring it here (rather than only at the next bytecode safepoint) keeps + // a thread doing rapid allow_threads calls from re-attaching and running + // past the requester forever, which would stall stop-the-world. Done + // outside the CURRENT_THREAD_SLOT borrow above because suspend re-borrows + // it. Safe against a concurrent start_the_world: suspend_if_needed only + // parks while the request is still live and self-recovers otherwise. + suspend_if_needed(&vm.state.stop_the_world); } /// Transition ATTACHED → DETACHED (like `_PyThreadState_Detach`). -#[cfg(all(unix, feature = "threading"))] +#[cfg(feature = "threading")] fn detach_thread() { CURRENT_THREAD_SLOT.with(|slot| { if let Some(s) = slot.borrow().as_ref() { @@ -281,7 +374,9 @@ fn detach_thread() { Ordering::AcqRel, Ordering::Acquire, ) { - Ok(_) => {} + Ok(_) => { + crate::object::qsbr::QSBR.offline(&s.qsbr); + } Err(THREAD_DETACHED) => { debug_assert!(false, "detach called while already DETACHED"); return; @@ -301,7 +396,7 @@ fn detach_thread() { /// to park this thread during blocking operations. /// /// `Py_BEGIN_ALLOW_THREADS` / `Py_END_ALLOW_THREADS` equivalent. -#[cfg(all(unix, feature = "threading"))] +#[cfg(feature = "threading")] pub fn allow_threads(vm: &VirtualMachine, f: impl FnOnce() -> R) -> R { // Preserve save/restore semantics: // only detach if this call observed ATTACHED at entry, and always restore @@ -322,8 +417,8 @@ pub fn allow_threads(vm: &VirtualMachine, f: impl FnOnce() -> R) -> R { result } -/// No-op on non-unix or non-threading builds. -#[cfg(not(all(unix, feature = "threading")))] +/// No-op on non-threading builds. +#[cfg(not(feature = "threading"))] pub fn allow_threads(_vm: &VirtualMachine, f: impl FnOnce() -> R) -> R { f() } @@ -331,7 +426,7 @@ pub fn allow_threads(_vm: &VirtualMachine, f: impl FnOnce() -> R) -> R { /// Called from check_signals when stop-the-world is requested. /// Transitions ATTACHED → SUSPENDED and waits until released /// (like `_PyThreadState_Suspend` + `_PyThreadState_Attach`). -#[cfg(all(unix, feature = "threading"))] +#[cfg(feature = "threading")] pub fn suspend_if_needed(stw: &super::StopTheWorldState) { let should_suspend = CURRENT_THREAD_SLOT.with(|slot| { slot.borrow() @@ -354,7 +449,7 @@ pub fn suspend_if_needed(stw: &super::StopTheWorldState) { do_suspend(stw); } -#[cfg(all(unix, feature = "threading"))] +#[cfg(feature = "threading")] #[cold] fn do_suspend(stw: &super::StopTheWorldState) { CURRENT_THREAD_SLOT.with(|slot| { @@ -431,7 +526,7 @@ fn do_suspend(stw: &super::StopTheWorldState) { }); } -#[cfg(all(unix, feature = "threading"))] +#[cfg(feature = "threading")] #[inline] #[must_use] pub fn stop_requested_for_current_thread() -> bool { @@ -442,9 +537,55 @@ pub fn stop_requested_for_current_thread() -> bool { }) } +/// Whether the QSBR subsystem asked this thread to pass a checkpoint. +/// A missed or racing read of this flag is harmless: the pending +/// retirement is still processed at the next checkpoint or by the GC +/// backstop. +#[cfg(feature = "threading")] +pub(crate) fn qsbr_break_requested() -> bool { + CURRENT_THREAD_SLOT.with(|slot| { + slot.borrow() + .as_ref() + .is_some_and(|s| s.qsbr.requested.load(Ordering::Relaxed)) + }) +} + +/// Pass a QSBR checkpoint: the calling thread holds no borrowed cache +/// pointers here (instruction boundary), so mark it quiescent and try to +/// free retired allocations. +#[cfg(feature = "threading")] +pub(crate) fn qsbr_checkpoint() { + use crate::object::qsbr::QSBR; + CURRENT_THREAD_SLOT.with(|slot| { + if let Some(s) = slot.borrow().as_ref() { + s.qsbr.requested.store(false, Ordering::Relaxed); + QSBR.quiescent_state(&s.qsbr); + } + }); + QSBR.process(); +} + +/// Debug check: lock-free type-cache reads are only sound on threads that +/// are registered with QSBR and currently ATTACHED. +#[cfg(all(feature = "threading", debug_assertions))] +pub(crate) fn debug_assert_current_thread_attached() { + CURRENT_THREAD_SLOT.with(|slot| { + if let Some(s) = slot.borrow().as_ref() { + debug_assert_eq!( + s.state.load(Ordering::Relaxed), + THREAD_ATTACHED, + "type cache read while thread not ATTACHED" + ); + } + }); +} + /// Push a frame pointer onto the current thread's shared frame stack. /// The pointed-to frame must remain alive until the matching pop. -#[cfg(feature = "threading")] +/// +/// Only used on non-unix threading builds; unix builds publish the top frame +/// through `set_current_frame` writing `ThreadSlot::top_frame`. +#[cfg(all(not(unix), feature = "threading"))] pub fn push_thread_frame(fp: FramePtr) { CURRENT_THREAD_SLOT.with(|slot| { if let Some(s) = slot.borrow().as_ref() { @@ -460,7 +601,7 @@ pub fn push_thread_frame(fp: FramePtr) { /// Pop a frame from the current thread's shared frame stack. /// Called when a frame is exited. -#[cfg(feature = "threading")] +#[cfg(all(not(unix), feature = "threading"))] pub fn pop_thread_frame() { CURRENT_THREAD_SLOT.with(|slot| { if let Some(s) = slot.borrow().as_ref() { @@ -477,6 +618,17 @@ pub fn pop_thread_frame() { /// Set the current thread's top frame pointer for signal-safe traceback walking. /// Returns the previous frame pointer so it can be restored on pop. pub fn set_current_frame(frame: *const Frame) -> *const Frame { + // Publish the top frame for cross-thread readers. The relaxed store is + // ordered by stop-the-world at read time (see `ThreadSlot::top_frame`). + #[cfg(all(unix, feature = "threading"))] + { + let slot_top = CURRENT_TOP_FRAME_SLOT.with(Cell::get); + if !slot_top.is_null() { + // SAFETY: points to this thread's `ThreadSlot::top_frame`, kept + // alive by the Arc in `CURRENT_THREAD_SLOT` for the thread's life. + unsafe { (*slot_top).store(frame as *mut Frame, Ordering::Relaxed) }; + } + } CURRENT_FRAME.with(|c| c.swap(frame as *mut Frame, Ordering::Relaxed) as *const Frame) } @@ -519,7 +671,7 @@ pub fn cleanup_current_thread_frames(vm: &VirtualMachine) { // A dying thread should not remain logically ATTACHED while its // thread-state slot is being removed. - #[cfg(all(unix, feature = "threading"))] + #[cfg(feature = "threading")] if let Some(slot) = ¤t_slot { let _ = slot.state.compare_exchange( THREAD_ATTACHED, @@ -541,7 +693,7 @@ pub fn cleanup_current_thread_frames(vm: &VirtualMachine) { None }; - #[cfg(all(unix, feature = "threading"))] + #[cfg(feature = "threading")] if let Some(slot) = &_removed && vm.state.stop_the_world.requested.load(Ordering::Acquire) && thread_id != vm.state.stop_the_world.requester_ident() @@ -551,6 +703,10 @@ pub fn cleanup_current_thread_frames(vm: &VirtualMachine) { // Unblock requester countdown progress. vm.state.stop_the_world.notify_thread_gone(); } + // Clear the cached top-frame pointer before dropping the slot Arc so no + // later `set_current_frame` dereferences freed slot memory. + #[cfg(all(unix, feature = "threading"))] + CURRENT_TOP_FRAME_SLOT.with(|c| c.set(core::ptr::null())); CURRENT_THREAD_SLOT.with(|s| { *s.borrow_mut() = None; }); @@ -558,24 +714,43 @@ pub fn cleanup_current_thread_frames(vm: &VirtualMachine) { /// Reinitialize thread slot after fork. Called in child process. /// Creates a fresh slot and registers it for the current thread, -/// preserving the current thread's frames from `vm.frames`. +/// preserving the current thread's frames from the signal-safe frame chain. /// /// Precondition: `reinit_locks_after_fork()` has already reset all /// VmState locks to unlocked. #[cfg(feature = "threading")] pub fn reinit_frame_slot_after_fork(vm: &VirtualMachine) { let current_ident = crate::stdlib::_thread::get_ident(); - let current_frames: Vec = vm.frames.borrow().clone(); + // On non-unix, rebuild the shared frame stack (bottom-to-top) from the + // current thread's frame chain, which walks top-to-bottom via `previous`. + #[cfg(not(unix))] + let current_frames: Vec = { + let mut current_frames = Vec::new(); + let mut cur = get_current_frame(); + while !cur.is_null() { + // SAFETY: the forking thread's chain frames are alive. + let py = unsafe { crate::Py::::from_payload_ptr(cur) }; + current_frames.push(FramePtr(unsafe { NonNull::new_unchecked(py as *mut _) })); + cur = unsafe { (*cur).previous_frame() }; + } + current_frames.reverse(); + current_frames + }; let new_slot = Arc::new(ThreadSlot { + // The surviving child thread keeps executing its current frame chain, + // whose top is the signal-safe `get_current_frame()`. + #[cfg(unix)] + top_frame: AtomicPtr::new(get_current_frame() as *mut Frame), + #[cfg(not(unix))] frames: parking_lot::Mutex::new(current_frames), exception: crate::PyAtomicRef::from(vm.topmost_exception()), - #[cfg(unix)] state: core::sync::atomic::AtomicI32::new(THREAD_ATTACHED), - #[cfg(unix)] stop_requested: core::sync::atomic::AtomicBool::new(false), - #[cfg(unix)] thread: std::thread::current(), + qsbr: crate::object::qsbr::QSBR.register(), }); + #[cfg(all(unix, feature = "threading"))] + CURRENT_TOP_FRAME_SLOT.with(|c| c.set(&new_slot.top_frame)); // Lock is safe: reinit_locks_after_fork() already reset it to unlocked. let mut registry = vm.state.thread_frames.lock(); @@ -710,7 +885,6 @@ impl VirtualMachine { builtins: self.builtins.clone(), sys_module: self.sys_module.clone(), ctx: self.ctx.clone(), - frames: RefCell::new(vec![]), datastack: core::cell::UnsafeCell::new(crate::datastack::DataStack::new()), wasm_id: self.wasm_id.clone(), exceptions: RefCell::default(), diff --git a/crates/vm/src/vm/vm_new.rs b/crates/vm/src/vm/vm_new.rs index c71cf842520..ae728aaba67 100644 --- a/crates/vm/src/vm/vm_new.rs +++ b/crates/vm/src/vm/vm_new.rs @@ -10,7 +10,7 @@ use rustpython_compiler_core::SourceLocation; use rustpython_compiler::{CompileError, ParseError}; use crate::{ - AsObject, Py, PyObject, PyObjectRef, PyRef, PyResult, + AsObject, Py, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, builtins::{ PyBaseException, PyBaseExceptionRef, PyBytesRef, PyDictRef, PyModule, PyOSError, PyStrRef, PyType, PyTypeRef, @@ -348,11 +348,9 @@ impl VirtualMachine { exc_type.name() ); - PyRef::new_ref( - PyBaseException::new(args, self), - exc_type, - Some(self.ctx.new_dict()), - ) + PyBaseException::new(args, self) + .into_ref_with_type_lazy_dict(self, exc_type) + .expect("vm.new_exception() called with an invalid exception type") } pub fn new_os_error(&self, msg: impl ToPyObject) -> PyRef { diff --git a/extra_tests/custom_text_test_runner.py b/extra_tests/custom_text_test_runner.py index 3457bdfd0e4..865265750f4 100644 --- a/extra_tests/custom_text_test_runner.py +++ b/extra_tests/custom_text_test_runner.py @@ -389,11 +389,9 @@ def startTest(self, test): } self.start_time = time.time() if self.test_types: - if "test_type" in getattr( - test, test._testMethodName - ).__func__.__dict__ and set([s.lower() for s in self.test_types]) == set( - [s.lower() for s in _get_method_dict(test)["test_type"]] - ): + if "test_type" in _get_method_dict(test) and set( + [s.lower() for s in self.test_types] + ) == set([s.lower() for s in _get_method_dict(test)["test_type"]]): pass else: _get_method_dict(test)["__unittest_skip_why__"] = ( diff --git a/extra_tests/snippets/builtin_type_bases.py b/extra_tests/snippets/builtin_type_bases.py new file mode 100644 index 00000000000..1d413e48e5c --- /dev/null +++ b/extra_tests/snippets/builtin_type_bases.py @@ -0,0 +1,293 @@ +from testutils import assert_raises + +# Reassigning __bases__ must rebuild slot dispatchers for the type and all its +# descendants: a slot whose method left the new MRO must be reset, not left stale. + + +# --- zelf itself loses __add__ (nb_add) --- +class OldAdd: + def __add__(self, other): + return "OLD" + + +class Bare: + pass + + +class C(OldAdd): + pass + + +c = C() +assert c + 1 == "OLD" +C.__bases__ = (Bare,) +with assert_raises(TypeError): + c + 1 + + +# --- 3-level descendant loses __iter__ (tp_iter) --- +class Itr: + def __iter__(self): + return iter([1, 2, 3]) + + +class New: + pass + + +class C2(Itr): + pass + + +class D2(C2): + pass + + +class E2(D2): + pass + + +e = E2() +assert list(e) == [1, 2, 3] +C2.__bases__ = (New,) +with assert_raises(TypeError): + list(e) + + +# --- descendant loses __len__ (sq_length), sibling slot untouched --- +class Sized: + def __len__(self): + return 7 + + +class C3(Sized): + pass + + +class D3(C3): + pass + + +d3 = D3() +assert len(d3) == 7 +C3.__bases__ = (Bare,) +with assert_raises(TypeError): + len(d3) + + +# --- descendant loses __getitem__ (mp_subscript) --- +class Subscriptable: + def __getitem__(self, key): + return key * 2 + + +class C4(Subscriptable): + pass + + +class D4(C4): + pass + + +d4 = D4() +assert d4[3] == 6 +C4.__bases__ = (Bare,) +with assert_raises(TypeError): + d4[3] + + +# --- descendant loses __call__ (tp_call) --- +class Callable: + def __call__(self): + return "called" + + +class C5(Callable): + pass + + +class D5(C5): + pass + + +d5 = D5() +assert d5() == "called" +C5.__bases__ = (Bare,) +with assert_raises(TypeError): + d5() + + +# --- guard: stale-wrong-target, name present in both bases must switch --- +class OldTarget: + def __add__(self, other): + return "OLD.__add__" + + +class NewTarget: + def __add__(self, other): + return "NEW.__add__" + + +class C6(OldTarget): + pass + + +class D6(C6): + pass + + +d6 = D6() +assert d6 + 1 == "OLD.__add__" +C6.__bases__ = (NewTarget,) +assert d6 + 1 == "NEW.__add__" + + +# --- guard: __getattr__ resolves at call time, stays correct --- +class OldGetattr: + def __getattr__(self, name): + return "OLD:" + name + + +class C7(OldGetattr): + pass + + +class D7(C7): + pass + + +d7 = D7() +assert d7.missing == "OLD:missing" +C7.__bases__ = (Bare,) +with assert_raises(AttributeError): + d7.missing + + +# --- mirror: new base ADDS a dunder the old chain lacked --- +class Adder: + def __add__(self, other): + return "ADDED" + + +class C8(Bare): + pass + + +class D8(C8): + pass + + +d8 = D8() +with assert_raises(TypeError): + d8 + 1 +C8.__bases__ = (Adder,) +assert d8 + 1 == "ADDED" + + +# --- round trip: swap away then back restores the slot --- +class C9(OldAdd): + pass + + +class D9(C9): + pass + + +d9 = D9() +assert d9 + 1 == "OLD" +C9.__bases__ = (Bare,) +with assert_raises(TypeError): + d9 + 1 +C9.__bases__ = (OldAdd,) +assert d9 + 1 == "OLD" + + +# --- left-only __add__ defined on the type itself survives a base swap --- +# __add__ and __radd__ share one accessor but occupy distinct fields; resolving +# the absent __radd__ must not overwrite the __add__ dispatcher. +class Mixin: + pass + + +class Other: + pass + + +class C10(Mixin): + def __add__(self, o): + return "C10" + + +c10 = C10() +assert c10 + 1 == "C10" +C10.__bases__ = (Other,) +assert c10 + 1 == "C10" + + +# --- right-only __radd__ survives a base swap --- +class C11(Mixin): + def __radd__(self, o): + return "C11" + + +c11 = C11() +assert 1 + c11 == "C11" +C11.__bases__ = (Other,) +assert 1 + c11 == "C11" + + +# --- subclass/grandchild shadowing __add__ keeps it when an ancestor swaps bases --- +class AddBase: + def __add__(self, o): + return "AddBase" + + +class Ancestor(AddBase): + pass + + +class Shadow(Ancestor): + def __add__(self, o): + return "Shadow" + + +class GrandShadow(Shadow): + pass + + +sh = Shadow() +gsh = GrandShadow() +assert sh + 1 == "Shadow" +assert gsh + 1 == "Shadow" +Ancestor.__bases__ = (Mixin,) +assert sh + 1 == "Shadow" +assert gsh + 1 == "Shadow" + + +# --- another Nb* pair: left-only __sub__ survives a base swap --- +class C12(Mixin): + def __sub__(self, o): + return "C12" + + +c12 = C12() +assert c12 - 1 == "C12" +C12.__bases__ = (Other,) +assert c12 - 1 == "C12" + + +# --- setattr/delattr-driven right-op updates keep the left op intact --- +class C13: + def __add__(self, o): + return "C13.add" + + +c13 = C13() +assert c13 + 1 == "C13.add" +C13.__radd__ = lambda self, o: "C13.radd" +assert c13 + 1 == "C13.add" +assert 1 + c13 == "C13.radd" +del C13.__radd__ +assert c13 + 1 == "C13.add" +with assert_raises(TypeError): + 1 + c13 diff --git a/extra_tests/snippets/stdlib_threading_gc_fork.py b/extra_tests/snippets/stdlib_threading_gc_fork.py new file mode 100644 index 00000000000..cd00cf00983 --- /dev/null +++ b/extra_tests/snippets/stdlib_threading_gc_fork.py @@ -0,0 +1,62 @@ +"""Fork while other threads drive concurrent GC stop-the-world. + +fork() and the cycle collector both stop the world through the same shared +state. Without a single exclusion around each stop->start span, an interleaving +of the fork requester and a GC requester clobbers that state (requester word, +suspension countdown) so the completion check never converges and a requester +waits on itself forever. + +Worker threads allocate cyclic garbage with GC enabled while the main thread +forks repeatedly; each child collects and exits. A regression shows up as a +hang in the parent (never finishing the fork loop). The allocation rate is kept +light so the collection stays cheap even in unoptimized builds. +""" + +import gc +import os +import threading +import time + +if not hasattr(os, "fork"): + print("skipped (no fork)") + raise SystemExit(0) + +gc.enable() +stop = threading.Event() + + +def churn(): + while not stop.is_set(): + a = {} + b = {"a": a} + a["b"] = b # cycle collectable only by the cycle collector + lst = [a, b] + lst.append(lst) + del a, b, lst + # Throttle so the collector keeps the heap small; the point is to + # interleave fork with concurrent collections, not to grow the heap. + time.sleep(0.001) + + +workers = [threading.Thread(target=churn) for _ in range(4)] +for w in workers: + w.start() + +# Let the workers get going before forking. +time.sleep(0.05) + +N = 25 +for _ in range(N): + pid = os.fork() + if pid == 0: + # Child: run its own stop-the-world collection, then exit. + gc.collect() + os._exit(0) + _, status = os.waitpid(pid, 0) + assert status == 0, status + +stop.set() +for w in workers: + w.join() + +print("ok") diff --git a/extra_tests/snippets/stdlib_threading_gc_frame_race.py b/extra_tests/snippets/stdlib_threading_gc_frame_race.py new file mode 100644 index 00000000000..37cdbbf122c --- /dev/null +++ b/extra_tests/snippets/stdlib_threading_gc_frame_race.py @@ -0,0 +1,101 @@ +"""Stress GC traversal against concurrently executing frames. + +The cycle collector reads each tracked object's interpreter state, including +the data stack and fast locals of frames that other threads are actively +executing. Those slots are written without synchronization by the running +thread, so the collector must only read them while the world is stopped. + +Workers churn frame state hard: deep recursion (many nested frames), heavy +local rebind / stack traffic, and generators repeatedly resumed. Meanwhile a +collector thread loops gc.collect() and an introspector walks live frame +objects via gc.get_objects(). A regression (torn read of a running frame) +shows up as a crash, a use-after-free, or a hang. +""" + +import gc +import sys +import threading +import time + +DURATION = 1.5 + + +def deep(n): + # Deep recursion + local rebind churns fast locals and the data stack. + a = n + b = [n, n + 1] + c = {"k": a} + if n <= 0: + return a + len(b) + len(c) + a = a - 1 + b.append(a) + return deep(n - 1) + a + + +def gen_worker(): + def counter(limit): + acc = 0 + i = 0 + while i < limit: + box = {"i": i} + box["self"] = box # a cycle held by the running generator frame + acc += i + yield acc + i += 1 + + g = counter(200) + total = 0 + for v in g: + total += v + return total + + +def make_frame_cycles(n): + for _ in range(n): + + def inner(): + fr = sys._getframe() + box = {"fr": fr} + box["self"] = box + return None + + inner() + + +def worker(stop): + # deep() nesting is kept modest so the recursion also fits the smaller + # worker-thread stack of unoptimized (debug) builds; the generators and + # frame cycles supply the rest of the frame churn. + while not stop.is_set(): + deep(12) + gen_worker() + make_frame_cycles(20) + + +def collector(stop): + while not stop.is_set(): + gc.collect() + + +def introspector(stop): + while not stop.is_set(): + for o in gc.get_objects(): + if type(o).__name__ == "frame": + try: + _ = o.f_lineno + _ = o.f_code.co_name + except Exception: + pass + + +stop = threading.Event() +threads = [threading.Thread(target=worker, args=(stop,)) for _ in range(4)] +threads.append(threading.Thread(target=collector, args=(stop,))) +threads.append(threading.Thread(target=introspector, args=(stop,))) +for t in threads: + t.start() +time.sleep(DURATION) +stop.set() +for t in threads: + t.join() +print("ok") diff --git a/extra_tests/snippets/stdlib_threading_gc_import.py b/extra_tests/snippets/stdlib_threading_gc_import.py new file mode 100644 index 00000000000..340184093f3 --- /dev/null +++ b/extra_tests/snippets/stdlib_threading_gc_import.py @@ -0,0 +1,58 @@ +"""Concurrent imports plus GC stop-the-world must not deadlock (no fork). + +The global import lock is held across bytecode by the importlib bootstrap, so +its holder can be parked at a safepoint mid-hold. If another thread blocks on +that lock while attached, a GC stop-the-world requester waits forever for that +attached thread to suspend while the lock holder stays parked -- a three-party +deadlock. Acquiring the import lock must therefore detach so the wait honors a +stop-the-world request. + +Two threads repeatedly re-import modules (contending the import lock) while a +third storms the cycle collector and a fourth allocates cyclic garbage. A +regression shows up as a hang (the importer threads never finishing). +""" + +import gc +import importlib +import sys +import threading + +gc.enable() +stop = threading.Event() + +# Modules cheap to import and safe to drop/re-import repeatedly. +MODS = ("colorsys", "stringprep") +ITERS = 2000 + + +def importer(mod): + for _ in range(ITERS): + if stop.is_set(): + break + sys.modules.pop(mod, None) + importlib.import_module(mod) + + +def collector(): + while not stop.is_set(): + gc.collect() + + +def allocator(): + while not stop.is_set(): + y = [{"i": i} for i in range(50)] + y[0]["self"] = y # cycle collectable only by the cycle collector + + +importers = [threading.Thread(target=importer, args=(m,)) for m in MODS] +helpers = [threading.Thread(target=collector), threading.Thread(target=allocator)] + +for t in importers + helpers: + t.start() +for t in importers: + t.join() +stop.set() +for t in helpers: + t.join() + +print("ok") diff --git a/extra_tests/snippets/stdlib_threading_type_cache.py b/extra_tests/snippets/stdlib_threading_type_cache.py new file mode 100644 index 00000000000..92368e8b83d --- /dev/null +++ b/extra_tests/snippets/stdlib_threading_type_cache.py @@ -0,0 +1,69 @@ +"""Stress the lock-free type method cache against concurrent type mutation. + +Readers hammer method lookups while a mutator continuously replaces and +deletes the method, dropping the old function objects. Guards against +use-after-free in the cache read protocol (QSBR deferred reclamation). + +Also churns a freelist-eligible published value (a tuple class attribute): +tuples normally go back through the freelist on dealloc, but once one is +published to the type cache it must instead go through the QSBR-deferred +reclamation path, so this exercises that bypass. +""" + +import threading +import time + + +class C: + def m(self): + return -1 + + +DURATION = 1.5 + + +def reader(stop): + obj = C() + while not stop.is_set(): + for _ in range(1000): + try: + obj.m() + except AttributeError: + pass + try: + obj.shape + except AttributeError: + pass + + +def mutator(stop): + i = 0 + while not stop.is_set(): + + def m(self, _i=i): + return _i + + C.m = m + C.shape = (i, i + 1) + i += 1 + if i % 97 == 0: + try: + del C.m + except AttributeError: + pass + try: + del C.shape + except AttributeError: + pass + + +stop = threading.Event() +threads = [threading.Thread(target=reader, args=(stop,)) for _ in range(4)] +threads.append(threading.Thread(target=mutator, args=(stop,))) +for t in threads: + t.start() +time.sleep(DURATION) +stop.set() +for t in threads: + t.join() +print("ok") diff --git a/tools/opcode_metadata/generate_rs_opcode_metadata.py b/tools/opcode_metadata/generate_rs_opcode_metadata.py index fd13e026613..97482f337a1 100644 --- a/tools/opcode_metadata/generate_rs_opcode_metadata.py +++ b/tools/opcode_metadata/generate_rs_opcode_metadata.py @@ -28,6 +28,7 @@ def fn_as_info_size(self) -> str: return f""" /// Returns [`Self`] as [`{self.size}`]. #[must_use] + #[inline] pub const fn as_{self.size}(self) -> {self.size} {{ self.as_numeric() }} @@ -113,6 +114,7 @@ def fn_to_base(self) -> str: return f""" #[must_use] + #[inline] pub const fn to_base(self) -> Option {{ {inner} }} @@ -146,25 +148,30 @@ def fn_to_instrumented(self) -> str: @property def fn_deopt(self) -> str: - arms = "" - for target, specialized in self.info.deopts.items(): - ops = "|".join(f"Self::{op}" for op in specialized) - arms += f"{ops} => Self::{target},\n" + specialized_to_base = self.specialized_to_base - arms = arms.strip() - - if not arms: + if not specialized_to_base: inner = "None" else: + table_type = f"super::{self.info.enum_name}" + entries = ",\n".join( + f"Some({table_type}::{specialized_to_base[name]})" + if name in specialized_to_base + else "None" + for name in self.rust_names_by_id + ) + inner = f""" - Some(match self {{ - {arms} - _ => return None, - }}) + const DEOPT: [Option<{table_type}>; {self.table_size}] = [ + {entries} + ]; + + DEOPT[self.as_numeric() as usize] """ return f""" #[must_use] + #[inline] pub const fn deopt(self) -> Option {{ {inner} }} @@ -172,7 +179,7 @@ def fn_deopt(self) -> str: @property def fn_cache_entries(self) -> str: - arms = "" + entries_by_base: dict[str, int] = {} for opcode in self: name = opcode.rust_name if opcode.is_instrumented: @@ -186,21 +193,25 @@ def fn_cache_entries(self) -> str: continue if size > 1: - arms += f"Self::{name} => {size - 1},\n" + entries_by_base[name] = size - 1 - arms = arms.strip() - if not arms: + if not entries_by_base: inner = "0" else: + entries = ", ".join( + str(entries_by_base.get(self.resolve_deoptimized(name), 0)) + for name in self.rust_names_by_id + ) + inner = f""" - match self.deoptimize() {{ - {arms} - _ => 0, - }} + const CACHE_ENTRIES: [u8; {self.table_size}] = [{entries}]; + + CACHE_ENTRIES[self.as_numeric() as usize] as usize """ return f""" #[must_use] + #[inline] pub const fn cache_entries(self) -> usize {{ {inner} }} @@ -323,6 +334,42 @@ def instrumented_mapping(self) -> dict[str, str]: return res + @property + def specialized_to_base(self) -> dict[str, str]: + """Maps a specialized opcode's name to its family's base opcode name.""" + res = {} + for target, specialized in self.info.deopts.items(): + for name in specialized: + res[name] = target + + return res + + @property + def instrumented_to_base(self) -> dict[str, str]: + """Maps an instrumented opcode's name to its base opcode name.""" + return {iname: name for name, iname in self.instrumented_mapping.items()} + + def resolve_deoptimized(self, name: str) -> str: + """ + Mirrors `deoptimize`: resolves a specialized opcode to its family's + base, an instrumented opcode to its base, or returns the name + unchanged. + """ + if name in self.specialized_to_base: + return self.specialized_to_base[name] + + return self.instrumented_to_base.get(name, name) + + @property + def table_size(self) -> int: + return {"u8": 256, "u16": 65536}[self.size] + + @property + def rust_names_by_id(self) -> list[str | None]: + """The opcode name at each numeric id, `None` where no opcode is assigned.""" + names_by_id = {opcode.id: opcode.rust_name for opcode in self} + return [names_by_id.get(i) for i in range(self.table_size)] + @property def size(self) -> str: return self.info.size