From fc5236e9403b47c4aeeabb3a1f594304785587b7 Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Sat, 23 May 2026 19:08:48 +0900 Subject: [PATCH 01/10] Share marshal ref table between code object and its internals read_marshal_bytes, _str, _str_vec, _name_tuple, and _const_tuple now take a shared ref table and resolve TYPE_REF / register FLAG_REF entries. deserialize_code is split into a public wrapper and an inner function that receives the ref table; deserialize_value_depth opens a fresh inner ref space when it hits Type::Code, mirroring CPython's behaviour of putting the code object itself at ref slot 0. Nested code objects inside const tuples reuse the surrounding code's ref space via the new read_const_value helper. --- crates/compiler-core/src/marshal.rs | 269 ++++++++++++++++++++++++---- 1 file changed, 230 insertions(+), 39 deletions(-) diff --git a/crates/compiler-core/src/marshal.rs b/crates/compiler-core/src/marshal.rs index 829c1dc9519..3a9acfd5aa1 100644 --- a/crates/compiler-core/src/marshal.rs +++ b/crates/compiler-core/src/marshal.rs @@ -193,6 +193,18 @@ impl> Read for Cursor { pub fn deserialize_code( rdr: &mut R, bag: Bag, +) -> Result> { + let mut refs: Vec> = Vec::new(); + deserialize_code_inner(rdr, bag, &mut refs) +} + +/// Inner code-object deserializer that shares a ref table with caller. +/// Used when decoding a code object embedded in another marshal stream so +/// that TYPE_REF entries inside the code can resolve across nested values. +fn deserialize_code_inner( + rdr: &mut R, + bag: Bag, + refs: &mut Vec>, ) -> Result> { // 1–5: scalar fields let arg_count = rdr.read_u32()?; @@ -202,24 +214,24 @@ pub fn deserialize_code( let flags = CodeFlags::from_bits_truncate(rdr.read_u32()?); // 6: co_code - let code_bytes = read_marshal_bytes(rdr)?; + let code_bytes = read_marshal_bytes(rdr, &bag, refs)?; // 7: co_consts - let constants = read_marshal_const_tuple(rdr, bag)?; + let constants = read_marshal_const_tuple(rdr, bag, refs)?; // 8: co_names - let names = read_marshal_name_tuple(rdr, &bag)?; + let names = read_marshal_name_tuple(rdr, &bag, refs)?; // 9: co_localsplusnames - let localsplusnames = read_marshal_str_vec(rdr)?; + let localsplusnames = read_marshal_str_vec(rdr, &bag, refs)?; // 10: co_localspluskinds - let localspluskinds = read_marshal_bytes(rdr)?; + let localspluskinds = read_marshal_bytes(rdr, &bag, refs)?; // 11–13: filename, name, qualname - let source_path = bag.make_name(&read_marshal_str(rdr)?); - let obj_name = bag.make_name(&read_marshal_str(rdr)?); - let qualname = bag.make_name(&read_marshal_str(rdr)?); + let source_path = bag.make_name(&read_marshal_str(rdr, &bag, refs)?); + let obj_name = bag.make_name(&read_marshal_str(rdr, &bag, refs)?); + let qualname = bag.make_name(&read_marshal_str(rdr, &bag, refs)?); // 14: co_firstlineno let first_line_raw = rdr.read_u32()? as i32; @@ -230,8 +242,8 @@ pub fn deserialize_code( }; // 15–16: linetable, exceptiontable - let linetable = read_marshal_bytes(rdr)?.to_vec().into_boxed_slice(); - let exceptiontable = read_marshal_bytes(rdr)?.to_vec().into_boxed_slice(); + let linetable = read_marshal_bytes(rdr, &bag, refs)?.into_boxed_slice(); + let exceptiontable = read_marshal_bytes(rdr, &bag, refs)?.into_boxed_slice(); // Split localsplusnames/kinds → varnames/cellvars/freevars let lp = split_localplus( @@ -275,72 +287,235 @@ pub fn deserialize_code( }) } -/// Read a marshal bytes object (TYPE_STRING = b's'). -fn read_marshal_bytes(rdr: &mut R) -> Result> { - let type_byte = rdr.read_u8()? & !FLAG_REF; +/// Reserve a ref slot if `FLAG_REF` was present, returning its index. +fn reserve_ref_slot(has_flag: bool, refs: &mut Vec>) -> Option { + if has_flag { + let idx = refs.len(); + refs.push(None); + Some(idx) + } else { + None + } +} + +/// Resolve a TYPE_REF index, returning the previously stored value. +fn resolve_ref(idx: usize, refs: &[Option]) -> Result { + refs.get(idx) + .and_then(|v| v.clone()) + .ok_or(MarshalError::InvalidBytecode) +} + +/// Read a marshal bytes object (TYPE_STRING = b's'), resolving TYPE_REF +/// and registering this read in the ref table when `FLAG_REF` is set. +fn read_marshal_bytes( + rdr: &mut R, + bag: &Bag, + refs: &mut Vec>, +) -> Result> { + let raw = rdr.read_u8()?; + let type_byte = raw & !FLAG_REF; + let has_flag = raw & FLAG_REF != 0; + + if type_byte == Type::Ref as u8 { + let idx = rdr.read_u32()? as usize; + let stored = resolve_ref(idx, refs)?; + return match stored.borrow_constant() { + BorrowedConstant::Bytes { value } => Ok(value.to_vec()), + _ => Err(MarshalError::BadType), + }; + } + if type_byte != Type::Bytes as u8 { return Err(MarshalError::BadType); } + + let slot = reserve_ref_slot(has_flag, refs); let len = rdr.read_u32()?; - Ok(rdr.read_slice(len)?.to_vec()) + let bytes = rdr.read_slice(len)?.to_vec(); + if let Some(idx) = slot { + refs[idx] = Some(bag.make_constant::(BorrowedConstant::Bytes { + value: &bytes, + })); + } + Ok(bytes) } -/// Read a marshal string object. -fn read_marshal_str(rdr: &mut R) -> Result { - let type_byte = rdr.read_u8()? & !FLAG_REF; - let s = match type_byte { +/// Read a marshal string object, resolving TYPE_REF and registering +/// this read in the ref table when `FLAG_REF` is set. +fn read_marshal_str( + rdr: &mut R, + bag: &Bag, + refs: &mut Vec>, +) -> Result { + let raw = rdr.read_u8()?; + let type_byte = raw & !FLAG_REF; + let has_flag = raw & FLAG_REF != 0; + + if type_byte == Type::Ref as u8 { + let idx = rdr.read_u32()? as usize; + let stored = resolve_ref(idx, refs)?; + return match stored.borrow_constant() { + BorrowedConstant::Str { value } => Ok(value.to_string_lossy().into_owned()), + _ => Err(MarshalError::BadType), + }; + } + + let slot = reserve_ref_slot(has_flag, refs); + let owned = match type_byte { b'u' | b't' | b'a' | b'A' => { let len = rdr.read_u32()?; - rdr.read_str(len)? + alloc::string::String::from(rdr.read_str(len)?) } b'z' | b'Z' => { let len = rdr.read_u8()? as u32; - rdr.read_str(len)? + alloc::string::String::from(rdr.read_str(len)?) } _ => return Err(MarshalError::BadType), }; - Ok(alloc::string::String::from(s)) + if let Some(idx) = slot { + refs[idx] = Some(bag.make_constant::(BorrowedConstant::Str { + value: Wtf8::new(owned.as_str()), + })); + } + Ok(owned) } /// Read a marshal tuple of strings, returning owned Strings. -fn read_marshal_str_vec(rdr: &mut R) -> Result> { - let type_byte = rdr.read_u8()? & !FLAG_REF; +fn read_marshal_str_vec( + rdr: &mut R, + bag: &Bag, + refs: &mut Vec>, +) -> Result> { + let raw = rdr.read_u8()?; + let type_byte = raw & !FLAG_REF; + let has_flag = raw & FLAG_REF != 0; + + if type_byte == Type::Ref as u8 { + let idx = rdr.read_u32()? as usize; + let stored = resolve_ref(idx, refs)?; + return match stored.borrow_constant() { + BorrowedConstant::Tuple { elements } => elements + .iter() + .map(|c| match c.borrow_constant() { + BorrowedConstant::Str { value } => Ok(value.to_string_lossy().into_owned()), + _ => Err(MarshalError::BadType), + }) + .collect(), + _ => Err(MarshalError::BadType), + }; + } + let n = match type_byte { b'(' => rdr.read_u32()? as usize, b')' => rdr.read_u8()? as usize, _ => return Err(MarshalError::BadType), }; - (0..n).map(|_| read_marshal_str(rdr)).collect() + let slot = reserve_ref_slot(has_flag, refs); + let items: Vec = (0..n) + .map(|_| read_marshal_str(rdr, bag, refs)) + .collect::>()?; + if let Some(idx) = slot { + let elements: Vec = items + .iter() + .map(|s| { + bag.make_constant::(BorrowedConstant::Str { + value: Wtf8::new(s.as_str()), + }) + }) + .collect(); + refs[idx] = Some(bag.make_constant::(BorrowedConstant::Tuple { + elements: &elements, + })); + } + Ok(items) } fn read_marshal_name_tuple( rdr: &mut R, bag: &Bag, + refs: &mut Vec>, ) -> Result::Name]>> { - let type_byte = rdr.read_u8()? & !FLAG_REF; - let n = match type_byte { - b'(' => rdr.read_u32()? as usize, - b')' => rdr.read_u8()? as usize, - _ => return Err(MarshalError::BadType), - }; - (0..n) - .map(|_| Ok(bag.make_name(&read_marshal_str(rdr)?))) - .collect::>>() - .map(Vec::into_boxed_slice) + let names = read_marshal_str_vec(rdr, bag, refs)?; + Ok(names + .iter() + .map(|s| bag.make_name(s)) + .collect::>() + .into_boxed_slice()) } -/// Read a marshal tuple of constants. +/// Read a marshal tuple of constants. Shares the ref table with the +/// surrounding code-object decode so that nested TYPE_REF entries (for +/// strings, bytes, code objects, etc.) resolve correctly. fn read_marshal_const_tuple( rdr: &mut R, bag: Bag, + refs: &mut Vec>, ) -> Result> { - let type_byte = rdr.read_u8()? & !FLAG_REF; + let raw = rdr.read_u8()?; + let type_byte = raw & !FLAG_REF; + let has_flag = raw & FLAG_REF != 0; + + if type_byte == Type::Ref as u8 { + let idx = rdr.read_u32()? as usize; + let stored = resolve_ref(idx, refs)?; + return match stored.borrow_constant() { + BorrowedConstant::Tuple { elements } => Ok(elements.iter().cloned().collect()), + _ => Err(MarshalError::BadType), + }; + } + let n = match type_byte { b'(' => rdr.read_u32()? as usize, b')' => rdr.read_u8()? as usize, _ => return Err(MarshalError::BadType), }; - (0..n).map(|_| deserialize_value(rdr, bag)).collect() + let slot = reserve_ref_slot(has_flag, refs); + let items: Vec = (0..n) + .map(|_| read_const_value(rdr, bag, MAX_MARSHAL_STACK_DEPTH, refs)) + .collect::>()?; + if let Some(idx) = slot { + refs[idx] = Some(bag.make_constant::(BorrowedConstant::Tuple { + elements: &items, + })); + } + Ok(items.into_iter().collect()) +} + +/// Read a single value while staying inside an existing code-object ref +/// space. Unlike `deserialize_value_depth`, encountering `Type::Code` +/// here reuses the caller's ref table instead of opening a fresh one — +/// this matches CPython's single global ref space for objects nested +/// inside a code object's const tuple. +fn read_const_value( + rdr: &mut R, + bag: Bag, + depth: usize, + refs: &mut Vec>, +) -> Result { + if depth == 0 { + return Err(MarshalError::InvalidBytecode); + } + let raw = rdr.read_u8()?; + let flag = raw & FLAG_REF != 0; + let type_code = raw & !FLAG_REF; + + if type_code == Type::Ref as u8 { + let idx = rdr.read_u32()? as usize; + return resolve_ref(idx, refs); + } + + let slot = reserve_ref_slot(flag, refs); + let typ = Type::try_from(type_code)?; + let value = if matches!(typ, Type::Code) { + let code = deserialize_code_inner(rdr, bag, refs)?; + bag.make_code(code) + } else { + deserialize_value_typed(rdr, bag, depth, refs, typ)? + }; + if let Some(idx) = slot { + refs[idx] = Some(value.clone()); + } + Ok(value) } pub trait MarshalBag: Copy { @@ -528,7 +703,23 @@ fn deserialize_value_depth( }; let typ = Type::try_from(type_code)?; - let value = deserialize_value_typed(rdr, bag, depth, refs, typ)?; + // Code-objects keep their own inner ref table because Bag::Value (the + // outer marshal value) and the constant-bag's Constant type are not + // in general the same. When the outer header carried FLAG_REF, the + // code object occupies slot 0 of CPython's single global ref space, + // so we mirror that by reserving slot 0 of the inner table. + let value = if matches!(typ, Type::Code) { + let mut inner_refs: Vec< + Option<::Constant>, + > = Vec::new(); + if flag { + inner_refs.push(None); + } + let code = deserialize_code_inner(rdr, bag.constant_bag(), &mut inner_refs)?; + bag.make_code(code) + } else { + deserialize_value_typed(rdr, bag, depth, refs, typ)? + }; if let Some(idx) = slot { refs[idx] = Some(value.clone()); @@ -667,7 +858,7 @@ fn deserialize_value_typed( let value = rdr.read_slice(len)?; bag.make_bytes(value) } - Type::Code => bag.make_code(deserialize_code(rdr, bag.constant_bag())?), + Type::Code => return Err(MarshalError::BadType), Type::Slice => { let d = depth - 1; let start = deserialize_value_depth(rdr, bag, d, refs)?; From c1653f3ffa5e08611df97a8a9a78975630ea6188 Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Sat, 23 May 2026 19:12:22 +0900 Subject: [PATCH 02/10] Align PYC magic number, FORMAT_VERSION, and header check with CPython 3.14 PYC_MAGIC_NUMBER changes from 2994 to 3627, matching CPython 3.14's pyc_magic_number_token (0x0a0d0e2b). marshal FORMAT_VERSION drops from 5 to 4 (the encoder/marshal.version value; the decoder already accepts both). check_pyc_magic_number_bytes now compares all four magic bytes instead of the first two. --- crates/compiler-core/src/marshal.rs | 2 +- crates/vm/src/import.rs | 2 +- crates/vm/src/version.rs | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/crates/compiler-core/src/marshal.rs b/crates/compiler-core/src/marshal.rs index 3a9acfd5aa1..447e1defb52 100644 --- a/crates/compiler-core/src/marshal.rs +++ b/crates/compiler-core/src/marshal.rs @@ -5,7 +5,7 @@ use malachite_bigint::{BigInt, Sign}; use num_complex::Complex64; use rustpython_wtf8::Wtf8; -pub const FORMAT_VERSION: u32 = 5; +pub const FORMAT_VERSION: u32 = 4; #[derive(Clone, Copy, Debug)] pub enum MarshalError { diff --git a/crates/vm/src/import.rs b/crates/vm/src/import.rs index f33290ff83b..798bc258b7f 100644 --- a/crates/vm/src/import.rs +++ b/crates/vm/src/import.rs @@ -9,7 +9,7 @@ use crate::{ }; pub(crate) fn check_pyc_magic_number_bytes(buf: &[u8]) -> bool { - buf.starts_with(&crate::version::PYC_MAGIC_NUMBER_BYTES[..2]) + buf.starts_with(&crate::version::PYC_MAGIC_NUMBER_BYTES) } pub(crate) fn init_importlib_base(vm: &mut VirtualMachine) -> PyResult { diff --git a/crates/vm/src/version.rs b/crates/vm/src/version.rs index f30a6a49426..05eb12a2942 100644 --- a/crates/vm/src/version.rs +++ b/crates/vm/src/version.rs @@ -69,8 +69,8 @@ pub const RUSTPYTHON_VERSION: &str = const { }; // Must be aligned to Lib/importlib/_bootstrap_external.py -// Bumped to 2994 for new CommonConstant discriminants (BuiltinList, BuiltinSet) -pub const PYC_MAGIC_NUMBER: u16 = 2994; +// Matches CPython 3.14 (Include/internal/pycore_magic_number.h). +pub const PYC_MAGIC_NUMBER: u16 = 3627; // CPython format: magic_number | ('\r' << 16) | ('\n' << 24) // This protects against text-mode file reads From 8c153177087338cf3fa024d1b16247e162b9534f Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Sat, 23 May 2026 20:08:12 +0900 Subject: [PATCH 03/10] Add CPython 3.14 .pyc decoding regression tests Two fixture-based tests pin the marshal decoder against actual CPython 3.14 marshal.dumps() output: a trivial module that exercises FLAG_REF plus TYPE_REF for qualname, and a module with a nested function that exercises ref sharing between a const tuple and its surrounding code object. --- crates/compiler-core/src/marshal.rs | 84 +++++++++++++++++++++++++++++ 1 file changed, 84 insertions(+) diff --git a/crates/compiler-core/src/marshal.rs b/crates/compiler-core/src/marshal.rs index 447e1defb52..19d66e6349b 100644 --- a/crates/compiler-core/src/marshal.rs +++ b/crates/compiler-core/src/marshal.rs @@ -1479,3 +1479,87 @@ fn lt_read_signed_varint(data: &[u8], pos: &mut usize) -> i32 { (val >> 1) as i32 } } + +#[cfg(test)] +mod tests { + use super::*; + use crate::bytecode::{BasicBag, ConstantData}; + + fn hex_to_bytes(hex: &str) -> Vec { + (0..hex.len()) + .step_by(2) + .map(|i| u8::from_str_radix(&hex[i..i + 2], 16).unwrap()) + .collect() + } + + fn decode_code(hex: &str) -> CodeObject { + let bytes = hex_to_bytes(hex); + let value = deserialize_value(&mut &bytes[..], BasicBag).expect("decode failed"); + match value { + ConstantData::Code { code } => *code, + other => panic!("expected Code, got {other:?}"), + } + } + + /// CPython 3.14 marshal output for: `compile("x = 1", "", "exec")`. + /// Exercises FLAG_REF on the code object and TYPE_REF for qualname + /// pointing back at the obj_name slot. + #[test] + fn cpython_314_trivial_assignment() { + let hex = "e30000000000000000000000000100000000000000f30a00000080005e017400520123002902\ + e9010000004e2901da0178a900f300000000da033c743eda083c6d6f64756c653e72070000000100\ + 0000730a000000f003010101d8040582017205000000"; + let code = decode_code(hex); + assert_eq!(code.obj_name.as_str(), ""); + assert_eq!(code.qualname.as_str(), ""); + assert_eq!(code.source_path.as_str(), ""); + assert_eq!(code.arg_count, 0); + assert_eq!(code.max_stackdepth, 1); + assert_eq!(code.names.len(), 1); + assert_eq!(code.names[0].as_str(), "x"); + assert_eq!(code.constants.len(), 2); + // (1, None) + let consts: &[ConstantData] = &code.constants; + assert!(matches!( + consts[0], + ConstantData::Integer { ref value } if *value == 1.into(), + )); + assert!(matches!(consts[1], ConstantData::None)); + } + + /// CPython 3.14 marshal output for a module with a nested function + /// and a string constant. Verifies that nested code objects inside + /// a const tuple share the surrounding code's ref space. + #[test] + fn cpython_314_nested_code_and_string_const() { + let hex = "e30000000000000000000000000100000000000000f310000000800052001700740052017401\ + 520223002903630200000000000000000000000200000003000000f3120000008000570\ + 12c0000000000000000000000230029014ea9002902da0161da016273020000002626da033c743e\ + da0361646472070000000200000073090000008000d80b0c8d35804cf300000000da0568656c6c\ + 6f4e29027207000000da084752454554494e47720300000072080000007206000000da083c6d6f\ + 64756c653e720b000000010000007311000000f003010101f204010111f006000c1382087208000000"; + let code = decode_code(hex); + assert_eq!(code.obj_name.as_str(), ""); + assert_eq!(code.names.len(), 2); + assert_eq!(code.names[0].as_str(), "add"); + assert_eq!(code.names[1].as_str(), "GREETING"); + assert_eq!(code.constants.len(), 3); + // Inner code, "hello", None + let consts: &[ConstantData] = &code.constants; + let inner = match &consts[0] { + ConstantData::Code { code } => code, + other => panic!("expected nested Code, got {other:?}"), + }; + assert_eq!(inner.obj_name.as_str(), "add"); + assert_eq!(inner.qualname.as_str(), "add"); + assert_eq!(inner.arg_count, 2); + assert_eq!(inner.varnames.len(), 2); + assert_eq!(inner.varnames[0].as_str(), "a"); + assert_eq!(inner.varnames[1].as_str(), "b"); + assert!(matches!( + consts[1], + ConstantData::Str { ref value } if value.as_str().ok() == Some("hello"), + )); + assert!(matches!(consts[2], ConstantData::None)); + } +} From 1fc426d0f4f337ad4fbdc8f4bbc2cd82e29063ab Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Sat, 23 May 2026 20:55:38 +0900 Subject: [PATCH 04/10] Accept CPython-tagged .pyc as read-only bytecode source SourceFileLoader.get_code now also looks for .pyc files using _RP_FALLBACK_CACHE_TAGS (currently ('cpython-314',)) in addition to sys.implementation.cache_tag. The matched .pyc is only used for reading; recompilation still writes to the RustPython-tagged path, so CPython's .pyc is never overwritten. Source-stat / hash / timestamp validation logic is unchanged. --- Lib/importlib/_bootstrap_external.py | 46 +++++++++++++++++++++++----- 1 file changed, 38 insertions(+), 8 deletions(-) diff --git a/Lib/importlib/_bootstrap_external.py b/Lib/importlib/_bootstrap_external.py index 95ce14b2c39..1eba51bfbb1 100644 --- a/Lib/importlib/_bootstrap_external.py +++ b/Lib/importlib/_bootstrap_external.py @@ -236,6 +236,13 @@ def _write_atomic(path, data, mode=0o666): # Deprecated. DEBUG_BYTECODE_SUFFIXES = OPTIMIZED_BYTECODE_SUFFIXES = BYTECODE_SUFFIXES +# RustPython: additional cache tags to try when looking up bytecode files. +# RustPython itself writes .pyc with its own cache_tag, but reading .pyc +# generated by CPython (e.g. via `python3 -m compileall`) requires accepting +# the CPython tag too. These act as read-only fallbacks; bytecode writes +# always use sys.implementation.cache_tag. +_RP_FALLBACK_CACHE_TAGS = ('cpython-314',) + def cache_from_source(path, debug_override=None, *, optimization=None): """Given the path to a .py file, return the path to its .pyc file. @@ -841,20 +848,43 @@ def get_code(self, fullname): except NotImplementedError: bytecode_path = None else: + # RustPython: include CPython-tagged variants as read-only + # fallbacks so .pyc generated by stock CPython can be reused. + _rp_primary_tag = sys.implementation.cache_tag + _rp_candidate_paths = [bytecode_path] + if _rp_primary_tag: + _rp_marker = f'.{_rp_primary_tag}.' + for _rp_alt_tag in _RP_FALLBACK_CACHE_TAGS: + if _rp_alt_tag and _rp_alt_tag != _rp_primary_tag: + _rp_alt = bytecode_path.replace( + _rp_marker, f'.{_rp_alt_tag}.', 1 + ) + if _rp_alt != bytecode_path: + _rp_candidate_paths.append(_rp_alt) + try: st = self.path_stats(source_path) except OSError: pass else: source_mtime = int(st['mtime']) - try: - data = self.get_data(bytecode_path) - except OSError: - pass - else: + # bytecode_path stays as the write target (primary RustPython + # tag); _rp_read_path tracks where the actual .pyc was found + # for verbose/error messages. + data = None + _rp_read_path = bytecode_path + for _rp_candidate in _rp_candidate_paths: + try: + data = self.get_data(_rp_candidate) + except OSError: + data = None + continue + _rp_read_path = _rp_candidate + break + if data is not None: exc_details = { 'name': fullname, - 'path': bytecode_path, + 'path': _rp_read_path, } try: flags = _classify_pyc(data, fullname, exc_details) @@ -883,10 +913,10 @@ def get_code(self, fullname): except (ImportError, EOFError): pass else: - _bootstrap._verbose_message('{} matches {}', bytecode_path, + _bootstrap._verbose_message('{} matches {}', _rp_read_path, source_path) return _compile_bytecode(bytes_data, name=fullname, - bytecode_path=bytecode_path, + bytecode_path=_rp_read_path, source_path=source_path) if source_bytes is None: source_bytes = self.get_data(source_path) From 8f1e64e1940f655bc7d41867e0bb8103e0ae51d4 Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Sat, 23 May 2026 21:22:40 +0900 Subject: [PATCH 05/10] Apply rustfmt to marshal helpers --- crates/compiler-core/src/marshal.rs | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/crates/compiler-core/src/marshal.rs b/crates/compiler-core/src/marshal.rs index 19d66e6349b..cde28c2cdd0 100644 --- a/crates/compiler-core/src/marshal.rs +++ b/crates/compiler-core/src/marshal.rs @@ -333,9 +333,8 @@ fn read_marshal_bytes( let len = rdr.read_u32()?; let bytes = rdr.read_slice(len)?.to_vec(); if let Some(idx) = slot { - refs[idx] = Some(bag.make_constant::(BorrowedConstant::Bytes { - value: &bytes, - })); + refs[idx] = + Some(bag.make_constant::(BorrowedConstant::Bytes { value: &bytes })); } Ok(bytes) } @@ -474,9 +473,8 @@ fn read_marshal_const_tuple( .map(|_| read_const_value(rdr, bag, MAX_MARSHAL_STACK_DEPTH, refs)) .collect::>()?; if let Some(idx) = slot { - refs[idx] = Some(bag.make_constant::(BorrowedConstant::Tuple { - elements: &items, - })); + refs[idx] = + Some(bag.make_constant::(BorrowedConstant::Tuple { elements: &items })); } Ok(items.into_iter().collect()) } @@ -709,9 +707,7 @@ fn deserialize_value_depth( // code object occupies slot 0 of CPython's single global ref space, // so we mirror that by reserving slot 0 of the inner table. let value = if matches!(typ, Type::Code) { - let mut inner_refs: Vec< - Option<::Constant>, - > = Vec::new(); + let mut inner_refs: Vec::Constant>> = Vec::new(); if flag { inner_refs.push(None); } From 3e25f4186929e2685e22a2306005967e611d5aa9 Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Sat, 23 May 2026 21:22:43 +0900 Subject: [PATCH 06/10] Marshal PySlice from format version 4 instead of 5 CPython's marshal supports TYPE_SLICE from format version 4 onwards and that is the default version. Rejecting slice dumps below version 5 made marshal.dumps(slice(...)) fail with the default version and broke test.test_marshal.SliceTestCase.test_slice. --- crates/vm/src/stdlib/marshal.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/vm/src/stdlib/marshal.rs b/crates/vm/src/stdlib/marshal.rs index 60ecb1792f0..c242384d72a 100644 --- a/crates/vm/src/stdlib/marshal.rs +++ b/crates/vm/src/stdlib/marshal.rs @@ -316,7 +316,7 @@ mod decl { buf.write_u8(b'c'); marshal::serialize_code(buf, &co.code); } else if let Some(sl) = obj.downcast_ref::() { - if version < 5 { + if version < 4 { return Err(vm.new_value_error("unmarshallable object".to_string())); } buf.write_u8(b':'); From 24b16e15e81c44cc9a5c05114970ccaa9c536709 Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Sat, 23 May 2026 22:42:55 +0900 Subject: [PATCH 07/10] Revert "Accept CPython-tagged .pyc as read-only bytecode source" Lib/importlib/_bootstrap_external.py is CPython's own code copied verbatim; local patches here defeat compatibility tracking. The cpython-XX cache_tag fallback needs to live on the RustPython side (Rust code or sys.implementation.cache_tag policy), not as edits to the imported standard library. This reverts commit 1fc426d0fb5fcdb50d35cad13bbb43e8f6ce1c7f. --- Lib/importlib/_bootstrap_external.py | 46 +++++----------------------- 1 file changed, 8 insertions(+), 38 deletions(-) diff --git a/Lib/importlib/_bootstrap_external.py b/Lib/importlib/_bootstrap_external.py index 1eba51bfbb1..95ce14b2c39 100644 --- a/Lib/importlib/_bootstrap_external.py +++ b/Lib/importlib/_bootstrap_external.py @@ -236,13 +236,6 @@ def _write_atomic(path, data, mode=0o666): # Deprecated. DEBUG_BYTECODE_SUFFIXES = OPTIMIZED_BYTECODE_SUFFIXES = BYTECODE_SUFFIXES -# RustPython: additional cache tags to try when looking up bytecode files. -# RustPython itself writes .pyc with its own cache_tag, but reading .pyc -# generated by CPython (e.g. via `python3 -m compileall`) requires accepting -# the CPython tag too. These act as read-only fallbacks; bytecode writes -# always use sys.implementation.cache_tag. -_RP_FALLBACK_CACHE_TAGS = ('cpython-314',) - def cache_from_source(path, debug_override=None, *, optimization=None): """Given the path to a .py file, return the path to its .pyc file. @@ -848,43 +841,20 @@ def get_code(self, fullname): except NotImplementedError: bytecode_path = None else: - # RustPython: include CPython-tagged variants as read-only - # fallbacks so .pyc generated by stock CPython can be reused. - _rp_primary_tag = sys.implementation.cache_tag - _rp_candidate_paths = [bytecode_path] - if _rp_primary_tag: - _rp_marker = f'.{_rp_primary_tag}.' - for _rp_alt_tag in _RP_FALLBACK_CACHE_TAGS: - if _rp_alt_tag and _rp_alt_tag != _rp_primary_tag: - _rp_alt = bytecode_path.replace( - _rp_marker, f'.{_rp_alt_tag}.', 1 - ) - if _rp_alt != bytecode_path: - _rp_candidate_paths.append(_rp_alt) - try: st = self.path_stats(source_path) except OSError: pass else: source_mtime = int(st['mtime']) - # bytecode_path stays as the write target (primary RustPython - # tag); _rp_read_path tracks where the actual .pyc was found - # for verbose/error messages. - data = None - _rp_read_path = bytecode_path - for _rp_candidate in _rp_candidate_paths: - try: - data = self.get_data(_rp_candidate) - except OSError: - data = None - continue - _rp_read_path = _rp_candidate - break - if data is not None: + try: + data = self.get_data(bytecode_path) + except OSError: + pass + else: exc_details = { 'name': fullname, - 'path': _rp_read_path, + 'path': bytecode_path, } try: flags = _classify_pyc(data, fullname, exc_details) @@ -913,10 +883,10 @@ def get_code(self, fullname): except (ImportError, EOFError): pass else: - _bootstrap._verbose_message('{} matches {}', _rp_read_path, + _bootstrap._verbose_message('{} matches {}', bytecode_path, source_path) return _compile_bytecode(bytes_data, name=fullname, - bytecode_path=_rp_read_path, + bytecode_path=bytecode_path, source_path=source_path) if source_bytes is None: source_bytes = self.get_data(source_path) From ce0c9ebaf3fa75a37c364ce3178968e1826656a6 Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Sun, 24 May 2026 13:35:46 +0900 Subject: [PATCH 08/10] Format sys.implementation.cache_tag as cpython-{MAJOR}{MINOR} Use the CPython compatibility version (e.g. cpython-314) instead of the rustpython-{MAJOR_IMPL}_{MINOR_IMPL} interpreter version string. --- crates/vm/src/stdlib/sys.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/crates/vm/src/stdlib/sys.rs b/crates/vm/src/stdlib/sys.rs index 44e692f1783..68d57d225a5 100644 --- a/crates/vm/src/stdlib/sys.rs +++ b/crates/vm/src/stdlib/sys.rs @@ -658,7 +658,8 @@ pub mod sys { fn implementation(vm: &VirtualMachine) -> PyRef { const NAME: &str = "rustpython"; - let cache_tag = format!("{NAME}-{}_{}", version::MAJOR_IMPL, version::MINOR_IMPL); + // cache tag uses 'cpython' because our compiler is cpython compatible + let cache_tag = format!("cpython-{}{}", version::MAJOR, version::MINOR); let ctx = &vm.ctx; py_namespace!(vm, { "name" => ctx.new_str(NAME), From 8c3b61fa4ef0c372e8f1727e44d32475205d6a5c Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Sun, 24 May 2026 16:54:47 +0900 Subject: [PATCH 09/10] Set marshal FORMAT_VERSION to 5 to match CPython 3.14.5 Py_MARSHAL_VERSION is 5 in CPython 3.14.5 (Include/marshal.h:16) and TYPE_SLICE serialization rejects version < 5 (Python/marshal.c:720). Restore the same threshold and constant so marshal.version and the slice-marshal gate match CPython. --- crates/compiler-core/src/marshal.rs | 2 +- crates/vm/src/stdlib/marshal.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/compiler-core/src/marshal.rs b/crates/compiler-core/src/marshal.rs index cde28c2cdd0..7e72a1e6365 100644 --- a/crates/compiler-core/src/marshal.rs +++ b/crates/compiler-core/src/marshal.rs @@ -5,7 +5,7 @@ use malachite_bigint::{BigInt, Sign}; use num_complex::Complex64; use rustpython_wtf8::Wtf8; -pub const FORMAT_VERSION: u32 = 4; +pub const FORMAT_VERSION: u32 = 5; #[derive(Clone, Copy, Debug)] pub enum MarshalError { diff --git a/crates/vm/src/stdlib/marshal.rs b/crates/vm/src/stdlib/marshal.rs index c242384d72a..60ecb1792f0 100644 --- a/crates/vm/src/stdlib/marshal.rs +++ b/crates/vm/src/stdlib/marshal.rs @@ -316,7 +316,7 @@ mod decl { buf.write_u8(b'c'); marshal::serialize_code(buf, &co.code); } else if let Some(sl) = obj.downcast_ref::() { - if version < 4 { + if version < 5 { return Err(vm.new_value_error("unmarshallable object".to_string())); } buf.write_u8(b':'); From cd633410c1abc58093c48af344bcf3a955cca95e Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Sun, 24 May 2026 16:59:28 +0900 Subject: [PATCH 10/10] Thread marshal recursion depth through nested code objects Code objects embedded in const-tuples reset the depth budget on each recursion, so a hostile or pathological marshal stream of code-in-tuple- in-code can blow the stack despite MAX_MARSHAL_STACK_DEPTH. Pass the current depth through deserialize_code_inner and read_marshal_const_tuple and decrement at each code-object/tuple boundary. Also route dict keys through deserialize_value_after_header so TYPE_CODE keys decode instead of failing with BadType. --- crates/compiler-core/src/marshal.rs | 66 +++++++++++++++-------------- 1 file changed, 35 insertions(+), 31 deletions(-) diff --git a/crates/compiler-core/src/marshal.rs b/crates/compiler-core/src/marshal.rs index 7e72a1e6365..503369a983e 100644 --- a/crates/compiler-core/src/marshal.rs +++ b/crates/compiler-core/src/marshal.rs @@ -195,7 +195,7 @@ pub fn deserialize_code( bag: Bag, ) -> Result> { let mut refs: Vec> = Vec::new(); - deserialize_code_inner(rdr, bag, &mut refs) + deserialize_code_inner(rdr, bag, MAX_MARSHAL_STACK_DEPTH, &mut refs) } /// Inner code-object deserializer that shares a ref table with caller. @@ -204,8 +204,12 @@ pub fn deserialize_code( fn deserialize_code_inner( rdr: &mut R, bag: Bag, + depth: usize, refs: &mut Vec>, ) -> Result> { + if depth == 0 { + return Err(MarshalError::InvalidBytecode); + } // 1–5: scalar fields let arg_count = rdr.read_u32()?; let posonlyarg_count = rdr.read_u32()?; @@ -217,7 +221,7 @@ fn deserialize_code_inner( let code_bytes = read_marshal_bytes(rdr, &bag, refs)?; // 7: co_consts - let constants = read_marshal_const_tuple(rdr, bag, refs)?; + let constants = read_marshal_const_tuple(rdr, bag, depth, refs)?; // 8: co_names let names = read_marshal_name_tuple(rdr, &bag, refs)?; @@ -448,8 +452,12 @@ fn read_marshal_name_tuple( fn read_marshal_const_tuple( rdr: &mut R, bag: Bag, + depth: usize, refs: &mut Vec>, ) -> Result> { + if depth == 0 { + return Err(MarshalError::InvalidBytecode); + } let raw = rdr.read_u8()?; let type_byte = raw & !FLAG_REF; let has_flag = raw & FLAG_REF != 0; @@ -469,8 +477,9 @@ fn read_marshal_const_tuple( _ => return Err(MarshalError::BadType), }; let slot = reserve_ref_slot(has_flag, refs); + let child_depth = depth - 1; let items: Vec = (0..n) - .map(|_| read_const_value(rdr, bag, MAX_MARSHAL_STACK_DEPTH, refs)) + .map(|_| read_const_value(rdr, bag, child_depth, refs)) .collect::>()?; if let Some(idx) = slot { refs[idx] = @@ -505,7 +514,7 @@ fn read_const_value( let slot = reserve_ref_slot(flag, refs); let typ = Type::try_from(type_code)?; let value = if matches!(typ, Type::Code) { - let code = deserialize_code_inner(rdr, bag, refs)?; + let code = deserialize_code_inner(rdr, bag, depth - 1, refs)?; bag.make_code(code) } else { deserialize_value_typed(rdr, bag, depth, refs, typ)? @@ -679,6 +688,23 @@ fn deserialize_value_depth( return Err(MarshalError::InvalidBytecode); } let raw = rdr.read_u8()?; + deserialize_value_after_header(rdr, bag, depth, refs, raw) +} + +/// Continue deserializing a value after the header byte has already been +/// consumed. Shared by `deserialize_value_depth` and the dict-key branch, +/// where the header byte is read up front to detect the TYPE_NULL +/// terminator. +fn deserialize_value_after_header( + rdr: &mut R, + bag: Bag, + depth: usize, + refs: &mut Vec>, + raw: u8, +) -> Result { + if depth == 0 { + return Err(MarshalError::InvalidBytecode); + } let flag = raw & FLAG_REF != 0; let type_code = raw & !FLAG_REF; @@ -704,14 +730,14 @@ fn deserialize_value_depth( // Code-objects keep their own inner ref table because Bag::Value (the // outer marshal value) and the constant-bag's Constant type are not // in general the same. When the outer header carried FLAG_REF, the - // code object occupies slot 0 of CPython's single global ref space, - // so we mirror that by reserving slot 0 of the inner table. + // code object occupies slot 0 of the single global ref space, so we + // mirror that by reserving slot 0 of the inner table. let value = if matches!(typ, Type::Code) { let mut inner_refs: Vec::Constant>> = Vec::new(); if flag { inner_refs.push(None); } - let code = deserialize_code_inner(rdr, bag.constant_bag(), &mut inner_refs)?; + let code = deserialize_code_inner(rdr, bag.constant_bag(), depth - 1, &mut inner_refs)?; bag.make_code(code) } else { deserialize_value_typed(rdr, bag, depth, refs, typ)? @@ -817,32 +843,10 @@ fn deserialize_value_typed( let mut pairs = Vec::new(); loop { let raw = rdr.read_u8()?; - let type_code = raw & !FLAG_REF; - if type_code == b'0' { + if raw & !FLAG_REF == b'0' { break; } - // TYPE_REF for key - let k = if type_code == Type::Ref as u8 { - let idx = rdr.read_u32()? as usize; - refs.get(idx) - .and_then(|v| v.clone()) - .ok_or(MarshalError::InvalidBytecode)? - } else { - let flag = raw & FLAG_REF != 0; - let key_slot = if flag { - let idx = refs.len(); - refs.push(None); - Some(idx) - } else { - None - }; - let key_type = Type::try_from(type_code)?; - let k = deserialize_value_typed(rdr, bag, d, refs, key_type)?; - if let Some(idx) = key_slot { - refs[idx] = Some(k.clone()); - } - k - }; + let k = deserialize_value_after_header(rdr, bag, d, refs, raw)?; let v = deserialize_value_depth(rdr, bag, d, refs)?; pairs.push((k, v)); }