diff --git a/Lib/test/test_flufl.py b/Lib/test/test_flufl.py index 62360d9f9e4..d77e481c81d 100644 --- a/Lib/test/test_flufl.py +++ b/Lib/test/test_flufl.py @@ -4,7 +4,6 @@ class FLUFLTests(unittest.TestCase): - @unittest.expectedFailure # TODO: RUSTPYTHON def test_barry_as_bdfl(self): code = "from __future__ import barry_as_FLUFL\n2 {0} 3" compile(code.format('<>'), '', 'exec', @@ -35,7 +34,6 @@ def test_guido_as_bdfl(self): # parser reports the start of the token self.assertEqual(cm.exception.offset, 3) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_barry_as_bdfl_look_ma_with_no_compiler_flags(self): # Check that the future import is handled by the parser # even if the compiler flags are not passed. diff --git a/Lib/test/test_future_stmt/test_future.py b/Lib/test/test_future_stmt/test_future.py index 02690919cf3..71f1e616116 100644 --- a/Lib/test/test_future_stmt/test_future.py +++ b/Lib/test/test_future_stmt/test_future.py @@ -111,7 +111,6 @@ def test_future_import_not_on_top(self): """ self.assertSyntaxError(code, lineno=3) - @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: SyntaxError not raised def test_future_import_with_extra_string(self): code = """ '''Docstring''' @@ -260,7 +259,6 @@ def _exec_future(self, code): ) return scope - @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: "t'{a + b}'" != "t'{a + b}'" def test_annotations(self): eq = self.assertAnnotationEqual eq('...') diff --git a/Lib/test/test_pyrepl/test_interact.py b/Lib/test/test_pyrepl/test_interact.py index 65b1eed5bdd..7cbe523a92f 100644 --- a/Lib/test/test_pyrepl/test_interact.py +++ b/Lib/test/test_pyrepl/test_interact.py @@ -166,7 +166,6 @@ def g(x: int): ... self.assertFalse(result) self.assertEqual(f.getvalue(), "{'x': 'int'}\n") - @unittest.expectedFailure # TODO: RUSTPYTHON def test_future_barry_as_flufl(self): console = InteractiveColoredConsole() f = io.StringIO() diff --git a/Lib/test/test_super.py b/Lib/test/test_super.py index 4d338bbbc5a..cde2352e6e1 100644 --- a/Lib/test/test_super.py +++ b/Lib/test/test_super.py @@ -90,7 +90,6 @@ def nested(): self.assertEqual(E().f(), 'AE') - @unittest.expectedFailure # TODO: RUSTPYTHON def test_various___class___pathologies(self): # See issue #12370 class X(A): diff --git a/Lib/test/test_symtable.py b/Lib/test/test_symtable.py index 16204bc45dd..9537de3756f 100644 --- a/Lib/test/test_symtable.py +++ b/Lib/test/test_symtable.py @@ -299,7 +299,6 @@ def test_symbol_lookup(self): self.assertRaises(KeyError, self.top.lookup, "not_here") - @unittest.expectedFailure # TODO: RUSTPYTHON def test_namespaces(self): self.assertTrue(self.top.lookup("Mine").is_namespace()) self.assertTrue(self.Mine.lookup("a_method").is_namespace()) @@ -360,7 +359,6 @@ def test_name(self): self.assertEqual(self.spam.lookup("x").get_name(), "x") self.assertEqual(self.Mine.get_name(), "Mine") - @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: Tuples differ: () != ('a_method',) def test_class_get_methods(self): deprecation_mess = ( re.escape('symtable.Class.get_methods() is deprecated ' @@ -442,7 +440,6 @@ def check_body(body, expected_methods): check_body('\n'.join((gen, func)), ('genexpr',)) check_body('\n'.join((func, gen)), ('genexpr',)) - @unittest.expectedFailure # TODO: RUSTPYTHON; SyntaxError: name 'x' is parameter and global def test_filename_correct(self): ### Bug tickler: SyntaxError file name correct whether error raised ### while parsing or building symbol table. @@ -474,7 +471,6 @@ def test_single(self): def test_exec(self): symbols = symtable.symtable("def f(x): return x", "?", "exec") - @unittest.expectedFailure # TODO: RUSTPYTHON; TypeError: Expected type 'str' but 'bytes' found. def test_bytes(self): top = symtable.symtable(TEST_CODE.encode('utf8'), "?", "exec") self.assertIsNotNone(find_block(top, "Mine")) @@ -563,7 +559,6 @@ def test_loopvar_in_only_one_scope(self): class CommandLineTest(unittest.TestCase): maxDiff = None - @unittest.expectedFailure # TODO: RUSTPYTHON; TypeError: Expected type 'str' but 'bytes' found. def test_file(self): filename = os_helper.TESTFN self.addCleanup(os_helper.unlink, filename) diff --git a/crates/codegen/src/compile.rs b/crates/codegen/src/compile.rs index fe0a983187a..bccb600f698 100644 --- a/crates/codegen/src/compile.rs +++ b/crates/codegen/src/compile.rs @@ -13,7 +13,7 @@ use crate::{ IndexMap, IndexSet, ToPythonName, ast_constant_value_to_constant_data, error::{CodegenError, CodegenErrorType, InternalError}, ir::{self, Block, BlockIdx, Blocks}, - preprocess, + preprocess, strip_python_comments, symboltable::{self, CompilerScope, Symbol, SymbolFlags, SymbolScope, SymbolTable}, unparse::UnparseExpr, }; @@ -2129,6 +2129,7 @@ impl<'warnings> Compiler<'warnings> { | bytecode::CodeFlags::FUTURE_WITH_STATEMENT | bytecode::CodeFlags::FUTURE_PRINT_FUNCTION | bytecode::CodeFlags::FUTURE_UNICODE_LITERALS + | bytecode::CodeFlags::FUTURE_BARRY_AS_BDFL | bytecode::CodeFlags::FUTURE_GENERATOR_STOP | bytecode::CodeFlags::FUTURE_ANNOTATIONS)); info.metadata.argcount = arg_count; @@ -2159,6 +2160,12 @@ impl<'warnings> Compiler<'warnings> { } } + fn configure_annotation_format_parameter(&mut self) { + let info = self.current_code_info(); + info.metadata.varnames.insert(".format".to_owned()); + info.nparams = 1; + } + /// Exit a function signature annotation scope. fn exit_annotation_scope(&mut self, saved_ctx: CompileContext) -> CodeObject { self.pop_annotation_symbol_table(); @@ -2208,10 +2215,7 @@ impl<'warnings> Compiler<'warnings> { // Keep the internal ".format" name; exit_annotation_scope() // renames it to "format" on the final code object. - self.current_code_info() - .metadata - .varnames - .insert(".format".to_owned()); + self.configure_annotation_format_parameter(); // Emit format validation: if format > VALUE_WITH_FAKE_GLOBALS: raise NotImplementedError // VALUE_WITH_FAKE_GLOBALS = 2 (from annotationlib.Format) @@ -2621,9 +2625,9 @@ impl<'warnings> Compiler<'warnings> { } /// Set the qualname of an annotation scope, qualified by the function whose - /// signature it annotates. CPython records that name on the annotation - /// block's symbol table entry (`ste_function_name`) and folds it into the - /// qualname, so `f`'s annotation scope is named `f.__annotate__`. + /// signature it annotates. The annotation block's symbol table entry records + /// that name (`ste_function_name`) and folds it into the qualname, so `f`'s + /// annotation scope is named `f.__annotate__`. fn set_annotation_qualname(&mut self, function_name: &str) { self.set_qualname_for_function(Some(function_name)); } @@ -2801,6 +2805,9 @@ impl<'warnings> Compiler<'warnings> { emit!(self, PseudoInstruction::AnnotationsPlaceholder); let (doc, statements) = split_doc_with_range(&body.body, &self.opts); + if doc.is_some() { + self.done_with_future_stmts = DoneWithFuture::DoneWithDoc; + } let module_start_loc = self.module_start_location(&body.body); let annotations_used = self.current_symbol_table().annotations_used; // Handle annotation bookkeeping before the docstring assignment, as @@ -3173,9 +3180,8 @@ impl<'warnings> Compiler<'warnings> { ) }; - // Special handling for class scope implicit cell variables - // These are treated as Cell even if not explicitly marked in symbol table - // __class__ and __classdict__: only LOAD uses Cell (stores go to class namespace) + // Special handling for class scope implicit cell variables. + // __classdict__: only LOAD uses Cell (stores go to class namespace) // __conditional_annotations__: both LOAD and STORE use Cell (it's a mutable set // that the annotation scope accesses through the closure) let symbol_scope = { @@ -3183,9 +3189,7 @@ impl<'warnings> Compiler<'warnings> { if current_table.typ == CompilerScope::Class && !self.current_code_info().in_inlined_comp && ((usage == NameUsage::Load - && (name == "__class__" - || name == "__classdict__" - || name == "__conditional_annotations__")) + && (name == "__classdict__" || name == "__conditional_annotations__")) || (name == "__conditional_annotations__" && usage == NameUsage::Store)) { Some(SymbolScope::Cell) @@ -3217,6 +3221,8 @@ impl<'warnings> Compiler<'warnings> { | "__firstlineno__" | "__doc__" | "__static_attributes__" + | "__annotate__" + | "__annotate_func__" | "__classdictcell__" | "__classcell__" ) { @@ -3988,10 +3994,7 @@ impl<'warnings> Compiler<'warnings> { // Enter scope with the type parameter name self.enter_scope(name, CompilerScope::TypeVariable, key, lineno)?; - self.current_code_info() - .metadata - .varnames - .insert(".format".to_owned()); + self.configure_annotation_format_parameter(); self.emit_format_validation(); @@ -4047,10 +4050,7 @@ impl<'warnings> Compiler<'warnings> { let key = self.symbol_table_stack.len() - 1; let lineno = self.get_source_line_number().get().to_u32(); self.enter_scope(alias_name, CompilerScope::TypeAlias, key, lineno)?; - self.current_code_info() - .metadata - .varnames - .insert(".format".to_owned()); + self.configure_annotation_format_parameter(); self.emit_format_validation(); let prev_ctx = self.ctx; @@ -5299,10 +5299,7 @@ impl<'warnings> Compiler<'warnings> { // Keep the internal ".format" name; the final code object // exposes this parameter as "format". - self.current_code_info() - .metadata - .varnames - .insert(".format".to_owned()); + self.configure_annotation_format_parameter(); // Emit format validation: if format > VALUE_WITH_FAKE_GLOBALS: raise NotImplementedError self.emit_format_validation(); @@ -10542,6 +10539,7 @@ impl<'warnings> Compiler<'warnings> { | bytecode::CodeFlags::FUTURE_WITH_STATEMENT | bytecode::CodeFlags::FUTURE_PRINT_FUNCTION | bytecode::CodeFlags::FUTURE_UNICODE_LITERALS + | bytecode::CodeFlags::FUTURE_BARRY_AS_BDFL | bytecode::CodeFlags::FUTURE_GENERATOR_STOP | bytecode::CodeFlags::FUTURE_ANNOTATIONS)); info.metadata.argcount = arg_count; @@ -11277,7 +11275,11 @@ impl<'warnings> Compiler<'warnings> { .insert(bytecode::CodeFlags::FUTURE_ANNOTATIONS); } FutureFeature::BarryAsFLUFL => { - // We do not support Barry-as-BDFL parser mode yet. This is a nop for now. + self.future_features + .insert(bytecode::CodeFlags::FUTURE_BARRY_AS_BDFL); + self.current_code_info() + .flags + .insert(bytecode::CodeFlags::FUTURE_BARRY_AS_BDFL); } FutureFeature::AbsoluteImport | FutureFeature::Division @@ -13001,20 +13003,14 @@ impl<'warnings> Compiler<'warnings> { if let Some(debug_text) = &fstring_expr.debug_text { let leading = debug_text.leading.as_str(); let trailing = debug_text.trailing.as_str(); - self.emit_pending_fstring_literal( - pending_literal, - pending_literal_range, - pending_literal_no_location, - element_count, - false, - join_append_range, - ); - let range = fstring_expr.expression.range(); - let leading = strip_fstring_debug_comments(leading); - let trailing = strip_fstring_debug_comments(trailing); let source = self.source_file.slice(range); - let text = [leading.as_str(), source, trailing.as_str()].concat(); + let text = [ + strip_python_comments(leading).as_str(), + source, + strip_python_comments(trailing).as_str(), + ] + .concat(); let debug_text_range = TextRange::new( range.start() - TextSize::new( @@ -13029,10 +13025,11 @@ impl<'warnings> Compiler<'warnings> { ); let text: Wtf8Buf = text.into(); - *pending_literal_range = Some(debug_text_range); + Self::extend_pending_literal_range(pending_literal_range, debug_text_range); *pending_literal_no_location = false; - *pending_literal = Some(Wtf8Buf::new()); - pending_literal.as_mut().unwrap().push_wtf8(text.as_ref()); + pending_literal + .get_or_insert_with(Wtf8Buf::new) + .push_wtf8(text.as_ref()); // If debug text is present, apply repr conversion when no `format_spec` specified. // See action_helpers.c: fstring_find_expr_replacement @@ -13145,20 +13142,19 @@ impl<'warnings> Compiler<'warnings> { if let Some(debug_text) = &fstring_expr.debug_text { let leading = debug_text.leading.as_str(); let trailing = debug_text.trailing.as_str(); - Self::count_pending_fstring_literal(pending_literal, element_count, false); let range = fstring_expr.expression.range(); let source = self.source_file.slice(range); let text = [ - strip_fstring_debug_comments(leading).as_str(), + strip_python_comments(leading).as_str(), source, - strip_fstring_debug_comments(trailing).as_str(), + strip_python_comments(trailing).as_str(), ] .concat(); let text: Wtf8Buf = text.into(); - let mut debug_text = Wtf8Buf::new(); - debug_text.push_wtf8(text.as_ref()); - *pending_literal = Some(debug_text); + pending_literal + .get_or_insert_with(Wtf8Buf::new) + .push_wtf8(text.as_ref()); } Self::count_pending_fstring_literal(pending_literal, element_count, false); @@ -13380,9 +13376,9 @@ impl<'warnings> Compiler<'warnings> { let range = interp.expression.range(); let source = self.source_file.slice(range); let text = [ - strip_fstring_debug_comments(leading).as_str(), + strip_python_comments(leading).as_str(), source, - strip_fstring_debug_comments(trailing).as_str(), + strip_python_comments(trailing).as_str(), ] .concat(); let debug_text_range = TextRange::new( @@ -13673,27 +13669,6 @@ impl ToU32 for usize { } } -/// Strip Python comments from f-string debug text (leading/trailing around `=`). -/// A comment starts with `#` and extends to the end of the line. -/// The newline character itself is preserved. -fn strip_fstring_debug_comments(text: &str) -> String { - let mut result = String::with_capacity(text.len()); - let mut in_comment = false; - for ch in text.chars() { - if in_comment { - if ch == '\n' { - in_comment = false; - result.push(ch); - } - } else if ch == '#' { - in_comment = true; - } else { - result.push(ch); - } - } - result -} - #[cfg(test)] mod ruff_tests { use super::*; @@ -17463,6 +17438,53 @@ class C: ); } + #[test] + #[expect( + clippy::literal_string_with_formatting_args, + reason = "the literal is the expected t-string annotation" + )] + fn future_tstring_annotation_preserves_interpolation_source_like_cpython() { + let code = compile_exec( + "from __future__ import annotations\nx: t'{a + b}'\ny: t'{ a + b }'\nz: f'{a + b =}'\nu: t'{a + b =}'\nv: t'{a + b =:>10}'\np: t'{(a)}'\nq: t'{((a))!r}'\nr: t'{ ((a)) = !r:>10}'\ns: t'{(a)=}'\nt: t'{a == b = }'\na1: t'''{a= # x=y\n}'''\na2: t'''{a # x=y\n}'''\na3: t'''{(a # x=y\n)}'''\na4: t'''{a # x=y\n!r}'''\na5: t'''{a # x=y\n:>10}'''\na6: t'''{'#'}'''\na7: t'''{('#', a) # c=d\n}'''\n", + ); + let annotation_strings = code + .constants + .iter() + .filter_map(|constant| match constant { + ConstantData::Str { value } + if value.starts_with("t'") + || value.starts_with("t\"") + || value.starts_with("f'") => + { + Some(value.to_string()) + } + _ => None, + }) + .collect::>(); + assert_eq!( + annotation_strings, + [ + "t'{a + b}'", + "t'{ a + b}'", + "f'a + b ={a + b!r}'", + "t'a + b ={a + b!r}'", + "t'a + b ={a + b:>10}'", + "t'{(a)}'", + "t'{((a))!r}'", + "t' ((a)) = { ((a))!r:>10}'", + "t'(a)={(a)!r}'", + "t'a == b = {a == b!r}'", + "t'a= \\n{a!r}'", + "t'{a}'", + "t'{(a \\n)}'", + "t'{a!r}'", + "t'{a:>10}'", + "t\"{'#'}\"", + "t\"{('#', a)}\"", + ] + ); + } + #[test] fn lambda_dict_literal_ops_use_dict_location_like_cpython() { let code = compile_exec( @@ -19518,7 +19540,7 @@ def spec(x): } #[test] - fn debug_fstring_literal_location_like_cpython() { + fn debug_fstring_literal_merging_and_location_like_cpython() { fn string_load_position(code: &CodeObject, expected: &str) -> (usize, usize, usize, usize) { code.instructions .iter() @@ -19541,16 +19563,20 @@ def spec(x): } let code = compile_exec( - "\ -def simple(x): + r#"def simple(x): return f'{x=}' def prefixed(x): return f'a {x=} b' -", + +def commented(x): + return f"""{ # comment +x=}""" +"#, ); let simple = find_code(&code, "simple").expect("missing simple code"); let prefixed = find_code(&code, "prefixed").expect("missing prefixed code"); + let commented = find_code(&code, "commented").expect("missing commented code"); assert_eq!( string_load_position(simple, "x="), @@ -19558,9 +19584,14 @@ def prefixed(x): "CPython represents f'{{x=}}' debug text as a literal at the expression/debug-text location" ); assert_eq!( - string_load_position(prefixed, "x="), - (5, 17, 5, 19), - "CPython keeps debug text as a separate JoinedStr Constant instead of merging it with the preceding literal" + string_load_position(prefixed, "a x="), + (5, 14, 5, 19), + "CPython merges debug text with the preceding JoinedStr literal" + ); + assert_eq!( + string_load_position(commented, " \nx="), + (8, 17, 9, 3), + "a stripped comment shortens the debug text but not the source range it spans" ); } @@ -26980,6 +27011,43 @@ class C: ); } + #[test] + fn explicit_class_dunder_class_store_uses_namespace_like_cpython() { + let code = compile_exec( + "\ +class C: + def method(self): + return __class__ + __class__ = 413 +", + ); + let class_code = find_code(&code, "C").expect("missing class code"); + let class_name_index = class_code + .names + .iter() + .position(|name| name.as_str() == "__class__") + .expect("missing __class__ name"); + + assert!(class_code.instructions.iter().any(|unit| { + matches!( + unit.op, + Instruction::StoreName { namei } + if namei.get(OpArg::new(u32::from(u8::from(unit.arg)))) as usize + == class_name_index + ) + })); + assert!(!class_code.instructions.iter().any(|unit| { + matches!( + unit.op, + Instruction::StoreDeref { i } + if class_code.cellvars + [usize::from(i.get(OpArg::new(u32::from(u8::from(unit.arg)))))] + .as_str() + == "__class__" + ) + })); + } + #[test] fn conditional_class_body_duplicates_no_location_exit_tail() { let code = compile_exec( @@ -27528,7 +27596,7 @@ def f(): } #[test] - fn future_barry_as_flufl_is_accepted_but_ignored() { + fn future_barry_as_flufl_sets_module_and_nested_code_flags() { let code = compile_exec( "\ from __future__ import barry_as_FLUFL @@ -27537,16 +27605,48 @@ def f(): pass ", ); - let future_flags = bytecode::CodeFlags::FUTURE_DIVISION - | bytecode::CodeFlags::FUTURE_ABSOLUTE_IMPORT - | bytecode::CodeFlags::FUTURE_WITH_STATEMENT - | bytecode::CodeFlags::FUTURE_PRINT_FUNCTION - | bytecode::CodeFlags::FUTURE_UNICODE_LITERALS - | bytecode::CodeFlags::FUTURE_GENERATOR_STOP - | bytecode::CodeFlags::FUTURE_ANNOTATIONS; - assert!((code.flags & future_flags).is_empty()); + assert!( + code.flags + .contains(bytecode::CodeFlags::FUTURE_BARRY_AS_BDFL) + ); let f = find_code(&code, "f").expect("missing f code"); - assert!((f.flags & future_flags).is_empty()); + assert!(f.flags.contains(bytecode::CodeFlags::FUTURE_BARRY_AS_BDFL)); + } + + #[test] + fn function_annotation_qualnames_include_the_annotated_function() { + let code = compile_exec( + "\ +def f(x: int): + pass +class C: + def m(self, x: int): + pass +def outer(): + def inner(x: int): + pass +", + ); + let mut qualnames = Vec::new(); + fn collect(code: &CodeObject, qualnames: &mut Vec) { + for constant in code.constants.iter() { + if let ConstantData::Code { code } = constant { + if code.obj_name == "__annotate__" { + qualnames.push(code.qualname.clone()); + } + collect(code.as_ref(), qualnames); + } + } + } + collect(&code, &mut qualnames); + assert_eq!( + qualnames, + [ + "f.__annotate__", + "C.m.__annotate__", + "outer..inner.__annotate__" + ] + ); } #[test] @@ -27560,6 +27660,20 @@ x: int assert!(!code.flags.contains(bytecode::CodeFlags::FUTURE_ANNOTATIONS)); } + #[test] + fn future_import_after_extra_string_is_rejected_like_cpython() { + assert_eq!( + compile_exec_error_message( + "\ +\"\"\"Docstring\"\"\" +\"this is not a docstring\" +from __future__ import nested_scopes +", + ), + "from __future__ imports must occur at the beginning of the file" + ); + } + #[test] fn future_braces_uses_cpython_special_error() { assert_eq!( @@ -27729,6 +27843,31 @@ class C: ); } + #[test] + fn nested_class_body_loads_outer_dunder_class_while_methods_use_own_cell() { + let code = compile_exec( + "\ +class Outer: + def method(self): + class Inner: + value = __class__ + def nested(): + return __class__ +", + ); + let inner = find_code(&code, "Inner").expect("missing nested class code"); + + assert!(inner.cellvars.iter().any(|name| name == "__class__")); + assert!(inner.freevars.iter().any(|name| name == "__class__")); + assert!( + inner + .instructions + .iter() + .any(|unit| matches!(unit.op, Instruction::LoadFromDictOrDeref { .. })), + "the class body must resolve __class__ from the enclosing method while the nested method closes over the new class cell" + ); + } + #[test] fn nested_closure_parameter_class_does_not_create_outer_class_closure() { let code = compile_exec( @@ -31236,6 +31375,16 @@ class C: .map(|name| name.as_str()) .collect::>(); assert_eq!(varnames, vec!["format"]); + assert!(annotate.instructions.iter().any(|unit| matches!( + unit.op, + Instruction::LoadFastBorrow { var_num } + if usize::from(var_num.get(OpArg::new(u32::from(u8::from(unit.arg))))) == 0 + ))); + assert!(!annotate.instructions.iter().any(|unit| matches!( + unit.op, + Instruction::LoadFastCheck { var_num } + if usize::from(var_num.get(OpArg::new(u32::from(u8::from(unit.arg))))) == 0 + ))); } #[test] @@ -31259,6 +31408,39 @@ def f(x: T): pass ); } + #[test] + fn future_generic_class_annotations_do_not_capture_type_params_like_cpython() { + let code = compile_exec( + "\ +from __future__ import annotations +class A[T, *Ts, **P]: + x: T + y: tuple[*Ts] + z: Callable[P, str] +", + ); + let type_params = + find_code(&code, "").expect("missing type parameter scope"); + let class = find_direct_child_code(type_params, "A").expect("missing class body"); + + assert_eq!( + type_params + .cellvars + .iter() + .map(|name| name.as_str()) + .collect::>(), + [".type_params"] + ); + assert_eq!( + class + .freevars + .iter() + .map(|name| name.as_str()) + .collect::>(), + [".type_params"] + ); + } + #[test] fn future_unannotated_function_does_not_hide_next_annotation_block() { let code = compile_exec( diff --git a/crates/codegen/src/lib.rs b/crates/codegen/src/lib.rs index a7349a5762f..f765eb74aba 100644 --- a/crates/codegen/src/lib.rs +++ b/crates/codegen/src/lib.rs @@ -8,6 +8,7 @@ extern crate log; extern crate alloc; +use alloc::{string::String, vec::Vec}; use rustpython_compiler_core::bytecode::ConstantData; type IndexMap = indexmap::IndexMap; @@ -93,6 +94,68 @@ pub(crate) fn ast_constant_value_to_constant_data(value: ast::ConstantValue) -> } } +fn strip_python_comments(text: &str) -> String { + let chars = text.chars().collect::>(); + let mut result = String::with_capacity(text.len()); + let mut quote = None; + let mut triple_quoted = false; + let mut escaped = false; + let mut in_comment = false; + let mut index = 0; + + while index < chars.len() { + let ch = chars[index]; + if in_comment { + if matches!(ch, '\n' | '\r') { + in_comment = false; + result.push(ch); + } + index += 1; + continue; + } + + if let Some(delimiter) = quote { + result.push(ch); + if escaped { + escaped = false; + } else if ch == '\\' { + escaped = true; + } else if triple_quoted + && ch == delimiter + && chars.get(index + 1) == Some(&delimiter) + && chars.get(index + 2) == Some(&delimiter) + { + result.push(delimiter); + result.push(delimiter); + quote = None; + index += 2; + } else if !triple_quoted && ch == delimiter { + quote = None; + } + index += 1; + continue; + } + + match ch { + '#' => in_comment = true, + '\'' | '"' => { + quote = Some(ch); + triple_quoted = + chars.get(index + 1) == Some(&ch) && chars.get(index + 2) == Some(&ch); + result.push(ch); + if triple_quoted { + result.push(ch); + result.push(ch); + index += 2; + } + } + _ => result.push(ch), + } + index += 1; + } + result +} + pub trait ToPythonName { /// Returns a short name for the node suitable for use in error messages. fn python_name(&self) -> &'static str; diff --git a/crates/codegen/src/preprocess.rs b/crates/codegen/src/preprocess.rs index 084b72c87f7..1bb65a76e62 100644 --- a/crates/codegen/src/preprocess.rs +++ b/crates/codegen/src/preprocess.rs @@ -257,7 +257,7 @@ pub fn checked_future_features_in_body( future_features.insert(bytecode::CodeFlags::FUTURE_ANNOTATIONS) } FutureFeature::BarryAsFLUFL => { - // We do not support Barry-as-BDFL parser mode yet. This is a nop for now. + future_features.insert(bytecode::CodeFlags::FUTURE_BARRY_AS_BDFL) } FutureFeature::AbsoluteImport | FutureFeature::Division diff --git a/crates/codegen/src/symboltable.rs b/crates/codegen/src/symboltable.rs index a771e19d36f..a09deb0bb3a 100644 --- a/crates/codegen/src/symboltable.rs +++ b/crates/codegen/src/symboltable.rs @@ -32,6 +32,9 @@ pub struct SymbolTable { /// The line number in the source code where this symboltable begins. pub line_number: u32, + /// Monotonic creation order used by public symtable children. + pub block_index: usize, + // Return True if the block is a nested class or function pub is_nested: bool, @@ -120,11 +123,18 @@ pub struct SymbolTable { } impl SymbolTable { - fn new(name: String, typ: CompilerScope, line_number: u32, is_nested: bool) -> Self { + fn new( + name: String, + typ: CompilerScope, + line_number: u32, + is_nested: bool, + block_index: usize, + ) -> Self { Self { name, typ, line_number, + block_index, is_nested, is_method: false, symbols: IndexMap::default(), @@ -993,7 +1003,7 @@ impl SymbolTableAnalyzer { || (sym.flags.contains(SymbolFlags::DEF_FREE_CLASS) && !matches!(st_typ, CompilerScope::Module)) { - if st_typ == CompilerScope::Class && name != "__class__" { + if st_typ == CompilerScope::Class { None } else { Some(SymbolScope::Cell) @@ -1044,6 +1054,7 @@ struct SymbolTableBuilder { // Mirrors symtable ENTER_RECURSIVE guards during compilation. recursion_depth: usize, recursion_limit: usize, + next_block_index: usize, } /// Enum to indicate in what mode an expression @@ -1074,6 +1085,7 @@ impl SymbolTableBuilder { in_conditional_block: false, recursion_depth: 0, recursion_limit: DEFAULT_RECURSION_LIMIT, + next_block_index: 0, }; this.enter_scope("top", CompilerScope::Module, 0); this @@ -1154,7 +1166,9 @@ impl SymbolTableBuilder { .last() .and_then(|t| t.mangled_names.clone()) .filter(|_| typ != CompilerScope::Class); - let mut table = SymbolTable::new(name.to_owned(), typ, line_number, is_nested); + let block_index = self.next_block_index; + self.next_block_index += 1; + let mut table = SymbolTable::new(name.to_owned(), typ, line_number, is_nested, block_index); table.is_method = is_method; table.future_annotations = self.future_annotations; table.mangled_names = inherited_mangled_names; @@ -1235,6 +1249,17 @@ impl SymbolTableBuilder { table } + fn resolve_future_annotation_names_as_globals(table: &mut SymbolTable) { + for symbol in table.symbols.values_mut() { + if symbol.scope == SymbolScope::Unknown + && symbol.flags.contains(SymbolFlags::USE) + && !symbol.is_bound() + { + symbol.scope = SymbolScope::GlobalImplicit; + } + } + } + /// Enter annotation scope (PEP 649) /// Creates or reuses the annotation block for the current scope fn enter_annotation_scope( @@ -1243,29 +1268,42 @@ impl SymbolTableBuilder { include_classdict_with_future: bool, include_conditional_annotations: bool, ) { - let current = self.tables.last_mut().unwrap(); - let can_see_class_scope = - current.typ == CompilerScope::Class || current.can_see_class_scope; - let has_conditional = current.has_conditional_annotations; - let is_nested = current.is_nested || Self::is_function_like_scope(current.typ); + let (can_see_class_scope, has_conditional, is_nested, needs_annotation_block) = { + let current = self.tables.last().unwrap(); + ( + current.typ == CompilerScope::Class || current.can_see_class_scope, + current.has_conditional_annotations, + current.is_nested || Self::is_function_like_scope(current.typ), + current.annotation_block.is_none(), + ) + }; // Create annotation block if not exists - if current.annotation_block.is_none() { + if needs_annotation_block { + let block_index = self.next_block_index; + self.next_block_index += 1; let mut annotation_table = SymbolTable::new( "__annotate__".to_owned(), CompilerScope::Annotation, line_number, is_nested, + block_index, ); // Annotation scope in class can see class scope annotation_table.can_see_class_scope = can_see_class_scope; annotation_table.skip_enclosing_function_scope = true; annotation_table.add_format_parameter(); - current.annotation_block = Some(Box::new(annotation_table)); + self.tables.last_mut().unwrap().annotation_block = Some(Box::new(annotation_table)); } // Take the annotation block and push to stack for processing - let annotation_table = current.annotation_block.take().unwrap(); + let annotation_table = self + .tables + .last_mut() + .unwrap() + .annotation_block + .take() + .unwrap(); self.tables.push(*annotation_table); // Save parent's varnames and seed with existing annotation varnames (e.g., "format") self.varnames_stack @@ -1287,6 +1325,9 @@ impl SymbolTableBuilder { let mut table = self.tables.pop().unwrap(); // Save the collected varnames to the symbol table table.varnames = core::mem::take(&mut self.current_varnames); + if self.future_annotations { + Self::resolve_future_annotation_names_as_globals(&mut table); + } // Store back to parent's annotation_block (not sub_tables) let parent = self.tables.last_mut().unwrap(); parent.annotation_block = Some(Box::new(table)); @@ -1320,6 +1361,13 @@ impl SymbolTableBuilder { .insert(SymbolFlags::USE | SymbolFlags::DEF_FREE_CLASS); } + fn add_format_parameter(&mut self) { + self.tables.last_mut().unwrap().add_format_parameter(); + if !self.current_varnames.iter().any(|name| name == ".format") { + self.current_varnames.push(".format".to_owned()); + } + } + /// Walk up the scope chain to determine if we're inside an async function. /// Annotation and TypeParams scopes act as async barriers (always non-async). /// Comprehension scopes are transparent (inherit parent's async context). @@ -1413,7 +1461,7 @@ impl SymbolTableBuilder { current.typ == CompilerScope::Class || current.can_see_class_scope; self.enter_scope("__annotate__", CompilerScope::Annotation, line_number); self.tables.last_mut().unwrap().can_see_class_scope = can_see_class_scope; - self.tables.last_mut().unwrap().add_format_parameter(); + self.add_format_parameter(); if can_see_class_scope { self.register_name("__classdict__", SymbolUsage::Used, TextRange::default())?; } @@ -1464,7 +1512,8 @@ impl SymbolTableBuilder { self.tables.last_mut().unwrap().in_unevaluated_annotation = was_in_unevaluated_annotation; if self.future_annotations { - let annotation_block = self.discard_scope(); + let mut annotation_block = self.discard_scope(); + Self::resolve_future_annotation_names_as_globals(&mut annotation_block); self.tables .last_mut() .unwrap() @@ -1657,8 +1706,8 @@ impl SymbolTableBuilder { self.in_conditional_block = false; self.class_name = Some(name.to_string()); if type_params.is_some() { - self.register_name(".type_params", SymbolUsage::Used, *range)?; self.register_name("__type_params__", SymbolUsage::Assigned, *range)?; + self.register_name(".type_params", SymbolUsage::Used, *range)?; } self.scan_statements(body)?; self.leave_scope(); @@ -1856,25 +1905,6 @@ impl SymbolTableBuilder { SymbolUsage::AnnotationAssigned, *target_range, )?; - // PEP 649: Register annotate function in module/class scope - let current_scope = self.tables.last().map(|t| t.typ); - match current_scope { - Some(CompilerScope::Module) => { - self.register_name( - "__annotate__", - SymbolUsage::Assigned, - *range, - )?; - } - Some(CompilerScope::Class) => { - self.register_name( - "__annotate_func__", - SymbolUsage::Assigned, - *range, - )?; - } - _ => {} - } } else if value.is_some() { self.register_name(id_str, SymbolUsage::Assigned, *target_range)?; } @@ -3522,6 +3552,7 @@ mod tests { let format = annotation_block .lookup(".format") .expect("missing annotation .format parameter"); + assert_eq!(annotation_block.varnames, [".format"]); assert!( format .flags @@ -3567,6 +3598,41 @@ mod tests { ); } + #[test] + fn deferred_annotation_store_names_are_not_public_symbols() { + let module = scan_source("x: int\n"); + assert!(module.lookup("__annotate__").is_none()); + assert!(module.annotation_block.is_some()); + + let module = scan_source("class C:\n y: str\n"); + let class = module + .sub_tables + .iter() + .find(|table| table.typ == CompilerScope::Class) + .expect("missing class scope"); + assert!(class.lookup("__annotate_func__").is_none()); + assert!(class.annotation_block.is_some()); + } + + #[test] + fn generic_class_symbols_follow_cpython_insertion_order() { + let module = scan_source("class C[T]:\n q = [lambda: i for i in range(2)]\n"); + let type_params = module + .sub_tables + .iter() + .find(|table| table.typ == CompilerScope::TypeParams) + .expect("missing type parameter scope"); + let class = type_params + .sub_tables + .iter() + .find(|table| table.typ == CompilerScope::Class) + .expect("missing generic class scope"); + assert_eq!( + class.symbols.keys().map(String::as_str).collect::>(), + ["__type_params__", ".type_params", "q", "range", "i"] + ); + } + #[test] fn function_signature_annotation_block_is_sibling_like_cpython() { let table = scan_source("def f(x: T): pass\n"); diff --git a/crates/codegen/src/unparse.rs b/crates/codegen/src/unparse.rs index 679560642e5..f890edf37a0 100644 --- a/crates/codegen/src/unparse.rs +++ b/crates/codegen/src/unparse.rs @@ -1,7 +1,8 @@ +use crate::strip_python_comments; use alloc::fmt; use core::fmt::Display as _; use ruff_python_ast as ast; -use ruff_text_size::Ranged; +use ruff_text_size::{Ranged, TextSize}; use rustpython_compiler_core::SourceFile; use rustpython_literal::escape::{AsciiEscape, UnicodeEscape}; @@ -610,7 +611,7 @@ impl<'a, 'b, 'c> Unparser<'a, 'b, 'c> { &mut self, val: &ast::Expr, debug_text: Option<&ast::DebugText>, - conversion: ast::ConversionFlag, + mut conversion: ast::ConversionFlag, spec: Option<&ast::InterpolatedStringFormatSpec>, ) -> fmt::Result { let buffered = @@ -622,6 +623,9 @@ impl<'a, 'b, 'c> Unparser<'a, 'b, 'c> { self.p(leading)?; self.p(self.source.slice(val.range()))?; self.p(trailing)?; + if conversion == ast::ConversionFlag::None && spec.is_none() { + conversion = ast::ConversionFlag::Repr; + } } let brace = if buffered.starts_with('{') { // put a space to avoid escaping the bracket @@ -709,7 +713,7 @@ impl<'a, 'b, 'c> Unparser<'a, 'b, 'c> { self.p("t")?; let body = fmt::from_fn(|f| { value.iter().try_for_each(|tstring| { - Unparser::new(f, self.source).unparse_fstring_body(&tstring.elements) + Unparser::new(f, self.source).unparse_tstring_body(&tstring.elements) }) }) .to_string(); @@ -717,6 +721,102 @@ impl<'a, 'b, 'c> Unparser<'a, 'b, 'c> { .str_repr() .write(self.f) } + + fn unparse_tstring_body(&mut self, elements: &[ast::InterpolatedStringElement]) -> fmt::Result { + for element in elements { + match element { + ast::InterpolatedStringElement::Literal(literal) => { + self.unparse_fstring_str(literal)?; + } + ast::InterpolatedStringElement::Interpolation(interpolation) => { + self.unparse_tstring_interpolation(interpolation)?; + } + } + } + Ok(()) + } + + fn unparse_tstring_interpolation( + &mut self, + interpolation: &ast::InterpolatedElement, + ) -> fmt::Result { + let source_conversion = interpolation.conversion; + let mut conversion = source_conversion; + let debug_parts = interpolation.debug_text.as_ref().map(|debug_text| { + ( + strip_python_comments(debug_text.leading.as_str()), + strip_python_comments(self.source.slice(interpolation.expression.range())), + strip_python_comments(debug_text.trailing.as_str()), + ) + }); + if let Some((leading, source, trailing)) = &debug_parts { + self.p(leading)?; + self.p(source)?; + self.p(trailing)?; + if conversion == ast::ConversionFlag::None && interpolation.format_spec.is_none() { + conversion = ast::ConversionFlag::Repr; + } + } + + let expression = if let Some(ast::ConstantValue::Str(value)) = &interpolation.runtime_str { + value.to_string() + } else if let Some((leading, source, trailing)) = &debug_parts { + let mut expression = leading.clone(); + expression.push_str(source); + let equal = trailing + .rfind('=') + .expect("debug interpolation must contain '='"); + expression.push_str(&trailing[..equal]); + expression.trim_end().to_owned() + } else { + let expression_range = interpolation.expression.range(); + let after_brace = interpolation.range.start() + TextSize::new(1); + let mut expression_end = interpolation.format_spec.as_ref().map_or_else( + || interpolation.range.end() - TextSize::new(1), + |format_spec| format_spec.range.start() - TextSize::new(1), + ); + if source_conversion != ast::ConversionFlag::None { + expression_end -= TextSize::new(2); + } + if interpolation.range.start() < expression_range.start() + && interpolation.range.end() >= expression_range.end() + && after_brace <= expression_end + { + strip_python_comments( + self.source + .slice(ruff_text_size::TextRange::new(after_brace, expression_end)), + ) + .trim_end() + .to_owned() + } else { + fmt::from_fn(|f| { + Unparser::new(f, self.source) + .unparse_expr(&interpolation.expression, precedence::TEST + 1) + }) + .to_string() + } + }; + + self.p(if expression.starts_with('{') { + "{ " + } else { + "{" + })?; + self.p(&expression)?; + + if conversion != ast::ConversionFlag::None { + self.p("!")?; + let conversion_byte = [conversion as u8]; + self.p(core::str::from_utf8(&conversion_byte).unwrap())?; + } + + if let Some(format_spec) = &interpolation.format_spec { + self.p(":")?; + self.unparse_tstring_body(&format_spec.elements)?; + } + + self.p("}") + } } pub(crate) struct UnparseExpr<'a> { diff --git a/crates/compiler-core/src/bytecode.rs b/crates/compiler-core/src/bytecode.rs index ba1639170a7..bcdacc18df9 100644 --- a/crates/compiler-core/src/bytecode.rs +++ b/crates/compiler-core/src/bytecode.rs @@ -483,6 +483,19 @@ bitflags! { } } +impl CodeFlags { + /// The `__future__` flags that `compile()` accepts and that a compiled code + /// object inherits from its caller. Mirrors `PyCF_MASK`. + pub const FUTURE_MASK: Self = Self::FUTURE_DIVISION + .union(Self::FUTURE_ABSOLUTE_IMPORT) + .union(Self::FUTURE_WITH_STATEMENT) + .union(Self::FUTURE_PRINT_FUNCTION) + .union(Self::FUTURE_UNICODE_LITERALS) + .union(Self::FUTURE_BARRY_AS_BDFL) + .union(Self::FUTURE_GENERATOR_STOP) + .union(Self::FUTURE_ANNOTATIONS); +} + #[repr(C)] #[derive(Copy, Clone, Debug)] pub struct CodeUnit { diff --git a/crates/compiler/src/lib.rs b/crates/compiler/src/lib.rs index fc9b67614b5..f262cc2270c 100644 --- a/crates/compiler/src/lib.rs +++ b/crates/compiler/src/lib.rs @@ -1,3 +1,6 @@ +extern crate alloc; + +use alloc::borrow::Cow; pub use ruff_python_ast::token::{TokenKind, Tokens}; use ruff_python_parser::ParseErrorType; use ruff_source_file::{PositionEncoding, SourceFile, SourceFileBuilder, SourceLocation}; @@ -362,12 +365,6 @@ fn cpython_parse_diagnostic_override( )); } - // `2 <> 3` outside Barry mode: ruff lexes `<` then an unexpected `>` and - // reports `ExpectedExpression` starting at the `>`. CPython's tokenizer - // treats `<>` as a single obsolete token and points at its start (the - // `<`) instead, so shift the reported location back over it. - source_error!(barry_flufl_obsolete_operator_error(error, source_text)); - // CPython's PEG parser collapses a bare "expected an expression" failure // into the generic "invalid syntax" message. rustpython-vm's `vm_new.rs` // does this same collapse for its own callers; rustpython-compiler has no @@ -384,23 +381,6 @@ fn cpython_parse_diagnostic_override( None } -fn barry_flufl_obsolete_operator_error( - error: &parser::ParseError, - source: &str, -) -> Option<(String, usize, usize)> { - if !matches!(&error.error, parser::ParseErrorType::ExpectedExpression) { - return None; - } - let start = error.location.start().to_usize(); - if start == 0 || source.as_bytes().get(start - 1) != Some(&b'<') { - return None; - } - if source.as_bytes().get(start) != Some(&b'>') { - return None; - } - Some(("invalid syntax".to_string(), start - 1, start + 1)) -} - fn eof_parse_diagnostic( error: &parser::ParseError, source_file: &SourceFile, @@ -5258,8 +5238,18 @@ fn _compile_with_syntax_warning_handler<'a>( Mode::Single | Mode::BlockExpr => parser::Mode::Module, }; let parser_options = parser::ParseOptions::from(parser_mode); - let parsed = parser::parse(source_file.source_text(), parser_options) - .map_err(|err| CompileError::from_ruff_parse_error(err, &source_file, mode))?; + let barry_source = prepare_barry_as_flufl_source( + source_file.source_text(), + parser_options.clone(), + opts.future_features + .contains(core::bytecode::CodeFlags::FUTURE_BARRY_AS_BDFL), + ); + let parsed = parser::parse(barry_source.source(), parser_options); + if let Some(error) = barry_source.diagnostic(parsed.as_ref().err(), &source_file) { + return Err(error); + } + let parsed = + parsed.map_err(|err| CompileError::from_ruff_parse_error(err, &source_file, mode))?; if opts.dont_imply_dedent && matches!(mode, Mode::Single) && let Some(error) = dont_imply_dedent_source_error(&source_file) @@ -5287,6 +5277,186 @@ fn _compile_with_syntax_warning_handler<'a>( Ok(code) } +#[doc(hidden)] +pub struct BarrySource<'a> { + source: Cow<'a, str>, + not_equal: Option, + legacy_not_equal: Vec, +} + +impl BarrySource<'_> { + #[must_use] + pub fn source(&self) -> &str { + &self.source + } + + #[must_use] + pub fn not_equal_before( + &self, + parse_error: Option<&parser::ParseError>, + ) -> Option { + self.not_equal.filter(|range| { + parse_error.is_none_or(|error| { + let diagnostic_start = if matches!( + &error.error, + parser::ParseErrorType::Lexical(parser::LexicalErrorType::Eof) + ) { + find_unclosed_bracket(&self.source).map_or_else( + || error.location.start(), + |(_, offset)| TextSize::new(offset as u32), + ) + } else { + error.location.start() + }; + range.start() <= diagnostic_start + }) + }) + } + + /// The obsolete `<>` operator the parse error points at, if any. In Barry + /// mode the operator was rewritten to `!=`, so the error lands on its + /// start; outside Barry mode ruff lexes `<` and then an unexpected `>`, so + /// the error lands one character in. Either way the whole operator is one + /// token to the tokenizer, so report it as one. + #[must_use] + pub fn invalid_legacy_operator( + &self, + parse_error: &parser::ParseError, + ) -> Option { + let location = parse_error.location.start(); + self.legacy_not_equal + .iter() + .copied() + .find(|range| range.contains(location) || range.start() == location) + .filter(|range| self.outranks_unclosed_bracket(*range)) + } + + /// Whether `range` outranks an unclosed bracket. The bracket is reported + /// at itself, so it wins over anything that starts after it. + fn outranks_unclosed_bracket(&self, range: ruff_text_size::TextRange) -> bool { + find_unclosed_bracket(&self.source) + .is_none_or(|(_, offset)| range.start() <= TextSize::new(offset as u32)) + } + + /// The diagnostic for this source, if any: an obsolete `<>` the parse + /// error points at, or -- in Barry mode only -- the first `!=`. A `<>` + /// takes precedence over a `!=` reported later in the source. + #[must_use] + pub fn diagnostic( + &self, + parse_error: Option<&parser::ParseError>, + source_file: &SourceFile, + ) -> Option { + if let Some(range) = parse_error.and_then(|error| self.invalid_legacy_operator(error)) { + return Some(barry_as_flufl_invalid_legacy_operator_error( + source_file, + range, + )); + } + self.not_equal_before(parse_error) + .map(|range| barry_as_flufl_not_equal_error(source_file, range)) + } +} + +#[doc(hidden)] +#[must_use] +pub fn barry_as_flufl_not_equal_error( + source_file: &SourceFile, + range: ruff_text_size::TextRange, +) -> CompileError { + CompileError::from_source_error( + source_file, + "with Barry as BDFL, use '<>' instead of '!='".to_owned(), + range.start().to_usize(), + range.end().to_usize(), + ) +} + +#[doc(hidden)] +#[must_use] +pub fn barry_as_flufl_invalid_legacy_operator_error( + source_file: &SourceFile, + range: ruff_text_size::TextRange, +) -> CompileError { + CompileError::from_source_error( + source_file, + "invalid syntax".to_owned(), + range.start().to_usize(), + range.end().to_usize(), + ) +} + +/// Every `<>` in `source`, located by plain text search. Only used where the +/// operator is not rewritten, so an occurrence inside a string or a comment +/// costs nothing: it can never coincide with the location of a parse error. +fn textual_legacy_not_equal(source: &str) -> Vec { + source + .match_indices("<>") + .map(|(offset, matched)| { + ruff_text_size::TextRange::at( + TextSize::new(offset as u32), + TextSize::new(matched.len() as u32), + ) + }) + .collect() +} + +#[doc(hidden)] +pub fn prepare_barry_as_flufl_source( + source: &str, + parser_options: parser::ParseOptions, + inherited: bool, +) -> BarrySource<'_> { + let scanned = (inherited || source.contains("barry_as_FLUFL")) + .then(|| parser::parse_unchecked(source, parser_options)); + let enabled = scanned.as_ref().is_some_and(|scanned| { + inherited + || codegen::preprocess::future_features(scanned.syntax()) + .contains(core::bytecode::CodeFlags::FUTURE_BARRY_AS_BDFL) + }); + let Some(scanned) = scanned.filter(|_| enabled) else { + return BarrySource { + source: Cow::Borrowed(source), + not_equal: None, + legacy_not_equal: textual_legacy_not_equal(source), + }; + }; + + let not_equal = scanned + .tokens() + .iter() + .find(|token| token.kind() == TokenKind::NotEqual) + .map(Ranged::range); + let replacements = scanned + .tokens() + .windows(2) + .filter_map(|tokens| { + let [less, greater] = tokens else { + return None; + }; + (less.kind() == TokenKind::Less + && greater.kind() == TokenKind::Greater + && less.end() == greater.start()) + .then(|| ruff_text_size::TextRange::new(less.start(), greater.end())) + }) + .collect::>(); + + let source = if replacements.is_empty() { + Cow::Borrowed(source) + } else { + let mut rewritten = source.to_owned(); + for range in replacements.iter().rev() { + rewritten.replace_range(range.start().to_usize()..range.end().to_usize(), "!="); + } + Cow::Owned(rewritten) + }; + BarrySource { + source, + not_equal, + legacy_not_equal: replacements, + } +} + pub fn compile_with_syntax_warning_handler<'a>( source: &str, mode: Mode, @@ -5315,16 +5485,27 @@ pub fn _compile_symtable( source_file: SourceFile, mode: Mode, ) -> Result { + let parser_mode = match mode { + Mode::Exec | Mode::Single | Mode::BlockExpr => parser::Mode::Module, + Mode::Eval => parser::Mode::Expression, + }; + let parser_options = parser::ParseOptions::from(parser_mode); + let barry_source = + prepare_barry_as_flufl_source(source_file.source_text(), parser_options.clone(), false); let res = match mode { Mode::Exec | Mode::Single | Mode::BlockExpr => { - let ast = ruff_python_parser::parse_module(source_file.source_text()) - .map_err(|e| CompileError::from_ruff_parse_error(e, &source_file, mode))?; + let parsed = ruff_python_parser::parse(barry_source.source(), parser_options); + if let Some(error) = barry_source.diagnostic(parsed.as_ref().err(), &source_file) { + return Err(error); + } + let ast = + parsed.map_err(|e| CompileError::from_ruff_parse_error(e, &source_file, mode))?; if let Some(error) = post_parse_source_error(&source_file, ast.tokens(), &CompileOpts::default()) { return Err(error); } - let ast = ast.into_syntax(); + let ast = ast.into_syntax().expect_module(); if matches!(mode, Mode::Single) && let Some(error) = single_mode_body_error(&ast.body, &source_file) { @@ -5333,11 +5514,12 @@ pub fn _compile_symtable( symboltable::SymbolTable::scan_program(&ast, source_file.clone()) } Mode::Eval => { - let ast = ruff_python_parser::parse( - source_file.source_text(), - parser::Mode::Expression.into(), - ) - .map_err(|e| CompileError::from_ruff_parse_error(e, &source_file, mode))?; + let parsed = ruff_python_parser::parse(barry_source.source(), parser_options); + if let Some(error) = barry_source.diagnostic(parsed.as_ref().err(), &source_file) { + return Err(error); + } + let ast = + parsed.map_err(|e| CompileError::from_ruff_parse_error(e, &source_file, mode))?; if let Some(error) = post_parse_source_error(&source_file, ast.tokens(), &CompileOpts::default()) { @@ -5378,6 +5560,120 @@ mod tests { compile(code, Mode::Single, "<>", CompileOpts::default()).expect("compile error"); } + #[test] + fn barry_as_flufl_rewrites_legacy_not_equal_after_future_import() { + let code = compile( + "from __future__ import barry_as_FLUFL\nresult = 2 <> 3\n", + Mode::Exec, + "", + CompileOpts::default(), + ) + .expect("Barry comparison should compile"); + assert!( + code.flags + .contains(core::bytecode::CodeFlags::FUTURE_BARRY_AS_BDFL) + ); + } + + #[test] + fn inherited_barry_as_flufl_rewrites_legacy_not_equal() { + let opts = CompileOpts { + future_features: core::bytecode::CodeFlags::FUTURE_BARRY_AS_BDFL, + ..CompileOpts::default() + }; + compile("2 <> 3", Mode::Single, "", opts) + .expect("inherited Barry comparison should compile"); + } + + #[test] + fn barry_as_flufl_rejects_modern_not_equal() { + let err = compile( + "from __future__ import barry_as_FLUFL\n2 != 3\n", + Mode::Exec, + "", + CompileOpts::default(), + ) + .expect_err("Barry mode should reject !="); + assert_eq!( + err.to_string(), + "with Barry as BDFL, use '<>' instead of '!='" + ); + assert_eq!(err.python_location(), (2, 3)); + } + + #[test] + fn obsolete_not_equal_diagnostic_spans_the_whole_operator() { + let err = compile("2 <> 3\n", Mode::Exec, "", CompileOpts::default()) + .expect_err("'<>' outside Barry mode is a syntax error"); + assert_eq!(err.to_string(), "invalid syntax"); + assert_eq!(err.python_location(), (1, 3)); + assert_eq!(err.python_end_location(), Some((1, 5))); + + // Only `<>` spans two characters; any other token that cannot start an + // expression keeps its own location. + let err = compile("2 <;\n", Mode::Exec, "", CompileOpts::default()) + .expect_err("'<;' is a syntax error"); + assert_eq!(err.to_string(), "invalid syntax"); + assert_eq!(err.python_location(), (1, 4)); + assert_eq!(err.python_end_location(), Some((1, 5))); + + // A `<>` that starts a statement is reported at the `<` too, where the + // parser stops instead of one character in. + let err = compile("<>\n", Mode::Exec, "", CompileOpts::default()) + .expect_err("a bare '<>' is a syntax error"); + assert_eq!(err.to_string(), "invalid syntax"); + assert_eq!(err.python_location(), (1, 1)); + assert_eq!(err.python_end_location(), Some((1, 3))); + + // A bracket left open earlier in the source outranks the operator. + let err = compile( + "(\n2 <> 3", + Mode::Exec, + "", + CompileOpts::default(), + ) + .expect_err("the bracket is never closed"); + assert_eq!(err.to_string(), "'(' was never closed"); + assert_eq!(err.python_location(), (1, 1)); + } + + #[test] + fn barry_as_flufl_does_not_rewrite_strings_or_comments() { + compile( + "from __future__ import barry_as_FLUFL\nx = '<>'\n# <>\n", + Mode::Exec, + "", + CompileOpts::default(), + ) + .expect("Barry markers in strings and comments should stay untouched"); + } + + #[test] + fn syntax_error_before_barry_not_equal_takes_precedence() { + let err = compile( + "from __future__ import barry_as_FLUFL\n<>\n2 != 3\n", + Mode::Exec, + "", + CompileOpts::default(), + ) + .expect_err("the earlier invalid comparison should fail"); + assert_eq!(err.to_string(), "invalid syntax"); + assert_eq!(err.python_location(), (2, 1)); + } + + #[test] + fn unclosed_bracket_before_barry_not_equal_takes_precedence() { + let err = compile( + "from __future__ import barry_as_FLUFL\n(\n2 != 3", + Mode::Exec, + "", + CompileOpts::default(), + ) + .expect_err("the earlier unclosed bracket should fail"); + assert_eq!(err.to_string(), "'(' was never closed"); + assert_eq!(err.python_location(), (2, 1)); + } + #[test] fn compile_phello() { let code = r#" diff --git a/crates/stdlib/src/snapshots/rustpython_stdlib___opcode__tests__bare_function_annotations_check_attribute_and_subscript_expressions.snap b/crates/stdlib/src/snapshots/rustpython_stdlib___opcode__tests__bare_function_annotations_check_attribute_and_subscript_expressions.snap index 4d78128b5e6..d7ca680d9c1 100644 --- a/crates/stdlib/src/snapshots/rustpython_stdlib___opcode__tests__bare_function_annotations_check_attribute_and_subscript_expressions.snap +++ b/crates/stdlib/src/snapshots/rustpython_stdlib___opcode__tests__bare_function_annotations_check_attribute_and_subscript_expressions.snap @@ -16,7 +16,7 @@ expression: "dis(r#\"\ndef f(one: int):\n int.new_attr: int\n [list][0].ne Disassembly of ", line 1>: 1 RESUME 0 - LOAD_FAST_CHECK 0 (format) + LOAD_FAST_BORROW 0 (format) LOAD_SMALL_INT 2 COMPARE_OP 132 (>) POP_JUMP_IF_FALSE 3 (to L1) diff --git a/crates/vm/src/object/core.rs b/crates/vm/src/object/core.rs index 5534666da5f..02ffe2c7e5e 100644 --- a/crates/vm/src/object/core.rs +++ b/crates/vm/src/object/core.rs @@ -434,6 +434,17 @@ pub(crate) const SIZEOF_PYOBJECT_HEAD: usize = core::mem::size_of::> // this holds. const _: () = assert!(SIZEOF_PYOBJECT_HEAD == 5 * core::mem::size_of::() + 8); +// `PyInner::drop_fields` names `payload` and `typ`; it is only complete while +// every other field stays trivially destructible. +const _: () = assert!( + !core::mem::needs_drop::() + && !core::mem::needs_drop::<&'static PyObjVTable>() + && !core::mem::needs_drop::>() + && !core::mem::needs_drop::>() + && !core::mem::needs_drop::>() + && !core::mem::needs_drop::>() +); + impl PyInner { /// Read type flags and member_count via raw pointers to avoid Stacked Borrows /// violations during bootstrap, where type objects have self-referential typ pointers. @@ -1124,6 +1135,20 @@ impl InstanceDict { } impl PyInner { + /// Run the destructors of the fields that have one, payload first. + /// + /// Declaration order would drop `typ` first, and `PyAtomicRef::drop` + /// leaves it null. A weakref payload is still linked into the list of the + /// object it points at until its own `Drop` unlinks it, and a thread + /// walking that list reads the class off every node it passes, so the + /// class has to outlive the payload. + unsafe fn drop_fields(ptr: *mut Self) { + unsafe { + core::ptr::drop_in_place(&raw mut (*ptr).payload); + core::ptr::drop_in_place(&raw mut (*ptr).typ); + } + } + /// Deallocate a PyInner, handling optional prefix(es). /// Layout: [ObjExt?][WeakRefList?][PyInner] /// @@ -1161,8 +1186,7 @@ impl PyInner { let alloc_ptr = (ptr as *mut u8).sub(inner_offset); - // Drop PyInner (payload, typ, etc.) - core::ptr::drop_in_place(ptr); + Self::drop_fields(ptr); // Drop ObjExt if present (dict, slots) if has_ext { @@ -1177,10 +1201,13 @@ impl PyInner { } } else if published { let layout = core::alloc::Layout::new::(); - core::ptr::drop_in_place(ptr); + Self::drop_fields(ptr); crate::object::qsbr::free_delayed(ptr as *mut u8, layout); } else { - drop(Box::from_raw(ptr)); + Self::drop_fields(ptr); + // The fields are gone; the box is only here to free the memory + // the matching `Box::new` in `new` allocated. + drop(Box::from_raw(ptr.cast::>())); } } } @@ -2907,4 +2934,53 @@ mod tests { let obj = ctx.new_bytes(b"dfghjkl".to_vec()); drop(obj); } + + /// A weakref node stays linked into its target's list until its own + /// `Drop` unlinks it, and `WeakRefList::add` reads the class off every + /// node it walks looking for a proxy to reuse. A node that lost its class + /// while still linked made that walk dereference a null type pointer. + #[cfg(feature = "threading")] + #[test] + fn weakref_proxies_keep_their_class_while_linked() { + const THREADS: usize = 8; + const ROUNDS: usize = 20_000; + + crate::Interpreter::without_stdlib(Default::default()).enter(|vm| { + let target: PyObjectRef = vm + .ctx + .new_class( + None, + "WeakrefTarget", + vm.ctx.types.object_type.to_owned(), + Default::default(), + ) + .into(); + let workers = (0..THREADS) + .map(|_| { + let thread_vm = vm.new_thread(); + let target = target.clone(); + std::thread::spawn(move || { + thread_vm.run(|vm| { + let proxy_type = vm.ctx.types.weakproxy_type.to_owned(); + for _ in 0..ROUNDS { + let proxy = target + .downgrade_with_typ(None, proxy_type.clone(), vm) + .expect("a type object takes weakrefs"); + drop(proxy); + vm.check_signals().unwrap(); + } + }) + }) + }) + .collect::>(); + // Detach while joining: a thread that blocks attached never + // reaches a safepoint, so a collection started by a worker could + // not finish. + vm.allow_threads(|| { + for worker in workers { + worker.join().unwrap(); + } + }); + }); + } } diff --git a/crates/vm/src/stdlib/_ast.rs b/crates/vm/src/stdlib/_ast.rs index b36277f5456..5cac1576676 100644 --- a/crates/vm/src/stdlib/_ast.rs +++ b/crates/vm/src/stdlib/_ast.rs @@ -1803,7 +1803,7 @@ pub(crate) fn parse( type_comments: bool, optimized_ast: bool, interactive: bool, - explicit_future_annotations: bool, + explicit_future_features: crate::bytecode::CodeFlags, dont_imply_dedent: bool, ) -> Result { let source_file = SourceFileBuilder::new("".to_owned(), source.to_owned()).finish(); @@ -1813,7 +1813,12 @@ pub(crate) fn parse( return Err(error); } options = options.with_target_version(target_version); - let parsed = parser::parse_unchecked(source, options); + let barry_source = rustpython_compiler::prepare_barry_as_flufl_source( + source, + options.clone(), + explicit_future_features.contains(crate::bytecode::CodeFlags::FUTURE_BARRY_AS_BDFL), + ); + let parsed = parser::parse_unchecked(barry_source.source(), options); let type_comment_source = type_comments.then(|| TypeCommentSource::new(source, parsed.tokens())); if let Some(lines) = &type_comment_source @@ -1823,6 +1828,9 @@ pub(crate) fn parse( } if let Err(errors) = parsed.as_result() { let parse_error = errors[0].clone(); + if let Some(error) = barry_source.diagnostic(Some(&parse_error), &source_file) { + return Err(error); + } let range = text_range_to_source_range(&source_file, parse_error.location); return Err(ParseError { error: parse_error.error, @@ -1834,6 +1842,9 @@ pub(crate) fn parse( } .into()); } + if let Some(error) = barry_source.diagnostic(None, &source_file) { + return Err(error); + } if dont_imply_dedent && interactive && let Some(error) = rustpython_compiler::dont_imply_dedent_source_error(&source_file) @@ -1880,7 +1891,8 @@ pub(crate) fn parse( { let future_features = codegen::preprocess::checked_future_features(&top) .map_err(|err| future_feature_compile_error(&source_file, err))?; - let future_annotations = explicit_future_annotations + let future_annotations = explicit_future_features + .contains(crate::bytecode::CodeFlags::FUTURE_ANNOTATIONS) || future_features.contains(crate::bytecode::CodeFlags::FUTURE_ANNOTATIONS); if interactive && let ast::Mod::Module(module) = &mut top { codegen::preprocess::preprocess_statements( diff --git a/crates/vm/src/stdlib/_symtable.rs b/crates/vm/src/stdlib/_symtable.rs index eb0ecaa87d7..cf3fa8bafc8 100644 --- a/crates/vm/src/stdlib/_symtable.rs +++ b/crates/vm/src/stdlib/_symtable.rs @@ -3,9 +3,10 @@ pub(crate) use _symtable::module_def; #[pymodule] mod _symtable { use crate::{ - Py, PyObjectRef, PyPayload, PyRef, PyResult, VirtualMachine, - builtins::{PyDictRef, PyUtf8StrRef}, + AsObject, Py, PyPayload, PyRef, PyResult, VirtualMachine, + builtins::{PyBaseExceptionRef, PyDictRef, PyListRef, PyStrRef, PyUtf8StrRef}, compiler, + function::{ArgStrOrBytesLike, FsPath}, types::Representable, }; use alloc::fmt; @@ -97,8 +98,8 @@ mod _symtable { #[pyfunction] fn symtable( - source: PyUtf8StrRef, - filename: PyUtf8StrRef, + source: ArgStrOrBytesLike, + filename: FsPath, mode: PyUtf8StrRef, vm: &VirtualMachine, ) -> PyResult> { @@ -107,15 +108,97 @@ mod _symtable { .parse::() .map_err(|err| vm.new_value_error(err.to_string()))?; - let symtable = compiler::compile_symtable(source.as_str(), mode, filename.as_str()) - .map_err(|err| vm.new_syntax_error(&err, Some(source.as_str())))?; + let filename_obj = match &filename { + FsPath::Str(filename) => filename.clone(), + FsPath::Bytes(filename) => { + let filename = FsPath::bytes_as_os_str(filename.as_bytes(), vm)?.to_owned(); + vm.fsdecode(filename) + } + }; + let filename = filename_obj.to_string_lossy(); + let source = match &source { + ArgStrOrBytesLike::Str(source) => source.try_as_utf8(vm)?.as_str().to_owned(), + ArgStrOrBytesLike::Buf(source) => vm + .decode_source_bytes(&source.borrow_buf(), &filename, false) + .map_err(|err| set_syntax_error_filename(err, &filename_obj, vm))?, + }; + if source.as_bytes().contains(&0) { + return Err(vm.new_exception_msg( + vm.ctx.exceptions.syntax_error.to_owned(), + "source code string cannot contain null bytes".into(), + )); + } + let symtable = compiler::compile_symtable(&source, mode, &filename).map_err(|err| { + let err = vm.new_syntax_error(&err, Some(&source)); + set_syntax_error_filename(err, &filename_obj, vm) + })?; + + Ok(to_py_symbol_table(symtable, vm)) + } + + fn set_syntax_error_filename( + err: PyBaseExceptionRef, + filename: &PyStrRef, + vm: &VirtualMachine, + ) -> PyBaseExceptionRef { + if err.fast_isinstance(vm.ctx.exceptions.syntax_error) { + err.as_object() + .set_attr("filename", filename.clone(), vm) + .unwrap(); + } + err + } - let py_symbol_table = to_py_symbol_table(symtable); - Ok(py_symbol_table.into_ref(&vm.ctx)) + fn append_visible_child(table: SymbolTable, children: &mut Vec) { + if table.comp_inlined { + for child in table.sub_tables { + append_visible_child(child, children); + } + } else { + children.push(table); + } } - const fn to_py_symbol_table(symtable: SymbolTable) -> PySymbolTable { - PySymbolTable { symtable } + fn to_py_symbol_table(mut symtable: SymbolTable, vm: &VirtualMachine) -> PyRef { + let mut child_tables = Vec::new(); + for table in core::mem::take(&mut symtable.sub_tables) { + append_visible_child(table, &mut child_tables); + } + if !symtable.future_annotations + && let Some(annotation_block) = symtable.annotation_block.take() + { + child_tables.push(*annotation_block); + } + child_tables.sort_by_key(|table| table.block_index); + + let children = vm.ctx.new_list( + child_tables + .into_iter() + .map(|table| to_py_symbol_table(table, vm).into()) + .collect(), + ); + let symbols = vm.ctx.new_dict(); + for (name, symbol) in &symtable.symbols { + let packed_flags = + i32::from(symbol.flags.bits()) | (symbol.scope.as_i32() << SCOPE_OFFSET); + symbols + .set_item(name, vm.new_pyobj(packed_flags), vm) + .unwrap(); + } + let varnames = vm.ctx.new_list( + symtable + .varnames + .iter() + .map(|name| vm.ctx.new_str(name.as_str()).into()) + .collect(), + ); + PySymbolTable { + symtable, + children, + symbols, + varnames, + } + .into_ref(&vm.ctx) } #[pyattr] @@ -123,6 +206,9 @@ mod _symtable { #[derive(PyPayload)] struct PySymbolTable { symtable: SymbolTable, + children: PyListRef, + symbols: PyDictRef, + varnames: PyListRef, } impl fmt::Debug for PySymbolTable { @@ -160,20 +246,8 @@ mod _symtable { } #[pygetset] - fn children(&self, vm: &VirtualMachine) -> Vec { - self.symtable - .sub_tables - .iter() - .flat_map(|t| { - if t.comp_inlined { - // Flatten: replace inlined comprehension tables with their children - t.sub_tables.iter().collect::>() - } else { - vec![t] - } - }) - .map(|t| to_py_symbol_table(t.clone()).into_pyobject(vm)) - .collect() + fn children(&self) -> PyListRef { + self.children.clone() } #[pygetset] @@ -182,14 +256,13 @@ mod _symtable { } #[pygetset] - fn symbols(&self, vm: &VirtualMachine) -> PyDictRef { - let dict = vm.ctx.new_dict(); - for (name, symbol) in &self.symtable.symbols { - let packed_flags = - i32::from(symbol.flags.bits()) | (symbol.scope.as_i32() << SCOPE_OFFSET); - dict.set_item(name, vm.new_pyobj(packed_flags), vm).unwrap(); - } - dict + fn symbols(&self) -> PyDictRef { + self.symbols.clone() + } + + #[pygetset] + fn varnames(&self) -> PyListRef { + self.varnames.clone() } #[pygetset] diff --git a/crates/vm/src/stdlib/builtins.rs b/crates/vm/src/stdlib/builtins.rs index c3eb200af6d..75d84abfb70 100644 --- a/crates/vm/src/stdlib/builtins.rs +++ b/crates/vm/src/stdlib/builtins.rs @@ -32,7 +32,7 @@ mod builtins { types::PyComparisonOp, vm::compile_mode::{ CompilerFlags, PY_EVAL_INPUT, PY_FILE_INPUT, PY_FUNC_TYPE_INPUT, PY_SINGLE_INPUT, - compile_future_feature_mask, compile_future_features_from_flags, + compile_future_features_from_flags, }, }; use itertools::Itertools; @@ -130,9 +130,7 @@ mod builtins { ) -> bytecode::CodeFlags { let mut future_features = compile_future_features_from_flags(flags); if !dont_inherit && let Some(code) = crate::frame::current_code() { - future_features |= bytecode::CodeFlags::from_bits_truncate( - code.flags.bits() & compile_future_feature_mask().bits(), - ); + future_features |= code.flags & bytecode::CodeFlags::FUTURE_MASK; } future_features } @@ -653,9 +651,7 @@ mod builtins { let source = string.as_str(); let mut opts = vm.compile_opts(); if let Some(code) = crate::frame::current_code() { - opts.future_features = bytecode::CodeFlags::from_bits_truncate( - code.flags.bits() & compile_future_feature_mask().bits(), - ); + opts.future_features = code.flags & bytecode::CodeFlags::FUTURE_MASK; } vm.compile_with_opts(source, mode, "", opts) .map_err(|err| err.into_pyexception(vm, Some(source)))? diff --git a/crates/vm/src/vm/compile.rs b/crates/vm/src/vm/compile.rs index 2beaf24f07c..9b6c727f0ef 100644 --- a/crates/vm/src/vm/compile.rs +++ b/crates/vm/src/vm/compile.rs @@ -278,8 +278,6 @@ impl VirtualMachine { let is_ast_only = cf.contains(CompilerFlags::ONLY_AST); let optimized_ast = cf.contains(CompilerFlags::OPTIMIZED_AST); let future_features = compile_future_features_from_flags(flags); - let explicit_future_annotations = - future_features.contains(crate::bytecode::CodeFlags::FUTURE_ANNOTATIONS); let target_version = if is_ast_only { Some(ruff_python_ast::PythonVersion { major: 3, @@ -313,7 +311,7 @@ impl VirtualMachine { type_comments, optimized_ast, interactive, - explicit_future_annotations, + future_features, dont_imply_dedent, ) .map_err(|e| (e, Some(source), allow_incomplete).to_pyexception(self))?; @@ -342,7 +340,7 @@ impl VirtualMachine { type_comments, false, start == PY_SINGLE_INPUT, - explicit_future_annotations, + future_features, dont_imply_dedent, ) .map_err(|e| (e, Some(source), allow_incomplete).to_pyexception(self))?; @@ -1116,6 +1114,58 @@ mod escape_warnings { }) } + #[test] + fn ast_only_compile_honors_barry_as_flufl() { + Interpreter::without_stdlib(Default::default()).enter(|vm| { + let flags = CompilerFlags::ONLY_AST.bits(); + vm.compile_string_object_with_flags( + b"from __future__ import barry_as_FLUFL\n2 <> 3\n", + "", + PY_FILE_INPUT, + flags, + -1, + -1, + ) + .expect("PyCF_ONLY_AST should accept <> in Barry mode"); + + let err = vm + .compile_string_object_with_flags( + b"from __future__ import barry_as_FLUFL\n2 != 3\n", + "", + PY_FILE_INPUT, + flags, + -1, + -1, + ) + .expect_err("PyCF_ONLY_AST should reject != in Barry mode"); + assert!( + err.as_object() + .str(vm) + .unwrap() + .as_wtf8() + .to_string() + .contains("with Barry as BDFL") + ); + }); + } + + #[test] + fn type_comment_preparse_honors_inherited_barry_as_flufl() { + Interpreter::without_stdlib(Default::default()).enter(|vm| { + let flags = CompilerFlags::TYPE_COMMENTS.bits() + | crate::bytecode::CodeFlags::FUTURE_BARRY_AS_BDFL.bits() as i32; + vm.compile_string_object_with_flags( + b"2 <> 3\n", + "", + PY_FILE_INPUT, + flags, + -1, + -1, + ) + .expect("type-comment preparse should accept <> in inherited Barry mode"); + }); + } + #[test] fn codegen_caller_warning_precedes_later_return_error() { let message = compile_error_message("(1)()\nreturn\n"); diff --git a/crates/vm/src/vm/compile_mode.rs b/crates/vm/src/vm/compile_mode.rs index 9885ba2e1f7..6da531bbc1f 100644 --- a/crates/vm/src/vm/compile_mode.rs +++ b/crates/vm/src/vm/compile_mode.rs @@ -67,17 +67,6 @@ pub(crate) const PY_CF_ALLOW_TOP_LEVEL_AWAIT: i32 = CompilerFlags::ALLOW_TOP_LEV pub(crate) const PY_CF_ALLOW_INCOMPLETE_INPUT: i32 = CompilerFlags::ALLOW_INCOMPLETE_INPUT.bits(); pub(crate) const PY_CF_OPTIMIZED_AST: i32 = CompilerFlags::OPTIMIZED_AST.bits(); -pub(crate) fn compile_future_feature_mask() -> bytecode::CodeFlags { - // RustPython accepts barry_as_FLUFL but leaves its parser mode disabled. - bytecode::CodeFlags::FUTURE_DIVISION - | bytecode::CodeFlags::FUTURE_ABSOLUTE_IMPORT - | bytecode::CodeFlags::FUTURE_WITH_STATEMENT - | bytecode::CodeFlags::FUTURE_PRINT_FUNCTION - | bytecode::CodeFlags::FUTURE_UNICODE_LITERALS - | bytecode::CodeFlags::FUTURE_GENERATOR_STOP - | bytecode::CodeFlags::FUTURE_ANNOTATIONS -} - pub(crate) fn compile_future_features_from_flags(flags: i32) -> bytecode::CodeFlags { - bytecode::CodeFlags::from_bits_truncate(flags as u32 & compile_future_feature_mask().bits()) + bytecode::CodeFlags::from_bits_truncate(flags as u32 & bytecode::CodeFlags::FUTURE_MASK.bits()) } diff --git a/crates/vm/src/vm/interpreter.rs b/crates/vm/src/vm/interpreter.rs index c538eb32ec8..6d4e1f75a22 100644 --- a/crates/vm/src/vm/interpreter.rs +++ b/crates/vm/src/vm/interpreter.rs @@ -1095,11 +1095,22 @@ mod tests { ); let (lock, ready) = &*state; - let mut state = lock.lock().unwrap(); - state.entered += 1; - ready.notify_all(); - while !state.release { - state = ready.wait(state).unwrap(); + { + let mut state = lock.lock().unwrap(); + state.entered += 1; + ready.notify_all(); + } + // Wait attached, but keep passing safepoints: a thread + // that blocks outright while attached never suspends, + // so a concurrent stop-the-world could not finish and + // the other worker could never attach. + loop { + vm.check_signals().unwrap(); + let state = lock.lock().unwrap(); + if state.release { + break; + } + let _ = ready.wait_timeout(state, Duration::from_millis(1)).unwrap(); } }); }) @@ -1165,6 +1176,11 @@ mod tests { let result = vm._add(&a, &b).unwrap(); assert_eq!(*int::get_value(&result), 42_i32.to_bigint().unwrap()); operations += 1; + // The protocol calls above never reach a safepoint on + // their own; a bytecode loop would. Without this, a + // concurrent stop-the-world could not finish while this + // thread stays attached. + vm.check_signals().unwrap(); std::thread::yield_now(); } (sub_finished_worker.load(Ordering::Acquire), operations) diff --git a/crates/wasm/src/vm_class.rs b/crates/wasm/src/vm_class.rs index 80f3ece1358..6780d4d1bc6 100644 --- a/crates/wasm/src/vm_class.rs +++ b/crates/wasm/src/vm_class.rs @@ -10,6 +10,7 @@ use ruff_text_size::Ranged; use rustpython_vm::{ Interpreter, PyObjectRef, PyRef, PyResult, Settings, VirtualMachine, builtins::PyWeak, + bytecode::CodeFlags, compiler::{self, Mode}, function::ArgMapping, scope::Scope, @@ -24,6 +25,7 @@ pub(crate) struct StoredVirtualMachine { /// you can put a Rc in here, keep it as a Weak, and it'll be held only for /// as long as the StoredVM is alive held_objects: RefCell>, + future_features: RefCell, } fn compile_err_to_js(vm: &VirtualMachine, err: VmCompileError) -> JsValue { @@ -33,8 +35,17 @@ fn compile_err_to_js(vm: &VirtualMachine, err: VmCompileError) -> JsValue { } } -fn statement_chunks(source: &str) -> Option> { - let module = compiler::parser::parse_module(source).ok()?.into_syntax(); +fn statement_chunks(source: &str, future_features: CodeFlags) -> Option> { + let parser_options = compiler::parser::ParseOptions::from(compiler::parser::Mode::Module); + let prepared = compiler::prepare_barry_as_flufl_source( + source, + parser_options.clone(), + future_features.contains(CodeFlags::FUTURE_BARRY_AS_BDFL), + ); + let module = compiler::parser::parse(prepared.source(), parser_options) + .ok()? + .into_syntax() + .expect_module(); module .body .iter() @@ -99,6 +110,7 @@ impl StoredVirtualMachine { interp, scope, held_objects: RefCell::new(Vec::new()), + future_features: RefCell::new(CodeFlags::empty()), } } } @@ -394,11 +406,24 @@ impl WASMVirtualMachine { source: &str, source_path: Option, ) -> Result { - self.with_vm(|vm, StoredVirtualMachine { scope, .. }| { + self.with_vm(|vm, stored| { + let scope = &stored.scope; let source_path = source_path.unwrap_or_else(|| "".to_owned()); - let Some(chunks) = statement_chunks(source) else { - let code = vm.compile(source, Mode::Single, source_path.as_str()); - let code = code.map_err(|err| compile_err_to_js(vm, err))?; + let compile = |source: &str, mode: Mode| -> Result<_, JsValue> { + let future_features = *stored.future_features.borrow(); + let opts = compiler::CompileOpts { + future_features, + ..vm.compile_opts() + }; + let code = vm + .compile_with_opts(source, mode, source_path.as_str(), opts) + .map_err(|err| compile_err_to_js(vm, err))?; + *stored.future_features.borrow_mut() |= code.code.flags & CodeFlags::FUTURE_MASK; + Ok(code) + }; + let future_features = *stored.future_features.borrow(); + let Some(chunks) = statement_chunks(source, future_features) else { + let code = compile(source, Mode::Single)?; let result = vm.run_code_obj(code, scope.clone()); return convert::pyresult_to_js_result(vm, result); }; @@ -413,8 +438,7 @@ impl WASMVirtualMachine { .map_err(|_| TypeError::new("lost sys.displayhook"))?; let mut result = vm.ctx.none(); for chunk in chunks { - let code = vm.compile(chunk, Mode::BlockExpr, source_path.as_str()); - let code = code.map_err(|err| compile_err_to_js(vm, err))?; + let code = compile(chunk, Mode::BlockExpr)?; result = vm.run_code_obj(code, scope.clone()).into_js(vm)?; displayhook.call((result.clone(),), vm).into_js(vm)?; } diff --git a/extra_tests/snippets/builtin_compile.py b/extra_tests/snippets/builtin_compile.py index 73247e50df1..ff11bbda2d8 100644 --- a/extra_tests/snippets/builtin_compile.py +++ b/extra_tests/snippets/builtin_compile.py @@ -1,7 +1,6 @@ import __future__ import ast -import sys from testutils import assert_raises @@ -66,8 +65,7 @@ def _check_flags_error(flags): barry_flag = __future__.barry_as_FLUFL.compiler_flag barry_code = compile("x = 1", "", "exec", flags=barry_flag) compile("from __future__ import barry_as_FLUFL\nx = 1\n", "", "exec") -if sys.implementation.name == "rustpython": - assert not (barry_code.co_flags & barry_flag) +assert barry_code.co_flags & barry_flag n = ast.parse('x = "# type: int"\n', type_comments=True) assert n.body[0].type_comment is None diff --git a/src/shell.rs b/src/shell.rs index 7fb9336af4b..d99d38765e0 100644 --- a/src/shell.rs +++ b/src/shell.rs @@ -7,6 +7,7 @@ use rustpython_compiler::{ use rustpython_vm::{ AsObject, PyResult, VirtualMachine, builtins::PyBaseExceptionRef, + bytecode::CodeFlags, compiler::{self}, readline::{Readline, ReadlineResult}, scope::Scope, @@ -26,6 +27,7 @@ fn shell_exec( scope: Scope, empty_line_given: bool, continuing_block: bool, + future_features: &mut CodeFlags, ) -> ShellExecResult { // compiling expects only UNIX style line endings, and will replace windows line endings // internally. Since we might need to analyze the source to determine if an error could be @@ -33,8 +35,13 @@ fn shell_exec( // was actually compiled. #[cfg(windows)] let source = &source.replace("\r\n", "\n"); - match vm.compile(source, compiler::Mode::Single, "") { + let opts = compiler::CompileOpts { + future_features: *future_features, + ..vm.compile_opts() + }; + match vm.compile_with_opts(source, compiler::Mode::Single, "", opts) { Ok(code) => { + *future_features |= code.code.flags & CodeFlags::FUTURE_MASK; if empty_line_given || !continuing_block { // We want to execute the full code match vm.run_code_obj(code, scope) { @@ -131,6 +138,7 @@ pub fn run_shell(vm: &VirtualMachine, scope: Scope) -> PyResult<()> { // valid. let mut continuing_block = false; let mut continuing_line = false; + let mut future_features = CodeFlags::empty(); loop { let prompt_name = if continuing_block || continuing_line { @@ -170,6 +178,7 @@ pub fn run_shell(vm: &VirtualMachine, scope: Scope) -> PyResult<()> { scope.clone(), empty_line_given, continuing_block, + &mut future_features, ) { ShellExecResult::Ok => { if continuing_block {