diff --git a/Lib/test/test_decimal.py b/Lib/test/test_decimal.py index 47ab0a79702..c621b7ac08c 100644 --- a/Lib/test/test_decimal.py +++ b/Lib/test/test_decimal.py @@ -854,10 +854,6 @@ class CExplicitConstructionTest(ExplicitConstructionTest, unittest.TestCase): class PyExplicitConstructionTest(ExplicitConstructionTest, unittest.TestCase): decimal = P - @unittest.expectedFailure # TODO: RUSTPYTHON - def test_unicode_digits(self): - return super().test_unicode_digits() - class ImplicitConstructionTest: '''Unit tests for Implicit Construction cases of Decimal.''' diff --git a/Lib/test/test_int.py b/Lib/test/test_int.py index a18683098e1..e281763a6c9 100644 --- a/Lib/test/test_int.py +++ b/Lib/test/test_int.py @@ -247,7 +247,6 @@ def test_invalid_signs(self): with self.assertRaises(ValueError): int(' + 1 ') - @unittest.expectedFailure # TODO: RUSTPYTHON def test_unicode(self): self.assertEqual(int("१२३४५६७८९०1234567890"), 12345678901234567890) self.assertEqual(int('١٢٣٤٥٦٧٨٩٠'), 1234567890) diff --git a/crates/common/src/str.rs b/crates/common/src/str.rs index 39ec7da1de5..c649c057de6 100644 --- a/crates/common/src/str.rs +++ b/crates/common/src/str.rs @@ -3,6 +3,7 @@ use crate::atomic::{OncePtr, PyAtomic, Radium}; use crate::format::CharLen; use crate::wtf8::{CodePoint, Wtf8, Wtf8Buf}; use crate::wtf8_index::Wtf8Index; +use alloc::borrow::Cow; use ascii::{AsciiChar, AsciiStr, AsciiString}; use core::fmt; use core::ops::{Bound, RangeBounds}; @@ -835,10 +836,63 @@ pub fn char_to_decimal(ch: char) -> Option { .map(|i| (i % 10) as u8) } +/// Replace Unicode decimal digits with their ASCII equivalents and any Unicode +/// whitespace with a plain space, so the byte-oriented numeric parsers can read +/// them. Mirrors CPython's `_PyUnicode_TransformDecimalAndSpaceToASCII`. +/// +/// The result is always ASCII. Any other non-ASCII character cannot appear in a +/// numeric literal, so it becomes a `?` and the rest of the string is dropped: +/// `?` is rejected by every parser at every base, which leaves the caller — the +/// one that knows the base and owns the original string — to raise the error. +#[must_use] +pub fn transform_decimal_and_space_to_ascii(s: &str) -> Cow<'_, str> { + if s.is_ascii() { + return Cow::Borrowed(s); + } + let mut out = String::with_capacity(s.len()); + for c in s.chars() { + if (c as u32) < 127 { + out.push(c); + } else if c.is_whitespace() { + out.push(' '); + } else if let Some(n) = char_to_decimal(c) { + out.push(char::from_digit(n.into(), 10).unwrap()); + } else { + out.push('?'); + break; + } + } + debug_assert!(out.is_ascii()); + Cow::Owned(out) +} + #[cfg(test)] mod tests { use super::*; + #[test] + fn transform_decimal_and_space() { + // ASCII input is passed through untouched, without allocating. + assert!(matches!( + transform_decimal_and_space_to_ascii("123"), + Cow::Borrowed("123") + )); + // Decimal digits from any script fold to ASCII. + assert_eq!(transform_decimal_and_space_to_ascii("١٢٣"), "123"); + assert_eq!(transform_decimal_and_space_to_ascii("１２३"), "123"); + assert_eq!(transform_decimal_and_space_to_ascii("1٢3"), "123"); + // Unicode whitespace folds to a plain space. + assert_eq!(transform_decimal_and_space_to_ascii("\u{3000}٣"), " 3"); + // ASCII characters ride through untouched, whatever they are. + assert_eq!(transform_decimal_and_space_to_ascii("0x١f"), "0x1f"); + assert_eq!(transform_decimal_and_space_to_ascii("-١_٢"), "-1_2"); + // Anything else poisons the literal and truncates it, so the result stays + // ASCII and the caller's parser is guaranteed to reject it. + assert_eq!(transform_decimal_and_space_to_ascii("½가"), "?"); + assert_eq!(transform_decimal_and_space_to_ascii("١٢가٣"), "12?"); + assert_eq!(transform_decimal_and_space_to_ascii("١\u{7f}"), "1?"); + } + #[test] fn get_chars_basic() { let s = "0123456789"; diff --git a/crates/vm/src/builtins/complex.rs b/crates/vm/src/builtins/complex.rs index c54b3bc1731..7dcbedf7e17 100644 --- a/crates/vm/src/builtins/complex.rs +++ b/crates/vm/src/builtins/complex.rs @@ -220,10 +220,10 @@ impl Constructor for PyComplex { "complex() can't take second arg if first is a string", )); } - let (re, im) = s - .to_str() - .and_then(rustpython_literal::complex::parse_str) - .ok_or_else(|| vm.new_value_error("complex() arg is a malformed string"))?; + let (re, im) = rustpython_literal::complex::parse_str( + &crate::protocol::numeric_literal_from_str(s), + ) + .ok_or_else(|| vm.new_value_error("complex() arg is a malformed string"))?; return Ok(Self::from(Complex64 { re, im })); } else { return Err(vm.new_type_error(format!( diff --git a/crates/vm/src/builtins/float.rs b/crates/vm/src/builtins/float.rs index 0b739694623..6a646440ec2 100644 --- a/crates/vm/src/builtins/float.rs +++ b/crates/vm/src/builtins/float.rs @@ -210,29 +210,8 @@ impl Constructor for PyFloat { pub fn float_from_string(val: PyObjectRef, vm: &VirtualMachine) -> PyResult { let (bytearray, buffer, buffer_lock, mapped_string); let b = if let Some(s) = val.downcast_ref::() { - use crate::common::str::PyKindStr; - match s.as_str_kind() { - PyKindStr::Ascii(s) => s.trim().as_bytes(), - PyKindStr::Utf8(s) => { - mapped_string = s - .trim() - .chars() - .map(|c| { - if let Some(n) = rustpython_common::str::char_to_decimal(c) { - char::from_digit(n.into(), 10).unwrap() - } else if c.is_whitespace() { - ' ' - } else { - c - } - }) - .collect::(); - mapped_string.as_bytes() - } - // if there are surrogates, it's not gonna parse anyway, - // so we can just choose a known bad value - PyKindStr::Wtf8(_) => b"", - } + mapped_string = crate::protocol::numeric_literal_from_str(s); + mapped_string.as_bytes() } else if let Some(bytes) = val.downcast_ref::() { bytes.as_bytes() } else if let Some(buf) = val.downcast_ref::() { diff --git a/crates/vm/src/builtins/int.rs b/crates/vm/src/builtins/int.rs index 60463ed0d58..bb7b5128073 100644 --- a/crates/vm/src/builtins/int.rs +++ b/crates/vm/src/builtins/int.rs @@ -16,7 +16,7 @@ use crate::{ ArgByteOrder, ArgIntoBool, FuncArgs, OptionalArg, OptionalOption, PyArithmeticValue, PyComparisonValue, }, - protocol::{PyNumberMethods, handle_bytes_to_int_err}, + protocol::{PyNumberMethods, handle_bytes_to_int_err, numeric_literal_from_str}, types::{AsNumber, Comparable, Constructor, Hashable, PyComparisonOp, Representable}, }; use alloc::fmt; @@ -822,7 +822,7 @@ struct IntToByteArgs { fn try_int_radix(obj: &PyObject, base: u32, vm: &VirtualMachine) -> PyResult { match_class!(match obj.to_owned() { string @ PyStr => { - let s = string.as_wtf8().trim(); + let s = numeric_literal_from_str(&string); bytes_to_int(s.as_bytes(), base, vm.state.int_max_str_digits.load()) .map_err(|e| handle_bytes_to_int_err(e, obj, vm)) } diff --git a/crates/vm/src/protocol/mod.rs b/crates/vm/src/protocol/mod.rs index 4061e06458a..1d191daee31 100644 --- a/crates/vm/src/protocol/mod.rs +++ b/crates/vm/src/protocol/mod.rs @@ -16,5 +16,6 @@ pub use mapping::{PyMapping, PyMappingMethods, PyMappingSlots}; pub use number::{ PyNumber, PyNumberBinaryFunc, PyNumberBinaryOp, PyNumberMethods, PyNumberSlots, PyNumberTernaryFunc, PyNumberTernaryOp, PyNumberUnaryFunc, handle_bytes_to_int_err, + numeric_literal_from_str, }; pub use sequence::{PySequence, PySequenceMethods, PySequenceSlots}; diff --git a/crates/vm/src/protocol/number.rs b/crates/vm/src/protocol/number.rs index 301499aa115..6f566431da7 100644 --- a/crates/vm/src/protocol/number.rs +++ b/crates/vm/src/protocol/number.rs @@ -8,11 +8,34 @@ use crate::{ builtins::{ PyBaseExceptionRef, PyByteArray, PyBytes, PyComplex, PyFloat, PyInt, PyIntRef, PyStr, int, }, - common::int::{BytesToIntError, bytes_to_int}, + common::{ + int::{BytesToIntError, bytes_to_int}, + str::{PyKindStr, transform_decimal_and_space_to_ascii}, + }, function::ArgBytesLike, object::{Traverse, TraverseFn}, stdlib::_warnings, }; +use alloc::borrow::Cow; + +/// Normalize a `str` for the byte-oriented numeric parsers: Unicode decimal digits +/// and whitespace fold to their ASCII equivalents, the way CPython runs every +/// numeric constructor's string argument through +/// `_PyUnicode_TransformDecimalAndSpaceToASCII` first. +/// +/// `int`, `float` and `complex` share this step and nothing else — only `int` takes +/// a base, and only `int` and `float` accept bytes-like input, so each keeps its own +/// entry point around this one. +/// +/// A string holding surrogates can never be a valid literal, so it folds to an +/// empty — and therefore invalid — one. +pub fn numeric_literal_from_str(s: &PyStr) -> Cow<'_, str> { + match s.as_str_kind() { + PyKindStr::Ascii(s) => Cow::Borrowed(s.trim().as_str()), + PyKindStr::Utf8(s) => transform_decimal_and_space_to_ascii(s.trim()), + PyKindStr::Wtf8(_) => Cow::Borrowed(""), + } +} pub type PyNumberUnaryFunc = fn(PyNumber<'_>, &VirtualMachine) -> PyResult; pub type PyNumberBinaryFunc = fn(&PyObject, &PyObject, &VirtualMachine) -> PyResult; @@ -59,7 +82,7 @@ impl PyObject { } else if let Some(i) = self.number().int(vm).or_else(|| self.try_index_opt(vm)) { i } else if let Some(s) = self.downcast_ref::() { - try_convert(self, s.as_wtf8().trim().as_bytes(), vm) + try_convert(self, numeric_literal_from_str(s).as_bytes(), vm) } else if let Some(bytes) = self.downcast_ref::() { try_convert(self, bytes, vm) } else if let Some(bytearray) = self.downcast_ref::() {