From c7b254be678c90b1bdb7b35b5580280a11d5d6e0 Mon Sep 17 00:00:00 2001 From: Josh Megnauth Date: Sat, 18 Apr 2026 15:42:37 -0400 Subject: [PATCH] Use Unicode properties for alnum, alpha, etc. Rust and Python differ in which properties they use for alphanumeric, numeric, et cetera. Both languages list which properties are used which makes it easy to mimic Python's behavior in Rust. My previous patch was a bit shortsighted because I filtered out combining characters from is_alphanumeric. Using properties is exact and also much cleaner. It also covers edge cases that my initial approach missed. Besides isalnum, I also fixed isnumeric and isdigit in the same way by using properties. --- Lib/test/test_str.py | 1 - crates/sre_engine/src/string.rs | 9 +++++---- crates/vm/src/builtins/str.rs | 27 ++++++++++++++++++--------- extra_tests/snippets/builtin_str.py | 13 +++++++++++++ 4 files changed, 36 insertions(+), 14 deletions(-) diff --git a/Lib/test/test_str.py b/Lib/test/test_str.py index 68037923283..15cee0d3a44 100644 --- a/Lib/test/test_str.py +++ b/Lib/test/test_str.py @@ -792,7 +792,6 @@ def test_isdecimal(self): for ch in ['\U0001D7F6', '\U00011066', '\U000104A0']: self.assertTrue(ch.isdecimal(), '{!a} is decimal.'.format(ch)) - @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: False != True def test_isdigit(self): super().test_isdigit() self.checkequalnofix(True, '\u2460', 'isdigit') diff --git a/crates/sre_engine/src/string.rs b/crates/sre_engine/src/string.rs index b4b3a6092d3..1350c9a07e1 100644 --- a/crates/sre_engine/src/string.rs +++ b/crates/sre_engine/src/string.rs @@ -1,4 +1,4 @@ -use icu_properties::props::{CanonicalCombiningClass, EnumeratedProperty}; +use icu_properties::props::{EnumeratedProperty, GeneralCategory, GeneralCategoryGroup}; use rustpython_wtf8::Wtf8; #[derive(Debug, Clone, Copy)] @@ -444,9 +444,10 @@ pub(crate) const fn is_uni_linebreak(ch: u32) -> bool { pub(crate) fn is_uni_alnum(ch: u32) -> bool { // TODO: check with cpython char::try_from(ch) - .map(|x| { - x.is_alphanumeric() - && CanonicalCombiningClass::for_char(x) == CanonicalCombiningClass::NotReordered + .map(|c| { + GeneralCategoryGroup::Letter + .union(GeneralCategoryGroup::Number) + .contains(GeneralCategory::for_char(c)) }) .unwrap_or(false) } diff --git a/crates/vm/src/builtins/str.rs b/crates/vm/src/builtins/str.rs index d74259b849c..af1c4a5ae92 100644 --- a/crates/vm/src/builtins/str.rs +++ b/crates/vm/src/builtins/str.rs @@ -45,8 +45,8 @@ use rustpython_common::{ }; use icu_properties::props::{ - BidiClass, BinaryProperty, CanonicalCombiningClass, EnumeratedProperty, GeneralCategory, - XidContinue, XidStart, + BidiClass, BinaryProperty, EnumeratedProperty, GeneralCategory, GeneralCategoryGroup, + NumericType, XidContinue, XidStart, }; use unicode_casing::CharExt; @@ -949,23 +949,30 @@ impl PyStr { fn isalnum(&self) -> bool { !self.data.is_empty() && self.char_all(|c| { - c.is_alphanumeric() - && CanonicalCombiningClass::for_char(c) == CanonicalCombiningClass::NotReordered + GeneralCategoryGroup::Letter + .union(GeneralCategoryGroup::Number) + .contains(GeneralCategory::for_char(c)) }) } #[pymethod] fn isnumeric(&self) -> bool { - !self.data.is_empty() && self.char_all(char::is_numeric) + !self.data.is_empty() + && self.char_all(|c| { + [ + NumericType::Decimal, + NumericType::Digit, + NumericType::Numeric, + ] + .contains(&NumericType::for_char(c)) + }) } #[pymethod] fn isdigit(&self) -> bool { - // python's isdigit also checks if exponents are digits, these are the unicode codepoints for exponents !self.data.is_empty() && self.char_all(|c| { - c.is_ascii_digit() - || matches!(c, '⁰' | '¹' | '²' | '³' | '⁴' | '⁵' | '⁶' | '⁷' | '⁸' | '⁹') + [NumericType::Digit, NumericType::Decimal].contains(&NumericType::for_char(c)) }) } @@ -1064,7 +1071,9 @@ impl PyStr { #[pymethod] fn isalpha(&self) -> bool { - !self.data.is_empty() && self.char_all(char::is_alphabetic) + !self.data.is_empty() + && self + .char_all(|c| GeneralCategoryGroup::Letter.contains(GeneralCategory::for_char(c))) } #[pymethod] diff --git a/extra_tests/snippets/builtin_str.py b/extra_tests/snippets/builtin_str.py index 61cbf63ea9a..3899c04956e 100644 --- a/extra_tests/snippets/builtin_str.py +++ b/extra_tests/snippets/builtin_str.py @@ -72,6 +72,7 @@ assert "\u1c89".istitle() # assert "DZ".title() == "Dz" assert a.isalpha() +assert not "\u093f".isalpha() # Combining characters differ slightly between Rust and Python assert "\u006e".isalnum() @@ -79,9 +80,21 @@ assert not "\u006e\u0303".isalnum() assert "\u00f1".isalnum() assert not "\u0345".isalnum() +assert not "\u093f".isalnum() for raw in range(0x0363, 0x036F): assert not chr(raw).isalnum() +# isdigit is true for exponents +assert "⁰".isdigit() +assert "⁰".isnumeric() +assert not "½".isdigit() +assert "½".isnumeric() +assert not "Ⅻ".isdigit() +assert "Ⅻ".isnumeric() + +# isnumeric is broader than Rust's +assert "\u3405".isnumeric() + s = "1 2 3" assert s.split(" ", 1) == ["1", "2 3"] assert s.rsplit(" ", 1) == ["1 2", "3"]