From 892b897ac62b31da4b3d2f56d2ad9023e9239cc8 Mon Sep 17 00:00:00 2001 From: Josh Megnauth Date: Fri, 15 May 2026 13:35:11 -0400 Subject: [PATCH] Use icu4x for UTF-8 float(); remove an alloc I removed an embedded table of non-ASCII numbers in favor of using `icu_decimal`. The benefits of using `icu4x` here are consistency plus Unicode updates. As Unicode is updated, we automatically reap the benefits without having to modify the table. `icu_decimal` is also useful beyond `float()`. I'm also using it to clean up `unicodedata` in #7947. --- Cargo.lock | 54 +++++++++++++++++++++++++++++++++ Cargo.toml | 1 + crates/common/src/str.rs | 44 --------------------------- crates/vm/Cargo.toml | 1 + crates/vm/src/builtins/float.rs | 31 +++++++++++-------- crates/vm/src/utils.rs | 40 ++++++++++++++++++++++++ 6 files changed, 114 insertions(+), 57 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d632ce7f741..0d0cf4b554f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1252,6 +1252,17 @@ version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" +[[package]] +name = "fixed_decimal" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "79c3c892f121fff406e5dd6b28c1b30096b95111c30701a899d4f2b18da6d1bd" +dependencies = [ + "displaydoc", + "smallvec", + "writeable", +] + [[package]] name = "flagset" version = "0.4.7" @@ -1679,6 +1690,29 @@ dependencies = [ "zerovec", ] +[[package]] +name = "icu_decimal" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "288247df2e32aa776ac54fdd64de552149ac43cb840f2761811f0e8d09719dd4" +dependencies = [ + "displaydoc", + "fixed_decimal", + "icu_decimal_data", + "icu_locale", + "icu_locale_core", + "icu_plurals", + "icu_provider", + "writeable", + "zerovec", +] + +[[package]] +name = "icu_decimal_data" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6f14a5ca9e8af29eef62064f269078424283d90dbaffeac5225addf62aaabc22" + [[package]] name = "icu_locale" version = "2.2.0" @@ -1737,6 +1771,25 @@ version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "da3be0ae77ea334f4da67c12f149704f19f81d1adf7c51cf482943e84a2bad38" +[[package]] +name = "icu_plurals" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a50023f1d49ad5c4333380328a0d4a19e4b9d6d842ec06639affd5ba47c8103" +dependencies = [ + "fixed_decimal", + "icu_locale", + "icu_plurals_data", + "icu_provider", + "zerovec", +] + +[[package]] +name = "icu_plurals_data" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8485497155dc865f901decb93ecc20d3e467df67bfeceb91e3ba34e2b11e8e1d" + [[package]] name = "icu_properties" version = "2.2.0" @@ -3848,6 +3901,7 @@ dependencies = [ "half", "hex", "icu_casemap", + "icu_decimal", "icu_locale", "icu_properties", "indexmap", diff --git a/Cargo.toml b/Cargo.toml index de2704842c3..dc726d37c87 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -306,6 +306,7 @@ thiserror = "2.0" timsort = "0.1.2" tk-sys = { git = "https://github.com/arihant2math/tkinter.git", tag = "v0.2.0" } icu_casemap = "2" +icu_decimal = "2" icu_locale = "2" icu_properties = "2" icu_normalizer = "2" diff --git a/crates/common/src/str.rs b/crates/common/src/str.rs index 3fddef04bb8..57a7a270fde 100644 --- a/crates/common/src/str.rs +++ b/crates/common/src/str.rs @@ -615,50 +615,6 @@ macro_rules! ascii { } pub use ascii; -// TODO: this should probably live in a crate like unic or unicode-properties -const UNICODE_DECIMAL_VALUES: &[char] = &[ - '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '٠', '١', '٢', '٣', '٤', '٥', '٦', '٧', '٨', - '٩', '۰', '۱', '۲', '۳', '۴', '۵', '۶', '۷', '۸', '۹', '߀', '߁', '߂', '߃', '߄', '߅', '߆', '߇', - '߈', '߉', '०', '१', '२', '३', '४', '५', '६', '७', '८', '९', '০', '১', '২', '৩', '৪', '৫', '৬', - '৭', '৮', '৯', '੦', '੧', '੨', '੩', '੪', '੫', '੬', '੭', '੮', '੯', '૦', '૧', '૨', '૩', '૪', '૫', - '૬', '૭', '૮', '૯', '୦', '୧', '୨', '୩', '୪', '୫', '୬', '୭', '୮', '୯', '௦', '௧', '௨', '௩', '௪', - '௫', '௬', '௭', '௮', '௯', '౦', '౧', '౨', '౩', '౪', '౫', '౬', '౭', '౮', '౯', '೦', '೧', '೨', '೩', - '೪', '೫', '೬', '೭', '೮', '೯', '൦', '൧', '൨', '൩', '൪', '൫', '൬', '൭', '൮', '൯', '෦', '෧', '෨', - '෩', '෪', '෫', '෬', '෭', '෮', '෯', '๐', '๑', '๒', '๓', '๔', '๕', '๖', '๗', '๘', '๙', '໐', '໑', - '໒', '໓', '໔', '໕', '໖', '໗', '໘', '໙', '༠', '༡', '༢', '༣', '༤', '༥', '༦', '༧', '༨', '༩', '၀', - '၁', '၂', '၃', '၄', '၅', '၆', '၇', '၈', '၉', '႐', '႑', '႒', '႓', '႔', '႕', '႖', '႗', '႘', '႙', - '០', '១', '២', '៣', '៤', '៥', '៦', '៧', '៨', '៩', '᠐', '᠑', '᠒', '᠓', '᠔', '᠕', '᠖', '᠗', '᠘', - '᠙', '᥆', '᥇', '᥈', '᥉', '᥊', '᥋', '᥌', '᥍', '᥎', '᥏', '᧐', '᧑', '᧒', '᧓', '᧔', '᧕', '᧖', '᧗', - '᧘', '᧙', '᪀', '᪁', '᪂', '᪃', '᪄', '᪅', '᪆', '᪇', '᪈', '᪉', '᪐', '᪑', '᪒', '᪓', '᪔', '᪕', '᪖', - '᪗', '᪘', '᪙', '᭐', '᭑', '᭒', '᭓', '᭔', '᭕', '᭖', '᭗', '᭘', '᭙', '᮰', '᮱', '᮲', '᮳', '᮴', '᮵', - '᮶', '᮷', '᮸', '᮹', '᱀', '᱁', '᱂', '᱃', '᱄', '᱅', '᱆', '᱇', '᱈', '᱉', '᱐', '᱑', '᱒', '᱓', '᱔', - '᱕', '᱖', '᱗', '᱘', '᱙', '꘠', '꘡', '꘢', '꘣', '꘤', '꘥', '꘦', '꘧', '꘨', '꘩', '꣐', '꣑', '꣒', '꣓', - '꣔', '꣕', '꣖', '꣗', '꣘', '꣙', '꤀', '꤁', '꤂', '꤃', '꤄', '꤅', '꤆', '꤇', '꤈', '꤉', '꧐', '꧑', '꧒', - '꧓', '꧔', '꧕', '꧖', '꧗', '꧘', '꧙', '꧰', '꧱', '꧲', '꧳', '꧴', '꧵', '꧶', '꧷', '꧸', '꧹', '꩐', '꩑', - '꩒', '꩓', '꩔', '꩕', '꩖', '꩗', '꩘', '꩙', '꯰', '꯱', '꯲', '꯳', '꯴', '꯵', '꯶', '꯷', '꯸', '꯹', '０', - '１', '２', '３', '４', '５', '６', '７', '８', '９', '𐒠', '𐒡', '𐒢', '𐒣', '𐒤', '𐒥', '𐒦', '𐒧', - '𐒨', '𐒩', '𑁦', '𑁧', '𑁨', '𑁩', '𑁪', '𑁫', '𑁬', '𑁭', '𑁮', '𑁯', '𑃰', '𑃱', '𑃲', '𑃳', '𑃴', '𑃵', '𑃶', - '𑃷', '𑃸', '𑃹', '𑄶', '𑄷', '𑄸', '𑄹', '𑄺', '𑄻', '𑄼', '𑄽', '𑄾', '𑄿', '𑇐', '𑇑', '𑇒', '𑇓', '𑇔', '𑇕', - '𑇖', '𑇗', '𑇘', '𑇙', '𑋰', '𑋱', '𑋲', '𑋳', '𑋴', '𑋵', '𑋶', '𑋷', '𑋸', '𑋹', '𑑐', '𑑑', '𑑒', '𑑓', '𑑔', - '𑑕', '𑑖', '𑑗', '𑑘', '𑑙', '𑓐', '𑓑', '𑓒', '𑓓', '𑓔', '𑓕', '𑓖', '𑓗', '𑓘', '𑓙', '𑙐', '𑙑', '𑙒', '𑙓', - '𑙔', '𑙕', '𑙖', '𑙗', '𑙘', '𑙙', '𑛀', '𑛁', '𑛂', '𑛃', '𑛄', '𑛅', '𑛆', '𑛇', '𑛈', '𑛉', '𑜰', '𑜱', '𑜲', - '𑜳', '𑜴', '𑜵', '𑜶', '𑜷', '𑜸', '𑜹', '𑣠', '𑣡', '𑣢', '𑣣', '𑣤', '𑣥', '𑣦', '𑣧', '𑣨', '𑣩', '𑱐', '𑱑', - '𑱒', '𑱓', '𑱔', '𑱕', '𑱖', '𑱗', '𑱘', '𑱙', '𑵐', '𑵑', '𑵒', '𑵓', '𑵔', '𑵕', '𑵖', '𑵗', '𑵘', '𑵙', '𖩠', - '𖩡', '𖩢', '𖩣', '𖩤', '𖩥', '𖩦', '𖩧', '𖩨', '𖩩', '𖭐', '𖭑', '𖭒', '𖭓', '𖭔', '𖭕', '𖭖', '𖭗', '𖭘', '𖭙', - '𝟎', '𝟏', '𝟐', '𝟑', '𝟒', '𝟓', '𝟔', '𝟕', '𝟖', '𝟗', '𝟘', '𝟙', '𝟚', '𝟛', '𝟜', '𝟝', '𝟞', '𝟟', '𝟠', - '𝟡', '𝟢', '𝟣', '𝟤', '𝟥', '𝟦', '𝟧', '𝟨', '𝟩', '𝟪', '𝟫', '𝟬', '𝟭', '𝟮', '𝟯', '𝟰', '𝟱', '𝟲', '𝟳', - '𝟴', '𝟵', '𝟶', '𝟷', '𝟸', '𝟹', '𝟺', '𝟻', '𝟼', '𝟽', '𝟾', '𝟿', '𞥐', '𞥑', '𞥒', '𞥓', '𞥔', '𞥕', '𞥖', - '𞥗', '𞥘', '𞥙', -]; - -#[must_use] -pub fn char_to_decimal(ch: char) -> Option { - UNICODE_DECIMAL_VALUES - .binary_search(&ch) - .ok() - .map(|i| (i % 10) as u8) -} - #[cfg(test)] mod tests { use super::*; diff --git a/crates/vm/Cargo.toml b/crates/vm/Cargo.toml index 83e41fa1f5f..b97045e3181 100644 --- a/crates/vm/Cargo.toml +++ b/crates/vm/Cargo.toml @@ -81,6 +81,7 @@ timsort = { workspace = true } ## unicode stuff icu_casemap = { workspace = true } +icu_decimal = { workspace = true } icu_locale = { workspace = true } icu_properties = { workspace = true } writeable = { workspace = true } diff --git a/crates/vm/src/builtins/float.rs b/crates/vm/src/builtins/float.rs index f36f9de79d4..69e2d8b3e4a 100644 --- a/crates/vm/src/builtins/float.rs +++ b/crates/vm/src/builtins/float.rs @@ -14,13 +14,16 @@ use crate::{ }, protocol::PyNumberMethods, types::{AsNumber, Callable, Comparable, Constructor, Hashable, PyComparisonOp, Representable}, + utils::SliceFmtWriter, }; use core::cell::Cell; use core::ptr::NonNull; +use icu_decimal::input::Decimal; use malachite_bigint::{BigInt, ToBigInt}; use num_complex::Complex64; use num_traits::{Signed, ToPrimitive, Zero}; use rustpython_common::int::float_to_ratio; +use writeable::Writeable; #[pyclass(module = false, name = "float")] #[derive(Debug, Copy, Clone, PartialEq)] @@ -206,26 +209,28 @@ impl Constructor for PyFloat { } fn float_from_string(val: PyObjectRef, vm: &VirtualMachine) -> PyResult { - let (bytearray, buffer, buffer_lock, mapped_string); + let (bytearray, buffer, buffer_lock); + let mut mapped_str_buf = [0u8; 1024]; let b = if let Some(s) = val.downcast_ref::() { use crate::common::str::PyKindStr; match s.as_str_kind() { PyKindStr::Ascii(s) => s.trim().as_bytes(), PyKindStr::Utf8(s) => { - mapped_string = s - .trim() - .chars() - .map(|c| { - if let Some(n) = rustpython_common::str::char_to_decimal(c) { - char::from_digit(n.into(), 10).unwrap() - } else if c.is_whitespace() { - ' ' + match Decimal::try_from_str(s.trim()) { + Ok(decimal) => { + let mut writer = SliceFmtWriter::new(&mut mapped_str_buf); + if decimal.write_to(&mut writer).is_err() { + s.as_bytes() } else { - c + let written = writer.written(); + &mapped_str_buf[..written] } - }) - .collect::(); - mapped_string.as_bytes() + } + Err(_) => { + // Let the string fail parsing below + s.as_bytes() + } + } } // if there are surrogates, it's not gonna parse anyway, // so we can just choose a known bad value diff --git a/crates/vm/src/utils.rs b/crates/vm/src/utils.rs index 51e27123fc8..bcbe907cbb8 100644 --- a/crates/vm/src/utils.rs +++ b/crates/vm/src/utils.rs @@ -87,3 +87,43 @@ impl fmt::Write for VecFmtWriter { Ok(()) } } + +/// Wrapper around a bytes slice that implements [`fmt::Write`]. +/// +/// # Errors +/// [`fmt::Error`] is returned if the string doesn't fit into the internal buffer. This +/// implementation writes as many bytes as can fit before returning an error. +/// Check [`Self::remainder`] if the amount of bytes that can be written is important. +pub(crate) struct SliceFmtWriter<'slice> { + buf: &'slice mut [u8], + written: usize, +} + +impl<'slice> SliceFmtWriter<'slice> { + pub(crate) const fn new(buf: &'slice mut [u8]) -> Self { + Self { buf, written: 0 } + } + + /// Amount of space left in the internal buffer. + pub(crate) const fn remainder(&self) -> usize { + self.buf.len() - self.written + } + + /// Amount of bytes written to the internal buffer. + pub(crate) const fn written(&self) -> usize { + self.written + } +} + +impl fmt::Write for SliceFmtWriter<'_> { + fn write_str(&mut self, s: &str) -> fmt::Result { + let to_copy = self.remainder().min(s.len()); + self.buf[self.written..self.written + to_copy].copy_from_slice(&s.as_bytes()[..to_copy]); + self.written += to_copy; + if to_copy == s.len() { + Ok(()) + } else { + Err(fmt::Error) + } + } +}