From b7e1e76822279a7b5db904c910f166b89973a4e9 Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Mon, 6 Jul 2026 01:43:26 +0900 Subject: [PATCH 1/9] Extract unicodedata core into rustpython-unicode crate MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Move the vm-free Unicode character database access — the generated UCD 3.2.0 / latest tables, their build-time generation, and the icu4x/unicode_names2 lookups for category/bidirectional/combining/east_asian_width/mirrored/decomposition/ normalize/is_normalized/digit/decimal/numeric/name/lookup — into a new leaf crate over char/CodePoint/&Wtf8. stdlib/unicodedata.rs keeps the UCD pyclass binding: it extracts the code point, boxes results, and maps errors. The unicode data files, build.rs table generation, and the icu_properties/icu_normalizer/unicode_names2 dependencies move out of stdlib. Assisted-by: Claude --- Cargo.lock | 14 +- Cargo.toml | 1 + crates/stdlib/Cargo.toml | 11 +- crates/stdlib/build.rs | 616 +----------------- crates/stdlib/src/unicodedata.rs | 393 ++--------- crates/unicode/Cargo.toml | 22 + crates/unicode/build.rs | 612 +++++++++++++++++ crates/unicode/src/data.rs | 343 ++++++++++ crates/unicode/src/lib.rs | 51 ++ crates/unicode/src/normalize.rs | 73 +++ crates/{stdlib => unicode}/unicode/README.md | 0 .../unicode/latest/DerivedNumericValues.txt | 0 .../latest/NormalizationCorrections.txt | 0 .../unicode/latest/UnicodeData.txt | 0 .../unicode/ucd32/DerivedBidiClass-3.2.0.txt | 0 .../ucd32/DerivedBinaryProperties-3.2.0.txt | 0 .../ucd32/DerivedCombiningClass-3.2.0.txt | 0 .../ucd32/DerivedEastAsianWidth-3.2.0.txt | 0 .../ucd32/DerivedGeneralCategory-3.2.0.txt | 0 .../ucd32/DerivedNumericType-3.2.0.txt | 0 .../ucd32/DerivedNumericValues-3.2.0.txt | 0 21 files changed, 1158 insertions(+), 978 deletions(-) create mode 100644 crates/unicode/Cargo.toml create mode 100644 crates/unicode/build.rs create mode 100644 crates/unicode/src/data.rs create mode 100644 crates/unicode/src/lib.rs create mode 100644 crates/unicode/src/normalize.rs rename crates/{stdlib => unicode}/unicode/README.md (100%) rename crates/{stdlib => unicode}/unicode/latest/DerivedNumericValues.txt (100%) rename crates/{stdlib => unicode}/unicode/latest/NormalizationCorrections.txt (100%) rename crates/{stdlib => unicode}/unicode/latest/UnicodeData.txt (100%) rename crates/{stdlib => unicode}/unicode/ucd32/DerivedBidiClass-3.2.0.txt (100%) rename crates/{stdlib => unicode}/unicode/ucd32/DerivedBinaryProperties-3.2.0.txt (100%) rename crates/{stdlib => unicode}/unicode/ucd32/DerivedCombiningClass-3.2.0.txt (100%) rename crates/{stdlib => unicode}/unicode/ucd32/DerivedEastAsianWidth-3.2.0.txt (100%) rename crates/{stdlib => unicode}/unicode/ucd32/DerivedGeneralCategory-3.2.0.txt (100%) rename crates/{stdlib => unicode}/unicode/ucd32/DerivedNumericType-3.2.0.txt (100%) rename crates/{stdlib => unicode}/unicode/ucd32/DerivedNumericValues-3.2.0.txt (100%) diff --git a/Cargo.lock b/Cargo.lock index bab008f86c0..3b4e815d257 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3571,8 +3571,6 @@ dependencies = [ "gethostname", "hex", "hmac", - "icu_normalizer", - "icu_properties", "indexmap", "insta", "itertools 0.15.0", @@ -3612,6 +3610,7 @@ dependencies = [ "rustpython-ruff_python_parser", "rustpython-ruff_source_file", "rustpython-ruff_text_size", + "rustpython-unicode", "rustpython-vm", "sha1 0.11.0", "sha2", @@ -3621,7 +3620,6 @@ dependencies = [ "system-configuration", "tcl-sys", "tk-sys", - "unicode_names2 3.1.0", "uuid", "webpki-roots", "widestring", @@ -3632,6 +3630,16 @@ dependencies = [ "xz-sys", ] +[[package]] +name = "rustpython-unicode" +version = "0.5.0" +dependencies = [ + "icu_normalizer", + "icu_properties", + "rustpython-wtf8", + "unicode_names2 3.1.0", +] + [[package]] name = "rustpython-venvlauncher" version = "0.5.0" diff --git a/Cargo.toml b/Cargo.toml index fb023e03e4a..20930e05120 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -179,6 +179,7 @@ rustpython-vm = { path = "crates/vm", default-features = false, version = "0.5.0 rustpython-pylib = { path = "crates/pylib", version = "0.5.0" } rustpython-stdlib = { path = "crates/stdlib", default-features = false, version = "0.5.0" } rustpython-sre_engine = { path = "crates/sre_engine", version = "0.5.0" } +rustpython-unicode = { path = "crates/unicode", version = "0.5.0" } rustpython-wtf8 = { path = "crates/wtf8", version = "0.5.0" } rustpython-doc = { path = "crates/doc", version = "0.5.0" } diff --git a/crates/stdlib/Cargo.toml b/crates/stdlib/Cargo.toml index a24811c4aea..1ba6148e8cf 100644 --- a/crates/stdlib/Cargo.toml +++ b/crates/stdlib/Cargo.toml @@ -32,6 +32,7 @@ rustpython-derive = { workspace = true } rustpython-vm = { workspace = true, default-features = false, features = ["compiler"]} rustpython-common = { workspace = true } rustpython-host_env = { workspace = true } +rustpython-unicode = { workspace = true } ruff_python_parser = { workspace = true } ruff_python_ast = { workspace = true } @@ -76,12 +77,6 @@ hmac = { workspace = true } pbkdf2 = { workspace = true, features = ["hmac"] } constant_time_eq = { workspace = true } -## unicode stuff -unicode_names2 = { workspace = true } -# update version all at the same time -icu_properties = { workspace = true } -icu_normalizer = { workspace = true } - # compression adler32 = { workspace = true } crc32fast = { workspace = true } @@ -141,9 +136,5 @@ system-configuration = { workspace = true } insta = { workspace = true } rustpython-pylib = { workspace = true, features = [ "freeze-stdlib" ] } -[build-dependencies] -icu_normalizer = { workspace = true } -icu_properties = { workspace = true } - [lints] workspace = true diff --git a/crates/stdlib/build.rs b/crates/stdlib/build.rs index 4cf7b21d4b7..95c34c4fb3c 100644 --- a/crates/stdlib/build.rs +++ b/crates/stdlib/build.rs @@ -1,606 +1,4 @@ -#![allow( - clippy::disallowed_methods, - reason = "build scripts cannot use rustpython-host_env" -)] - -// spell-checker:ignore decomp DECOMP ossl osslconf - -extern crate alloc; - -use core::num::NonZeroUsize; - -use alloc::collections::{BTreeMap, BTreeSet}; - -use std::{ - env, - fs::{self, File}, - io::{self, BufRead, BufReader, BufWriter, Write}, - path::{Path, PathBuf}, - thread, -}; - -use icu_properties::props::{EnumeratedProperty, GeneralCategory, NumericType}; - -fn generate_unicode_3_2() { - let path = PathBuf::from(env::var("OUT_DIR").unwrap()) - .join("generated") - .join("unicode_3_2.rs"); - fs::create_dir_all(path.parent().unwrap()).unwrap(); - let mut writer = BufWriter::new(File::create(&path).unwrap()); - - let base = Path::new(env!("CARGO_MANIFEST_DIR")) - .join("unicode") - .join("ucd32"); - - write_derived( - &base, - "DerivedGeneralCategory-3.2.0.txt", - "GENERAL_CATEGORY", - "(u32, u32, GeneralCategory)", - NonZeroUsize::new(1).unwrap(), - &mut writer, - |start, end, id, _| { - let id = parse_general(id); - if id != GeneralCategory::Unassigned { - Some((start, end, id)) - } else { - None - } - }, - |writer, mut values| { - values.sort_unstable_by_key(|(start, _, _)| *start); - write!(writer, "[").unwrap(); - for (start, end, id) in values { - write!(writer, "({start}, {end}, GeneralCategory::{id:?}),").unwrap(); - } - write!(writer, "];").unwrap(); - }, - ); - - write_derived( - &base, - "DerivedEastAsianWidth-3.2.0.txt", - "EAST_ASIAN_WIDTH", - "(u32, u32, EastAsianWidth)", - NonZeroUsize::new(1).unwrap(), - &mut writer, - |start, end, id, _| { - let id = parse_eaw(id); - if id != "EastAsianWidth::Neutral" { - Some((start, end, id)) - } else { - None - } - }, - |writer, mut values| { - values.sort_unstable_by_key(|(start, _, _)| *start); - write!(writer, "[").unwrap(); - for (start, end, id) in values { - write!(writer, "({start}, {end}, {id}),").unwrap(); - } - write!(writer, "];").unwrap(); - }, - ); - - write_derived( - &base, - "DerivedBidiClass-3.2.0.txt", - "BIDI_CLASS", - "(u32, u32, BidiClass)", - NonZeroUsize::new(1).unwrap(), - &mut writer, - |start, end, id, _| { - let id = parse_bidi(id); - if id != "BidiClass::LeftToRight" { - Some((start, end, id)) - } else { - None - } - }, - |writer, mut values| { - values.sort_unstable_by_key(|(start, _, _)| *start); - write!(writer, "[").unwrap(); - for (start, end, id) in values { - write!(writer, "({start}, {end}, {id}),").unwrap(); - } - write!(writer, "];").unwrap(); - }, - ); - - write_derived( - &base, - "DerivedBinaryProperties-3.2.0.txt", - "BIDI_MIRRORED", - "(u32, u32)", - NonZeroUsize::new(1).unwrap(), - &mut writer, - |start, end, id, _| { - assert_eq!( - "Bidi_Mirrored", - id.trim(), - "DerivedBinaryProperties-3.2.0 only has Bidi_Mirrored" - ); - Some((start, end)) - }, - |writer, mut values| { - values.sort_unstable_by_key(|(start, _)| *start); - writeln!(writer, "{values:?};").unwrap(); - }, - ); - - write_derived( - &base, - "DerivedCombiningClass-3.2.0.txt", - "COMBINING_CLASS", - "(u32, u32, CanonicalCombiningClass)", - NonZeroUsize::new(1).unwrap(), - &mut writer, - |start, end, id, _| { - let id: u8 = id.parse().unwrap(); - if id == 0 { - return None; - } - Some((start, end, id)) - }, - |writer, mut values| { - values.sort_unstable_by_key(|(start, _, _)| *start); - write!(writer, "[").unwrap(); - for (start, end, id) in values { - write!( - writer, - "({start}, {end}, CanonicalCombiningClass::from_icu4c_value({id}))," - ) - .unwrap(); - } - writeln!(writer, "];").unwrap(); - }, - ); -} - -fn generate_numeric_type() { - let path = PathBuf::from(env::var("OUT_DIR").unwrap()) - .join("generated") - .join("unicode_num_type.rs"); - fs::create_dir_all(path.parent().unwrap()).unwrap(); - let mut writer = BufWriter::new(File::create(&path).unwrap()); - - let base = Path::new(env!("CARGO_MANIFEST_DIR")) - .join("unicode") - .join("ucd32"); - - write_derived( - &base, - "DerivedNumericType-3.2.0.txt", - "NUMERIC_TYPE_DIFF", - "(u32, u32, NumericType)", - NonZeroUsize::new(1).unwrap(), - &mut writer, - |start, end, id, _| { - let id = parse_numeric_type_str(id); - let differs = (start..=end).any(|c| match char::from_u32(c) { - Some(c) => { - let modern = parse_numeric_type_val(NumericType::for_char(c)); - modern != id - } - None => true, - }); - - if differs { - Some((start, end, id)) - } else { - None - } - }, - |writer, mut values| { - values.sort_unstable_by_key(|(start, _, _)| *start); - write!(writer, "[").unwrap(); - for (start, end, id) in values { - write!(writer, "({start}, {end}, {id}),").unwrap(); - } - writeln!(writer, "];").unwrap(); - }, - ); -} - -fn generate_numeric_value() { - let path = PathBuf::from(env::var("OUT_DIR").unwrap()) - .join("generated") - .join("unicode_numeric_value.rs"); - fs::create_dir_all(path.parent().unwrap()).unwrap(); - let mut writer = BufWriter::new(File::create(&path).unwrap()); - - // Ideally, this would store the diffs between the two tables. However, we need 3.2.0 - // membership as well as different chars. The final tables are both smaller than storing the - // full 3.2.0 value table. - let ucd32 = Path::new(env!("CARGO_MANIFEST_DIR")) - .join("unicode") - .join("ucd32"); - let mut ucd32_diffs = BTreeMap::new(); - let mut ucd32_member = BTreeSet::new(); - let numeric_32 = - BufReader::new(File::open(ucd32.join("DerivedNumericValues-3.2.0.txt")).unwrap()); - parse_unicode_3_2( - numeric_32, - NonZeroUsize::new(1).unwrap(), - &mut io::empty(), - |start, end, value, _| { - let value: f64 = value - .parse() - .expect("Unicode data contains valid properties"); - ucd32_diffs.insert((start, end), value); - ucd32_member.insert((start, end)); - Option::<()>::None - }, - |_writer, _values| {}, - ); - - let ucd_latest = Path::new(env!("CARGO_MANIFEST_DIR")) - .join("unicode") - .join("latest"); - - write_derived( - &ucd_latest, - "DerivedNumericValues.txt", - "NUMERIC_VALUES", - "(u32, u32, f64)", - NonZeroUsize::new(1).unwrap(), - &mut writer, - |start, end, value, _| { - let value: f64 = value - .parse() - .expect("Unicode data contains valid properties"); - - if ucd32_diffs - .get(&(start, end)) - .is_some_and(|old_v| *old_v == value) - { - ucd32_diffs.remove(&(start, end)); - } - - Some((start, end, value)) - }, - |writer, mut values| { - values.sort_unstable_by_key(|(ch, _, _)| *ch); - writeln!(writer, "{values:?};").unwrap(); - }, - ); - - // TODO: More flexible parser - writeln!( - writer, - "static NUMERIC_VALUES_DIFF: &[(u32, u32, f64)] = &[" - ) - .unwrap(); - for ((start, end), value) in ucd32_diffs { - write!(writer, "({start}, {end}, {value:?}),").unwrap(); - } - writeln!(writer, "];").unwrap(); - - // Compress membership table - let mut iter = ucd32_member.iter(); - let &(mut start_prev, mut end_prev) = iter.next().unwrap(); - let mut membership = Vec::new(); - - for &(start, end) in iter { - if start <= end_prev + 1 { - end_prev = end_prev.max(end); - } else { - membership.push((start_prev, end_prev)); - start_prev = start; - end_prev = end; - } - } - membership.push((start_prev, end_prev)); - membership.sort_unstable_by_key(|&(start, _)| start); - - writeln!(writer, "static NUMERIC_VAL_EXISTS_32: &[(u32, u32)] = &").unwrap(); - write!(writer, "{membership:?};").unwrap(); -} - -fn generate_unicode_latest() { - let path = PathBuf::from(env::var("OUT_DIR").unwrap()) - .join("generated") - .join("unicode_latest.rs"); - fs::create_dir_all(path.parent().unwrap()).unwrap(); - let mut writer = BufWriter::new(File::create(&path).unwrap()); - - let base = Path::new(env!("CARGO_MANIFEST_DIR")) - .join("unicode") - .join("latest"); - - // NOTE: - // This ONLY parses compatibility decomposition because Python exposes the tags. The tags are - // the "", "", et cetera bits before the decomposition. Thus, we can save space - // by using icu4x's CanonicalDecomposer for non-compatibility decomposition. - let mut decomp_ranges = Vec::new(); - write_derived( - &base, - "UnicodeData.txt", - "DECOMP_COMPAT", - "(u32, DecompositionType, usize)", - NonZeroUsize::new(5).unwrap(), - &mut writer, - |start, _end, value, _| { - // We're building a sparse array. Most characters don't decompose, so we don't - // need to literally store a row for each char. - if value.is_empty() { - return None; - } - - let (dtype, decomp) = value.split_once('>').map(|(dtype, decomp)| { - let dtype = dtype.strip_prefix('<').unwrap_or_else(|| { - panic!("Compatibility decomp; expected \n\tgot: {value}") - }); - ( - parse_decomp_type(dtype), - decomp - .split_whitespace() - .map(|s| u32::from_str_radix(s, 16).unwrap()), - ) - })?; - - decomp_ranges.extend(decomp); - let end = decomp_ranges.len(); - - Some((start, dtype, end)) - }, - |writer, values| { - // UnicodeData.txt should already be sorted - write!(writer, "[").unwrap(); - for (start, dtype, end) in values { - write!(writer, "({start}, DecompositionType::{dtype:?}, {end}),").unwrap(); - } - writeln!(writer, "];").unwrap(); - }, - ); - - writeln!(writer, "static DECOMP_RANGE: &[u32] = &{decomp_ranges:?};").unwrap(); - - // Normalization corrections is super small - only a handful chars at the time of writing. - write_derived( - &base, - "NormalizationCorrections.txt", - "DECOMP_UPDATES", - "(u32, u32)", - NonZeroUsize::new(1).unwrap(), - &mut writer, - |start, _end, value, line| { - let original = u32::from_str_radix(value.trim(), 16).unwrap_or_else(|e| { - panic!("field 2 of decomp corrections should be a char in hex: {value} {e}") - }); - let version = line - .rsplit(';') - .next() - .unwrap_or_else(|| { - panic!("field 4 of decomp corrections should be a UCD version: {line}") - }) - .split_once('#') - .unwrap() - .0 - .trim(); - - // `version` = when the char was updated. Therefore, we use the incorrect chars past - // 3.2.0 but skip the chars fixed in 3.2.0 because they'll already be right. - if version != "3.2.0" { - Some((start, original)) - } else { - None - } - }, - |writer, mut values| { - values.sort_unstable_by_key(|(c, _)| *c); - write!(writer, "{values:?};").unwrap(); - }, - ); -} - -#[expect(clippy::too_many_arguments)] -fn write_derived( - base: &Path, - file_name: &str, - static_name: &str, - array_type: &str, - field: NonZeroUsize, - writer: &mut W, - parse: P, - write_vec: FW, -) where - W: Write, - P: FnMut(u32, u32, &str, &str) -> Option, - FW: FnMut(&mut W, Vec), -{ - let path = base.join(file_name); - let reader = BufReader::new(File::open(path).unwrap()); - writeln!(writer, "static {static_name}: &[{array_type}] = &").unwrap(); - parse_unicode_3_2(reader, field, writer, parse, write_vec); -} - -/// Parse Unicode 3.2.0 property files. -fn parse_unicode_3_2( - reader: impl BufRead, - field: NonZeroUsize, - writer: &mut W, - mut parse: P, - mut write_vec: FW, -) where - W: Write, - P: FnMut(u32, u32, &str, &str) -> Option, - FW: FnMut(&mut W, Vec), -{ - let mut parsed = Vec::new(); - - for line in reader.lines().map(Result::unwrap) { - let line = line.trim(); - if line.is_empty() || line.starts_with('#') { - continue; - } - - let mut fields = line.split(';'); - let range = fields.next().expect("Unicode data is missing a char range"); - let id = fields - .nth(field.get().saturating_sub(1)) - .expect("Unicode data is missing a property"); - let (start, end) = match range.split_once("..") { - Some((left, right)) => { - let start = u32::from_str_radix(left.trim(), 16).unwrap(); - let end = u32::from_str_radix(right.trim(), 16).unwrap(); - (start, end) - } - None => { - let start = u32::from_str_radix(range.trim(), 16).unwrap(); - (start, start) - } - }; - - let id = id.split_once('#').map_or(id, |(left, _)| left).trim(); - if let Some(val) = parse(start, end, id, line) { - parsed.push(val); - } - } - write_vec(writer, parsed); -} - -fn parse_general(id: &str) -> GeneralCategory { - match id.trim() { - "Cn" => GeneralCategory::Unassigned, - "Lu" => GeneralCategory::UppercaseLetter, - "Ll" => GeneralCategory::LowercaseLetter, - "Lt" => GeneralCategory::TitlecaseLetter, - "Lm" => GeneralCategory::ModifierLetter, - "Lo" => GeneralCategory::OtherLetter, - "Mn" => GeneralCategory::NonspacingMark, - "Mc" => GeneralCategory::SpacingMark, - "Me" => GeneralCategory::EnclosingMark, - "Nd" => GeneralCategory::DecimalNumber, - "Nl" => GeneralCategory::LetterNumber, - "No" => GeneralCategory::OtherNumber, - "Zs" => GeneralCategory::SpaceSeparator, - "Zl" => GeneralCategory::LineSeparator, - "Zp" => GeneralCategory::ParagraphSeparator, - "Cc" => GeneralCategory::Control, - "Cf" => GeneralCategory::Format, - "Co" => GeneralCategory::PrivateUse, - "Cs" => GeneralCategory::Surrogate, - "Pd" => GeneralCategory::DashPunctuation, - "Ps" => GeneralCategory::OpenPunctuation, - "Pe" => GeneralCategory::ClosePunctuation, - "Pc" => GeneralCategory::ConnectorPunctuation, - "Pi" => GeneralCategory::InitialPunctuation, - "Pf" => GeneralCategory::FinalPunctuation, - "Po" => GeneralCategory::OtherPunctuation, - "Sm" => GeneralCategory::MathSymbol, - "Sc" => GeneralCategory::CurrencySymbol, - "Sk" => GeneralCategory::ModifierSymbol, - "So" => GeneralCategory::OtherSymbol, - invalid => unreachable!("Unicode data contains valid properties: {invalid}"), - } -} - -fn parse_eaw(id: &str) -> &'static str { - match id.trim() { - "N" => "EastAsianWidth::Neutral", - "A" => "EastAsianWidth::Ambiguous", - "H" => "EastAsianWidth::Halfwidth", - "F" => "EastAsianWidth::Fullwidth", - "Na" => "EastAsianWidth::Narrow", - "W" => "EastAsianWidth::Wide", - invalid => unreachable!("Unicode data contains valid properties: {invalid}"), - } -} - -fn parse_bidi(id: &str) -> &'static str { - match id.trim() { - "L" => "BidiClass::LeftToRight", - "R" => "BidiClass::RightToLeft", - "EN" => "BidiClass::EuropeanNumber", - "ES" => "BidiClass::EuropeanSeparator", - "ET" => "BidiClass::EuropeanTerminator", - "AN" => "BidiClass::ArabicNumber", - "CS" => "BidiClass::CommonSeparator", - "B" => "BidiClass::ParagraphSeparator", - "S" => "BidiClass::SegmentSeparator", - "WS" => "BidiClass::WhiteSpace", - "ON" => "BidiClass::OtherNeutral", - "LRE" => "BidiClass::LeftToRightEmbedding", - "LRO" => "BidiClass::LeftToRightOverride", - "AL" => "BidiClass::ArabicLetter", - "RLE" => "BidiClass::RightToLeftEmbedding", - "RLO" => "BidiClass::RightToLeftOverride", - "PDF" => "BidiClass::PopDirectionalFormat", - "NSM" => "BidiClass::NonspacingMark", - "BN" => "BidiClass::BoundaryNeutral", - "FSI" => "BidiClass::FirstStrongIsolate", - "LRI" => "BidiClass::LeftToRightIsolate", - "RLI" => "BidiClass::RightToLeftIsolate", - "PDI" => "BidiClass::PopDirectionalIsolate", - invalid => unreachable!("Unicode data contains valid properties: {invalid}"), - } -} - -fn parse_numeric_type_val(val: NumericType) -> &'static str { - match val { - NumericType::None => "none", - NumericType::Decimal => "decimal", - NumericType::Digit => "digit", - NumericType::Numeric => "numeric", - _ => unreachable!("Unicode data contains valid properties"), - } -} - -fn parse_numeric_type_str(id: &str) -> &'static str { - match id { - "none" => "NumericType::None", - "decimal" => "NumericType::Decimal", - "digit" => "NumericType::Digit", - "numeric" => "NumericType::Numeric", - invalid => unreachable!("Unicode data contains valid properties: {invalid}"), - } -} - -#[derive(Debug, Default)] -enum DecompositionType { - #[default] - Canonical, - Compat, - Circle, - Final, - Font, - Fraction, - Initial, - Isolated, - Medial, - Narrow, - Nobreak, - Small, - Square, - Sub, - Super, - Vertical, - Wide, -} - -fn parse_decomp_type(id: &str) -> DecompositionType { - match id { - "canonical" => DecompositionType::Canonical, - "compat" => DecompositionType::Compat, - "circle" => DecompositionType::Circle, - "final" => DecompositionType::Final, - "font" => DecompositionType::Font, - "fraction" => DecompositionType::Fraction, - "initial" => DecompositionType::Initial, - "isolated" => DecompositionType::Isolated, - "medial" => DecompositionType::Medial, - "narrow" => DecompositionType::Narrow, - "noBreak" => DecompositionType::Nobreak, - "small" => DecompositionType::Small, - "square" => DecompositionType::Square, - "sub" => DecompositionType::Sub, - "super" => DecompositionType::Super, - "vertical" => DecompositionType::Vertical, - "wide" => DecompositionType::Wide, - invalid => unreachable!("Unicode data contains valid properties: {invalid}"), - } -} +// spell-checker:ignore ossl osslconf fn main() { println!(r#"cargo::rustc-check-cfg=cfg(osslconf, values("OPENSSL_NO_COMP"))"#); @@ -655,16 +53,4 @@ fn main() { println!("cargo::rustc-cfg=openssl_vendored") } } - - println!("cargo:rerun-if-changed=unicode/ucd32"); - println!("cargo:rerun-if-changed=unicode/latest"); - - let t_32 = thread::spawn(generate_unicode_3_2); - let t_numeric_type = thread::spawn(generate_numeric_type); - let t_numeric_value = thread::spawn(generate_numeric_value); - let t_latest = thread::spawn(generate_unicode_latest); - t_32.join().unwrap(); - t_numeric_type.join().unwrap(); - t_numeric_value.join().unwrap(); - t_latest.join().unwrap(); } diff --git a/crates/stdlib/src/unicodedata.rs b/crates/stdlib/src/unicodedata.rs index 0d6e7b97226..b7f097125a8 100644 --- a/crates/stdlib/src/unicodedata.rs +++ b/crates/stdlib/src/unicodedata.rs @@ -2,130 +2,26 @@ See also: https://docs.python.org/3/library/unicodedata.html */ -// spell-checker:ignore codep decomp DECOMP nfkc unistr unidata - -use core::{cmp::Ordering, hint::cold_path}; +// spell-checker:ignore nfkc unistr unidata pub(crate) use unicodedata::module_def; -use icu_properties::props::{ - BidiClass, CanonicalCombiningClass, EastAsianWidth, GeneralCategory, NumericType, -}; +use rustpython_unicode::{self as unicode_core, NormalizeForm}; use crate::vm::{ PyObject, PyResult, VirtualMachine, builtins::PyStr, convert::TryFromBorrowedObject, }; -include!(concat!(env!("OUT_DIR"), "/generated/unicode_3_2.rs")); -include!(concat!(env!("OUT_DIR"), "/generated/unicode_latest.rs")); -include!(concat!(env!("OUT_DIR"), "/generated/unicode_num_type.rs")); -include!(concat!( - env!("OUT_DIR"), - "/generated/unicode_numeric_value.rs" -)); - -#[derive(Clone, Copy)] -#[repr(u8)] -enum DecompositionType { - #[allow(unused)] - Canonical, - Compat, - Circle, - Final, - Font, - Fraction, - Initial, - Isolated, - Medial, - Narrow, - Nobreak, - Small, - Square, - Sub, - Super, - Vertical, - Wide, -} - -impl DecompositionType { - const fn type_tag(self) -> &'static str { - match self { - Self::Canonical => "canonical", - Self::Compat => "compat", - Self::Circle => "circle", - Self::Final => "final", - Self::Font => "font", - Self::Fraction => "fraction", - Self::Initial => "initial", - Self::Isolated => "isolated", - Self::Medial => "medial", - Self::Narrow => "narrow", - Self::Nobreak => "noBreak", - Self::Small => "small", - Self::Square => "square", - Self::Sub => "sub", - Self::Super => "super", - Self::Vertical => "vertical", - Self::Wide => "wide", - } - } -} - -#[derive(Clone, Copy, Eq, PartialEq)] -enum NormalizeForm { - Nfc, - Nfkc, - Nfd, - Nfkd, -} +struct NormalizeFormArg(NormalizeForm); -fn lookup_property(table: &[(u32, u32, T)], ch: char) -> Option { - let ch = ch as u32; - table - .binary_search_by(|&(start, end, _)| { - if ch > end { - Ordering::Less - } else if ch < start { - Ordering::Greater - } else { - Ordering::Equal - } - }) - .ok() - .map(|i| table[i].2) -} - -fn lookup_numeric_val(ch: char, modern: bool) -> Option { - if modern { - lookup_property(NUMERIC_VALUES, ch) - } else { - cold_path(); - lookup_property(NUMERIC_VALUES_DIFF, ch).or_else(|| { - NUMERIC_VAL_EXISTS_32 - .binary_search_by(|&(start, end)| { - let ch = ch as u32; - if ch > end { - Ordering::Less - } else if ch < start { - Ordering::Greater - } else { - Ordering::Equal - } - }) - .ok() - .and_then(|_| lookup_property(NUMERIC_VALUES, ch)) - }) - } -} - -impl<'a> TryFromBorrowedObject<'a> for NormalizeForm { +impl<'a> TryFromBorrowedObject<'a> for NormalizeFormArg { fn try_from_borrowed_object(vm: &VirtualMachine, obj: &'a PyObject) -> PyResult { obj.try_value_with( |form: &PyStr| match form.as_bytes() { - b"NFC" => Ok(Self::Nfc), - b"NFKC" => Ok(Self::Nfkc), - b"NFD" => Ok(Self::Nfd), - b"NFKD" => Ok(Self::Nfkd), + b"NFC" => Ok(Self(NormalizeForm::Nfc)), + b"NFKC" => Ok(Self(NormalizeForm::Nfkc)), + b"NFD" => Ok(Self(NormalizeForm::Nfd)), + b"NFKD" => Ok(Self(NormalizeForm::Nfkd)), _ => Err(vm.new_value_error("invalid normalization form")), }, vm, @@ -135,27 +31,12 @@ impl<'a> TryFromBorrowedObject<'a> for NormalizeForm { #[pymodule] mod unicodedata { - use core::{cmp::Ordering, fmt::Write, hint::cold_path}; - - use super::{ - BIDI_CLASS, BIDI_MIRRORED, COMBINING_CLASS, DECOMP_COMPAT, DECOMP_RANGE, DECOMP_UPDATES, - EAST_ASIAN_WIDTH, GENERAL_CATEGORY, NUMERIC_TYPE_DIFF, NormalizeForm, lookup_numeric_val, - lookup_property, - }; + use super::{NormalizeFormArg, unicode_core}; use crate::vm::{ Py, PyObjectRef, PyPayload, PyRef, PyResult, VirtualMachine, builtins::{PyModule, PyStrRef}, function::OptionalArg, }; - - use icu_normalizer::{ - ComposingNormalizerBorrowed, DecomposingNormalizerBorrowed, - properties::{CanonicalDecomposition, Decomposed}, - }; - use icu_properties::props::{ - BidiClass, BidiMirrored, BinaryProperty, CanonicalCombiningClass, EastAsianWidth, - EnumeratedProperty, GeneralCategory, NamedEnumeratedProperty, NumericType, - }; use itertools::Itertools; use rustpython_common::wtf8::{CodePoint, Wtf8Buf}; @@ -190,12 +71,14 @@ mod unicodedata { #[pyclass(name = "UCD")] #[derive(Debug, PyPayload)] pub(super) struct Ucd { - modern: bool, + inner: unicode_core::Ucd, } impl Ucd { pub(super) const fn new(modern: bool) -> Self { - Self { modern } + Self { + inner: unicode_core::Ucd::new(modern), + } } fn extract_char(&self, character: PyStrRef, vm: &VirtualMachine) -> PyResult { @@ -211,26 +94,14 @@ mod unicodedata { impl Ucd { #[pymethod] fn category(&self, character: PyStrRef, vm: &VirtualMachine) -> PyResult<&'static str> { - self.extract_char(character, vm).map(|c| { - let Some(c) = c.to_char() else { - return GeneralCategory::Surrogate.short_name(); - }; - if self.modern { - Some(GeneralCategory::for_char(c)) - } else { - cold_path(); - lookup_property(GENERAL_CATEGORY, c) - } - .unwrap_or(GeneralCategory::Unassigned) - .short_name() - }) + self.extract_char(character, vm) + .map(|c| self.inner.category(c)) } - // TODO: Names needs to account for Unicode 3.2.0 and 16.0.0 #[pymethod] fn lookup(&self, name: PyStrRef, vm: &VirtualMachine) -> PyResult { if let Some(name_str) = name.to_str() - && let Some(character) = unicode_names2::character(name_str) + && let Some(character) = unicode_core::lookup_character(name_str) { return Ok(character.to_string()); } @@ -241,7 +112,6 @@ mod unicodedata { )) } - // TODO: Names needs to account for Unicode 3.2.0 and 16.0.0 #[pymethod] fn name( &self, @@ -252,9 +122,9 @@ mod unicodedata { if let Some(name) = self .extract_char(character, vm)? .to_char() - .and_then(unicode_names2::name) + .and_then(unicode_core::character_name) { - return Ok(vm.ctx.new_str(name.to_string()).into()); + return Ok(vm.ctx.new_str(name).into()); } default.ok_or_else(|| vm.new_value_error("no such name")) } @@ -265,19 +135,8 @@ mod unicodedata { character: PyStrRef, vm: &VirtualMachine, ) -> PyResult<&'static str> { - self.extract_char(character, vm).map(|c| { - c.to_char() - .and_then(|c| { - if self.modern { - Some(BidiClass::for_char(c)) - } else { - cold_path(); - lookup_property(BIDI_CLASS, c) - } - }) - .unwrap_or(BidiClass::LeftToRight) - .short_name() - }) + self.extract_char(character, vm) + .map(|c| self.inner.bidirectional(c)) } #[pymethod] @@ -286,180 +145,36 @@ mod unicodedata { character: PyStrRef, vm: &VirtualMachine, ) -> PyResult<&'static str> { - self.extract_char(character, vm).map(|c| { - c.to_char() - .and_then(|c| { - if self.modern { - Some(EastAsianWidth::for_char(c)) - } else { - cold_path(); - // CPython overrides characters in the PUA for 3.2.0. - // Basic Multilingual Plane: - // https://en.wikipedia.org/wiki/Plane_(Unicode)#Basic_Multilingual_Plane - // https://en.wikipedia.org/wiki/Private_Use_Areas - // https://www.unicode.org/reports/tr11/tr11-10.html - // https://www.unicode.org/reports/tr11/ - // - // Currently, this implementation is incomplete because I can't figure - // out what CPython is doing. - lookup_property(EAST_ASIAN_WIDTH, c) - } - }) - .unwrap_or(EastAsianWidth::Neutral) - .short_name() - }) + self.extract_char(character, vm) + .map(|c| self.inner.east_asian_width(c)) } #[pymethod] - fn normalize(&self, form: super::NormalizeForm, unistr: PyStrRef) -> Wtf8Buf { - let text = unistr.as_wtf8(); - match form { - NormalizeForm::Nfc => { - let normalizer = ComposingNormalizerBorrowed::new_nfc(); - text.map_utf8(|s| normalizer.normalize_iter(s.chars())) - .collect() - } - NormalizeForm::Nfkc => { - let normalizer = ComposingNormalizerBorrowed::new_nfkc(); - text.map_utf8(|s| normalizer.normalize_iter(s.chars())) - .collect() - } - NormalizeForm::Nfd => { - let normalizer = DecomposingNormalizerBorrowed::new_nfd(); - text.map_utf8(|s| normalizer.normalize_iter(s.chars())) - .collect() - } - NormalizeForm::Nfkd => { - let normalizer = DecomposingNormalizerBorrowed::new_nfkd(); - text.map_utf8(|s| normalizer.normalize_iter(s.chars())) - .collect() - } - } + fn normalize(&self, form: NormalizeFormArg, unistr: PyStrRef) -> Wtf8Buf { + unicode_core::normalize(form.0, unistr.as_wtf8()) } #[pymethod] - fn is_normalized(&self, form: super::NormalizeForm, unistr: PyStrRef) -> bool { - match form { - NormalizeForm::Nfc => { - ComposingNormalizerBorrowed::new_nfc().is_normalized_utf8(unistr.as_bytes()) - } - NormalizeForm::Nfkc => { - ComposingNormalizerBorrowed::new_nfkc().is_normalized_utf8(unistr.as_bytes()) - } - NormalizeForm::Nfd => { - DecomposingNormalizerBorrowed::new_nfd().is_normalized_utf8(unistr.as_bytes()) - } - NormalizeForm::Nfkd => { - DecomposingNormalizerBorrowed::new_nfkd().is_normalized_utf8(unistr.as_bytes()) - } - } + fn is_normalized(&self, form: NormalizeFormArg, unistr: PyStrRef) -> bool { + unicode_core::is_normalized(form.0, unistr.as_bytes()) } #[pymethod] fn mirrored(&self, character: PyStrRef, vm: &VirtualMachine) -> PyResult { - self.extract_char(character, vm).map(|c| { - c.to_char().map_or(0, |c| { - (if self.modern { - BidiMirrored::for_char(c) - } else { - cold_path(); - let c = c as u32; - BIDI_MIRRORED - .binary_search_by(|&(start, end)| { - if c > end { - Ordering::Less - } else if c < start { - Ordering::Greater - } else { - Ordering::Equal - } - }) - .is_ok() - }) as i32 - }) - }) + self.extract_char(character, vm) + .map(|c| self.inner.mirrored(c)) } #[pymethod] fn combining(&self, character: PyStrRef, vm: &VirtualMachine) -> PyResult { - self.extract_char(character, vm).map(|c| { - c.to_char() - .and_then(|c| { - if self.modern { - Some(CanonicalCombiningClass::for_char(c)) - } else { - cold_path(); - lookup_property(COMBINING_CLASS, c) - } - }) - .unwrap_or(CanonicalCombiningClass::NotReordered) - .to_icu4c_value() - }) + self.extract_char(character, vm) + .map(|c| self.inner.combining(c)) } #[pymethod] fn decomposition(&self, character: PyStrRef, vm: &VirtualMachine) -> PyResult { - let Some(ch) = self.extract_char(character, vm).map(CodePoint::to_char)? else { - return Ok(String::new()); - }; - - // Decomposition is remarkable stable according to the normalization file, - // so the updates slice is very small - only about four char pairs. Linearly searching - // it is very fast. The file lists the original, incorrect decomp and the fixed char. - // For 3.2.0, we use the original decomp for compatibility while ignoring the update. - // - // Finally, we don't have to do anything for the latest UCD as it's already updated. - if self.modern - && let Some((_, original)) = DECOMP_UPDATES - .iter() - .find(|&&(codep, _original)| codep == ch as u32) - { - Ok(format!("{original:04X}")) - } else if let Ok(i) = - DECOMP_COMPAT.binary_search_by_key(&(ch as u32), |&(codep, _, _)| codep) - { - // Compatibility decomposition - // `icu4x` doesn't expose a non-recursive, compatibility decomposer so we - // have to do it manually for now. - let tag = DECOMP_COMPAT[i].1.type_tag(); - let end = DECOMP_COMPAT[i].2; - let start = i - .checked_sub(1) - .map(|i| DECOMP_COMPAT[i].2) - .unwrap_or_default(); - - let decomp = &DECOMP_RANGE[start..end]; - let cap = decomp.len() * 10 + decomp.len() + tag.len() + 1; - let mut out = String::with_capacity(cap); - - write!(out, "<{tag}>").unwrap(); - for ch in decomp { - write!(out, " {ch:04X}").unwrap(); - } - - Ok(out) - } else { - // Canonical decomposition - let decomposed = CanonicalDecomposition::new().decompose(ch); - match decomposed { - Decomposed::Default => Ok(String::new()), - Decomposed::Singleton(ch) => Ok(format!("{:04X}", ch as u32)), - Decomposed::Expansion(l, r) => Ok(format!("{:04X} {:04X}", l as u32, r as u32)), - } - } - } - - fn numeric_type_matches(&self, ch: CodePoint, expected: &[NumericType]) -> Option { - let ch = ch.to_char()?; - - let actual = if self.modern { - NumericType::for_char(ch) - } else { - cold_path(); - lookup_property(NUMERIC_TYPE_DIFF, ch).unwrap_or_else(|| NumericType::for_char(ch)) - }; - - expected.contains(&actual).then_some(ch) + self.extract_char(character, vm) + .map(|c| self.inner.decomposition(c)) } #[pymethod] @@ -470,12 +185,9 @@ mod unicodedata { vm: &VirtualMachine, ) -> PyResult> { let ch = self.extract_char(character, vm)?; - let expected = [NumericType::Decimal, NumericType::Digit]; - self.numeric_type_matches(ch, &expected) - .and_then(|ch| { - let value = lookup_numeric_val(ch, true)?; - (value.trunc() == value).then(|| vm.ctx.new_int(value as u64).into()) - }) + self.inner + .digit(ch) + .map(|value| vm.ctx.new_int(value).into()) .or_else(|| default.present()) .map(Option::Some) .ok_or_else(|| vm.new_value_error("not a digit")) @@ -489,12 +201,9 @@ mod unicodedata { vm: &VirtualMachine, ) -> PyResult> { let ch = self.extract_char(character, vm)?; - let expected = [NumericType::Decimal]; - self.numeric_type_matches(ch, &expected) - .and_then(|ch| { - let value = lookup_numeric_val(ch, self.modern)?; - (value.trunc() == value).then(|| vm.ctx.new_int(value as u64).into()) - }) + self.inner + .decimal(ch) + .map(|value| vm.ctx.new_int(value).into()) .or_else(|| default.present()) .map(Option::Some) .ok_or_else(|| vm.new_value_error("not a decimal")) @@ -508,11 +217,9 @@ mod unicodedata { vm: &VirtualMachine, ) -> PyResult> { let ch = self.extract_char(character, vm)?; - let expected = &NumericType::ALL_VALUES[1..]; - self.numeric_type_matches(ch, expected) - .and_then(|ch| { - lookup_numeric_val(ch, self.modern).map(|value| vm.ctx.new_float(value).into()) - }) + self.inner + .numeric(ch) + .map(|value| vm.ctx.new_float(value).into()) .or_else(|| default.present()) .map(Option::Some) .ok_or_else(|| vm.new_value_error("not a numeric character")) @@ -520,16 +227,7 @@ mod unicodedata { #[pygetset] fn unidata_version(&self) -> String { - if self.modern { - format!( - "{}.{}.{}", - char::UNICODE_VERSION.0, - char::UNICODE_VERSION.1, - char::UNICODE_VERSION.2 - ) - } else { - "3.2.0".into() - } + self.inner.unidata_version() } } @@ -540,11 +238,6 @@ mod unicodedata { #[pyattr] fn unidata_version(_vm: &VirtualMachine) -> String { - format!( - "{}.{}.{}", - char::UNICODE_VERSION.0, - char::UNICODE_VERSION.1, - char::UNICODE_VERSION.2 - ) + unicode_core::unicode_version() } } diff --git a/crates/unicode/Cargo.toml b/crates/unicode/Cargo.toml new file mode 100644 index 00000000000..05f114ee3b7 --- /dev/null +++ b/crates/unicode/Cargo.toml @@ -0,0 +1,22 @@ +[package] +name = "rustpython-unicode" +description = "Runtime-independent CPython-compatible Unicode semantics and data for RustPython and related Python tooling." +edition = { workspace = true } +version = { workspace = true } +authors = { workspace = true } +repository = { workspace = true } +license = { workspace = true } +rust-version = { workspace = true } + +[dependencies] +rustpython-wtf8 = { workspace = true } + +icu_properties = { workspace = true } +icu_normalizer = { workspace = true } +unicode_names2 = { workspace = true } + +[build-dependencies] +icu_properties = { workspace = true } + +[lints] +workspace = true diff --git a/crates/unicode/build.rs b/crates/unicode/build.rs new file mode 100644 index 00000000000..3a82df85eb8 --- /dev/null +++ b/crates/unicode/build.rs @@ -0,0 +1,612 @@ +// spell-checker:ignore decomp DECOMP + +extern crate alloc; + +use core::num::NonZeroUsize; + +use alloc::collections::{BTreeMap, BTreeSet}; + +use std::{ + env, + fs::{self, File}, + io::{self, BufRead, BufReader, BufWriter, Write}, + path::{Path, PathBuf}, + thread, +}; + +use icu_properties::props::{EnumeratedProperty, GeneralCategory, NumericType}; + +fn generate_unicode_3_2() { + let path = PathBuf::from(env::var("OUT_DIR").unwrap()) + .join("generated") + .join("unicode_3_2.rs"); + fs::create_dir_all(path.parent().unwrap()).unwrap(); + let mut writer = BufWriter::new(File::create(&path).unwrap()); + + let base = Path::new(env!("CARGO_MANIFEST_DIR")) + .join("unicode") + .join("ucd32"); + + write_derived( + &base, + "DerivedGeneralCategory-3.2.0.txt", + "GENERAL_CATEGORY", + "(u32, u32, GeneralCategory)", + NonZeroUsize::new(1).unwrap(), + &mut writer, + |start, end, id, _| { + let id = parse_general(id); + if id != GeneralCategory::Unassigned { + Some((start, end, id)) + } else { + None + } + }, + |writer, mut values| { + values.sort_unstable_by_key(|(start, _, _)| *start); + write!(writer, "[").unwrap(); + for (start, end, id) in values { + write!(writer, "({start}, {end}, GeneralCategory::{id:?}),").unwrap(); + } + write!(writer, "];").unwrap(); + }, + ); + + write_derived( + &base, + "DerivedEastAsianWidth-3.2.0.txt", + "EAST_ASIAN_WIDTH", + "(u32, u32, EastAsianWidth)", + NonZeroUsize::new(1).unwrap(), + &mut writer, + |start, end, id, _| { + let id = parse_eaw(id); + if id != "EastAsianWidth::Neutral" { + Some((start, end, id)) + } else { + None + } + }, + |writer, mut values| { + values.sort_unstable_by_key(|(start, _, _)| *start); + write!(writer, "[").unwrap(); + for (start, end, id) in values { + write!(writer, "({start}, {end}, {id}),").unwrap(); + } + write!(writer, "];").unwrap(); + }, + ); + + write_derived( + &base, + "DerivedBidiClass-3.2.0.txt", + "BIDI_CLASS", + "(u32, u32, BidiClass)", + NonZeroUsize::new(1).unwrap(), + &mut writer, + |start, end, id, _| { + let id = parse_bidi(id); + if id != "BidiClass::LeftToRight" { + Some((start, end, id)) + } else { + None + } + }, + |writer, mut values| { + values.sort_unstable_by_key(|(start, _, _)| *start); + write!(writer, "[").unwrap(); + for (start, end, id) in values { + write!(writer, "({start}, {end}, {id}),").unwrap(); + } + write!(writer, "];").unwrap(); + }, + ); + + write_derived( + &base, + "DerivedBinaryProperties-3.2.0.txt", + "BIDI_MIRRORED", + "(u32, u32)", + NonZeroUsize::new(1).unwrap(), + &mut writer, + |start, end, id, _| { + assert_eq!( + "Bidi_Mirrored", + id.trim(), + "DerivedBinaryProperties-3.2.0 only has Bidi_Mirrored" + ); + Some((start, end)) + }, + |writer, mut values| { + values.sort_unstable_by_key(|(start, _)| *start); + writeln!(writer, "{values:?};").unwrap(); + }, + ); + + write_derived( + &base, + "DerivedCombiningClass-3.2.0.txt", + "COMBINING_CLASS", + "(u32, u32, CanonicalCombiningClass)", + NonZeroUsize::new(1).unwrap(), + &mut writer, + |start, end, id, _| { + let id: u8 = id.parse().unwrap(); + if id == 0 { + return None; + } + Some((start, end, id)) + }, + |writer, mut values| { + values.sort_unstable_by_key(|(start, _, _)| *start); + write!(writer, "[").unwrap(); + for (start, end, id) in values { + write!( + writer, + "({start}, {end}, CanonicalCombiningClass::from_icu4c_value({id}))," + ) + .unwrap(); + } + writeln!(writer, "];").unwrap(); + }, + ); +} + +fn generate_numeric_type() { + let path = PathBuf::from(env::var("OUT_DIR").unwrap()) + .join("generated") + .join("unicode_num_type.rs"); + fs::create_dir_all(path.parent().unwrap()).unwrap(); + let mut writer = BufWriter::new(File::create(&path).unwrap()); + + let base = Path::new(env!("CARGO_MANIFEST_DIR")) + .join("unicode") + .join("ucd32"); + + write_derived( + &base, + "DerivedNumericType-3.2.0.txt", + "NUMERIC_TYPE_DIFF", + "(u32, u32, NumericType)", + NonZeroUsize::new(1).unwrap(), + &mut writer, + |start, end, id, _| { + let id = parse_numeric_type_str(id); + let differs = (start..=end).any(|c| match char::from_u32(c) { + Some(c) => { + let modern = parse_numeric_type_val(NumericType::for_char(c)); + modern != id + } + None => true, + }); + + if differs { + Some((start, end, id)) + } else { + None + } + }, + |writer, mut values| { + values.sort_unstable_by_key(|(start, _, _)| *start); + write!(writer, "[").unwrap(); + for (start, end, id) in values { + write!(writer, "({start}, {end}, {id}),").unwrap(); + } + writeln!(writer, "];").unwrap(); + }, + ); +} + +fn generate_numeric_value() { + let path = PathBuf::from(env::var("OUT_DIR").unwrap()) + .join("generated") + .join("unicode_numeric_value.rs"); + fs::create_dir_all(path.parent().unwrap()).unwrap(); + let mut writer = BufWriter::new(File::create(&path).unwrap()); + + // Ideally, this would store the diffs between the two tables. However, we need 3.2.0 + // membership as well as different chars. The final tables are both smaller than storing the + // full 3.2.0 value table. + let ucd32 = Path::new(env!("CARGO_MANIFEST_DIR")) + .join("unicode") + .join("ucd32"); + let mut ucd32_diffs = BTreeMap::new(); + let mut ucd32_member = BTreeSet::new(); + let numeric_32 = + BufReader::new(File::open(ucd32.join("DerivedNumericValues-3.2.0.txt")).unwrap()); + parse_unicode_3_2( + numeric_32, + NonZeroUsize::new(1).unwrap(), + &mut io::empty(), + |start, end, value, _| { + let value: f64 = value + .parse() + .expect("Unicode data contains valid properties"); + ucd32_diffs.insert((start, end), value); + ucd32_member.insert((start, end)); + Option::<()>::None + }, + |_writer, _values| {}, + ); + + let ucd_latest = Path::new(env!("CARGO_MANIFEST_DIR")) + .join("unicode") + .join("latest"); + + write_derived( + &ucd_latest, + "DerivedNumericValues.txt", + "NUMERIC_VALUES", + "(u32, u32, f64)", + NonZeroUsize::new(1).unwrap(), + &mut writer, + |start, end, value, _| { + let value: f64 = value + .parse() + .expect("Unicode data contains valid properties"); + + if ucd32_diffs + .get(&(start, end)) + .is_some_and(|old_v| *old_v == value) + { + ucd32_diffs.remove(&(start, end)); + } + + Some((start, end, value)) + }, + |writer, mut values| { + values.sort_unstable_by_key(|(ch, _, _)| *ch); + writeln!(writer, "{values:?};").unwrap(); + }, + ); + + // TODO: More flexible parser + writeln!( + writer, + "static NUMERIC_VALUES_DIFF: &[(u32, u32, f64)] = &[" + ) + .unwrap(); + for ((start, end), value) in ucd32_diffs { + write!(writer, "({start}, {end}, {value:?}),").unwrap(); + } + writeln!(writer, "];").unwrap(); + + // Compress membership table + let mut iter = ucd32_member.iter(); + let &(mut start_prev, mut end_prev) = iter.next().unwrap(); + let mut membership = Vec::new(); + + for &(start, end) in iter { + if start <= end_prev + 1 { + end_prev = end_prev.max(end); + } else { + membership.push((start_prev, end_prev)); + start_prev = start; + end_prev = end; + } + } + membership.push((start_prev, end_prev)); + membership.sort_unstable_by_key(|&(start, _)| start); + + writeln!(writer, "static NUMERIC_VAL_EXISTS_32: &[(u32, u32)] = &").unwrap(); + write!(writer, "{membership:?};").unwrap(); +} + +fn generate_unicode_latest() { + let path = PathBuf::from(env::var("OUT_DIR").unwrap()) + .join("generated") + .join("unicode_latest.rs"); + fs::create_dir_all(path.parent().unwrap()).unwrap(); + let mut writer = BufWriter::new(File::create(&path).unwrap()); + + let base = Path::new(env!("CARGO_MANIFEST_DIR")) + .join("unicode") + .join("latest"); + + // NOTE: + // This ONLY parses compatibility decomposition because Python exposes the tags. The tags are + // the "", "", et cetera bits before the decomposition. Thus, we can save space + // by using icu4x's CanonicalDecomposer for non-compatibility decomposition. + let mut decomp_ranges = Vec::new(); + write_derived( + &base, + "UnicodeData.txt", + "DECOMP_COMPAT", + "(u32, DecompositionType, usize)", + NonZeroUsize::new(5).unwrap(), + &mut writer, + |start, _end, value, _| { + // We're building a sparse array. Most characters don't decompose, so we don't + // need to literally store a row for each char. + if value.is_empty() { + return None; + } + + let (dtype, decomp) = value.split_once('>').map(|(dtype, decomp)| { + let dtype = dtype.strip_prefix('<').unwrap_or_else(|| { + panic!("Compatibility decomp; expected \n\tgot: {value}") + }); + ( + parse_decomp_type(dtype), + decomp + .split_whitespace() + .map(|s| u32::from_str_radix(s, 16).unwrap()), + ) + })?; + + decomp_ranges.extend(decomp); + let end = decomp_ranges.len(); + + Some((start, dtype, end)) + }, + |writer, values| { + // UnicodeData.txt should already be sorted + write!(writer, "[").unwrap(); + for (start, dtype, end) in values { + write!(writer, "({start}, DecompositionType::{dtype:?}, {end}),").unwrap(); + } + writeln!(writer, "];").unwrap(); + }, + ); + + writeln!(writer, "static DECOMP_RANGE: &[u32] = &{decomp_ranges:?};").unwrap(); + + // Normalization corrections is super small - only a handful chars at the time of writing. + write_derived( + &base, + "NormalizationCorrections.txt", + "DECOMP_UPDATES", + "(u32, u32)", + NonZeroUsize::new(1).unwrap(), + &mut writer, + |start, _end, value, line| { + let original = u32::from_str_radix(value.trim(), 16).unwrap_or_else(|e| { + panic!("field 2 of decomp corrections should be a char in hex: {value} {e}") + }); + let version = line + .rsplit(';') + .next() + .unwrap_or_else(|| { + panic!("field 4 of decomp corrections should be a UCD version: {line}") + }) + .split_once('#') + .unwrap() + .0 + .trim(); + + // `version` = when the char was updated. Therefore, we use the incorrect chars past + // 3.2.0 but skip the chars fixed in 3.2.0 because they'll already be right. + if version != "3.2.0" { + Some((start, original)) + } else { + None + } + }, + |writer, mut values| { + values.sort_unstable_by_key(|(c, _)| *c); + write!(writer, "{values:?};").unwrap(); + }, + ); +} + +#[expect(clippy::too_many_arguments)] +fn write_derived( + base: &Path, + file_name: &str, + static_name: &str, + array_type: &str, + field: NonZeroUsize, + writer: &mut W, + parse: P, + write_vec: FW, +) where + W: Write, + P: FnMut(u32, u32, &str, &str) -> Option, + FW: FnMut(&mut W, Vec), +{ + let path = base.join(file_name); + let reader = BufReader::new(File::open(path).unwrap()); + writeln!(writer, "static {static_name}: &[{array_type}] = &").unwrap(); + parse_unicode_3_2(reader, field, writer, parse, write_vec); +} + +/// Parse Unicode 3.2.0 property files. +fn parse_unicode_3_2( + reader: impl BufRead, + field: NonZeroUsize, + writer: &mut W, + mut parse: P, + mut write_vec: FW, +) where + W: Write, + P: FnMut(u32, u32, &str, &str) -> Option, + FW: FnMut(&mut W, Vec), +{ + let mut parsed = Vec::new(); + + for line in reader.lines().map(Result::unwrap) { + let line = line.trim(); + if line.is_empty() || line.starts_with('#') { + continue; + } + + let mut fields = line.split(';'); + let range = fields.next().expect("Unicode data is missing a char range"); + let id = fields + .nth(field.get().saturating_sub(1)) + .expect("Unicode data is missing a property"); + let (start, end) = match range.split_once("..") { + Some((left, right)) => { + let start = u32::from_str_radix(left.trim(), 16).unwrap(); + let end = u32::from_str_radix(right.trim(), 16).unwrap(); + (start, end) + } + None => { + let start = u32::from_str_radix(range.trim(), 16).unwrap(); + (start, start) + } + }; + + let id = id.split_once('#').map_or(id, |(left, _)| left).trim(); + if let Some(val) = parse(start, end, id, line) { + parsed.push(val); + } + } + write_vec(writer, parsed); +} + +fn parse_general(id: &str) -> GeneralCategory { + match id.trim() { + "Cn" => GeneralCategory::Unassigned, + "Lu" => GeneralCategory::UppercaseLetter, + "Ll" => GeneralCategory::LowercaseLetter, + "Lt" => GeneralCategory::TitlecaseLetter, + "Lm" => GeneralCategory::ModifierLetter, + "Lo" => GeneralCategory::OtherLetter, + "Mn" => GeneralCategory::NonspacingMark, + "Mc" => GeneralCategory::SpacingMark, + "Me" => GeneralCategory::EnclosingMark, + "Nd" => GeneralCategory::DecimalNumber, + "Nl" => GeneralCategory::LetterNumber, + "No" => GeneralCategory::OtherNumber, + "Zs" => GeneralCategory::SpaceSeparator, + "Zl" => GeneralCategory::LineSeparator, + "Zp" => GeneralCategory::ParagraphSeparator, + "Cc" => GeneralCategory::Control, + "Cf" => GeneralCategory::Format, + "Co" => GeneralCategory::PrivateUse, + "Cs" => GeneralCategory::Surrogate, + "Pd" => GeneralCategory::DashPunctuation, + "Ps" => GeneralCategory::OpenPunctuation, + "Pe" => GeneralCategory::ClosePunctuation, + "Pc" => GeneralCategory::ConnectorPunctuation, + "Pi" => GeneralCategory::InitialPunctuation, + "Pf" => GeneralCategory::FinalPunctuation, + "Po" => GeneralCategory::OtherPunctuation, + "Sm" => GeneralCategory::MathSymbol, + "Sc" => GeneralCategory::CurrencySymbol, + "Sk" => GeneralCategory::ModifierSymbol, + "So" => GeneralCategory::OtherSymbol, + invalid => unreachable!("Unicode data contains valid properties: {invalid}"), + } +} + +fn parse_eaw(id: &str) -> &'static str { + match id.trim() { + "N" => "EastAsianWidth::Neutral", + "A" => "EastAsianWidth::Ambiguous", + "H" => "EastAsianWidth::Halfwidth", + "F" => "EastAsianWidth::Fullwidth", + "Na" => "EastAsianWidth::Narrow", + "W" => "EastAsianWidth::Wide", + invalid => unreachable!("Unicode data contains valid properties: {invalid}"), + } +} + +fn parse_bidi(id: &str) -> &'static str { + match id.trim() { + "L" => "BidiClass::LeftToRight", + "R" => "BidiClass::RightToLeft", + "EN" => "BidiClass::EuropeanNumber", + "ES" => "BidiClass::EuropeanSeparator", + "ET" => "BidiClass::EuropeanTerminator", + "AN" => "BidiClass::ArabicNumber", + "CS" => "BidiClass::CommonSeparator", + "B" => "BidiClass::ParagraphSeparator", + "S" => "BidiClass::SegmentSeparator", + "WS" => "BidiClass::WhiteSpace", + "ON" => "BidiClass::OtherNeutral", + "LRE" => "BidiClass::LeftToRightEmbedding", + "LRO" => "BidiClass::LeftToRightOverride", + "AL" => "BidiClass::ArabicLetter", + "RLE" => "BidiClass::RightToLeftEmbedding", + "RLO" => "BidiClass::RightToLeftOverride", + "PDF" => "BidiClass::PopDirectionalFormat", + "NSM" => "BidiClass::NonspacingMark", + "BN" => "BidiClass::BoundaryNeutral", + "FSI" => "BidiClass::FirstStrongIsolate", + "LRI" => "BidiClass::LeftToRightIsolate", + "RLI" => "BidiClass::RightToLeftIsolate", + "PDI" => "BidiClass::PopDirectionalIsolate", + invalid => unreachable!("Unicode data contains valid properties: {invalid}"), + } +} + +fn parse_numeric_type_val(val: NumericType) -> &'static str { + match val { + NumericType::None => "none", + NumericType::Decimal => "decimal", + NumericType::Digit => "digit", + NumericType::Numeric => "numeric", + _ => unreachable!("Unicode data contains valid properties"), + } +} + +fn parse_numeric_type_str(id: &str) -> &'static str { + match id { + "none" => "NumericType::None", + "decimal" => "NumericType::Decimal", + "digit" => "NumericType::Digit", + "numeric" => "NumericType::Numeric", + invalid => unreachable!("Unicode data contains valid properties: {invalid}"), + } +} + +#[derive(Debug, Default)] +enum DecompositionType { + #[default] + Canonical, + Compat, + Circle, + Final, + Font, + Fraction, + Initial, + Isolated, + Medial, + Narrow, + Nobreak, + Small, + Square, + Sub, + Super, + Vertical, + Wide, +} + +fn parse_decomp_type(id: &str) -> DecompositionType { + match id { + "canonical" => DecompositionType::Canonical, + "compat" => DecompositionType::Compat, + "circle" => DecompositionType::Circle, + "final" => DecompositionType::Final, + "font" => DecompositionType::Font, + "fraction" => DecompositionType::Fraction, + "initial" => DecompositionType::Initial, + "isolated" => DecompositionType::Isolated, + "medial" => DecompositionType::Medial, + "narrow" => DecompositionType::Narrow, + "noBreak" => DecompositionType::Nobreak, + "small" => DecompositionType::Small, + "square" => DecompositionType::Square, + "sub" => DecompositionType::Sub, + "super" => DecompositionType::Super, + "vertical" => DecompositionType::Vertical, + "wide" => DecompositionType::Wide, + invalid => unreachable!("Unicode data contains valid properties: {invalid}"), + } +} + +fn main() { + println!("cargo:rerun-if-changed=unicode/ucd32"); + println!("cargo:rerun-if-changed=unicode/latest"); + + let t_32 = thread::spawn(generate_unicode_3_2); + let t_numeric_type = thread::spawn(generate_numeric_type); + let t_numeric_value = thread::spawn(generate_numeric_value); + let t_latest = thread::spawn(generate_unicode_latest); + t_32.join().unwrap(); + t_numeric_type.join().unwrap(); + t_numeric_value.join().unwrap(); + t_latest.join().unwrap(); +} diff --git a/crates/unicode/src/data.rs b/crates/unicode/src/data.rs new file mode 100644 index 00000000000..e98c7d8e7cb --- /dev/null +++ b/crates/unicode/src/data.rs @@ -0,0 +1,343 @@ +//! Access to the Unicode character database (`unicodedata`). +//! +//! Owns the generated Unicode 3.2.0 / latest tables and the +//! `icu4x`/`unicode_names2` lookups behind them. + +// spell-checker:ignore codep decomp DECOMP unidata + +use core::{cmp::Ordering, fmt::Write, hint::cold_path}; + +use icu_normalizer::properties::{CanonicalDecomposition, Decomposed}; +use icu_properties::props::{ + BidiClass, BidiMirrored, BinaryProperty, CanonicalCombiningClass, EastAsianWidth, + EnumeratedProperty, GeneralCategory, NamedEnumeratedProperty, NumericType, +}; +use rustpython_wtf8::CodePoint; + +include!(concat!(env!("OUT_DIR"), "/generated/unicode_3_2.rs")); +include!(concat!(env!("OUT_DIR"), "/generated/unicode_latest.rs")); +include!(concat!(env!("OUT_DIR"), "/generated/unicode_num_type.rs")); +include!(concat!( + env!("OUT_DIR"), + "/generated/unicode_numeric_value.rs" +)); + +#[derive(Clone, Copy)] +#[repr(u8)] +enum DecompositionType { + #[allow(unused)] + Canonical, + Compat, + Circle, + Final, + Font, + Fraction, + Initial, + Isolated, + Medial, + Narrow, + Nobreak, + Small, + Square, + Sub, + Super, + Vertical, + Wide, +} + +impl DecompositionType { + const fn type_tag(self) -> &'static str { + match self { + Self::Canonical => "canonical", + Self::Compat => "compat", + Self::Circle => "circle", + Self::Final => "final", + Self::Font => "font", + Self::Fraction => "fraction", + Self::Initial => "initial", + Self::Isolated => "isolated", + Self::Medial => "medial", + Self::Narrow => "narrow", + Self::Nobreak => "noBreak", + Self::Small => "small", + Self::Square => "square", + Self::Sub => "sub", + Self::Super => "super", + Self::Vertical => "vertical", + Self::Wide => "wide", + } + } +} + +fn lookup_property(table: &[(u32, u32, T)], ch: char) -> Option { + let ch = ch as u32; + table + .binary_search_by(|&(start, end, _)| { + if ch > end { + Ordering::Less + } else if ch < start { + Ordering::Greater + } else { + Ordering::Equal + } + }) + .ok() + .map(|i| table[i].2) +} + +fn lookup_numeric_val(ch: char, modern: bool) -> Option { + if modern { + lookup_property(NUMERIC_VALUES, ch) + } else { + cold_path(); + lookup_property(NUMERIC_VALUES_DIFF, ch).or_else(|| { + NUMERIC_VAL_EXISTS_32 + .binary_search_by(|&(start, end)| { + let ch = ch as u32; + if ch > end { + Ordering::Less + } else if ch < start { + Ordering::Greater + } else { + Ordering::Equal + } + }) + .ok() + .and_then(|_| lookup_property(NUMERIC_VALUES, ch)) + }) + } +} + +/// The version string of the latest Unicode database bundled with the standard +/// library (`unicodedata.unidata_version`). +#[must_use] +pub fn unicode_version() -> String { + format!( + "{}.{}.{}", + char::UNICODE_VERSION.0, + char::UNICODE_VERSION.1, + char::UNICODE_VERSION.2 + ) +} + +/// Look up a character by its Unicode name (`unicodedata.lookup`). +#[must_use] +pub fn lookup_character(name: &str) -> Option { + unicode_names2::character(name) +} + +/// The Unicode name of `ch` (`unicodedata.name`), if any. +#[must_use] +pub fn character_name(ch: char) -> Option { + unicode_names2::name(ch).map(|name| name.to_string()) +} + +/// A view over the Unicode character database at a fixed version. +/// +/// `modern` selects the latest bundled UCD; otherwise the Unicode 3.2.0 tables +/// used by `unicodedata.ucd_3_2_0` are consulted. +#[derive(Debug, Clone, Copy)] +pub struct Ucd { + modern: bool, +} + +impl Ucd { + #[must_use] + pub const fn new(modern: bool) -> Self { + Self { modern } + } + + #[must_use] + pub fn category(&self, c: CodePoint) -> &'static str { + let Some(c) = c.to_char() else { + return GeneralCategory::Surrogate.short_name(); + }; + if self.modern { + Some(GeneralCategory::for_char(c)) + } else { + cold_path(); + lookup_property(GENERAL_CATEGORY, c) + } + .unwrap_or(GeneralCategory::Unassigned) + .short_name() + } + + #[must_use] + pub fn bidirectional(&self, c: CodePoint) -> &'static str { + c.to_char() + .and_then(|c| { + if self.modern { + Some(BidiClass::for_char(c)) + } else { + cold_path(); + lookup_property(BIDI_CLASS, c) + } + }) + .unwrap_or(BidiClass::LeftToRight) + .short_name() + } + + #[must_use] + pub fn east_asian_width(&self, c: CodePoint) -> &'static str { + c.to_char() + .and_then(|c| { + if self.modern { + Some(EastAsianWidth::for_char(c)) + } else { + cold_path(); + // CPython overrides characters in the PUA for 3.2.0. + // Basic Multilingual Plane: + // https://en.wikipedia.org/wiki/Plane_(Unicode)#Basic_Multilingual_Plane + // https://en.wikipedia.org/wiki/Private_Use_Areas + // https://www.unicode.org/reports/tr11/tr11-10.html + // https://www.unicode.org/reports/tr11/ + // + // Currently, this implementation is incomplete because I can't figure + // out what CPython is doing. + lookup_property(EAST_ASIAN_WIDTH, c) + } + }) + .unwrap_or(EastAsianWidth::Neutral) + .short_name() + } + + #[must_use] + pub fn mirrored(&self, c: CodePoint) -> i32 { + c.to_char().map_or(0, |c| { + (if self.modern { + BidiMirrored::for_char(c) + } else { + cold_path(); + let c = c as u32; + BIDI_MIRRORED + .binary_search_by(|&(start, end)| { + if c > end { + Ordering::Less + } else if c < start { + Ordering::Greater + } else { + Ordering::Equal + } + }) + .is_ok() + }) as i32 + }) + } + + #[must_use] + pub fn combining(&self, c: CodePoint) -> u8 { + c.to_char() + .and_then(|c| { + if self.modern { + Some(CanonicalCombiningClass::for_char(c)) + } else { + cold_path(); + lookup_property(COMBINING_CLASS, c) + } + }) + .unwrap_or(CanonicalCombiningClass::NotReordered) + .to_icu4c_value() + } + + #[must_use] + pub fn decomposition(&self, c: CodePoint) -> String { + let Some(ch) = c.to_char() else { + return String::new(); + }; + + // Decomposition is remarkable stable according to the normalization file, + // so the updates slice is very small - only about four char pairs. Linearly searching + // it is very fast. The file lists the original, incorrect decomp and the fixed char. + // For 3.2.0, we use the original decomp for compatibility while ignoring the update. + // + // Finally, we don't have to do anything for the latest UCD as it's already updated. + if self.modern + && let Some((_, original)) = DECOMP_UPDATES + .iter() + .find(|&&(codep, _original)| codep == ch as u32) + { + format!("{original:04X}") + } else if let Ok(i) = + DECOMP_COMPAT.binary_search_by_key(&(ch as u32), |&(codep, _, _)| codep) + { + // Compatibility decomposition + // `icu4x` doesn't expose a non-recursive, compatibility decomposer so we + // have to do it manually for now. + let tag = DECOMP_COMPAT[i].1.type_tag(); + let end = DECOMP_COMPAT[i].2; + let start = i + .checked_sub(1) + .map(|i| DECOMP_COMPAT[i].2) + .unwrap_or_default(); + + let decomp = &DECOMP_RANGE[start..end]; + let cap = decomp.len() * 10 + decomp.len() + tag.len() + 1; + let mut out = String::with_capacity(cap); + + write!(out, "<{tag}>").unwrap(); + for ch in decomp { + write!(out, " {ch:04X}").unwrap(); + } + + out + } else { + // Canonical decomposition + let decomposed = CanonicalDecomposition::new().decompose(ch); + match decomposed { + Decomposed::Default => String::new(), + Decomposed::Singleton(ch) => format!("{:04X}", ch as u32), + Decomposed::Expansion(l, r) => format!("{:04X} {:04X}", l as u32, r as u32), + } + } + } + + fn numeric_type_matches(self, ch: CodePoint, expected: &[NumericType]) -> Option { + let ch = ch.to_char()?; + + let actual = if self.modern { + NumericType::for_char(ch) + } else { + cold_path(); + lookup_property(NUMERIC_TYPE_DIFF, ch).unwrap_or_else(|| NumericType::for_char(ch)) + }; + + expected.contains(&actual).then_some(ch) + } + + /// The integer digit value of `c` (`unicodedata.digit`), if it has one. + #[must_use] + pub fn digit(&self, c: CodePoint) -> Option { + let expected = [NumericType::Decimal, NumericType::Digit]; + self.numeric_type_matches(c, &expected).and_then(|ch| { + let value = lookup_numeric_val(ch, true)?; + (value.trunc() == value).then_some(value as u64) + }) + } + + /// The integer decimal value of `c` (`unicodedata.decimal`), if it has one. + #[must_use] + pub fn decimal(&self, c: CodePoint) -> Option { + let expected = [NumericType::Decimal]; + self.numeric_type_matches(c, &expected).and_then(|ch| { + let value = lookup_numeric_val(ch, self.modern)?; + (value.trunc() == value).then_some(value as u64) + }) + } + + /// The numeric value of `c` (`unicodedata.numeric`), if it has one. + #[must_use] + pub fn numeric(&self, c: CodePoint) -> Option { + let expected = &NumericType::ALL_VALUES[1..]; + self.numeric_type_matches(c, expected) + .and_then(|ch| lookup_numeric_val(ch, self.modern)) + } + + #[must_use] + pub fn unidata_version(&self) -> String { + if self.modern { + unicode_version() + } else { + "3.2.0".into() + } + } +} diff --git a/crates/unicode/src/lib.rs b/crates/unicode/src/lib.rs new file mode 100644 index 00000000000..093a31d1f8c --- /dev/null +++ b/crates/unicode/src/lib.rs @@ -0,0 +1,51 @@ +//! Runtime-independent CPython-compatible Unicode semantics and data. +//! +//! Every entry point operates on plain `char`/`u32`/`CodePoint`/`&Wtf8` values +//! so it can be shared by any Python runtime; argument extraction and Python +//! exception mapping stay with the caller. There is no global mutable state and +//! results depend only on inputs. + +pub mod data; +pub mod normalize; + +pub use data::{Ucd, character_name, lookup_character, unicode_version}; +pub use normalize::{NormalizeForm, is_normalized, normalize}; + +#[cfg(test)] +mod tests { + use rustpython_wtf8::{CodePoint, Wtf8Buf}; + + use crate::{NormalizeForm, Ucd, character_name, is_normalized, lookup_character, normalize}; + + fn cp(ch: char) -> CodePoint { + CodePoint::from(ch) + } + + #[test] + fn data_queries_match_unicodedata_behavior() { + let ucd = Ucd::new(true); + assert_eq!(ucd.category(cp('A')), "Lu"); + assert_eq!(ucd.category(CodePoint::from_u32(0xD800).unwrap()), "Cs"); + assert_eq!(lookup_character("SNOWMAN"), Some('☃')); + assert_eq!(character_name('☃').as_deref(), Some("SNOWMAN")); + assert_eq!(ucd.decimal(cp('५')), Some(5)); + assert_eq!(ucd.digit(cp('²')), Some(2)); + let third = ucd.numeric(cp('⅓')).unwrap(); + assert!((third - 1.0 / 3.0).abs() < 1e-6, "got {third}"); + } + + #[test] + fn ucd_3_2_0_view_differs_from_modern() { + let legacy = Ucd::new(false); + assert_eq!(legacy.unidata_version(), "3.2.0"); + } + + #[test] + fn normalization_round_trips() { + let composed = Wtf8Buf::from("é"); + let decomposed = normalize(NormalizeForm::Nfd, &composed); + assert_eq!(normalize(NormalizeForm::Nfc, &decomposed), composed); + assert!(is_normalized(NormalizeForm::Nfc, "é".as_bytes())); + assert!(!is_normalized(NormalizeForm::Nfd, "é".as_bytes())); + } +} diff --git a/crates/unicode/src/normalize.rs b/crates/unicode/src/normalize.rs new file mode 100644 index 00000000000..04a6194f419 --- /dev/null +++ b/crates/unicode/src/normalize.rs @@ -0,0 +1,73 @@ +//! Unicode normalization (`unicodedata.normalize` / `is_normalized`). + +// spell-checker:ignore nfkc + +use core::str::FromStr; + +use icu_normalizer::{ComposingNormalizerBorrowed, DecomposingNormalizerBorrowed}; +use rustpython_wtf8::{Wtf8, Wtf8Buf}; + +/// One of the four Unicode normalization forms. +#[derive(Debug, Clone, Copy, Eq, PartialEq)] +pub enum NormalizeForm { + Nfc, + Nfkc, + Nfd, + Nfkd, +} + +impl FromStr for NormalizeForm { + type Err = (); + + fn from_str(s: &str) -> Result { + match s { + "NFC" => Ok(Self::Nfc), + "NFKC" => Ok(Self::Nfkc), + "NFD" => Ok(Self::Nfd), + "NFKD" => Ok(Self::Nfkd), + _ => Err(()), + } + } +} + +/// Normalize `text` to `form` (`unicodedata.normalize`). +/// +/// Lone surrogates are passed through unchanged; only the valid UTF-8 runs are +/// normalized. +#[must_use] +pub fn normalize(form: NormalizeForm, text: &Wtf8) -> Wtf8Buf { + match form { + NormalizeForm::Nfc => { + let normalizer = ComposingNormalizerBorrowed::new_nfc(); + text.map_utf8(|s| normalizer.normalize_iter(s.chars())) + .collect() + } + NormalizeForm::Nfkc => { + let normalizer = ComposingNormalizerBorrowed::new_nfkc(); + text.map_utf8(|s| normalizer.normalize_iter(s.chars())) + .collect() + } + NormalizeForm::Nfd => { + let normalizer = DecomposingNormalizerBorrowed::new_nfd(); + text.map_utf8(|s| normalizer.normalize_iter(s.chars())) + .collect() + } + NormalizeForm::Nfkd => { + let normalizer = DecomposingNormalizerBorrowed::new_nfkd(); + text.map_utf8(|s| normalizer.normalize_iter(s.chars())) + .collect() + } + } +} + +/// Whether `bytes` (interpreted as UTF-8) is already in `form` +/// (`unicodedata.is_normalized`). +#[must_use] +pub fn is_normalized(form: NormalizeForm, bytes: &[u8]) -> bool { + match form { + NormalizeForm::Nfc => ComposingNormalizerBorrowed::new_nfc().is_normalized_utf8(bytes), + NormalizeForm::Nfkc => ComposingNormalizerBorrowed::new_nfkc().is_normalized_utf8(bytes), + NormalizeForm::Nfd => DecomposingNormalizerBorrowed::new_nfd().is_normalized_utf8(bytes), + NormalizeForm::Nfkd => DecomposingNormalizerBorrowed::new_nfkd().is_normalized_utf8(bytes), + } +} diff --git a/crates/stdlib/unicode/README.md b/crates/unicode/unicode/README.md similarity index 100% rename from crates/stdlib/unicode/README.md rename to crates/unicode/unicode/README.md diff --git a/crates/stdlib/unicode/latest/DerivedNumericValues.txt b/crates/unicode/unicode/latest/DerivedNumericValues.txt similarity index 100% rename from crates/stdlib/unicode/latest/DerivedNumericValues.txt rename to crates/unicode/unicode/latest/DerivedNumericValues.txt diff --git a/crates/stdlib/unicode/latest/NormalizationCorrections.txt b/crates/unicode/unicode/latest/NormalizationCorrections.txt similarity index 100% rename from crates/stdlib/unicode/latest/NormalizationCorrections.txt rename to crates/unicode/unicode/latest/NormalizationCorrections.txt diff --git a/crates/stdlib/unicode/latest/UnicodeData.txt b/crates/unicode/unicode/latest/UnicodeData.txt similarity index 100% rename from crates/stdlib/unicode/latest/UnicodeData.txt rename to crates/unicode/unicode/latest/UnicodeData.txt diff --git a/crates/stdlib/unicode/ucd32/DerivedBidiClass-3.2.0.txt b/crates/unicode/unicode/ucd32/DerivedBidiClass-3.2.0.txt similarity index 100% rename from crates/stdlib/unicode/ucd32/DerivedBidiClass-3.2.0.txt rename to crates/unicode/unicode/ucd32/DerivedBidiClass-3.2.0.txt diff --git a/crates/stdlib/unicode/ucd32/DerivedBinaryProperties-3.2.0.txt b/crates/unicode/unicode/ucd32/DerivedBinaryProperties-3.2.0.txt similarity index 100% rename from crates/stdlib/unicode/ucd32/DerivedBinaryProperties-3.2.0.txt rename to crates/unicode/unicode/ucd32/DerivedBinaryProperties-3.2.0.txt diff --git a/crates/stdlib/unicode/ucd32/DerivedCombiningClass-3.2.0.txt b/crates/unicode/unicode/ucd32/DerivedCombiningClass-3.2.0.txt similarity index 100% rename from crates/stdlib/unicode/ucd32/DerivedCombiningClass-3.2.0.txt rename to crates/unicode/unicode/ucd32/DerivedCombiningClass-3.2.0.txt diff --git a/crates/stdlib/unicode/ucd32/DerivedEastAsianWidth-3.2.0.txt b/crates/unicode/unicode/ucd32/DerivedEastAsianWidth-3.2.0.txt similarity index 100% rename from crates/stdlib/unicode/ucd32/DerivedEastAsianWidth-3.2.0.txt rename to crates/unicode/unicode/ucd32/DerivedEastAsianWidth-3.2.0.txt diff --git a/crates/stdlib/unicode/ucd32/DerivedGeneralCategory-3.2.0.txt b/crates/unicode/unicode/ucd32/DerivedGeneralCategory-3.2.0.txt similarity index 100% rename from crates/stdlib/unicode/ucd32/DerivedGeneralCategory-3.2.0.txt rename to crates/unicode/unicode/ucd32/DerivedGeneralCategory-3.2.0.txt diff --git a/crates/stdlib/unicode/ucd32/DerivedNumericType-3.2.0.txt b/crates/unicode/unicode/ucd32/DerivedNumericType-3.2.0.txt similarity index 100% rename from crates/stdlib/unicode/ucd32/DerivedNumericType-3.2.0.txt rename to crates/unicode/unicode/ucd32/DerivedNumericType-3.2.0.txt diff --git a/crates/stdlib/unicode/ucd32/DerivedNumericValues-3.2.0.txt b/crates/unicode/unicode/ucd32/DerivedNumericValues-3.2.0.txt similarity index 100% rename from crates/stdlib/unicode/ucd32/DerivedNumericValues-3.2.0.txt rename to crates/unicode/unicode/ucd32/DerivedNumericValues-3.2.0.txt From e0b0e72640d1a63132084a6c485a2b25b8931a2c Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Mon, 6 Jul 2026 01:56:18 +0900 Subject: [PATCH 2/9] Route str predicates, casefold, and printable through rustpython-unicode Add classify/case/identifier modules to the shared crate over char/&str/&Wtf8: isalpha/isalnum/isdecimal/isdigit/isnumeric/isspace/isprintable classification, XID identifier predicates, and full-mapping casefold. vm/builtins/str.rs and literal/char.rs now call these instead of icu_properties/icu_casemap directly. String-level iteration with final-sigma handling (lower/upper/title/capitalize, islower/isupper) stays in the runtime. Assisted-by: Claude --- Cargo.lock | 5 +- crates/literal/Cargo.toml | 2 +- crates/literal/src/char.rs | 16 +----- crates/unicode/Cargo.toml | 2 + crates/unicode/src/case.rs | 57 ++++++++++++++++++++ crates/unicode/src/classify.rs | 93 ++++++++++++++++++++++++++++++++ crates/unicode/src/data.rs | 5 ++ crates/unicode/src/identifier.rs | 27 ++++++++++ crates/unicode/src/lib.rs | 46 ++++++++++++++++ crates/vm/Cargo.toml | 1 + crates/vm/src/builtins/str.rs | 77 ++++++-------------------- 11 files changed, 252 insertions(+), 79 deletions(-) create mode 100644 crates/unicode/src/case.rs create mode 100644 crates/unicode/src/classify.rs create mode 100644 crates/unicode/src/identifier.rs diff --git a/Cargo.lock b/Cargo.lock index 3b4e815d257..3fcc8ca3423 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3452,11 +3452,11 @@ name = "rustpython-literal" version = "0.5.0" dependencies = [ "hexf-parse", - "icu_properties", "is-macro", "lexical-parse-float", "num-traits", "rand 0.10.1", + "rustpython-unicode", "rustpython-wtf8", ] @@ -3634,10 +3634,12 @@ dependencies = [ name = "rustpython-unicode" version = "0.5.0" dependencies = [ + "icu_casemap", "icu_normalizer", "icu_properties", "rustpython-wtf8", "unicode_names2 3.1.0", + "writeable", ] [[package]] @@ -3692,6 +3694,7 @@ dependencies = [ "rustpython-ruff_python_parser", "rustpython-ruff_text_size", "rustpython-sre_engine", + "rustpython-unicode", "rustyline", "scopeguard", "serde_core", diff --git a/crates/literal/Cargo.toml b/crates/literal/Cargo.toml index b9795a771eb..60350f7937d 100644 --- a/crates/literal/Cargo.toml +++ b/crates/literal/Cargo.toml @@ -9,13 +9,13 @@ license = { workspace = true } rust-version = { workspace = true } [dependencies] +rustpython-unicode = { workspace = true } rustpython-wtf8 = { workspace = true } hexf-parse = { workspace = true } is-macro.workspace = true lexical-parse-float = { workspace = true, features = ["format"] } num-traits = { workspace = true } -icu_properties = { workspace = true } [dev-dependencies] rand = { workspace = true } diff --git a/crates/literal/src/char.rs b/crates/literal/src/char.rs index 5b446cc1a19..a23e41eec83 100644 --- a/crates/literal/src/char.rs +++ b/crates/literal/src/char.rs @@ -1,5 +1,3 @@ -use icu_properties::props::{EnumeratedProperty, GeneralCategory}; - /// According to python following categories aren't printable: /// * Cc (Other, Control) /// * Cf (Other, Format) @@ -10,17 +8,5 @@ use icu_properties::props::{EnumeratedProperty, GeneralCategory}; /// * Zp Separator, Paragraph ('\u2029', PARAGRAPH SEPARATOR) /// * Zs (Separator, Space) other than ASCII space('\x20'). pub fn is_printable(c: char) -> bool { - let cat = GeneralCategory::for_char(c); - - !matches!( - cat, - GeneralCategory::SpaceSeparator - | GeneralCategory::LineSeparator - | GeneralCategory::ParagraphSeparator - | GeneralCategory::Control - | GeneralCategory::Format - | GeneralCategory::Surrogate - | GeneralCategory::PrivateUse - | GeneralCategory::Unassigned - ) + rustpython_unicode::classify::is_repr_printable(c) } diff --git a/crates/unicode/Cargo.toml b/crates/unicode/Cargo.toml index 05f114ee3b7..678305ee09e 100644 --- a/crates/unicode/Cargo.toml +++ b/crates/unicode/Cargo.toml @@ -11,9 +11,11 @@ rust-version = { workspace = true } [dependencies] rustpython-wtf8 = { workspace = true } +icu_casemap = { workspace = true } icu_properties = { workspace = true } icu_normalizer = { workspace = true } unicode_names2 = { workspace = true } +writeable = { workspace = true } [build-dependencies] icu_properties = { workspace = true } diff --git a/crates/unicode/src/case.rs b/crates/unicode/src/case.rs new file mode 100644 index 00000000000..0e38eea11dc --- /dev/null +++ b/crates/unicode/src/case.rs @@ -0,0 +1,57 @@ +//! Case folding for Python `str.casefold`. +//! +//! Lower, upper, and title casing of `str` objects stay with the runtime +//! because they iterate the string with special final-sigma handling. Case +//! folding has no such context dependence, so it lives here and is shared with +//! other runtimes. + +use alloc::{ + string::{String, ToString}, + vec::Vec, +}; + +use icu_casemap::CaseMapper; +use rustpython_wtf8::{Wtf8, Wtf8Buf, Wtf8Chunk}; +use writeable::Writeable; + +/// Full Unicode case fold of `text` (`str.casefold`). +#[must_use] +pub fn casefold_str(text: &str) -> String { + CaseMapper::new().fold_string(text).to_string() +} + +/// Full Unicode case fold of `text`, passing lone surrogates through unchanged. +#[must_use] +pub fn casefold_wtf8(text: &Wtf8) -> Wtf8Buf { + let mut out = Vec::with_capacity(text.len()); + let mapper = CaseMapper::new(); + for chunk in text.chunks() { + match chunk { + Wtf8Chunk::Utf8(s) => { + mapper + .fold(s) + .write_to(&mut FmtWriter(&mut out)) + .expect("writing to an in-memory buffer cannot fail"); + } + Wtf8Chunk::Surrogate(c) => { + let mut buf = Wtf8Buf::new(); + buf.push(c); + out.extend_from_slice(buf.as_bytes()); + } + } + } + // SAFETY: + // * CaseMapper only produces valid UTF-8. + // * Surrogates are appended as valid WTF-8 (encoded via Wtf8Buf::push). + unsafe { Wtf8Buf::from_bytes_unchecked(out) } +} + +/// Adapter so `icu`'s `Writeable` output can be appended to a byte buffer. +struct FmtWriter<'a>(&'a mut Vec); + +impl core::fmt::Write for FmtWriter<'_> { + fn write_str(&mut self, s: &str) -> core::fmt::Result { + self.0.extend_from_slice(s.as_bytes()); + Ok(()) + } +} diff --git a/crates/unicode/src/classify.rs b/crates/unicode/src/classify.rs new file mode 100644 index 00000000000..dd5b54db942 --- /dev/null +++ b/crates/unicode/src/classify.rs @@ -0,0 +1,93 @@ +//! Character classification predicates for Python `str` methods. +//! +//! Each predicate operates on a single Unicode scalar. Callers iterating over +//! WTF-8 text apply these per code point, treating lone surrogates as failing +//! every predicate. + +use icu_properties::props::{ + BidiClass, EnumeratedProperty, GeneralCategory, GeneralCategoryGroup, NumericType, +}; + +/// `str.isalpha` for a single character: any `Letter` general category. +#[must_use] +pub fn is_alpha(c: char) -> bool { + GeneralCategoryGroup::Letter.contains(GeneralCategory::for_char(c)) +} + +/// `str.isalnum` for a single character: any `Letter` or `Number` category. +#[must_use] +pub fn is_alnum(c: char) -> bool { + GeneralCategoryGroup::Letter + .union(GeneralCategoryGroup::Number) + .contains(GeneralCategory::for_char(c)) +} + +/// `str.isdecimal` for a single character: `Decimal_Number` general category. +#[must_use] +pub fn is_decimal(c: char) -> bool { + matches!(GeneralCategory::for_char(c), GeneralCategory::DecimalNumber) +} + +/// `str.isdigit` for a single character: `Numeric_Type` of `Digit` or `Decimal`. +#[must_use] +pub fn is_digit(c: char) -> bool { + matches!( + NumericType::for_char(c), + NumericType::Digit | NumericType::Decimal + ) +} + +/// `str.isnumeric` for a single character: any numeric `Numeric_Type`. +#[must_use] +pub fn is_numeric(c: char) -> bool { + matches!( + NumericType::for_char(c), + NumericType::Decimal | NumericType::Digit | NumericType::Numeric + ) +} + +/// `str.isspace` for a single character: `Space_Separator`, or a bidi +/// whitespace / paragraph / segment separator. +#[must_use] +pub fn is_space(c: char) -> bool { + matches!( + GeneralCategory::for_char(c), + GeneralCategory::SpaceSeparator + ) || matches!( + BidiClass::for_char(c), + BidiClass::WhiteSpace | BidiClass::ParagraphSeparator | BidiClass::SegmentSeparator + ) +} + +/// `str.isprintable` for a single character: ASCII space is printable, as are +/// all characters that survive [`is_repr_printable`]. +#[must_use] +pub fn is_printable(c: char) -> bool { + c == '\u{0020}' || is_repr_printable(c) +} + +/// Repr/escape printable semantics. +/// +/// The following categories are not printable: +/// * Cc (Other, Control) +/// * Cf (Other, Format) +/// * Cs (Other, Surrogate) +/// * Co (Other, Private Use) +/// * Cn (Other, Not Assigned) +/// * Zl (Separator, Line) +/// * Zp (Separator, Paragraph) +/// * Zs (Separator, Space), including ASCII space +#[must_use] +pub fn is_repr_printable(c: char) -> bool { + !matches!( + GeneralCategory::for_char(c), + GeneralCategory::SpaceSeparator + | GeneralCategory::LineSeparator + | GeneralCategory::ParagraphSeparator + | GeneralCategory::Control + | GeneralCategory::Format + | GeneralCategory::Surrogate + | GeneralCategory::PrivateUse + | GeneralCategory::Unassigned + ) +} diff --git a/crates/unicode/src/data.rs b/crates/unicode/src/data.rs index e98c7d8e7cb..7d437968ea0 100644 --- a/crates/unicode/src/data.rs +++ b/crates/unicode/src/data.rs @@ -7,6 +7,11 @@ use core::{cmp::Ordering, fmt::Write, hint::cold_path}; +use alloc::{ + format, + string::{String, ToString}, +}; + use icu_normalizer::properties::{CanonicalDecomposition, Decomposed}; use icu_properties::props::{ BidiClass, BidiMirrored, BinaryProperty, CanonicalCombiningClass, EastAsianWidth, diff --git a/crates/unicode/src/identifier.rs b/crates/unicode/src/identifier.rs new file mode 100644 index 00000000000..8ea6fb26761 --- /dev/null +++ b/crates/unicode/src/identifier.rs @@ -0,0 +1,27 @@ +//! Python identifier predicates (`str.isidentifier`). + +use icu_properties::props::{BinaryProperty, XidContinue, XidStart}; + +/// Whether `c` has the `XID_Start` property. +#[must_use] +pub fn is_xid_start(c: char) -> bool { + XidStart::for_char(c) +} + +/// Whether `c` has the `XID_Continue` property. +#[must_use] +pub fn is_xid_continue(c: char) -> bool { + XidContinue::for_char(c) +} + +/// Whether `c` may start a Python identifier: `_` or `XID_Start`. +#[must_use] +pub fn is_start(c: char) -> bool { + c == '_' || is_xid_start(c) +} + +/// Whether `c` may continue a Python identifier: `XID_Continue`. +#[must_use] +pub fn is_continue(c: char) -> bool { + is_xid_continue(c) +} diff --git a/crates/unicode/src/lib.rs b/crates/unicode/src/lib.rs index 093a31d1f8c..6dc9a8c40d2 100644 --- a/crates/unicode/src/lib.rs +++ b/crates/unicode/src/lib.rs @@ -5,7 +5,14 @@ //! exception mapping stay with the caller. There is no global mutable state and //! results depend only on inputs. +#![no_std] + +extern crate alloc; + +pub mod case; +pub mod classify; pub mod data; +pub mod identifier; pub mod normalize; pub use data::{Ucd, character_name, lookup_character, unicode_version}; @@ -40,6 +47,45 @@ mod tests { assert_eq!(legacy.unidata_version(), "3.2.0"); } + #[test] + fn numeric_type_chain_holds() { + use crate::classify::{is_decimal, is_digit, is_numeric}; + + // isdecimal ⊂ isdigit ⊂ isnumeric + for c in ('\0'..='\u{2FFFF}').filter_map(|c| char::from_u32(c as u32)) { + if is_decimal(c) { + assert!(is_digit(c), "{c:?} decimal but not digit"); + } + if is_digit(c) { + assert!(is_numeric(c), "{c:?} digit but not numeric"); + } + } + assert!(crate::classify::is_decimal('5')); + assert!(!crate::classify::is_decimal('²')); + assert!(crate::classify::is_digit('²')); + assert!(!crate::classify::is_digit('⅓')); + assert!(crate::classify::is_numeric('⅓')); + } + + #[test] + fn identifier_predicates() { + use crate::identifier::{is_continue, is_start}; + + assert!(is_start('_')); + assert!(is_start('가')); + assert!(!is_start('1')); + assert!(is_continue('1')); + } + + #[test] + fn casefold_full_mappings() { + use crate::case::casefold_str; + + // ß case-folds to "ss" + assert_eq!(casefold_str("ß"), "ss"); + assert_eq!(casefold_str("Σ"), "σ"); + } + #[test] fn normalization_round_trips() { let composed = Wtf8Buf::from("é"); diff --git a/crates/vm/Cargo.toml b/crates/vm/Cargo.toml index b3479e017a1..f4e59e6acc4 100644 --- a/crates/vm/Cargo.toml +++ b/crates/vm/Cargo.toml @@ -42,6 +42,7 @@ ruff_text_size = { workspace = true, optional = true } rustpython-compiler-core = { workspace = true } rustpython-literal = { workspace = true } rustpython-sre_engine = { workspace = true } +rustpython-unicode = { workspace = true } ascii = { workspace = true } bitflags = { workspace = true } diff --git a/crates/vm/src/builtins/str.rs b/crates/vm/src/builtins/str.rs index 69109c943e8..a72a272679b 100644 --- a/crates/vm/src/builtins/str.rs +++ b/crates/vm/src/builtins/str.rs @@ -48,12 +48,13 @@ use rustpython_common::{ wtf8::{CodePoint, Wtf8, Wtf8Buf, Wtf8Concat}, }; -use icu_casemap::{CaseMapper, TitlecaseMapper}; +use icu_casemap::TitlecaseMapper; use icu_locale::LanguageIdentifier; use icu_properties::props::{ - BidiClass, BinaryProperty, CaseIgnorable, Cased, EnumeratedProperty, GeneralCategory, - GeneralCategoryGroup, Lowercase, NumericType, Uppercase, XidContinue, XidStart, + BinaryProperty, CaseIgnorable, Cased, EnumeratedProperty, GeneralCategory, + GeneralCategoryGroup, Lowercase, Uppercase, }; +use rustpython_unicode as unicode; use writeable::Writeable; impl<'a> TryFromBorrowedObject<'a> for String { @@ -752,22 +753,8 @@ impl PyStr { fn casefold(&self) -> Self { match self.as_str_kind() { PyKindStr::Ascii(s) => s.to_ascii_lowercase().into(), - PyKindStr::Utf8(s) => CaseMapper::new().fold_string(s).to_string().into(), - PyKindStr::Wtf8(w) => { - let mut out = VecFmtWriter(Vec::with_capacity(w.len())); - let mapper = CaseMapper::new(); - for chunk in w.as_bytes().utf8_chunks() { - mapper - .fold(chunk.valid()) - .write_to(&mut out) - .expect("Writing to an in-memory buffer cannot fail."); - out.0.extend(chunk.invalid()); - } - // SAFETY: - // * CaseMapper only produces valid UTF-8 - // * Surrogates are appended as-is - unsafe { Wtf8Buf::from_bytes_unchecked(out.0) }.into() - } + PyKindStr::Utf8(s) => unicode::case::casefold_str(s).into(), + PyKindStr::Wtf8(w) => unicode::case::casefold_wtf8(w).into(), } } @@ -1011,41 +998,22 @@ impl PyStr { #[pymethod] fn isalnum(&self) -> bool { - !self.data.is_empty() - && self.char_all(|c| { - GeneralCategoryGroup::Letter - .union(GeneralCategoryGroup::Number) - .contains(GeneralCategory::for_char(c)) - }) + !self.data.is_empty() && self.char_all(unicode::classify::is_alnum) } #[pymethod] fn isnumeric(&self) -> bool { - !self.data.is_empty() - && self.char_all(|c| { - [ - NumericType::Decimal, - NumericType::Digit, - NumericType::Numeric, - ] - .contains(&NumericType::for_char(c)) - }) + !self.data.is_empty() && self.char_all(unicode::classify::is_numeric) } #[pymethod] fn isdigit(&self) -> bool { - !self.data.is_empty() - && self.char_all(|c| { - [NumericType::Digit, NumericType::Decimal].contains(&NumericType::for_char(c)) - }) + !self.data.is_empty() && self.char_all(unicode::classify::is_digit) } #[pymethod] fn isdecimal(&self) -> bool { - !self.data.is_empty() - && self.char_all(|c| { - matches!(GeneralCategory::for_char(c), GeneralCategory::DecimalNumber) - }) + !self.data.is_empty() && self.char_all(unicode::classify::is_decimal) } fn __mod__(&self, values: PyObjectRef, vm: &VirtualMachine) -> PyResult { @@ -1143,9 +1111,7 @@ impl PyStr { #[pymethod] fn isalpha(&self) -> bool { - !self.data.is_empty() - && self - .char_all(|c| GeneralCategoryGroup::Letter.contains(GeneralCategory::for_char(c))) + !self.data.is_empty() && self.char_all(unicode::classify::is_alpha) } #[pymethod] @@ -1175,23 +1141,12 @@ impl PyStr { #[pymethod] fn isprintable(&self) -> bool { - self.char_all(|c| c == '\u{0020}' || rustpython_literal::char::is_printable(c)) + self.char_all(unicode::classify::is_printable) } #[pymethod] fn isspace(&self) -> bool { - !self.data.is_empty() - && self.char_all(|c| { - matches!( - GeneralCategory::for_char(c), - GeneralCategory::SpaceSeparator - ) || matches!( - BidiClass::for_char(c), - BidiClass::WhiteSpace - | BidiClass::ParagraphSeparator - | BidiClass::SegmentSeparator - ) - }) + !self.data.is_empty() && self.char_all(unicode::classify::is_space) } // Return true if all cased characters in the string are lowercase and there is at least one cased character, false otherwise. @@ -1452,12 +1407,10 @@ impl PyStr { let Some(s) = self.to_str() else { return false }; let mut chars = s.chars(); - let is_identifier_start = chars - .next() - .is_some_and(|c| c == '_' || XidStart::for_char(c)); + let is_identifier_start = chars.next().is_some_and(unicode::identifier::is_start); // a string is not an identifier if it has whitespace or starts with a number - is_identifier_start && chars.all(XidContinue::for_char) + is_identifier_start && chars.all(unicode::identifier::is_continue) } // https://docs.python.org/3/library/stdtypes.html#str.translate From 6dc0556db95e1a626eba91a5a28bcd8db18abdf2 Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Mon, 6 Jul 2026 02:00:12 +0900 Subject: [PATCH 3/9] Route sre_engine char classes through rustpython-unicode MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add the regex module with the SRE character-class and case predicates (is_word/is_space/is_digit, is_uni_* Unicode variants, ascii/locale/unicode case folding). sre_engine/string.rs now forwards to it, keeping its public API and behavior identical — including the ASCII-only is_uni_digit and the hardcoded is_uni_space table. Assisted-by: Claude --- crates/sre_engine/Cargo.toml | 2 +- crates/sre_engine/src/string.rs | 81 ++++--------------- crates/unicode/src/lib.rs | 1 + crates/unicode/src/regex.rs | 137 ++++++++++++++++++++++++++++++++ 4 files changed, 154 insertions(+), 67 deletions(-) create mode 100644 crates/unicode/src/regex.rs diff --git a/crates/sre_engine/Cargo.toml b/crates/sre_engine/Cargo.toml index 8400a34b567..03b3f609801 100644 --- a/crates/sre_engine/Cargo.toml +++ b/crates/sre_engine/Cargo.toml @@ -15,11 +15,11 @@ name = "benches" harness = false [dependencies] +rustpython-unicode = { workspace = true } rustpython-wtf8 = { workspace = true } num_enum = { workspace = true } bitflags = { workspace = true } optional = { workspace = true } -icu_properties = { workspace = true } [dev-dependencies] criterion = { workspace = true } diff --git a/crates/sre_engine/src/string.rs b/crates/sre_engine/src/string.rs index 6c8b9a567b4..1038607a2c0 100644 --- a/crates/sre_engine/src/string.rs +++ b/crates/sre_engine/src/string.rs @@ -1,4 +1,4 @@ -use icu_properties::props::{EnumeratedProperty, GeneralCategory, GeneralCategoryGroup}; +use rustpython_unicode as unicode; use rustpython_wtf8::Wtf8; #[derive(Debug, Clone, Copy)] @@ -333,26 +333,21 @@ const fn utf8_is_cont_byte(byte: u8) -> bool { /// Mask of the value bits of a continuation byte. const CONT_MASK: u8 = 0b0011_1111; -const fn is_py_ascii_whitespace(b: u8) -> bool { - matches!(b, b'\t' | b'\n' | b'\x0C' | b'\r' | b' ' | b'\x0B') -} - #[inline] pub(crate) fn is_word(ch: u32) -> bool { - ch == '_' as u32 || u8::try_from(ch).is_ok_and(|x| x.is_ascii_alphanumeric()) + unicode::regex::is_word(ch) } #[inline] pub(crate) fn is_space(ch: u32) -> bool { - u8::try_from(ch).is_ok_and(is_py_ascii_whitespace) + unicode::regex::is_space(ch) } #[inline] pub(crate) fn is_digit(ch: u32) -> bool { - u8::try_from(ch).is_ok_and(|x| x.is_ascii_digit()) + unicode::regex::is_digit(ch) } #[inline] pub(crate) fn is_loc_alnum(ch: u32) -> bool { - // FIXME: Ignore the locales - u8::try_from(ch).is_ok_and(|x| x.is_ascii_alphanumeric()) + unicode::regex::is_loc_alnum(ch) } #[inline] pub(crate) fn is_loc_word(ch: u32) -> bool { @@ -360,80 +355,36 @@ pub(crate) fn is_loc_word(ch: u32) -> bool { } #[inline] pub(crate) const fn is_linebreak(ch: u32) -> bool { - ch == '\n' as u32 + unicode::regex::is_linebreak(ch) } #[inline] #[must_use] pub fn lower_ascii(ch: u32) -> u32 { - u8::try_from(ch).map_or(ch, |x| x.to_ascii_lowercase() as u32) + unicode::regex::lower_ascii(ch) } #[inline] pub(crate) fn lower_locate(ch: u32) -> u32 { - // FIXME: Ignore the locales - lower_ascii(ch) + unicode::regex::lower_locate(ch) } #[inline] pub(crate) fn upper_locate(ch: u32) -> u32 { - // FIXME: Ignore the locales - u8::try_from(ch).map_or(ch, |x| x.to_ascii_uppercase() as u32) + unicode::regex::upper_locate(ch) } #[inline] pub(crate) fn is_uni_digit(ch: u32) -> bool { - // TODO: check with cpython - char::try_from(ch).is_ok_and(|x| x.is_ascii_digit()) + unicode::regex::is_uni_digit(ch) } #[inline] pub(crate) fn is_uni_space(ch: u32) -> bool { - // TODO: check with cpython - is_space(ch) - || matches!( - ch, - 0x0009 - | 0x000A - | 0x000B - | 0x000C - | 0x000D - | 0x001C - | 0x001D - | 0x001E - | 0x001F - | 0x0020 - | 0x0085 - | 0x00A0 - | 0x1680 - | 0x2000 - | 0x2001 - | 0x2002 - | 0x2003 - | 0x2004 - | 0x2005 - | 0x2006 - | 0x2007 - | 0x2008 - | 0x2009 - | 0x200A - | 0x2028 - | 0x2029 - | 0x202F - | 0x205F - | 0x3000 - ) + unicode::regex::is_uni_space(ch) } #[inline] pub(crate) const fn is_uni_linebreak(ch: u32) -> bool { - matches!( - ch, - 0x000A | 0x000B | 0x000C | 0x000D | 0x001C | 0x001D | 0x001E | 0x0085 | 0x2028 | 0x2029 - ) + unicode::regex::is_uni_linebreak(ch) } #[inline] pub(crate) fn is_uni_alnum(ch: u32) -> bool { - // TODO: check with cpython - char::try_from(ch).is_ok_and(|c| { - GeneralCategoryGroup::Letter - .union(GeneralCategoryGroup::Number) - .contains(GeneralCategory::for_char(c)) - }) + unicode::regex::is_uni_alnum(ch) } #[inline] pub(crate) fn is_uni_word(ch: u32) -> bool { @@ -442,12 +393,10 @@ pub(crate) fn is_uni_word(ch: u32) -> bool { #[inline] #[must_use] pub fn lower_unicode(ch: u32) -> u32 { - // TODO: check with cpython - char::try_from(ch).map_or(ch, |x| x.to_lowercase().next().unwrap() as u32) + unicode::regex::lower_unicode(ch) } #[inline] #[must_use] pub fn upper_unicode(ch: u32) -> u32 { - // TODO: check with cpython - char::try_from(ch).map_or(ch, |x| x.to_uppercase().next().unwrap() as u32) + unicode::regex::upper_unicode(ch) } diff --git a/crates/unicode/src/lib.rs b/crates/unicode/src/lib.rs index 6dc9a8c40d2..58fe9c7478f 100644 --- a/crates/unicode/src/lib.rs +++ b/crates/unicode/src/lib.rs @@ -14,6 +14,7 @@ pub mod classify; pub mod data; pub mod identifier; pub mod normalize; +pub mod regex; pub use data::{Ucd, character_name, lookup_character, unicode_version}; pub use normalize::{NormalizeForm, is_normalized, normalize}; diff --git a/crates/unicode/src/regex.rs b/crates/unicode/src/regex.rs new file mode 100644 index 00000000000..595f5e6f2f1 --- /dev/null +++ b/crates/unicode/src/regex.rs @@ -0,0 +1,137 @@ +//! Character-class and case predicates for the SRE regex engine. +//! +//! Every predicate takes a raw `u32` code point (SRE decodes strings into +//! `u32`s, including lone surrogates) and returns whether it belongs to the +//! class. ASCII-mode predicates only ever consider byte values; Unicode-mode +//! predicates consult the shared property tables. + +use crate::classify; + +const UNDERSCORE: u32 = '_' as u32; + +const fn is_py_ascii_whitespace(b: u8) -> bool { + matches!(b, b'\t' | b'\n' | b'\x0C' | b'\r' | b' ' | b'\x0B') +} + +#[must_use] +pub fn is_word(ch: u32) -> bool { + ch == UNDERSCORE || u8::try_from(ch).is_ok_and(|x| x.is_ascii_alphanumeric()) +} + +#[must_use] +pub fn is_space(ch: u32) -> bool { + u8::try_from(ch).is_ok_and(is_py_ascii_whitespace) +} + +#[must_use] +pub fn is_digit(ch: u32) -> bool { + u8::try_from(ch).is_ok_and(|x| x.is_ascii_digit()) +} + +#[must_use] +pub fn is_loc_alnum(ch: u32) -> bool { + // FIXME: Ignore the locales + u8::try_from(ch).is_ok_and(|x| x.is_ascii_alphanumeric()) +} + +#[must_use] +pub fn is_loc_word(ch: u32) -> bool { + ch == UNDERSCORE || is_loc_alnum(ch) +} + +#[must_use] +pub const fn is_linebreak(ch: u32) -> bool { + ch == '\n' as u32 +} + +#[must_use] +pub fn lower_ascii(ch: u32) -> u32 { + u8::try_from(ch).map_or(ch, |x| x.to_ascii_lowercase() as u32) +} + +#[must_use] +pub fn lower_locate(ch: u32) -> u32 { + // FIXME: Ignore the locales + lower_ascii(ch) +} + +#[must_use] +pub fn upper_locate(ch: u32) -> u32 { + // FIXME: Ignore the locales + u8::try_from(ch).map_or(ch, |x| x.to_ascii_uppercase() as u32) +} + +#[must_use] +pub fn is_uni_digit(ch: u32) -> bool { + // TODO: check with cpython + char::try_from(ch).is_ok_and(|x| x.is_ascii_digit()) +} + +#[must_use] +pub fn is_uni_space(ch: u32) -> bool { + // TODO: check with cpython + is_space(ch) + || matches!( + ch, + 0x0009 + | 0x000A + | 0x000B + | 0x000C + | 0x000D + | 0x001C + | 0x001D + | 0x001E + | 0x001F + | 0x0020 + | 0x0085 + | 0x00A0 + | 0x1680 + | 0x2000 + | 0x2001 + | 0x2002 + | 0x2003 + | 0x2004 + | 0x2005 + | 0x2006 + | 0x2007 + | 0x2008 + | 0x2009 + | 0x200A + | 0x2028 + | 0x2029 + | 0x202F + | 0x205F + | 0x3000 + ) +} + +#[must_use] +pub const fn is_uni_linebreak(ch: u32) -> bool { + matches!( + ch, + 0x000A | 0x000B | 0x000C | 0x000D | 0x001C | 0x001D | 0x001E | 0x0085 | 0x2028 | 0x2029 + ) +} + +#[must_use] +pub fn is_uni_alnum(ch: u32) -> bool { + // TODO: check with cpython + char::try_from(ch).is_ok_and(classify::is_alnum) +} + +#[must_use] +pub fn is_uni_word(ch: u32) -> bool { + ch == UNDERSCORE || is_uni_alnum(ch) +} + +#[must_use] +pub fn lower_unicode(ch: u32) -> u32 { + // TODO: check with cpython + char::try_from(ch).map_or(ch, |x| x.to_lowercase().next().unwrap() as u32) +} + +#[must_use] +pub fn upper_unicode(ch: u32) -> u32 { + // TODO: check with cpython + char::try_from(ch).map_or(ch, |x| x.to_uppercase().next().unwrap() as u32) +} From ea9efabe304232611dc51d345293f27dc3a684d7 Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Mon, 6 Jul 2026 02:03:56 +0900 Subject: [PATCH 4/9] Route \N{} name lookups through rustpython-unicode codegen/string_parser.rs (\N{...} escapes) and common/encodings.rs (the namereplace error handler) now resolve character names via rustpython-unicode instead of depending on unicode_names2 directly, leaving the crate as the sole owner of the name database. Assisted-by: Claude --- Cargo.lock | 6 +++--- crates/codegen/Cargo.toml | 2 +- crates/codegen/src/string_parser.rs | 2 +- crates/common/Cargo.toml | 2 +- crates/common/src/encodings.rs | 2 +- 5 files changed, 7 insertions(+), 7 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 3fcc8ca3423..31086945509 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3313,9 +3313,9 @@ dependencies = [ "rustpython-ruff_python_ast", "rustpython-ruff_python_parser", "rustpython-ruff_text_size", + "rustpython-unicode", "rustpython-wtf8", "thiserror", - "unicode_names2 3.1.0", ] [[package]] @@ -3336,9 +3336,9 @@ dependencies = [ "parking_lot", "radium", "rustpython-literal", + "rustpython-unicode", "rustpython-wtf8", "siphasher", - "unicode_names2 3.1.0", ] [[package]] @@ -3541,9 +3541,9 @@ version = "0.5.0" dependencies = [ "bitflags 2.13.0", "criterion", - "icu_properties", "num_enum", "optional", + "rustpython-unicode", "rustpython-wtf8", ] diff --git a/crates/codegen/Cargo.toml b/crates/codegen/Cargo.toml index 031f3b96521..c43bf1bcb08 100644 --- a/crates/codegen/Cargo.toml +++ b/crates/codegen/Cargo.toml @@ -15,6 +15,7 @@ std = ["thiserror/std", "itertools/use_std"] [dependencies] rustpython-compiler-core = { workspace = true } rustpython-literal = {workspace = true } +rustpython-unicode = { workspace = true } rustpython-wtf8 = { workspace = true } ruff_python_ast = { workspace = true } ruff_text_size = { workspace = true } @@ -29,7 +30,6 @@ thiserror = { workspace = true } malachite-bigint = { workspace = true } memchr = { workspace = true } rapidhash = { workspace = true } -unicode_names2 = { workspace = true } [dev-dependencies] ruff_python_parser = { workspace = true } diff --git a/crates/codegen/src/string_parser.rs b/crates/codegen/src/string_parser.rs index 0b5bcfffc9c..622488a2177 100644 --- a/crates/codegen/src/string_parser.rs +++ b/crates/codegen/src/string_parser.rs @@ -114,7 +114,7 @@ impl StringParser { let name_and_ending = self.skip_bytes(close_idx + 1); let name = &name_and_ending[..name_and_ending.len() - 1]; - unicode_names2::character(name).ok_or_else(|| unreachable!()) + rustpython_unicode::lookup_character(name).ok_or_else(|| unreachable!()) } /// Parse an escaped character, returning the new character. diff --git a/crates/common/Cargo.toml b/crates/common/Cargo.toml index 725be665f73..4498e74ca49 100644 --- a/crates/common/Cargo.toml +++ b/crates/common/Cargo.toml @@ -16,6 +16,7 @@ wasm_js = ["getrandom/wasm_js"] [dependencies] rustpython-literal = { workspace = true } +rustpython-unicode = { workspace = true } rustpython-wtf8 = { workspace = true } ascii = { workspace = true } @@ -28,7 +29,6 @@ malachite-q = { workspace = true } malachite-base = { workspace = true } num-traits = { workspace = true } parking_lot = { workspace = true, optional = true } -unicode_names2 = { workspace = true } radium = { workspace = true } lock_api = { workspace = true } diff --git a/crates/common/src/encodings.rs b/crates/common/src/encodings.rs index 913f0521e16..b9ce02b88cb 100644 --- a/crates/common/src/encodings.rs +++ b/crates/common/src/encodings.rs @@ -414,7 +414,7 @@ pub mod errors { let mut out = String::with_capacity(num_chars * 4); for c in err_str.code_points() { let c_u32 = c.to_u32(); - if let Some(c_name) = c.to_char().and_then(unicode_names2::name) { + if let Some(c_name) = c.to_char().and_then(rustpython_unicode::character_name) { write!(out, "\\N{{{c_name}}}").unwrap(); } else if c_u32 >= 0x10000 { write!(out, "\\U{c_u32:08x}").unwrap(); From e8ffc1279755a4ccd6df976748b6283caebb355c Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Mon, 6 Jul 2026 02:09:32 +0900 Subject: [PATCH 5/9] Add differential Unicode sweep and shared-crate snippet test tests/differential.rs sweeps the full 0..0x110000 range and compares each str classification predicate against a committed CPython 3.14 reference dataset (tests/data/cpython3.14_predicates.txt, produced by generate_reference.py). Code points that differ only because the Rust std / icu4x build ships a later Unicode release than CPython 3.14's 16.0.0 are recorded in tests/data/version_skew_cpython3.14.txt, regenerable via RUSTPYTHON_UNICODE_REGEN_SKEW=1; the regen refuses to record any cpython=true/crate=false divergence, so only newly-assigned code points are allowed. Any other divergence fails. Both data files use a run-length `predicate start:end,...` encoding. extra_tests/snippets/stdlib_unicode_shared.py exercises the routed surface end to end (str predicates, casefold, identifiers, unicodedata, normalize, \N{}, and re character classes) and passes identically on CPython 3.14 and RustPython. Assisted-by: Claude --- .../tests/data/cpython3.14_predicates.txt | 9 + .../tests/data/version_skew_cpython3.14.txt | 11 + crates/unicode/tests/differential.rs | 236 ++++++++++++++++++ crates/unicode/tests/generate_reference.py | 74 ++++++ extra_tests/snippets/stdlib_unicode_shared.py | 89 +++++++ 5 files changed, 419 insertions(+) create mode 100644 crates/unicode/tests/data/cpython3.14_predicates.txt create mode 100644 crates/unicode/tests/data/version_skew_cpython3.14.txt create mode 100644 crates/unicode/tests/differential.rs create mode 100644 crates/unicode/tests/generate_reference.py create mode 100644 extra_tests/snippets/stdlib_unicode_shared.py diff --git a/crates/unicode/tests/data/cpython3.14_predicates.txt b/crates/unicode/tests/data/cpython3.14_predicates.txt new file mode 100644 index 00000000000..9339f4e4744 --- /dev/null +++ b/crates/unicode/tests/data/cpython3.14_predicates.txt @@ -0,0 +1,9 @@ +# unidata_version 16.0.0 +isalpha 41:5A,61:7A,AA:AA,B5:B5,BA:BA,C0:D6,D8:F6,F8:2C1,2C6:2D1,2E0:2E4,2EC:2EC,2EE:2EE,370:374,376:377,37A:37D,37F:37F,386:386,388:38A,38C:38C,38E:3A1,3A3:3F5,3F7:481,48A:52F,531:556,559:559,560:588,5D0:5EA,5EF:5F2,620:64A,66E:66F,671:6D3,6D5:6D5,6E5:6E6,6EE:6EF,6FA:6FC,6FF:6FF,710:710,712:72F,74D:7A5,7B1:7B1,7CA:7EA,7F4:7F5,7FA:7FA,800:815,81A:81A,824:824,828:828,840:858,860:86A,870:887,889:88E,8A0:8C9,904:939,93D:93D,950:950,958:961,971:980,985:98C,98F:990,993:9A8,9AA:9B0,9B2:9B2,9B6:9B9,9BD:9BD,9CE:9CE,9DC:9DD,9DF:9E1,9F0:9F1,9FC:9FC,A05:A0A,A0F:A10,A13:A28,A2A:A30,A32:A33,A35:A36,A38:A39,A59:A5C,A5E:A5E,A72:A74,A85:A8D,A8F:A91,A93:AA8,AAA:AB0,AB2:AB3,AB5:AB9,ABD:ABD,AD0:AD0,AE0:AE1,AF9:AF9,B05:B0C,B0F:B10,B13:B28,B2A:B30,B32:B33,B35:B39,B3D:B3D,B5C:B5D,B5F:B61,B71:B71,B83:B83,B85:B8A,B8E:B90,B92:B95,B99:B9A,B9C:B9C,B9E:B9F,BA3:BA4,BA8:BAA,BAE:BB9,BD0:BD0,C05:C0C,C0E:C10,C12:C28,C2A:C39,C3D:C3D,C58:C5A,C5D:C5D,C60:C61,C80:C80,C85:C8C,C8E:C90,C92:CA8,CAA:CB3,CB5:CB9,CBD:CBD,CDD:CDE,CE0:CE1,CF1:CF2,D04:D0C,D0E:D10,D12:D3A,D3D:D3D,D4E:D4E,D54:D56,D5F:D61,D7A:D7F,D85:D96,D9A:DB1,DB3:DBB,DBD:DBD,DC0:DC6,E01:E30,E32:E33,E40:E46,E81:E82,E84:E84,E86:E8A,E8C:EA3,EA5:EA5,EA7:EB0,EB2:EB3,EBD:EBD,EC0:EC4,EC6:EC6,EDC:EDF,F00:F00,F40:F47,F49:F6C,F88:F8C,1000:102A,103F:103F,1050:1055,105A:105D,1061:1061,1065:1066,106E:1070,1075:1081,108E:108E,10A0:10C5,10C7:10C7,10CD:10CD,10D0:10FA,10FC:1248,124A:124D,1250:1256,1258:1258,125A:125D,1260:1288,128A:128D,1290:12B0,12B2:12B5,12B8:12BE,12C0:12C0,12C2:12C5,12C8:12D6,12D8:1310,1312:1315,1318:135A,1380:138F,13A0:13F5,13F8:13FD,1401:166C,166F:167F,1681:169A,16A0:16EA,16F1:16F8,1700:1711,171F:1731,1740:1751,1760:176C,176E:1770,1780:17B3,17D7:17D7,17DC:17DC,1820:1878,1880:1884,1887:18A8,18AA:18AA,18B0:18F5,1900:191E,1950:196D,1970:1974,1980:19AB,19B0:19C9,1A00:1A16,1A20:1A54,1AA7:1AA7,1B05:1B33,1B45:1B4C,1B83:1BA0,1BAE:1BAF,1BBA:1BE5,1C00:1C23,1C4D:1C4F,1C5A:1C7D,1C80:1C8A,1C90:1CBA,1CBD:1CBF,1CE9:1CEC,1CEE:1CF3,1CF5:1CF6,1CFA:1CFA,1D00:1DBF,1E00:1F15,1F18:1F1D,1F20:1F45,1F48:1F4D,1F50:1F57,1F59:1F59,1F5B:1F5B,1F5D:1F5D,1F5F:1F7D,1F80:1FB4,1FB6:1FBC,1FBE:1FBE,1FC2:1FC4,1FC6:1FCC,1FD0:1FD3,1FD6:1FDB,1FE0:1FEC,1FF2:1FF4,1FF6:1FFC,2071:2071,207F:207F,2090:209C,2102:2102,2107:2107,210A:2113,2115:2115,2119:211D,2124:2124,2126:2126,2128:2128,212A:212D,212F:2139,213C:213F,2145:2149,214E:214E,2183:2184,2C00:2CE4,2CEB:2CEE,2CF2:2CF3,2D00:2D25,2D27:2D27,2D2D:2D2D,2D30:2D67,2D6F:2D6F,2D80:2D96,2DA0:2DA6,2DA8:2DAE,2DB0:2DB6,2DB8:2DBE,2DC0:2DC6,2DC8:2DCE,2DD0:2DD6,2DD8:2DDE,2E2F:2E2F,3005:3006,3031:3035,303B:303C,3041:3096,309D:309F,30A1:30FA,30FC:30FF,3105:312F,3131:318E,31A0:31BF,31F0:31FF,3400:4DBF,4E00:A48C,A4D0:A4FD,A500:A60C,A610:A61F,A62A:A62B,A640:A66E,A67F:A69D,A6A0:A6E5,A717:A71F,A722:A788,A78B:A7CD,A7D0:A7D1,A7D3:A7D3,A7D5:A7DC,A7F2:A801,A803:A805,A807:A80A,A80C:A822,A840:A873,A882:A8B3,A8F2:A8F7,A8FB:A8FB,A8FD:A8FE,A90A:A925,A930:A946,A960:A97C,A984:A9B2,A9CF:A9CF,A9E0:A9E4,A9E6:A9EF,A9FA:A9FE,AA00:AA28,AA40:AA42,AA44:AA4B,AA60:AA76,AA7A:AA7A,AA7E:AAAF,AAB1:AAB1,AAB5:AAB6,AAB9:AABD,AAC0:AAC0,AAC2:AAC2,AADB:AADD,AAE0:AAEA,AAF2:AAF4,AB01:AB06,AB09:AB0E,AB11:AB16,AB20:AB26,AB28:AB2E,AB30:AB5A,AB5C:AB69,AB70:ABE2,AC00:D7A3,D7B0:D7C6,D7CB:D7FB,F900:FA6D,FA70:FAD9,FB00:FB06,FB13:FB17,FB1D:FB1D,FB1F:FB28,FB2A:FB36,FB38:FB3C,FB3E:FB3E,FB40:FB41,FB43:FB44,FB46:FBB1,FBD3:FD3D,FD50:FD8F,FD92:FDC7,FDF0:FDFB,FE70:FE74,FE76:FEFC,FF21:FF3A,FF41:FF5A,FF66:FFBE,FFC2:FFC7,FFCA:FFCF,FFD2:FFD7,FFDA:FFDC,10000:1000B,1000D:10026,10028:1003A,1003C:1003D,1003F:1004D,10050:1005D,10080:100FA,10280:1029C,102A0:102D0,10300:1031F,1032D:10340,10342:10349,10350:10375,10380:1039D,103A0:103C3,103C8:103CF,10400:1049D,104B0:104D3,104D8:104FB,10500:10527,10530:10563,10570:1057A,1057C:1058A,1058C:10592,10594:10595,10597:105A1,105A3:105B1,105B3:105B9,105BB:105BC,105C0:105F3,10600:10736,10740:10755,10760:10767,10780:10785,10787:107B0,107B2:107BA,10800:10805,10808:10808,1080A:10835,10837:10838,1083C:1083C,1083F:10855,10860:10876,10880:1089E,108E0:108F2,108F4:108F5,10900:10915,10920:10939,10980:109B7,109BE:109BF,10A00:10A00,10A10:10A13,10A15:10A17,10A19:10A35,10A60:10A7C,10A80:10A9C,10AC0:10AC7,10AC9:10AE4,10B00:10B35,10B40:10B55,10B60:10B72,10B80:10B91,10C00:10C48,10C80:10CB2,10CC0:10CF2,10D00:10D23,10D4A:10D65,10D6F:10D85,10E80:10EA9,10EB0:10EB1,10EC2:10EC4,10F00:10F1C,10F27:10F27,10F30:10F45,10F70:10F81,10FB0:10FC4,10FE0:10FF6,11003:11037,11071:11072,11075:11075,11083:110AF,110D0:110E8,11103:11126,11144:11144,11147:11147,11150:11172,11176:11176,11183:111B2,111C1:111C4,111DA:111DA,111DC:111DC,11200:11211,11213:1122B,1123F:11240,11280:11286,11288:11288,1128A:1128D,1128F:1129D,1129F:112A8,112B0:112DE,11305:1130C,1130F:11310,11313:11328,1132A:11330,11332:11333,11335:11339,1133D:1133D,11350:11350,1135D:11361,11380:11389,1138B:1138B,1138E:1138E,11390:113B5,113B7:113B7,113D1:113D1,113D3:113D3,11400:11434,11447:1144A,1145F:11461,11480:114AF,114C4:114C5,114C7:114C7,11580:115AE,115D8:115DB,11600:1162F,11644:11644,11680:116AA,116B8:116B8,11700:1171A,11740:11746,11800:1182B,118A0:118DF,118FF:11906,11909:11909,1190C:11913,11915:11916,11918:1192F,1193F:1193F,11941:11941,119A0:119A7,119AA:119D0,119E1:119E1,119E3:119E3,11A00:11A00,11A0B:11A32,11A3A:11A3A,11A50:11A50,11A5C:11A89,11A9D:11A9D,11AB0:11AF8,11BC0:11BE0,11C00:11C08,11C0A:11C2E,11C40:11C40,11C72:11C8F,11D00:11D06,11D08:11D09,11D0B:11D30,11D46:11D46,11D60:11D65,11D67:11D68,11D6A:11D89,11D98:11D98,11EE0:11EF2,11F02:11F02,11F04:11F10,11F12:11F33,11FB0:11FB0,12000:12399,12480:12543,12F90:12FF0,13000:1342F,13441:13446,13460:143FA,14400:14646,16100:1611D,16800:16A38,16A40:16A5E,16A70:16ABE,16AD0:16AED,16B00:16B2F,16B40:16B43,16B63:16B77,16B7D:16B8F,16D40:16D6C,16E40:16E7F,16F00:16F4A,16F50:16F50,16F93:16F9F,16FE0:16FE1,16FE3:16FE3,17000:187F7,18800:18CD5,18CFF:18D08,1AFF0:1AFF3,1AFF5:1AFFB,1AFFD:1AFFE,1B000:1B122,1B132:1B132,1B150:1B152,1B155:1B155,1B164:1B167,1B170:1B2FB,1BC00:1BC6A,1BC70:1BC7C,1BC80:1BC88,1BC90:1BC99,1D400:1D454,1D456:1D49C,1D49E:1D49F,1D4A2:1D4A2,1D4A5:1D4A6,1D4A9:1D4AC,1D4AE:1D4B9,1D4BB:1D4BB,1D4BD:1D4C3,1D4C5:1D505,1D507:1D50A,1D50D:1D514,1D516:1D51C,1D51E:1D539,1D53B:1D53E,1D540:1D544,1D546:1D546,1D54A:1D550,1D552:1D6A5,1D6A8:1D6C0,1D6C2:1D6DA,1D6DC:1D6FA,1D6FC:1D714,1D716:1D734,1D736:1D74E,1D750:1D76E,1D770:1D788,1D78A:1D7A8,1D7AA:1D7C2,1D7C4:1D7CB,1DF00:1DF1E,1DF25:1DF2A,1E030:1E06D,1E100:1E12C,1E137:1E13D,1E14E:1E14E,1E290:1E2AD,1E2C0:1E2EB,1E4D0:1E4EB,1E5D0:1E5ED,1E5F0:1E5F0,1E7E0:1E7E6,1E7E8:1E7EB,1E7ED:1E7EE,1E7F0:1E7FE,1E800:1E8C4,1E900:1E943,1E94B:1E94B,1EE00:1EE03,1EE05:1EE1F,1EE21:1EE22,1EE24:1EE24,1EE27:1EE27,1EE29:1EE32,1EE34:1EE37,1EE39:1EE39,1EE3B:1EE3B,1EE42:1EE42,1EE47:1EE47,1EE49:1EE49,1EE4B:1EE4B,1EE4D:1EE4F,1EE51:1EE52,1EE54:1EE54,1EE57:1EE57,1EE59:1EE59,1EE5B:1EE5B,1EE5D:1EE5D,1EE5F:1EE5F,1EE61:1EE62,1EE64:1EE64,1EE67:1EE6A,1EE6C:1EE72,1EE74:1EE77,1EE79:1EE7C,1EE7E:1EE7E,1EE80:1EE89,1EE8B:1EE9B,1EEA1:1EEA3,1EEA5:1EEA9,1EEAB:1EEBB,20000:2A6DF,2A700:2B739,2B740:2B81D,2B820:2CEA1,2CEB0:2EBE0,2EBF0:2EE5D,2F800:2FA1D,30000:3134A,31350:323AF +isalnum 30:39,41:5A,61:7A,AA:AA,B2:B3,B5:B5,B9:BA,BC:BE,C0:D6,D8:F6,F8:2C1,2C6:2D1,2E0:2E4,2EC:2EC,2EE:2EE,370:374,376:377,37A:37D,37F:37F,386:386,388:38A,38C:38C,38E:3A1,3A3:3F5,3F7:481,48A:52F,531:556,559:559,560:588,5D0:5EA,5EF:5F2,620:64A,660:669,66E:66F,671:6D3,6D5:6D5,6E5:6E6,6EE:6FC,6FF:6FF,710:710,712:72F,74D:7A5,7B1:7B1,7C0:7EA,7F4:7F5,7FA:7FA,800:815,81A:81A,824:824,828:828,840:858,860:86A,870:887,889:88E,8A0:8C9,904:939,93D:93D,950:950,958:961,966:96F,971:980,985:98C,98F:990,993:9A8,9AA:9B0,9B2:9B2,9B6:9B9,9BD:9BD,9CE:9CE,9DC:9DD,9DF:9E1,9E6:9F1,9F4:9F9,9FC:9FC,A05:A0A,A0F:A10,A13:A28,A2A:A30,A32:A33,A35:A36,A38:A39,A59:A5C,A5E:A5E,A66:A6F,A72:A74,A85:A8D,A8F:A91,A93:AA8,AAA:AB0,AB2:AB3,AB5:AB9,ABD:ABD,AD0:AD0,AE0:AE1,AE6:AEF,AF9:AF9,B05:B0C,B0F:B10,B13:B28,B2A:B30,B32:B33,B35:B39,B3D:B3D,B5C:B5D,B5F:B61,B66:B6F,B71:B77,B83:B83,B85:B8A,B8E:B90,B92:B95,B99:B9A,B9C:B9C,B9E:B9F,BA3:BA4,BA8:BAA,BAE:BB9,BD0:BD0,BE6:BF2,C05:C0C,C0E:C10,C12:C28,C2A:C39,C3D:C3D,C58:C5A,C5D:C5D,C60:C61,C66:C6F,C78:C7E,C80:C80,C85:C8C,C8E:C90,C92:CA8,CAA:CB3,CB5:CB9,CBD:CBD,CDD:CDE,CE0:CE1,CE6:CEF,CF1:CF2,D04:D0C,D0E:D10,D12:D3A,D3D:D3D,D4E:D4E,D54:D56,D58:D61,D66:D78,D7A:D7F,D85:D96,D9A:DB1,DB3:DBB,DBD:DBD,DC0:DC6,DE6:DEF,E01:E30,E32:E33,E40:E46,E50:E59,E81:E82,E84:E84,E86:E8A,E8C:EA3,EA5:EA5,EA7:EB0,EB2:EB3,EBD:EBD,EC0:EC4,EC6:EC6,ED0:ED9,EDC:EDF,F00:F00,F20:F33,F40:F47,F49:F6C,F88:F8C,1000:102A,103F:1049,1050:1055,105A:105D,1061:1061,1065:1066,106E:1070,1075:1081,108E:108E,1090:1099,10A0:10C5,10C7:10C7,10CD:10CD,10D0:10FA,10FC:1248,124A:124D,1250:1256,1258:1258,125A:125D,1260:1288,128A:128D,1290:12B0,12B2:12B5,12B8:12BE,12C0:12C0,12C2:12C5,12C8:12D6,12D8:1310,1312:1315,1318:135A,1369:137C,1380:138F,13A0:13F5,13F8:13FD,1401:166C,166F:167F,1681:169A,16A0:16EA,16EE:16F8,1700:1711,171F:1731,1740:1751,1760:176C,176E:1770,1780:17B3,17D7:17D7,17DC:17DC,17E0:17E9,17F0:17F9,1810:1819,1820:1878,1880:1884,1887:18A8,18AA:18AA,18B0:18F5,1900:191E,1946:196D,1970:1974,1980:19AB,19B0:19C9,19D0:19DA,1A00:1A16,1A20:1A54,1A80:1A89,1A90:1A99,1AA7:1AA7,1B05:1B33,1B45:1B4C,1B50:1B59,1B83:1BA0,1BAE:1BE5,1C00:1C23,1C40:1C49,1C4D:1C7D,1C80:1C8A,1C90:1CBA,1CBD:1CBF,1CE9:1CEC,1CEE:1CF3,1CF5:1CF6,1CFA:1CFA,1D00:1DBF,1E00:1F15,1F18:1F1D,1F20:1F45,1F48:1F4D,1F50:1F57,1F59:1F59,1F5B:1F5B,1F5D:1F5D,1F5F:1F7D,1F80:1FB4,1FB6:1FBC,1FBE:1FBE,1FC2:1FC4,1FC6:1FCC,1FD0:1FD3,1FD6:1FDB,1FE0:1FEC,1FF2:1FF4,1FF6:1FFC,2070:2071,2074:2079,207F:2089,2090:209C,2102:2102,2107:2107,210A:2113,2115:2115,2119:211D,2124:2124,2126:2126,2128:2128,212A:212D,212F:2139,213C:213F,2145:2149,214E:214E,2150:2189,2460:249B,24EA:24FF,2776:2793,2C00:2CE4,2CEB:2CEE,2CF2:2CF3,2CFD:2CFD,2D00:2D25,2D27:2D27,2D2D:2D2D,2D30:2D67,2D6F:2D6F,2D80:2D96,2DA0:2DA6,2DA8:2DAE,2DB0:2DB6,2DB8:2DBE,2DC0:2DC6,2DC8:2DCE,2DD0:2DD6,2DD8:2DDE,2E2F:2E2F,3005:3007,3021:3029,3031:3035,3038:303C,3041:3096,309D:309F,30A1:30FA,30FC:30FF,3105:312F,3131:318E,3192:3195,31A0:31BF,31F0:31FF,3220:3229,3248:324F,3251:325F,3280:3289,32B1:32BF,3400:4DBF,4E00:A48C,A4D0:A4FD,A500:A60C,A610:A62B,A640:A66E,A67F:A69D,A6A0:A6EF,A717:A71F,A722:A788,A78B:A7CD,A7D0:A7D1,A7D3:A7D3,A7D5:A7DC,A7F2:A801,A803:A805,A807:A80A,A80C:A822,A830:A835,A840:A873,A882:A8B3,A8D0:A8D9,A8F2:A8F7,A8FB:A8FB,A8FD:A8FE,A900:A925,A930:A946,A960:A97C,A984:A9B2,A9CF:A9D9,A9E0:A9E4,A9E6:A9FE,AA00:AA28,AA40:AA42,AA44:AA4B,AA50:AA59,AA60:AA76,AA7A:AA7A,AA7E:AAAF,AAB1:AAB1,AAB5:AAB6,AAB9:AABD,AAC0:AAC0,AAC2:AAC2,AADB:AADD,AAE0:AAEA,AAF2:AAF4,AB01:AB06,AB09:AB0E,AB11:AB16,AB20:AB26,AB28:AB2E,AB30:AB5A,AB5C:AB69,AB70:ABE2,ABF0:ABF9,AC00:D7A3,D7B0:D7C6,D7CB:D7FB,F900:FA6D,FA70:FAD9,FB00:FB06,FB13:FB17,FB1D:FB1D,FB1F:FB28,FB2A:FB36,FB38:FB3C,FB3E:FB3E,FB40:FB41,FB43:FB44,FB46:FBB1,FBD3:FD3D,FD50:FD8F,FD92:FDC7,FDF0:FDFB,FE70:FE74,FE76:FEFC,FF10:FF19,FF21:FF3A,FF41:FF5A,FF66:FFBE,FFC2:FFC7,FFCA:FFCF,FFD2:FFD7,FFDA:FFDC,10000:1000B,1000D:10026,10028:1003A,1003C:1003D,1003F:1004D,10050:1005D,10080:100FA,10107:10133,10140:10178,1018A:1018B,10280:1029C,102A0:102D0,102E1:102FB,10300:10323,1032D:1034A,10350:10375,10380:1039D,103A0:103C3,103C8:103CF,103D1:103D5,10400:1049D,104A0:104A9,104B0:104D3,104D8:104FB,10500:10527,10530:10563,10570:1057A,1057C:1058A,1058C:10592,10594:10595,10597:105A1,105A3:105B1,105B3:105B9,105BB:105BC,105C0:105F3,10600:10736,10740:10755,10760:10767,10780:10785,10787:107B0,107B2:107BA,10800:10805,10808:10808,1080A:10835,10837:10838,1083C:1083C,1083F:10855,10858:10876,10879:1089E,108A7:108AF,108E0:108F2,108F4:108F5,108FB:1091B,10920:10939,10980:109B7,109BC:109CF,109D2:10A00,10A10:10A13,10A15:10A17,10A19:10A35,10A40:10A48,10A60:10A7E,10A80:10A9F,10AC0:10AC7,10AC9:10AE4,10AEB:10AEF,10B00:10B35,10B40:10B55,10B58:10B72,10B78:10B91,10BA9:10BAF,10C00:10C48,10C80:10CB2,10CC0:10CF2,10CFA:10D23,10D30:10D39,10D40:10D65,10D6F:10D85,10E60:10E7E,10E80:10EA9,10EB0:10EB1,10EC2:10EC4,10F00:10F27,10F30:10F45,10F51:10F54,10F70:10F81,10FB0:10FCB,10FE0:10FF6,11003:11037,11052:1106F,11071:11072,11075:11075,11083:110AF,110D0:110E8,110F0:110F9,11103:11126,11136:1113F,11144:11144,11147:11147,11150:11172,11176:11176,11183:111B2,111C1:111C4,111D0:111DA,111DC:111DC,111E1:111F4,11200:11211,11213:1122B,1123F:11240,11280:11286,11288:11288,1128A:1128D,1128F:1129D,1129F:112A8,112B0:112DE,112F0:112F9,11305:1130C,1130F:11310,11313:11328,1132A:11330,11332:11333,11335:11339,1133D:1133D,11350:11350,1135D:11361,11380:11389,1138B:1138B,1138E:1138E,11390:113B5,113B7:113B7,113D1:113D1,113D3:113D3,11400:11434,11447:1144A,11450:11459,1145F:11461,11480:114AF,114C4:114C5,114C7:114C7,114D0:114D9,11580:115AE,115D8:115DB,11600:1162F,11644:11644,11650:11659,11680:116AA,116B8:116B8,116C0:116C9,116D0:116E3,11700:1171A,11730:1173B,11740:11746,11800:1182B,118A0:118F2,118FF:11906,11909:11909,1190C:11913,11915:11916,11918:1192F,1193F:1193F,11941:11941,11950:11959,119A0:119A7,119AA:119D0,119E1:119E1,119E3:119E3,11A00:11A00,11A0B:11A32,11A3A:11A3A,11A50:11A50,11A5C:11A89,11A9D:11A9D,11AB0:11AF8,11BC0:11BE0,11BF0:11BF9,11C00:11C08,11C0A:11C2E,11C40:11C40,11C50:11C6C,11C72:11C8F,11D00:11D06,11D08:11D09,11D0B:11D30,11D46:11D46,11D50:11D59,11D60:11D65,11D67:11D68,11D6A:11D89,11D98:11D98,11DA0:11DA9,11EE0:11EF2,11F02:11F02,11F04:11F10,11F12:11F33,11F50:11F59,11FB0:11FB0,11FC0:11FD4,12000:12399,12400:1246E,12480:12543,12F90:12FF0,13000:1342F,13441:13446,13460:143FA,14400:14646,16100:1611D,16130:16139,16800:16A38,16A40:16A5E,16A60:16A69,16A70:16ABE,16AC0:16AC9,16AD0:16AED,16B00:16B2F,16B40:16B43,16B50:16B59,16B5B:16B61,16B63:16B77,16B7D:16B8F,16D40:16D6C,16D70:16D79,16E40:16E96,16F00:16F4A,16F50:16F50,16F93:16F9F,16FE0:16FE1,16FE3:16FE3,17000:187F7,18800:18CD5,18CFF:18D08,1AFF0:1AFF3,1AFF5:1AFFB,1AFFD:1AFFE,1B000:1B122,1B132:1B132,1B150:1B152,1B155:1B155,1B164:1B167,1B170:1B2FB,1BC00:1BC6A,1BC70:1BC7C,1BC80:1BC88,1BC90:1BC99,1CCF0:1CCF9,1D2C0:1D2D3,1D2E0:1D2F3,1D360:1D378,1D400:1D454,1D456:1D49C,1D49E:1D49F,1D4A2:1D4A2,1D4A5:1D4A6,1D4A9:1D4AC,1D4AE:1D4B9,1D4BB:1D4BB,1D4BD:1D4C3,1D4C5:1D505,1D507:1D50A,1D50D:1D514,1D516:1D51C,1D51E:1D539,1D53B:1D53E,1D540:1D544,1D546:1D546,1D54A:1D550,1D552:1D6A5,1D6A8:1D6C0,1D6C2:1D6DA,1D6DC:1D6FA,1D6FC:1D714,1D716:1D734,1D736:1D74E,1D750:1D76E,1D770:1D788,1D78A:1D7A8,1D7AA:1D7C2,1D7C4:1D7CB,1D7CE:1D7FF,1DF00:1DF1E,1DF25:1DF2A,1E030:1E06D,1E100:1E12C,1E137:1E13D,1E140:1E149,1E14E:1E14E,1E290:1E2AD,1E2C0:1E2EB,1E2F0:1E2F9,1E4D0:1E4EB,1E4F0:1E4F9,1E5D0:1E5ED,1E5F0:1E5FA,1E7E0:1E7E6,1E7E8:1E7EB,1E7ED:1E7EE,1E7F0:1E7FE,1E800:1E8C4,1E8C7:1E8CF,1E900:1E943,1E94B:1E94B,1E950:1E959,1EC71:1ECAB,1ECAD:1ECAF,1ECB1:1ECB4,1ED01:1ED2D,1ED2F:1ED3D,1EE00:1EE03,1EE05:1EE1F,1EE21:1EE22,1EE24:1EE24,1EE27:1EE27,1EE29:1EE32,1EE34:1EE37,1EE39:1EE39,1EE3B:1EE3B,1EE42:1EE42,1EE47:1EE47,1EE49:1EE49,1EE4B:1EE4B,1EE4D:1EE4F,1EE51:1EE52,1EE54:1EE54,1EE57:1EE57,1EE59:1EE59,1EE5B:1EE5B,1EE5D:1EE5D,1EE5F:1EE5F,1EE61:1EE62,1EE64:1EE64,1EE67:1EE6A,1EE6C:1EE72,1EE74:1EE77,1EE79:1EE7C,1EE7E:1EE7E,1EE80:1EE89,1EE8B:1EE9B,1EEA1:1EEA3,1EEA5:1EEA9,1EEAB:1EEBB,1F100:1F10C,1FBF0:1FBF9,20000:2A6DF,2A700:2B739,2B740:2B81D,2B820:2CEA1,2CEB0:2EBE0,2EBF0:2EE5D,2F800:2FA1D,30000:3134A,31350:323AF +isdecimal 30:39,660:669,6F0:6F9,7C0:7C9,966:96F,9E6:9EF,A66:A6F,AE6:AEF,B66:B6F,BE6:BEF,C66:C6F,CE6:CEF,D66:D6F,DE6:DEF,E50:E59,ED0:ED9,F20:F29,1040:1049,1090:1099,17E0:17E9,1810:1819,1946:194F,19D0:19D9,1A80:1A89,1A90:1A99,1B50:1B59,1BB0:1BB9,1C40:1C49,1C50:1C59,A620:A629,A8D0:A8D9,A900:A909,A9D0:A9D9,A9F0:A9F9,AA50:AA59,ABF0:ABF9,FF10:FF19,104A0:104A9,10D30:10D39,10D40:10D49,11066:1106F,110F0:110F9,11136:1113F,111D0:111D9,112F0:112F9,11450:11459,114D0:114D9,11650:11659,116C0:116C9,116D0:116E3,11730:11739,118E0:118E9,11950:11959,11BF0:11BF9,11C50:11C59,11D50:11D59,11DA0:11DA9,11F50:11F59,16130:16139,16A60:16A69,16AC0:16AC9,16B50:16B59,16D70:16D79,1CCF0:1CCF9,1D7CE:1D7FF,1E140:1E149,1E2F0:1E2F9,1E4F0:1E4F9,1E5F1:1E5FA,1E950:1E959,1FBF0:1FBF9 +isdigit 30:39,B2:B3,B9:B9,660:669,6F0:6F9,7C0:7C9,966:96F,9E6:9EF,A66:A6F,AE6:AEF,B66:B6F,BE6:BEF,C66:C6F,CE6:CEF,D66:D6F,DE6:DEF,E50:E59,ED0:ED9,F20:F29,1040:1049,1090:1099,1369:1371,17E0:17E9,1810:1819,1946:194F,19D0:19DA,1A80:1A89,1A90:1A99,1B50:1B59,1BB0:1BB9,1C40:1C49,1C50:1C59,2070:2070,2074:2079,2080:2089,2460:2468,2474:247C,2488:2490,24EA:24EA,24F5:24FD,24FF:24FF,2776:277E,2780:2788,278A:2792,A620:A629,A8D0:A8D9,A900:A909,A9D0:A9D9,A9F0:A9F9,AA50:AA59,ABF0:ABF9,FF10:FF19,104A0:104A9,10A40:10A43,10D30:10D39,10D40:10D49,10E60:10E68,11052:1105A,11066:1106F,110F0:110F9,11136:1113F,111D0:111D9,112F0:112F9,11450:11459,114D0:114D9,11650:11659,116C0:116C9,116D0:116E3,11730:11739,118E0:118E9,11950:11959,11BF0:11BF9,11C50:11C59,11D50:11D59,11DA0:11DA9,11F50:11F59,16130:16139,16A60:16A69,16AC0:16AC9,16B50:16B59,16D70:16D79,1CCF0:1CCF9,1D7CE:1D7FF,1E140:1E149,1E2F0:1E2F9,1E4F0:1E4F9,1E5F1:1E5FA,1E950:1E959,1F100:1F10A,1FBF0:1FBF9 +isnumeric 30:39,B2:B3,B9:B9,BC:BE,660:669,6F0:6F9,7C0:7C9,966:96F,9E6:9EF,9F4:9F9,A66:A6F,AE6:AEF,B66:B6F,B72:B77,BE6:BF2,C66:C6F,C78:C7E,CE6:CEF,D58:D5E,D66:D78,DE6:DEF,E50:E59,ED0:ED9,F20:F33,1040:1049,1090:1099,1369:137C,16EE:16F0,17E0:17E9,17F0:17F9,1810:1819,1946:194F,19D0:19DA,1A80:1A89,1A90:1A99,1B50:1B59,1BB0:1BB9,1C40:1C49,1C50:1C59,2070:2070,2074:2079,2080:2089,2150:2182,2185:2189,2460:249B,24EA:24FF,2776:2793,2CFD:2CFD,3007:3007,3021:3029,3038:303A,3192:3195,3220:3229,3248:324F,3251:325F,3280:3289,32B1:32BF,3405:3405,3483:3483,382A:382A,3B4D:3B4D,4E00:4E00,4E03:4E03,4E07:4E07,4E09:4E09,4E24:4E24,4E5D:4E5D,4E8C:4E8C,4E94:4E94,4E96:4E96,4EAC:4EAC,4EBF:4EC0,4EDF:4EDF,4EE8:4EE8,4F0D:4F0D,4F70:4F70,4FE9:4FE9,5006:5006,5104:5104,5146:5146,5169:5169,516B:516B,516D:516D,5341:5341,5343:5345,534C:534C,53C1:53C4,56DB:56DB,58F1:58F1,58F9:58F9,5E7A:5E7A,5EFE:5EFF,5F0C:5F0E,5F10:5F10,62D0:62D0,62FE:62FE,634C:634C,67D2:67D2,6D1E:6D1E,6F06:6F06,7396:7396,767E:767E,7695:7695,79ED:79ED,8086:8086,842C:842C,8CAE:8CAE,8CB3:8CB3,8D30:8D30,920E:920E,94A9:94A9,9621:9621,9646:9646,964C:964C,9678:9678,96F6:96F6,A620:A629,A6E6:A6EF,A830:A835,A8D0:A8D9,A900:A909,A9D0:A9D9,A9F0:A9F9,AA50:AA59,ABF0:ABF9,F96B:F96B,F973:F973,F978:F978,F9B2:F9B2,F9D1:F9D1,F9D3:F9D3,F9FD:F9FD,FF10:FF19,10107:10133,10140:10178,1018A:1018B,102E1:102FB,10320:10323,10341:10341,1034A:1034A,103D1:103D5,104A0:104A9,10858:1085F,10879:1087F,108A7:108AF,108FB:108FF,10916:1091B,109BC:109BD,109C0:109CF,109D2:109FF,10A40:10A48,10A7D:10A7E,10A9D:10A9F,10AEB:10AEF,10B58:10B5F,10B78:10B7F,10BA9:10BAF,10CFA:10CFF,10D30:10D39,10D40:10D49,10E60:10E7E,10F1D:10F26,10F51:10F54,10FC5:10FCB,11052:1106F,110F0:110F9,11136:1113F,111D0:111D9,111E1:111F4,112F0:112F9,11450:11459,114D0:114D9,11650:11659,116C0:116C9,116D0:116E3,11730:1173B,118E0:118F2,11950:11959,11BF0:11BF9,11C50:11C6C,11D50:11D59,11DA0:11DA9,11F50:11F59,11FC0:11FD4,12400:1246E,16130:16139,16A60:16A69,16AC0:16AC9,16B50:16B59,16B5B:16B61,16D70:16D79,16E80:16E96,1CCF0:1CCF9,1D2C0:1D2D3,1D2E0:1D2F3,1D360:1D378,1D7CE:1D7FF,1E140:1E149,1E2F0:1E2F9,1E4F0:1E4F9,1E5F1:1E5FA,1E8C7:1E8CF,1E950:1E959,1EC71:1ECAB,1ECAD:1ECAF,1ECB1:1ECB4,1ED01:1ED2D,1ED2F:1ED3D,1F100:1F10C,1FBF0:1FBF9,20001:20001,20064:20064,200E2:200E2,20121:20121,2092A:2092A,20983:20983,2098C:2098C,2099C:2099C,20AEA:20AEA,20AFD:20AFD,20B19:20B19,22390:22390,22998:22998,23B1B:23B1B,2626D:2626D,2F890:2F890 +isspace 9:D,1C:20,85:85,A0:A0,1680:1680,2000:200A,2028:2029,202F:202F,205F:205F,3000:3000 +isprintable 20:7E,A1:AC,AE:377,37A:37F,384:38A,38C:38C,38E:3A1,3A3:52F,531:556,559:58A,58D:58F,591:5C7,5D0:5EA,5EF:5F4,606:61B,61D:6DC,6DE:70D,710:74A,74D:7B1,7C0:7FA,7FD:82D,830:83E,840:85B,85E:85E,860:86A,870:88E,897:8E1,8E3:983,985:98C,98F:990,993:9A8,9AA:9B0,9B2:9B2,9B6:9B9,9BC:9C4,9C7:9C8,9CB:9CE,9D7:9D7,9DC:9DD,9DF:9E3,9E6:9FE,A01:A03,A05:A0A,A0F:A10,A13:A28,A2A:A30,A32:A33,A35:A36,A38:A39,A3C:A3C,A3E:A42,A47:A48,A4B:A4D,A51:A51,A59:A5C,A5E:A5E,A66:A76,A81:A83,A85:A8D,A8F:A91,A93:AA8,AAA:AB0,AB2:AB3,AB5:AB9,ABC:AC5,AC7:AC9,ACB:ACD,AD0:AD0,AE0:AE3,AE6:AF1,AF9:AFF,B01:B03,B05:B0C,B0F:B10,B13:B28,B2A:B30,B32:B33,B35:B39,B3C:B44,B47:B48,B4B:B4D,B55:B57,B5C:B5D,B5F:B63,B66:B77,B82:B83,B85:B8A,B8E:B90,B92:B95,B99:B9A,B9C:B9C,B9E:B9F,BA3:BA4,BA8:BAA,BAE:BB9,BBE:BC2,BC6:BC8,BCA:BCD,BD0:BD0,BD7:BD7,BE6:BFA,C00:C0C,C0E:C10,C12:C28,C2A:C39,C3C:C44,C46:C48,C4A:C4D,C55:C56,C58:C5A,C5D:C5D,C60:C63,C66:C6F,C77:C8C,C8E:C90,C92:CA8,CAA:CB3,CB5:CB9,CBC:CC4,CC6:CC8,CCA:CCD,CD5:CD6,CDD:CDE,CE0:CE3,CE6:CEF,CF1:CF3,D00:D0C,D0E:D10,D12:D44,D46:D48,D4A:D4F,D54:D63,D66:D7F,D81:D83,D85:D96,D9A:DB1,DB3:DBB,DBD:DBD,DC0:DC6,DCA:DCA,DCF:DD4,DD6:DD6,DD8:DDF,DE6:DEF,DF2:DF4,E01:E3A,E3F:E5B,E81:E82,E84:E84,E86:E8A,E8C:EA3,EA5:EA5,EA7:EBD,EC0:EC4,EC6:EC6,EC8:ECE,ED0:ED9,EDC:EDF,F00:F47,F49:F6C,F71:F97,F99:FBC,FBE:FCC,FCE:FDA,1000:10C5,10C7:10C7,10CD:10CD,10D0:1248,124A:124D,1250:1256,1258:1258,125A:125D,1260:1288,128A:128D,1290:12B0,12B2:12B5,12B8:12BE,12C0:12C0,12C2:12C5,12C8:12D6,12D8:1310,1312:1315,1318:135A,135D:137C,1380:1399,13A0:13F5,13F8:13FD,1400:167F,1681:169C,16A0:16F8,1700:1715,171F:1736,1740:1753,1760:176C,176E:1770,1772:1773,1780:17DD,17E0:17E9,17F0:17F9,1800:180D,180F:1819,1820:1878,1880:18AA,18B0:18F5,1900:191E,1920:192B,1930:193B,1940:1940,1944:196D,1970:1974,1980:19AB,19B0:19C9,19D0:19DA,19DE:1A1B,1A1E:1A5E,1A60:1A7C,1A7F:1A89,1A90:1A99,1AA0:1AAD,1AB0:1ACE,1B00:1B4C,1B4E:1BF3,1BFC:1C37,1C3B:1C49,1C4D:1C8A,1C90:1CBA,1CBD:1CC7,1CD0:1CFA,1D00:1F15,1F18:1F1D,1F20:1F45,1F48:1F4D,1F50:1F57,1F59:1F59,1F5B:1F5B,1F5D:1F5D,1F5F:1F7D,1F80:1FB4,1FB6:1FC4,1FC6:1FD3,1FD6:1FDB,1FDD:1FEF,1FF2:1FF4,1FF6:1FFE,2010:2027,2030:205E,2070:2071,2074:208E,2090:209C,20A0:20C0,20D0:20F0,2100:218B,2190:2429,2440:244A,2460:2B73,2B76:2B95,2B97:2CF3,2CF9:2D25,2D27:2D27,2D2D:2D2D,2D30:2D67,2D6F:2D70,2D7F:2D96,2DA0:2DA6,2DA8:2DAE,2DB0:2DB6,2DB8:2DBE,2DC0:2DC6,2DC8:2DCE,2DD0:2DD6,2DD8:2DDE,2DE0:2E5D,2E80:2E99,2E9B:2EF3,2F00:2FD5,2FF0:2FFF,3001:303F,3041:3096,3099:30FF,3105:312F,3131:318E,3190:31E5,31EF:321E,3220:A48C,A490:A4C6,A4D0:A62B,A640:A6F7,A700:A7CD,A7D0:A7D1,A7D3:A7D3,A7D5:A7DC,A7F2:A82C,A830:A839,A840:A877,A880:A8C5,A8CE:A8D9,A8E0:A953,A95F:A97C,A980:A9CD,A9CF:A9D9,A9DE:A9FE,AA00:AA36,AA40:AA4D,AA50:AA59,AA5C:AAC2,AADB:AAF6,AB01:AB06,AB09:AB0E,AB11:AB16,AB20:AB26,AB28:AB2E,AB30:AB6B,AB70:ABED,ABF0:ABF9,AC00:D7A3,D7B0:D7C6,D7CB:D7FB,F900:FA6D,FA70:FAD9,FB00:FB06,FB13:FB17,FB1D:FB36,FB38:FB3C,FB3E:FB3E,FB40:FB41,FB43:FB44,FB46:FBC2,FBD3:FD8F,FD92:FDC7,FDCF:FDCF,FDF0:FE19,FE20:FE52,FE54:FE66,FE68:FE6B,FE70:FE74,FE76:FEFC,FF01:FFBE,FFC2:FFC7,FFCA:FFCF,FFD2:FFD7,FFDA:FFDC,FFE0:FFE6,FFE8:FFEE,FFFC:FFFD,10000:1000B,1000D:10026,10028:1003A,1003C:1003D,1003F:1004D,10050:1005D,10080:100FA,10100:10102,10107:10133,10137:1018E,10190:1019C,101A0:101A0,101D0:101FD,10280:1029C,102A0:102D0,102E0:102FB,10300:10323,1032D:1034A,10350:1037A,10380:1039D,1039F:103C3,103C8:103D5,10400:1049D,104A0:104A9,104B0:104D3,104D8:104FB,10500:10527,10530:10563,1056F:1057A,1057C:1058A,1058C:10592,10594:10595,10597:105A1,105A3:105B1,105B3:105B9,105BB:105BC,105C0:105F3,10600:10736,10740:10755,10760:10767,10780:10785,10787:107B0,107B2:107BA,10800:10805,10808:10808,1080A:10835,10837:10838,1083C:1083C,1083F:10855,10857:1089E,108A7:108AF,108E0:108F2,108F4:108F5,108FB:1091B,1091F:10939,1093F:1093F,10980:109B7,109BC:109CF,109D2:10A03,10A05:10A06,10A0C:10A13,10A15:10A17,10A19:10A35,10A38:10A3A,10A3F:10A48,10A50:10A58,10A60:10A9F,10AC0:10AE6,10AEB:10AF6,10B00:10B35,10B39:10B55,10B58:10B72,10B78:10B91,10B99:10B9C,10BA9:10BAF,10C00:10C48,10C80:10CB2,10CC0:10CF2,10CFA:10D27,10D30:10D39,10D40:10D65,10D69:10D85,10D8E:10D8F,10E60:10E7E,10E80:10EA9,10EAB:10EAD,10EB0:10EB1,10EC2:10EC4,10EFC:10F27,10F30:10F59,10F70:10F89,10FB0:10FCB,10FE0:10FF6,11000:1104D,11052:11075,1107F:110BC,110BE:110C2,110D0:110E8,110F0:110F9,11100:11134,11136:11147,11150:11176,11180:111DF,111E1:111F4,11200:11211,11213:11241,11280:11286,11288:11288,1128A:1128D,1128F:1129D,1129F:112A9,112B0:112EA,112F0:112F9,11300:11303,11305:1130C,1130F:11310,11313:11328,1132A:11330,11332:11333,11335:11339,1133B:11344,11347:11348,1134B:1134D,11350:11350,11357:11357,1135D:11363,11366:1136C,11370:11374,11380:11389,1138B:1138B,1138E:1138E,11390:113B5,113B7:113C0,113C2:113C2,113C5:113C5,113C7:113CA,113CC:113D5,113D7:113D8,113E1:113E2,11400:1145B,1145D:11461,11480:114C7,114D0:114D9,11580:115B5,115B8:115DD,11600:11644,11650:11659,11660:1166C,11680:116B9,116C0:116C9,116D0:116E3,11700:1171A,1171D:1172B,11730:11746,11800:1183B,118A0:118F2,118FF:11906,11909:11909,1190C:11913,11915:11916,11918:11935,11937:11938,1193B:11946,11950:11959,119A0:119A7,119AA:119D7,119DA:119E4,11A00:11A47,11A50:11AA2,11AB0:11AF8,11B00:11B09,11BC0:11BE1,11BF0:11BF9,11C00:11C08,11C0A:11C36,11C38:11C45,11C50:11C6C,11C70:11C8F,11C92:11CA7,11CA9:11CB6,11D00:11D06,11D08:11D09,11D0B:11D36,11D3A:11D3A,11D3C:11D3D,11D3F:11D47,11D50:11D59,11D60:11D65,11D67:11D68,11D6A:11D8E,11D90:11D91,11D93:11D98,11DA0:11DA9,11EE0:11EF8,11F00:11F10,11F12:11F3A,11F3E:11F5A,11FB0:11FB0,11FC0:11FF1,11FFF:12399,12400:1246E,12470:12474,12480:12543,12F90:12FF2,13000:1342F,13440:13455,13460:143FA,14400:14646,16100:16139,16800:16A38,16A40:16A5E,16A60:16A69,16A6E:16ABE,16AC0:16AC9,16AD0:16AED,16AF0:16AF5,16B00:16B45,16B50:16B59,16B5B:16B61,16B63:16B77,16B7D:16B8F,16D40:16D79,16E40:16E9A,16F00:16F4A,16F4F:16F87,16F8F:16F9F,16FE0:16FE4,16FF0:16FF1,17000:187F7,18800:18CD5,18CFF:18D08,1AFF0:1AFF3,1AFF5:1AFFB,1AFFD:1AFFE,1B000:1B122,1B132:1B132,1B150:1B152,1B155:1B155,1B164:1B167,1B170:1B2FB,1BC00:1BC6A,1BC70:1BC7C,1BC80:1BC88,1BC90:1BC99,1BC9C:1BC9F,1CC00:1CCF9,1CD00:1CEB3,1CF00:1CF2D,1CF30:1CF46,1CF50:1CFC3,1D000:1D0F5,1D100:1D126,1D129:1D172,1D17B:1D1EA,1D200:1D245,1D2C0:1D2D3,1D2E0:1D2F3,1D300:1D356,1D360:1D378,1D400:1D454,1D456:1D49C,1D49E:1D49F,1D4A2:1D4A2,1D4A5:1D4A6,1D4A9:1D4AC,1D4AE:1D4B9,1D4BB:1D4BB,1D4BD:1D4C3,1D4C5:1D505,1D507:1D50A,1D50D:1D514,1D516:1D51C,1D51E:1D539,1D53B:1D53E,1D540:1D544,1D546:1D546,1D54A:1D550,1D552:1D6A5,1D6A8:1D7CB,1D7CE:1DA8B,1DA9B:1DA9F,1DAA1:1DAAF,1DF00:1DF1E,1DF25:1DF2A,1E000:1E006,1E008:1E018,1E01B:1E021,1E023:1E024,1E026:1E02A,1E030:1E06D,1E08F:1E08F,1E100:1E12C,1E130:1E13D,1E140:1E149,1E14E:1E14F,1E290:1E2AE,1E2C0:1E2F9,1E2FF:1E2FF,1E4D0:1E4F9,1E5D0:1E5FA,1E5FF:1E5FF,1E7E0:1E7E6,1E7E8:1E7EB,1E7ED:1E7EE,1E7F0:1E7FE,1E800:1E8C4,1E8C7:1E8D6,1E900:1E94B,1E950:1E959,1E95E:1E95F,1EC71:1ECB4,1ED01:1ED3D,1EE00:1EE03,1EE05:1EE1F,1EE21:1EE22,1EE24:1EE24,1EE27:1EE27,1EE29:1EE32,1EE34:1EE37,1EE39:1EE39,1EE3B:1EE3B,1EE42:1EE42,1EE47:1EE47,1EE49:1EE49,1EE4B:1EE4B,1EE4D:1EE4F,1EE51:1EE52,1EE54:1EE54,1EE57:1EE57,1EE59:1EE59,1EE5B:1EE5B,1EE5D:1EE5D,1EE5F:1EE5F,1EE61:1EE62,1EE64:1EE64,1EE67:1EE6A,1EE6C:1EE72,1EE74:1EE77,1EE79:1EE7C,1EE7E:1EE7E,1EE80:1EE89,1EE8B:1EE9B,1EEA1:1EEA3,1EEA5:1EEA9,1EEAB:1EEBB,1EEF0:1EEF1,1F000:1F02B,1F030:1F093,1F0A0:1F0AE,1F0B1:1F0BF,1F0C1:1F0CF,1F0D1:1F0F5,1F100:1F1AD,1F1E6:1F202,1F210:1F23B,1F240:1F248,1F250:1F251,1F260:1F265,1F300:1F6D7,1F6DC:1F6EC,1F6F0:1F6FC,1F700:1F776,1F77B:1F7D9,1F7E0:1F7EB,1F7F0:1F7F0,1F800:1F80B,1F810:1F847,1F850:1F859,1F860:1F887,1F890:1F8AD,1F8B0:1F8BB,1F8C0:1F8C1,1F900:1FA53,1FA60:1FA6D,1FA70:1FA7C,1FA80:1FA89,1FA8F:1FAC6,1FACE:1FADC,1FADF:1FAE9,1FAF0:1FAF8,1FB00:1FB92,1FB94:1FBF9,20000:2A6DF,2A700:2B739,2B740:2B81D,2B820:2CEA1,2CEB0:2EBE0,2EBF0:2EE5D,2F800:2FA1D,30000:3134A,31350:323AF,E0100:E01EF +isidentifier 41:5A,5F:5F,61:7A,AA:AA,B5:B5,BA:BA,C0:D6,D8:F6,F8:2C1,2C6:2D1,2E0:2E4,2EC:2EC,2EE:2EE,370:374,376:377,37B:37D,37F:37F,386:386,388:38A,38C:38C,38E:3A1,3A3:3F5,3F7:481,48A:52F,531:556,559:559,560:588,5D0:5EA,5EF:5F2,620:64A,66E:66F,671:6D3,6D5:6D5,6E5:6E6,6EE:6EF,6FA:6FC,6FF:6FF,710:710,712:72F,74D:7A5,7B1:7B1,7CA:7EA,7F4:7F5,7FA:7FA,800:815,81A:81A,824:824,828:828,840:858,860:86A,870:887,889:88E,8A0:8C9,904:939,93D:93D,950:950,958:961,971:980,985:98C,98F:990,993:9A8,9AA:9B0,9B2:9B2,9B6:9B9,9BD:9BD,9CE:9CE,9DC:9DD,9DF:9E1,9F0:9F1,9FC:9FC,A05:A0A,A0F:A10,A13:A28,A2A:A30,A32:A33,A35:A36,A38:A39,A59:A5C,A5E:A5E,A72:A74,A85:A8D,A8F:A91,A93:AA8,AAA:AB0,AB2:AB3,AB5:AB9,ABD:ABD,AD0:AD0,AE0:AE1,AF9:AF9,B05:B0C,B0F:B10,B13:B28,B2A:B30,B32:B33,B35:B39,B3D:B3D,B5C:B5D,B5F:B61,B71:B71,B83:B83,B85:B8A,B8E:B90,B92:B95,B99:B9A,B9C:B9C,B9E:B9F,BA3:BA4,BA8:BAA,BAE:BB9,BD0:BD0,C05:C0C,C0E:C10,C12:C28,C2A:C39,C3D:C3D,C58:C5A,C5D:C5D,C60:C61,C80:C80,C85:C8C,C8E:C90,C92:CA8,CAA:CB3,CB5:CB9,CBD:CBD,CDD:CDE,CE0:CE1,CF1:CF2,D04:D0C,D0E:D10,D12:D3A,D3D:D3D,D4E:D4E,D54:D56,D5F:D61,D7A:D7F,D85:D96,D9A:DB1,DB3:DBB,DBD:DBD,DC0:DC6,E01:E30,E32:E32,E40:E46,E81:E82,E84:E84,E86:E8A,E8C:EA3,EA5:EA5,EA7:EB0,EB2:EB2,EBD:EBD,EC0:EC4,EC6:EC6,EDC:EDF,F00:F00,F40:F47,F49:F6C,F88:F8C,1000:102A,103F:103F,1050:1055,105A:105D,1061:1061,1065:1066,106E:1070,1075:1081,108E:108E,10A0:10C5,10C7:10C7,10CD:10CD,10D0:10FA,10FC:1248,124A:124D,1250:1256,1258:1258,125A:125D,1260:1288,128A:128D,1290:12B0,12B2:12B5,12B8:12BE,12C0:12C0,12C2:12C5,12C8:12D6,12D8:1310,1312:1315,1318:135A,1380:138F,13A0:13F5,13F8:13FD,1401:166C,166F:167F,1681:169A,16A0:16EA,16EE:16F8,1700:1711,171F:1731,1740:1751,1760:176C,176E:1770,1780:17B3,17D7:17D7,17DC:17DC,1820:1878,1880:18A8,18AA:18AA,18B0:18F5,1900:191E,1950:196D,1970:1974,1980:19AB,19B0:19C9,1A00:1A16,1A20:1A54,1AA7:1AA7,1B05:1B33,1B45:1B4C,1B83:1BA0,1BAE:1BAF,1BBA:1BE5,1C00:1C23,1C4D:1C4F,1C5A:1C7D,1C80:1C8A,1C90:1CBA,1CBD:1CBF,1CE9:1CEC,1CEE:1CF3,1CF5:1CF6,1CFA:1CFA,1D00:1DBF,1E00:1F15,1F18:1F1D,1F20:1F45,1F48:1F4D,1F50:1F57,1F59:1F59,1F5B:1F5B,1F5D:1F5D,1F5F:1F7D,1F80:1FB4,1FB6:1FBC,1FBE:1FBE,1FC2:1FC4,1FC6:1FCC,1FD0:1FD3,1FD6:1FDB,1FE0:1FEC,1FF2:1FF4,1FF6:1FFC,2071:2071,207F:207F,2090:209C,2102:2102,2107:2107,210A:2113,2115:2115,2118:211D,2124:2124,2126:2126,2128:2128,212A:2139,213C:213F,2145:2149,214E:214E,2160:2188,2C00:2CE4,2CEB:2CEE,2CF2:2CF3,2D00:2D25,2D27:2D27,2D2D:2D2D,2D30:2D67,2D6F:2D6F,2D80:2D96,2DA0:2DA6,2DA8:2DAE,2DB0:2DB6,2DB8:2DBE,2DC0:2DC6,2DC8:2DCE,2DD0:2DD6,2DD8:2DDE,3005:3007,3021:3029,3031:3035,3038:303C,3041:3096,309D:309F,30A1:30FA,30FC:30FF,3105:312F,3131:318E,31A0:31BF,31F0:31FF,3400:4DBF,4E00:A48C,A4D0:A4FD,A500:A60C,A610:A61F,A62A:A62B,A640:A66E,A67F:A69D,A6A0:A6EF,A717:A71F,A722:A788,A78B:A7CD,A7D0:A7D1,A7D3:A7D3,A7D5:A7DC,A7F2:A801,A803:A805,A807:A80A,A80C:A822,A840:A873,A882:A8B3,A8F2:A8F7,A8FB:A8FB,A8FD:A8FE,A90A:A925,A930:A946,A960:A97C,A984:A9B2,A9CF:A9CF,A9E0:A9E4,A9E6:A9EF,A9FA:A9FE,AA00:AA28,AA40:AA42,AA44:AA4B,AA60:AA76,AA7A:AA7A,AA7E:AAAF,AAB1:AAB1,AAB5:AAB6,AAB9:AABD,AAC0:AAC0,AAC2:AAC2,AADB:AADD,AAE0:AAEA,AAF2:AAF4,AB01:AB06,AB09:AB0E,AB11:AB16,AB20:AB26,AB28:AB2E,AB30:AB5A,AB5C:AB69,AB70:ABE2,AC00:D7A3,D7B0:D7C6,D7CB:D7FB,F900:FA6D,FA70:FAD9,FB00:FB06,FB13:FB17,FB1D:FB1D,FB1F:FB28,FB2A:FB36,FB38:FB3C,FB3E:FB3E,FB40:FB41,FB43:FB44,FB46:FBB1,FBD3:FC5D,FC64:FD3D,FD50:FD8F,FD92:FDC7,FDF0:FDF9,FE71:FE71,FE73:FE73,FE77:FE77,FE79:FE79,FE7B:FE7B,FE7D:FE7D,FE7F:FEFC,FF21:FF3A,FF41:FF5A,FF66:FF9D,FFA0:FFBE,FFC2:FFC7,FFCA:FFCF,FFD2:FFD7,FFDA:FFDC,10000:1000B,1000D:10026,10028:1003A,1003C:1003D,1003F:1004D,10050:1005D,10080:100FA,10140:10174,10280:1029C,102A0:102D0,10300:1031F,1032D:1034A,10350:10375,10380:1039D,103A0:103C3,103C8:103CF,103D1:103D5,10400:1049D,104B0:104D3,104D8:104FB,10500:10527,10530:10563,10570:1057A,1057C:1058A,1058C:10592,10594:10595,10597:105A1,105A3:105B1,105B3:105B9,105BB:105BC,105C0:105F3,10600:10736,10740:10755,10760:10767,10780:10785,10787:107B0,107B2:107BA,10800:10805,10808:10808,1080A:10835,10837:10838,1083C:1083C,1083F:10855,10860:10876,10880:1089E,108E0:108F2,108F4:108F5,10900:10915,10920:10939,10980:109B7,109BE:109BF,10A00:10A00,10A10:10A13,10A15:10A17,10A19:10A35,10A60:10A7C,10A80:10A9C,10AC0:10AC7,10AC9:10AE4,10B00:10B35,10B40:10B55,10B60:10B72,10B80:10B91,10C00:10C48,10C80:10CB2,10CC0:10CF2,10D00:10D23,10D4A:10D65,10D6F:10D85,10E80:10EA9,10EB0:10EB1,10EC2:10EC4,10F00:10F1C,10F27:10F27,10F30:10F45,10F70:10F81,10FB0:10FC4,10FE0:10FF6,11003:11037,11071:11072,11075:11075,11083:110AF,110D0:110E8,11103:11126,11144:11144,11147:11147,11150:11172,11176:11176,11183:111B2,111C1:111C4,111DA:111DA,111DC:111DC,11200:11211,11213:1122B,1123F:11240,11280:11286,11288:11288,1128A:1128D,1128F:1129D,1129F:112A8,112B0:112DE,11305:1130C,1130F:11310,11313:11328,1132A:11330,11332:11333,11335:11339,1133D:1133D,11350:11350,1135D:11361,11380:11389,1138B:1138B,1138E:1138E,11390:113B5,113B7:113B7,113D1:113D1,113D3:113D3,11400:11434,11447:1144A,1145F:11461,11480:114AF,114C4:114C5,114C7:114C7,11580:115AE,115D8:115DB,11600:1162F,11644:11644,11680:116AA,116B8:116B8,11700:1171A,11740:11746,11800:1182B,118A0:118DF,118FF:11906,11909:11909,1190C:11913,11915:11916,11918:1192F,1193F:1193F,11941:11941,119A0:119A7,119AA:119D0,119E1:119E1,119E3:119E3,11A00:11A00,11A0B:11A32,11A3A:11A3A,11A50:11A50,11A5C:11A89,11A9D:11A9D,11AB0:11AF8,11BC0:11BE0,11C00:11C08,11C0A:11C2E,11C40:11C40,11C72:11C8F,11D00:11D06,11D08:11D09,11D0B:11D30,11D46:11D46,11D60:11D65,11D67:11D68,11D6A:11D89,11D98:11D98,11EE0:11EF2,11F02:11F02,11F04:11F10,11F12:11F33,11FB0:11FB0,12000:12399,12400:1246E,12480:12543,12F90:12FF0,13000:1342F,13441:13446,13460:143FA,14400:14646,16100:1611D,16800:16A38,16A40:16A5E,16A70:16ABE,16AD0:16AED,16B00:16B2F,16B40:16B43,16B63:16B77,16B7D:16B8F,16D40:16D6C,16E40:16E7F,16F00:16F4A,16F50:16F50,16F93:16F9F,16FE0:16FE1,16FE3:16FE3,17000:187F7,18800:18CD5,18CFF:18D08,1AFF0:1AFF3,1AFF5:1AFFB,1AFFD:1AFFE,1B000:1B122,1B132:1B132,1B150:1B152,1B155:1B155,1B164:1B167,1B170:1B2FB,1BC00:1BC6A,1BC70:1BC7C,1BC80:1BC88,1BC90:1BC99,1D400:1D454,1D456:1D49C,1D49E:1D49F,1D4A2:1D4A2,1D4A5:1D4A6,1D4A9:1D4AC,1D4AE:1D4B9,1D4BB:1D4BB,1D4BD:1D4C3,1D4C5:1D505,1D507:1D50A,1D50D:1D514,1D516:1D51C,1D51E:1D539,1D53B:1D53E,1D540:1D544,1D546:1D546,1D54A:1D550,1D552:1D6A5,1D6A8:1D6C0,1D6C2:1D6DA,1D6DC:1D6FA,1D6FC:1D714,1D716:1D734,1D736:1D74E,1D750:1D76E,1D770:1D788,1D78A:1D7A8,1D7AA:1D7C2,1D7C4:1D7CB,1DF00:1DF1E,1DF25:1DF2A,1E030:1E06D,1E100:1E12C,1E137:1E13D,1E14E:1E14E,1E290:1E2AD,1E2C0:1E2EB,1E4D0:1E4EB,1E5D0:1E5ED,1E5F0:1E5F0,1E7E0:1E7E6,1E7E8:1E7EB,1E7ED:1E7EE,1E7F0:1E7FE,1E800:1E8C4,1E900:1E943,1E94B:1E94B,1EE00:1EE03,1EE05:1EE1F,1EE21:1EE22,1EE24:1EE24,1EE27:1EE27,1EE29:1EE32,1EE34:1EE37,1EE39:1EE39,1EE3B:1EE3B,1EE42:1EE42,1EE47:1EE47,1EE49:1EE49,1EE4B:1EE4B,1EE4D:1EE4F,1EE51:1EE52,1EE54:1EE54,1EE57:1EE57,1EE59:1EE59,1EE5B:1EE5B,1EE5D:1EE5D,1EE5F:1EE5F,1EE61:1EE62,1EE64:1EE64,1EE67:1EE6A,1EE6C:1EE72,1EE74:1EE77,1EE79:1EE7C,1EE7E:1EE7E,1EE80:1EE89,1EE8B:1EE9B,1EEA1:1EEA3,1EEA5:1EEA9,1EEAB:1EEBB,20000:2A6DF,2A700:2B739,2B740:2B81D,2B820:2CEA1,2CEB0:2EBE0,2EBF0:2EE5D,2F800:2FA1D,30000:3134A,31350:323AF diff --git a/crates/unicode/tests/data/version_skew_cpython3.14.txt b/crates/unicode/tests/data/version_skew_cpython3.14.txt new file mode 100644 index 00000000000..acdb26089ba --- /dev/null +++ b/crates/unicode/tests/data/version_skew_cpython3.14.txt @@ -0,0 +1,11 @@ +# Code points whose classification differs between CPython 3.14 (Unicode 16.0.0) +# and the Rust std / icu4x build used here (a later Unicode release assigns them). +# Regenerate with RUSTPYTHON_UNICODE_REGEN_SKEW=1. +# Format: `predicate start:end,...` with inclusive hex ranges. +isalnum 88F:88F,C5C:C5C,CDC:CDC,A7CE:A7CF,A7D2:A7D2,A7D4:A7D4,A7F1:A7F1,10940:10959,10EC5:10EC7,11DB0:11DDB,11DE0:11DE9,16EA0:16EB8,16EBB:16ED3,16FF2:16FF6,187F8:187FF,18D09:18D1E,18D80:18DF2,1E6C0:1E6DE,1E6E0:1E6E2,1E6E4:1E6E5,1E6E7:1E6ED,1E6F0:1E6F4,1E6FE:1E6FF,2B73A:2B73F,2CEA2:2CEAD,323B0:33479 +isalpha 88F:88F,C5C:C5C,CDC:CDC,A7CE:A7CF,A7D2:A7D2,A7D4:A7D4,A7F1:A7F1,10940:10959,10EC5:10EC7,11DB0:11DDB,16EA0:16EB8,16EBB:16ED3,16FF2:16FF3,187F8:187FF,18D09:18D1E,18D80:18DF2,1E6C0:1E6DE,1E6E0:1E6E2,1E6E4:1E6E5,1E6E7:1E6ED,1E6F0:1E6F4,1E6FE:1E6FF,2B73A:2B73F,2CEA2:2CEAD,323B0:33479 +isdecimal 11DE0:11DE9 +isdigit 11DE0:11DE9 +isidentifier 88F:88F,C5C:C5C,CDC:CDC,A7CE:A7CF,A7D2:A7D2,A7D4:A7D4,A7F1:A7F1,10940:10959,10EC5:10EC7,11DB0:11DDB,16EA0:16EB8,16EBB:16ED3,16FF2:16FF6,187F8:187FF,18D09:18D1E,18D80:18DF2,1E6C0:1E6DE,1E6E0:1E6E2,1E6E4:1E6E5,1E6E7:1E6ED,1E6F0:1E6F4,1E6FE:1E6FF,2B73A:2B73F,2CEA2:2CEAD,323B0:33479 +isnumeric 11DE0:11DE9,12038:12039,12079:12079,12226:12226,1222B:1222B,1230B:1230B,1230D:1230D,12399:12399,16FF4:16FF6 +isprintable 88F:88F,C5C:C5C,CDC:CDC,1ACF:1ADD,1AE0:1AEB,20C1:20C1,2B96:2B96,A7CE:A7CF,A7D2:A7D2,A7D4:A7D4,A7F1:A7F1,FBC3:FBD2,FD90:FD91,FDC8:FDCE,10940:10959,10EC5:10EC7,10ED0:10ED8,10EFA:10EFB,11B60:11B67,11DB0:11DDB,11DE0:11DE9,16EA0:16EB8,16EBB:16ED3,16FF2:16FF6,187F8:187FF,18D09:18D1E,18D80:18DF2,1CCFA:1CCFC,1CEBA:1CED0,1CEE0:1CEF0,1E6C0:1E6DE,1E6E0:1E6F5,1E6FE:1E6FF,1F6D8:1F6D8,1F777:1F77A,1F8D0:1F8D8,1FA54:1FA57,1FA8A:1FA8A,1FA8E:1FA8E,1FAC8:1FAC8,1FACD:1FACD,1FAEA:1FAEA,1FAEF:1FAEF,1FBFA:1FBFA,2B73A:2B73F,2CEA2:2CEAD,323B0:33479 diff --git a/crates/unicode/tests/differential.rs b/crates/unicode/tests/differential.rs new file mode 100644 index 00000000000..ba73d3149da --- /dev/null +++ b/crates/unicode/tests/differential.rs @@ -0,0 +1,236 @@ +#![allow( + clippy::std_instead_of_alloc, + clippy::tests_outside_test_module, + reason = "integration test target always links std and its functions are the test entry points" +)] + +//! Differential sweep of the classification predicates over the full scalar +//! range `0..0x110000` against a committed CPython reference dataset. +//! +//! CPython 3.14 ships Unicode 16.0.0 while the Rust standard library / icu4x +//! build used here may be a later release. Code points whose classification +//! changed between those Unicode versions are expected to differ; those are +//! recorded in `data/version_skew_cpython3.14.txt` as an explicit allow-list. +//! Any divergence outside that list fails the test — a real regression, not a +//! version bump. +//! +//! Both data files use the same run-length format: one `predicate` line per +//! str method, followed by comma-separated hex `start:end` inclusive ranges. + +use std::collections::{BTreeMap, BTreeSet}; + +use rustpython_unicode::classify; + +const MAX: u32 = 0x110000; +const REFERENCE: &str = include_str!("data/cpython3.14_predicates.txt"); +const VERSION_SKEW: &str = include_str!("data/version_skew_cpython3.14.txt"); + +fn crate_predicate(name: &str, cp: u32) -> bool { + let Some(c) = char::from_u32(cp) else { + // Lone surrogates are not scalars; every str predicate is false. + return false; + }; + match name { + "isalpha" => classify::is_alpha(c), + "isalnum" => classify::is_alnum(c), + "isdecimal" => classify::is_decimal(c), + "isdigit" => classify::is_digit(c), + "isnumeric" => classify::is_numeric(c), + "isspace" => classify::is_space(c), + "isprintable" => classify::is_printable(c), + "isidentifier" => { + // str.isidentifier is a whole-string predicate; for a single char it + // is "may start an identifier". + classify_is_identifier_char(c) + } + other => panic!("unknown predicate {other}"), + } +} + +fn classify_is_identifier_char(c: char) -> bool { + rustpython_unicode::identifier::is_start(c) +} + +/// Parse a `name -> sorted set of code points` map from a run-length file. +/// +/// Each non-comment line is `predicate start:end,start:end,...` with inclusive +/// hex ranges; a predicate with no members is a bare `predicate`. +fn parse_ranges(text: &str) -> BTreeMap> { + let mut map = BTreeMap::new(); + for line in text.lines() { + let line = line.trim(); + if line.is_empty() || line.starts_with('#') { + continue; + } + let (name, packed) = match line.split_once(' ') { + Some((name, packed)) => (name, packed.trim()), + None => (line, ""), + }; + let mut set = BTreeSet::new(); + if !packed.is_empty() { + for run in packed.split(',') { + let (s, e) = run.split_once(':').expect("run is start:end"); + let start = u32::from_str_radix(s, 16).unwrap(); + let end = u32::from_str_radix(e, 16).unwrap(); + for cp in start..=end { + set.insert(cp); + } + } + } + map.insert(name.to_string(), set); + } + map +} + +/// Collapse a sorted code-point set into inclusive `start:end` runs. +fn encode_ranges(set: &BTreeSet) -> String { + let mut runs = Vec::new(); + let mut iter = set.iter().copied(); + if let Some(first) = iter.next() { + let (mut start, mut end) = (first, first); + for cp in iter { + if cp == end + 1 { + end = cp; + } else { + runs.push((start, end)); + start = cp; + end = cp; + } + } + runs.push((start, end)); + } + runs.iter() + .map(|(s, e)| format!("{s:X}:{e:X}")) + .collect::>() + .join(",") +} + +/// Recompute the full divergence set. Every entry is a `(predicate, code +/// point)` where the crate and the CPython reference disagree. +fn all_divergences(reference: &BTreeMap>) -> Vec<(String, u32, bool)> { + let mut out = Vec::new(); + for (name, truth) in reference { + for cp in 0..MAX { + let expected = truth.contains(&cp); + let actual = crate_predicate(name, cp); + if expected != actual { + out.push((name.clone(), cp, expected)); + } + } + } + out +} + +/// Regenerate `data/version_skew_cpython3.14.txt` from the current toolchain. +/// +/// Run with `RUSTPYTHON_UNICODE_REGEN_SKEW=1 cargo test -p rustpython-unicode +/// --test differential` after bumping the Rust/icu toolchain. All divergences +/// must be one-directional (crate=true, cpython=false) — newly-assigned code +/// points from a later Unicode release. A `cpython=true, crate=false` entry +/// means a code point lost a property, which is a real regression, so this +/// refuses to record it. +#[test] +fn regen_version_skew() { + if std::env::var_os("RUSTPYTHON_UNICODE_REGEN_SKEW").is_none() { + return; + } + let reference = parse_ranges(REFERENCE); + let divergences = all_divergences(&reference); + + let regressions: Vec<_> = divergences + .iter() + .filter(|(_, _, expected)| *expected) + .collect(); + assert!( + regressions.is_empty(), + "refusing to record {} cpython=true/crate=false divergence(s) — these are \ + regressions, not version skew: {:?}", + regressions.len(), + ®ressions[..regressions.len().min(20)] + ); + + let mut by_predicate: BTreeMap> = BTreeMap::new(); + for (name, cp, _) in &divergences { + by_predicate.entry(name.clone()).or_default().insert(*cp); + } + + let mut body = String::from( + "# Code points whose classification differs between CPython 3.14 (Unicode 16.0.0)\n\ + # and the Rust std / icu4x build used here (a later Unicode release assigns them).\n\ + # Regenerate with RUSTPYTHON_UNICODE_REGEN_SKEW=1.\n\ + # Format: `predicate start:end,...` with inclusive hex ranges.\n", + ); + for (name, set) in &by_predicate { + body.push_str(&format!("{name} {}\n", encode_ranges(set))); + } + let path = concat!( + env!("CARGO_MANIFEST_DIR"), + "/tests/data/version_skew_cpython3.14.txt" + ); + std::fs::write(path, body).unwrap(); + eprintln!( + "wrote {} skew code points across {} predicates to {path}", + divergences.len(), + by_predicate.len() + ); +} + +#[test] +fn predicates_match_cpython_except_documented_version_skew() { + let reference = parse_ranges(REFERENCE); + let skew = parse_ranges(VERSION_SKEW); + + let allowed = |name: &str, cp: u32| skew.get(name).is_some_and(|set| set.contains(&cp)); + + let mut unexpected: Vec<(String, u32, bool, bool)> = Vec::new(); + + for (name, truth) in &reference { + for cp in 0..MAX { + let expected = truth.contains(&cp); + let actual = crate_predicate(name, cp); + if expected != actual && !allowed(name, cp) { + unexpected.push((name.clone(), cp, expected, actual)); + } + } + } + + // Also flag stale allow-list entries: code points that no longer diverge. + let mut stale: Vec<(String, u32)> = Vec::new(); + for (name, set) in &skew { + let Some(truth) = reference.get(name) else { + continue; + }; + for &cp in set { + let expected = truth.contains(&cp); + let actual = crate_predicate(name, cp); + if expected == actual { + stale.push((name.clone(), cp)); + } + } + } + + if !unexpected.is_empty() || !stale.is_empty() { + let mut msg = String::new(); + if !unexpected.is_empty() { + msg.push_str(&format!( + "{} undocumented divergence(s) from CPython:\n", + unexpected.len() + )); + for (name, cp, expected, actual) in unexpected.iter().take(50) { + msg.push_str(&format!( + " {name} U+{cp:04X}: cpython={expected} crate={actual}\n" + )); + } + } + if !stale.is_empty() { + msg.push_str(&format!( + "{} stale version_skew_cpython3.14.txt entries that now agree:\n", + stale.len() + )); + for (name, cp) in stale.iter().take(50) { + msg.push_str(&format!(" {name} U+{cp:04X}\n")); + } + } + panic!("{msg}"); + } +} diff --git a/crates/unicode/tests/generate_reference.py b/crates/unicode/tests/generate_reference.py new file mode 100644 index 00000000000..c7b21e66fc4 --- /dev/null +++ b/crates/unicode/tests/generate_reference.py @@ -0,0 +1,74 @@ +#!/usr/bin/env python3.14 +"""Generate the CPython reference dataset for the differential Unicode sweep. + +Run with a CPython interpreter whose ``unicodedata.unidata_version`` matches the +Unicode release this crate targets (16.0.0 for CPython 3.14). The output is a +compact run-length encoding of every predicate's true-set over the full scalar +range, consumed by ``tests/differential.rs``. + +Usage: + python3.14 crates/unicode/tests/generate_reference.py + +Writes ``tests/data/cpython3.14_predicates.txt``. Commit the result. +""" + +from __future__ import annotations + +import pathlib +import sys +import unicodedata + +MAX = 0x110000 + +# str predicates: name -> single-char method. +STR_PREDICATES = { + "isalpha": str.isalpha, + "isalnum": str.isalnum, + "isdecimal": str.isdecimal, + "isdigit": str.isdigit, + "isnumeric": str.isnumeric, + "isspace": str.isspace, + "isprintable": str.isprintable, + "isidentifier": str.isidentifier, +} + + +def encode_ranges(is_true) -> list[tuple[int, int]]: + """Collapse the true-set of ``is_true`` into inclusive ``[start, end]`` runs.""" + ranges: list[tuple[int, int]] = [] + start: int | None = None + for cp in range(MAX): + if is_true(cp): + if start is None: + start = cp + elif start is not None: + ranges.append((start, cp - 1)) + start = None + if start is not None: + ranges.append((start, MAX - 1)) + return ranges + + +def main() -> int: + if unicodedata.unidata_version != "16.0.0": + sys.stderr.write( + f"warning: unidata_version is {unicodedata.unidata_version}, " + "expected 16.0.0 (CPython 3.14); regenerating anyway\n" + ) + + out = pathlib.Path(__file__).parent / "data" / "cpython3.14_predicates.txt" + out.parent.mkdir(parents=True, exist_ok=True) + + lines = [f"# unidata_version {unicodedata.unidata_version}"] + for name, method in STR_PREDICATES.items(): + ranges = encode_ranges(lambda cp, m=method: m(chr(cp))) + packed = ",".join(f"{s:X}:{e:X}" for s, e in ranges) + lines.append(f"{name} {packed}") + + out.write_text("\n".join(lines) + "\n") + print(f"wrote {out} ({out.stat().st_size} bytes)") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/extra_tests/snippets/stdlib_unicode_shared.py b/extra_tests/snippets/stdlib_unicode_shared.py new file mode 100644 index 00000000000..dd46231e5b1 --- /dev/null +++ b/extra_tests/snippets/stdlib_unicode_shared.py @@ -0,0 +1,89 @@ +# Exercises the Unicode semantics routed through the shared rustpython-unicode +# crate: str predicates, casefold, identifier rules, unicodedata queries, +# normalization, \N{} escapes, and re character classes. + +import re +import unicodedata + +# --- str classification predicates --------------------------------------- + +# Numeric_Type chain: isdecimal ⊂ isdigit ⊂ isnumeric +assert "5".isdecimal() and "5".isdigit() and "5".isnumeric() +assert not "²".isdecimal() # SUPERSCRIPT TWO: digit but not decimal +assert "²".isdigit() and "²".isnumeric() +assert not "⅓".isdigit() # VULGAR FRACTION ONE THIRD: numeric only +assert "⅓".isnumeric() + +assert "abc".isalpha() +assert "abc123".isalnum() +assert not "abc123".isalpha() +assert "あ".isalpha() # HIRAGANA LETTER A + +assert " \t\n".isspace() +assert "　".isspace() # IDEOGRAPHIC SPACE +assert "hello world".isprintable() +assert not "\x00".isprintable() +assert " ".isprintable() # ASCII space is printable + +# identifier rules (XID_Start / XID_Continue, plus leading underscore) +assert "_var".isidentifier() +assert "유니코드".isidentifier() # Hangul identifier +assert not "1abc".isidentifier() +assert not "a b".isidentifier() + +# --- case mapping / casefold --------------------------------------------- + +assert "ABC".lower() == "abc" +assert "abc".upper() == "ABC" +# casefold uses full mappings, unlike lower() +assert "ß".casefold() == "ss" # LATIN SMALL LETTER SHARP S +assert "Σ".casefold() == "σ" # GREEK CAPITAL SIGMA -> small sigma +assert "Straße".casefold() == "strasse" + +# lone-surrogate safety: casefold must not panic on surrogates +surrogate = "\ud800" +assert surrogate.casefold() == surrogate + +# --- unicodedata ---------------------------------------------------------- + +assert unicodedata.category("A") == "Lu" +assert unicodedata.category("1") == "Nd" +assert unicodedata.bidirectional("A") == "L" +assert unicodedata.decimal("٥") == 5 # ARABIC-INDIC DIGIT FIVE +assert unicodedata.digit("²") == 2 +assert abs(unicodedata.numeric("⅓") - (1 / 3)) < 1e-6 +assert unicodedata.name("☃") == "SNOWMAN" +assert unicodedata.lookup("SNOWMAN") == "☃" +assert unicodedata.combining("́") == 230 # COMBINING ACUTE ACCENT +assert unicodedata.mirrored("(") == 1 +assert unicodedata.east_asian_width("あ") == "W" + +# ucd_3_2_0 legacy view (used by stringprep) +assert unicodedata.ucd_3_2_0.unidata_version == "3.2.0" + +# --- normalization -------------------------------------------------------- + +composed = "é" # é +decomposed = "é" +assert unicodedata.normalize("NFC", decomposed) == composed +assert unicodedata.normalize("NFD", composed) == decomposed +assert unicodedata.is_normalized("NFC", composed) +assert not unicodedata.is_normalized("NFD", composed) + +# --- \N{} escapes (compiler) --------------------------------------------- + +assert "\N{SNOWMAN}" == "☃" +assert "\N{GREEK SMALL LETTER ALPHA}" == "α" + +# --- re character classes ------------------------------------------------- + +assert re.fullmatch(r"\w+", "abc_123") is not None +assert re.fullmatch(r"\w+", "유니코드") is not None # \w is Unicode-aware +assert re.fullmatch(r"\d+", "123") is not None +assert re.fullmatch(r"\s+", " \t\n") is not None +# ASCII flag restricts \w to ASCII +assert re.fullmatch(r"\w+", "유", re.ASCII) is None +# case-insensitive matching routes through the shared case helpers +assert re.fullmatch(r"straße", "STRAßE", re.IGNORECASE) is not None + +print("stdlib_unicode_shared: OK") From 9fa5456b83ba860270ca29ad0966cb5a9659ad03 Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Mon, 6 Jul 2026 02:12:41 +0900 Subject: [PATCH 6/9] Match regex \d to Unicode decimal digits MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit is_uni_digit previously matched only ASCII 0-9, so re's \d in Unicode mode missed decimal digits like ٥ and ५. SRE_UNI_IS_DIGIT matches Py_UNICODE_ISDECIMAL (category Nd), so route it through classify::is_decimal. Unmasks test_bug_6561 in test_re and extends the shared-crate snippet with the Nd/Nl/No cases. Assisted-by: Claude --- Lib/test/test_pkgutil.py | 1 - Lib/test/test_re.py | 1 - crates/unicode/src/regex.rs | 5 +++-- extra_tests/snippets/stdlib_unicode_shared.py | 4 ++++ 4 files changed, 7 insertions(+), 4 deletions(-) diff --git a/Lib/test/test_pkgutil.py b/Lib/test/test_pkgutil.py index f5444409593..d4faaaeca00 100644 --- a/Lib/test/test_pkgutil.py +++ b/Lib/test/test_pkgutil.py @@ -231,7 +231,6 @@ def test_walk_packages_raises_on_string_or_bytes_input(self): with self.assertRaises((TypeError, ValueError)): list(pkgutil.walk_packages(bytes_input)) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_name_resolution(self): import logging import logging.handlers diff --git a/Lib/test/test_re.py b/Lib/test/test_re.py index 8ac6daecc32..1d396e4f31c 100644 --- a/Lib/test/test_re.py +++ b/Lib/test/test_re.py @@ -1734,7 +1734,6 @@ def test_bug_817234(self): self.assertEqual(next(iter).span(), (4, 4)) self.assertRaises(StopIteration, next, iter) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_bug_6561(self): # '\d' should match characters in Unicode category 'Nd' # (Number, Decimal Digit), but not those in 'Nl' (Number, diff --git a/crates/unicode/src/regex.rs b/crates/unicode/src/regex.rs index 595f5e6f2f1..0dcc7dbb392 100644 --- a/crates/unicode/src/regex.rs +++ b/crates/unicode/src/regex.rs @@ -63,8 +63,9 @@ pub fn upper_locate(ch: u32) -> u32 { #[must_use] pub fn is_uni_digit(ch: u32) -> bool { - // TODO: check with cpython - char::try_from(ch).is_ok_and(|x| x.is_ascii_digit()) + // SRE_UNI_IS_DIGIT matches Unicode decimal digits (Py_UNICODE_ISDECIMAL), + // not just ASCII 0-9. + char::try_from(ch).is_ok_and(classify::is_decimal) } #[must_use] diff --git a/extra_tests/snippets/stdlib_unicode_shared.py b/extra_tests/snippets/stdlib_unicode_shared.py index dd46231e5b1..ff8bc0533e0 100644 --- a/extra_tests/snippets/stdlib_unicode_shared.py +++ b/extra_tests/snippets/stdlib_unicode_shared.py @@ -80,6 +80,10 @@ assert re.fullmatch(r"\w+", "abc_123") is not None assert re.fullmatch(r"\w+", "유니코드") is not None # \w is Unicode-aware assert re.fullmatch(r"\d+", "123") is not None +# \d matches Unicode decimal digits (category Nd), not just ASCII +assert re.fullmatch(r"\d", "٥") is not None # ARABIC-INDIC DIGIT FIVE +assert re.fullmatch(r"\d", "५") is not None # DEVANAGARI DIGIT FIVE +assert re.fullmatch(r"\d", "²") is None # SUPERSCRIPT TWO (No), not decimal assert re.fullmatch(r"\s+", " \t\n") is not None # ASCII flag restricts \w to ASCII assert re.fullmatch(r"\w+", "유", re.ASCII) is None From 96f822736bc64777d09afa3dc03662993df85a74 Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Mon, 6 Jul 2026 09:56:59 +0900 Subject: [PATCH 7/9] Replace shallow Unicode wrappers with re-exports Address review feedback on the extraction. Functions that only forwarded to the shared crate become `use ... as` re-exports rather than hand-written wrappers: - literal: is_printable is dropped; escape.rs calls rustpython_unicode::classify::is_repr_printable directly. - unicode::identifier::is_continue re-exports is_xid_continue. - unicode::data::lookup_character re-exports unicode_names2::character. - Drop the unused repr(u8) on DecompositionType. The SRE character-class and case predicates move back into sre_engine::string (their pre-extraction home) instead of living in a unicode::regex module that only sre_engine and the vm _sre binding used; is_uni_digit/is_uni_alnum still delegate to rustpython_unicode::classify. engine.rs and _sre consume them from sre_engine::string again. The unicode crate's tests move from a single root tests module in lib.rs into per-module test submodules (case, classify, data, identifier, normalize). Assisted-by: Claude --- crates/literal/src/char.rs | 12 --- crates/literal/src/escape.rs | 4 +- crates/literal/src/lib.rs | 1 - crates/sre_engine/src/string.rs | 105 +++++++++++++++++++---- crates/unicode/src/case.rs | 12 +++ crates/unicode/src/classify.rs | 23 ++++++ crates/unicode/src/data.rs | 36 ++++++-- crates/unicode/src/identifier.rs | 16 +++- crates/unicode/src/lib.rs | 79 ------------------ crates/unicode/src/normalize.rs | 16 ++++ crates/unicode/src/regex.rs | 138 ------------------------------- 11 files changed, 185 insertions(+), 257 deletions(-) delete mode 100644 crates/literal/src/char.rs delete mode 100644 crates/unicode/src/regex.rs diff --git a/crates/literal/src/char.rs b/crates/literal/src/char.rs deleted file mode 100644 index a23e41eec83..00000000000 --- a/crates/literal/src/char.rs +++ /dev/null @@ -1,12 +0,0 @@ -/// According to python following categories aren't printable: -/// * Cc (Other, Control) -/// * Cf (Other, Format) -/// * Cs (Other, Surrogate) -/// * Co (Other, Private Use) -/// * Cn (Other, Not Assigned) -/// * Zl Separator, Line ('\u2028', LINE SEPARATOR) -/// * Zp Separator, Paragraph ('\u2029', PARAGRAPH SEPARATOR) -/// * Zs (Separator, Space) other than ASCII space('\x20'). -pub fn is_printable(c: char) -> bool { - rustpython_unicode::classify::is_repr_printable(c) -} diff --git a/crates/literal/src/escape.rs b/crates/literal/src/escape.rs index 1099c0a02bc..50dce8b264c 100644 --- a/crates/literal/src/escape.rs +++ b/crates/literal/src/escape.rs @@ -204,7 +204,7 @@ impl UnicodeEscape<'_> { '\\' | '\t' | '\r' | '\n' => 2, ch if ch < ' ' || ch as u32 == 0x7f => 4, // \xHH ch if ch.is_ascii() => 1, - ch if crate::char::is_printable(ch) => { + ch if rustpython_unicode::classify::is_repr_printable(ch) => { // max = std::cmp::max(ch, max); ch.len_utf8() } @@ -238,7 +238,7 @@ impl UnicodeEscape<'_> { ch if ch.is_ascii() => { write!(formatter, "\\x{:02x}", ch as u8) } - ch if crate::char::is_printable(ch) => formatter.write_char(ch), + ch if rustpython_unicode::classify::is_repr_printable(ch) => formatter.write_char(ch), '\0'..='\u{ff}' => { write!(formatter, "\\x{:02x}", ch as u32) } diff --git a/crates/literal/src/lib.rs b/crates/literal/src/lib.rs index a863dd87738..6d520900142 100644 --- a/crates/literal/src/lib.rs +++ b/crates/literal/src/lib.rs @@ -2,7 +2,6 @@ extern crate alloc; -pub mod char; pub mod complex; pub mod escape; pub mod float; diff --git a/crates/sre_engine/src/string.rs b/crates/sre_engine/src/string.rs index 1038607a2c0..6468c6d0cfd 100644 --- a/crates/sre_engine/src/string.rs +++ b/crates/sre_engine/src/string.rs @@ -1,4 +1,3 @@ -use rustpython_unicode as unicode; use rustpython_wtf8::Wtf8; #[derive(Debug, Clone, Copy)] @@ -333,70 +332,142 @@ const fn utf8_is_cont_byte(byte: u8) -> bool { /// Mask of the value bits of a continuation byte. const CONT_MASK: u8 = 0b0011_1111; +// Character-class and case predicates for the SRE engine. +// +// Every predicate takes a raw `u32` code point (SRE decodes strings into `u32`s, +// including lone surrogates) and returns whether it belongs to the class. +// ASCII-mode predicates only ever consider byte values; Unicode-mode predicates +// consult the shared property tables in `rustpython_unicode::classify`. + +const UNDERSCORE: u32 = '_' as u32; + +const fn is_py_ascii_whitespace(b: u8) -> bool { + matches!(b, b'\t' | b'\n' | b'\x0C' | b'\r' | b' ' | b'\x0B') +} + #[inline] pub(crate) fn is_word(ch: u32) -> bool { - unicode::regex::is_word(ch) + ch == UNDERSCORE || u8::try_from(ch).is_ok_and(|x| x.is_ascii_alphanumeric()) } + #[inline] pub(crate) fn is_space(ch: u32) -> bool { - unicode::regex::is_space(ch) + u8::try_from(ch).is_ok_and(is_py_ascii_whitespace) } + #[inline] pub(crate) fn is_digit(ch: u32) -> bool { - unicode::regex::is_digit(ch) + u8::try_from(ch).is_ok_and(|x| x.is_ascii_digit()) } + #[inline] pub(crate) fn is_loc_alnum(ch: u32) -> bool { - unicode::regex::is_loc_alnum(ch) + // FIXME: Ignore the locales + u8::try_from(ch).is_ok_and(|x| x.is_ascii_alphanumeric()) } + #[inline] pub(crate) fn is_loc_word(ch: u32) -> bool { - ch == '_' as u32 || is_loc_alnum(ch) + ch == UNDERSCORE || is_loc_alnum(ch) } + #[inline] pub(crate) const fn is_linebreak(ch: u32) -> bool { - unicode::regex::is_linebreak(ch) + ch == '\n' as u32 } + #[inline] #[must_use] pub fn lower_ascii(ch: u32) -> u32 { - unicode::regex::lower_ascii(ch) + u8::try_from(ch).map_or(ch, |x| x.to_ascii_lowercase() as u32) } + #[inline] pub(crate) fn lower_locate(ch: u32) -> u32 { - unicode::regex::lower_locate(ch) + // FIXME: Ignore the locales + lower_ascii(ch) } + #[inline] pub(crate) fn upper_locate(ch: u32) -> u32 { - unicode::regex::upper_locate(ch) + // FIXME: Ignore the locales + u8::try_from(ch).map_or(ch, |x| x.to_ascii_uppercase() as u32) } + #[inline] pub(crate) fn is_uni_digit(ch: u32) -> bool { - unicode::regex::is_uni_digit(ch) + // SRE_UNI_IS_DIGIT matches Unicode decimal digits (Py_UNICODE_ISDECIMAL), + // not just ASCII 0-9. + char::try_from(ch).is_ok_and(rustpython_unicode::classify::is_decimal) } + #[inline] pub(crate) fn is_uni_space(ch: u32) -> bool { - unicode::regex::is_uni_space(ch) + // TODO: check with cpython + is_space(ch) + || matches!( + ch, + 0x0009 + | 0x000A + | 0x000B + | 0x000C + | 0x000D + | 0x001C + | 0x001D + | 0x001E + | 0x001F + | 0x0020 + | 0x0085 + | 0x00A0 + | 0x1680 + | 0x2000 + | 0x2001 + | 0x2002 + | 0x2003 + | 0x2004 + | 0x2005 + | 0x2006 + | 0x2007 + | 0x2008 + | 0x2009 + | 0x200A + | 0x2028 + | 0x2029 + | 0x202F + | 0x205F + | 0x3000 + ) } + #[inline] pub(crate) const fn is_uni_linebreak(ch: u32) -> bool { - unicode::regex::is_uni_linebreak(ch) + matches!( + ch, + 0x000A | 0x000B | 0x000C | 0x000D | 0x001C | 0x001D | 0x001E | 0x0085 | 0x2028 | 0x2029 + ) } + #[inline] pub(crate) fn is_uni_alnum(ch: u32) -> bool { - unicode::regex::is_uni_alnum(ch) + // TODO: check with cpython + char::try_from(ch).is_ok_and(rustpython_unicode::classify::is_alnum) } + #[inline] pub(crate) fn is_uni_word(ch: u32) -> bool { - ch == '_' as u32 || is_uni_alnum(ch) + ch == UNDERSCORE || is_uni_alnum(ch) } + #[inline] #[must_use] pub fn lower_unicode(ch: u32) -> u32 { - unicode::regex::lower_unicode(ch) + // TODO: check with cpython + char::try_from(ch).map_or(ch, |x| x.to_lowercase().next().unwrap() as u32) } + #[inline] #[must_use] pub fn upper_unicode(ch: u32) -> u32 { - unicode::regex::upper_unicode(ch) + // TODO: check with cpython + char::try_from(ch).map_or(ch, |x| x.to_uppercase().next().unwrap() as u32) } diff --git a/crates/unicode/src/case.rs b/crates/unicode/src/case.rs index 0e38eea11dc..872d0b29ac1 100644 --- a/crates/unicode/src/case.rs +++ b/crates/unicode/src/case.rs @@ -55,3 +55,15 @@ impl core::fmt::Write for FmtWriter<'_> { Ok(()) } } + +#[cfg(test)] +mod tests { + use super::casefold_str; + + #[test] + fn casefold_full_mappings() { + // ß case-folds to "ss" + assert_eq!(casefold_str("ß"), "ss"); + assert_eq!(casefold_str("Σ"), "σ"); + } +} diff --git a/crates/unicode/src/classify.rs b/crates/unicode/src/classify.rs index dd5b54db942..7a333763a57 100644 --- a/crates/unicode/src/classify.rs +++ b/crates/unicode/src/classify.rs @@ -91,3 +91,26 @@ pub fn is_repr_printable(c: char) -> bool { | GeneralCategory::Unassigned ) } + +#[cfg(test)] +mod tests { + use super::{is_decimal, is_digit, is_numeric}; + + #[test] + fn numeric_type_chain_holds() { + // isdecimal ⊂ isdigit ⊂ isnumeric + for c in ('\0'..='\u{2FFFF}').filter_map(|c| char::from_u32(c as u32)) { + if is_decimal(c) { + assert!(is_digit(c), "{c:?} decimal but not digit"); + } + if is_digit(c) { + assert!(is_numeric(c), "{c:?} digit but not numeric"); + } + } + assert!(is_decimal('5')); + assert!(!is_decimal('²')); + assert!(is_digit('²')); + assert!(!is_digit('⅓')); + assert!(is_numeric('⅓')); + } +} diff --git a/crates/unicode/src/data.rs b/crates/unicode/src/data.rs index 7d437968ea0..df0a9002bf0 100644 --- a/crates/unicode/src/data.rs +++ b/crates/unicode/src/data.rs @@ -28,7 +28,6 @@ include!(concat!( )); #[derive(Clone, Copy)] -#[repr(u8)] enum DecompositionType { #[allow(unused)] Canonical, @@ -126,10 +125,7 @@ pub fn unicode_version() -> String { } /// Look up a character by its Unicode name (`unicodedata.lookup`). -#[must_use] -pub fn lookup_character(name: &str) -> Option { - unicode_names2::character(name) -} +pub use unicode_names2::character as lookup_character; /// The Unicode name of `ch` (`unicodedata.name`), if any. #[must_use] @@ -346,3 +342,33 @@ impl Ucd { } } } + +#[cfg(test)] +mod tests { + use rustpython_wtf8::CodePoint; + + use super::{Ucd, character_name, lookup_character}; + + fn cp(ch: char) -> CodePoint { + CodePoint::from(ch) + } + + #[test] + fn data_queries_match_unicodedata_behavior() { + let ucd = Ucd::new(true); + assert_eq!(ucd.category(cp('A')), "Lu"); + assert_eq!(ucd.category(CodePoint::from_u32(0xD800).unwrap()), "Cs"); + assert_eq!(lookup_character("SNOWMAN"), Some('☃')); + assert_eq!(character_name('☃').as_deref(), Some("SNOWMAN")); + assert_eq!(ucd.decimal(cp('५')), Some(5)); + assert_eq!(ucd.digit(cp('²')), Some(2)); + let third = ucd.numeric(cp('⅓')).unwrap(); + assert!((third - 1.0 / 3.0).abs() < 1e-6, "got {third}"); + } + + #[test] + fn ucd_3_2_0_view_differs_from_modern() { + let legacy = Ucd::new(false); + assert_eq!(legacy.unidata_version(), "3.2.0"); + } +} diff --git a/crates/unicode/src/identifier.rs b/crates/unicode/src/identifier.rs index 8ea6fb26761..413c722feb0 100644 --- a/crates/unicode/src/identifier.rs +++ b/crates/unicode/src/identifier.rs @@ -21,7 +21,17 @@ pub fn is_start(c: char) -> bool { } /// Whether `c` may continue a Python identifier: `XID_Continue`. -#[must_use] -pub fn is_continue(c: char) -> bool { - is_xid_continue(c) +pub use is_xid_continue as is_continue; + +#[cfg(test)] +mod tests { + use super::{is_continue, is_start}; + + #[test] + fn identifier_predicates() { + assert!(is_start('_')); + assert!(is_start('가')); + assert!(!is_start('1')); + assert!(is_continue('1')); + } } diff --git a/crates/unicode/src/lib.rs b/crates/unicode/src/lib.rs index 58fe9c7478f..a3f3eceb1c7 100644 --- a/crates/unicode/src/lib.rs +++ b/crates/unicode/src/lib.rs @@ -14,85 +14,6 @@ pub mod classify; pub mod data; pub mod identifier; pub mod normalize; -pub mod regex; pub use data::{Ucd, character_name, lookup_character, unicode_version}; pub use normalize::{NormalizeForm, is_normalized, normalize}; - -#[cfg(test)] -mod tests { - use rustpython_wtf8::{CodePoint, Wtf8Buf}; - - use crate::{NormalizeForm, Ucd, character_name, is_normalized, lookup_character, normalize}; - - fn cp(ch: char) -> CodePoint { - CodePoint::from(ch) - } - - #[test] - fn data_queries_match_unicodedata_behavior() { - let ucd = Ucd::new(true); - assert_eq!(ucd.category(cp('A')), "Lu"); - assert_eq!(ucd.category(CodePoint::from_u32(0xD800).unwrap()), "Cs"); - assert_eq!(lookup_character("SNOWMAN"), Some('☃')); - assert_eq!(character_name('☃').as_deref(), Some("SNOWMAN")); - assert_eq!(ucd.decimal(cp('५')), Some(5)); - assert_eq!(ucd.digit(cp('²')), Some(2)); - let third = ucd.numeric(cp('⅓')).unwrap(); - assert!((third - 1.0 / 3.0).abs() < 1e-6, "got {third}"); - } - - #[test] - fn ucd_3_2_0_view_differs_from_modern() { - let legacy = Ucd::new(false); - assert_eq!(legacy.unidata_version(), "3.2.0"); - } - - #[test] - fn numeric_type_chain_holds() { - use crate::classify::{is_decimal, is_digit, is_numeric}; - - // isdecimal ⊂ isdigit ⊂ isnumeric - for c in ('\0'..='\u{2FFFF}').filter_map(|c| char::from_u32(c as u32)) { - if is_decimal(c) { - assert!(is_digit(c), "{c:?} decimal but not digit"); - } - if is_digit(c) { - assert!(is_numeric(c), "{c:?} digit but not numeric"); - } - } - assert!(crate::classify::is_decimal('5')); - assert!(!crate::classify::is_decimal('²')); - assert!(crate::classify::is_digit('²')); - assert!(!crate::classify::is_digit('⅓')); - assert!(crate::classify::is_numeric('⅓')); - } - - #[test] - fn identifier_predicates() { - use crate::identifier::{is_continue, is_start}; - - assert!(is_start('_')); - assert!(is_start('가')); - assert!(!is_start('1')); - assert!(is_continue('1')); - } - - #[test] - fn casefold_full_mappings() { - use crate::case::casefold_str; - - // ß case-folds to "ss" - assert_eq!(casefold_str("ß"), "ss"); - assert_eq!(casefold_str("Σ"), "σ"); - } - - #[test] - fn normalization_round_trips() { - let composed = Wtf8Buf::from("é"); - let decomposed = normalize(NormalizeForm::Nfd, &composed); - assert_eq!(normalize(NormalizeForm::Nfc, &decomposed), composed); - assert!(is_normalized(NormalizeForm::Nfc, "é".as_bytes())); - assert!(!is_normalized(NormalizeForm::Nfd, "é".as_bytes())); - } -} diff --git a/crates/unicode/src/normalize.rs b/crates/unicode/src/normalize.rs index 04a6194f419..e79df05a7a1 100644 --- a/crates/unicode/src/normalize.rs +++ b/crates/unicode/src/normalize.rs @@ -71,3 +71,19 @@ pub fn is_normalized(form: NormalizeForm, bytes: &[u8]) -> bool { NormalizeForm::Nfkd => DecomposingNormalizerBorrowed::new_nfkd().is_normalized_utf8(bytes), } } + +#[cfg(test)] +mod tests { + use rustpython_wtf8::Wtf8Buf; + + use super::{NormalizeForm, is_normalized, normalize}; + + #[test] + fn normalization_round_trips() { + let composed = Wtf8Buf::from("é"); + let decomposed = normalize(NormalizeForm::Nfd, &composed); + assert_eq!(normalize(NormalizeForm::Nfc, &decomposed), composed); + assert!(is_normalized(NormalizeForm::Nfc, "é".as_bytes())); + assert!(!is_normalized(NormalizeForm::Nfd, "é".as_bytes())); + } +} diff --git a/crates/unicode/src/regex.rs b/crates/unicode/src/regex.rs deleted file mode 100644 index 0dcc7dbb392..00000000000 --- a/crates/unicode/src/regex.rs +++ /dev/null @@ -1,138 +0,0 @@ -//! Character-class and case predicates for the SRE regex engine. -//! -//! Every predicate takes a raw `u32` code point (SRE decodes strings into -//! `u32`s, including lone surrogates) and returns whether it belongs to the -//! class. ASCII-mode predicates only ever consider byte values; Unicode-mode -//! predicates consult the shared property tables. - -use crate::classify; - -const UNDERSCORE: u32 = '_' as u32; - -const fn is_py_ascii_whitespace(b: u8) -> bool { - matches!(b, b'\t' | b'\n' | b'\x0C' | b'\r' | b' ' | b'\x0B') -} - -#[must_use] -pub fn is_word(ch: u32) -> bool { - ch == UNDERSCORE || u8::try_from(ch).is_ok_and(|x| x.is_ascii_alphanumeric()) -} - -#[must_use] -pub fn is_space(ch: u32) -> bool { - u8::try_from(ch).is_ok_and(is_py_ascii_whitespace) -} - -#[must_use] -pub fn is_digit(ch: u32) -> bool { - u8::try_from(ch).is_ok_and(|x| x.is_ascii_digit()) -} - -#[must_use] -pub fn is_loc_alnum(ch: u32) -> bool { - // FIXME: Ignore the locales - u8::try_from(ch).is_ok_and(|x| x.is_ascii_alphanumeric()) -} - -#[must_use] -pub fn is_loc_word(ch: u32) -> bool { - ch == UNDERSCORE || is_loc_alnum(ch) -} - -#[must_use] -pub const fn is_linebreak(ch: u32) -> bool { - ch == '\n' as u32 -} - -#[must_use] -pub fn lower_ascii(ch: u32) -> u32 { - u8::try_from(ch).map_or(ch, |x| x.to_ascii_lowercase() as u32) -} - -#[must_use] -pub fn lower_locate(ch: u32) -> u32 { - // FIXME: Ignore the locales - lower_ascii(ch) -} - -#[must_use] -pub fn upper_locate(ch: u32) -> u32 { - // FIXME: Ignore the locales - u8::try_from(ch).map_or(ch, |x| x.to_ascii_uppercase() as u32) -} - -#[must_use] -pub fn is_uni_digit(ch: u32) -> bool { - // SRE_UNI_IS_DIGIT matches Unicode decimal digits (Py_UNICODE_ISDECIMAL), - // not just ASCII 0-9. - char::try_from(ch).is_ok_and(classify::is_decimal) -} - -#[must_use] -pub fn is_uni_space(ch: u32) -> bool { - // TODO: check with cpython - is_space(ch) - || matches!( - ch, - 0x0009 - | 0x000A - | 0x000B - | 0x000C - | 0x000D - | 0x001C - | 0x001D - | 0x001E - | 0x001F - | 0x0020 - | 0x0085 - | 0x00A0 - | 0x1680 - | 0x2000 - | 0x2001 - | 0x2002 - | 0x2003 - | 0x2004 - | 0x2005 - | 0x2006 - | 0x2007 - | 0x2008 - | 0x2009 - | 0x200A - | 0x2028 - | 0x2029 - | 0x202F - | 0x205F - | 0x3000 - ) -} - -#[must_use] -pub const fn is_uni_linebreak(ch: u32) -> bool { - matches!( - ch, - 0x000A | 0x000B | 0x000C | 0x000D | 0x001C | 0x001D | 0x001E | 0x0085 | 0x2028 | 0x2029 - ) -} - -#[must_use] -pub fn is_uni_alnum(ch: u32) -> bool { - // TODO: check with cpython - char::try_from(ch).is_ok_and(classify::is_alnum) -} - -#[must_use] -pub fn is_uni_word(ch: u32) -> bool { - ch == UNDERSCORE || is_uni_alnum(ch) -} - -#[must_use] -pub fn lower_unicode(ch: u32) -> u32 { - // TODO: check with cpython - char::try_from(ch).map_or(ch, |x| x.to_lowercase().next().unwrap() as u32) -} - -#[must_use] -pub fn upper_unicode(ch: u32) -> u32 { - // TODO: check with cpython - char::try_from(ch).map_or(ch, |x| x.to_uppercase().next().unwrap() as u32) -} From 399c647c541f0422396ab851276924cd1acb7013 Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Mon, 6 Jul 2026 18:42:36 +0900 Subject: [PATCH 8/9] no-std --- Cargo.toml | 10 +++++----- crates/unicode/src/data.rs | 6 ++++-- crates/wtf8/Cargo.toml | 4 ++-- crates/wtf8/src/lib.rs | 2 +- 4 files changed, 12 insertions(+), 10 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 20930e05120..d8081f50166 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -193,12 +193,12 @@ der = { version = "0.8", features = ["alloc", "oid", "pem", "zeroize"] } phf = { version = "0.14.0", default-features = false, features = ["macros"]} adler32 = "1.2.0" approx = "0.5.1" -ascii = "1.1" +ascii = { version = "1.1", default-features = false } base64 = "0.22" blake2 = "0.11.0-rc.6" bitflags = "2.11.0" bitflagset = "0.0.3" -bstr = "1" +bstr = { version = "1", default-features = false, features = ["unicode"] } bzip2 = "0.6" chrono = { version = "0.4.44", default-features = false, features = ["clock", "std"] } console_error_panic_hook = "0.1" @@ -228,7 +228,7 @@ hexf-parse = "0.2.1" hmac = "0.13" indexmap = { version = "2.14.0", features = ["std"] } insta = "1.47" -itertools = "0.15.0" +itertools = { version = "0.15.0", default-features = false, features = ["use_alloc"] } is-macro = "0.3.7" js-sys = "0.3" junction = "2.0.0" @@ -249,7 +249,7 @@ malachite-bigint = "0.9.1" malachite-q = "0.9.1" malachite-base = "0.9.1" md-5 = "0.11" -memchr = "2.8.1" +memchr = { version = "2.8.1", default-features = false, features = ["alloc"] } memmap2 = "0.9.10" mt19937 = "3.3" num-complex = "0.4.6" @@ -311,7 +311,7 @@ icu_locale = "2" icu_properties = "2" icu_normalizer = "2" uuid = "1.23.2" -unicode_names2 = "3" +unicode_names2 = { version = "3", default-features = false, features = ["no_std"] } widestring = "1.2.0" windows-sys = "0.61.2" wasm-bindgen = "0.2.106" diff --git a/crates/unicode/src/data.rs b/crates/unicode/src/data.rs index df0a9002bf0..e8b8318fe50 100644 --- a/crates/unicode/src/data.rs +++ b/crates/unicode/src/data.rs @@ -311,7 +311,8 @@ impl Ucd { let expected = [NumericType::Decimal, NumericType::Digit]; self.numeric_type_matches(c, &expected).and_then(|ch| { let value = lookup_numeric_val(ch, true)?; - (value.trunc() == value).then_some(value as u64) + let int = value as u64; + (int as f64 == value).then_some(int) }) } @@ -321,7 +322,8 @@ impl Ucd { let expected = [NumericType::Decimal]; self.numeric_type_matches(c, &expected).and_then(|ch| { let value = lookup_numeric_val(ch, self.modern)?; - (value.trunc() == value).then_some(value as u64) + let int = value as u64; + (int as f64 == value).then_some(int) }) } diff --git a/crates/wtf8/Cargo.toml b/crates/wtf8/Cargo.toml index 110b54ad0ca..20bf824898a 100644 --- a/crates/wtf8/Cargo.toml +++ b/crates/wtf8/Cargo.toml @@ -9,7 +9,7 @@ repository.workspace = true license.workspace = true [dependencies] -ascii = { workspace = true } -bstr = { workspace = true } +ascii = { workspace = true, features = ["alloc"] } +bstr = { workspace = true, features = ["alloc"] } itertools = { workspace = true } memchr = { workspace = true } diff --git a/crates/wtf8/src/lib.rs b/crates/wtf8/src/lib.rs index 772a2879944..b31ed1cf09c 100644 --- a/crates/wtf8/src/lib.rs +++ b/crates/wtf8/src/lib.rs @@ -31,7 +31,7 @@ //! to match CPython's behavior. //! //! [WTF-8]: https://simonsapin.github.io/wtf-8 -//! [`OsStr`]: std::ffi::OsStr +//! [`OsStr`]: https://doc.rust-lang.org/std/ffi/struct.OsStr.html #![no_std] #![allow(clippy::precedence, clippy::match_overlapping_arm)] From a6271d25e426ecae4acab70f95271f0ef9badfaa Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Mon, 6 Jul 2026 23:16:11 +0900 Subject: [PATCH 9/9] Address review feedback: test module, dead variant, WTF-8 normalization - differential.rs: move the sweep tests into a `mod tests` block and drop the file-level allow of clippy::tests_outside_test_module and std_instead_of_alloc; the test uses alloc collections instead. - data.rs: remove the never-constructed DecompositionType::Canonical variant and its allow(unused); compatibility decomposition never produces it and canonical decomposition is handled through icu4x. - normalize.rs: is_normalized now takes &Wtf8 and checks each UTF-8 run, skipping lone surrogates, matching normalize's run-wise behavior. - unicodedata.rs: pass as_wtf8() to is_normalized. Assisted-by: Claude --- crates/stdlib/src/unicodedata.rs | 2 +- crates/unicode/src/data.rs | 3 - crates/unicode/src/normalize.rs | 48 +++- crates/unicode/tests/differential.rs | 373 +++++++++++++-------------- 4 files changed, 222 insertions(+), 204 deletions(-) diff --git a/crates/stdlib/src/unicodedata.rs b/crates/stdlib/src/unicodedata.rs index b7f097125a8..134332bc9d5 100644 --- a/crates/stdlib/src/unicodedata.rs +++ b/crates/stdlib/src/unicodedata.rs @@ -156,7 +156,7 @@ mod unicodedata { #[pymethod] fn is_normalized(&self, form: NormalizeFormArg, unistr: PyStrRef) -> bool { - unicode_core::is_normalized(form.0, unistr.as_bytes()) + unicode_core::is_normalized(form.0, unistr.as_wtf8()) } #[pymethod] diff --git a/crates/unicode/src/data.rs b/crates/unicode/src/data.rs index e8b8318fe50..83d81612b6a 100644 --- a/crates/unicode/src/data.rs +++ b/crates/unicode/src/data.rs @@ -29,8 +29,6 @@ include!(concat!( #[derive(Clone, Copy)] enum DecompositionType { - #[allow(unused)] - Canonical, Compat, Circle, Final, @@ -52,7 +50,6 @@ enum DecompositionType { impl DecompositionType { const fn type_tag(self) -> &'static str { match self { - Self::Canonical => "canonical", Self::Compat => "compat", Self::Circle => "circle", Self::Final => "final", diff --git a/crates/unicode/src/normalize.rs b/crates/unicode/src/normalize.rs index e79df05a7a1..e2f80d02439 100644 --- a/crates/unicode/src/normalize.rs +++ b/crates/unicode/src/normalize.rs @@ -5,7 +5,7 @@ use core::str::FromStr; use icu_normalizer::{ComposingNormalizerBorrowed, DecomposingNormalizerBorrowed}; -use rustpython_wtf8::{Wtf8, Wtf8Buf}; +use rustpython_wtf8::{Wtf8, Wtf8Buf, Wtf8Chunk}; /// One of the four Unicode normalization forms. #[derive(Debug, Clone, Copy, Eq, PartialEq)] @@ -60,21 +60,27 @@ pub fn normalize(form: NormalizeForm, text: &Wtf8) -> Wtf8Buf { } } -/// Whether `bytes` (interpreted as UTF-8) is already in `form` -/// (`unicodedata.is_normalized`). +/// Whether `text` is already in `form` (`unicodedata.is_normalized`). +/// +/// Lone surrogates split the text into valid UTF-8 runs; each run is checked +/// independently, matching the run-wise normalization performed by [`normalize`]. #[must_use] -pub fn is_normalized(form: NormalizeForm, bytes: &[u8]) -> bool { - match form { - NormalizeForm::Nfc => ComposingNormalizerBorrowed::new_nfc().is_normalized_utf8(bytes), - NormalizeForm::Nfkc => ComposingNormalizerBorrowed::new_nfkc().is_normalized_utf8(bytes), - NormalizeForm::Nfd => DecomposingNormalizerBorrowed::new_nfd().is_normalized_utf8(bytes), - NormalizeForm::Nfkd => DecomposingNormalizerBorrowed::new_nfkd().is_normalized_utf8(bytes), - } +pub fn is_normalized(form: NormalizeForm, text: &Wtf8) -> bool { + let check: fn(&str) -> bool = match form { + NormalizeForm::Nfc => |s| ComposingNormalizerBorrowed::new_nfc().is_normalized(s), + NormalizeForm::Nfkc => |s| ComposingNormalizerBorrowed::new_nfkc().is_normalized(s), + NormalizeForm::Nfd => |s| DecomposingNormalizerBorrowed::new_nfd().is_normalized(s), + NormalizeForm::Nfkd => |s| DecomposingNormalizerBorrowed::new_nfkd().is_normalized(s), + }; + text.chunks().all(|chunk| match chunk { + Wtf8Chunk::Utf8(s) => check(s), + Wtf8Chunk::Surrogate(_) => true, + }) } #[cfg(test)] mod tests { - use rustpython_wtf8::Wtf8Buf; + use rustpython_wtf8::{CodePoint, Wtf8Buf}; use super::{NormalizeForm, is_normalized, normalize}; @@ -83,7 +89,23 @@ mod tests { let composed = Wtf8Buf::from("é"); let decomposed = normalize(NormalizeForm::Nfd, &composed); assert_eq!(normalize(NormalizeForm::Nfc, &decomposed), composed); - assert!(is_normalized(NormalizeForm::Nfc, "é".as_bytes())); - assert!(!is_normalized(NormalizeForm::Nfd, "é".as_bytes())); + assert!(is_normalized( + NormalizeForm::Nfc, + Wtf8Buf::from("é").as_ref() + )); + assert!(!is_normalized( + NormalizeForm::Nfd, + Wtf8Buf::from("é").as_ref() + )); + } + + #[test] + fn is_normalized_skips_lone_surrogates() { + // A lone surrogate splits the text into UTF-8 runs; each run is checked + // independently, so a surrogate next to normalized text stays normalized. + let mut buf = Wtf8Buf::from("é"); + buf.push(CodePoint::from_u32(0xD800).unwrap()); + assert!(is_normalized(NormalizeForm::Nfc, &buf)); + assert!(!is_normalized(NormalizeForm::Nfd, &buf)); } } diff --git a/crates/unicode/tests/differential.rs b/crates/unicode/tests/differential.rs index ba73d3149da..fef3451fdb9 100644 --- a/crates/unicode/tests/differential.rs +++ b/crates/unicode/tests/differential.rs @@ -1,9 +1,3 @@ -#![allow( - clippy::std_instead_of_alloc, - clippy::tests_outside_test_module, - reason = "integration test target always links std and its functions are the test entry points" -)] - //! Differential sweep of the classification predicates over the full scalar //! range `0..0x110000` against a committed CPython reference dataset. //! @@ -17,220 +11,225 @@ //! Both data files use the same run-length format: one `predicate` line per //! str method, followed by comma-separated hex `start:end` inclusive ranges. -use std::collections::{BTreeMap, BTreeSet}; - -use rustpython_unicode::classify; - -const MAX: u32 = 0x110000; -const REFERENCE: &str = include_str!("data/cpython3.14_predicates.txt"); -const VERSION_SKEW: &str = include_str!("data/version_skew_cpython3.14.txt"); - -fn crate_predicate(name: &str, cp: u32) -> bool { - let Some(c) = char::from_u32(cp) else { - // Lone surrogates are not scalars; every str predicate is false. - return false; - }; - match name { - "isalpha" => classify::is_alpha(c), - "isalnum" => classify::is_alnum(c), - "isdecimal" => classify::is_decimal(c), - "isdigit" => classify::is_digit(c), - "isnumeric" => classify::is_numeric(c), - "isspace" => classify::is_space(c), - "isprintable" => classify::is_printable(c), - "isidentifier" => { - // str.isidentifier is a whole-string predicate; for a single char it - // is "may start an identifier". - classify_is_identifier_char(c) +#[cfg(test)] +mod tests { + extern crate alloc; + + use alloc::collections::{BTreeMap, BTreeSet}; + + use rustpython_unicode::classify; + + const MAX: u32 = 0x110000; + const REFERENCE: &str = include_str!("data/cpython3.14_predicates.txt"); + const VERSION_SKEW: &str = include_str!("data/version_skew_cpython3.14.txt"); + + fn crate_predicate(name: &str, cp: u32) -> bool { + let Some(c) = char::from_u32(cp) else { + // Lone surrogates are not scalars; every str predicate is false. + return false; + }; + match name { + "isalpha" => classify::is_alpha(c), + "isalnum" => classify::is_alnum(c), + "isdecimal" => classify::is_decimal(c), + "isdigit" => classify::is_digit(c), + "isnumeric" => classify::is_numeric(c), + "isspace" => classify::is_space(c), + "isprintable" => classify::is_printable(c), + "isidentifier" => { + // str.isidentifier is a whole-string predicate; for a single char it + // is "may start an identifier". + classify_is_identifier_char(c) + } + other => panic!("unknown predicate {other}"), } - other => panic!("unknown predicate {other}"), } -} -fn classify_is_identifier_char(c: char) -> bool { - rustpython_unicode::identifier::is_start(c) -} + fn classify_is_identifier_char(c: char) -> bool { + rustpython_unicode::identifier::is_start(c) + } -/// Parse a `name -> sorted set of code points` map from a run-length file. -/// -/// Each non-comment line is `predicate start:end,start:end,...` with inclusive -/// hex ranges; a predicate with no members is a bare `predicate`. -fn parse_ranges(text: &str) -> BTreeMap> { - let mut map = BTreeMap::new(); - for line in text.lines() { - let line = line.trim(); - if line.is_empty() || line.starts_with('#') { - continue; - } - let (name, packed) = match line.split_once(' ') { - Some((name, packed)) => (name, packed.trim()), - None => (line, ""), - }; - let mut set = BTreeSet::new(); - if !packed.is_empty() { - for run in packed.split(',') { - let (s, e) = run.split_once(':').expect("run is start:end"); - let start = u32::from_str_radix(s, 16).unwrap(); - let end = u32::from_str_radix(e, 16).unwrap(); - for cp in start..=end { - set.insert(cp); + /// Parse a `name -> sorted set of code points` map from a run-length file. + /// + /// Each non-comment line is `predicate start:end,start:end,...` with inclusive + /// hex ranges; a predicate with no members is a bare `predicate`. + fn parse_ranges(text: &str) -> BTreeMap> { + let mut map = BTreeMap::new(); + for line in text.lines() { + let line = line.trim(); + if line.is_empty() || line.starts_with('#') { + continue; + } + let (name, packed) = match line.split_once(' ') { + Some((name, packed)) => (name, packed.trim()), + None => (line, ""), + }; + let mut set = BTreeSet::new(); + if !packed.is_empty() { + for run in packed.split(',') { + let (s, e) = run.split_once(':').expect("run is start:end"); + let start = u32::from_str_radix(s, 16).unwrap(); + let end = u32::from_str_radix(e, 16).unwrap(); + for cp in start..=end { + set.insert(cp); + } } } + map.insert(name.to_string(), set); } - map.insert(name.to_string(), set); + map } - map -} -/// Collapse a sorted code-point set into inclusive `start:end` runs. -fn encode_ranges(set: &BTreeSet) -> String { - let mut runs = Vec::new(); - let mut iter = set.iter().copied(); - if let Some(first) = iter.next() { - let (mut start, mut end) = (first, first); - for cp in iter { - if cp == end + 1 { - end = cp; - } else { - runs.push((start, end)); - start = cp; - end = cp; + /// Collapse a sorted code-point set into inclusive `start:end` runs. + fn encode_ranges(set: &BTreeSet) -> String { + let mut runs = Vec::new(); + let mut iter = set.iter().copied(); + if let Some(first) = iter.next() { + let (mut start, mut end) = (first, first); + for cp in iter { + if cp == end + 1 { + end = cp; + } else { + runs.push((start, end)); + start = cp; + end = cp; + } } + runs.push((start, end)); } - runs.push((start, end)); + runs.iter() + .map(|(s, e)| format!("{s:X}:{e:X}")) + .collect::>() + .join(",") } - runs.iter() - .map(|(s, e)| format!("{s:X}:{e:X}")) - .collect::>() - .join(",") -} -/// Recompute the full divergence set. Every entry is a `(predicate, code -/// point)` where the crate and the CPython reference disagree. -fn all_divergences(reference: &BTreeMap>) -> Vec<(String, u32, bool)> { - let mut out = Vec::new(); - for (name, truth) in reference { - for cp in 0..MAX { - let expected = truth.contains(&cp); - let actual = crate_predicate(name, cp); - if expected != actual { - out.push((name.clone(), cp, expected)); + /// Recompute the full divergence set. Every entry is a `(predicate, code + /// point)` where the crate and the CPython reference disagree. + fn all_divergences(reference: &BTreeMap>) -> Vec<(String, u32, bool)> { + let mut out = Vec::new(); + for (name, truth) in reference { + for cp in 0..MAX { + let expected = truth.contains(&cp); + let actual = crate_predicate(name, cp); + if expected != actual { + out.push((name.clone(), cp, expected)); + } } } + out } - out -} -/// Regenerate `data/version_skew_cpython3.14.txt` from the current toolchain. -/// -/// Run with `RUSTPYTHON_UNICODE_REGEN_SKEW=1 cargo test -p rustpython-unicode -/// --test differential` after bumping the Rust/icu toolchain. All divergences -/// must be one-directional (crate=true, cpython=false) — newly-assigned code -/// points from a later Unicode release. A `cpython=true, crate=false` entry -/// means a code point lost a property, which is a real regression, so this -/// refuses to record it. -#[test] -fn regen_version_skew() { - if std::env::var_os("RUSTPYTHON_UNICODE_REGEN_SKEW").is_none() { - return; - } - let reference = parse_ranges(REFERENCE); - let divergences = all_divergences(&reference); - - let regressions: Vec<_> = divergences - .iter() - .filter(|(_, _, expected)| *expected) - .collect(); - assert!( - regressions.is_empty(), - "refusing to record {} cpython=true/crate=false divergence(s) — these are \ - regressions, not version skew: {:?}", - regressions.len(), - ®ressions[..regressions.len().min(20)] - ); - - let mut by_predicate: BTreeMap> = BTreeMap::new(); - for (name, cp, _) in &divergences { - by_predicate.entry(name.clone()).or_default().insert(*cp); - } + /// Regenerate `data/version_skew_cpython3.14.txt` from the current toolchain. + /// + /// Run with `RUSTPYTHON_UNICODE_REGEN_SKEW=1 cargo test -p rustpython-unicode + /// --test differential` after bumping the Rust/icu toolchain. All divergences + /// must be one-directional (crate=true, cpython=false) — newly-assigned code + /// points from a later Unicode release. A `cpython=true, crate=false` entry + /// means a code point lost a property, which is a real regression, so this + /// refuses to record it. + #[test] + fn regen_version_skew() { + if std::env::var_os("RUSTPYTHON_UNICODE_REGEN_SKEW").is_none() { + return; + } + let reference = parse_ranges(REFERENCE); + let divergences = all_divergences(&reference); + + let regressions: Vec<_> = divergences + .iter() + .filter(|(_, _, expected)| *expected) + .collect(); + assert!( + regressions.is_empty(), + "refusing to record {} cpython=true/crate=false divergence(s) — these are \ + regressions, not version skew: {:?}", + regressions.len(), + ®ressions[..regressions.len().min(20)] + ); + + let mut by_predicate: BTreeMap> = BTreeMap::new(); + for (name, cp, _) in &divergences { + by_predicate.entry(name.clone()).or_default().insert(*cp); + } - let mut body = String::from( - "# Code points whose classification differs between CPython 3.14 (Unicode 16.0.0)\n\ - # and the Rust std / icu4x build used here (a later Unicode release assigns them).\n\ - # Regenerate with RUSTPYTHON_UNICODE_REGEN_SKEW=1.\n\ - # Format: `predicate start:end,...` with inclusive hex ranges.\n", - ); - for (name, set) in &by_predicate { - body.push_str(&format!("{name} {}\n", encode_ranges(set))); + let mut body = String::from( + "# Code points whose classification differs between CPython 3.14 (Unicode 16.0.0)\n\ + # and the Rust std / icu4x build used here (a later Unicode release assigns them).\n\ + # Regenerate with RUSTPYTHON_UNICODE_REGEN_SKEW=1.\n\ + # Format: `predicate start:end,...` with inclusive hex ranges.\n", + ); + for (name, set) in &by_predicate { + body.push_str(&format!("{name} {}\n", encode_ranges(set))); + } + let path = concat!( + env!("CARGO_MANIFEST_DIR"), + "/tests/data/version_skew_cpython3.14.txt" + ); + std::fs::write(path, body).unwrap(); + eprintln!( + "wrote {} skew code points across {} predicates to {path}", + divergences.len(), + by_predicate.len() + ); } - let path = concat!( - env!("CARGO_MANIFEST_DIR"), - "/tests/data/version_skew_cpython3.14.txt" - ); - std::fs::write(path, body).unwrap(); - eprintln!( - "wrote {} skew code points across {} predicates to {path}", - divergences.len(), - by_predicate.len() - ); -} -#[test] -fn predicates_match_cpython_except_documented_version_skew() { - let reference = parse_ranges(REFERENCE); - let skew = parse_ranges(VERSION_SKEW); + #[test] + fn predicates_match_cpython_except_documented_version_skew() { + let reference = parse_ranges(REFERENCE); + let skew = parse_ranges(VERSION_SKEW); - let allowed = |name: &str, cp: u32| skew.get(name).is_some_and(|set| set.contains(&cp)); + let allowed = |name: &str, cp: u32| skew.get(name).is_some_and(|set| set.contains(&cp)); - let mut unexpected: Vec<(String, u32, bool, bool)> = Vec::new(); + let mut unexpected: Vec<(String, u32, bool, bool)> = Vec::new(); - for (name, truth) in &reference { - for cp in 0..MAX { - let expected = truth.contains(&cp); - let actual = crate_predicate(name, cp); - if expected != actual && !allowed(name, cp) { - unexpected.push((name.clone(), cp, expected, actual)); + for (name, truth) in &reference { + for cp in 0..MAX { + let expected = truth.contains(&cp); + let actual = crate_predicate(name, cp); + if expected != actual && !allowed(name, cp) { + unexpected.push((name.clone(), cp, expected, actual)); + } } } - } - // Also flag stale allow-list entries: code points that no longer diverge. - let mut stale: Vec<(String, u32)> = Vec::new(); - for (name, set) in &skew { - let Some(truth) = reference.get(name) else { - continue; - }; - for &cp in set { - let expected = truth.contains(&cp); - let actual = crate_predicate(name, cp); - if expected == actual { - stale.push((name.clone(), cp)); + // Also flag stale allow-list entries: code points that no longer diverge. + let mut stale: Vec<(String, u32)> = Vec::new(); + for (name, set) in &skew { + let Some(truth) = reference.get(name) else { + continue; + }; + for &cp in set { + let expected = truth.contains(&cp); + let actual = crate_predicate(name, cp); + if expected == actual { + stale.push((name.clone(), cp)); + } } } - } - if !unexpected.is_empty() || !stale.is_empty() { - let mut msg = String::new(); - if !unexpected.is_empty() { - msg.push_str(&format!( - "{} undocumented divergence(s) from CPython:\n", - unexpected.len() - )); - for (name, cp, expected, actual) in unexpected.iter().take(50) { + if !unexpected.is_empty() || !stale.is_empty() { + let mut msg = String::new(); + if !unexpected.is_empty() { msg.push_str(&format!( - " {name} U+{cp:04X}: cpython={expected} crate={actual}\n" + "{} undocumented divergence(s) from CPython:\n", + unexpected.len() )); + for (name, cp, expected, actual) in unexpected.iter().take(50) { + msg.push_str(&format!( + " {name} U+{cp:04X}: cpython={expected} crate={actual}\n" + )); + } } - } - if !stale.is_empty() { - msg.push_str(&format!( - "{} stale version_skew_cpython3.14.txt entries that now agree:\n", - stale.len() - )); - for (name, cp) in stale.iter().take(50) { - msg.push_str(&format!(" {name} U+{cp:04X}\n")); + if !stale.is_empty() { + msg.push_str(&format!( + "{} stale version_skew_cpython3.14.txt entries that now agree:\n", + stale.len() + )); + for (name, cp) in stale.iter().take(50) { + msg.push_str(&format!(" {name} U+{cp:04X}\n")); + } } + panic!("{msg}"); } - panic!("{msg}"); } }