From 02c70c453585a6496b6e592eaae62ea9e9771f1e Mon Sep 17 00:00:00 2001 From: Yubin Kim Date: Fri, 24 Jul 2026 21:04:13 +0900 Subject: [PATCH 1/2] Replace malformed unicode error helper calls with the _real variants new_unicode_decode_error / new_unicode_encode_error build the exception from a bare message without running the initializer, so the result has none of the five attributes a unicode error must carry and str() renders as an empty string. Convert the non-Windows call sites in csv, socket, getlogin and the fs-path decoders to the _real constructors, passing the source bytes/str and the failing offset from the captured Utf8Error. The Windows-gated sites (nt, mbcs/oem codecs) and the sites whose source object is not reachable (uname, array) are left for follow-ups. Assisted-by: Claude Code:claude-fable-5 --- crates/stdlib/src/csv.rs | 34 +++++++++++++++++++++--------- crates/stdlib/src/socket.rs | 36 ++++++++++++++++++++++++++++---- crates/vm/src/function/fspath.rs | 11 ++++++++-- crates/vm/src/stdlib/os.rs | 11 ++++++++-- crates/vm/src/stdlib/posix.rs | 13 ++++++++---- 5 files changed, 83 insertions(+), 22 deletions(-) diff --git a/crates/stdlib/src/csv.rs b/crates/stdlib/src/csv.rs index a5203e8a373..7d2a14ea3c2 100644 --- a/crates/stdlib/src/csv.rs +++ b/crates/stdlib/src/csv.rs @@ -60,6 +60,20 @@ mod _csv { vm.new_exception_msg(super::_csv::error(vm), msg.into()) } + fn new_not_utf8_error( + vm: &VirtualMachine, + bytes: &[u8], + err: core::str::Utf8Error, + ) -> PyBaseExceptionRef { + vm.new_unicode_decode_error_real( + vm.ctx.new_str("utf-8"), + vm.ctx.new_bytes(bytes.to_vec()), + err.valid_up_to(), + err.valid_up_to() + 1, + vm.ctx.new_str("csv not utf8"), + ) + } + #[pyattr] #[pyclass(module = "csv", name = "Dialect")] #[derive(Debug, PyPayload, Clone, Copy)] @@ -1111,8 +1125,8 @@ mod _csv { { return Ok(vm.ctx.none()); } - let field = core::str::from_utf8(&field) - .map_err(|_| vm.new_unicode_decode_error("csv not utf8"))?; + let field = + core::str::from_utf8(&field).map_err(|e| new_not_utf8_error(vm, &field, e))?; Ok(vm.ctx.new_str(field).into()) }) .collect() @@ -1248,7 +1262,7 @@ mod _csv { prev_end = end; let s = core::str::from_utf8(&buffer[range.clone()]) // not sure if this is possible - the input was all strings - .map_err(|_e| vm.new_unicode_decode_error("csv not utf8"))?; + .map_err(|e| new_not_utf8_error(vm, &buffer[range.clone()], e))?; // TODO: RUSTPYTHON; Incomplete implementation if let QuoteStyle::Nonnumeric = zelf.dialect.quoting { @@ -1423,8 +1437,8 @@ mod _csv { } write_lineterminator(&mut output, self.dialect.lineterminator); - let s = core::str::from_utf8(&output) - .map_err(|_| vm.new_unicode_decode_error("csv not utf8"))?; + let s = + core::str::from_utf8(&output).map_err(|e| new_not_utf8_error(vm, &output, e))?; self.write.call((s,), vm) } @@ -1469,8 +1483,8 @@ mod _csv { write_lineterminator(&mut output, self.dialect.lineterminator); - let s = core::str::from_utf8(&output) - .map_err(|_| vm.new_unicode_decode_error("csv not utf8"))?; + let s = + core::str::from_utf8(&output).map_err(|e| new_not_utf8_error(vm, &output, e))?; self.write.call((s,), vm) } @@ -1518,8 +1532,8 @@ mod _csv { write_lineterminator(&mut output, self.dialect.lineterminator); - let s = core::str::from_utf8(&output) - .map_err(|_| vm.new_unicode_decode_error("csv not utf8"))?; + let s = + core::str::from_utf8(&output).map_err(|e| new_not_utf8_error(vm, &output, e))?; self.write.call((s,), vm) } @@ -1593,7 +1607,7 @@ mod _csv { } let s = core::str::from_utf8(&buffer[..buffer_offset]) - .map_err(|_| vm.new_unicode_decode_error("csv not utf8"))?; + .map_err(|e| new_not_utf8_error(vm, &buffer[..buffer_offset], e))?; self.write.call((s,), vm) } diff --git a/crates/stdlib/src/socket.rs b/crates/stdlib/src/socket.rs index 283aa408339..e5e6d92c302 100644 --- a/crates/stdlib/src/socket.rs +++ b/crates/stdlib/src/socket.rs @@ -2612,8 +2612,15 @@ mod _socket { } Some(ArgStrOrBytesLike::Buf(b)) => { let bytes = b.borrow_buf(); - let host_str = core::str::from_utf8(&bytes) - .map_err(|_| vm.new_unicode_decode_error("host bytes is not utf8"))?; + let host_str = core::str::from_utf8(&bytes).map_err(|e| { + vm.new_unicode_decode_error_real( + vm.ctx.new_str("utf-8"), + vm.ctx.new_bytes(bytes.to_vec()), + e.valid_up_to(), + e.valid_up_to() + 1, + vm.ctx.new_str("host bytes is not utf8"), + ) + })?; Some(host_str.to_owned()) } None => None, @@ -2627,14 +2634,35 @@ mod _socket { ArgStrOrBytesLike::Str(s) => { // For str, check for surrogates and raise UnicodeEncodeError if found s.to_str() - .ok_or_else(|| vm.new_unicode_encode_error("surrogates not allowed"))? + .ok_or_else(|| { + let start = s + .as_wtf8() + .code_points() + .position(|c| c.to_char().is_none()) + .unwrap(); + vm.new_unicode_encode_error_real( + vm.ctx.new_str("utf-8"), + (*s).clone(), + start, + start + 1, + vm.ctx.new_str("surrogates not allowed"), + ) + })? .to_owned() } ArgStrOrBytesLike::Buf(b) => { // For bytes, check if it's valid UTF-8 let bytes = b.borrow_buf(); core::str::from_utf8(&bytes) - .map_err(|_| vm.new_unicode_decode_error("port is not utf8"))? + .map_err(|e| { + vm.new_unicode_decode_error_real( + vm.ctx.new_str("utf-8"), + vm.ctx.new_bytes(bytes.to_vec()), + e.valid_up_to(), + e.valid_up_to() + 1, + vm.ctx.new_str("port is not utf8"), + ) + })? .to_owned() } }; diff --git a/crates/vm/src/function/fspath.rs b/crates/vm/src/function/fspath.rs index 053802285cd..325dc2374ae 100644 --- a/crates/vm/src/function/fspath.rs +++ b/crates/vm/src/function/fspath.rs @@ -125,8 +125,15 @@ impl FsPath { } pub fn bytes_as_os_str<'a>(b: &'a [u8], vm: &VirtualMachine) -> PyResult<&'a std::ffi::OsStr> { - rustpython_host_env::os::bytes_as_os_str(b) - .map_err(|_| vm.new_unicode_decode_error("can't decode path for utf-8")) + rustpython_host_env::os::bytes_as_os_str(b).map_err(|e| { + vm.new_unicode_decode_error_real( + vm.ctx.new_str("utf-8"), + vm.ctx.new_bytes(b.to_vec()), + e.valid_up_to(), + e.valid_up_to() + 1, + vm.ctx.new_str("can't decode path for utf-8"), + ) + }) } } diff --git a/crates/vm/src/stdlib/os.rs b/crates/vm/src/stdlib/os.rs index f885fb1db05..6781b86ea3b 100644 --- a/crates/vm/src/stdlib/os.rs +++ b/crates/vm/src/stdlib/os.rs @@ -130,8 +130,15 @@ pub(super) struct FollowSymlinks( #[cfg(not(windows))] fn bytes_as_os_str<'a>(b: &'a [u8], vm: &VirtualMachine) -> PyResult<&'a std::ffi::OsStr> { - rustpython_host_env::os::bytes_as_os_str(b) - .map_err(|_| vm.new_unicode_decode_error("can't decode path for utf-8")) + rustpython_host_env::os::bytes_as_os_str(b).map_err(|e| { + vm.new_unicode_decode_error_real( + vm.ctx.new_str("utf-8"), + vm.ctx.new_bytes(b.to_vec()), + e.valid_up_to(), + e.valid_up_to() + 1, + vm.ctx.new_str("can't decode path for utf-8"), + ) + }) } pub(crate) fn warn_if_bool_fd(obj: &PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { diff --git a/crates/vm/src/stdlib/posix.rs b/crates/vm/src/stdlib/posix.rs index 80245aed08f..1c99b61b6f0 100644 --- a/crates/vm/src/stdlib/posix.rs +++ b/crates/vm/src/stdlib/posix.rs @@ -1731,10 +1731,15 @@ pub mod module { let Some(login) = rustpython_host_env::posix::getlogin() else { return Err(vm.new_os_error("unable to determine login name")); }; - login - .to_str() - .map(|s| s.to_owned()) - .map_err(|e| vm.new_unicode_decode_error(format!("unable to decode login name: {e}"))) + login.to_str().map(|s| s.to_owned()).map_err(|e| { + vm.new_unicode_decode_error_real( + vm.ctx.new_str("utf-8"), + vm.ctx.new_bytes(login.as_bytes().to_vec()), + e.valid_up_to(), + e.valid_up_to() + 1, + vm.ctx.new_str("unable to decode login name"), + ) + }) } // cfg from nix From b63c408cd5375e640d184f8a4c9ce7cf3006a703 Mon Sep 17 00:00:00 2001 From: Yubin Kim Date: Sun, 26 Jul 2026 11:04:24 +0900 Subject: [PATCH 2/2] Report the full invalid UTF-8 span in the converted decode errors The conversions used valid_up_to() + 1 for the decode-error end offset, which under-reports multi-byte invalid sequences. Take the span from the Utf8Error instead: valid_up_to() + error_len(), or the input length for a truncated sequence (error_len() == None), matching CPython. Assisted-by: Claude Code:claude-opus-4-8 --- crates/stdlib/src/csv.rs | 3 ++- crates/stdlib/src/socket.rs | 4 ++-- crates/vm/src/function/fspath.rs | 2 +- crates/vm/src/stdlib/os.rs | 2 +- crates/vm/src/stdlib/posix.rs | 3 ++- 5 files changed, 8 insertions(+), 6 deletions(-) diff --git a/crates/stdlib/src/csv.rs b/crates/stdlib/src/csv.rs index 7d2a14ea3c2..91717801bc4 100644 --- a/crates/stdlib/src/csv.rs +++ b/crates/stdlib/src/csv.rs @@ -69,7 +69,8 @@ mod _csv { vm.ctx.new_str("utf-8"), vm.ctx.new_bytes(bytes.to_vec()), err.valid_up_to(), - err.valid_up_to() + 1, + err.error_len() + .map_or(bytes.len(), |n| err.valid_up_to() + n), vm.ctx.new_str("csv not utf8"), ) } diff --git a/crates/stdlib/src/socket.rs b/crates/stdlib/src/socket.rs index e5e6d92c302..968399ca782 100644 --- a/crates/stdlib/src/socket.rs +++ b/crates/stdlib/src/socket.rs @@ -2617,7 +2617,7 @@ mod _socket { vm.ctx.new_str("utf-8"), vm.ctx.new_bytes(bytes.to_vec()), e.valid_up_to(), - e.valid_up_to() + 1, + e.error_len().map_or(bytes.len(), |n| e.valid_up_to() + n), vm.ctx.new_str("host bytes is not utf8"), ) })?; @@ -2659,7 +2659,7 @@ mod _socket { vm.ctx.new_str("utf-8"), vm.ctx.new_bytes(bytes.to_vec()), e.valid_up_to(), - e.valid_up_to() + 1, + e.error_len().map_or(bytes.len(), |n| e.valid_up_to() + n), vm.ctx.new_str("port is not utf8"), ) })? diff --git a/crates/vm/src/function/fspath.rs b/crates/vm/src/function/fspath.rs index 325dc2374ae..cd3cd2276f7 100644 --- a/crates/vm/src/function/fspath.rs +++ b/crates/vm/src/function/fspath.rs @@ -130,7 +130,7 @@ impl FsPath { vm.ctx.new_str("utf-8"), vm.ctx.new_bytes(b.to_vec()), e.valid_up_to(), - e.valid_up_to() + 1, + e.error_len().map_or(b.len(), |n| e.valid_up_to() + n), vm.ctx.new_str("can't decode path for utf-8"), ) }) diff --git a/crates/vm/src/stdlib/os.rs b/crates/vm/src/stdlib/os.rs index 6781b86ea3b..73e4918ccd4 100644 --- a/crates/vm/src/stdlib/os.rs +++ b/crates/vm/src/stdlib/os.rs @@ -135,7 +135,7 @@ fn bytes_as_os_str<'a>(b: &'a [u8], vm: &VirtualMachine) -> PyResult<&'a std::ff vm.ctx.new_str("utf-8"), vm.ctx.new_bytes(b.to_vec()), e.valid_up_to(), - e.valid_up_to() + 1, + e.error_len().map_or(b.len(), |n| e.valid_up_to() + n), vm.ctx.new_str("can't decode path for utf-8"), ) }) diff --git a/crates/vm/src/stdlib/posix.rs b/crates/vm/src/stdlib/posix.rs index 1c99b61b6f0..c16da1ee703 100644 --- a/crates/vm/src/stdlib/posix.rs +++ b/crates/vm/src/stdlib/posix.rs @@ -1736,7 +1736,8 @@ pub mod module { vm.ctx.new_str("utf-8"), vm.ctx.new_bytes(login.as_bytes().to_vec()), e.valid_up_to(), - e.valid_up_to() + 1, + e.error_len() + .map_or(login.as_bytes().len(), |n| e.valid_up_to() + n), vm.ctx.new_str("unable to decode login name"), ) })