diff --git a/crates/stdlib/src/csv.rs b/crates/stdlib/src/csv.rs index a5203e8a373..91717801bc4 100644 --- a/crates/stdlib/src/csv.rs +++ b/crates/stdlib/src/csv.rs @@ -60,6 +60,21 @@ mod _csv { vm.new_exception_msg(super::_csv::error(vm), msg.into()) } + fn new_not_utf8_error( + vm: &VirtualMachine, + bytes: &[u8], + err: core::str::Utf8Error, + ) -> PyBaseExceptionRef { + vm.new_unicode_decode_error_real( + vm.ctx.new_str("utf-8"), + vm.ctx.new_bytes(bytes.to_vec()), + err.valid_up_to(), + err.error_len() + .map_or(bytes.len(), |n| err.valid_up_to() + n), + vm.ctx.new_str("csv not utf8"), + ) + } + #[pyattr] #[pyclass(module = "csv", name = "Dialect")] #[derive(Debug, PyPayload, Clone, Copy)] @@ -1111,8 +1126,8 @@ mod _csv { { return Ok(vm.ctx.none()); } - let field = core::str::from_utf8(&field) - .map_err(|_| vm.new_unicode_decode_error("csv not utf8"))?; + let field = + core::str::from_utf8(&field).map_err(|e| new_not_utf8_error(vm, &field, e))?; Ok(vm.ctx.new_str(field).into()) }) .collect() @@ -1248,7 +1263,7 @@ mod _csv { prev_end = end; let s = core::str::from_utf8(&buffer[range.clone()]) // not sure if this is possible - the input was all strings - .map_err(|_e| vm.new_unicode_decode_error("csv not utf8"))?; + .map_err(|e| new_not_utf8_error(vm, &buffer[range.clone()], e))?; // TODO: RUSTPYTHON; Incomplete implementation if let QuoteStyle::Nonnumeric = zelf.dialect.quoting { @@ -1423,8 +1438,8 @@ mod _csv { } write_lineterminator(&mut output, self.dialect.lineterminator); - let s = core::str::from_utf8(&output) - .map_err(|_| vm.new_unicode_decode_error("csv not utf8"))?; + let s = + core::str::from_utf8(&output).map_err(|e| new_not_utf8_error(vm, &output, e))?; self.write.call((s,), vm) } @@ -1469,8 +1484,8 @@ mod _csv { write_lineterminator(&mut output, self.dialect.lineterminator); - let s = core::str::from_utf8(&output) - .map_err(|_| vm.new_unicode_decode_error("csv not utf8"))?; + let s = + core::str::from_utf8(&output).map_err(|e| new_not_utf8_error(vm, &output, e))?; self.write.call((s,), vm) } @@ -1518,8 +1533,8 @@ mod _csv { write_lineterminator(&mut output, self.dialect.lineterminator); - let s = core::str::from_utf8(&output) - .map_err(|_| vm.new_unicode_decode_error("csv not utf8"))?; + let s = + core::str::from_utf8(&output).map_err(|e| new_not_utf8_error(vm, &output, e))?; self.write.call((s,), vm) } @@ -1593,7 +1608,7 @@ mod _csv { } let s = core::str::from_utf8(&buffer[..buffer_offset]) - .map_err(|_| vm.new_unicode_decode_error("csv not utf8"))?; + .map_err(|e| new_not_utf8_error(vm, &buffer[..buffer_offset], e))?; self.write.call((s,), vm) } diff --git a/crates/stdlib/src/socket.rs b/crates/stdlib/src/socket.rs index 283aa408339..968399ca782 100644 --- a/crates/stdlib/src/socket.rs +++ b/crates/stdlib/src/socket.rs @@ -2612,8 +2612,15 @@ mod _socket { } Some(ArgStrOrBytesLike::Buf(b)) => { let bytes = b.borrow_buf(); - let host_str = core::str::from_utf8(&bytes) - .map_err(|_| vm.new_unicode_decode_error("host bytes is not utf8"))?; + let host_str = core::str::from_utf8(&bytes).map_err(|e| { + vm.new_unicode_decode_error_real( + vm.ctx.new_str("utf-8"), + vm.ctx.new_bytes(bytes.to_vec()), + e.valid_up_to(), + e.error_len().map_or(bytes.len(), |n| e.valid_up_to() + n), + vm.ctx.new_str("host bytes is not utf8"), + ) + })?; Some(host_str.to_owned()) } None => None, @@ -2627,14 +2634,35 @@ mod _socket { ArgStrOrBytesLike::Str(s) => { // For str, check for surrogates and raise UnicodeEncodeError if found s.to_str() - .ok_or_else(|| vm.new_unicode_encode_error("surrogates not allowed"))? + .ok_or_else(|| { + let start = s + .as_wtf8() + .code_points() + .position(|c| c.to_char().is_none()) + .unwrap(); + vm.new_unicode_encode_error_real( + vm.ctx.new_str("utf-8"), + (*s).clone(), + start, + start + 1, + vm.ctx.new_str("surrogates not allowed"), + ) + })? .to_owned() } ArgStrOrBytesLike::Buf(b) => { // For bytes, check if it's valid UTF-8 let bytes = b.borrow_buf(); core::str::from_utf8(&bytes) - .map_err(|_| vm.new_unicode_decode_error("port is not utf8"))? + .map_err(|e| { + vm.new_unicode_decode_error_real( + vm.ctx.new_str("utf-8"), + vm.ctx.new_bytes(bytes.to_vec()), + e.valid_up_to(), + e.error_len().map_or(bytes.len(), |n| e.valid_up_to() + n), + vm.ctx.new_str("port is not utf8"), + ) + })? .to_owned() } }; diff --git a/crates/vm/src/function/fspath.rs b/crates/vm/src/function/fspath.rs index 053802285cd..cd3cd2276f7 100644 --- a/crates/vm/src/function/fspath.rs +++ b/crates/vm/src/function/fspath.rs @@ -125,8 +125,15 @@ impl FsPath { } pub fn bytes_as_os_str<'a>(b: &'a [u8], vm: &VirtualMachine) -> PyResult<&'a std::ffi::OsStr> { - rustpython_host_env::os::bytes_as_os_str(b) - .map_err(|_| vm.new_unicode_decode_error("can't decode path for utf-8")) + rustpython_host_env::os::bytes_as_os_str(b).map_err(|e| { + vm.new_unicode_decode_error_real( + vm.ctx.new_str("utf-8"), + vm.ctx.new_bytes(b.to_vec()), + e.valid_up_to(), + e.error_len().map_or(b.len(), |n| e.valid_up_to() + n), + vm.ctx.new_str("can't decode path for utf-8"), + ) + }) } } diff --git a/crates/vm/src/stdlib/os.rs b/crates/vm/src/stdlib/os.rs index f885fb1db05..73e4918ccd4 100644 --- a/crates/vm/src/stdlib/os.rs +++ b/crates/vm/src/stdlib/os.rs @@ -130,8 +130,15 @@ pub(super) struct FollowSymlinks( #[cfg(not(windows))] fn bytes_as_os_str<'a>(b: &'a [u8], vm: &VirtualMachine) -> PyResult<&'a std::ffi::OsStr> { - rustpython_host_env::os::bytes_as_os_str(b) - .map_err(|_| vm.new_unicode_decode_error("can't decode path for utf-8")) + rustpython_host_env::os::bytes_as_os_str(b).map_err(|e| { + vm.new_unicode_decode_error_real( + vm.ctx.new_str("utf-8"), + vm.ctx.new_bytes(b.to_vec()), + e.valid_up_to(), + e.error_len().map_or(b.len(), |n| e.valid_up_to() + n), + vm.ctx.new_str("can't decode path for utf-8"), + ) + }) } pub(crate) fn warn_if_bool_fd(obj: &PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { diff --git a/crates/vm/src/stdlib/posix.rs b/crates/vm/src/stdlib/posix.rs index 80245aed08f..c16da1ee703 100644 --- a/crates/vm/src/stdlib/posix.rs +++ b/crates/vm/src/stdlib/posix.rs @@ -1731,10 +1731,16 @@ pub mod module { let Some(login) = rustpython_host_env::posix::getlogin() else { return Err(vm.new_os_error("unable to determine login name")); }; - login - .to_str() - .map(|s| s.to_owned()) - .map_err(|e| vm.new_unicode_decode_error(format!("unable to decode login name: {e}"))) + login.to_str().map(|s| s.to_owned()).map_err(|e| { + vm.new_unicode_decode_error_real( + vm.ctx.new_str("utf-8"), + vm.ctx.new_bytes(login.as_bytes().to_vec()), + e.valid_up_to(), + e.error_len() + .map_or(login.as_bytes().len(), |n| e.valid_up_to() + n), + vm.ctx.new_str("unable to decode login name"), + ) + }) } // cfg from nix