Fix handling of Windows (WTF-16) and WASI (UTF-8) paths

Windows paths now use WTF-16 <-> WTF-8 conversion everywhere, which is lossless. Previously, conversion of ill-formed UTF-16 paths would either fail or invoke illegal behavior.

WASI paths must be valid UTF-8, and the relevant function calls have been updated to handle the possibility of failure due to paths not being encoded/encodable as valid UTF-8.

Closes #18694
Closes #1774
Closes #2565

Author: squeek502

deps/aro/aro/Compilation.zig

@@ -69,7 +69,7 @@ pub const Environment = struct {
             const val: ?[]const u8 = std.process.getEnvVarOwned(allocator, env_var_name) catch |err| switch (err) {
                 error.OutOfMemory => |e| return e,
                 error.EnvironmentVariableNotFound => null,
-                error.InvalidUtf8 => null,
+                error.InvalidWtf8 => null,
             };
             @field(env, field.name) = val;
         }

deps/aro/aro/Driver.zig

@@ -523,7 +523,8 @@ pub fn errorDescription(e: anyerror) []const u8 {
         error.NotDir => "is not a directory",
         error.NotOpenForReading => "file is not open for reading",
         error.NotOpenForWriting => "file is not open for writing",
-        error.InvalidUtf8 => "input is not valid UTF-8",
+        error.InvalidUtf8 => "path is not valid UTF-8",
+        error.InvalidWtf8 => "path is not valid WTF-8",
         error.FileBusy => "file is busy",
         error.NameTooLong => "file name is too long",
         error.AccessDenied => "access denied",

lib/std/Build/Cache.zig

@@ -162,7 +162,7 @@ fn findPrefixResolved(cache: *const Cache, resolved_path: []u8) !PrefixedPath {
 fn getPrefixSubpath(allocator: Allocator, prefix: []const u8, path: []u8) ![]u8 {
     const relative = try std.fs.path.relative(allocator, prefix, path);
     errdefer allocator.free(relative);
-    var component_iterator = std.fs.path.NativeUtf8ComponentIterator.init(relative) catch {
+    var component_iterator = std.fs.path.NativeComponentIterator.init(relative) catch {
         return error.NotASubPath;
     };
     if (component_iterator.root() != null) {

lib/std/Thread.zig

@@ -91,7 +91,7 @@ pub fn setName(self: Thread, name: []const u8) SetNameError!void {
         },
         .windows => {
             var buf: [max_name_len]u16 = undefined;
-            const len = try std.unicode.utf8ToUtf16Le(&buf, name);
+            const len = try std.unicode.wtf8ToWtf16Le(&buf, name);
             const byte_len = math.cast(c_ushort, len * 2) orelse return error.NameTooLong;
 
             // Note: NT allocates its own copy, no use-after-free here.
@@ -157,17 +157,12 @@ pub fn setName(self: Thread, name: []const u8) SetNameError!void {
 }
 
 pub const GetNameError = error{
-    // For Windows, the name is converted from UTF16 to UTF8
-    CodepointTooLarge,
-    Utf8CannotEncodeSurrogateHalf,
-    DanglingSurrogateHalf,
-    ExpectedSecondSurrogateHalf,
-    UnexpectedSecondSurrogateHalf,
-
     Unsupported,
     Unexpected,
 } || os.PrctlError || os.ReadError || std.fs.File.OpenError || std.fmt.BufPrintError;
 
+/// On Windows, the result is encoded as [WTF-8](https://simonsapin.github.io/wtf-8/).
+/// On other platforms, the result is an opaque sequence of bytes with no particular encoding.
 pub fn getName(self: Thread, buffer_ptr: *[max_name_len:0]u8) GetNameError!?[]const u8 {
     buffer_ptr[max_name_len] = 0;
     var buffer: [:0]u8 = buffer_ptr;
@@ -213,7 +208,7 @@ pub fn getName(self: Thread, buffer_ptr: *[max_name_len:0]u8) GetNameError!?[]co
             )) {
                 .SUCCESS => {
                     const string = @as(*const os.windows.UNICODE_STRING, @ptrCast(&buf));
-                    const len = try std.unicode.utf16LeToUtf8(buffer, string.Buffer[0 .. string.Length / 2]);
+                    const len = std.unicode.wtf16LeToWtf8(buffer, string.Buffer[0 .. string.Length / 2]);
                     return if (len > 0) buffer[0..len] else null;
                 },
                 .NOT_IMPLEMENTED => return error.Unsupported,

lib/std/child_process.zig

@@ -129,10 +129,9 @@ pub const ChildProcess = struct {
         /// POSIX-only. `StdIo.Ignore` was selected and opening `/dev/null` returned ENODEV.
         NoDevice,
 
-        /// Windows-only. One of:
-        /// * `cwd` was provided and it could not be re-encoded into UTF16LE, or
-        /// * The `PATH` or `PATHEXT` environment variable contained invalid UTF-8.
-        InvalidUtf8,
+        /// Windows-only. `cwd` or `argv` was provided and it was invalid WTF-8.
+        /// https://simonsapin.github.io/wtf-8/
+        InvalidWtf8,
 
         /// Windows-only. `cwd` was provided, but the path did not exist when spawning the child process.
         CurrentWorkingDirectoryUnlinked,
@@ -767,16 +766,16 @@ pub const ChildProcess = struct {
         };
         var piProcInfo: windows.PROCESS_INFORMATION = undefined;
 
-        const cwd_w = if (self.cwd) |cwd| try unicode.utf8ToUtf16LeAllocZ(self.allocator, cwd) else null;
+        const cwd_w = if (self.cwd) |cwd| try unicode.wtf8ToWtf16LeAllocZ(self.allocator, cwd) else null;
         defer if (cwd_w) |cwd| self.allocator.free(cwd);
         const cwd_w_ptr = if (cwd_w) |cwd| cwd.ptr else null;
 
         const maybe_envp_buf = if (self.env_map) |env_map| try createWindowsEnvBlock(self.allocator, env_map) else null;
         defer if (maybe_envp_buf) |envp_buf| self.allocator.free(envp_buf);
         const envp_ptr = if (maybe_envp_buf) |envp_buf| envp_buf.ptr else null;
 
-        const app_name_utf8 = self.argv[0];
-        const app_name_is_absolute = fs.path.isAbsolute(app_name_utf8);
+        const app_name_wtf8 = self.argv[0];
+        const app_name_is_absolute = fs.path.isAbsolute(app_name_wtf8);
 
         // the cwd set in ChildProcess is in effect when choosing the executable path
         // to match posix semantics
@@ -785,11 +784,11 @@ pub const ChildProcess = struct {
             // If the app name is absolute, then we need to use its dirname as the cwd
             if (app_name_is_absolute) {
                 cwd_path_w_needs_free = true;
-                const dir = fs.path.dirname(app_name_utf8).?;
-                break :x try unicode.utf8ToUtf16LeAllocZ(self.allocator, dir);
+                const dir = fs.path.dirname(app_name_wtf8).?;
+                break :x try unicode.wtf8ToWtf16LeAllocZ(self.allocator, dir);
             } else if (self.cwd) |cwd| {
                 cwd_path_w_needs_free = true;
-                break :x try unicode.utf8ToUtf16LeAllocZ(self.allocator, cwd);
+                break :x try unicode.wtf8ToWtf16LeAllocZ(self.allocator, cwd);
             } else {
                 break :x &[_:0]u16{}; // empty for cwd
             }
@@ -800,19 +799,19 @@ pub const ChildProcess = struct {
         // into the basename and dirname and use the dirname as an addition to the cwd
         // path. This is because NtQueryDirectoryFile cannot accept FileName params with
         // path separators.
-        const app_basename_utf8 = fs.path.basename(app_name_utf8);
+        const app_basename_wtf8 = fs.path.basename(app_name_wtf8);
         // If the app name is absolute, then the cwd will already have the app's dirname in it,
         // so only populate app_dirname if app name is a relative path with > 0 path separators.
-        const maybe_app_dirname_utf8 = if (!app_name_is_absolute) fs.path.dirname(app_name_utf8) else null;
+        const maybe_app_dirname_wtf8 = if (!app_name_is_absolute) fs.path.dirname(app_name_wtf8) else null;
         const app_dirname_w: ?[:0]u16 = x: {
-            if (maybe_app_dirname_utf8) |app_dirname_utf8| {
-                break :x try unicode.utf8ToUtf16LeAllocZ(self.allocator, app_dirname_utf8);
+            if (maybe_app_dirname_wtf8) |app_dirname_wtf8| {
+                break :x try unicode.wtf8ToWtf16LeAllocZ(self.allocator, app_dirname_wtf8);
             }
             break :x null;
         };
         defer if (app_dirname_w != null) self.allocator.free(app_dirname_w.?);
 
-        const app_name_w = try unicode.utf8ToUtf16LeAllocZ(self.allocator, app_basename_utf8);
+        const app_name_w = try unicode.wtf8ToWtf16LeAllocZ(self.allocator, app_basename_wtf8);
         defer self.allocator.free(app_name_w);
 
         const cmd_line_w = argvToCommandLineWindows(self.allocator, self.argv) catch |err| switch (err) {
@@ -1173,7 +1172,7 @@ const CreateProcessSupportedExtension = enum {
     exe,
 };
 
-/// Case-insensitive UTF-16 lookup
+/// Case-insensitive WTF-16 lookup
 fn windowsCreateProcessSupportsExtension(ext: []const u16) ?CreateProcessSupportedExtension {
     if (ext.len != 4) return null;
     const State = enum {
@@ -1237,7 +1236,7 @@ test "windowsCreateProcessSupportsExtension" {
     try std.testing.expect(windowsCreateProcessSupportsExtension(&[_]u16{ '.', 'e', 'X', 'e', 'c' }) == null);
 }
 
-pub const ArgvToCommandLineError = error{ OutOfMemory, InvalidUtf8, InvalidArg0 };
+pub const ArgvToCommandLineError = error{ OutOfMemory, InvalidWtf8, InvalidArg0 };
 
 /// Serializes `argv` to a Windows command-line string suitable for passing to a child process and
 /// parsing by the `CommandLineToArgvW` algorithm. The caller owns the returned slice.
@@ -1320,7 +1319,7 @@ pub fn argvToCommandLineWindows(
         }
     }
 
-    return try unicode.utf8ToUtf16LeAllocZ(allocator, buf.items);
+    return try unicode.wtf8ToWtf16LeAllocZ(allocator, buf.items);
 }
 
 test "argvToCommandLineWindows" {
@@ -1386,7 +1385,7 @@ fn testArgvToCommandLineWindows(argv: []const []const u8, expected_cmd_line: []c
     const cmd_line_w = try argvToCommandLineWindows(std.testing.allocator, argv);
     defer std.testing.allocator.free(cmd_line_w);
 
-    const cmd_line = try unicode.utf16LeToUtf8Alloc(std.testing.allocator, cmd_line_w);
+    const cmd_line = try unicode.wtf16LeToWtf8Alloc(std.testing.allocator, cmd_line_w);
     defer std.testing.allocator.free(cmd_line);
 
     try std.testing.expectEqualStrings(expected_cmd_line, cmd_line);
@@ -1424,7 +1423,7 @@ fn windowsMakeAsyncPipe(rd: *?windows.HANDLE, wr: *?windows.HANDLE, sattr: *cons
             "\\\\.\\pipe\\zig-childprocess-{d}-{d}",
             .{ windows.kernel32.GetCurrentProcessId(), pipe_name_counter.fetchAdd(1, .Monotonic) },
         ) catch unreachable;
-        const len = std.unicode.utf8ToUtf16Le(&tmp_bufw, pipe_path) catch unreachable;
+        const len = std.unicode.wtf8ToWtf16Le(&tmp_bufw, pipe_path) catch unreachable;
         tmp_bufw[len] = 0;
         break :blk tmp_bufw[0..len :0];
     };
@@ -1521,10 +1520,10 @@ pub fn createWindowsEnvBlock(allocator: mem.Allocator, env_map: *const EnvMap) !
     var it = env_map.iterator();
     var i: usize = 0;
     while (it.next()) |pair| {
-        i += try unicode.utf8ToUtf16Le(result[i..], pair.key_ptr.*);
+        i += try unicode.wtf8ToWtf16Le(result[i..], pair.key_ptr.*);
         result[i] = '=';
         i += 1;
-        i += try unicode.utf8ToUtf16Le(result[i..], pair.value_ptr.*);
+        i += try unicode.wtf8ToWtf16Le(result[i..], pair.value_ptr.*);
         result[i] = 0;
         i += 1;
     }