Windows: Support UNC, rooted, drive relative, and namespaced/device paths

There are many different types of Windows paths, and there are a few different possible namespaces on top of that. Before this commit, NT namespaced paths were somewhat supported, and for Win32 paths (those without a namespace prefix), only relative and drive absolute paths were supported. After this commit, all of the following are supported:

- Device namespaced paths (`\\.\`)
- Verbatim paths (`\\?\`)
- NT-namespaced paths (`\??\`)
- Relative paths (`foo`)
- Drive-absolute paths (`C:\foo`)
- Drive-relative paths (`C:foo`)
- Rooted paths (`\foo`)
- UNC absolute paths (`\\server\share\foo`)
- Root local device paths (`\\.` or `\\?` exactly)

Plus:

- Any of the path types and namespace types can be mixed and matched together as appropriate.
- All of the `std.os.windows.*ToPrefixedFileW` functions will accept any path type, prefixed or not, and do the appropriate thing to convert them to an NT-prefixed path if necessary.

This is achieved by making the `std.os.windows.*ToPrefixedFileW` functions behave like `ntdll.RtlDosPathNameToNtPathName_U`, but with a few differences:

- Does not allocate on the heap (this is why we can't use `ntdll.RtlDosPathNameToNtPathName_U` directly, it does internal heap allocation).
- Relative paths are kept as relative unless they contain too many .. components, in which case they are treated as 'drive relative' and resolved against the CWD (this is how it behaved before this commit as well).
- Special case device names like COM1, NUL, etc are not handled specially (TODO)
- `.` and space are not stripped from the end of relative paths (potential TODO)

Most of the non-trivial conversion of non-relative paths is done via `ntdll.RtlGetFullPathName_U`, which AFAIK is used internally by `ntdll.RtlDosPathNameToNtPathName_U`.

Some relevant reading on Windows paths:

- https://googleprojectzero.blogspot.com/2016/02/the-definitive-guide-on-win32-to-nt.html
- https://chrisdenton.github.io/omnipath/Overview.html

Closes #8205
Might close (untested) #12729

Note:
- This removes checking for illegal characters in `std.os.windows.sliceToPrefixedFileW`, since the previous solution (iterate the whole string and error if any illegal characters were found) was naive and won't work for all path types. This is further complicated by things like file streams (where `:` is used as a delimiter, e.g. `file.ext:stream_name:$DATA`) and things in the device namespace (where a path like `\\.\GLOBALROOT\??\UNC\localhost\C$\foo` is valid despite the `?`s in the path and is effectively equivalent to `C:\foo`). Truly validating paths is complicated and would need to be tailored to each path type. The illegal character checking being removed may open up users to more instances of hitting `OBJECT_NAME_INVALID => unreachable` when using `fs` APIs.
  + This is related to #15607

Author: squeek502

lib/std/child_process.zig

@@ -957,15 +957,12 @@ fn windowsCreateProcessPathExt(
     // NtQueryDirectoryFile calls.
 
     var dir = dir: {
-        if (fs.path.isAbsoluteWindowsWTF16(dir_buf.items[0..dir_path_len])) {
-            const prefixed_path = try windows.wToPrefixedFileW(dir_buf.items[0..dir_path_len]);
-            break :dir fs.cwd().openDirW(prefixed_path.span().ptr, .{}, true) catch return error.FileNotFound;
-        }
         // needs to be null-terminated
         try dir_buf.append(allocator, 0);
-        defer dir_buf.shrinkRetainingCapacity(dir_buf.items[0..dir_path_len].len);
+        defer dir_buf.shrinkRetainingCapacity(dir_path_len);
         const dir_path_z = dir_buf.items[0 .. dir_buf.items.len - 1 :0];
-        break :dir std.fs.cwd().openDirW(dir_path_z.ptr, .{}, true) catch return error.FileNotFound;
+        const prefixed_path = try windows.wToPrefixedFileW(dir_path_z);
+        break :dir fs.cwd().openDirW(prefixed_path.span().ptr, .{}, true) catch return error.FileNotFound;
     };
     defer dir.close();
 

lib/std/os/windows.zig

@@ -1157,9 +1157,9 @@ pub fn GetFinalPathNameByHandle(
 
             // This surprising path is a filesystem path to the mount manager on Windows.
             // Source: https://stackoverflow.com/questions/3012828/using-ioctl-mountmgr-query-points
-            const mgmt_path = "\\MountPointManager";
-            const mgmt_path_u16 = sliceToPrefixedFileW(mgmt_path) catch unreachable;
-            const mgmt_handle = OpenFile(mgmt_path_u16.span(), .{
+            // This is the NT namespaced version of \\.\MountPointManager
+            const mgmt_path_u16 = std.unicode.utf8ToUtf16LeStringLiteral("\\??\\MountPointManager");
+            const mgmt_handle = OpenFile(mgmt_path_u16, .{
                 .access_mask = SYNCHRONIZE,
                 .share_access = FILE_SHARE_READ | FILE_SHARE_WRITE,
                 .creation = FILE_OPEN,
@@ -1997,43 +1997,248 @@ pub fn cStrToPrefixedFileW(s: [*:0]const u8) !PathSpace {
     return sliceToPrefixedFileW(mem.sliceTo(s, 0));
 }
 
-/// Converts the path `s` to WTF16, null-terminated. If the path is absolute,
-/// it will get NT-style prefix `\??\` prepended automatically.
-pub fn sliceToPrefixedFileW(s: []const u8) !PathSpace {
-    // TODO https://github.com/ziglang/zig/issues/2765
-    var path_space: PathSpace = undefined;
-    const prefix = "\\??\\";
-    const prefix_index: usize = if (mem.startsWith(u8, s, prefix)) prefix.len else 0;
-    for (s[prefix_index..]) |byte| {
-        switch (byte) {
-            '*', '?', '"', '<', '>', '|' => return error.BadPathName,
-            else => {},
-        }
-    }
-    const prefix_u16 = [_]u16{ '\\', '?', '?', '\\' };
-    const start_index = if (prefix_index > 0 or !std.fs.path.isAbsolute(s)) 0 else blk: {
-        path_space.data[0..prefix_u16.len].* = prefix_u16;
-        break :blk prefix_u16.len;
-    };
-    path_space.len = start_index + try std.unicode.utf8ToUtf16Le(path_space.data[start_index..], s);
-    if (path_space.len > path_space.data.len) return error.NameTooLong;
-    path_space.len = start_index + (normalizePath(u16, path_space.data[start_index..path_space.len]) catch |err| switch (err) {
-        error.TooManyParentDirs => {
-            if (!std.fs.path.isAbsolute(s)) {
-                var temp_path: PathSpace = undefined;
-                temp_path.len = try std.unicode.utf8ToUtf16Le(&temp_path.data, s);
-                std.debug.assert(temp_path.len == path_space.len);
-                temp_path.data[path_space.len] = 0;
-                path_space.len = prefix_u16.len + try getFullPathNameW(&temp_path.data, path_space.data[prefix_u16.len..]);
-                path_space.data[0..prefix_u16.len].* = prefix_u16;
-                std.debug.assert(path_space.data[path_space.len] == 0);
+/// Same as `wToPrefixedFileW` but accepts a UTF-8 encoded path.
+pub fn sliceToPrefixedFileW(path: []const u8) !PathSpace {
+    var temp_path: PathSpace = undefined;
+    temp_path.len = try std.unicode.utf8ToUtf16Le(&temp_path.data, path);
+    temp_path.data[temp_path.len] = 0;
+    return wToPrefixedFileW(temp_path.span());
+}
+
+/// Converts the `path` to WTF16, null-terminated. If the path contains any
+/// namespace prefix, or is anything but a relative path (rooted, drive relative,
+/// etc) the result will have the NT-style prefix `\??\`.
+///
+/// Similar to RtlDosPathNameToNtPathName_U with a few differences:
+/// - Does not allocate on the heap.
+/// - Relative paths are kept as relative unless they contain too many ..
+///   components, in which case they are treated as drive-relative and resolved
+///   against the CWD.
+/// - Special case device names like COM1, NUL, etc are not handled specially (TODO)
+/// - . and space are not stripped from the end of relative paths (potential TODO)
+pub fn wToPrefixedFileW(path: [:0]const u16) !PathSpace {
+    const nt_prefix = [_]u16{ '\\', '?', '?', '\\' };
+    switch (getNamespacePrefix(u16, path)) {
+        // TODO: Figure out a way to design an API that can avoid the copy for .nt,
+        //       since it is always returned fully unmodified.
+        .nt, .verbatim => {
+            var path_space: PathSpace = undefined;
+            path_space.data[0..nt_prefix.len].* = nt_prefix;
+            const len_after_prefix = path.len - nt_prefix.len;
+            @memcpy(path_space.data[nt_prefix.len..][0..len_after_prefix], path[nt_prefix.len..]);
+            path_space.len = path.len;
+            path_space.data[path_space.len] = 0;
+            return path_space;
+        },
+        .local_device, .fake_verbatim => {
+            var path_space: PathSpace = undefined;
+            const path_byte_len = ntdll.RtlGetFullPathName_U(
+                path.ptr,
+                path_space.data.len * 2,
+                &path_space.data,
+                null,
+            );
+            if (path_byte_len == 0) {
+                // TODO: This may not be the right error
+                return error.BadPathName;
+            } else if (path_byte_len / 2 > path_space.data.len) {
+                return error.NameTooLong;
+            }
+            path_space.len = path_byte_len / 2;
+            // Both prefixes will be normalized but retained, so all
+            // we need to do now is replace them with the NT prefix
+            path_space.data[0..nt_prefix.len].* = nt_prefix;
+            return path_space;
+        },
+        .none => {
+            const path_type = getUnprefixedPathType(u16, path);
+            var path_space: PathSpace = undefined;
+            relative: {
+                if (path_type == .relative) {
+                    // TODO: Handle special case device names like COM1, AUX, NUL, CONIN$, CONOUT$, etc.
+                    //       See https://googleprojectzero.blogspot.com/2016/02/the-definitive-guide-on-win32-to-nt.html
+
+                    // TODO: Potentially strip all trailing . and space characters from the
+                    //       end of the path. This is something that both RtlDosPathNameToNtPathName_U
+                    //       and RtlGetFullPathName_U do. Technically, trailing . and spaces
+                    //       are allowed, but such paths may not interact well with Windows (i.e.
+                    //       files with these paths can't be deleted from explorer.exe, etc).
+                    //       This could be something that normalizePath may want to do.
+
+                    @memcpy(path_space.data[0..path.len], path);
+                    // Try to normalize, but if we get too many parent directories,
+                    // then this is effectively a 'drive relative' path, so we need to
+                    // start over and use RtlGetFullPathName_U instead.
+                    path_space.len = normalizePath(u16, path_space.data[0..path.len]) catch |err| switch (err) {
+                        error.TooManyParentDirs => break :relative,
+                    };
+                    path_space.data[path_space.len] = 0;
+                    return path_space;
+                }
+            }
+            // We now know we are going to return an absolute NT path, so
+            // we can unconditionally prefix it with the NT prefix.
+            path_space.data[0..nt_prefix.len].* = nt_prefix;
+            if (path_type == .root_local_device) {
+                // `\\.` and `\\?` always get converted to `\??\` exactly, so
+                // we can just stop here
+                path_space.len = nt_prefix.len;
+                path_space.data[path_space.len] = 0;
                 return path_space;
             }
-            return error.BadPathName;
+            const path_buf_offset = switch (path_type) {
+                // UNC paths will always start with `\\`. However, we want to
+                // end up with something like `\??\UNC\server\share`, so to get
+                // RtlGetFullPathName to write into the spot we want the `server`
+                // part to end up, we need to provide an offset such that
+                // the `\\` part gets written where the `C\` of `UNC\` will be
+                // in the final NT path.
+                .unc_absolute => nt_prefix.len + 2,
+                else => nt_prefix.len,
+            };
+            const buf_len = @intCast(u32, path_space.data.len - path_buf_offset);
+            const path_byte_len = ntdll.RtlGetFullPathName_U(
+                path.ptr,
+                buf_len * 2,
+                path_space.data[path_buf_offset..].ptr,
+                null,
+            );
+            if (path_byte_len == 0) {
+                // TODO: This may not be the right error
+                return error.BadPathName;
+            } else if (path_byte_len / 2 > buf_len) {
+                return error.NameTooLong;
+            }
+            path_space.len = path_buf_offset + (path_byte_len / 2);
+            if (path_type == .unc_absolute) {
+                // Now add in the UNC, the `C` should overwrite the first `\` of the
+                // FullPathName, ultimately resulting in `\??\UNC\<the rest of the path>`
+                std.debug.assert(path_space.data[path_buf_offset] == '\\');
+                std.debug.assert(path_space.data[path_buf_offset + 1] == '\\');
+                const unc = [_]u16{ 'U', 'N', 'C' };
+                path_space.data[nt_prefix.len..][0..unc.len].* = unc;
+            }
+            return path_space;
         },
-    });
-    path_space.data[path_space.len] = 0;
-    return path_space;
+    }
+}
+
+pub const NamespacePrefix = enum {
+    none,
+    /// `\\.\` (path separators can be `\` or `/`)
+    local_device,
+    /// `\\?\`
+    /// When converted to an NT path, everything past the prefix is left
+    /// untouched and `\\?\` is replaced by `\??\`.
+    verbatim,
+    /// `\\?\` without all path separators being `\`.
+    /// This seems to be recognized as a prefix, but the 'verbatim' aspect
+    /// is not respected (i.e. if `//?/C:/foo` is converted to an NT path,
+    /// it will become `\??\C:\foo` [it will be canonicalized and the //?/ won't
+    /// be treated as part of the final path])
+    fake_verbatim,
+    /// `\??\`
+    nt,
+};
+
+pub fn getNamespacePrefix(comptime T: type, path: []const T) NamespacePrefix {
+    if (path.len < 4) return .none;
+    var all_backslash = switch (path[0]) {
+        '\\' => true,
+        '/' => false,
+        else => return .none,
+    };
+    all_backslash = all_backslash and switch (path[3]) {
+        '\\' => true,
+        '/' => false,
+        else => return .none,
+    };
+    switch (path[1]) {
+        '?' => if (path[2] == '?' and all_backslash) return .nt else return .none,
+        '\\' => {},
+        '/' => all_backslash = false,
+        else => return .none,
+    }
+    return switch (path[2]) {
+        '?' => if (all_backslash) .verbatim else .fake_verbatim,
+        '.' => .local_device,
+        else => .none,
+    };
+}
+
+test getNamespacePrefix {
+    try std.testing.expectEqual(NamespacePrefix.none, getNamespacePrefix(u8, ""));
+    try std.testing.expectEqual(NamespacePrefix.nt, getNamespacePrefix(u8, "\\??\\"));
+    try std.testing.expectEqual(NamespacePrefix.none, getNamespacePrefix(u8, "/??/"));
+    try std.testing.expectEqual(NamespacePrefix.none, getNamespacePrefix(u8, "/??\\"));
+    try std.testing.expectEqual(NamespacePrefix.none, getNamespacePrefix(u8, "\\?\\\\"));
+    try std.testing.expectEqual(NamespacePrefix.local_device, getNamespacePrefix(u8, "\\\\.\\"));
+    try std.testing.expectEqual(NamespacePrefix.local_device, getNamespacePrefix(u8, "\\\\./"));
+    try std.testing.expectEqual(NamespacePrefix.local_device, getNamespacePrefix(u8, "/\\./"));
+    try std.testing.expectEqual(NamespacePrefix.local_device, getNamespacePrefix(u8, "//./"));
+    try std.testing.expectEqual(NamespacePrefix.none, getNamespacePrefix(u8, "/.//"));
+    try std.testing.expectEqual(NamespacePrefix.verbatim, getNamespacePrefix(u8, "\\\\?\\"));
+    try std.testing.expectEqual(NamespacePrefix.fake_verbatim, getNamespacePrefix(u8, "\\/?\\"));
+    try std.testing.expectEqual(NamespacePrefix.fake_verbatim, getNamespacePrefix(u8, "\\/?/"));
+    try std.testing.expectEqual(NamespacePrefix.fake_verbatim, getNamespacePrefix(u8, "//?/"));
+}
+
+pub const UnprefixedPathType = enum {
+    unc_absolute,
+    drive_absolute,
+    drive_relative,
+    rooted,
+    relative,
+    root_local_device,
+};
+
+inline fn isSepW(c: u16) bool {
+    return c == '/' or c == '\\';
+}
+
+/// Get the path type of a path that is known to not have any namespace prefixes
+/// (`\\?\`, `\\.\`, `\??\`).
+pub fn getUnprefixedPathType(comptime T: type, path: []const T) UnprefixedPathType {
+    if (path.len < 1) return .relative;
+
+    if (std.debug.runtime_safety) {
+        std.debug.assert(getNamespacePrefix(T, path) == .none);
+    }
+
+    if (isSepW(path[0])) {
+        // \x
+        if (path.len < 2 or !isSepW(path[1])) return .rooted;
+        // exactly \\. or \\? with nothing trailing
+        if (path.len == 3 and (path[2] == '.' or path[2] == '?')) return .root_local_device;
+        // \\x
+        return .unc_absolute;
+    } else {
+        // x
+        if (path.len < 2 or path[1] != ':') return .relative;
+        // x:\
+        if (path.len > 2 and isSepW(path[2])) return .drive_absolute;
+        // x:
+        return .drive_relative;
+    }
+}
+
+test getUnprefixedPathType {
+    try std.testing.expectEqual(UnprefixedPathType.relative, getUnprefixedPathType(u8, ""));
+    try std.testing.expectEqual(UnprefixedPathType.relative, getUnprefixedPathType(u8, "x"));
+    try std.testing.expectEqual(UnprefixedPathType.relative, getUnprefixedPathType(u8, "x\\"));
+    try std.testing.expectEqual(UnprefixedPathType.root_local_device, getUnprefixedPathType(u8, "//."));
+    try std.testing.expectEqual(UnprefixedPathType.root_local_device, getUnprefixedPathType(u8, "/\\?"));
+    try std.testing.expectEqual(UnprefixedPathType.root_local_device, getUnprefixedPathType(u8, "\\\\?"));
+    try std.testing.expectEqual(UnprefixedPathType.unc_absolute, getUnprefixedPathType(u8, "\\\\x"));
+    try std.testing.expectEqual(UnprefixedPathType.unc_absolute, getUnprefixedPathType(u8, "//x"));
+    try std.testing.expectEqual(UnprefixedPathType.rooted, getUnprefixedPathType(u8, "\\x"));
+    try std.testing.expectEqual(UnprefixedPathType.rooted, getUnprefixedPathType(u8, "/"));
+    try std.testing.expectEqual(UnprefixedPathType.drive_relative, getUnprefixedPathType(u8, "x:"));
+    try std.testing.expectEqual(UnprefixedPathType.drive_relative, getUnprefixedPathType(u8, "x:abc"));
+    try std.testing.expectEqual(UnprefixedPathType.drive_relative, getUnprefixedPathType(u8, "x:a/b/c"));
+    try std.testing.expectEqual(UnprefixedPathType.drive_absolute, getUnprefixedPathType(u8, "x:\\"));
+    try std.testing.expectEqual(UnprefixedPathType.drive_absolute, getUnprefixedPathType(u8, "x:\\abc"));
+    try std.testing.expectEqual(UnprefixedPathType.drive_absolute, getUnprefixedPathType(u8, "x:/a/b/c"));
 }
 
 fn getFullPathNameW(path: [*:0]const u16, out: []u16) !usize {
@@ -2046,34 +2251,6 @@ fn getFullPathNameW(path: [*:0]const u16, out: []u16) !usize {
     return result;
 }
 
-/// Assumes an absolute path.
-pub fn wToPrefixedFileW(s: []const u16) !PathSpace {
-    // TODO https://github.com/ziglang/zig/issues/2765
-    var path_space: PathSpace = undefined;
-
-    const start_index = if (mem.startsWith(u16, s, &[_]u16{ '\\', '?' })) 0 else blk: {
-        const prefix = [_]u16{ '\\', '?', '?', '\\' };
-        path_space.data[0..prefix.len].* = prefix;
-        break :blk prefix.len;
-    };
-    path_space.len = start_index + s.len;
-    if (path_space.len > path_space.data.len) return error.NameTooLong;
-    @memcpy(path_space.data[start_index..][0..s.len], s);
-    // > File I/O functions in the Windows API convert "/" to "\" as part of
-    // > converting the name to an NT-style name, except when using the "\\?\"
-    // > prefix as detailed in the following sections.
-    // from https://docs.microsoft.com/en-us/windows/desktop/FileIO/naming-a-file#maximum-path-length-limitation
-    // Because we want the larger maximum path length for absolute paths, we
-    // convert forward slashes to backward slashes here.
-    for (path_space.data[0..path_space.len]) |*elem| {
-        if (elem.* == '/') {
-            elem.* = '\\';
-        }
-    }
-    path_space.data[path_space.len] = 0;
-    return path_space;
-}
-
 inline fn MAKELANGID(p: c_ushort, s: c_ushort) LANGID {
     return (s << 10) | p;
 }

lib/std/os/windows/ntdll.zig

@@ -158,6 +158,16 @@ pub extern "ntdll" fn RtlDosPathNameToNtPathName_U(
 ) callconv(WINAPI) BOOL;
 pub extern "ntdll" fn RtlFreeUnicodeString(UnicodeString: *UNICODE_STRING) callconv(WINAPI) void;
 
+/// Returns the number of bytes written to `Buffer`.
+/// If the returned count is larger than `BufferByteLength`, the buffer was too small.
+/// If the returned count is zero, an error occurred.
+pub extern "ntdll" fn RtlGetFullPathName_U(
+    FileName: [*:0]const u16,
+    BufferByteLength: ULONG,
+    Buffer: [*]u16,
+    ShortName: ?*[*:0]const u16,
+) callconv(windows.WINAPI) windows.ULONG;
+
 pub extern "ntdll" fn NtQueryDirectoryFile(
     FileHandle: HANDLE,
     Event: ?HANDLE,