Skip to content

Commit 47e9e32

Browse files
gcoakesgcoakes
committed
std: Add fs.MemoryMap and fs.MemoryMapGrowable.
These new types provide a cross platform memory mapping API which encompasses the common subset of both POSIX and Windows APIs. `MemoryMap` provides a view of a file with a fixed size and offset, analogous to Window's `MapViewOfFile` or POSIX's `mmap`. `MemoryMapGrowable` provides a view of a whole file and has the ability to safely grow the file/mapping.
1 parent 30a769a commit 47e9e32

File tree

4 files changed

+328
-2
lines changed

4 files changed

+328
-2
lines changed

‎lib/std/fs.zig

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,8 @@ const is_darwin = native_os.isDarwin();
1717
pub const AtomicFile = @import("fs/AtomicFile.zig");
1818
pub const Dir = @import("fs/Dir.zig");
1919
pub const File = @import("fs/File.zig");
20+
pub const MemoryMap = @import("fs/MemoryMap.zig");
21+
pub const MemoryMapGrowable = @import("fs/MemoryMapGrowable.zig");
2022
pub const path = @import("fs/path.zig");
2123

2224
pub const has_executable_bit = switch (native_os) {
@@ -710,6 +712,8 @@ test {
710712
_ = &AtomicFile;
711713
_ = &Dir;
712714
_ = &File;
715+
_ = &MemoryMap;
716+
_ = &MemoryMapGrowable;
713717
_ = &path;
714718
_ = @import("fs/test.zig");
715719
_ = @import("fs/get_app_data_dir.zig");

‎lib/std/fs/MemoryMap.zig

Lines changed: 184 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,184 @@
1+
const std = @import("../std.zig");
2+
const builtin = @import("builtin");
3+
4+
const MemoryMap = @This();
5+
6+
/// An OS-specific reference to a kernel object for this mapping.
7+
handle: switch (builtin.os.tag) {
8+
.windows => std.os.windows.HANDLE,
9+
else => void,
10+
},
11+
/// The region of virtual memory in which the file is mapped.
12+
///
13+
/// Accesses to this are subject to the protection semantics specified upon
14+
/// initialization of the mapping. Failure to abide by those semantics has undefined
15+
/// behavior (though should be well-defined by the OS).
16+
mapped: []align(std.mem.page_size) volatile u8,
17+
18+
test MemoryMap {
19+
var tmp = std.testing.tmpDir(.{});
20+
defer tmp.cleanup();
21+
22+
var file = try tmp.dir.createFile("mmap.bin", .{
23+
.exclusive = true,
24+
.truncate = true,
25+
.read = true,
26+
});
27+
defer file.close();
28+
29+
const magic = "\xde\xca\xfb\xad";
30+
try file.writeAll(magic);
31+
32+
const len = try file.getEndPos();
33+
34+
var view = try MemoryMap.init(file, .{ .length = @intCast(len) });
35+
defer view.deinit();
36+
37+
try std.testing.expectEqualSlices(u8, magic, @volatileCast(view.mapped));
38+
}
39+
40+
pub const InitOptions = struct {
41+
protection: ProtectionFlags = .{},
42+
exclusivity: Exclusivity = .private,
43+
/// The desired offset of the mapping.
44+
///
45+
/// The backing file must be of at least `offset` size.
46+
offset: usize = 0,
47+
/// The desired length of the mapping.
48+
///
49+
/// The backing file must be of at least `offset + length` size.
50+
length: usize,
51+
hint: ?[*]align(std.mem.page_size) u8 = null,
52+
};
53+
54+
/// A description of OS protections to be applied to a memory-mapped region.
55+
pub const ProtectionFlags = struct {
56+
write: bool = false,
57+
execute: bool = false,
58+
};
59+
60+
pub const Exclusivity = enum {
61+
/// The file's content may be read or written by external processes.
62+
shared,
63+
/// The file's content is exclusive to this process.
64+
///
65+
/// TODO: Does POSIX semantics propogate changes from external processes into private
66+
/// mapping? I know it doesn't propogate local changes back into the file, but this
67+
/// is about the opposite.
68+
private,
69+
};
70+
71+
/// Create a memory-mapped view into `file`.
72+
///
73+
/// The `file` may be closed after this returns (TODO: check if this can be done on
74+
/// Windows).
75+
///
76+
/// Asserts `opts.length` is non-zero.
77+
pub fn init(file: std.fs.File, opts: InitOptions) !MemoryMap {
78+
std.debug.assert(opts.length > 0);
79+
switch (builtin.os.tag) {
80+
.windows => {
81+
// Create the kernel resource for the memory mapping.
82+
const prot: std.os.windows.DWORD = switch (opts.protection.execute) {
83+
true => switch (opts.protection.write) {
84+
true => std.os.windows.PAGE_EXECUTE_READWRITE,
85+
false => std.os.windows.PAGE_EXECUTE_READ,
86+
},
87+
false => switch (opts.protection.write) {
88+
true => std.os.windows.PAGE_READWRITE,
89+
false => std.os.windows.PAGE_READONLY,
90+
},
91+
};
92+
const handle = try std.os.windows.CreateFileMapping(
93+
file.handle,
94+
null,
95+
prot,
96+
opts.length,
97+
null,
98+
);
99+
errdefer std.os.windows.CloseHandle(handle);
100+
101+
// Convert the public options into Windows specific options.
102+
var flags: std.os.windows.DWORD = std.os.windows.FILE_MAP_READ;
103+
if (opts.protection.write)
104+
flags |= std.os.windows.FILE_MAP_WRITE;
105+
if (opts.protection.execute)
106+
flags |= std.os.windows.FILE_MAP_EXECUTE;
107+
if (opts.exclusivity == .private)
108+
flags |= std.os.windows.FILE_MAP_COPY;
109+
110+
// Create the mapping.
111+
const mapped = try std.os.windows.MapViewOfFile(
112+
handle,
113+
flags,
114+
opts.offset,
115+
opts.length,
116+
opts.hint,
117+
);
118+
119+
return .{
120+
.handle = handle,
121+
.mapped = mapped,
122+
};
123+
},
124+
else => {
125+
// The man page indicates the flags must be either `NONE` or an OR of the
126+
// flags. That doesn't explicitly state that the absence of those flags is
127+
// the same as `NONE`, so this static assertion is made. That'll break the
128+
// build rather than behaving unexpectedly if some weird system comes up.
129+
comptime std.debug.assert(std.posix.PROT.NONE == 0);
130+
131+
// Convert the public options into POSIX specific options.
132+
var prot: u32 = std.posix.PROT.READ;
133+
if (opts.protection.write)
134+
prot |= std.posix.PROT.WRITE;
135+
if (opts.protection.execute)
136+
prot |= std.posix.PROT.EXEC;
137+
const flags: std.posix.MAP = .{
138+
.TYPE = switch (opts.exclusivity) {
139+
.shared => .SHARED,
140+
.private => .PRIVATE,
141+
},
142+
};
143+
144+
// Create the mapping.
145+
const mapped = try std.posix.mmap(
146+
opts.hint,
147+
opts.length,
148+
prot,
149+
@bitCast(flags),
150+
file.handle,
151+
opts.offset,
152+
);
153+
154+
return .{
155+
.handle = {},
156+
.mapped = mapped,
157+
};
158+
},
159+
}
160+
}
161+
162+
/// Unmap the file from virtual memory and deallocate kernel resources.
163+
///
164+
/// Invalidates references to `self.mapped`.
165+
pub fn deinit(self: MemoryMap) void {
166+
switch (builtin.os.tag) {
167+
.windows => {
168+
std.os.windows.UnmapViewOfFile(@volatileCast(self.mapped.ptr));
169+
std.os.windows.CloseHandle(self.handle);
170+
},
171+
else => {
172+
std.posix.munmap(@volatileCast(self.mapped));
173+
},
174+
}
175+
}
176+
177+
/// Reinterpret `self.mapped` as `T`.
178+
///
179+
/// The returned pointer is aligned to the beginning of the mapping. The mapping may be
180+
/// larger than `T`. The caller is responsible for determining whether volatility can be
181+
/// stripped away through external synchronization.
182+
pub inline fn cast(self: MemoryMap, comptime T: type) *align(std.mem.page_size) volatile T {
183+
return std.mem.bytesAsValue(T, self.mapped[0..@sizeOf(T)]);
184+
}

‎lib/std/fs/MemoryMapGrowable.zig

Lines changed: 135 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,135 @@
1+
const std = @import("../std.zig");
2+
const builtin = @import("builtin");
3+
4+
const MemoryMap = @import("./MemoryMap.zig");
5+
6+
const MemoryMapGrowable = @This();
7+
8+
protection: ProtectionFlags,
9+
exclusivity: Exclusivity,
10+
view: MemoryMap,
11+
12+
pub const ProtectionFlags = MemoryMap.ProtectionFlags;
13+
pub const Exclusivity = MemoryMap.Exclusivity;
14+
15+
test MemoryMapGrowable {
16+
var tmp = std.testing.tmpDir(.{});
17+
defer tmp.cleanup();
18+
19+
var file = try tmp.dir.createFile("mmap.bin", .{
20+
.exclusive = true,
21+
.truncate = true,
22+
.read = true,
23+
});
24+
defer file.close();
25+
26+
const magic = "\xde\xca\xfb\xad";
27+
try file.writeAll(magic);
28+
29+
const len = try file.getEndPos();
30+
31+
var arr = try MemoryMapGrowable.init(file, .{
32+
.protection = .{ .write = true },
33+
.exclusivity = .shared,
34+
.length = @intCast(len),
35+
});
36+
defer arr.deinit();
37+
38+
try std.testing.expectEqualSlices(u8, magic, @volatileCast(arr.view.mapped));
39+
40+
const double_magic = magic ** 2;
41+
42+
try arr.ensureTotalCapacityPrecise(file, double_magic.len);
43+
44+
@memcpy(@volatileCast(arr.view.mapped)[magic.len..], magic);
45+
46+
try std.testing.expectEqualSlices(u8, double_magic, @volatileCast(arr.view.mapped));
47+
}
48+
49+
/// Create a memory-mapped array list from `file`.
50+
///
51+
/// Asserts `opts.length` is non-zero.
52+
pub fn init(file: std.fs.File, opts: MemoryMap.InitOptions) !MemoryMapGrowable {
53+
return .{
54+
.protection = opts.protection,
55+
.exclusivity = opts.exclusivity,
56+
.view = try MemoryMap.init(file, opts),
57+
};
58+
}
59+
60+
pub fn deinit(self: MemoryMapGrowable) void {
61+
self.view.deinit();
62+
}
63+
64+
/// Grow `file` as necessary to map `len` bytes.
65+
///
66+
/// Invalidates references to `self.view.mapped` if `self.view.mapped.len > len`.
67+
///
68+
/// If the file must be grown, then this may block until a lock can be acquired on the
69+
/// file. For Linux, this does not utilize locks since the growth can be done safely with
70+
/// `fallocate`.
71+
///
72+
/// If an error occurs while growing the file, `self.view.mapped` is invalidated.
73+
/// TODO: Provide some mechanism of recovering from a failed growth.
74+
///
75+
/// This function is precise in that it specifically grows to `len`. That does not
76+
/// prevent external processes from growing the file beyond `len`, and this function does
77+
/// not truncate the file to a lower size.
78+
///
79+
/// Asserts `len` is non-zero.
80+
pub fn ensureTotalCapacityPrecise(
81+
self: *MemoryMapGrowable,
82+
file: std.fs.File,
83+
len: usize,
84+
) !void {
85+
std.debug.assert(len > 0);
86+
if (len <= self.view.mapped.len)
87+
return;
88+
const hint = self.view.mapped.ptr;
89+
const prev_len = self.view.mapped.len;
90+
self.view.deinit();
91+
self.view = undefined;
92+
const actual_len = try file.grow(prev_len, len);
93+
self.view = try MemoryMap.init(file, .{
94+
.protection = self.protection,
95+
.exclusivity = self.exclusivity,
96+
.length = actual_len,
97+
.hint = @volatileCast(hint),
98+
});
99+
}
100+
101+
/// Grow `file` as necessary to map `len` bytes.
102+
///
103+
/// Invalidates references to `self.view.mapped` if `self.view.mapped.len > len`.
104+
///
105+
/// The caller must ensure it is safe to set the end position of the file even in the
106+
/// face of external processes interacting with the file. If concurrent modification of
107+
/// the file length is expected, then `ensureTotalCapacityPrecise` should be used.
108+
///
109+
/// If an error occurs while growing the file, `self.view.mapped` is invalidated.
110+
/// TODO: Provide some mechanism of recovering from a failed growth.
111+
///
112+
/// This function is precise in that it specifically grows to `len`. That does not
113+
/// prevent external processes from growing the file beyond `len`, and this function does
114+
/// not truncate the file to a lower size.
115+
///
116+
/// Asserts `len` is non-zero.
117+
pub fn ensureTotalCapacityPreciseExclusive(
118+
self: *MemoryMapGrowable,
119+
file: std.fs.File,
120+
len: usize,
121+
) !void {
122+
std.debug.assert(len > 0);
123+
if (len <= self.view.mapped.len)
124+
return;
125+
const hint = self.view.mapped.ptr;
126+
self.view.deinit();
127+
self.view = undefined;
128+
try file.setEndPos(len);
129+
self.view = try MemoryMap.init(file, .{
130+
.protection = self.protection,
131+
.exclusivity = self.exclusivity,
132+
.length = len,
133+
.hint = @volatileCast(hint),
134+
});
135+
}

‎lib/std/os/windows.zig

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -201,8 +201,11 @@ pub fn MapViewOfFile(
201201
size,
202202
hint,
203203
);
204-
if (ptr) |p|
205-
return @alignCast(p[0..size]);
204+
if (ptr) |p| {
205+
const cast: [*]align(std.mem.page_size) volatile u8 =
206+
@alignCast(@ptrCast(p));
207+
return @alignCast(cast[0..size]);
208+
}
206209
switch (GetLastError()) {
207210
else => |err| return unexpectedError(err),
208211
}

0 commit comments

Comments
 (0)