Skip to content

Commit a2651cb

Browse files
andrewrkandrewrk
authored
Merge pull request ziglang#19388 from ziglang/cache-dedup
cache system file deduplication
2 parents 54c0857 + 9503590 commit a2651cb

File tree

11 files changed

+450
-372
lines changed

11 files changed

+450
-372
lines changed

‎lib/std/Build/Cache.zig

Lines changed: 151 additions & 140 deletions
Large diffs are not rendered by default.

‎lib/std/Build/Cache/Directory.zig

Lines changed: 74 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,74 @@
1+
const Directory = @This();
2+
const std = @import("../../std.zig");
3+
const fs = std.fs;
4+
const fmt = std.fmt;
5+
const Allocator = std.mem.Allocator;
6+
7+
/// This field is redundant for operations that can act on the open directory handle
8+
/// directly, but it is needed when passing the directory to a child process.
9+
/// `null` means cwd.
10+
path: ?[]const u8,
11+
handle: fs.Dir,
12+
13+
pub fn clone(d: Directory, arena: Allocator) Allocator.Error!Directory {
14+
return .{
15+
.path = if (d.path) |p| try arena.dupe(u8, p) else null,
16+
.handle = d.handle,
17+
};
18+
}
19+
20+
pub fn cwd() Directory {
21+
return .{
22+
.path = null,
23+
.handle = fs.cwd(),
24+
};
25+
}
26+
27+
pub fn join(self: Directory, allocator: Allocator, paths: []const []const u8) ![]u8 {
28+
if (self.path) |p| {
29+
// TODO clean way to do this with only 1 allocation
30+
const part2 = try fs.path.join(allocator, paths);
31+
defer allocator.free(part2);
32+
return fs.path.join(allocator, &[_][]const u8{ p, part2 });
33+
} else {
34+
return fs.path.join(allocator, paths);
35+
}
36+
}
37+
38+
pub fn joinZ(self: Directory, allocator: Allocator, paths: []const []const u8) ![:0]u8 {
39+
if (self.path) |p| {
40+
// TODO clean way to do this with only 1 allocation
41+
const part2 = try fs.path.join(allocator, paths);
42+
defer allocator.free(part2);
43+
return fs.path.joinZ(allocator, &[_][]const u8{ p, part2 });
44+
} else {
45+
return fs.path.joinZ(allocator, paths);
46+
}
47+
}
48+
49+
/// Whether or not the handle should be closed, or the path should be freed
50+
/// is determined by usage, however this function is provided for convenience
51+
/// if it happens to be what the caller needs.
52+
pub fn closeAndFree(self: *Directory, gpa: Allocator) void {
53+
self.handle.close();
54+
if (self.path) |p| gpa.free(p);
55+
self.* = undefined;
56+
}
57+
58+
pub fn format(
59+
self: Directory,
60+
comptime fmt_string: []const u8,
61+
options: fmt.FormatOptions,
62+
writer: anytype,
63+
) !void {
64+
_ = options;
65+
if (fmt_string.len != 0) fmt.invalidFmtError(fmt_string, self);
66+
if (self.path) |p| {
67+
try writer.writeAll(p);
68+
try writer.writeAll(fs.path.sep_str);
69+
}
70+
}
71+
72+
pub fn eql(self: Directory, other: Directory) bool {
73+
return self.handle.fd == other.handle.fd;
74+
}

‎lib/std/Build/Cache/Path.zig

Lines changed: 154 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,154 @@
1+
root_dir: Cache.Directory,
2+
/// The path, relative to the root dir, that this `Path` represents.
3+
/// Empty string means the root_dir is the path.
4+
sub_path: []const u8 = "",
5+
6+
pub fn clone(p: Path, arena: Allocator) Allocator.Error!Path {
7+
return .{
8+
.root_dir = try p.root_dir.clone(arena),
9+
.sub_path = try arena.dupe(u8, p.sub_path),
10+
};
11+
}
12+
13+
pub fn cwd() Path {
14+
return .{ .root_dir = Cache.Directory.cwd() };
15+
}
16+
17+
pub fn join(p: Path, arena: Allocator, sub_path: []const u8) Allocator.Error!Path {
18+
if (sub_path.len == 0) return p;
19+
const parts: []const []const u8 =
20+
if (p.sub_path.len == 0) &.{sub_path} else &.{ p.sub_path, sub_path };
21+
return .{
22+
.root_dir = p.root_dir,
23+
.sub_path = try fs.path.join(arena, parts),
24+
};
25+
}
26+
27+
pub fn resolvePosix(p: Path, arena: Allocator, sub_path: []const u8) Allocator.Error!Path {
28+
if (sub_path.len == 0) return p;
29+
return .{
30+
.root_dir = p.root_dir,
31+
.sub_path = try fs.path.resolvePosix(arena, &.{ p.sub_path, sub_path }),
32+
};
33+
}
34+
35+
pub fn joinString(p: Path, allocator: Allocator, sub_path: []const u8) Allocator.Error![]u8 {
36+
const parts: []const []const u8 =
37+
if (p.sub_path.len == 0) &.{sub_path} else &.{ p.sub_path, sub_path };
38+
return p.root_dir.join(allocator, parts);
39+
}
40+
41+
pub fn joinStringZ(p: Path, allocator: Allocator, sub_path: []const u8) Allocator.Error![:0]u8 {
42+
const parts: []const []const u8 =
43+
if (p.sub_path.len == 0) &.{sub_path} else &.{ p.sub_path, sub_path };
44+
return p.root_dir.joinZ(allocator, parts);
45+
}
46+
47+
pub fn openFile(
48+
p: Path,
49+
sub_path: []const u8,
50+
flags: fs.File.OpenFlags,
51+
) !fs.File {
52+
var buf: [fs.MAX_PATH_BYTES]u8 = undefined;
53+
const joined_path = if (p.sub_path.len == 0) sub_path else p: {
54+
break :p std.fmt.bufPrint(&buf, "{s}" ++ fs.path.sep_str ++ "{s}", .{
55+
p.sub_path, sub_path,
56+
}) catch return error.NameTooLong;
57+
};
58+
return p.root_dir.handle.openFile(joined_path, flags);
59+
}
60+
61+
pub fn makeOpenPath(p: Path, sub_path: []const u8, opts: fs.OpenDirOptions) !fs.Dir {
62+
var buf: [fs.MAX_PATH_BYTES]u8 = undefined;
63+
const joined_path = if (p.sub_path.len == 0) sub_path else p: {
64+
break :p std.fmt.bufPrint(&buf, "{s}" ++ fs.path.sep_str ++ "{s}", .{
65+
p.sub_path, sub_path,
66+
}) catch return error.NameTooLong;
67+
};
68+
return p.root_dir.handle.makeOpenPath(joined_path, opts);
69+
}
70+
71+
pub fn statFile(p: Path, sub_path: []const u8) !fs.Dir.Stat {
72+
var buf: [fs.MAX_PATH_BYTES]u8 = undefined;
73+
const joined_path = if (p.sub_path.len == 0) sub_path else p: {
74+
break :p std.fmt.bufPrint(&buf, "{s}" ++ fs.path.sep_str ++ "{s}", .{
75+
p.sub_path, sub_path,
76+
}) catch return error.NameTooLong;
77+
};
78+
return p.root_dir.handle.statFile(joined_path);
79+
}
80+
81+
pub fn atomicFile(
82+
p: Path,
83+
sub_path: []const u8,
84+
options: fs.Dir.AtomicFileOptions,
85+
buf: *[fs.MAX_PATH_BYTES]u8,
86+
) !fs.AtomicFile {
87+
const joined_path = if (p.sub_path.len == 0) sub_path else p: {
88+
break :p std.fmt.bufPrint(buf, "{s}" ++ fs.path.sep_str ++ "{s}", .{
89+
p.sub_path, sub_path,
90+
}) catch return error.NameTooLong;
91+
};
92+
return p.root_dir.handle.atomicFile(joined_path, options);
93+
}
94+
95+
pub fn access(p: Path, sub_path: []const u8, flags: fs.File.OpenFlags) !void {
96+
var buf: [fs.MAX_PATH_BYTES]u8 = undefined;
97+
const joined_path = if (p.sub_path.len == 0) sub_path else p: {
98+
break :p std.fmt.bufPrint(&buf, "{s}" ++ fs.path.sep_str ++ "{s}", .{
99+
p.sub_path, sub_path,
100+
}) catch return error.NameTooLong;
101+
};
102+
return p.root_dir.handle.access(joined_path, flags);
103+
}
104+
105+
pub fn makePath(p: Path, sub_path: []const u8) !void {
106+
var buf: [fs.MAX_PATH_BYTES]u8 = undefined;
107+
const joined_path = if (p.sub_path.len == 0) sub_path else p: {
108+
break :p std.fmt.bufPrint(&buf, "{s}" ++ fs.path.sep_str ++ "{s}", .{
109+
p.sub_path, sub_path,
110+
}) catch return error.NameTooLong;
111+
};
112+
return p.root_dir.handle.makePath(joined_path);
113+
}
114+
115+
pub fn format(
116+
self: Path,
117+
comptime fmt_string: []const u8,
118+
options: std.fmt.FormatOptions,
119+
writer: anytype,
120+
) !void {
121+
if (fmt_string.len == 1) {
122+
// Quote-escape the string.
123+
const stringEscape = std.zig.stringEscape;
124+
const f = switch (fmt_string[0]) {
125+
'q' => "",
126+
'\'' => '\'',
127+
else => @compileError("unsupported format string: " ++ fmt_string),
128+
};
129+
if (self.root_dir.path) |p| {
130+
try stringEscape(p, f, options, writer);
131+
if (self.sub_path.len > 0) try stringEscape(fs.path.sep_str, f, options, writer);
132+
}
133+
if (self.sub_path.len > 0) {
134+
try stringEscape(self.sub_path, f, options, writer);
135+
}
136+
return;
137+
}
138+
if (fmt_string.len > 0)
139+
std.fmt.invalidFmtError(fmt_string, self);
140+
if (self.root_dir.path) |p| {
141+
try writer.writeAll(p);
142+
try writer.writeAll(fs.path.sep_str);
143+
}
144+
if (self.sub_path.len > 0) {
145+
try writer.writeAll(self.sub_path);
146+
try writer.writeAll(fs.path.sep_str);
147+
}
148+
}
149+
150+
const Path = @This();
151+
const std = @import("../../std.zig");
152+
const fs = std.fs;
153+
const Allocator = std.mem.Allocator;
154+
const Cache = std.Build.Cache;

‎lib/std/Build/Step.zig

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -544,7 +544,7 @@ pub fn cacheHit(s: *Step, man: *std.Build.Cache.Manifest) !bool {
544544

545545
fn failWithCacheError(s: *Step, man: *const std.Build.Cache.Manifest, err: anyerror) anyerror {
546546
const i = man.failed_file_index orelse return err;
547-
const pp = man.files.items[i].prefixed_path orelse return err;
547+
const pp = man.files.keys()[i].prefixed_path;
548548
const prefix = man.cache.prefixes()[pp.prefix].path orelse "";
549549
return s.fail("{s}: {s}/{s}", .{ @errorName(err), prefix, pp.sub_path });
550550
}

0 commit comments

Comments
 (0)