Merge pull request #17392 from ziglang/fetch

rework package manager

Author: andrewrk

CMakeLists.txt

@@ -528,7 +528,7 @@ set(ZIG_STAGE2_SOURCES
     "${CMAKE_SOURCE_DIR}/src/Liveness.zig"
     "${CMAKE_SOURCE_DIR}/src/Module.zig"
     "${CMAKE_SOURCE_DIR}/src/Package.zig"
-    "${CMAKE_SOURCE_DIR}/src/Package/hash.zig"
+    "${CMAKE_SOURCE_DIR}/src/Package/Fetch.zig"
     "${CMAKE_SOURCE_DIR}/src/RangeSet.zig"
     "${CMAKE_SOURCE_DIR}/src/Sema.zig"
     "${CMAKE_SOURCE_DIR}/src/TypedValue.zig"

build.zig

@@ -88,7 +88,7 @@ pub fn build(b: *std.Build) !void {
         .name = "check-case",
         .root_source_file = .{ .path = "test/src/Cases.zig" },
         .optimize = optimize,
-        .main_pkg_path = .{ .path = "." },
+        .main_mod_path = .{ .path = "." },
     });
     check_case_exe.stack_size = stack_size;
     check_case_exe.single_threaded = single_threaded;

doc/build.zig.zon.md

@@ -0,0 +1,65 @@
+# build.zig.zon Documentation
+
+This is the manifest file for build.zig scripts. It is named build.zig.zon in
+order to make it clear that it is metadata specifically pertaining to
+build.zig.
+
+- **build root** - the directory that contains `build.zig`
+
+## Top-Level Fields
+
+### `name`
+
+String. Required.
+
+### `version`
+
+String. Required.
+
+[semver](https://semver.org/)
+
+### `dependencies`
+
+Struct.
+
+Each dependency must either provide a `url` and `hash`, or a `path`.
+
+#### `url`
+
+String. 
+
+When updating this field to a new URL, be sure to delete the corresponding
+`hash`, otherwise you are communicating that you expect to find the old hash at
+the new URL.
+
+#### `hash`
+
+String. 
+
+[multihash](https://multiformats.io/multihash/)
+
+This is computed from the file contents of the directory of files that is
+obtained after fetching `url` and applying the inclusion rules given by
+`paths`.
+
+This field is the source of truth; packages do not come from an `url`; they
+come from a `hash`. `url` is just one of many possible mirrors for how to
+obtain a package matching this `hash`.
+
+#### `path`
+
+String.
+
+When this is provided, the package is found in a directory relative to the
+build root. In this case the package's hash is irrelevant and therefore not
+computed.
+
+### `paths`
+
+List. Required.
+
+Specifies the set of files and directories that are included in this package.
+Paths are relative to the build root. Use the empty string (`""`) to refer to
+the build root itself.
+
+Only files included in the package are used to compute a package's `hash`.

lib/build_runner.zig

@@ -997,6 +997,7 @@ fn usage(builder: *std.Build, already_ran_build: bool, out_stream: anytype) !voi
         \\  -j<N>                        Limit concurrent jobs (default is to use all CPU cores)
         \\  --maxrss <bytes>             Limit memory usage (default is to use available memory)
         \\  --skip-oom-steps             Instead of failing, skip steps that would exceed --maxrss
+        \\  --fetch                      Exit after fetching dependency tree
         \\
         \\Project-Specific Options:
         \\

lib/std/Build.zig

@@ -634,6 +634,9 @@ pub const ExecutableOptions = struct {
     use_llvm: ?bool = null,
     use_lld: ?bool = null,
     zig_lib_dir: ?LazyPath = null,
+    main_mod_path: ?LazyPath = null,
+
+    /// Deprecated; use `main_mod_path`.
     main_pkg_path: ?LazyPath = null,
 };
 
@@ -652,7 +655,7 @@ pub fn addExecutable(b: *Build, options: ExecutableOptions) *Step.Compile {
         .use_llvm = options.use_llvm,
         .use_lld = options.use_lld,
         .zig_lib_dir = options.zig_lib_dir orelse b.zig_lib_dir,
-        .main_pkg_path = options.main_pkg_path,
+        .main_mod_path = options.main_mod_path orelse options.main_pkg_path,
     });
 }
 
@@ -667,6 +670,9 @@ pub const ObjectOptions = struct {
     use_llvm: ?bool = null,
     use_lld: ?bool = null,
     zig_lib_dir: ?LazyPath = null,
+    main_mod_path: ?LazyPath = null,
+
+    /// Deprecated; use `main_mod_path`.
     main_pkg_path: ?LazyPath = null,
 };
 
@@ -683,7 +689,7 @@ pub fn addObject(b: *Build, options: ObjectOptions) *Step.Compile {
         .use_llvm = options.use_llvm,
         .use_lld = options.use_lld,
         .zig_lib_dir = options.zig_lib_dir orelse b.zig_lib_dir,
-        .main_pkg_path = options.main_pkg_path,
+        .main_mod_path = options.main_mod_path orelse options.main_pkg_path,
     });
 }
 
@@ -699,6 +705,9 @@ pub const SharedLibraryOptions = struct {
     use_llvm: ?bool = null,
     use_lld: ?bool = null,
     zig_lib_dir: ?LazyPath = null,
+    main_mod_path: ?LazyPath = null,
+
+    /// Deprecated; use `main_mod_path`.
     main_pkg_path: ?LazyPath = null,
 };
 
@@ -717,7 +726,7 @@ pub fn addSharedLibrary(b: *Build, options: SharedLibraryOptions) *Step.Compile
         .use_llvm = options.use_llvm,
         .use_lld = options.use_lld,
         .zig_lib_dir = options.zig_lib_dir orelse b.zig_lib_dir,
-        .main_pkg_path = options.main_pkg_path,
+        .main_mod_path = options.main_mod_path orelse options.main_pkg_path,
     });
 }
 
@@ -733,6 +742,9 @@ pub const StaticLibraryOptions = struct {
     use_llvm: ?bool = null,
     use_lld: ?bool = null,
     zig_lib_dir: ?LazyPath = null,
+    main_mod_path: ?LazyPath = null,
+
+    /// Deprecated; use `main_mod_path`.
     main_pkg_path: ?LazyPath = null,
 };
 
@@ -751,7 +763,7 @@ pub fn addStaticLibrary(b: *Build, options: StaticLibraryOptions) *Step.Compile
         .use_llvm = options.use_llvm,
         .use_lld = options.use_lld,
         .zig_lib_dir = options.zig_lib_dir orelse b.zig_lib_dir,
-        .main_pkg_path = options.main_pkg_path,
+        .main_mod_path = options.main_mod_path orelse options.main_pkg_path,
     });
 }
 
@@ -769,6 +781,9 @@ pub const TestOptions = struct {
     use_llvm: ?bool = null,
     use_lld: ?bool = null,
     zig_lib_dir: ?LazyPath = null,
+    main_mod_path: ?LazyPath = null,
+
+    /// Deprecated; use `main_mod_path`.
     main_pkg_path: ?LazyPath = null,
 };
 
@@ -787,7 +802,7 @@ pub fn addTest(b: *Build, options: TestOptions) *Step.Compile {
         .use_llvm = options.use_llvm,
         .use_lld = options.use_lld,
         .zig_lib_dir = options.zig_lib_dir orelse b.zig_lib_dir,
-        .main_pkg_path = options.main_pkg_path,
+        .main_mod_path = options.main_mod_path orelse options.main_pkg_path,
     });
 }
 

lib/std/Build/Cache.zig

@@ -9,6 +9,20 @@ pub const Directory = struct {
     path: ?[]const u8,
     handle: fs.Dir,
 
+    pub fn clone(d: Directory, arena: Allocator) Allocator.Error!Directory {
+        return .{
+            .path = if (d.path) |p| try arena.dupe(u8, p) else null,
+            .handle = d.handle,
+        };
+    }
+
+    pub fn cwd() Directory {
+        return .{
+            .path = null,
+            .handle = fs.cwd(),
+        };
+    }
+
     pub fn join(self: Directory, allocator: Allocator, paths: []const []const u8) ![]u8 {
         if (self.path) |p| {
             // TODO clean way to do this with only 1 allocation
@@ -47,12 +61,16 @@ pub const Directory = struct {
         writer: anytype,
     ) !void {
         _ = options;
-        if (fmt_string.len != 0) fmt.invalidFmtError(fmt, self);
+        if (fmt_string.len != 0) fmt.invalidFmtError(fmt_string, self);
         if (self.path) |p| {
             try writer.writeAll(p);
             try writer.writeAll(fs.path.sep_str);
         }
     }
+
+    pub fn eql(self: Directory, other: Directory) bool {
+        return self.handle.fd == other.handle.fd;
+    }
 };
 
 gpa: Allocator,

lib/std/Build/Step/Compile.zig

@@ -68,7 +68,7 @@ c_std: std.Build.CStd,
 /// Set via options; intended to be read-only after that.
 zig_lib_dir: ?LazyPath,
 /// Set via options; intended to be read-only after that.
-main_pkg_path: ?LazyPath,
+main_mod_path: ?LazyPath,
 exec_cmd_args: ?[]const ?[]const u8,
 filter: ?[]const u8,
 test_evented_io: bool = false,
@@ -316,6 +316,9 @@ pub const Options = struct {
     use_llvm: ?bool = null,
     use_lld: ?bool = null,
     zig_lib_dir: ?LazyPath = null,
+    main_mod_path: ?LazyPath = null,
+
+    /// deprecated; use `main_mod_path`.
     main_pkg_path: ?LazyPath = null,
 };
 
@@ -480,7 +483,7 @@ pub fn create(owner: *std.Build, options: Options) *Compile {
         .installed_headers = ArrayList(*Step).init(owner.allocator),
         .c_std = std.Build.CStd.C99,
         .zig_lib_dir = null,
-        .main_pkg_path = null,
+        .main_mod_path = null,
         .exec_cmd_args = null,
         .filter = options.filter,
         .test_runner = options.test_runner,
@@ -515,8 +518,8 @@ pub fn create(owner: *std.Build, options: Options) *Compile {
         lp.addStepDependencies(&self.step);
     }
 
-    if (options.main_pkg_path) |lp| {
-        self.main_pkg_path = lp.dupe(self.step.owner);
+    if (options.main_mod_path orelse options.main_pkg_path) |lp| {
+        self.main_mod_path = lp.dupe(self.step.owner);
         lp.addStepDependencies(&self.step);
     }
 
@@ -1998,8 +2001,8 @@ fn make(step: *Step, prog_node: *std.Progress.Node) !void {
         try zig_args.append(dir.getPath(b));
     }
 
-    if (self.main_pkg_path) |dir| {
-        try zig_args.append("--main-pkg-path");
+    if (self.main_mod_path) |dir| {
+        try zig_args.append("--main-mod-path");
         try zig_args.append(dir.getPath(b));
     }
 

lib/std/array_hash_map.zig

@@ -1229,14 +1229,41 @@ pub fn ArrayHashMapUnmanaged(
         /// Sorts the entries and then rebuilds the index.
         /// `sort_ctx` must have this method:
         /// `fn lessThan(ctx: @TypeOf(ctx), a_index: usize, b_index: usize) bool`
+        /// Uses a stable sorting algorithm.
         pub inline fn sort(self: *Self, sort_ctx: anytype) void {
             if (@sizeOf(ByIndexContext) != 0)
                 @compileError("Cannot infer context " ++ @typeName(Context) ++ ", call sortContext instead.");
-            return self.sortContext(sort_ctx, undefined);
+            return sortContextInternal(self, .stable, sort_ctx, undefined);
         }
 
-        pub fn sortContext(self: *Self, sort_ctx: anytype, ctx: Context) void {
-            self.entries.sort(sort_ctx);
+        /// Sorts the entries and then rebuilds the index.
+        /// `sort_ctx` must have this method:
+        /// `fn lessThan(ctx: @TypeOf(ctx), a_index: usize, b_index: usize) bool`
+        /// Uses an unstable sorting algorithm.
+        pub inline fn sortUnstable(self: *Self, sort_ctx: anytype) void {
+            if (@sizeOf(ByIndexContext) != 0)
+                @compileError("Cannot infer context " ++ @typeName(Context) ++ ", call sortUnstableContext instead.");
+            return self.sortContextInternal(.unstable, sort_ctx, undefined);
+        }
+
+        pub inline fn sortContext(self: *Self, sort_ctx: anytype, ctx: Context) void {
+            return sortContextInternal(self, .stable, sort_ctx, ctx);
+        }
+
+        pub inline fn sortUnstableContext(self: *Self, sort_ctx: anytype, ctx: Context) void {
+            return sortContextInternal(self, .unstable, sort_ctx, ctx);
+        }
+
+        fn sortContextInternal(
+            self: *Self,
+            comptime mode: std.sort.Mode,
+            sort_ctx: anytype,
+            ctx: Context,
+        ) void {
+            switch (mode) {
+                .stable => self.entries.sort(sort_ctx),
+                .unstable => self.entries.sortUnstable(sort_ctx),
+            }
             const header = self.index_header orelse return;
             header.reset();
             self.insertAllEntriesIntoNewHeader(if (store_hash) {} else ctx, header);

lib/std/fs/path.zig

@@ -728,15 +728,17 @@ pub fn resolvePosix(allocator: Allocator, paths: []const []const u8) Allocator.E
     }
 }
 
-test "resolve" {
+test resolve {
     try testResolveWindows(&[_][]const u8{ "a\\b\\c\\", "..\\..\\.." }, ".");
     try testResolveWindows(&[_][]const u8{"."}, ".");
+    try testResolveWindows(&[_][]const u8{""}, ".");
 
     try testResolvePosix(&[_][]const u8{ "a/b/c/", "../../.." }, ".");
     try testResolvePosix(&[_][]const u8{"."}, ".");
+    try testResolvePosix(&[_][]const u8{""}, ".");
 }
 
-test "resolveWindows" {
+test resolveWindows {
     try testResolveWindows(
         &[_][]const u8{ "Z:\\", "/usr/local", "lib\\zig\\std\\array_list.zig" },
         "Z:\\usr\\local\\lib\\zig\\std\\array_list.zig",
@@ -764,7 +766,7 @@ test "resolveWindows" {
     try testResolveWindows(&[_][]const u8{"a/b"}, "a\\b");
 }
 
-test "resolvePosix" {
+test resolvePosix {
     try testResolvePosix(&.{ "/a/b", "c" }, "/a/b/c");
     try testResolvePosix(&.{ "/a/b", "c", "//d", "e///" }, "/d/e");
     try testResolvePosix(&.{ "/a/b/c", "..", "../" }, "/a");

lib/std/multi_array_list.zig

@@ -467,7 +467,7 @@ pub fn MultiArrayList(comptime T: type) type {
 
         /// `ctx` has the following method:
         /// `fn lessThan(ctx: @TypeOf(ctx), a_index: usize, b_index: usize) bool`
-        fn sortInternal(self: Self, a: usize, b: usize, ctx: anytype, comptime mode: enum { stable, unstable }) void {
+        fn sortInternal(self: Self, a: usize, b: usize, ctx: anytype, comptime mode: std.sort.Mode) void {
             const sort_context: struct {
                 sub_ctx: @TypeOf(ctx),
                 slice: Slice,

lib/std/process.zig

@@ -46,7 +46,7 @@ pub fn getCwdAlloc(allocator: Allocator) ![]u8 {
     }
 }
 
-test "getCwdAlloc" {
+test getCwdAlloc {
     if (builtin.os.tag == .wasi) return error.SkipZigTest;
 
     const cwd = try getCwdAlloc(testing.allocator);

lib/std/sort.zig

@@ -4,6 +4,8 @@ const testing = std.testing;
 const mem = std.mem;
 const math = std.math;
 
+pub const Mode = enum { stable, unstable };
+
 pub const block = @import("sort/block.zig").block;
 pub const pdq = @import("sort/pdq.zig").pdq;
 pub const pdqContext = @import("sort/pdq.zig").pdqContext;