Merge branch 'clean up writeFileAllUnseekable by using readers'

closes #7156

Author: andrewrk

CMakeLists.txt

@@ -375,10 +375,11 @@ set(ZIG_STAGE2_SOURCES
     "${CMAKE_SOURCE_DIR}/lib/std/io/buffered_atomic_file.zig"
     "${CMAKE_SOURCE_DIR}/lib/std/io/buffered_writer.zig"
     "${CMAKE_SOURCE_DIR}/lib/std/io/change_detection_stream.zig"
-    "${CMAKE_SOURCE_DIR}/lib/std/io/counting_writer.zig"
     "${CMAKE_SOURCE_DIR}/lib/std/io/counting_reader.zig"
+    "${CMAKE_SOURCE_DIR}/lib/std/io/counting_writer.zig"
     "${CMAKE_SOURCE_DIR}/lib/std/io/find_byte_writer.zig"
     "${CMAKE_SOURCE_DIR}/lib/std/io/fixed_buffer_stream.zig"
+    "${CMAKE_SOURCE_DIR}/lib/std/io/limited_reader.zig"
     "${CMAKE_SOURCE_DIR}/lib/std/io/reader.zig"
     "${CMAKE_SOURCE_DIR}/lib/std/io/seekable_stream.zig"
     "${CMAKE_SOURCE_DIR}/lib/std/io/writer.zig"

lib/std/fs/file.zig

@@ -698,7 +698,7 @@ pub const File = struct {
         header_count: usize = 0,
     };
 
-    pub const WriteFileError = ReadError || WriteError;
+    pub const WriteFileError = ReadError || error{EndOfStream} || WriteError;
 
     pub fn writeFileAll(self: File, in_file: File, args: WriteFileOptions) WriteFileError!void {
         return self.writeFileAllSendfile(in_file, args) catch |err| switch (err) {
@@ -722,23 +722,14 @@ pub const File = struct {
 
         try self.writevAll(headers);
 
-        var buffer: [4096]u8 = undefined;
-        {
-            var index: usize = 0;
-            // Skip in_offset bytes.
-            while (index < args.in_offset) {
-                const ask = math.min(buffer.len, args.in_offset - index);
-                const amt = try in_file.read(buffer[0..ask]);
-                index += amt;
-            }
-        }
-        const in_len = args.in_len orelse math.maxInt(u64);
-        var index: usize = 0;
-        while (index < in_len) {
-            const ask = math.min(buffer.len, in_len - index);
-            const amt = try in_file.read(buffer[0..ask]);
-            if (amt == 0) break;
-            index += try self.write(buffer[0..amt]);
+        try in_file.reader().skipBytes(args.in_offset, .{ .buf_size = 4096 });
+
+        var fifo = std.fifo.LinearFifo(u8, .{ .Static = 4096 }).init();
+        if (args.in_len) |len| {
+            var stream = std.io.limitedReader(in_file.reader(), len);
+            try fifo.pump(stream.reader(), self.writer());
+        } else {
+            try fifo.pump(in_file.reader(), self.writer());
         }
 
         try self.writevAll(trailers);

lib/std/io.zig

@@ -125,6 +125,9 @@ pub const fixedBufferStream = @import("io/fixed_buffer_stream.zig").fixedBufferS
 pub const CWriter = @import("io/c_writer.zig").CWriter;
 pub const cWriter = @import("io/c_writer.zig").cWriter;
 
+pub const LimitedReader = @import("io/limited_reader.zig").LimitedReader;
+pub const limitedReader = @import("io/limited_reader.zig").limitedReader;
+
 pub const CountingWriter = @import("io/counting_writer.zig").CountingWriter;
 pub const countingWriter = @import("io/counting_writer.zig").countingWriter;
 pub const CountingReader = @import("io/counting_reader.zig").CountingReader;

lib/std/io/limited_reader.zig

@@ -0,0 +1,50 @@
+// SPDX-License-Identifier: MIT
+// Copyright (c) 2015-2020 Zig Contributors
+// This file is part of [zig](https://ziglang.org/), which is MIT licensed.
+// The MIT license requires this copyright notice to be included in all copies
+// and substantial portions of the software.
+const std = @import("../std.zig");
+const io = std.io;
+const assert = std.debug.assert;
+const testing = std.testing;
+
+pub fn LimitedReader(comptime ReaderType: type) type {
+    return struct {
+        inner_reader: ReaderType,
+        bytes_left: u64,
+
+        pub const Error = ReaderType.Error;
+        pub const Reader = io.Reader(*Self, Error, read);
+
+        const Self = @This();
+
+        pub fn read(self: *Self, dest: []u8) Error!usize {
+            const max_read = std.math.min(self.bytes_left, dest.len);
+            const n = try self.inner_reader.read(dest[0..max_read]);
+            self.bytes_left -= n;
+            return n;
+        }
+
+        pub fn reader(self: *Self) Reader {
+            return .{ .context = self };
+        }
+    };
+}
+
+/// Returns an initialised `LimitedReader`
+/// `bytes_left` is a `u64` to be able to take 64 bit file offsets
+pub fn limitedReader(inner_reader: anytype, bytes_left: u64) LimitedReader(@TypeOf(inner_reader)) {
+    return .{ .inner_reader = inner_reader, .bytes_left = bytes_left };
+}
+
+test "basic usage" {
+    const data = "hello world";
+    var fbs = std.io.fixedBufferStream(data);
+    var early_stream = limitedReader(fbs.reader(), 3);
+
+    var buf: [5]u8 = undefined;
+    testing.expectEqual(@as(usize, 3), try early_stream.reader().read(&buf));
+    testing.expectEqualSlices(u8, data[0..3], buf[0..3]);
+    testing.expectEqual(@as(usize, 0), try early_stream.reader().read(&buf));
+    testing.expectError(error.EndOfStream, early_stream.reader().skipBytes(10, .{}));
+}

lib/std/io/reader.zig

@@ -271,8 +271,9 @@ pub fn Reader(
             buf_size: usize = 512,
         };
 
+        // `num_bytes` is a `u64` to match `off_t`
         /// Reads `num_bytes` bytes from the stream and discards them
-        pub fn skipBytes(self: Self, num_bytes: usize, comptime options: SkipBytesOptions) !void {
+        pub fn skipBytes(self: Self, num_bytes: u64, comptime options: SkipBytesOptions) !void {
             var buf: [options.buf_size]u8 = undefined;
             var remaining = num_bytes;