DO NOT MERGE: Repro atomics crash

.github/workflows/pull_request.yml

@@ -37,7 +37,7 @@ jobs:
       - name: Prepare the action
         run: ./scripts/prep-gh-action.sh --install-swiftly
       - name: Build and Test
-        run: swift test
+        run: SWIFT_DEBUG_FAILED_TYPE_LOOKUP=y swift test --no-parallel --filter HTTPClientTests
 
   releasebuildcheck:
     name: Release Build Check

Package.swift

@@ -26,6 +26,7 @@ let package = Package(
         .package(url: "https://github.com/apple/swift-docc-plugin", from: "1.3.0"),
         .package(url: "https://github.com/apple/swift-openapi-generator", from: "1.6.0"),
         .package(url: "https://github.com/apple/swift-openapi-runtime", from: "1.7.0"),
+        .package(url: "https://github.com/apple/swift-system", from: "1.4.2"),
         // This dependency provides the correct version of the formatter so that you can run `swift run swiftformat Package.swift Plugins/ Sources/ Tests/`
         .package(url: "https://github.com/nicklockwood/SwiftFormat", exact: "0.49.18"),
     ],
@@ -38,6 +39,7 @@ let package = Package(
                 .target(name: "LinuxPlatform", condition: .when(platforms: [.linux])),
                 .target(name: "MacOSPlatform", condition: .when(platforms: [.macOS])),
                 .product(name: "SwiftToolsSupport-auto", package: "swift-tools-support-core"),
+                .product(name: "SystemPackage", package: "swift-system"),
             ]
         ),
         .executableTarget(
@@ -58,6 +60,7 @@ let package = Package(
                 .product(name: "NIOFoundationCompat", package: "swift-nio"),
                 .product(name: "OpenAPIRuntime", package: "swift-openapi-runtime"),
                 .product(name: "OpenAPIAsyncHTTPClient", package: "swift-openapi-async-http-client"),
+                .product(name: "SystemPackage", package: "swift-system"),
             ],
         ),
         .target(
@@ -114,6 +117,7 @@ let package = Package(
             dependencies: [
                 "SwiftlyCore",
                 "CLibArchive",
+                .product(name: "SystemPackage", package: "swift-system"),
             ],
             linkerSettings: [
                 .linkedLibrary("z"),
@@ -123,6 +127,7 @@ let package = Package(
             name: "MacOSPlatform",
             dependencies: [
                 "SwiftlyCore",
+                .product(name: "SystemPackage", package: "swift-system"),
             ]
         ),
         .systemLibrary(
@@ -134,7 +139,10 @@ let package = Package(
         ),
         .testTarget(
             name: "SwiftlyTests",
-            dependencies: ["Swiftly"],
+            dependencies: [
+                "Swiftly",
+                .product(name: "SystemPackage", package: "swift-system"),
+            ],
             resources: [
                 .embedInCode("mock-signing-key-private.pgp"),
             ]

Sources/LinuxPlatform/Extract.swift

@@ -1,5 +1,6 @@
 import CLibArchive
 import Foundation
+import SystemPackage
 
 // The code in this file consists mainly of a Swift port of the "Complete Extractor" example included in the libarchive
 // documentation: https://github.com/libarchive/libarchive/wiki/Examples#a-complete-extractor
@@ -44,7 +45,7 @@ func copyData(readArchive: OpaquePointer?, writeArchive: OpaquePointer?) throws
 /// the provided closure which will return the path the file will be written to.
 ///
 /// This uses libarchive under the hood, so a wide variety of archive formats are supported (e.g. .tar.gz).
-func extractArchive(atPath archivePath: URL, transform: (String) -> URL) throws {
+func extractArchive(atPath archivePath: FilePath, transform: (String) -> FilePath) throws {
     var flags = Int32(0)
     flags = ARCHIVE_EXTRACT_TIME
     flags |= ARCHIVE_EXTRACT_PERM
@@ -66,8 +67,8 @@ func extractArchive(atPath archivePath: URL, transform: (String) -> URL) throws
         archive_write_free(ext)
     }
 
-    if archive_read_open_filename(a, archivePath.path, 10240) != 0 {
-        throw ExtractError(message: "Failed to open \"\(archivePath.path)\"")
+    if archive_read_open_filename(a, archivePath.string, 10240) != 0 {
+        throw ExtractError(message: "Failed to open \"\(archivePath)\"")
     }
 
     while true {
@@ -82,7 +83,7 @@ func extractArchive(atPath archivePath: URL, transform: (String) -> URL) throws
         }
 
         let currentPath = String(cString: archive_entry_pathname(entry))
-        archive_entry_set_pathname(entry, transform(currentPath).path)
+        archive_entry_set_pathname(entry, transform(currentPath).string)
         r = archive_write_header(ext, entry)
         guard r == ARCHIVE_OK else {
             throw ExtractError(archive: ext)

Sources/LinuxPlatform/Linux.swift

@@ -1,5 +1,6 @@
 import Foundation
 import SwiftlyCore
+import SystemPackage
 
 /// `Platform` implementation for Linux systems.
 /// This implementation can be reused for any supported Linux platform.
@@ -18,24 +19,22 @@ public struct Linux: Platform {
 
     public init() {}
 
-    public var defaultSwiftlyHomeDirectory: URL {
+    public var defaultSwiftlyHomeDir: FilePath {
         if let dir = ProcessInfo.processInfo.environment["XDG_DATA_HOME"] {
-            return URL(fileURLWithPath: dir).appendingPathComponent("swiftly", isDirectory: true)
+            return FilePath(dir) / "swiftly"
         } else {
-            return FileManager.default.homeDirectoryForCurrentUser
-                .appendingPathComponent(".local/share/swiftly", isDirectory: true)
+            return homeDir / ".local/share/swiftly"
         }
     }
 
-    public func swiftlyBinDir(_ ctx: SwiftlyCoreContext) -> URL {
-        ctx.mockedHomeDir.map { $0.appendingPathComponent("bin", isDirectory: true) }
-            ?? ProcessInfo.processInfo.environment["SWIFTLY_BIN_DIR"].map { URL(fileURLWithPath: $0) }
-            ?? FileManager.default.homeDirectoryForCurrentUser
-            .appendingPathComponent(".local/share/swiftly/bin", isDirectory: true)
+    public func swiftlyBinDir(_ ctx: SwiftlyCoreContext) -> FilePath {
+        ctx.mockedHomeDir.map { $0 / "bin" }
+            ?? ProcessInfo.processInfo.environment["SWIFTLY_BIN_DIR"].map { FilePath($0) }
+            ?? homeDir / ".local/share/swiftly/bin"
     }
 
-    public func swiftlyToolchainsDir(_ ctx: SwiftlyCoreContext) -> URL {
-        self.swiftlyHomeDir(ctx).appendingPathComponent("toolchains", isDirectory: true)
+    public func swiftlyToolchainsDir(_ ctx: SwiftlyCoreContext) -> FilePath {
+        self.swiftlyHomeDir(ctx) / "toolchains"
     }
 
     public var toolchainFileExtension: String {
@@ -45,12 +44,12 @@ public struct Linux: Platform {
     private static let skipVerificationMessage: String =
         "To skip signature verification, specify the --no-verify flag."
 
-    public func verifySwiftlySystemPrerequisites() throws {
+    public func verifySwiftlySystemPrerequisites() async throws {
         // Check if the root CA certificates are installed on this system for NIOSSL to use.
         // This list comes from LinuxCABundle.swift in NIOSSL.
         var foundTrustedCAs = false
         for crtFile in ["/etc/ssl/certs/ca-certificates.crt", "/etc/pki/tls/certs/ca-bundle.crt"] {
-            if URL(fileURLWithPath: crtFile).fileExists() {
+            if try await fileExists(atPath: FilePath(crtFile)) {
                 foundTrustedCAs = true
                 break
             }
@@ -267,21 +266,17 @@ public struct Linux: Platform {
             }
 
             let tmpFile = self.getTempFilePath()
-            let _ = FileManager.default.createFile(
-                atPath: tmpFile.path, contents: nil, attributes: [.posixPermissions: 0o600]
-            )
-            defer {
-                try? FileManager.default.removeItem(at: tmpFile)
-            }
-
-            try await ctx.httpClient.getGpgKeys().download(to: tmpFile)
-            if let mockedHomeDir = ctx.mockedHomeDir {
-                try self.runProgram(
-                    "gpg", "--import", tmpFile.path, quiet: true,
-                    env: ["GNUPGHOME": mockedHomeDir.appendingPathComponent(".gnupg").path]
-                )
-            } else {
-                try self.runProgram("gpg", "--import", tmpFile.path, quiet: true)
+            try await create(file: tmpFile, contents: nil, mode: 0o600)
+            try await withTemporary(files: tmpFile) {
+                try await ctx.httpClient.getGpgKeys().download(to: tmpFile)
+                if let mockedHomeDir = ctx.mockedHomeDir {
+                    try self.runProgram(
+                        "gpg", "--import", "\(tmpFile)", quiet: true,
+                        env: ["GNUPGHOME": (mockedHomeDir / ".gnupg").string]
+                    )
+                } else {
+                    try self.runProgram("gpg", "--import", "\(tmpFile)", quiet: true)
+                }
             }
         }
 
@@ -333,69 +328,63 @@ public struct Linux: Platform {
     }
 
     public func install(
-        _ ctx: SwiftlyCoreContext, from tmpFile: URL, version: ToolchainVersion, verbose: Bool
+        _ ctx: SwiftlyCoreContext, from tmpFile: FilePath, version: ToolchainVersion, verbose: Bool
     ) async throws {
-        guard tmpFile.fileExists() else {
+        guard try await fileExists(atPath: tmpFile) else {
             throw SwiftlyError(message: "\(tmpFile) doesn't exist")
         }
 
-        if !self.swiftlyToolchainsDir(ctx).fileExists() {
-            try FileManager.default.createDirectory(
-                at: self.swiftlyToolchainsDir(ctx), withIntermediateDirectories: false
-            )
+        if !(try await fileExists(atPath: self.swiftlyToolchainsDir(ctx))) {
+            try await mkdir(atPath: self.swiftlyToolchainsDir(ctx))
         }
 
         await ctx.print("Extracting toolchain...")
-        let toolchainDir = self.swiftlyToolchainsDir(ctx).appendingPathComponent(version.name)
+        let toolchainDir = self.swiftlyToolchainsDir(ctx) / version.name
 
-        if toolchainDir.fileExists() {
-            try FileManager.default.removeItem(at: toolchainDir)
+        if try await fileExists(atPath: toolchainDir) {
+            try await remove(atPath: toolchainDir)
         }
 
         try extractArchive(atPath: tmpFile) { name in
             // drop swift-a.b.c-RELEASE etc name from the extracted files.
             let relativePath = name.drop { c in c != "/" }.dropFirst()
 
             // prepend /path/to/swiftlyHomeDir/toolchains/<toolchain> to each file name
-            let destination = toolchainDir.appendingPathComponent(String(relativePath))
+            let destination = toolchainDir / String(relativePath)
 
             if verbose {
                 // To avoid having to make extractArchive async this is a regular print
                 //  to stdout. Note that it is unlikely that the test mocking will require
                 //  capturing this output.
-                print("\(destination.path)")
+                print("\(destination)")
             }
 
             // prepend /path/to/swiftlyHomeDir/toolchains/<toolchain> to each file name
             return destination
         }
     }
 
-    public func extractSwiftlyAndInstall(_ ctx: SwiftlyCoreContext, from archive: URL) async throws {
-        guard archive.fileExists() else {
+    public func extractSwiftlyAndInstall(_ ctx: SwiftlyCoreContext, from archive: FilePath) async throws {
+        guard try await fileExists(atPath: archive) else {
             throw SwiftlyError(message: "\(archive) doesn't exist")
         }
 
         let tmpDir = self.getTempFilePath()
-        defer {
-            try? FileManager.default.removeItem(at: tmpDir)
-        }
-        try FileManager.default.createDirectory(atPath: tmpDir.path, withIntermediateDirectories: true)
+        try await mkdir(atPath: tmpDir, parents: true)
+        try await withTemporary(files: tmpDir) {
+            await ctx.print("Extracting new swiftly...")
+            try extractArchive(atPath: archive) { name in
+                // Extract to the temporary directory
+                tmpDir / String(name)
+            }
 
-        await ctx.print("Extracting new swiftly...")
-        try extractArchive(atPath: archive) { name in
-            // Extract to the temporary directory
-            tmpDir.appendingPathComponent(String(name))
+            try self.runProgram((tmpDir / "swiftly").string, "init")
         }
-
-        try self.runProgram(tmpDir.appendingPathComponent("swiftly").path, "init")
     }
 
-    public func uninstall(_ ctx: SwiftlyCoreContext, _ toolchain: ToolchainVersion, verbose _: Bool)
-        throws
-    {
-        let toolchainDir = self.swiftlyToolchainsDir(ctx).appendingPathComponent(toolchain.name)
-        try FileManager.default.removeItem(at: toolchainDir)
+    public func uninstall(_ ctx: SwiftlyCoreContext, _ toolchain: ToolchainVersion, verbose _: Bool) async throws {
+        let toolchainDir = self.swiftlyToolchainsDir(ctx) / toolchain.name
+        try await remove(atPath: toolchainDir)
     }
 
     public func getExecutableName() -> String {
@@ -404,69 +393,65 @@ public struct Linux: Platform {
         return "swiftly-\(arch)-unknown-linux-gnu"
     }
 
-    public func getTempFilePath() -> URL {
-        FileManager.default.temporaryDirectory.appendingPathComponent("swiftly-\(UUID())")
+    public func getTempFilePath() -> FilePath {
+        tmpDir / "swiftly-\(UUID())"
     }
 
     public func verifyToolchainSignature(
-        _ ctx: SwiftlyCoreContext, toolchainFile: ToolchainFile, archive: URL, verbose: Bool
+        _ ctx: SwiftlyCoreContext, toolchainFile: ToolchainFile, archive: FilePath, verbose: Bool
     ) async throws {
         if verbose {
             await ctx.print("Downloading toolchain signature...")
         }
 
         let sigFile = self.getTempFilePath()
-        let _ = FileManager.default.createFile(atPath: sigFile.path, contents: nil)
-        defer {
-            try? FileManager.default.removeItem(at: sigFile)
-        }
-
-        try await ctx.httpClient.getSwiftToolchainFileSignature(toolchainFile).download(to: sigFile)
-
-        await ctx.print("Verifying toolchain signature...")
-        do {
-            if let mockedHomeDir = ctx.mockedHomeDir {
-                try self.runProgram(
-                    "gpg", "--verify", sigFile.path, archive.path, quiet: false,
-                    env: ["GNUPGHOME": mockedHomeDir.appendingPathComponent(".gnupg").path]
-                )
-            } else {
-                try self.runProgram("gpg", "--verify", sigFile.path, archive.path, quiet: !verbose)
+        try await create(file: sigFile, contents: nil)
+        try await withTemporary(files: sigFile) {
+            try await ctx.httpClient.getSwiftToolchainFileSignature(toolchainFile).download(to: sigFile)
+
+            await ctx.print("Verifying toolchain signature...")
+            do {
+                if let mockedHomeDir = ctx.mockedHomeDir {
+                    try self.runProgram(
+                        "gpg", "--verify", "\(sigFile)", "\(archive)", quiet: false,
+                        env: ["GNUPGHOME": (mockedHomeDir / ".gnupg").string]
+                    )
+                } else {
+                    try self.runProgram("gpg", "--verify", "\(sigFile)", "\(archive)", quiet: !verbose)
+                }
+            } catch {
+                throw SwiftlyError(message: "Signature verification failed: \(error).")
             }
-        } catch {
-            throw SwiftlyError(message: "Signature verification failed: \(error).")
         }
     }
 
     public func verifySwiftlySignature(
-        _ ctx: SwiftlyCoreContext, archiveDownloadURL: URL, archive: URL, verbose: Bool
+        _ ctx: SwiftlyCoreContext, archiveDownloadURL: URL, archive: FilePath, verbose: Bool
     ) async throws {
         if verbose {
             await ctx.print("Downloading swiftly signature...")
         }
 
         let sigFile = self.getTempFilePath()
-        let _ = FileManager.default.createFile(atPath: sigFile.path, contents: nil)
-        defer {
-            try? FileManager.default.removeItem(at: sigFile)
-        }
-
-        try await ctx.httpClient.getSwiftlyReleaseSignature(
-            url: archiveDownloadURL.appendingPathExtension("sig")
-        ).download(to: sigFile)
-
-        await ctx.print("Verifying swiftly signature...")
-        do {
-            if let mockedHomeDir = ctx.mockedHomeDir {
-                try self.runProgram(
-                    "gpg", "--verify", sigFile.path, archive.path, quiet: false,
-                    env: ["GNUPGHOME": mockedHomeDir.appendingPathComponent(".gnupg").path]
-                )
-            } else {
-                try self.runProgram("gpg", "--verify", sigFile.path, archive.path, quiet: !verbose)
+        try await create(file: sigFile, contents: nil)
+        try await withTemporary(files: sigFile) {
+            try await ctx.httpClient.getSwiftlyReleaseSignature(
+                url: archiveDownloadURL.appendingPathExtension("sig")
+            ).download(to: sigFile)
+
+            await ctx.print("Verifying swiftly signature...")
+            do {
+                if let mockedHomeDir = ctx.mockedHomeDir {
+                    try self.runProgram(
+                        "gpg", "--verify", "\(sigFile)", "\(archive)", quiet: false,
+                        env: ["GNUPGHOME": (mockedHomeDir / ".gnupg").string]
+                    )
+                } else {
+                    try self.runProgram("gpg", "--verify", "\(sigFile)", "\(archive)", quiet: !verbose)
+                }
+            } catch {
+                throw SwiftlyError(message: "Signature verification failed: \(error).")
             }
-        } catch {
-            throw SwiftlyError(message: "Signature verification failed: \(error).")
         }
     }
 
@@ -631,9 +616,9 @@ public struct Linux: Platform {
         return "/bin/bash"
     }
 
-    public func findToolchainLocation(_ ctx: SwiftlyCoreContext, _ toolchain: ToolchainVersion) -> URL
+    public func findToolchainLocation(_ ctx: SwiftlyCoreContext, _ toolchain: ToolchainVersion) -> FilePath
     {
-        self.swiftlyToolchainsDir(ctx).appendingPathComponent("\(toolchain.name)")
+        self.swiftlyToolchainsDir(ctx) / "\(toolchain.name)"
     }
 
     public static let currentPlatform: any Platform = Linux()