[DNM] Add prototype implementation of a binary static library artifact bundle auditing tool

.dockerignore

@@ -0,0 +1,5 @@
+.build/
+.git/
+Dockerfile
+.dockerignore
+README.md

.gitignore

@@ -20,3 +20,6 @@ Package.resolved
 .vscode
 Utilities/InstalledSwiftPMConfiguration/config.json
 .devcontainer
+
+# Ignore temp stuff from building artifacts
+.o

BinaryStaticLibraryAuditTool/Dockerfile

@@ -0,0 +1,18 @@
+FROM swift:6.0-amazonlinux2
+
+RUN yum update -y && \
+    yum install -y llvm-devel
+
+WORKDIR /build
+
+COPY ./Package.* ./
+RUN swift package resolve
+
+COPY . ./
+RUN swift build -c release --static-swift-stdlib
+
+RUN chmod a+rx entrypoint.sh
+
+ENTRYPOINT ["/build/entrypoint.sh", "/usr/bin/llvm-objdump"]
+
+

BinaryStaticLibraryAuditTool/Package.resolved

@@ -0,0 +1,24 @@
+{
+  "originHash" : "fc7f45f6dd3d234173d788063ed950bea2c5f0a94d5d8a051b7ee6132c9110be",
+  "pins" : [
+    {
+      "identity" : "swift-argument-parser",
+      "kind" : "remoteSourceControl",
+      "location" : "https://github.com/apple/swift-argument-parser.git",
+      "state" : {
+        "revision" : "41982a3656a71c768319979febd796c6fd111d5c",
+        "version" : "1.5.0"
+      }
+    },
+    {
+      "identity" : "swift-system",
+      "kind" : "remoteSourceControl",
+      "location" : "https://github.com/apple/swift-system.git",
+      "state" : {
+        "revision" : "a34201439c74b53f0fd71ef11741af7e7caf01e1",
+        "version" : "1.4.2"
+      }
+    }
+  ],
+  "version" : 3
+}

BinaryStaticLibraryAuditTool/Package.swift

@@ -0,0 +1,50 @@
+// swift-tools-version: 6.0
+
+import PackageDescription
+
+let swiftSettings: [SwiftSetting] = [
+    .enableUpcomingFeature("InternalImportsByDefault"), // SE-0409: https://github.com/swiftlang/swift-evolution/blob/main/proposals/0409-access-level-on-imports.md
+]
+
+let package = Package(
+    name: "BinaryArtifactAudit",
+    platforms: [
+        .macOS(.v13),
+    ],
+    products: [
+        .executable(name: "binary-artifact-audit", targets: ["BinaryArtifactAuditExec"]),
+    ],
+    dependencies: [
+        .package(url: "https://github.com/apple/swift-argument-parser.git", from: "1.5.0"),
+        .package(url: "https://github.com/apple/swift-system.git", from: "1.4.2"),
+    ],
+    targets: [
+        .executableTarget(
+            name: "BinaryArtifactAuditExec",
+            dependencies: [
+                .target(name: "BinaryArtifactAudit"),
+                .product(name: "ArgumentParser", package: "swift-argument-parser"),
+                .product(name: "SystemPackage", package: "swift-system"),
+            ],
+            swiftSettings: swiftSettings
+        ),
+        .target(
+            name: "BinaryArtifactAudit",
+            dependencies: [
+                .product(name: "SystemPackage", package: "swift-system"),
+            ],
+            resources: [
+                .copy("Resources")
+            ],
+            swiftSettings: swiftSettings
+        ),
+        .testTarget(
+            name: "BinaryArtifactAuditTests",
+            dependencies: [.target(name: "BinaryArtifactAudit")],
+            resources: [
+                .copy("TestBundles")
+            ],
+            swiftSettings: swiftSettings
+        ),
+    ]
+)

BinaryStaticLibraryAuditTool/README.md

@@ -0,0 +1,24 @@
+# Binary Artifact Auditing Tool
+
+This tool allows checking that a binary static library artifact will be compatible with any supported Linux Swift deployment platform.
+
+It is intended to be used in conjunction with the provided `Dockerfile` as follows:
+```
+$ docker build -t my-tag .
+$ docker run --rm my-tag <validate-local/validate-remote> <artifact-path/artifact-url>
+```
+
+## Operation
+
+The tool uses the installed `llvm-objdump` to inspect the static library as well as the local libc and any C runtime libraries and/or object files to
+determine if any symbols aren't defined.
+
+## Known Supported Platforms
+
+| Docker Image          | libc.so version |
+| --------------------- | --------------- |
+| 6.0-fedora39          | GNU libc 2.38   |
+| 6.0-rhel-ubi9-slim    | GNU libc 2.34   |
+| 6.0-amazonlinux2-slim | GNU libc 2.26   |
+| 6.0-bookworm          | GNU libc 2.36   |
+| 6.0-focal-slim        | GNU libc 2.31   |

BinaryStaticLibraryAuditTool/Sources/BinaryArtifactAudit/ArtifactBundle.swift

@@ -0,0 +1,58 @@
+package import SystemPackage
+private import Foundation
+
+package struct ArtifactBundle {
+    package let root: FilePath
+    package let manifest: ArtifactManifest
+
+    private init(root: FilePath, manifest: ArtifactManifest) {
+        self.root = root
+        self.manifest = manifest
+    }
+
+    package static func create(reading root: FilePath, in fileSystem: some FileSystem = LocalFileSystem()) throws -> ArtifactBundle {
+        let manifestPath = root.appending("info.json")
+        guard fileSystem.isRegularFile(manifestPath) else {
+            throw Err.noManifest
+        }
+
+        let manifest = try JSONDecoder().decode(ArtifactManifest.self, from: .init(contentsOf: .init(filePath: manifestPath.string)))
+
+        let bundle = ArtifactBundle(root: root, manifest: manifest)
+
+        for (_, artifact) in bundle.manifest.artifacts {
+            try bundle.validateMetadata(artifact, fileSystem: fileSystem)
+        }
+
+        return bundle
+    }
+
+    private func validateMetadata(_ artifact: ArtifactMetadata, fileSystem: some FileSystem) throws {
+        for variant in artifact.variants {
+            guard fileSystem.isRegularFile(root.appending(variant.path.components)) else {
+                throw Err.noSuchVariantPath(variant.path.string)
+            }
+
+            for header in variant.headerPaths {
+                guard fileSystem.isDirectory(root.appending(header.components)) else {
+                    throw Err.noSuchHeaderPath(header.string)
+                }
+            }
+
+            if let modulePath = variant.moduleMapPath {
+                guard fileSystem.isRegularFile(root.appending(modulePath.components)) else {
+                    throw Err.noSuchModuleMapPath(modulePath.string)
+                }
+            }
+        }
+    }
+}
+
+extension ArtifactBundle {
+    package enum Err: Error {
+        case noManifest
+        case noSuchVariantPath(String)
+        case noSuchHeaderPath(String)
+        case noSuchModuleMapPath(String)
+    }
+}

BinaryStaticLibraryAuditTool/Sources/BinaryArtifactAudit/ArtifactBundleProvider.swift

@@ -0,0 +1,5 @@
+package import Foundation
+
+package protocol ArtifactBundleProvider {
+    func artifact(for: URL) async throws -> ArtifactBundle
+}

BinaryStaticLibraryAuditTool/Sources/BinaryArtifactAudit/ArtifactManifest.swift

@@ -0,0 +1,4 @@
+package struct ArtifactManifest: Codable {
+    package var schemaVersion: String
+    package var artifacts: [String: ArtifactMetadata]
+}

BinaryStaticLibraryAuditTool/Sources/BinaryArtifactAudit/ArtifactMetadata+Type.swift

@@ -0,0 +1,5 @@
+extension ArtifactMetadata {
+    package enum ArtifactType: String, Equatable, Codable {
+        case staticLibrary
+    }
+}

BinaryStaticLibraryAuditTool/Sources/BinaryArtifactAudit/ArtifactMetadata+Variant.swift

@@ -0,0 +1,40 @@
+package import SystemPackage
+
+extension ArtifactMetadata {
+    package struct Variant: Codable {
+        package var path: FilePath
+        package var headerPaths: [FilePath]
+        package var moduleMapPath: FilePath? = nil
+        package var supportedTriples: [String]
+
+        enum CodingKeys: String, CodingKey {
+            case path
+            case headerPaths
+            case moduleMapPath
+            case supportedTriples
+        }
+
+        package func encode(to encoder: any Encoder) throws {
+            var container = encoder.container(keyedBy: CodingKeys.self)
+            try container.encode(path.string , forKey: .path)
+            try container.encode(headerPaths.map { $0.string }, forKey: .headerPaths)
+            try container.encodeIfPresent(moduleMapPath.map { $0.string }, forKey: .moduleMapPath)
+            try container.encode(supportedTriples, forKey: .supportedTriples)
+        }
+
+        package init(from decoder: any Decoder) throws {
+            let values = try decoder.container(keyedBy: CodingKeys.self)
+
+            let pathString = try values.decode(String.self, forKey: .path)
+            self.path = FilePath(pathString)
+
+            let headerPathStrings = try values.decode([String].self, forKey: .headerPaths)
+            self.headerPaths = headerPathStrings.map { FilePath($0) }
+
+            let moduleMapPathString = try values.decodeIfPresent(String.self, forKey: .moduleMapPath)
+            self.moduleMapPath = moduleMapPathString.map { FilePath($0) }
+
+            self.supportedTriples = try values.decode([String].self, forKey: .supportedTriples)
+        }
+    }
+}

BinaryStaticLibraryAuditTool/Sources/BinaryArtifactAudit/ArtifactMetadata.swift

@@ -0,0 +1,5 @@
+package struct ArtifactMetadata: Codable {
+    package var version: String
+    package var type: ArtifactType
+    package var variants: [Variant]
+}

BinaryStaticLibraryAuditTool/Sources/BinaryArtifactAudit/ClangAdditionalLibrariesDetector.swift

@@ -0,0 +1,68 @@
+package import SystemPackage
+private import Foundation
+
+package func detectAdditionalObjects() async throws -> Set<FilePath> {
+    guard let sampleExecutable = Bundle.module.url(forResource: "main", withExtension: "c") else {
+        throw AdditionalObjectsDetectorError.missingTestFile
+    }
+
+    let clangOutput = try await Process.run(executable: "/usr/bin/clang", arguments: "-###", sampleExecutable.path())
+    guard let commandStrings = String(data: clangOutput.error, encoding: .utf8)?.split(whereSeparator: \.isNewline) else {
+        throw AdditionalObjectsDetectorError.failedToParseClangOutput
+    }
+
+    let commands = commandStrings.map { $0.split(whereSeparator: \.isWhitespace) }
+    guard let linkerCommand = commands.last(where: { $0.first?.contains("ld") == true }) else {
+        throw AdditionalObjectsDetectorError.couldNotFindLinkerCommand
+    }
+
+    var libraryExtensionsToTry: [String] = []
+#if canImport(Darwin)
+    libraryExtensionsToTry.append(contentsOf: [".a", ".dylib", ".tbd"])
+#elseif os(Windows)
+    libraryExtensionsToTry.append(contentsOf: [".lib", ".dll"])
+#else
+    libraryExtensionsToTry.append(contentsOf: [".a", ".so"])
+#endif
+
+    var objects: Set<FilePath> = []
+    var searchPaths: [FilePath] = []
+    let fileSystem = LocalFileSystem()
+
+    let linkerArguments = linkerCommand.dropFirst().map { $0.replacingOccurrences(of: "\"", with: "") }
+
+    for argument in linkerArguments {
+        if argument.hasPrefix("-L") {
+            let path = FilePath(String(argument.dropFirst(2)))
+            searchPaths.append(path.lexicallyNormalized())
+        } else if argument.hasPrefix("-l") {
+            let libName = String(argument.dropFirst(2))
+            searchPathLoop: for path in searchPaths {
+                for ext in libraryExtensionsToTry {
+                    let potentialLibrary = path.appending("lib\(libName)\(ext)")
+                    if fileSystem.isRegularFile(potentialLibrary) {
+                        objects.insert(potentialLibrary.lexicallyNormalized())
+                        break searchPathLoop
+                    }
+                }
+            }
+
+            assertionFailure("Could not find lib for \(libName)")
+        } else if argument.hasSuffix(".o") && fileSystem.isRegularFile(FilePath(String(argument))) {
+            objects.insert(FilePath(String(argument)).lexicallyNormalized())
+        } else if let dotIndex = argument.firstIndex(of: "."),
+            ["so", "dylib"].first(where: { argument[dotIndex...].contains($0) }) != nil {
+            objects.insert(FilePath(String(argument)).lexicallyNormalized())
+        }
+    }
+
+    objects.remove("/usr/lib/gcc/aarch64-redhat-linux/7/libgcc_s.so")
+    return objects
+}
+
+enum AdditionalObjectsDetectorError: Error {
+    case missingTestFile
+    case failedToParseClangOutput
+    case couldNotFindLinkerCommand
+}
+

BinaryStaticLibraryAuditTool/Sources/BinaryArtifactAudit/FileInfo.swift

@@ -0,0 +1,12 @@
+package import Foundation
+
+package struct FileInfo {
+    package let type: FileAttributeType
+    package init(type: FileAttributeType) {
+        self.type = type
+    }
+
+    package var isRegularFile: Bool { self.type == .typeRegular }
+
+    package var isDirectory: Bool { self.type == .typeDirectory }
+}

BinaryStaticLibraryAuditTool/Sources/BinaryArtifactAudit/FileSystem.swift

@@ -0,0 +1,29 @@
+package import SystemPackage
+
+package protocol FileSystem {
+    func fileInfo(_ path: FilePath) throws -> FileInfo
+
+    func isRegularFile(_ path: FilePath) -> Bool
+
+    func isDirectory(_ path: FilePath) -> Bool
+
+    func createTemporaryDirectory() throws -> FilePath
+}
+
+extension FileSystem {
+    package func isRegularFile(_ path: FilePath) -> Bool {
+        do {
+            return try fileInfo(path).isRegularFile
+        } catch {
+            return false
+        }
+    }
+
+    package func isDirectory(_ path: FilePath) -> Bool {
+        do {
+            return try fileInfo(path).isDirectory
+        } catch {
+            return false
+        }
+    }
+}

BinaryStaticLibraryAuditTool/Sources/BinaryArtifactAudit/LocalArtifactBundleProvider.swift

@@ -0,0 +1,23 @@
+package import Foundation
+private import SystemPackage
+
+package struct LocalArtifactBundleProvider: ArtifactBundleProvider {
+    package init () { }
+
+    package func artifact(for artifactURL: URL) async throws -> ArtifactBundle {
+        let path = FilePath(artifactURL.path())
+        let fileSystem = LocalFileSystem()
+
+        guard fileSystem.isDirectory(path) else {
+            throw Err.noArtifactBundle(path.string)
+        }
+
+        return try ArtifactBundle.create(reading: path, in: fileSystem)
+    }
+}
+
+extension LocalArtifactBundleProvider {
+    package enum Err: Error {
+        case noArtifactBundle(String)
+    }
+}