diff --git a/include/swift/Option/Options.td b/include/swift/Option/Options.td index efc95fc590596..6db1e5d81394f 100644 --- a/include/swift/Option/Options.td +++ b/include/swift/Option/Options.td @@ -173,12 +173,12 @@ def verify_incremental_dependencies : Flag<["-"], "verify-incremental-dependencies">, Flags<[FrontendOption, HelpHidden]>, HelpText<"Enable the dependency verifier for each frontend job">; - + def strict_implicit_module_context : Flag<["-"], "strict-implicit-module-context">, Flags<[FrontendOption, HelpHidden]>, HelpText<"Enable the strict forwarding of compilation context to downstream implicit module dependencies">; - + def no_strict_implicit_module_context : Flag<["-"], "no-strict-implicit-module-context">, Flags<[FrontendOption, HelpHidden]>, @@ -315,7 +315,7 @@ def tools_directory : Separate<["-"], "tools-directory">, def D : JoinedOrSeparate<["-"], "D">, Flags<[FrontendOption]>, HelpText<"Marks a conditional compilation flag as true">; - + def e : Separate<["-"], "e">, Flags<[NewDriverOnlyOption]>, HelpText<"Executes a line of code provided on the command line">; @@ -2106,7 +2106,7 @@ def external_plugin_path : Separate<["-"], "external-plugin-path">, Group, HelpText<"Add directory to the plugin search path with a plugin server executable">, MetaVarName<"#">; - + def cas_backend: Flag<["-"], "cas-backend">, Flags<[FrontendOption, NoDriverOption]>, HelpText<"Enable using CASBackend for object file output">; diff --git a/lib/Basic/Sandbox.cpp b/lib/Basic/Sandbox.cpp index 3c60ef9c10101..f0055feaeb1f6 100644 --- a/lib/Basic/Sandbox.cpp +++ b/lib/Basic/Sandbox.cpp @@ -24,8 +24,8 @@ static StringRef sandboxProfile(llvm::BumpPtrAllocator &Alloc) { // Allow reading file metadata of any files. contents += "(allow file-read-metadata)\n"; - // Allow reading dylibs. - contents += "(allow file-read* (regex #\"\\.dylib$\"))\n"; + // Allow reading dylibs and WebAssembly macros. + contents += "(allow file-read* (regex #\"\\.(dylib|wasm)$\"))\n"; // This is required to launch any processes (execve(2)). contents += "(allow process-exec*)\n"; diff --git a/test/Macros/Inputs/wasi_shim.h b/test/Macros/Inputs/wasi_shim.h new file mode 100644 index 0000000000000..cc7ea544bf643 --- /dev/null +++ b/test/Macros/Inputs/wasi_shim.h @@ -0,0 +1,55 @@ +// definitions from +// https://github.com/WebAssembly/wasi-libc/blob/320bbbcced68ce8e564b0dc4c8f80a5a5ad21a9c/libc-bottom-half/headers/public/wasi/api.h + +typedef unsigned char uint8_t; +typedef unsigned short uint16_t; +typedef unsigned int uint32_t; +typedef unsigned long long uint64_t; + +typedef uint8_t wasi_signal_t; +typedef uint16_t wasi_errno_t; +typedef uint32_t wasi_exitcode_t; +typedef int wasi_fd_t; +typedef __SIZE_TYPE__ wasi_size_t; + +typedef struct wasi_ciovec_t { + const uint8_t *buf; + wasi_size_t buf_len; +} wasi_ciovec_t; + +wasi_errno_t wasi_fd_write( + wasi_fd_t fd, + const wasi_ciovec_t *iovs, + wasi_size_t iovs_len, + wasi_size_t *nwritten +) __attribute__(( + __import_module__("wasi_snapshot_preview1"), + __import_name__("fd_write"), + __warn_unused_result__ +)); + +_Noreturn void wasi_proc_exit( + wasi_exitcode_t code +) __attribute__(( + __import_module__("wasi_snapshot_preview1"), + __import_name__("proc_exit") +)); + +// libc shims + +static inline wasi_size_t swift_strlen(const char *buf) { + wasi_size_t len = 0; + while (buf[len]) len++; + return len; +} + +static inline wasi_errno_t swift_write(int fd, const void *buf, wasi_size_t len) { + struct wasi_ciovec_t vec = { .buf = (const uint8_t *)buf, .buf_len = len }; + wasi_size_t nwritten = 0; + return wasi_fd_write(fd, &vec, 1, &nwritten); +} + +_Noreturn static inline void swift_abort(const char *message) { + swift_write(2, message, swift_strlen(message)); + wasi_proc_exit(1); +} diff --git a/test/Macros/lit.local.cfg b/test/Macros/lit.local.cfg index fe1101a502530..53d28816c7ded 100644 --- a/test/Macros/lit.local.cfg +++ b/test/Macros/lit.local.cfg @@ -31,3 +31,11 @@ else: ) config.substitutions.append(('%c-flags', config.c_flags)) config.substitutions.append(('%exe-linker-flags', config.exe_linker_flags)) + +config.substitutions.insert( + 0, + ( + '%swift-build-wasm-c-plugin', + f'{config.clang} -target wasm32 -nostdlib -iquote %S' + ) +) diff --git a/test/Macros/macro_plugin_wasm.swift b/test/Macros/macro_plugin_wasm.swift new file mode 100644 index 0000000000000..871a17414ea10 --- /dev/null +++ b/test/Macros/macro_plugin_wasm.swift @@ -0,0 +1,60 @@ +// REQUIRES: swift_swift_parser +// REQUIRES: CODEGENERATOR=WebAssembly + +// RUN: %empty-directory(%t) + +// RUN: split-file %s %t + +//#-- Prepare the Wasm macro plugin. +// RUN: %swift-build-wasm-c-plugin %t/MacroDefinition.c -o %t/Plugin.wasm + +// RUN: env SWIFT_DUMP_PLUGIN_MESSAGING=1 %target-swift-frontend \ +// RUN: -typecheck \ +// RUN: -swift-version 5 \ +// RUN: -load-resolved-plugin %t/Plugin.wasm#%swift-plugin-server#MacroDefinition \ +// RUN: -Rmacro-loading \ +// RUN: -module-name MyApp \ +// RUN: %t/test.swift \ +// RUN: > %t/macro-loading.txt 2>&1 + +// RUN: %FileCheck %s < %t/macro-loading.txt + +// CHECK: ->(plugin:[[#PID:]]) {"getCapability":{"capability":{"protocolVersion":[[#PROTOCOL_VERSION:]]}}} +// CHECK: <-(plugin:[[#PID]]) {"getCapabilityResult":{"capability":{"features":["load-plugin-library"],"protocolVersion":7}}} +// CHECK: ->(plugin:[[#PID]]) {"loadPluginLibrary":{"libraryPath":"{{.*[\\/]}}Plugin.wasm","moduleName":"MacroDefinition"}} +// CHECK: <-(plugin:[[#PID]]) {"loadPluginLibraryResult":{"diagnostics":[],"loaded":true}} +// CHECK: ->(plugin:[[#PID]]) {"expandFreestandingMacro":{"discriminator":"$s{{.+}}","lexicalContext":[{{.*}}],"macro":{"moduleName":"MacroDefinition","name":"constInt","typeName":"ConstMacro"},"macroRole":"expression","syntax":{"kind":"expression","location":{"column":16,"fileID":"MyApp/test.swift","fileName":"{{.+}}test.swift","line":4,"offset":143},"source":"#constInt"}}} +// CHECK: <-(plugin:[[#PID]]) {"expandMacroResult":{"diagnostics":[],"expandedSource":"1"}} + +//--- test.swift +@freestanding(expression) macro constInt() -> Int = #externalMacro(module: "MacroDefinition", type: "ConstMacro") + +func foo() { + let _: Int = #constInt +} + +//--- MacroDefinition.c +#include "Inputs/wasi_shim.h" + +static void write_json(const char *json) { + wasi_size_t len = swift_strlen(json); + uint64_t len64 = (uint64_t)len; + swift_write(1, &len64, sizeof(len64)); + swift_write(1, json, len); +} + +int was_start_called = 0; +int pump_calls = 0; + +__attribute__((export_name("_start"))) +void _start(void) { + if (was_start_called) swift_abort("_start called twice!"); + was_start_called = 1; +} + +__attribute__((export_name("swift_wasm_macro_v1_pump"))) +void pump(void) { + if (!was_start_called) swift_abort("_start not called!"); + if (pump_calls++ != 0) swift_abort("expected pump to be called once"); + write_json("{\"expandMacroResult\": {\"expandedSource\": \"1\", \"diagnostics\": []}}"); +} diff --git a/test/Macros/macro_plugin_wasm_badversion.swift b/test/Macros/macro_plugin_wasm_badversion.swift new file mode 100644 index 0000000000000..7d63493732c30 --- /dev/null +++ b/test/Macros/macro_plugin_wasm_badversion.swift @@ -0,0 +1,33 @@ +// REQUIRES: swift_swift_parser +// REQUIRES: CODEGENERATOR=WebAssembly + +// RUN: %empty-directory(%t) + +// RUN: split-file %s %t + +//#-- Prepare the Wasm macro plugin. +// RUN: %swift-build-wasm-c-plugin %t/MacroDefinition.c -o %t/Plugin.wasm + +// RUN: %target-swift-frontend \ +// RUN: -typecheck -verify \ +// RUN: -swift-version 5 \ +// RUN: -load-resolved-plugin %t/Plugin.wasm#%swift-plugin-server#MacroDefinition \ +// RUN: -module-name MyApp \ +// RUN: %t/test.swift + +//--- test.swift +// expected-warning @+2 {{Wasm plugin has an unknown ABI (could not find 'swift_wasm_macro_v1_pump')}} +// expected-note @+1 {{declared here}} +@freestanding(expression) macro constInt() -> Int = #externalMacro(module: "MacroDefinition", type: "ConstMacro") + +func foo() { + // expected-error @+1 {{Wasm plugin has an unknown ABI (could not find 'swift_wasm_macro_v1_pump')}} + let _: Int = #constInt +} + +//--- MacroDefinition.c +__attribute__((export_name("_start"))) +void _start(void) {} + +__attribute__((export_name("swift_wasm_macro_v100_pump"))) +void pump(void) {} diff --git a/test/Macros/macro_plugin_wasm_guest_error.swift b/test/Macros/macro_plugin_wasm_guest_error.swift new file mode 100644 index 0000000000000..e0ff5a9a4b112 --- /dev/null +++ b/test/Macros/macro_plugin_wasm_guest_error.swift @@ -0,0 +1,40 @@ +// REQUIRES: swift_swift_parser +// REQUIRES: CODEGENERATOR=WebAssembly + +// RUN: %empty-directory(%t) + +// RUN: split-file %s %t + +//#-- Prepare the Wasm macro plugin. +// RUN: %swift-build-wasm-c-plugin %t/MacroDefinition.c -o %t/Plugin.wasm + +// RUN: %target-swift-frontend \ +// RUN: -typecheck -verify \ +// RUN: -swift-version 5 \ +// RUN: -load-resolved-plugin %t/Plugin.wasm#%swift-plugin-server#MacroDefinition \ +// RUN: -module-name MyApp \ +// RUN: %t/test.swift \ +// RUN: 2>%t/macro-loading.txt + +// RUN: %FileCheck %s < %t/macro-loading.txt + +// CHECK: guest error! + +//--- test.swift +@freestanding(expression) macro constInt() -> Int = #externalMacro(module: "MacroDefinition", type: "ConstMacro") + +func foo() { + // expected-error @+1 {{failed to communicate with external macro}} + let _: Int = #constInt +} + +//--- MacroDefinition.c +#include "Inputs/wasi_shim.h" + +__attribute__((export_name("_start"))) +void _start(void) {} + +__attribute__((export_name("swift_wasm_macro_v1_pump"))) +void pump(void) { + swift_abort("guest error!\n"); +} diff --git a/test/Macros/macro_plugin_wasm_nostart.swift b/test/Macros/macro_plugin_wasm_nostart.swift new file mode 100644 index 0000000000000..18929f911e7b6 --- /dev/null +++ b/test/Macros/macro_plugin_wasm_nostart.swift @@ -0,0 +1,33 @@ +// REQUIRES: swift_swift_parser +// REQUIRES: CODEGENERATOR=WebAssembly + +// RUN: %empty-directory(%t) + +// RUN: split-file %s %t + +//#-- Prepare the Wasm macro plugin. +// RUN: %swift-build-wasm-c-plugin %t/MacroDefinition.c -o %t/Plugin.wasm + +// RUN: %target-swift-frontend \ +// RUN: -typecheck -verify \ +// RUN: -swift-version 5 \ +// RUN: -load-resolved-plugin %t/Plugin.wasm#%swift-plugin-server#MacroDefinition \ +// RUN: -module-name MyApp \ +// RUN: %t/test.swift + +//--- test.swift +// expected-warning @+2 {{Wasm plugin does not have a '_start' entrypoint}} +// expected-note @+1 {{declared here}} +@freestanding(expression) macro constInt() -> Int = #externalMacro(module: "MacroDefinition", type: "ConstMacro") + +func foo() { + // expected-error @+1 {{Wasm plugin does not have a '_start' entrypoint}} + let _: Int = #constInt +} + +//--- MacroDefinition.c +__attribute__((export_name("_tart"))) +void _start(void) {} + +__attribute__((export_name("swift_wasm_macro_v1_pump"))) +void pump(void) {} diff --git a/tools/swift-ide-test/swift-ide-test.cpp b/tools/swift-ide-test/swift-ide-test.cpp index 37df87bbdab03..d3106934cc7e5 100644 --- a/tools/swift-ide-test/swift-ide-test.cpp +++ b/tools/swift-ide-test/swift-ide-test.cpp @@ -356,7 +356,6 @@ LoadPluginExecutable("load-plugin-executable", llvm::cl::desc("load plugin executable"), llvm::cl::cat(Category)); - static llvm::cl::opt EnableSourceImport("enable-source-import", llvm::cl::Hidden, llvm::cl::cat(Category), llvm::cl::init(false)); diff --git a/tools/swift-plugin-server/.editorconfig b/tools/swift-plugin-server/.editorconfig new file mode 100644 index 0000000000000..410ff6c9ed018 --- /dev/null +++ b/tools/swift-plugin-server/.editorconfig @@ -0,0 +1,9 @@ +# editorconfig.org + +root = true + +[*] +indent_style = space +indent_size = 2 +trim_trailing_whitespace = true +insert_final_newline = true diff --git a/tools/swift-plugin-server/.gitignore b/tools/swift-plugin-server/.gitignore new file mode 100644 index 0000000000000..f4096fee41368 --- /dev/null +++ b/tools/swift-plugin-server/.gitignore @@ -0,0 +1 @@ +Package.resolved diff --git a/tools/swift-plugin-server/.swiftformat b/tools/swift-plugin-server/.swiftformat new file mode 100644 index 0000000000000..57b8fe01a6a08 --- /dev/null +++ b/tools/swift-plugin-server/.swiftformat @@ -0,0 +1,3 @@ +--indent 2 +--self insert + diff --git a/tools/swift-plugin-server/CMakeLists.txt b/tools/swift-plugin-server/CMakeLists.txt index 6db1f512fa63a..26430ea6adb69 100644 --- a/tools/swift-plugin-server/CMakeLists.txt +++ b/tools/swift-plugin-server/CMakeLists.txt @@ -1,11 +1,24 @@ if (SWIFT_BUILD_SWIFT_SYNTAX) + # override the remote SwiftSystem dependency in WasmKit + # TODO: create new variables instead of piggybacking off of SWIFT_PATH_TO_SWIFT_SYNTAX_SOURCE + file(TO_CMAKE_PATH "${SWIFT_PATH_TO_SWIFT_SYNTAX_SOURCE}" swift_syntax_path) + FetchContent_Declare(SwiftSystem SOURCE_DIR "${swift_syntax_path}/../swift-system") + + set(WASMKIT_BUILD_CLI OFF) + FetchContent_Declare(WasmKit SOURCE_DIR "${swift_syntax_path}/../wasmkit") + FetchContent_MakeAvailable(WasmKit) + add_pure_swift_host_tool(swift-plugin-server - Sources/swift-plugin-server/swift-plugin-server.swift + Sources/swift-plugin-server/SwiftPluginServer.swift + Sources/swift-plugin-server/WasmEngine.swift + Sources/swift-plugin-server/WasmKitEngine.swift + Sources/swift-plugin-server/WasmMessageHandler.swift SWIFT_COMPONENT compiler SWIFT_DEPENDENCIES SwiftCompilerPluginMessageHandling SwiftLibraryPluginProvider + WasmKitWASI PACKAGE_NAME Toolchain ) diff --git a/tools/swift-plugin-server/Package.swift b/tools/swift-plugin-server/Package.swift index 90ae3bfd2aed3..bbea463dcd651 100644 --- a/tools/swift-plugin-server/Package.swift +++ b/tools/swift-plugin-server/Package.swift @@ -13,6 +13,7 @@ let package = Package( ], dependencies: [ .package(path: "../../../swift-syntax"), + .package(path: "../../../wasmkit"), ], targets: [ .executableTarget( @@ -20,6 +21,8 @@ let package = Package( dependencies: [ .product(name: "_SwiftCompilerPluginMessageHandling", package: "swift-syntax"), .product(name: "_SwiftLibraryPluginProvider", package: "swift-syntax"), + .product(name: "WASI", package: "WasmKit"), + .product(name: "WasmKitWASI", package: "WasmKit"), ] ), .target( @@ -29,6 +32,14 @@ let package = Package( .product(name: "_SwiftLibraryPluginProvider", package: "swift-syntax"), ] ), + .testTarget( + name: "PluginServerTests", + dependencies: [ + .product(name: "WAT", package: "WasmKit"), + .product(name: "WasmKit", package: "WasmKit"), + "swift-plugin-server", + ] + ), ], cxxLanguageStandard: .cxx17 ) diff --git a/tools/swift-plugin-server/Sources/SwiftInProcPluginServer/InProcPluginServer.swift b/tools/swift-plugin-server/Sources/SwiftInProcPluginServer/InProcPluginServer.swift index 2902f8c200eba..fa61d6acab2fb 100644 --- a/tools/swift-plugin-server/Sources/SwiftInProcPluginServer/InProcPluginServer.swift +++ b/tools/swift-plugin-server/Sources/SwiftInProcPluginServer/InProcPluginServer.swift @@ -14,17 +14,17 @@ @_spi(PluginMessage) import SwiftLibraryPluginProvider #if canImport(Darwin) -import Darwin + import Darwin #elseif canImport(Glibc) -import Glibc + import Glibc #elseif canImport(Bionic) -import Bionic + import Bionic #elseif canImport(Musl) -import Musl + import Musl #elseif canImport(ucrt) -import ucrt + import ucrt #else -#error("'malloc' not found") + #error("'malloc' not found") #endif /// Entry point. @@ -80,11 +80,10 @@ struct InProcPluginServer { func handleMessage(_ input: UnsafeBufferPointer) throws -> [UInt8] { let request = try JSON.decode(HostToPluginMessage.self, from: input) - let response = handler.handleMessage(request) + let response = self.handler.handleMessage(request) return try JSON.encode(response) } @MainActor static let shared = Self() } - diff --git a/tools/swift-plugin-server/Sources/swift-plugin-server/FilePathTemp.swift b/tools/swift-plugin-server/Sources/swift-plugin-server/FilePathTemp.swift new file mode 100644 index 0000000000000..324bd603020be --- /dev/null +++ b/tools/swift-plugin-server/Sources/swift-plugin-server/FilePathTemp.swift @@ -0,0 +1,111 @@ +/* + This source file is part of the Swift System open source project + + Copyright (c) 2024 Apple Inc. and the Swift System project authors + Licensed under Apache License v2.0 with Runtime Library Exception + + See https://swift.org/LICENSE.txt for license information +*/ + +#if canImport(Darwin) +import Darwin +#elseif canImport(Glibc) +import Glibc +#elseif canImport(Musl) +import Musl +#elseif canImport(WASILibc) +import WASILibc +#else +#error("Unsupported platform") +#endif + +import SystemPackage + +// MARK: - API + +/// Create a temporary path for the duration of the closure. +/// +/// - Parameters: +/// - basename: The base name for the temporary path. +/// - body: The closure to execute. +/// +/// Creates a temporary directory with a name based on the given `basename`, +/// executes `body`, passing in the path of the created directory, then +/// deletes the directory and all of its contents before returning. +func withTemporaryFilePath( + basename: FilePath.Component, + _ body: (FilePath) throws -> R +) throws -> R { + let temporaryDir = try createUniqueTemporaryDirectory(basename: basename) + defer { + try? _recursiveRemove(at: temporaryDir) + } + + return try body(temporaryDir) +} + +// MARK: - Internals + +fileprivate let base64 = Array( + "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_".utf8 +) + +/// Create a directory that is only accessible to the current user. +/// +/// - Parameters: +/// - path: The path of the directory to create. +/// - Returns: `true` if a new directory was created. +/// +/// This function will throw if there is an error, except if the error +/// is that the directory exists, in which case it returns `false`. +fileprivate func makeLockedDownDirectory(at path: FilePath) throws -> Bool { + return try path.withPlatformString { + if mkdir($0, 0o700) == 0 { + return true + } + let err = errno + if err == Errno.fileExists.rawValue { + return false + } else { + throw Errno(rawValue: err) + } + } +} + +/// Generate a random string of base64 filename safe characters. +/// +/// - Parameters: +/// - length: The number of characters in the returned string. +/// - Returns: A random string of length `length`. +fileprivate func createRandomString(length: Int) -> String { + return String( + decoding: (0.. FilePath { + var tempDir = try _getTemporaryDirectory() + tempDir.append(basename) + + while true { + tempDir.extension = createRandomString(length: 16) + + if try makeLockedDownDirectory(at: tempDir) { + return tempDir + } + } +} diff --git a/tools/swift-plugin-server/Sources/swift-plugin-server/FilePathTempPOSIX.swift b/tools/swift-plugin-server/Sources/swift-plugin-server/FilePathTempPOSIX.swift new file mode 100644 index 0000000000000..be1a7f797aaef --- /dev/null +++ b/tools/swift-plugin-server/Sources/swift-plugin-server/FilePathTempPOSIX.swift @@ -0,0 +1,176 @@ +/* + This source file is part of the Swift System open source project + + Copyright (c) 2024 Apple Inc. and the Swift System project authors + Licensed under Apache License v2.0 with Runtime Library Exception + + See https://swift.org/LICENSE.txt for license information + */ + +#if !os(Windows) + +#if canImport(Darwin) +import Darwin +#elseif canImport(Glibc) +import Glibc +#elseif canImport(Musl) +import Musl +#elseif canImport(WASILibc) +import WASILibc +#else +#error("Unsupported platform") +#endif + +import SystemPackage + +/// Get the path to the system temporary directory. +internal func _getTemporaryDirectory() throws -> FilePath { + guard let tmp = getenv("TMPDIR") else { + return "/tmp" + } + + return FilePath(stringLiteral: .init(cString: tmp)) +} + +/// Delete the entire contents of a directory, including its subdirectories. +/// +/// - Parameters: +/// - path: The directory to be deleted. +/// +/// Removes a directory completely, including all of its contents. +internal func _recursiveRemove( + at path: FilePath +) throws { + let dirfd = try FileDescriptor.open(path, .readOnly, options: .directory) + defer { + try? dirfd.close() + } + + let dot: (CInterop.PlatformChar, CInterop.PlatformChar) = (46, 0) + try withUnsafeBytes(of: dot) { + try recursiveRemove( + in: dirfd.rawValue, + name: $0.assumingMemoryBound(to: CInterop.PlatformChar.self).baseAddress! + ) + } + + try path.withPlatformString { + let error = rmdir($0) + if error != 0 { + throw Errno(rawValue: error) + } + } +} + +internal let SYSTEM_AT_REMOVE_DIR = AT_REMOVEDIR +internal let SYSTEM_DT_DIR = DT_DIR +internal typealias system_dirent = dirent +#if os(Linux) || os(Android) +internal typealias system_DIRPtr = OpaquePointer +#else +internal typealias system_DIRPtr = UnsafeMutablePointer +#endif + + +/// Open a directory by reference to its parent and name. +/// +/// - Parameters: +/// - dirfd: An open file descriptor for the parent directory. +/// - name: The name of the directory to open. +/// - Returns: A pointer to a `DIR` structure. +/// +/// This is like `opendir()`, but instead of taking a path, it uses a +/// file descriptor pointing at the parent, thus avoiding path length +/// limits. +fileprivate func impl_opendirat( + _ dirfd: CInt, + _ name: UnsafePointer +) -> system_DIRPtr? { + let fd = openat(dirfd, name, + FileDescriptor.AccessMode.readOnly.rawValue + | FileDescriptor.OpenOptions.directory.rawValue) + if fd < 0 { + return nil + } + return fdopendir(fd) +} + +/// Invoke a closure for each file within a particular directory. +/// +/// - Parameters: +/// - dirfd: The parent of the directory to be enumerated. +/// - subdir: The subdirectory to be enumerated. +/// - body: The closure that will be invoked. +/// +/// We skip the `.` and `..` pseudo-entries. +fileprivate func forEachFile( + in dirfd: CInt, + subdir: UnsafePointer, + _ body: (system_dirent) throws -> () +) throws { + guard let dir = impl_opendirat(dirfd, subdir) else { + throw Errno(rawValue: errno) + } + defer { + _ = closedir(dir) + } + + while let dirent = readdir(dir) { + // Skip . and .. + if dirent.pointee.d_name.0 == 46 + && (dirent.pointee.d_name.1 == 0 + || (dirent.pointee.d_name.1 == 46 + && dirent.pointee.d_name.2 == 0)) { + continue + } + + try body(dirent.pointee) + } +} + +/// Delete the entire contents of a directory, including its subdirectories. +/// +/// - Parameters: +/// - dirfd: The parent of the directory to be removed. +/// - name: The name of the directory to be removed. +/// +/// Removes a directory completely, including all of its contents. +fileprivate func recursiveRemove( + in dirfd: CInt, + name: UnsafePointer +) throws { + // First, deal with subdirectories + try forEachFile(in: dirfd, subdir: name) { dirent in + if dirent.d_type == SYSTEM_DT_DIR { + try withUnsafeBytes(of: dirent.d_name) { + try recursiveRemove( + in: dirfd, + name: $0.assumingMemoryBound(to: CInterop.PlatformChar.self) + .baseAddress! + ) + } + } + } + + // Now delete the contents of this directory + try forEachFile(in: dirfd, subdir: name) { dirent in + let flag: CInt + + if dirent.d_type == SYSTEM_DT_DIR { + flag = SYSTEM_AT_REMOVE_DIR + } else { + flag = 0 + } + + let result = withUnsafeBytes(of: dirent.d_name) { + unlinkat(dirfd, $0.assumingMemoryBound(to: CInterop.PlatformChar.self).baseAddress!, flag) + } + + if result != 0 { + throw Errno(rawValue: errno) + } + } +} + +#endif // !os(Windows) + diff --git a/tools/swift-plugin-server/Sources/swift-plugin-server/FilePathWindows.swift b/tools/swift-plugin-server/Sources/swift-plugin-server/FilePathWindows.swift new file mode 100644 index 0000000000000..b19f0c7603c34 --- /dev/null +++ b/tools/swift-plugin-server/Sources/swift-plugin-server/FilePathWindows.swift @@ -0,0 +1,116 @@ +/* + This source file is part of the Swift System open source project + + Copyright (c) 2024 Apple Inc. and the Swift System project authors + Licensed under Apache License v2.0 with Runtime Library Exception + + See https://swift.org/LICENSE.txt for license information +*/ + +#if os(Windows) + +import SystemPackage + +import WinSDK + +/// Get the path to the system temporary directory. +internal func _getTemporaryDirectory() throws -> FilePath { + return try withUnsafeTemporaryAllocation(of: CInterop.PlatformChar.self, + capacity: Int(MAX_PATH) + 1) { + buffer in + + guard GetTempPath2W(DWORD(buffer.count), buffer.baseAddress) != 0 else { + throw Errno(windowsError: GetLastError()) + } + + return FilePath(SystemString(platformString: buffer.baseAddress!)) + } +} + +/// Invoke a closure for each file within a particular directory. +/// +/// - Parameters: +/// - path: The path at which we should enumerate items. +/// - body: The closure that will be invoked. +/// +/// We skip the `.` and `..` pseudo-entries. +fileprivate func forEachFile( + at path: FilePath, + _ body: (WIN32_FIND_DATAW) throws -> () +) rethrows { + let searchPath = path.appending("\\*") + + try searchPath.withPlatformString { szPath in + var findData = WIN32_FIND_DATAW() + let hFind = FindFirstFileW(szPath, &findData) + if hFind == INVALID_HANDLE_VALUE { + throw Errno(windowsError: GetLastError()) + } + defer { + FindClose(hFind) + } + + repeat { + // Skip . and .. + if findData.cFileName.0 == 46 + && (findData.cFileName.1 == 0 + || (findData.cFileName.1 == 46 + && findData.cFileName.2 == 0)) { + continue + } + + try body(findData) + } while FindNextFileW(hFind, &findData) + } +} + +/// Delete the entire contents of a directory, including its subdirectories. +/// +/// - Parameters: +/// - path: The directory to be deleted. +/// +/// Removes a directory completely, including all of its contents. +internal func _recursiveRemove( + at path: FilePath +) throws { + // First, deal with subdirectories + try forEachFile(at: path) { findData in + if (findData.dwFileAttributes & DWORD(FILE_ATTRIBUTE_DIRECTORY)) != 0 { + let name = withUnsafeBytes(of: findData.cFileName) { + return SystemString(platformString: $0.assumingMemoryBound( + to: CInterop.PlatformChar.self).baseAddress!) + } + let component = FilePath.Component(name)! + let subpath = path.appending(component) + + try _recursiveRemove(at: subpath) + } + } + + // Now delete everything else + try forEachFile(at: path) { findData in + let name = withUnsafeBytes(of: findData.cFileName) { + return SystemString(platformString: $0.assumingMemoryBound( + to: CInterop.PlatformChar.self).baseAddress!) + } + let component = FilePath.Component(name)! + let subpath = path.appending(component) + + if (findData.dwFileAttributes & DWORD(FILE_ATTRIBUTE_DIRECTORY)) == 0 { + try subpath.withPlatformString { + if !DeleteFileW($0) { + throw Errno(windowsError: GetLastError()) + } + } + } + } + + // Finally, delete the parent + try path.withPlatformString { + if !RemoveDirectoryW($0) { + throw Errno(windowsError: GetLastError()) + } + } +} + +#endif // os(Windows) diff --git a/tools/swift-plugin-server/Sources/swift-plugin-server/swift-plugin-server.swift b/tools/swift-plugin-server/Sources/swift-plugin-server/SwiftPluginServer.swift similarity index 81% rename from tools/swift-plugin-server/Sources/swift-plugin-server/swift-plugin-server.swift rename to tools/swift-plugin-server/Sources/swift-plugin-server/SwiftPluginServer.swift index 3fb2d9fe3d6a7..31ebc78353eb3 100644 --- a/tools/swift-plugin-server/Sources/swift-plugin-server/swift-plugin-server.swift +++ b/tools/swift-plugin-server/Sources/swift-plugin-server/SwiftPluginServer.swift @@ -14,12 +14,16 @@ @_spi(PluginMessage) import SwiftLibraryPluginProvider @main -final class SwiftPluginServer { +enum SwiftPluginServer { static func main() throws { let connection = try StandardIOMessageConnection() let listener = CompilerPluginMessageListener( connection: connection, - provider: LibraryPluginProvider.shared + messageHandler: WasmInterceptingMessageHandler( + base: PluginProviderMessageHandler( + provider: LibraryPluginProvider.shared + ) + ) ) try listener.main() } diff --git a/tools/swift-plugin-server/Sources/swift-plugin-server/WasmEngine.swift b/tools/swift-plugin-server/Sources/swift-plugin-server/WasmEngine.swift new file mode 100644 index 0000000000000..d5f04c1bc350d --- /dev/null +++ b/tools/swift-plugin-server/Sources/swift-plugin-server/WasmEngine.swift @@ -0,0 +1,115 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift open source project +// +// Copyright (c) 2024 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See http://swift.org/LICENSE.txt for license information +// See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +import SystemPackage +import WASI +import WasmTypes + +typealias WasmFunction = () throws -> Void + +protocol WasmEngine { + init(pluginPath: FilePath) throws + + func function(named name: String) throws -> WasmFunction? + + func writeToPlugin(_ storage: some Sequence) throws + func readFromPlugin(into storage: UnsafeMutableRawBufferPointer) throws -> Int + + func shutDown() throws +} + +typealias DefaultWasmPlugin = WasmEnginePlugin + +// a WasmPlugin implementation that delegates to a WasmEngine +struct WasmEnginePlugin: WasmPlugin { + private let pumpFunction: WasmFunction + let engine: Engine + + init(path: FilePath) throws { + self.engine = try Engine(pluginPath: path) + + let exportName = "swift_wasm_macro_v1_pump" + guard let pump = try engine.function(named: exportName) else { + throw WasmEngineError(message: "Wasm plugin has an unknown ABI (could not find '\(exportName)')") + } + self.pumpFunction = pump + + guard let start = try engine.function(named: "_start") else { + throw WasmEngineError(message: "Wasm plugin does not have a '_start' entrypoint") + } + try start() + } + + func handleMessage(_ json: [UInt8]) throws -> [UInt8] { + try withUnsafeBytes(of: UInt64(json.count).littleEndian) { + _ = try engine.writeToPlugin($0) + } + try engine.writeToPlugin(json) + + try self.pumpFunction() + + let lengthRaw = try withUnsafeTemporaryAllocation(of: UInt8.self, capacity: 8) { buffer in + let lengthCount = try engine.readFromPlugin(into: UnsafeMutableRawBufferPointer(buffer)) + guard lengthCount == 8 else { + throw WasmEngineError(message: "Wasm plugin sent invalid response") + } + return buffer.withMemoryRebound(to: UInt64.self, \.baseAddress!.pointee) + } + let length = Int(UInt64(littleEndian: lengthRaw)) + return try [UInt8](unsafeUninitializedCapacity: length) { buffer, size in + let received = try engine.readFromPlugin(into: UnsafeMutableRawBufferPointer(buffer)) + guard received == length else { + throw WasmEngineError(message: "Wasm plugin sent truncated response") + } + size = received + } + } + + func shutDown() throws { + try self.engine.shutDown() + } +} + +struct WasmEngineError: Error, CustomStringConvertible { + let description: String + + init(message: String) { + self.description = message + } +} + +// `pipe` support on Windows is not included in the latest swift-system release 1.3.1 +// but it is available in the latest main branch. Remove the following code when we +// update to the next release of swift-system. +#if os(Windows) + import ucrt + + var system_errno: CInt { + var value: CInt = 0 + _ = ucrt._get_errno(&value) + return value + } + + extension FileDescriptor { + static func pipe() throws -> (readEnd: FileDescriptor, writeEnd: FileDescriptor) { + var fds: (Int32, Int32) = (-1, -1) + return try withUnsafeMutablePointer(to: &fds) { pointer in + try pointer.withMemoryRebound(to: Int32.self, capacity: 2) { fds in + guard _pipe(fds, 4096, _O_BINARY | _O_NOINHERIT) == 0 else { + throw Errno(rawValue: system_errno) + } + return (FileDescriptor(rawValue: fds[0]), FileDescriptor(rawValue: fds[1])) + } + } + } + } +#endif diff --git a/tools/swift-plugin-server/Sources/swift-plugin-server/WasmKitEngine.swift b/tools/swift-plugin-server/Sources/swift-plugin-server/WasmKitEngine.swift new file mode 100644 index 0000000000000..5e6139d33cdbd --- /dev/null +++ b/tools/swift-plugin-server/Sources/swift-plugin-server/WasmKitEngine.swift @@ -0,0 +1,73 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift open source project +// +// Copyright (c) 2024 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See http://swift.org/LICENSE.txt for license information +// See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +import SystemPackage +import WASI +import WasmKit +import WasmKitWASI + +typealias DefaultWasmEngine = WasmKitEngine + +struct WasmKitEngine: WasmEngine { + private let engine: Engine + private let functions: [String: Function] + private let hostToPlugin: FileDescriptor + private let pluginToHost: FileDescriptor + + init(pluginPath: FilePath) throws { + let hostToPluginPipe = try FileDescriptor.pipe() + let pluginToHostPipe = try FileDescriptor.pipe() + self.hostToPlugin = hostToPluginPipe.writeEnd + self.pluginToHost = pluginToHostPipe.readEnd + + var configuration = EngineConfiguration() + configuration.stackSize = 1 << 20 + self.engine = Engine(configuration: configuration) + let store = Store(engine: engine) + + let module = try parseWasm(filePath: pluginPath) + var moduleImports = Imports() + + let imports = try WASIBridgeToHost( + stdin: hostToPluginPipe.readEnd, + stdout: pluginToHostPipe.writeEnd, + stderr: .standardError + ) + imports.link(to: &moduleImports, store: store) + let instance = try module.instantiate(store: store, imports: moduleImports) + var functions = [String: Function]() + for (name, export) in instance.exports { + guard case let .function(function) = export else { continue } + functions[name] = function + } + + self.functions = functions + } + + func function(named name: String) throws -> WasmFunction? { + guard let function = functions[name] else { return nil } + return { _ = try function.invoke() } + } + + func writeToPlugin(_ storage: some Sequence) throws { + try self.hostToPlugin.writeAll(storage) + } + + func readFromPlugin(into storage: UnsafeMutableRawBufferPointer) throws -> Int { + try self.pluginToHost.read(into: storage) + } + + func shutDown() throws { + try self.hostToPlugin.close() + try self.pluginToHost.close() + } +} diff --git a/tools/swift-plugin-server/Sources/swift-plugin-server/WasmMessageHandler.swift b/tools/swift-plugin-server/Sources/swift-plugin-server/WasmMessageHandler.swift new file mode 100644 index 0000000000000..e397c06c1335d --- /dev/null +++ b/tools/swift-plugin-server/Sources/swift-plugin-server/WasmMessageHandler.swift @@ -0,0 +1,122 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift open source project +// +// Copyright (c) 2024 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See http://swift.org/LICENSE.txt for license information +// See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +import SystemPackage +@_spi(PluginMessage) import SwiftCompilerPluginMessageHandling + +/// A `PluginMessageHandler` that intercepts messages intended for Wasm plugins. +final class WasmInterceptingMessageHandler: PluginMessageHandler { + private var loadedWasmPlugins: [String: any WasmPlugin] = [:] + + let base: Base + init(base: Base) { + self.base = base + } + + /// Handle the message ourselves if it references a Wasm plugin. + /// Otherwise, forward it to `base`. + func handleMessage(_ message: HostToPluginMessage) -> PluginToHostMessage { + switch message { + case let .loadPluginLibrary(libraryPath, moduleName): + guard libraryPath.hasSuffix(".wasm") else { break } + let libraryFilePath = FilePath(libraryPath) + do { + self.loadedWasmPlugins[moduleName] = try DefaultWasmPlugin(path: libraryFilePath) + } catch { + return .loadPluginLibraryResult( + loaded: false, + diagnostics: [PluginMessage.Diagnostic(errorMessage: "\(error)")] + ) + } + return .loadPluginLibraryResult(loaded: true, diagnostics: []) + case let .expandAttachedMacro(macro, _, _, syntax, _, _, _, _, _), + let .expandFreestandingMacro(macro, _, _, syntax, _): + if let response = self.expandMacro(macro, message: message, location: syntax.location) { + return response + } // else break + case .getCapability: + break + #if !SWIFT_PACKAGE + @unknown default: + break + #endif + } + return self.base.handleMessage(message) + } + + func shutDown() throws { + for plugin in self.loadedWasmPlugins.values { + try plugin.shutDown() + } + + self.loadedWasmPlugins = [:] + } + + private func expandMacro( + _ macro: PluginMessage.MacroReference, + message: HostToPluginMessage, + location: PluginMessage.SourceLocation? + ) -> PluginToHostMessage? { + guard let plugin = self.loadedWasmPlugins[macro.moduleName] else { return nil } + do { + let request = try JSON.encode(message) + let responseRaw = try plugin.handleMessage(request) + return try responseRaw.withUnsafeBytes { + try $0.withMemoryRebound(to: UInt8.self) { + try JSON.decode(PluginToHostMessage.self, from: $0) + } + } + } catch { + return .expandMacroResult( + expandedSource: nil, + diagnostics: [PluginMessage.Diagnostic( + errorMessage: """ + failed to communicate with external macro implementation type \ + '\(macro.moduleName).\(macro.typeName)' to expand macro '\(macro.name)'; \ + \(error) + """, + position: location?.position ?? .invalid + )] + ) + } + } +} + +fileprivate extension PluginMessage.Diagnostic { + init( + errorMessage: String, + position: PluginMessage.Diagnostic.Position = .invalid + ) { + self.init( + message: errorMessage, + severity: .error, + position: position, + highlights: [], + notes: [], + fixIts: [] + ) + } +} + +fileprivate extension PluginMessage.SourceLocation { + var position: PluginMessage.Diagnostic.Position { + .init(fileName: fileName, offset: offset) + } +} + +protocol WasmPlugin { + init(path: FilePath) throws + + func handleMessage(_ json: [UInt8]) throws -> [UInt8] + + func shutDown() throws +} diff --git a/tools/swift-plugin-server/Tests/PluginServerTests/WasmEngineTests.swift b/tools/swift-plugin-server/Tests/PluginServerTests/WasmEngineTests.swift new file mode 100644 index 0000000000000..73ce5451a6949 --- /dev/null +++ b/tools/swift-plugin-server/Tests/PluginServerTests/WasmEngineTests.swift @@ -0,0 +1,80 @@ +@testable import swift_plugin_server +import Foundation +import SystemPackage +import Testing +import WasmKit +import WasmKitWASI +import WAT + +private let echo = #""" +(module + (import "wasi_snapshot_preview1" "fd_read" + (func $fd_read + (param (; fd ;) i32 (; ioVecOffset ;) i32 (; ioVecCount ;) i32 (; readCountPointer ;) i32) + (result (; errno ;) i32) + ) + ) + + (import "wasi_snapshot_preview1" "fd_write" + (func $fd_write + (param (; fd ;) i32 (; ioVecOffset ;) i32 (; ioVecCount ;) i32 (; writtenCountPointer ;) i32) + (result (; errno ;) i32) + ) + ) + + (export "_start" (func $start)) + (export "swift_wasm_macro_v1_pump" (func $pump)) + (memory $memory 1) + (export "memory" (memory $memory)) + + (func $start) + + (func $pump + (memory.fill + (i32.const 0) + (i32.const 0) + (i32.const 1024) + ) + + (i32.store + (i32.const 0) (; first byte is the buffer pointer ;) + (i32.const 8) (; 4 bytes for the pointer + 4 bytes for the count ;) + ) + (i32.store + (i32.const 4 ) (; 4 bytes already used for the pointer ;) + (i32.const 1024) (; 1024 bytes should be enough for everyone (probably) ;) + ) + + (call $fd_read + (i32.const 0) (; stdin FD ;) + (i32.const 0) (; ioVecOffset ;) + (i32.const 1) (; ioVecCount ;) + (i32.const 1028) (; skip buffer and its count and store read counter after that ;) + ) + (drop) (; ignore the result ;) + + (call $fd_write + (i32.const 1) (; stdout FD ;) + (i32.const 0) (; ioVecOffset ;) + (i32.const 1) (; ioVecCount ;) + (i32.const 1028) (; skip buffer and its count and store write counter after that ;) + ) + (drop) (; ignore the result ;) + ) +) +"""# + +@Suite +struct WasmEngineTests { + @Test + func basic() throws { + let binary = try wat2wasm(echo) + let echoURL = FileManager.default.temporaryDirectory.appendingPathExtension("echo.wasm") + try Data(binary).write(to: echoURL) + defer { try! FileManager.default.removeItem(at: echoURL) } + + let wasmKit = try WasmEnginePlugin(path: FilePath(echoURL.path)) + let input: [UInt8] = [1,2,3,4,5] + #expect(try wasmKit.handleMessage(input) == input) + } +} diff --git a/utils/update_checkout/update-checkout-config.json b/utils/update_checkout/update-checkout-config.json index 1a6217c71c7bb..783ea51323c4a 100644 --- a/utils/update_checkout/update-checkout-config.json +++ b/utils/update_checkout/update-checkout-config.json @@ -13,7 +13,7 @@ "swift-async-algorithms": { "remote": { "id": "apple/swift-async-algorithms" } }, "swift-atomics": { - "remote": { "id": "apple/swift-atomics" } }, + "remote": { "id": "apple/swift-atomics" } }, "swift-collections": { "remote": { "id": "apple/swift-collections" } }, "swift-crypto": { @@ -27,7 +27,7 @@ "swift-log": { "remote": { "id": "apple/swift-log" } }, "swift-numerics": { - "remote": { "id": "apple/swift-numerics" } }, + "remote": { "id": "apple/swift-numerics" } }, "swift-toolchain-sqlite": { "remote": { "id": "swiftlang/swift-toolchain-sqlite" } }, "swift-tools-support-core": { @@ -165,7 +165,7 @@ "swift-experimental-string-processing": "swift/main", "swift-sdk-generator": "main", "wasi-libc": "wasi-sdk-24", - "wasmkit": "0.1.2", + "wasmkit": "0.1.3", "curl": "curl-8_9_1", "libxml2": "v2.11.5", "zlib": "v1.3.1" @@ -372,7 +372,7 @@ "swift-experimental-string-processing": "swift/main", "swift-sdk-generator": "main", "wasi-libc": "wasi-sdk-24", - "wasmkit": "0.1.2", + "wasmkit": "0.1.3", "curl": "curl-8_9_1", "libxml2": "v2.11.5", "zlib": "v1.3.1" @@ -420,7 +420,7 @@ "swift-nio": "2.65.0", "swift-experimental-string-processing": "swift/main", "wasi-libc": "wasi-sdk-24", - "wasmkit": "0.1.2", + "wasmkit": "0.1.3", "curl": "curl-8_9_1", "icu": "maint/maint-69", "libxml2": "v2.11.5",