Tests: Augment SwiftBuild build provider test coverage

Augment some test to run against the native and swiftbuild build
providers to gain extra confidence with the swift build integration.

Author: bkhouri

Sources/_InternalTestSupport/XCTAssertHelpers.swift

@@ -50,6 +50,10 @@ public func XCTSkipIfCI(file: StaticString = #filePath, line: UInt = #line) thro
     }
 }
 
+public func XCTSkipSwiftBuildTodo(because reason: String) throws {
+    throw XCTSkip("SWBINTTODO: \(reason)")
+}
+
 /// An `async`-friendly replacement for `XCTAssertThrowsError`.
 public func XCTAssertAsyncThrowsError<T>(
     _ expression: @autoclosure () async throws -> T,

Tests/CommandsTests/BuildCommandTests.swift

@@ -855,7 +855,7 @@ class BuildCommandSwiftBuildTests: BuildCommandTestCases {
     }
 
     override func testNonReachableProductsAndTargetsFunctional() async throws {
-        throw XCTSkip("SWBINTTODO: Test failed. This needs to be investigated")
+        try XCTSkipSwiftBuildTodo(because: "Test failed. This needs to be investigated")
     }
 
     override func testParseableInterfaces() async throws {
@@ -877,7 +877,7 @@ class BuildCommandSwiftBuildTests: BuildCommandTestCases {
     }
 
     override func testAutomaticParseableInterfacesWithLibraryEvolution() async throws {
-        throw XCTSkip("SWBINTTODO: Test failed because of missing 'A.swiftmodule/*.swiftinterface' files")
+        try XCTSkipSwiftBuildTodo(because: "Test failed because of missing 'A.swiftmodule/*.swiftinterface' files")
         // TODO: We still need to override this test just like we did for `testParseableInterfaces` above.
     }
 
@@ -896,50 +896,50 @@ class BuildCommandSwiftBuildTests: BuildCommandTestCases {
     }
 
     override func testGetTaskAllowEntitlement() async throws {
-        throw XCTSkip("SWBINTTODO: Test failed because swiftbuild doesn't output precis codesign commands. Once swift run works with swiftbuild the test can be investigated.")
+        try XCTSkipSwiftBuildTodo(because: "Test failed because swiftbuild doesn't output precis codesign commands. Once swift run works with swiftbuild the test can be investigated.")
     }
 
     override func testCodeCoverage() async throws {
-        throw XCTSkip("SWBINTTODO: Test failed because of missing plugin support in the PIF builder. This can be reinvestigated after the support is there.")
+        try XCTSkipSwiftBuildTodo(because: "Test failed because of missing plugin support in the PIF builder. This can be reinvestigated after the support is there.")
     }
 
     override func testAtMainSupport() async throws {
         #if !os(macOS)
-        throw XCTSkip("SWBINTTODO: File not found or missing libclang errors on non-macOS platforms. This needs to be investigated")
+        try XCTSkipSwiftBuildTodo(because: "File not found or missing libclang errors on non-macOS platforms. This needs to be investigated")
         #else
         try await super.testAtMainSupport()
         #endif
     }
 
     override func testImportOfMissedDepWarning() async throws {
-        throw XCTSkip("SWBINTTODO: Test fails because the warning message regarding missing imports is expected to be more verbose and actionable at the SwiftPM level with mention of the involved targets. This needs to be investigated. See case targetDiagnostic(TargetDiagnosticInfo) as a message type that may help.")
+        try XCTSkipSwiftBuildTodo(because: "Test fails because the warning message regarding missing imports is expected to be more verbose and actionable at the SwiftPM level with mention of the involved targets. This needs to be investigated. See case targetDiagnostic(TargetDiagnosticInfo) as a message type that may help.")
     }
 
     override func testProductAndTarget() async throws {
-        throw XCTSkip("SWBINTTODO: Test fails because there isn't a clear warning message about the lib1 being an automatic product and that the default product is being built instead. This needs to be investigated")
+        try XCTSkipSwiftBuildTodo(because: "Test fails because there isn't a clear warning message about the lib1 being an automatic product and that the default product is being built instead. This needs to be investigated")
     }
 
     override func testSwiftGetVersion() async throws {
-        throw XCTSkip("SWBINTTODO: Test fails because the dummy-swiftc used in the test isn't accepted by swift-build. This needs to be investigated")
+        try XCTSkipSwiftBuildTodo(because: "Test fails because the dummy-swiftc used in the test isn't accepted by swift-build. This needs to be investigated")
     }
 
     override func testSymlink() async throws {
-        throw XCTSkip("SWBINTTODO: Test fails because of a difference in the build layout. This needs to be updated to the expected path")
+        try XCTSkipSwiftBuildTodo(because: "Test fails because of a difference in the build layout. This needs to be updated to the expected path")
     }
 
 #if !canImport(Darwin)
     override func testIgnoresLinuxMain() async throws {
-        throw XCTSkip("SWBINTTODO: Swift build doesn't currently ignore Linux main when linking on Linux. This needs further investigation.")
+        try XCTSkipSwiftBuildTodo(because: "Swift build doesn't currently ignore Linux main when linking on Linux. This needs further investigation.")
     }
 #endif
 
 #if !os(macOS)
     override func testBuildStartMessage() async throws {
-        throw XCTSkip("SWBINTTODO: Swift build produces an error building the fixture for this test.")
+        try XCTSkipSwiftBuildTodo(because: "Swift build produces an error building the fixture for this test.")
     }
 
     override func testSwiftDriverRawOutputGetsNewlines() async throws {
-        throw XCTSkip("SWBINTTODO: Swift build produces an error building the fixture for this test.")
+        try XCTSkipSwiftBuildTodo(because: "Swift build produces an error building the fixture for this test.")
     }
 #endif
 
@@ -959,7 +959,7 @@ class BuildCommandSwiftBuildTests: BuildCommandTestCases {
 
     override func testBuildCompleteMessage() async throws {
         #if os(Linux)
-        throw XCTSkip("SWBINTTODO: Need to properly set LD_LIBRARY_PATH on linux")
+        try XCTSkipSwiftBuildTodo(because: "Need to properly set LD_LIBRARY_PATH on linux")
         #else
         try await super.testBuildCompleteMessage()
         #endif

Tests/CommandsTests/MultiRootSupportTests.swift

@@ -16,7 +16,7 @@ import _InternalTestSupport
 import Workspace
 import XCTest
 
-final class MultiRootSupportTests: CommandsTestCase {
+final class MultiRootSupportTests: XCTestCase {
     func testWorkspaceLoader() throws {
         let fs = InMemoryFileSystem(emptyFiles: [
             "/tmp/test/dep/Package.swift",

Tests/CommandsTests/PackageCommandTests.swift

@@ -1385,7 +1385,7 @@ class PackageCommandTestCase: CommandsBuildProviderTestCase {
 
     func testPackageClean() async throws {
         try await fixture(name: "DependencyResolution/External/Simple") { fixturePath in
-            let packageRoot = fixturePath.appending("Bar")
+            let packageRoot: AbsolutePath = fixturePath.appending("Bar")
 
             // Build it.
             await XCTAssertBuilds(packageRoot)
@@ -1397,6 +1397,8 @@ class PackageCommandTestCase: CommandsBuildProviderTestCase {
             // Clean, and check for removal of the build directory but not Packages.
             _ = try await execute(["clean"], packagePath: packageRoot)
             XCTAssertNoSuchPath(binFile)
+            XCTAssertFalse(try localFileSystem.getDirectoryContents(buildPath.appending("repositories")).isEmpty)
+
             // Clean again to ensure we get no error.
             _ = try await execute(["clean"], packagePath: packageRoot)
         }
@@ -1412,15 +1414,10 @@ class PackageCommandTestCase: CommandsBuildProviderTestCase {
             let binFile = buildPath.appending(components: try UserToolchain.default.targetTriple.platformBuildPathComponent, "debug", "Bar")
             XCTAssertFileExists(binFile)
             XCTAssert(localFileSystem.isDirectory(buildPath))
-            // Clean, and check for removal of the build directory but not Packages.
-
-            _ = try await execute(["clean"], packagePath: packageRoot)
-            XCTAssertNoSuchPath(binFile)
-            XCTAssertFalse(try localFileSystem.getDirectoryContents(buildPath.appending("repositories")).isEmpty)
 
             // Fully clean.
             _ = try await execute(["reset"], packagePath: packageRoot)
-            XCTAssertFalse(localFileSystem.isDirectory(buildPath))
+            XCTAssertFalse(localFileSystem.exists(buildPath))
 
             // Test that we can successfully run reset again.
             _ = try await execute(["reset"], packagePath: packageRoot)
@@ -3898,15 +3895,15 @@ class PackageCommandSwiftBuildTests: PackageCommandTestCase {
     }
 
     override func testCommandPluginBuildingCallbacks() async throws {
-        throw XCTSkip("SWBINTTODO: Test fails as plugins are not currenty supported")
+        try XCTSkipSwiftBuildTodo(because: "Test fails as plugins are not currenty supported")
     }
     override func testCommandPluginBuildTestability() async throws {
-        throw XCTSkip("SWBINTTODO: Test fails as plugins are not currenty supported")
+        try XCTSkipSwiftBuildTodo(because: "Test fails as plugins are not currenty supported")
     }
 
 #if !os(macOS)
     override func testCommandPluginTestingCallbacks() async throws {
-        throw XCTSkip("SWBINTTODO: Test fails on inability to find libclang on Linux. Also, plugins are not currently supported")
+        try XCTSkipSwiftBuildTodo(because: "Test fails on inability to find libclang on Linux. Also, plugins are not currently supported")
     }
 #endif
 }

Tests/CommandsTests/RunCommandTests.swift

@@ -255,18 +255,18 @@ class RunCommandSwiftBuildTests: RunCommandTestCase {
     }
 
     override func testMultipleExecutableAndExplicitExecutable() async throws {
-        throw XCTSkip("SWBINTTODO: https://github.com/swiftlang/swift-package-manager/issues/8279: Swift run using Swift Build does not output executable content to the terminal")
+        try XCTSkipSwiftBuildTodo(because: "https://github.com/swiftlang/swift-package-manager/issues/8279: Swift run using Swift Build does not output executable content to the terminal")
     }
 
     override func testUnknownProductAndArgumentPassing() async throws {
-        throw XCTSkip("SWBINTTODO: https://github.com/swiftlang/swift-package-manager/issues/8279: Swift run using Swift Build does not output executable content to the terminal")
+        try XCTSkipSwiftBuildTodo(because: "https://github.com/swiftlang/swift-package-manager/issues/8279: Swift run using Swift Build does not output executable content to the terminal")
     }
 
     override func testToolsetDebugger() async throws {
-        throw XCTSkip("SWBINTTODO: Test fixture fails to build")
+        try XCTSkipSwiftBuildTodo(because: "Test fixture fails to build")
     }
 
     override func testUnreachableExecutable() async throws {
-        throw XCTSkip("SWBINTTODO: Test fails because of build layout differences.")
+        try XCTSkipSwiftBuildTodo(because: "Test fails because of build layout differences.")
     }
 }

Tests/CommandsTests/TestCommandTests.swift

@@ -648,54 +648,54 @@ class TestCommandSwiftBuildTests: TestCommandTestCase {
     }
 
     override func testList() async throws {
-        throw XCTSkip("SWBINTTODO: Test currently fails due to 'error: build failed'")
+        try XCTSkipSwiftBuildTodo(because: "Test currently fails due to 'error: build failed'")
     }
 
     override func testEnableTestDiscoveryDeprecation() async throws {
-        throw XCTSkip("SWBINTTODO: Test currently fails due to 'error: build failed'")
+        try XCTSkipSwiftBuildTodo(because: "Test currently fails due to 'error: build failed'")
     }
 
     override func testEnableDisableTestability() async throws {
-        throw XCTSkip("SWBINTTODO: Test currently fails due to 'error: build failed'")
+        try XCTSkipSwiftBuildTodo(because: "Test currently fails due to 'error: build failed'")
     }
 
     override func testToolsetRunner() async throws {
-        throw XCTSkip("SWBINTTODO: Test currently fails, as some assertions are not met")
+        try XCTSkipSwiftBuildTodo(because: "Test currently fails, as some assertions are not met")
     }
 
     override func testWithReleaseConfiguration() async throws {
-        throw XCTSkip("SWBINTTODO: Test currently fails with 'error: toolchain is invalid: could not find CLI tool `swiftpm-testing-helper` at any of these directories: [..., ...]'")
+        try XCTSkipSwiftBuildTodo(because: "Test currently fails with 'error: toolchain is invalid: could not find CLI tool `swiftpm-testing-helper` at any of these directories: [..., ...]'")
     }
 
     override func testXCTestOnlyDoesNotLogAboutNoMatchingTests() async throws {
-        throw XCTSkip("SWBINTTODO: Test currently fails assertion as the there is a different error message 'error: no tests found; create a target in the 'Tests' directory'")
+        try XCTSkipSwiftBuildTodo(because: "Test currently fails assertion as the there is a different error message 'error: no tests found; create a target in the 'Tests' directory'")
     }
 
     override func testSwiftTestXMLOutputVerifyMultipleTestFailureMessageWithFlagEnabledSwiftTesting() async throws {
-        throw XCTSkip("SWBINTTODO: Test currently fails assertion as the there is a different error message 'error: no tests found; create a target in the 'Tests' directory'")
+        try XCTSkipSwiftBuildTodo(because: "Test currently fails assertion as the there is a different error message 'error: no tests found; create a target in the 'Tests' directory'")
     }
 
     override func testSwiftTestXMLOutputVerifySingleTestFailureMessageWithFlagDisabledSwiftTesting() async throws {
-        throw XCTSkip("SWBINTTODO: Test currently fails, further investigation is needed")
+        try XCTSkipSwiftBuildTodo(because: "Test currently fails, further investigation is needed")
     }
 
     override func testSwiftTestXMLOutputVerifySingleTestFailureMessageWithFlagEnabledSwiftTesting() async throws {
-        throw XCTSkip("SWBINTTODO: Test currently fails, further investigation is needed")
+        try XCTSkipSwiftBuildTodo(because: "Test currently fails, further investigation is needed")
     }
 
     override func testSwiftTestXMLOutputVerifyMultipleTestFailureMessageWithFlagDisabledSwiftTesting() async throws {
-        throw XCTSkip("SWBINTTODO: Test currently fails, further investigation is needed")
+        try XCTSkipSwiftBuildTodo(because: "Test currently fails, further investigation is needed")
     }
 
 #if !canImport(Darwin)
     override func testGeneratedMainIsExistentialAnyClean() async throws {
-        throw XCTSkip("SWBINTTODO: This is a PIF builder missing GUID problem. Further investigation is needed.")
+        try XCTSkipSwiftBuildTodo(because: "This is a PIF builder missing GUID problem. Further investigation is needed.")
     }
 #endif
 
 #if !canImport(Darwin)
     override func testGeneratedMainIsConcurrencySafe_XCTest() async throws {
-        throw XCTSkip("SWBINTTODO: This is a PIF builder missing GUID problem. Further investigation is needed.")
+        try XCTSkipSwiftBuildTodo(because: "This is a PIF builder missing GUID problem. Further investigation is needed.")
     }
 #endif