Skip to content

Commit 7c53f09

Browse files
adam-fowleradam-fowler
authored
Use autobahn for testing client (#1)
* Use autobahn for testing client * Remove reportDirectory as we don't use it * Fix isCI * Set autobahn server name * Check AUTOBAHN_ALL_TESTS environment variable * fix up name of autobahn service
1 parent 1d885aa commit 7c53f09

File tree

4 files changed

+199
-0
lines changed

4 files changed

+199
-0
lines changed

‎.github/workflows/ci.yml

+9
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,16 @@ jobs:
3535
steps:
3636
- name: Checkout
3737
uses: actions/checkout@v4
38+
- name: Restart Autobahn
39+
# The autobahn service container is started *before* swift-websocket is checked
40+
# out. Restarting the container after the checkout step is needed for the
41+
# container to see volumes populated from the checked out workspace.
42+
uses: docker://docker
43+
with:
44+
args: docker restart fuzzingserver
3845
- name: Test
46+
env:
47+
FUZZING_SERVER: autobahn
3948
run: |
4049
swift test --enable-code-coverage
4150
- name: Convert coverage files

‎Tests/WebSocketTests/AutobahnTests.swift

+178
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,178 @@
1+
//===----------------------------------------------------------------------===//
2+
//
3+
// This source file is part of the Hummingbird server framework project
4+
//
5+
// Copyright (c) 2024 the Hummingbird authors
6+
// Licensed under Apache License v2.0
7+
//
8+
// See LICENSE.txt for license information
9+
// See hummingbird/CONTRIBUTORS.txt for the list of Hummingbird authors
10+
//
11+
// SPDX-License-Identifier: Apache-2.0
12+
//
13+
//===----------------------------------------------------------------------===//
14+
15+
import Foundation
16+
import Logging
17+
import NIOConcurrencyHelpers
18+
import NIOPosix
19+
import WSClient
20+
import WSCompression
21+
import XCTest
22+
23+
final class AutobahnTests: XCTestCase {
24+
/// To run all the autobahn tests takes a long time. By default we only run a selection.
25+
/// The `AUTOBAHN_ALL_TESTS` environment flag triggers running all of them.
26+
var runAllTests: Bool { ProcessInfo.processInfo.environment["AUTOBAHN_ALL_TESTS"] == "true" }
27+
var autobahnServer: String { ProcessInfo.processInfo.environment["FUZZING_SERVER"] ?? "localhost" }
28+
29+
func getValue<T: Decodable & Sendable>(_ path: String, as: T.Type) async throws -> T {
30+
let result: NIOLockedValueBox<T?> = .init(nil)
31+
try await WebSocketClient.connect(
32+
url: .init("ws://\(self.autobahnServer):9001/\(path)"),
33+
logger: Logger(label: "Autobahn")
34+
) { inbound, _, _ in
35+
var inboundIterator = inbound.messages(maxSize: .max).makeAsyncIterator()
36+
switch try await inboundIterator.next() {
37+
case .text(let text):
38+
let data = Data(text.utf8)
39+
let report = try JSONDecoder().decode(T.self, from: data)
40+
result.withLockedValue { $0 = report }
41+
42+
case .binary:
43+
preconditionFailure("Received unexpected data")
44+
45+
case .none:
46+
return
47+
}
48+
}
49+
return try result.withLockedValue { try XCTUnwrap($0) }
50+
}
51+
52+
func autobahnTests(
53+
cases: Set<Int>,
54+
extensions: [WebSocketExtensionFactory] = [.perMessageDeflate(maxDecompressedFrameSize: 16_777_216)]
55+
) async throws {
56+
struct CaseInfo: Decodable {
57+
let id: String
58+
let description: String
59+
}
60+
struct CaseStatus: Decodable {
61+
let behavior: String
62+
}
63+
64+
let logger = Logger(label: "Autobahn")
65+
66+
// Run tests
67+
do {
68+
for index in cases.sorted() {
69+
// get case info
70+
let info = try await getValue("getCaseInfo?case=\(index)&agent=swift-websocket", as: CaseInfo.self)
71+
logger.info("\(info.id): \(info.description)")
72+
73+
// run case
74+
try await WebSocketClient.connect(
75+
url: .init("ws://\(self.autobahnServer):9001/runCase?case=\(index)&agent=swift-websocket"),
76+
configuration: .init(maxFrameSize: 16_777_216, extensions: extensions),
77+
logger: logger
78+
) { inbound, outbound, _ in
79+
for try await msg in inbound.messages(maxSize: .max) {
80+
switch msg {
81+
case .binary(let buffer):
82+
try await outbound.write(.binary(buffer))
83+
case .text(let string):
84+
try await outbound.write(.text(string))
85+
}
86+
}
87+
}
88+
89+
// get case status
90+
let status = try await getValue("getCaseStatus?case=\(index)&agent=swift-websocket", as: CaseStatus.self)
91+
XCTAssert(status.behavior == "OK" || status.behavior == "INFORMATIONAL")
92+
}
93+
} catch let error as NIOConnectionError {
94+
logger.error("Autobahn tests require a running Autobahn fuzzing server. Run ./scripts/autobahn-server.sh")
95+
throw error
96+
}
97+
}
98+
99+
func test_1_Framing() async throws {
100+
try await self.autobahnTests(cases: .init(1..<17))
101+
}
102+
103+
func test_2_PingPongs() async throws {
104+
try await self.autobahnTests(cases: .init(17..<28))
105+
}
106+
107+
func test_3_ReservedBits() async throws {
108+
// Reserved bits tests fail
109+
try XCTSkipIf(true)
110+
try await self.autobahnTests(cases: .init(28..<35))
111+
}
112+
113+
func test_4_Opcodes() async throws {
114+
try await self.autobahnTests(cases: .init(35..<45))
115+
}
116+
117+
func test_5_Fragmentation() async throws {
118+
try await self.autobahnTests(cases: .init(45..<65))
119+
}
120+
121+
func test_6_UTF8Handling() async throws {
122+
// UTF8 validation fails
123+
try XCTSkipIf(true)
124+
try await self.autobahnTests(cases: .init(65..<210))
125+
}
126+
127+
func test_7_CloseHandling() async throws {
128+
try await self.autobahnTests(cases: .init(210..<222))
129+
// UTF8 validation fails so skip 222
130+
try await self.autobahnTests(cases: .init(223..<247))
131+
}
132+
133+
func test_9_Performance() async throws {
134+
if !self.runAllTests {
135+
try await self.autobahnTests(cases: .init([247, 260, 270, 281, 291, 296]))
136+
} else {
137+
try await self.autobahnTests(cases: .init(247..<301))
138+
}
139+
}
140+
141+
func test_10_AutoFragmentation() async throws {
142+
try await self.autobahnTests(cases: .init([301]))
143+
}
144+
145+
func test_12_CompressionDifferentPayloads() async throws {
146+
if !self.runAllTests {
147+
try await self.autobahnTests(cases: .init([302, 330, 349, 360, 388]))
148+
} else {
149+
try await self.autobahnTests(cases: .init(302..<391))
150+
}
151+
}
152+
153+
func test_13_CompressionDifferentParameters() async throws {
154+
if !self.runAllTests {
155+
try await self.autobahnTests(cases: .init([392]), extensions: [.perMessageDeflate(noContextTakeover: false, maxDecompressedFrameSize: 131_072)])
156+
try await self.autobahnTests(cases: .init([427]), extensions: [.perMessageDeflate(noContextTakeover: true, maxDecompressedFrameSize: 131_072)])
157+
try await self.autobahnTests(cases: .init([440]), extensions: [.perMessageDeflate(maxWindow: 9, noContextTakeover: false, maxDecompressedFrameSize: 131_072)])
158+
try await self.autobahnTests(cases: .init([451]), extensions: [.perMessageDeflate(maxWindow: 15, noContextTakeover: false, maxDecompressedFrameSize: 131_072)])
159+
try await self.autobahnTests(cases: .init([473]), extensions: [.perMessageDeflate(maxWindow: 9, noContextTakeover: true, maxDecompressedFrameSize: 131_072)])
160+
try await self.autobahnTests(cases: .init([498]), extensions: [.perMessageDeflate(maxWindow: 15, noContextTakeover: true, maxDecompressedFrameSize: 131_072)])
161+
// case 13.7.x are repeated with different setups
162+
try await self.autobahnTests(cases: .init([509]), extensions: [.perMessageDeflate(maxWindow: 9, noContextTakeover: true, maxDecompressedFrameSize: 131_072)])
163+
try await self.autobahnTests(cases: .init([517]), extensions: [.perMessageDeflate(noContextTakeover: true, maxDecompressedFrameSize: 131_072)])
164+
try await self.autobahnTests(cases: .init([504]), extensions: [.perMessageDeflate(noContextTakeover: false, maxDecompressedFrameSize: 131_072)])
165+
} else {
166+
try await self.autobahnTests(cases: .init(392..<410), extensions: [.perMessageDeflate(noContextTakeover: false, maxDecompressedFrameSize: 131_072)])
167+
try await self.autobahnTests(cases: .init(410..<428), extensions: [.perMessageDeflate(noContextTakeover: true, maxDecompressedFrameSize: 131_072)])
168+
try await self.autobahnTests(cases: .init(428..<446), extensions: [.perMessageDeflate(maxWindow: 9, noContextTakeover: false, maxDecompressedFrameSize: 131_072)])
169+
try await self.autobahnTests(cases: .init(446..<464), extensions: [.perMessageDeflate(maxWindow: 15, noContextTakeover: false, maxDecompressedFrameSize: 131_072)])
170+
try await self.autobahnTests(cases: .init(464..<482), extensions: [.perMessageDeflate(maxWindow: 9, noContextTakeover: true, maxDecompressedFrameSize: 131_072)])
171+
try await self.autobahnTests(cases: .init(482..<500), extensions: [.perMessageDeflate(maxWindow: 15, noContextTakeover: true, maxDecompressedFrameSize: 131_072)])
172+
// case 13.7.x are repeated with different setups
173+
try await self.autobahnTests(cases: .init(500..<518), extensions: [.perMessageDeflate(maxWindow: 9, noContextTakeover: true, maxDecompressedFrameSize: 131_072)])
174+
try await self.autobahnTests(cases: .init(500..<518), extensions: [.perMessageDeflate(noContextTakeover: true, maxDecompressedFrameSize: 131_072)])
175+
try await self.autobahnTests(cases: .init(500..<518), extensions: [.perMessageDeflate(noContextTakeover: false, maxDecompressedFrameSize: 131_072)])
176+
}
177+
}
178+
}

‎scripts/autobahn-config/fuzzingserver.json

+6
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
{
2+
"url": "ws://127.0.0.1:9001",
3+
"outdir": "./reports/",
4+
"cases": ["*"],
5+
"exclude-agent-cases": {}
6+
}

‎scripts/autobahn-server.sh

+6
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
docker run -it --rm \
2+
-v "${PWD}/scripts/autobahn-config:/config" \
3+
-v "${PWD}/.build/reports:/reports" \
4+
-p 9001:9001 \
5+
--name fuzzingserver \
6+
crossbario/autobahn-testsuite

0 commit comments

Comments
 (0)