Merge #103

103: Add a simple test runner in GDScript r=Bromeon a=ttencate

I made the output look like `cargo test` to reduce the cognitive burden while switching between the two languages:

![2023-01-31T20:52:24_1057x1384](https://user-images.githubusercontent.com/90930/215867771-ace18fd1-d1f6-4af2-8f59-b3d98b012328.png)

Nothing has been done on the Rust side yet, so there is one big test `IntegrationTests::test_all()` (renamed from `run()`).

Co-authored-by: Thomas ten Cate <[email protected]>

Author: bors[bot]

.gitignore

@@ -7,11 +7,12 @@
 target
 Cargo.lock
 
-# Godot (extension list needed to run tests without prior editor opening)
-.godot
-godot.log
-!/itest/.godot/extension_list.cfg
-*.import
+# Godot
+**/.import/
+**/.godot/**
+# Needed to run projects without having to open the editor first.
+!**/.godot/extension_list.cfg
+!**/.godot/global_script_class_cache.cfg
 
 # This project: input JSONs and code generation
 gen
@@ -20,4 +21,4 @@ gen
 /script
 *.bat
 config.toml
-exp.rs
+exp.rs

examples/dodge-the-creeps/godot/.gitignore

@@ -1,2 +1,3 @@
-.import
+.godot/
+.import/
 logs/

itest/godot/.godot/global_script_class_cache.cfg

@@ -0,0 +1,13 @@
+list=[{
+"base": &"RefCounted",
+"class": &"TestStats",
+"icon": "",
+"language": &"GDScript",
+"path": "res://TestStats.gd"
+}, {
+"base": &"Node",
+"class": &"TestSuite",
+"icon": "",
+"language": &"GDScript",
+"path": "res://TestSuite.gd"
+}]

itest/godot/ManualFfiTests.gd

@@ -2,60 +2,39 @@
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at https://mozilla.org/MPL/2.0/.
 
-extends Node
+extends TestSuite
 
-func run() -> bool:
-	print("[GD] Test ManualFfi...")
-	var ok = true
-	ok = ok && test_missing_init()
-	ok = ok && test_to_string()
-	ok = ok && test_export()
-
-	print("[GD] ManualFfi tested (passed=", ok, ")")
-	return ok
-
-func test_missing_init() -> bool:
-	return true # TODO: fix dynamic eval
+func test_missing_init():
+	return # TODO: fix dynamic eval
 
 	var expr = Expression.new()
 	var error = expr.parse("WithoutInit.new()")
-	if error != OK:
-		print("Failed to parse dynamic expression")
-		return false
+	if !assert_eq(error, OK, "Failed to parse dynamic expression"):
+		return
 
 	var instance = expr.execute()
-	if expr.has_execute_failed():
-		print("Failed to evaluate dynamic expression")
-		return false
+	if !assert_that(!expr.has_execute_failed(), "Failed to evaluate dynamic expression"):
+		return
 
 	print("[GD] WithoutInit is: ", instance)
-	return true
 
-func test_to_string() -> bool:
+func test_to_string():
 	var ffi = VirtualMethodTest.new()
-	var s = str(ffi)
-
-	print("to_string: ", s)
-	print("to_string: ", ffi)
-	return true
 
-func test_export() -> bool:
+	assert_eq(str(ffi), "VirtualMethodTest[integer=0]")
+
+func test_export():
 	var obj = HasProperty.new()
 
 	obj.int_val = 5
-	print("[GD] HasProperty's int_val property is: ", obj.int_val, " and should be 5")
-	var int_val_correct = obj.int_val == 5
+	assert_eq(obj.int_val, 5)
 
 	obj.string_val = "test val"
-	print("[GD] HasProperty's string_val property is: ", obj.string_val, " and should be \"test val\"")
-	var string_val_correct = obj.string_val == "test val"
+	assert_eq(obj.string_val, "test val")
 
 	var node = Node.new()
 	obj.object_val = node
-	print("[GD] HasProperty's object_val property is: ", obj.object_val, " and should be ", node)
-	var object_val_correct = obj.object_val == node
+	assert_eq(obj.object_val, node)
 
 	obj.free()
 	node.free()
-
-	return int_val_correct && string_val_correct && object_val_correct

itest/godot/TestRunner.gd

@@ -5,23 +5,74 @@
 extends Node
 
 func _ready():
-	var rust_tests := IntegrationTests.new()
-
-	var rust = rust_tests.run()
-	var manual = $ManualFfiTests.run()
-	var generated = $GenFfiTests.run()
-
-	var status: bool = rust && manual && generated
-
+	var test_suites: Array = [
+		IntegrationTests.new(),
+		preload("res://ManualFfiTests.gd").new(),
+		preload("res://gen/GenFfiTests.gd").new(),
+	]
+	
+	var tests: Array[_Test] = []
+	for suite in test_suites:
+		for method in suite.get_method_list():
+			var method_name: String = method.name
+			if method_name.begins_with("test_"):
+				tests.push_back(_Test.new(suite, method_name))
+	
 	print()
-	var exit_code: int
-	if status:
-		print(" All tests PASSED.")
-		exit_code = 0
-	else:
-		print(" Tests FAILED.")
-		exit_code = 1
-
-	print(" -- exiting.")
-	rust_tests.free()
+	print_rich("  [b][color=green]Running[/color][/b] test project %s" % [
+		ProjectSettings.get_setting("application/config/name", ""),
+	])
+	print()
+	
+	var stats: TestStats = TestStats.new()
+	stats.start_stopwatch()
+	for test in tests:
+		printraw("  -- %s ... " % [test.test_name])
+		var ok: bool = test.run()
+		print_rich("[color=green]ok[/color]" if ok else "[color=red]FAILED[/color]")
+		stats.add(ok)
+	stats.stop_stopwatch()
+	
+	print()
+	print_rich("test result: %s. %d passed; %d failed; finished in %.2fs" % [
+		"[color=green]ok[/color]" if stats.all_passed() else "[color=red]FAILED[/color]",
+		stats.num_ok,
+		stats.num_failed,
+		stats.runtime_seconds(),
+	])
+	print()
+	
+	for suite in test_suites:
+		suite.free()
+	
+	var exit_code: int = 0 if stats.all_passed() else 1
 	get_tree().quit(exit_code)
+
+class _Test:
+	var suite: Object
+	var method_name: String
+	var test_name: String
+	
+	func _init(suite: Object, method_name: String):
+		self.suite = suite
+		self.method_name = method_name
+		self.test_name = "%s::%s" % [_suite_name(suite), method_name]
+	
+	func run():
+		# This is a no-op if the suite doesn't have this property.
+		suite.set("_assertion_failed", false)
+		var result = suite.call(method_name)
+		var ok: bool = (
+			(result == true or result == null)
+			&& !suite.get("_assertion_failed")
+		)
+		return ok
+	
+	static func _suite_name(suite: Object) -> String:
+		var script := suite.get_script()
+		if script:
+			# Test suite written in GDScript.
+			return script.resource_path.get_file().get_basename()
+		else:
+			# Test suite written in Rust.
+			return suite.get_class()

itest/godot/TestRunner.tscn

@@ -1,14 +1,6 @@
-[gd_scene load_steps=4 format=3 uid="uid://dgcj68l8n6wpb"]
+[gd_scene load_steps=2 format=3 uid="uid://dgcj68l8n6wpb"]
 
 [ext_resource type="Script" path="res://TestRunner.gd" id="1_wdbrf"]
-[ext_resource type="Script" path="res://ManualFfiTests.gd" id="2_gmtlc"]
-[ext_resource type="Script" path="res://gen/GenFfiTests.gd" id="3_vivae"]
 
 [node name="TestRunner" type="Node"]
 script = ExtResource("1_wdbrf")
-
-[node name="ManualFfiTests" type="Node" parent="."]
-script = ExtResource("2_gmtlc")
-
-[node name="GenFfiTests" type="Node" parent="."]
-script = ExtResource("3_vivae")

itest/godot/TestStats.gd

@@ -0,0 +1,33 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at https://mozilla.org/MPL/2.0/.
+
+class_name TestStats
+extends RefCounted
+
+var num_run := 0
+var num_ok := 0
+var num_failed := 0
+
+var _start_time_usec := 0
+var _runtime_usec := 0
+
+func add(ok: bool):
+	num_run += 1
+	if ok:
+		num_ok += 1
+	else:
+		num_failed += 1
+
+func all_passed() -> bool:
+	# Consider 0 tests run as a failure too, because it's probably a problem with the run itself.
+	return num_failed == 0 && num_run > 0
+
+func start_stopwatch():
+	_start_time_usec = Time.get_ticks_usec()
+
+func stop_stopwatch():
+	_runtime_usec += Time.get_ticks_usec() - _start_time_usec
+
+func runtime_seconds() -> float:
+	return _runtime_usec * 1.0e-6

itest/godot/TestSuite.gd

@@ -0,0 +1,29 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at https://mozilla.org/MPL/2.0/.
+
+class_name TestSuite
+extends Node
+
+var _assertion_failed: bool = false
+
+## Asserts that `what` is `true`, but does not abort the test. Returns `what` so you can return
+## early from the test function if the assertion failed.
+func assert_that(what: bool, message: String = "") -> bool:
+	if !what:
+		_assertion_failed = true
+		if message:
+			print("assertion failed: %s" % message)
+		else:
+			print("assertion failed")
+	return what
+
+func assert_eq(left, right, message: String = "") -> bool:
+	if left != right:
+		_assertion_failed = true
+		if message:
+			print("assertion failed: %s\n  left: %s\n right: %s" % [message, left, right])
+		else:
+			print("assertion failed: `(left == right)`\n  left: %s\n right: %s" % [left, right])
+		return false
+	return true

itest/godot/input/GenFfiTests.template.gd

@@ -2,28 +2,16 @@
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at https://mozilla.org/MPL/2.0/.
 
-extends Node
+extends TestSuite
 
-func run() -> bool:
+#(
+func test_varcall_IDENT():
 	var ffi = GenFfi.new()
-	print("[GD] GenFfi constructed: ", ffi.get_instance_id())
-	var ok := true#(
-	if !test_varcall_IDENT(ffi):
-		ok = false
-		push_error("  -- FFI test failed: test_varcall_IDENT")
-	#)
-
-	print("[GD] GenFfi destructing...")
-	return ok
 
-#(
-func test_varcall_IDENT(ffi: GenFfi) -> bool:
 	var from_rust = ffi.return_IDENT()
-	var ok1: bool = ffi.accept_IDENT(from_rust)
+	assert_that(ffi.accept_IDENT(from_rust))
 
 	var from_gdscript = VAL
 	var mirrored = ffi.mirror_IDENT(from_gdscript)
-	var ok2: bool = (mirrored == from_gdscript)
-
-	return ok1 && ok2
+	assert_eq(mirrored, from_gdscript)
 #)

itest/godot/project.godot

@@ -13,3 +13,8 @@ config_version=5
 config/name="IntegrationTests"
 run/main_scene="res://TestRunner.tscn"
 config/features=PackedStringArray("4.0")
+run/flush_stdout_on_print=true
+
+[debug]
+
+gdscript/warnings/shadowed_variable=0

itest/rust/src/lib.rs

@@ -61,7 +61,7 @@ struct IntegrationTests {}
 #[godot_api]
 impl IntegrationTests {
     #[func]
-    fn run(&mut self) -> bool {
+    fn test_all(&mut self) -> bool {
         println!("Run Godot integration tests...");
         run_tests()
     }