Improvements to test-go CI job

.github/workflows/test-go.yaml

@@ -42,18 +42,26 @@ jobs:
   test-go:
     strategy:
       matrix:
-        suite: ["integration-core", "integration-enterprise", "integration-mdm", "core", "mysql", "fleetctl", "vuln"]
+        suite: ["integration-core", "integration-enterprise", "integration-mdm", "fast", "fleetctl", "main", "mysql", "scripts", "service", "vuln"]
         os: [ubuntu-latest]
         mysql: ["mysql:8.0.36", "mysql:8.4.3", "mysql:9.1.0"] # make sure to update supported versions docs when this changes
         isCron:
           - ${{ github.event_name == 'schedule' }}
-        # Only run MySQL 9 tests on cron schedule
         exclude:
           - isCron: false
+            mysql: "mysql:9.1.0" # Only run MySQL 9 tests on cron schedule
+          # The suites below do not need MySQL, so we exclude additional MySQL options from the above matrix.
+          - suite: "fast"
+            mysql: "mysql:8.4.3"
+          - suite: "fast"
             mysql: "mysql:9.1.0"
-    # Don't cancel all tests if an integration suite fails as integration suites are more likely to be flakey.
-    # Don't cancel all tests if the vulns suite fails as vulns may start failing for unrelated reasons to PR'd code.
-    continue-on-error: ${{ matrix.suite == 'vuln' || startsWith(matrix.suite, 'integration-') }}
+          - suite: "scripts"
+            mysql: "mysql:8.4.3"
+          - suite: "scripts"
+            mysql: "mysql:9.1.0"
+    # Don't cancel other jobs if one test suite fails. Since our product teams are tightly coupled, we never want to see our tests fail due
+    # to an unrelated issue in another product area.
+    continue-on-error: true
     runs-on: ${{ matrix.os }}
 
     env:
@@ -66,6 +74,47 @@ jobs:
       with:
         egress-policy: audit
 
+    - name: Configure job
+      run: |
+        echo "RUN_TESTS_ARG=" >> $GITHUB_ENV
+        if [[ "${{ matrix.suite }}" == "main" ]]; then
+          echo "CI_TEST_PKG=main" >> $GITHUB_ENV
+          echo "NEED_DOCKER=1" >> $GITHUB_ENV
+        elif [[ "${{ matrix.suite }}" == "fast" ]]; then
+          # DO NOT add any dependencies in this test suite.
+          echo "CI_TEST_PKG=${{ matrix.suite }}" >> $GITHUB_ENV
+        elif [[ "${{ matrix.suite }}" == "service" ]]; then
+          echo "CI_TEST_PKG=service" >> $GITHUB_ENV
+          echo "RUN_TESTS_ARG=-skip=^TestIntegrations" >> $GITHUB_ENV
+          echo "NEED_DOCKER=1" >> $GITHUB_ENV
+        elif [[ "${{ matrix.suite }}" == "integration-core" ]]; then
+          echo "CI_TEST_PKG=service" >> $GITHUB_ENV
+          echo "RUN_TESTS_ARG=-run=^TestIntegrations -skip '^(TestIntegrationsMDM|TestIntegrationsEnterprise)'" >> $GITHUB_ENV
+          # We re-generate test schema just in case there is an issue with the schema. We only do this for one test.
+          echo "GENERATE_TEST_SCHEMA=1" >> $GITHUB_ENV
+          echo "NEED_DOCKER=1" >> $GITHUB_ENV
+        elif [[ "${{ matrix.suite }}" == "integration-mdm" ]]; then
+          echo "CI_TEST_PKG=service" >> $GITHUB_ENV
+          echo "RUN_TESTS_ARG=-run=^TestIntegrationsMDM" >> $GITHUB_ENV
+          echo "NEED_DOCKER=1" >> $GITHUB_ENV
+        elif [[ "${{ matrix.suite }}" == "integration-enterprise" ]]; then
+          echo "CI_TEST_PKG=service" >> $GITHUB_ENV
+          echo "RUN_TESTS_ARG=-run=^TestIntegrationsEnterprise" >> $GITHUB_ENV
+          echo "NEED_DOCKER=1" >> $GITHUB_ENV
+        elif [[ "${{ matrix.suite }}" == "scripts" ]]; then
+          echo "CI_TEST_PKG=${{ matrix.suite }}" >> $GITHUB_ENV
+          echo "NEED_ZSH=1" >> $GITHUB_ENV
+        else
+          echo "CI_TEST_PKG=${{ matrix.suite }}" >> $GITHUB_ENV
+          echo "NEED_DOCKER=1" >> $GITHUB_ENV
+        fi
+
+    - name: Set Go race setting on schedule
+      if: github.event.schedule == '0 4 * * *'
+      run: |
+        echo "RACE_ENABLED=true" >> $GITHUB_ENV
+        echo "GO_TEST_TIMEOUT=1h" >> $GITHUB_ENV
+
     - name: Checkout Code
       uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3
 
@@ -76,30 +125,27 @@ jobs:
 
     # Pre-starting dependencies here means they are ready to go when we need them.
     - name: Start Infra Dependencies
+      if: ${{ env.NEED_DOCKER }}
       # Use & to background this
       run: FLEET_MYSQL_IMAGE=${{ matrix.mysql }} docker compose -f docker-compose.yml -f docker-compose-redis-cluster.yml up -d mysql_test mysql_replica_test redis redis-cluster-1 redis-cluster-2 redis-cluster-3 redis-cluster-4 redis-cluster-5 redis-cluster-6 redis-cluster-setup minio saml_idp mailhog mailpit smtp4dev_test &
 
     - name: Add TLS certificate for SMTP Tests
+      if: ${{ env.NEED_DOCKER }}
       run: |
         sudo cp tools/smtp4dev/fleet.crt /usr/local/share/ca-certificates/
         sudo update-ca-certificates
 
-    # This step takes >20 seconds and should be skipped for jobs that don't need it. https://github.com/fleetdm/fleet/issues/27435
     - name: Install ZSH
+      if: ${{ env.NEED_ZSH }}
       run: sudo apt update && sudo apt install -y zsh
 
     - name: Generate static files
       run: |
         export PATH=$PATH:~/go/bin
         make generate-go
 
-    - name: Set Go race setting on schedule
-      if: github.event.schedule == '0 4 * * *'
-      run: |
-        echo "RACE_ENABLED=true" >> $GITHUB_ENV
-        echo "GO_TEST_TIMEOUT=1h" >> $GITHUB_ENV
-
     - name: Wait for mysql
+      if: ${{ env.NEED_DOCKER }}
       run: |
         echo "waiting for mysql..."
         until docker compose exec -T mysql_test sh -c "mysql -uroot -p\"\${MYSQL_ROOT_PASSWORD}\" -e \"SELECT 1=1\" fleet" &> /dev/null; do
@@ -114,25 +160,13 @@ jobs:
         done
         echo "mysql replica is ready"
 
+    - name: Generate test schema
+      if: ${{ env.GENERATE_TEST_SCHEMA }}
+      run: make test-schema
+
     - name: Run Go Tests
       run: |
-        if [[ "${{ matrix.suite }}" == "core" ]]; then
-          CI_TEST_PKG=main
-          RUN_TESTS_ARG='-skip=^TestIntegrations'
-        elif [[ "${{ matrix.suite }}" == "integration-core" ]]; then
-          CI_TEST_PKG=integration
-          RUN_TESTS_ARG='-run=^TestIntegrations -skip "^(TestIntegrationsMDM|TestIntegrationsEnterprise)"'
-        elif [[ "${{ matrix.suite }}" == "integration-mdm" ]]; then
-          CI_TEST_PKG=integration
-          RUN_TESTS_ARG='-run=^TestIntegrationsMDM'
-        elif [[ "${{ matrix.suite }}" == "integration-enterprise" ]]; then
-          CI_TEST_PKG=integration
-          RUN_TESTS_ARG='-run=^TestIntegrationsEnterprise'
-        else
-          CI_TEST_PKG="${{ matrix.suite }}"
-          RUN_TESTS_ARG=''
-        fi
-        GO_TEST_EXTRA_FLAGS="-v -race=$RACE_ENABLED -timeout=$GO_TEST_TIMEOUT $RUN_TESTS_ARG" \
+        GO_TEST_EXTRA_FLAGS="-v -race=$RACE_ENABLED -timeout=$GO_TEST_TIMEOUT ${{ env.RUN_TESTS_ARG }}" \
           TEST_LOCK_FILE_PATH=$(pwd)/lock \
           TEST_CRON_NO_RECOVER=1 \
           NETWORK_TEST=1 \
@@ -143,7 +177,7 @@ jobs:
           SAML_IDP_TEST=1 \
           MAIL_TEST=1 \
           NETWORK_TEST_GITHUB_TOKEN=${{ secrets.FLEET_RELEASE_GITHUB_PAT }} \
-          CI_TEST_PKG="${CI_TEST_PKG}" \
+          CI_TEST_PKG="${{ env.CI_TEST_PKG }}" \
           make test-go 2>&1 | tee /tmp/gotest.log
 
     - name: Create mysql identifier without colon

Makefile

@@ -121,14 +121,14 @@ fdm:
 		sudo ln -sf "$$(pwd)/build/fdm" /usr/local/bin/fdm; \
 	fi
 
-.help-short--serve: 
+.help-short--serve:
 	@echo "Start the fleet server"
-.help-short--up: 
+.help-short--up:
 	@echo "Start the fleet server (alias for \`serve\`)"
 .help-long--serve: SERVE_CMD:=serve
 .help-long--up: SERVE_CMD:=up
 .help-long--serve .help-long--up:
-	@echo "Starts an instance of the Fleet web and API server." 
+	@echo "Starts an instance of the Fleet web and API server."
 	@echo
 	@echo "  By default the server will listen on localhost:8080, in development mode with a premium license."
 	@echo "  If different options are used to start the server, the options will become 'sticky' and will be used the next time \`$(TOOL_CMD) $(SERVE_CMD)\` is called."
@@ -291,19 +291,48 @@ debug-go-tests:
 	@MYSQL_TEST=1 REDIS_TEST=1 MINIO_STORAGE_TEST=1 SAML_IDP_TEST=1 NETWORK_TEST=1 make .debug-go-tests
 
 # Set up packages for CI testing.
-DEFAULT_PKG_TO_TEST := ./cmd/... ./ee/... ./orbit/pkg/... ./orbit/cmd/orbit ./pkg/... ./server/... ./tools/...
+DEFAULT_PKGS_TO_TEST := ./cmd/... ./ee/... ./orbit/pkg/... ./orbit/cmd/orbit ./pkg/... ./server/... ./tools/...
+# fast tests are quick and do not require out-of-process dependencies (such as MySQL, etc.)
+FAST_PKGS_TO_TEST := \
+	./ee/tools/mdm \
+	./orbit/pkg/cryptoinfo \
+	./orbit/pkg/dataflatten \
+	./orbit/pkg/keystore \
+	./server/goose \
+	./server/mdm/apple/appmanifest \
+	./server/mdm/lifecycle \
+	./server/mdm/scep/challenge \
+	./server/mdm/scep/x509util \
+	./server/policies
+FLEETCTL_PKGS_TO_TEST := ./cmd/fleetctl/...
+MYSQL_PKGS_TO_TEST := ./server/datastore/mysql/... ./server/mdm/android/mysql
+SCRIPTS_PKGS_TO_TEST := ./orbit/pkg/scripts
+SERVICE_PKGS_TO_TEST := ./server/service
+VULN_PKGS_TO_TEST := ./server/vulnerabilities/...
 ifeq ($(CI_TEST_PKG), main)
-	CI_PKG_TO_TEST=$(shell go list ${DEFAULT_PKG_TO_TEST} | grep -v "server/datastore/mysql" | grep -v "cmd/fleetctl" | grep -v "server/vulnerabilities" | sed -e 's|github.com/fleetdm/fleet/v4/||g')
-else ifeq ($(CI_TEST_PKG), integration)
-	CI_PKG_TO_TEST="server/service"
-else ifeq ($(CI_TEST_PKG), mysql)
-	CI_PKG_TO_TEST="server/datastore/mysql/..."
+    # This is the bucket of all the tests that are not in a specific group. We take a diff between DEFAULT_PKG_TO_TEST and all the specific *_PKGS_TO_TEST.
+	CI_PKG_TO_TEST=$(shell /bin/bash -c "comm -23 <(go list ${DEFAULT_PKGS_TO_TEST} | sort) <({ \
+	go list $(FAST_PKGS_TO_TEST) && \
+	go list $(FLEETCTL_PKGS_TO_TEST) && \
+	go list $(MYSQL_PKGS_TO_TEST) && \
+	go list $(SCRIPTS_PKGS_TO_TEST) && \
+	go list $(SERVICE_PKGS_TO_TEST) && \
+	go list $(VULN_PKGS_TO_TEST) \
+	;} | sort) | sed -e 's|github.com/fleetdm/fleet/v4/||g'")
+else ifeq ($(CI_TEST_PKG), fast)
+	CI_PKG_TO_TEST=$(FAST_PKGS_TO_TEST)
 else ifeq ($(CI_TEST_PKG), fleetctl)
-	CI_PKG_TO_TEST="cmd/fleetctl/..."
+	CI_PKG_TO_TEST=$(FLEETCTL_PKGS_TO_TEST)
+else ifeq ($(CI_TEST_PKG), mysql)
+	CI_PKG_TO_TEST=$(MYSQL_PKGS_TO_TEST)
+else ifeq ($(CI_TEST_PKG), scripts)
+	CI_PKG_TO_TEST=$(SCRIPTS_PKGS_TO_TEST)
+else ifeq ($(CI_TEST_PKG), service)
+	CI_PKG_TO_TEST=$(SERVICE_PKGS_TO_TEST)
 else ifeq ($(CI_TEST_PKG), vuln)
-	CI_PKG_TO_TEST="server/vulnerabilities/..."
+	CI_PKG_TO_TEST=$(VULN_PKGS_TO_TEST)
 else
-	CI_PKG_TO_TEST=$(DEFAULT_PKG_TO_TEST)
+	CI_PKG_TO_TEST=$(DEFAULT_PKGS_TO_TEST)
 endif
 # Command used in CI to run all tests.
 .help-short--test-go:
@@ -315,12 +344,14 @@ endif
 	@echo "The test package bundle to run.  If not specified, all Go tests will run."
 .help-extra--test-go:
 	@echo "AVAILABLE TEST BUNDLES:"
-	@echo "  integration"
+	@echo "  fast"
+	@echo "  service"
+	@echo "  scripts"
 	@echo "  mysql"
 	@echo "  fleetctl"
 	@echo "  vuln"
 	@echo "  main        (all tests not included in other bundles)"
-test-go: test-schema mock
+test-go:
 	make .run-go-tests PKG_TO_TEST="$(CI_PKG_TO_TEST)"
 
 analyze-go:
@@ -573,7 +604,7 @@ db-restore:
 .help-short--snap .help-short--snapshot:
 	@echo "Snapshot the database"
 .help-long--snap .help-long--snapshot:
-	@echo "Interactively take a snapshot of the present database state. Restore snapshots with \`$(TOOL_CMD) restore\`."	
+	@echo "Interactively take a snapshot of the present database state. Restore snapshots with \`$(TOOL_CMD) restore\`."
 
 SNAPSHOT_BINARY = ./build/snapshot
 snap snapshot: $(SNAPSHOT_BINARY)
@@ -584,7 +615,7 @@ $(SNAPSHOT_BINARY): tools/snapshot/*.go
 .help-short--restore:
 	@echo "Restore a database snapshot"
 .help-long--restore:
-	@echo "Interactively restore database state using a snapshot taken with \`$(TOOL_CMD) snapshot\`."	
+	@echo "Interactively restore database state using a snapshot taken with \`$(TOOL_CMD) snapshot\`."
 .help-options--restore:
 	@echo "PREPARE (alias: PREP)"
 	@echo "Run migrations after restoring the snapshot"