WIP: Get total runs

.pre-commit-config.yaml

@@ -7,6 +7,7 @@ repos:
     rev: v2.2.1
     hooks:
       - id: prettier
+        verbose: true
   - repo: https://github.com/igorshubovych/markdownlint-cli
     rev: v0.43.0
     hooks:

tests/conftest.py

@@ -964,6 +964,27 @@ def _fetch_testrun_matrix():
     )
 
 
+@pytest.fixture
+def mock_summary_groups_request(monkeypatch):
+    """
+    Mock get_summary_groups()
+    """
+
+    def _fetch_summary_groups(start_day, end_day):
+        tests_folder = os.path.dirname(__file__)
+        file_name = "summary_groups.json"
+        data_path = os.path.join(tests_folder, "sample_data", file_name)
+        with open(data_path) as f:
+            data = json.load(f)
+        return data
+
+    monkeypatch.setattr(
+        treeherder.intermittents_commenter.commenter.fetch,
+        "fetch_summary_groups",
+        _fetch_summary_groups,
+    )
+
+
 class MockResponse:
     def __init__(self):
         self.status_code = 200

tests/intermittents_commenter/expected_comment.text

@@ -7,7 +7,7 @@ This is the #1 most frequent failure this week.
 ## Table
 | |**debug**|**no_variant**|**opt**|**spi-nw**|
 |---|:-:|:-:|:-:|:-:|
-|**macosx1015-x86_64/debug**| | | |1|
+|**macosx1015-x86_64/debug**| | | |1/22|
 
 ## For more details, see:
 https://treeherder.mozilla.org/intermittent-failures/bugdetails?bug=1&startday=2022-05-09&endday=2025-05-10&tree=all

tests/intermittents_commenter/expected_comment_with_5_failures.text

@@ -7,10 +7,10 @@ This is the #1 most frequent failure this week.
 ## Table
 | |**asan**|**debug**|**headless**|**no_variant**|**opt**|**spi-nw**|**swr**|**tsan**|
 |---|:-:|:-:|:-:|:-:|:-:|:-:|:-:|:-:|
-|**linux-x86/unknown build**| | |2| | | | | |
-|**linux1804-x86_64/debug**| | | | | |1| | |
-|**macosx1015-x86_64/debug**| | | | | |1| | |
-|**macosx1470-x86_64/debug**| | | | | | |1| |
+|**linux-x86/unknown build**| | |2/0| | | | | |
+|**linux1804-x86_64/debug**| | | | | |1/92| | |
+|**macosx1015-x86_64/debug**| | | | | |1/22| | |
+|**macosx1470-x86_64/debug**| | | | | | |1/0| |
 
 ## For more details, see:
 https://treeherder.mozilla.org/intermittent-failures/bugdetails?bug=1&startday=2022-05-09&endday=2025-05-10&tree=all

tests/intermittents_commenter/test_commenter.py

@@ -10,6 +10,7 @@ def test_intermittents_commenter(
     mock_test_variants_firefoxci_request,
     mock_test_manifests_firefoxci_request,
     mock_testrun_matrix_firefoxci_request,
+    mock_summary_groups_request,
 ):
     startday = "2022-05-09"
     endday = "2025-05-10"
@@ -53,6 +54,7 @@ def test_intermittents_commenter_with_failures(
     mock_test_variants_firefoxci_request,
     mock_test_manifests_firefoxci_request,
     mock_testrun_matrix_firefoxci_request,
+    mock_summary_groups_request,
 ):
     startday = "2022-05-09"
     endday = "2025-05-10"

treeherder/intermittents_commenter/comment.template

@@ -35,7 +35,11 @@ This is the #{{rank}} most frequent failure this week.{% endif %}
 {% for platform_and_build, test_by_variant in data_table.items() | sort() -%}
 |**{{platform_and_build}}**|
   {%- for variant in test_variants -%}
-    {{test_by_variant.get(variant, " ")}}|
+    {%- if variant in test_by_variant -%}
+    {{test_by_variant[variant].get('count', ' ')}}/{{test_by_variant[variant].get('runs', ' ')}}|
+    {%- else -%}
+    {{" |"}}
+    {%- endif -%}
   {%- endfor %}
 {% endfor %}
 ## For more details, see:

treeherder/intermittents_commenter/commenter.py

@@ -14,7 +14,7 @@
     COMPONENTS,
     WHITEBOARD_NEEDSWORK_OWNER,
 )
-from treeherder.model.models import BugJobMap, Bugscache, OptionCollection
+from treeherder.model.models import BugJobMap, OptionCollection
 
 from . import fetch
 
@@ -42,6 +42,7 @@ class BugRunInfo:
 
 @dataclass
 class BugsDetails:
+    run_count: dict[str, int] = field(default_factory=dict)  # {job_name: run_count}
     total: int = 0
     test_variants: set = field(default_factory=set)
     per_repositories: dict[str, int] = field(default_factory=dict)  # {repo1: 1, repo2: 2, ...}
@@ -59,6 +60,7 @@ class Commenter:
     test_variants = None
     manifests = None
     testrun_matrix = None
+    summary_groups = None
 
     def __init__(self, weekly_mode, dry_run=False):
         self.weekly_mode = weekly_mode
@@ -79,7 +81,7 @@ def generate_bug_changes(self, startday, endday, alt_startday, alt_endday):
 
         bug_ids, bugs = self.get_bugs(startday, endday)
         option_collection_map = OptionCollection.objects.get_option_collection_map()
-        bug_map = self.build_bug_map(bugs, option_collection_map)
+        bug_map = self.build_bug_map(bugs, option_collection_map, startday, endday)
 
         alt_date_bug_totals = self.get_alt_date_bug_totals(alt_startday, alt_endday, bug_ids)
 
@@ -306,6 +308,7 @@ def get_bugs(self, startday, endday):
                 "bug__bugzilla_id",
                 "job__option_collection_hash",
                 "job__signature__job_type_name",
+                "bug__summary",
             )
         )
         return bug_ids, bugs
@@ -385,7 +388,32 @@ def get_bug_run_info(self, bug):
             info.build_type = "unknown build"
         return info
 
-    def build_bug_map(self, bugs, option_collection_map):
+    def get_task_labels_and_count(self, manifest, start_day, end_day):
+        tasks_and_count = {}
+        summary_groups = (
+            fetch.fetch_summary_groups(start_day, end_day)
+            if self.summary_groups is None
+            else self.summary_groups
+        )
+        days = [start_day]
+        if self.weekly_mode:
+            for j in range(6):
+                jj = datetime.strptime(days[-1], "%Y-%m-%d") + timedelta(days=1)
+                days.append(jj.strftime("%Y-%m-%d"))
+        for day in days:
+            if day not in summary_groups:
+                continue
+            all_task_labels = summary_groups[day]["job_type_names"]
+            for tasks_by_manifest in summary_groups[day]["manifests"]:
+                for man, tasks in tasks_by_manifest.items():
+                    if manifest == man:
+                        for task_index, _, _, count in tasks:
+                            task_label = all_task_labels[task_index]
+                            tasks_and_count.setdefault(task_label, 0)
+                            tasks_and_count[task_label] += count
+        return tasks_and_count
+
+    def build_bug_map(self, bugs, option_collection_map, start_day, end_day):
         """Build bug_map
          eg:
         {
@@ -413,22 +441,30 @@ def build_bug_map(self, bugs, option_collection_map):
         }
         """
         bug_map = {}
-        bug_ids = [b["bug__bugzilla_id"] for b in bugs]
-        bug_summaries = Bugscache.objects.filter(bugzilla_id__in=bug_ids).values(
-            "summary", "bugzilla_id"
-        )
         all_variants = set()
-        for bug, bug_id in zip(bugs, bug_ids):
-            manifest = self.get_test_manifest(bug_summaries.filter(bugzilla_id=bug_id))
+        for bug in bugs:
+            bug_id = bug["bug__bugzilla_id"]
+            all_tests = self.get_tests_from_manifests()
+            test_name = self.get_test_name(all_tests, bug["bug__summary"])
+            manifest = None
             bug_testrun_matrix = []
-            if manifest:
-                testrun_matrix = (
-                    fetch.fetch_testrun_matrix()
-                    if self.testrun_matrix is None
-                    else self.testrun_matrix
-                )
-                self.testrun_matrix = testrun_matrix
-                bug_testrun_matrix = testrun_matrix[manifest]
+            run_count = 0
+            if test_name:
+                manifest = all_tests[test_name][0]
+                if manifest:
+                    tasks_count = self.get_task_labels_and_count(manifest, start_day, end_day)
+                    job_name = bug["job__signature__job_type_name"]
+                    for task_name, count in tasks_count.items():
+                        if task_name in [job_name, job_name.rsplit("-", 1)[0]]:
+                            run_count = count
+                            break
+                    testrun_matrix = (
+                        fetch.fetch_testrun_matrix()
+                        if self.testrun_matrix is None
+                        else self.testrun_matrix
+                    )
+                    self.testrun_matrix = testrun_matrix
+                    bug_testrun_matrix = testrun_matrix[manifest]
             bug_run_info = self.get_bug_run_info(bug)
             all_variants = bug_run_info.variants
             if bug_testrun_matrix and bug_run_info.os_name in bug_testrun_matrix:
@@ -447,7 +483,9 @@ def build_bug_map(self, bugs, option_collection_map):
                 bug_infos.total = 1
                 bug_infos.test_variants |= all_variants
                 bug_infos.per_repositories[repo] = 1
-                bug_infos.data_table[platform_and_build] = {test_variant: 1}
+                bug_infos.data_table[platform_and_build] = {
+                    test_variant: {"count": 1, "runs": run_count},
+                }
                 bug_map[bug_id] = bug_infos
             else:
                 bug_infos = bug_map[bug_id]
@@ -459,9 +497,10 @@ def build_bug_map(self, bugs, option_collection_map):
                 data_table = bug_infos.data_table
                 platform_and_build_data = data_table.get(platform_and_build, {})
                 data_table[platform_and_build] = platform_and_build_data
-                data_table[platform_and_build][test_variant] = (
-                    platform_and_build_data.get(test_variant, 0) + 1
-                )
+                if test_variant in data_table[platform_and_build]:
+                    data_table[platform_and_build][test_variant]["count"] += 1
+                else:
+                    data_table[platform_and_build][test_variant] = {"count": 1, "runs": run_count}
         return bug_map
 
     def get_alt_date_bug_totals(self, startday, endday, bug_ids):
@@ -530,8 +569,7 @@ def fix_wpt_name(self, test_name):
         test_name = test_name.split("?")[0]
         return test_name
 
-    def get_test_manifest(self, bug_summaries):
-        all_tests = self.get_tests_from_manifests()
+    def get_test_name(self, all_tests, summary):
         tv_strings = [
             " TV ",
             " TV-nofis ",
@@ -554,49 +592,45 @@ def get_test_manifest(self, bug_summaries):
             "svg",
             "mp4",
         ]
-        for bug_summary_dict in bug_summaries:
-            summary = bug_summary_dict["summary"]
-            # ensure format we want
-            if "| single tracking bug" not in summary:
-                continue
-            # ignore chrome://, file://, resource://, http[s]://, etc.
-            if "://" in summary:
-                continue
-            # ignore test-verify as these run only on demand when the specific test is modified
-            if any(k for k in tv_strings if k.lower() in summary.lower()):
-                continue
-            # now parse and try to find file in list of tests
-            if any(k for k in test_file_extensions if f"{k} | single" in summary):
-                if " (finished)" in summary:
-                    summary = summary.replace(" (finished)", "")
-                # get <test_name> from: "TEST-UNEXPECTED-FAIL | <test_name> | single tracking bug"
-                # TODO: fix reftest
-                test_name = summary.split("|")[-2].strip()
-                if " == " in test_name or " != " in test_name:
-                    test_name = test_name.split(" ")[0]
+        # ensure format we want
+        if "| single tracking bug" not in summary:
+            return None
+        # ignore chrome://, file://, resource://, http[s]://, etc.
+        if "://" in summary:
+            return None
+        # ignore test-verify as these run only on demand when the specific test is modified
+        if any(k for k in tv_strings if k.lower() in summary.lower()):
+            return None
+        # now parse and try to find file in list of tests
+        if any(k for k in test_file_extensions if f"{k} | single" in summary):
+            if " (finished)" in summary:
+                summary = summary.replace(" (finished)", "")
+            # get <test_name> from: "TEST-UNEXPECTED-FAIL | <test_name> | single tracking bug"
+            # TODO: fix reftest
+            test_name = summary.split("|")[-2].strip()
+            if " == " in test_name or " != " in test_name:
+                test_name = test_name.split(" ")[0]
+            else:
+                test_name = test_name.split(" ")[-1]
+            # comm/ is thunderbird, not in mozilla-central repo
+            # "-ref" is related to a reftest reference file, not what we want to target
+            # if no <path>/<filename>, then we won't be able to find in repo, ignore
+            if test_name.startswith("comm/") or "-ref" in test_name or "/" not in test_name:
+                return None
+            # handle known WPT mapping
+            if test_name.startswith("/") or test_name.startswith("mozilla/tests"):
+                test_name = self.fix_wpt_name(test_name)
+            if test_name not in all_tests:
+                # try reftest:
+                if f"layout/reftests/{test_name}" in all_tests:
+                    test_name = f"layout/reftests/{test_name}"
                 else:
-                    test_name = test_name.split(" ")[-1]
-                # comm/ is thunderbird, not in mozilla-central repo
-                # "-ref" is related to a reftest reference file, not what we want to target
-                # if no <path>/<filename>, then we won't be able to find in repo, ignore
-                if test_name.startswith("comm/") or "-ref" in test_name or "/" not in test_name:
-                    continue
-                # handle known WPT mapping
-                if test_name.startswith("/") or test_name.startswith("mozilla/tests"):
-                    test_name = self.fix_wpt_name(test_name)
-                if test_name not in all_tests:
-                    # try reftest:
-                    if f"layout/reftests/{test_name}" in all_tests:
-                        test_name = f"layout/reftests/{test_name}"
-                    else:
-                        # unknown test
-                        # TODO: we get here for a few reasons:
-                        # 1) test has moved in the source tree
-                        # 2) test has typo in summary
-                        # 3) test has been deleted from the source tree
-                        # 4) sometimes test was deleted but is valid on beta
-                        continue
-                # matching test- we can access manifest
-                manifest = all_tests[test_name]
-                return manifest[0]
+                    # unknown test
+                    # TODO: we get here for a few reasons:
+                    # 1) test has moved in the source tree
+                    # 2) test has typo in summary
+                    # 3) test has been deleted from the source tree
+                    # 4) sometimes test was deleted but is valid on beta
+                    return None
+            return test_name
         return None

treeherder/intermittents_commenter/fetch.py

@@ -22,3 +22,9 @@ def fetch_testrun_matrix():
     testrun_matrix_url = f"{firefoxci_artefact_api_url}/test-info-testrun-matrix.json"
     response = requests.get(testrun_matrix_url, headers={"User-agent": "mach-test-info/1.0"})
     return response.json()
+
+
+def fetch_summary_groups(start_day, end_day):
+    testrun_info_url = f"{firefoxci_artefact_api_url}/test-run-info.json"
+    response = requests.get(testrun_info_url, headers={"User-agent": "mach-test-info/1.0"})
+    return response.json()