Add post log parser to look for repeated test runs and annotate as intermittent.

Author: jmaher

tests/log_parser/test_tasks.py

@@ -228,3 +228,23 @@ def test_bug_suggestion_line_no_stb(
             ),
         }
     ]
+
+
+@pytest.mark.django_db
+def test_confirm_failure_intermittent(
+    failure_classifications, jobs_with_local_log, sample_push, test_repository
+):
+    """
+    TODO: write tests for testing intermittents.py handling in the parser.
+     * test retrigger with 1 similar failure, but both jobs have different failures - both orange
+     * test 5 jobs, 2 fail, 2 fail for other reasons, 1 pass - all green
+     * test infra/tooling error + 1x green - both green
+     * test failure w/3x tests + 3x confirm-failure tasks green - all green
+     * test failure w/3x tests + 2x confirm-failure tasks green - original task still orange
+    """
+    store_push_data(test_repository, sample_push)
+    for job in jobs_with_local_log:
+        job["job"]["result"] = "testfailed"
+        job["revision"] = sample_push[0]["revision"]
+    store_job_data(test_repository, jobs_with_local_log)
+    assert 1 == 0

treeherder/log_parser/intermittents.py

@@ -0,0 +1,55 @@
+from treeherder.model.models import Group, GroupStatus, Job
+
+
+def check_and_mark_intermittent(job_id):
+    current_job = Job.objects.get(id=job_id)
+
+    if current_job.job_type.name.endswith("-cf"):
+        jtname = [current_job.job_type.name, current_job.job_type.name.strip("-cf")]
+    else:
+        jtname = [current_job.job_type.name, f"{current_job.job_type.name}-cf"]
+
+    all_groups = Group.objects.filter(
+        job_logs__job__push__id=current_job.push.id,
+        job_logs__job__job_type__name__in=jtname,
+        group_result__status__in=[GroupStatus.OK, GroupStatus.ERROR],
+    ).values(
+        "name",
+        "job_logs__job__id",
+        "group_result__status",
+    )
+
+    groups = {}
+    jobs = {}
+    for item in all_groups:
+        if item["name"] not in groups:
+            groups[item["name"]] = {}
+        if item["job_logs__job__id"] not in groups[item["name"]]:
+            groups[item["name"]][item["job_logs__job__id"]] = item["group_result__status"]
+
+        if item["job_logs__job__id"] not in jobs:
+            jobs[item["job_logs__job__id"]] = {}
+        if item["name"] not in jobs[item["job_logs__job__id"]]:
+            jobs[item["job_logs__job__id"]][item["name"]] = item["group_result__status"]
+
+    if len(jobs.keys()) <= 1:
+        # zero jobs == no groups reported (i.e. marionette)
+        # 1 job == no additional data
+        return
+
+    for job in jobs.keys():
+        # for each similar task.label, ensure all groups have >=50% pass rate, if so flag failing
+        # job as intermittent.  for non test failures, ensure all groups are green
+        all_green = True
+        failed_groups = [g for g in jobs[job] if int(jobs[job][g]) == GroupStatus.ERROR]
+        for group in failed_groups:
+            all_status = [groups[group][j] for j in groups[group]]
+            pass_rate = len([s for s in all_status if s == GroupStatus.OK]) / len(all_status)
+            if pass_rate < 0.5:
+                all_green = False
+                break
+
+        target_job = Job.objects.filter(id=job)
+
+        if all_green and target_job[0].result != "success":
+            target_job.update(failure_classification_id=4)

treeherder/log_parser/tasks.py

@@ -13,7 +13,7 @@
 from treeherder.model.models import Job, JobLog
 from treeherder.workers.task import retryable_task
 
-from . import failureline
+from . import failureline, intermittents
 
 logger = logging.getLogger(__name__)
 
@@ -81,6 +81,8 @@ def store_failure_lines(job_log):
     errorsummary file."""
     logger.info("Running store_failure_lines for job %s", job_log.job.id)
     failureline.store_failure_lines(job_log)
+    logger.info("Running check_and_mark_intermittent for job %s", job_log.job.id)
+    intermittents.check_and_mark_intermittent(job_log.job.id)
 
 
 def post_log_artifacts(job_log):