Add post log parser to look for repeated test runs and annotate as intermittent.

Author: jmaher

treeherder/log_parser/intermittents.py

@@ -0,0 +1,54 @@
+from treeherder.model.models import Group, GroupStatus, Job
+
+def check_and_mark_intermittent(job_id):
+    current_job = Job.objects.get(id=job_id)
+
+    if current_job.job_type.name.endswith("-cf"):
+        jtname = [current_job.job_type.name, current_job.job_type.name.strip("-cf")]
+    else:
+        jtname = [current_job.job_type.name, f"{current_job.job_type.name}-cf"]
+
+    all_groups = Group.objects.filter(
+        job_logs__job__push__id=current_job.push.id,
+        job_logs__job__job_type__name__in=jtname,
+        group_result__status__in=[GroupStatus.OK, GroupStatus.ERROR],
+    ).values(
+        "name",
+        "job_logs__job__id",
+        "group_result__status",
+    )
+
+    groups = {}
+    jobs = {}
+    for item in all_groups:
+        if item["name"] not in groups:
+            groups[item["name"]] = {}
+        if item["job_logs__job__id"] not in groups[item["name"]]:
+            groups[item["name"]][item["job_logs__job__id"]] = item["group_result__status"]
+
+        if item["job_logs__job__id"] not in jobs:
+            jobs[item["job_logs__job__id"]] = {}
+        if item["name"] not in jobs[item["job_logs__job__id"]]:
+            jobs[item["job_logs__job__id"]][item["name"]] = item["group_result__status"]
+
+    if len(jobs.keys()) <= 1:
+        # zero jobs == no groups reported (i.e. marionette)
+        # 1 job == no additional data
+        return
+
+    for job in jobs.keys():
+        # for each similar task.label, ensure all groups have >=50% pass rate, if so flag failing
+        # job as intermittent.  for non test failures, ensure all groups are green
+        all_green = True
+        failed_groups = [g for g in jobs[job] if int(jobs[job][g]) == GroupStatus.ERROR]
+        for group in failed_groups:
+            all_status = [groups[group][j] for j in groups[group]]
+            pass_rate = len([s for s in all_status if s == GroupStatus.OK]) / len(all_status)
+            if pass_rate < 0.5:
+                all_green = False
+                break
+
+        target_job = Job.objects.filter(id=job)
+
+        if all_green and target_job[0].result != "success":
+            target_job.update(failure_classification_id=4)

treeherder/log_parser/tasks.py

@@ -14,6 +14,7 @@
 from treeherder.workers.task import retryable_task
 
 from . import failureline
+from . import intermittents
 
 logger = logging.getLogger(__name__)
 
@@ -81,7 +82,8 @@ def store_failure_lines(job_log):
     errorsummary file."""
     logger.info("Running store_failure_lines for job %s", job_log.job.id)
     failureline.store_failure_lines(job_log)
-
+    logger.info("Running check_and_mark_intermittent for job %s", job_log.job.id)
+    intermittents.check_and_mark_intermittent(job_log.job.id)
 
 def post_log_artifacts(job_log):
     """Post a list of artifacts to a job."""