Allow endpoint to change the statistics calculation based on the replicates query param (#8678)

Author: beatrice-acasandrei

tests/webapp/api/test_perfcompare_api.py

@@ -4,7 +4,7 @@
 from django.urls import reverse
 
 from treeherder.model.models import Job
-from treeherder.perf.models import PerformanceDatum
+from treeherder.perf.models import PerformanceDatum, PerformanceDatumReplicate
 from treeherder.webapp.api import perfcompare_utils
 
 NOW = datetime.datetime.now()
@@ -224,6 +224,8 @@ def test_perfcompare_results_with_only_one_run_and_diff_repo(
 
     base_perf_data_values = [32.4]
     new_perf_data_values = [40.2]
+    base_perf_data_replicates = [20]
+    new_perf_data_replicates = [15]
 
     job = perf_jobs[0]
     job.push = test_perfcomp_push
@@ -238,6 +240,7 @@ def test_perfcompare_results_with_only_one_run_and_diff_repo(
     )
     perf_datum.push.time = job.push.time
     perf_datum.push.save()
+    PerformanceDatumReplicate.objects.create(performance_datum=perf_datum, value=20)
 
     new_sig = create_signature(
         signature_hash=(20 * "t2"),
@@ -264,6 +267,7 @@ def test_perfcompare_results_with_only_one_run_and_diff_repo(
     )
     perf_datum.push.time = job.push.time
     perf_datum.push.save()
+    PerformanceDatumReplicate.objects.create(performance_datum=perf_datum, value=15)
 
     response = get_expected(
         base_sig,
@@ -293,8 +297,8 @@ def test_perfcompare_results_with_only_one_run_and_diff_repo(
             "new_retriggerable_job_ids": [10],
             "base_runs": base_perf_data_values,
             "new_runs": new_perf_data_values,
-            "base_runs_replicates": [],
-            "new_runs_replicates": [],
+            "base_runs_replicates": [20],
+            "new_runs_replicates": [15],
             "base_avg_value": round(response["base_avg_value"], 2),
             "new_avg_value": round(response["new_avg_value"], 2),
             "base_median_value": round(response["base_median_value"], 2),
@@ -348,6 +352,38 @@ def test_perfcompare_results_with_only_one_run_and_diff_repo(
     assert response.status_code == 200
     assert expected[0] == response.json()[0]
 
+    # test the same comparison by setting the replicates values to true
+    query_params = (
+        "?base_repository={}&new_repository={}&base_revision={}&new_revision={}&framework={"
+        "}&no_subtests=true&replicates=true".format(
+            try_repository.name,
+            test_repository.name,
+            test_perfcomp_push.revision,
+            test_perfcomp_push_2.revision,
+            test_perf_signature.framework_id,
+        )
+    )
+
+    expected_response = get_expected(
+        base_sig,
+        new_sig,
+        extra_options,
+        test_option_collection,
+        # get expected_response using the replicates values
+        new_perf_data_replicates,
+        base_perf_data_replicates,
+    )
+    response = client.get(reverse("perfcompare-results") + query_params)
+
+    # test to see how the response changed based on the replicates values
+    assert response.status_code == 200
+    response.json()[0]["base_avg_value"] = round(expected_response["base_avg_value"], 2)
+    response.json()[0]["base_stddev"] = round(expected_response["base_stddev"], 2)
+    response.json()[0]["base_median_value"] = round(expected_response["base_median_value"], 2)
+    response.json()[0]["new_avg_value"] = round(expected_response["new_avg_value"], 2)
+    response.json()[0]["new_stddev"] = round(expected_response["new_stddev"], 2)
+    response.json()[0]["new_median_value"] = round(expected_response["new_median_value"], 2)
+
 
 def test_perfcompare_results_without_base_signature(
     client,

treeherder/webapp/api/performance_data.py

@@ -919,6 +919,7 @@ def list(self, request):
         no_subtests = query_params.validated_data["no_subtests"]
         base_parent_signature = query_params.validated_data["base_parent_signature"]
         new_parent_signature = query_params.validated_data["new_parent_signature"]
+        replicates = query_params.validated_data["replicates"]
 
         try:
             new_push = models.Push.objects.get(revision=new_rev, repository__name=new_repo_name)
@@ -979,11 +980,17 @@ def list(self, request):
             new_grouped_replicates,
         ) = self._get_grouped_perf_data(new_perf_data)
 
+        statistics_base_grouped_data = base_grouped_values
+        statistics_new_grouped_data = new_grouped_values
+        if replicates:
+            statistics_base_grouped_data = base_grouped_replicates
+            statistics_new_grouped_data = new_grouped_replicates
+
         base_signatures_map, base_header_names, base_platforms = self._get_signatures_map(
-            base_signatures, base_grouped_values, option_collection_map
+            base_signatures, statistics_base_grouped_data, option_collection_map
         )
         new_signatures_map, new_header_names, new_platforms = self._get_signatures_map(
-            new_signatures, new_grouped_values, option_collection_map
+            new_signatures, statistics_new_grouped_data, option_collection_map
         )
 
         header_names = list(set(base_header_names + new_header_names))
@@ -1020,22 +1027,24 @@ def list(self, request):
                 new_perf_data_values = new_grouped_values.get(new_sig_id, [])
                 base_perf_data_replicates = base_grouped_replicates.get(base_sig_id, [])
                 new_perf_data_replicates = new_grouped_replicates.get(new_sig_id, [])
-                base_runs_count = len(base_perf_data_values)
-                new_runs_count = len(new_perf_data_values)
+                statistics_base_perf_data = statistics_base_grouped_data.get(base_sig_id, [])
+                statistics_new_perf_data = statistics_new_grouped_data.get(new_sig_id, [])
+                base_runs_count = len(statistics_base_perf_data)
+                new_runs_count = len(statistics_new_perf_data)
                 is_complete = base_runs_count and new_runs_count
                 no_results_to_show = not base_runs_count and not new_runs_count
                 if no_results_to_show:
                     continue
-                base_avg_value = perfcompare_utils.get_avg(base_perf_data_values, header)
-                base_stddev = perfcompare_utils.get_stddev(base_perf_data_values, header)
-                base_median_value = perfcompare_utils.get_median(base_perf_data_values)
-                new_avg_value = perfcompare_utils.get_avg(new_perf_data_values, header)
-                new_stddev = perfcompare_utils.get_stddev(new_perf_data_values, header)
-                new_median_value = perfcompare_utils.get_median(new_perf_data_values)
+                base_avg_value = perfcompare_utils.get_avg(statistics_base_perf_data, header)
+                base_stddev = perfcompare_utils.get_stddev(statistics_base_perf_data, header)
+                base_median_value = perfcompare_utils.get_median(statistics_base_perf_data)
+                new_avg_value = perfcompare_utils.get_avg(statistics_new_perf_data, header)
+                new_stddev = perfcompare_utils.get_stddev(statistics_new_perf_data, header)
+                new_median_value = perfcompare_utils.get_median(statistics_new_perf_data)
                 base_stddev_pct = perfcompare_utils.get_stddev_pct(base_avg_value, base_stddev)
                 new_stddev_pct = perfcompare_utils.get_stddev_pct(new_avg_value, new_stddev)
                 confidence = perfcompare_utils.get_abs_ttest_value(
-                    base_perf_data_values, new_perf_data_values
+                    statistics_base_perf_data, statistics_new_perf_data
                 )
                 confidence_text = perfcompare_utils.get_confidence_text(confidence)
                 delta_value = perfcompare_utils.get_delta_value(new_avg_value, base_avg_value)

treeherder/webapp/api/performance_serializers.py

@@ -513,6 +513,7 @@ class PerfCompareResultsQueryParamsSerializer(serializers.Serializer):
     no_subtests = serializers.BooleanField(required=False)
     base_parent_signature = serializers.CharField(required=False, allow_null=True, default=None)
     new_parent_signature = serializers.CharField(required=False, allow_null=True, default=None)
+    replicates = serializers.BooleanField(required=False)
 
     def validate(self, data):
         if data["base_revision"] is None and data["interval"] is None: