Adds relative inferences for DAU statistic (#2047)

Author: danielkberry

jetstream/statistics.py

@@ -510,18 +510,35 @@ def transform(
 
         results = []
         for branch in experiment.branches:
-            branch_data = [
+            # for absolute differences we report the absolute difference in per-user
+            # sum(DAU) scaled by total user enrollment. The interpretation here is the
+            # DAU gain achieved if we deploy this branch to the same population as the
+            # experiment
+            branch_data_abs = [
                 x
                 for x in bootstrap_results.__root__
                 if x.branch == branch.slug and x.comparison == "difference"
             ]
-            for d in branch_data:
+            for d in branch_data_abs:
                 d.point = d.point * num_enrolled_clients
                 d.upper = d.upper * num_enrolled_clients
                 d.lower = d.lower * num_enrolled_clients
                 d.statistic = "per_client_dau_impact"
 
                 results.append(d)
+
+            # for relative differences we simply report the relative difference in
+            # per-user sum(DAU)
+            branch_data_rel = [
+                x
+                for x in bootstrap_results.__root__
+                if x.branch == branch.slug and x.comparison == "relative_uplift"
+            ]
+            for d in branch_data_rel:
+                d.statistic = "per_client_dau_impact"
+
+                results.append(d)
+
         return StatisticResultCollection.parse_obj(results)
 
 

jetstream/tests/test_statistics.py

@@ -205,13 +205,23 @@ def test_per_client_dau_impact(self):
             test_data, "value", "control", experiment, AnalysisBasis.ENROLLMENTS, "all"
         ).__root__
 
-        difference = [r for r in result if r.comparison == "difference"][0]
+        abs_difference = [r for r in result if r.comparison == "difference"][0]
         # analytically, we should see a point estimate of 10, with 95% CI of (7.155,12.844)
         # at these small sample sizes, mozanalysis's bootstrap can be quite variable
         # so use a large tolerance
-        assert np.abs(difference.point - 10) < 1.0
-        assert np.abs(difference.lower - 7.155) < 1.0
-        assert np.abs(difference.upper - 12.844) < 1.0
+        assert np.abs(abs_difference.point - 10) < 1.0
+        assert np.abs(abs_difference.lower - 7.155) < 1.0
+        assert np.abs(abs_difference.upper - 12.844) < 1.0
+
+        rel_difference = [r for r in result if r.comparison == "relative_uplift"][0]
+
+        # analytically, we should see a point estimate of 222%, with 95% CI of (108%,398%)
+        # at these small sample sizes, mozanalysis's bootstrap can be quite variable
+        # so use a large tolerance
+
+        assert np.isclose(rel_difference.point, 2.22, atol=0.25)
+        assert np.isclose(rel_difference.lower, 1.08, atol=0.25)
+        assert np.isclose(rel_difference.upper, 3.98, atol=0.25)
 
     def test_binomial(self):
         stat = Binomial()