Merge pull request #2010 from rust-lang/compare-backends

Add UI for comparing backend results

Author: Kobzol

site/frontend/src/pages/compare/compile/benchmarks.vue

@@ -16,6 +16,7 @@ export interface BenchmarkProps {
   benchmarkMap: CompileBenchmarkMap;
   filter: CompileBenchmarkFilter;
   stat: string;
+  showBackend: boolean;
 }
 
 const props = defineProps<BenchmarkProps>();
@@ -77,6 +78,7 @@ const secondaryHasNonRelevant = computed(
       :commit-b="data.b"
       :stat="stat"
       :benchmark-map="benchmarkMap"
+      :show-backend="showBackend"
     >
       <template #header>
         <Section title="Primary" link="secondary" :linkUp="false"></Section>
@@ -92,6 +94,7 @@ const secondaryHasNonRelevant = computed(
       :commit-b="data.b"
       :stat="stat"
       :benchmark-map="benchmarkMap"
+      :show-backend="showBackend"
     >
       <template #header>
         <Section title="Secondary" link="primary" :linkUp="true"></Section>

site/frontend/src/pages/compare/compile/common.ts

@@ -27,6 +27,7 @@ export type CompileBenchmarkFilter = {
     binary: boolean;
     library: boolean;
   };
+  selfCompareBackend: boolean;
 } & BenchmarkFilter;
 
 export const defaultCompileFilter: CompileBenchmarkFilter = {
@@ -57,6 +58,7 @@ export const defaultCompileFilter: CompileBenchmarkFilter = {
     binary: true,
     library: true,
   },
+  selfCompareBackend: false,
 };
 
 export type Profile = "check" | "debug" | "opt" | "doc";
@@ -206,5 +208,57 @@ export function createCompileBenchmarkMap(
 }
 
 export function testCaseKey(testCase: CompileTestCase): string {
-  return `${testCase.benchmark};${testCase.profile};${testCase.scenario};${testCase.category}`;
+  return `${testCase.benchmark};${testCase.profile};${testCase.scenario};${testCase.backend};${testCase.category}`;
+}
+
+// Transform compile comparisons to compare LLVM vs Cranelift, instead of
+// before/after. Assumes that the data comes from the same commit.
+export function transformDataForBackendComparison(
+  comparisons: CompileBenchmarkComparison[]
+): CompileBenchmarkComparison[] {
+  const benchmarkMap: Map<
+    string,
+    {
+      llvm: number | null;
+      cranelift: number | null;
+      benchmark: string;
+      profile: Profile;
+      scenario: string;
+    }
+  > = new Map();
+  for (const comparison of comparisons) {
+    const key = `${comparison.benchmark};${comparison.profile};${comparison.scenario}`;
+    if (!benchmarkMap.has(key)) {
+      benchmarkMap.set(key, {
+        llvm: null,
+        cranelift: null,
+        benchmark: comparison.benchmark,
+        profile: comparison.profile,
+        scenario: comparison.scenario,
+      });
+    }
+    const record = benchmarkMap.get(key);
+    if (comparison.backend === "llvm") {
+      record.llvm = comparison.comparison.statistics[0];
+    } else if (comparison.backend === "cranelift") {
+      record.cranelift = comparison.comparison.statistics[0];
+    }
+  }
+
+  return Array.from(benchmarkMap, ([_, entry]) => {
+    const comparison: CompileBenchmarkComparison = {
+      benchmark: entry.benchmark,
+      profile: entry.profile,
+      scenario: entry.scenario,
+      // Treat LLVM as the baseline
+      backend: "llvm",
+      comparison: {
+        statistics: [entry.llvm, entry.cranelift],
+        is_relevant: true,
+        significance_factor: 1.0,
+        significance_threshold: 1.0,
+      },
+    };
+    return comparison;
+  });
 }

site/frontend/src/pages/compare/compile/compile-page.vue

@@ -14,6 +14,7 @@ import {
   computeCompileComparisonsWithNonRelevant,
   createCompileBenchmarkMap,
   defaultCompileFilter,
+  transformDataForBackendComparison,
 } from "./common";
 import {BenchmarkInfo} from "../../../api";
 import {importantCompileMetrics} from "../metrics";
@@ -101,6 +102,11 @@ function loadFilterFromUrl(
         defaultFilter.artifact.library
       ),
     },
+    selfCompareBackend: getBoolOrDefault(
+      urlParams,
+      "selfCompareBackend",
+      defaultFilter.selfCompareBackend
+    ),
   };
 }
 
@@ -172,6 +178,11 @@ function storeFilterToUrl(
     filter.artifact.library,
     defaultFilter.artifact.library
   );
+  storeOrReset(
+    "selfCompareBackend",
+    filter.selfCompareBackend,
+    defaultFilter.selfCompareBackend
+  );
 
   changeUrl(urlParams);
 }
@@ -182,6 +193,13 @@ function updateFilter(newFilter: CompileBenchmarkFilter) {
   refreshQuickLinks();
 }
 
+// We pass the event target here, because Parcel cannot handle the `as`
+// cast directly in the template.
+function updateSelfCompareBackend(target: EventTarget) {
+  const element = target as HTMLInputElement;
+  updateFilter({...filter.value, selfCompareBackend: element.checked});
+}
+
 /**
  * When the filter changes, the URL is updated.
  * After that happens, we want to re-render the quick links component, because
@@ -197,15 +215,36 @@ const urlParams = getUrlParams();
 const quickLinksKey = ref(0);
 const filter = ref(loadFilterFromUrl(urlParams, defaultCompileFilter));
 
+// Should we use the backend as the source of before/after data?
+const selfCompareBackend = computed(() => {
+  return canCompareBackends.value && filter.value.selfCompareBackend;
+});
+const canCompareBackends = computed(() => {
+  const hasMultipleBackends =
+    new Set(props.data.compile_comparisons.map((c) => c.backend)).size > 1;
+  // Are we currently comparing the same commit in the before/after toolchains?
+  const comparesSameCommit = props.data.a.commit === props.data.b.commit;
+  return hasMultipleBackends && comparesSameCommit;
+});
+
 function exportData() {
   exportToMarkdown(comparisons.value, filter.value.showRawData);
 }
 
 const benchmarkMap = createCompileBenchmarkMap(props.data);
+
+const compileComparisons = computed(() => {
+  // If requested, artificially restructure the data to create a comparison between backends
+  if (selfCompareBackend.value) {
+    return transformDataForBackendComparison(props.data.compile_comparisons);
+  } else {
+    return props.data.compile_comparisons;
+  }
+});
 const allComparisons = computed(() =>
   computeCompileComparisonsWithNonRelevant(
     filter.value,
-    props.data.compile_comparisons,
+    compileComparisons.value,
     benchmarkMap
   )
 );
@@ -222,6 +261,17 @@ const filteredSummary = computed(() => computeSummary(comparisons.value));
     :selected-metric="selector.stat"
     :metrics="benchmarkInfo.compile_metrics"
   />
+  <div
+    v-if="canCompareBackends"
+    :title="`Compare codegen backends for commit '${props.data.a.commit}'`"
+  >
+    Compare codegen backends for this commit:
+    <input
+      type="checkbox"
+      :checked="selfCompareBackend"
+      @change="(e) => updateSelfCompareBackend(e.target)"
+    />
+  </div>
   <Filters
     :defaultFilter="defaultCompileFilter"
     :initialFilter="filter"
@@ -230,12 +280,23 @@ const filteredSummary = computed(() => computeSummary(comparisons.value));
   />
   <OverallSummary :summary="filteredSummary" />
   <Aggregations :cases="comparisons" />
+  <div class="warning" v-if="selfCompareBackend">
+    Note: comparing results of the baseline LLVM backend to the Cranelift
+    backend.
+  </div>
   <Benchmarks
     :data="data"
     :test-cases="comparisons"
     :all-test-cases="allComparisons"
     :filter="filter"
     :stat="selector.stat"
     :benchmark-map="benchmarkMap"
+    :show-backend="!selfCompareBackend"
   ></Benchmarks>
 </template>
+<style lang="scss" scoped>
+.warning {
+  color: red;
+  font-weight: bold;
+}
+</style>

site/frontend/src/pages/compare/compile/table/comparisons-table.vue

@@ -18,6 +18,7 @@ const props = defineProps<{
   commitA: ArtifactDescription;
   commitB: ArtifactDescription;
   stat: string;
+  showBackend: boolean;
 }>();
 
 function prettifyRawNumber(number: number): string {
@@ -56,7 +57,7 @@ const unit = computed(() => {
           <th>Benchmark</th>
           <th>Profile</th>
           <th>Scenario</th>
-          <th>Backend</th>
+          <th v-if="showBackend">Backend</th>
           <th>% Change</th>
           <th class="narrow">
             Significance Threshold
@@ -95,7 +96,7 @@ const unit = computed(() => {
                 {{ comparison.testCase.profile }}
               </td>
               <td>{{ comparison.testCase.scenario }}</td>
-              <td>{{ comparison.testCase.backend }}</td>
+              <td v-if="showBackend">{{ comparison.testCase.backend }}</td>
               <td>
                 <div class="numeric-aligned">
                   <span v-bind:class="percentClass(comparison.percent)">