Merge upstream/HEAD@main

Author: gjkeller

.buildkite/common.py

@@ -6,28 +6,41 @@
 """
 
 import argparse
+import ast
 import json
 import os
 import random
 import string
 import subprocess
 from pathlib import Path
 
-DEFAULT_INSTANCES = {
-    "c5n.metal": "x86_64",  # Intel Skylake
-    "m5n.metal": "x86_64",  # Intel Cascade Lake
-    "m6i.metal": "x86_64",  # Intel Icelake
-    "m6a.metal": "x86_64",  # AMD Milan
-    "m6g.metal": "aarch64",  # Graviton2
-    "m7g.metal": "aarch64",  # Graviton3
-}
+# fmt: off
+DEFAULT_INSTANCES = [
+    "c5n.metal",      # Intel Skylake
+    "m5n.metal",      # Intel Cascade Lake
+    "m6i.metal",      # Intel Icelake
+    "m7i.metal-24xl", # Intel Sapphire Rapids
+    "m7i.metal-48xl", # Intel Sapphire Rapids
+    "m6a.metal",      # AMD Milan
+    "m7a.metal-48xl", # AMD Genoa
+    "m6g.metal",      # Graviton2
+    "m7g.metal",      # Graviton3
+    "m8g.metal-24xl", # Graviton4 1 socket
+    "m8g.metal-48xl", # Graviton4 2 sockets
+]
+# fmt: on
 
 DEFAULT_PLATFORMS = [
     ("al2", "linux_5.10"),
     ("al2023", "linux_6.1"),
 ]
 
 
+def get_arch_for_instance(instance):
+    """Return instance architecture"""
+    return "x86_64" if instance[2] != "g" else "aarch64"
+
+
 def overlay_dict(base: dict, update: dict):
     """Overlay a dict over a base one"""
     base = base.copy()
@@ -133,7 +146,8 @@ def __call__(self, parser, namespace, value, option_string=None):
         res = getattr(namespace, self.dest, {})
         key_str, val = value.split("=", maxsplit=1)
         keys = key_str.split("/")
-        update = {keys[-1]: val}
+        # Interpret it as a literal iff it starts like one
+        update = {keys[-1]: ast.literal_eval(val) if val[0] in "[{'" else val}
         for key in list(reversed(keys))[1:]:
             update = {key: update}
         res = overlay_dict(res, update)
@@ -145,7 +159,7 @@ def __call__(self, parser, namespace, value, option_string=None):
     "--instances",
     required=False,
     nargs="+",
-    default=DEFAULT_INSTANCES.keys(),
+    default=DEFAULT_INSTANCES,
 )
 COMMON_PARSER.add_argument(
     "--platforms",
@@ -171,6 +185,12 @@ def __call__(self, parser, namespace, value, option_string=None):
     default=None,
     type=str,
 )
+COMMON_PARSER.add_argument(
+    "--no-kani",
+    help="Don't add kani step",
+    action="store_true",
+    default=False,
+)
 
 
 def random_str(k: int):
@@ -180,16 +200,7 @@ def random_str(k: int):
 
 def ab_revision_build(revision):
     """Generate steps for building an A/B-test revision"""
-    # Copied from framework/ab_test. Double dollar signs needed for Buildkite (otherwise it will try to interpolate itself)
-    return [
-        f"commitish={revision}",
-        f"if ! git cat-file -t $$commitish; then commitish=origin/{revision}; fi",
-        "branch_name=tmp-$$commitish",
-        "git branch $$branch_name $$commitish",
-        f"git clone -b $$branch_name . build/{revision}",
-        f"cd build/{revision} && ./tools/devtool -y build --release && cd -",
-        "git branch -D $$branch_name",
-    ]
+    return [f"./tools/devtool -y build --rev {revision} --release"]
 
 
 def shared_build():
@@ -252,7 +263,7 @@ def __init__(self, with_build_step=True, **kwargs):
         self.per_instance = overlay_dict(per_instance, args.step_param)
         self.per_arch = self.per_instance.copy()
         self.per_arch["instances"] = ["m6i.metal", "m7g.metal"]
-        self.per_arch["platforms"] = [("al2", "linux_5.10")]
+        self.per_arch["platforms"] = [("al2023", "linux_6.1")]
         self.binary_dir = args.binary_dir
         # Build sharing
         if with_build_step:
@@ -297,7 +308,7 @@ def _adapt_group(self, group):
             step["command"] = prepend + step["command"]
             if self.shared_build is not None:
                 step["depends_on"] = self.build_key(
-                    DEFAULT_INSTANCES[step["agents"]["instance"]]
+                    get_arch_for_instance(step["agents"]["instance"])
                 )
         return group
 
@@ -332,7 +343,7 @@ def build_group_per_arch(self, label, *args, **kwargs):
         if set_key:
             for step in grp["steps"]:
                 step["key"] = self.build_key(
-                    DEFAULT_INSTANCES[step["agents"]["instance"]]
+                    get_arch_for_instance(step["agents"]["instance"])
                 )
         return self.add_step(grp, depends_on_build=depends_on_build)
 

.buildkite/pipeline_cpu_template.py

@@ -6,7 +6,7 @@
 
 from enum import Enum
 
-from common import DEFAULT_INSTANCES, DEFAULT_PLATFORMS, BKPipeline, group
+from common import DEFAULT_PLATFORMS, BKPipeline, group
 
 
 class BkStep(str, Enum):
@@ -23,24 +23,29 @@ class BkStep(str, Enum):
 cpu_template_test = {
     "rdmsr": {
         BkStep.COMMAND: [
-            "tools/devtool -y test --no-build -- -m nonci -n4 --dist worksteal integration_tests/functional/test_cpu_features.py -k 'test_cpu_rdmsr' "
+            "tools/devtool -y test --no-build -- -m nonci -n4 --dist worksteal integration_tests/functional/test_cpu_features_x86_64.py -k 'test_cpu_rdmsr' "
         ],
         BkStep.LABEL: "📖 rdmsr",
-        "instances": ["c5n.metal", "m5n.metal", "m6a.metal", "m6i.metal"],
-        "platforms": DEFAULT_PLATFORMS,
+        "instances": [
+            "c5n.metal",
+            "m5n.metal",
+            "m6i.metal",
+            "m7i.metal-24xl",
+            "m7i.metal-48xl",
+            "m6a.metal",
+            "m7a.metal-48xl",
+        ],
     },
     "fingerprint": {
         BkStep.COMMAND: [
             "tools/devtool -y test --no-build -- -m no_block_pr integration_tests/functional/test_cpu_template_helper.py -k test_guest_cpu_config_change",
         ],
         BkStep.LABEL: "🖐️ fingerprint",
-        "instances": DEFAULT_INSTANCES.keys(),
-        "platforms": DEFAULT_PLATFORMS,
     },
     "cpuid_wrmsr": {
         "snapshot": {
             BkStep.COMMAND: [
-                "tools/devtool -y test --no-build -- -m nonci -n4 --dist worksteal integration_tests/functional/test_cpu_features.py -k 'test_cpu_wrmsr_snapshot or test_cpu_cpuid_snapshot'",
+                "tools/devtool -y test --no-build -- -m nonci -n4 --dist worksteal integration_tests/functional/test_cpu_features_x86_64.py -k 'test_cpu_wrmsr_snapshot or test_cpu_cpuid_snapshot'",
                 "mkdir -pv tests/snapshot_artifacts_upload/{instance}_{os}_{kv}",
                 "sudo mv tests/snapshot_artifacts/* tests/snapshot_artifacts_upload/{instance}_{os}_{kv}",
             ],
@@ -52,7 +57,7 @@ class BkStep(str, Enum):
             BkStep.COMMAND: [
                 "buildkite-agent artifact download tests/snapshot_artifacts_upload/{instance}_{os}_{kv}/**/* .",
                 "mv tests/snapshot_artifacts_upload/{instance}_{os}_{kv} tests/snapshot_artifacts",
-                "tools/devtool -y test --no-build -- -m nonci -n4 --dist worksteal integration_tests/functional/test_cpu_features.py -k 'test_cpu_wrmsr_restore or test_cpu_cpuid_restore'",
+                "tools/devtool -y test --no-build -- -m nonci -n4 --dist worksteal integration_tests/functional/test_cpu_features_x86_64.py -k 'test_cpu_wrmsr_restore or test_cpu_cpuid_restore'",
             ],
             BkStep.LABEL: "📸 load snapshot artifacts created on {instance} {snapshot_os} {snapshot_kv} to {restore_instance} {restore_os} {restore_kv}",
             BkStep.TIMEOUT: 30,
@@ -62,7 +67,14 @@ class BkStep(str, Enum):
             "c5n.metal": ["m5n.metal", "m6i.metal"],
             "m6i.metal": ["m5n.metal", "c5n.metal"],
         },
-        "instances": ["c5n.metal", "m5n.metal", "m6i.metal", "m6a.metal"],
+        "instances": [
+            "c5n.metal",
+            "m5n.metal",
+            "m6i.metal",
+            "m7i.metal-24xl",
+            "m7i.metal-48xl",
+            "m6a.metal",
+        ],
     },
 }
 

.buildkite/pipeline_cross.py

@@ -18,13 +18,23 @@
     per_instance = pipeline.per_instance.copy()
     per_instance.pop("instances")
     per_instance.pop("platforms")
-    instances_x86_64 = ["c5n.metal", "m5n.metal", "m6i.metal", "m6a.metal"]
+    instances_x86_64 = [
+        "c5n.metal",
+        "m5n.metal",
+        "m6i.metal",
+        "m7i.metal-24xl",
+        "m7i.metal-48xl",
+        "m6a.metal",
+        "m7a.metal-48xl",
+    ]
     instances_aarch64 = ["m7g.metal"]
     commands = [
-        "./tools/devtool -y sh ./tools/create_snapshot_artifact/main.py",
-        "mkdir -pv snapshots/{instance}_{kv}",
-        "sudo chown -Rc $USER: snapshot_artifacts",
-        "mv -v snapshot_artifacts/* snapshots/{instance}_{kv}",
+        "./tools/devtool -y test --no-build -- -m nonci -n4 integration_tests/functional/test_snapshot_phase1.py",
+        # punch holes in mem snapshot tiles and tar them so they are preserved in S3
+        "find test_results/test_snapshot_phase1 -type f -name mem |xargs -P4 -t -n1 fallocate -d",
+        "mv -v test_results/test_snapshot_phase1 snapshot_artifacts",
+        "mkdir -pv snapshots",
+        "tar cSvf snapshots/{instance}_{kv}.tar snapshot_artifacts",
     ]
     pipeline.build_group(
         "📸 create snapshots",
@@ -80,10 +90,10 @@
         k_val = pytest_keyword_for_instance.get(dst_instance, "")
         step = {
             "command": [
-                f"buildkite-agent artifact download snapshots/{src_instance}_{src_kv}/* .",
-                f"mv -v snapshots/{src_instance}_{src_kv} snapshot_artifacts",
+                f"buildkite-agent artifact download snapshots/{src_instance}_{src_kv}.tar .",
+                f"tar xSvf snapshots/{src_instance}_{src_kv}.tar",
                 *pipeline.devtool_test(
-                    pytest_opts=f"-m nonci {k_val} integration_tests/functional/test_snapshot_restore_cross_kernel.py",
+                    pytest_opts=f"-m nonci -n8 --dist worksteal {k_val} integration_tests/functional/test_snapshot_restore_cross_kernel.py",
                 ),
             ],
             "label": f"🎬 {src_instance} {src_kv} ➡️ {dst_instance} {dst_kv}",

.buildkite/pipeline_perf.py

@@ -16,42 +16,48 @@
 # the operating system sometimes uses it for book-keeping tasks. The memory node (-m parameter)
 # has to be the node associated with the NUMA node from which we picked CPUs.
 perf_test = {
-    "virtio-block": {
-        "label": "💿 Virtio Block Performance",
-        "test_path": "integration_tests/performance/test_block_ab.py::test_block_performance",
+    "virtio-block-sync": {
+        "label": "💿 Virtio Sync Block Performance",
+        "tests": "integration_tests/performance/test_block_ab.py::test_block_performance -k 'not Async'",
+        "devtool_opts": "-c 1-10 -m 0",
+    },
+    "virtio-block-async": {
+        "label": "💿 Virtio Async Block Performance",
+        "tests": "integration_tests/performance/test_block_ab.py::test_block_performance -k Async",
         "devtool_opts": "-c 1-10 -m 0",
     },
     "vhost-user-block": {
         "label": "💿 vhost-user Block Performance",
-        "test_path": "integration_tests/performance/test_block_ab.py::test_block_vhost_user_performance",
+        "tests": "integration_tests/performance/test_block_ab.py::test_block_vhost_user_performance",
         "devtool_opts": "-c 1-10 -m 0",
         "ab_opts": "--noise-threshold 0.1",
     },
-    "network-latency": {
-        "label": "📠 Network Latency",
-        "test_path": "integration_tests/performance/test_network_ab.py::test_network_latency",
+    "network": {
+        "label": "📠 Network Latency and Throughput",
+        "tests": "integration_tests/performance/test_network_ab.py",
         "devtool_opts": "-c 1-10 -m 0",
         # Triggers if delta is > 0.01ms (10µs) or default relative threshold (5%)
+        # only relevant for latency test, throughput test will always be magnitudes above this anyway
         "ab_opts": "--absolute-strength 0.010",
     },
-    "network-throughput": {
-        "label": "📠 Network TCP Throughput",
-        "test_path": "integration_tests/performance/test_network_ab.py::test_network_tcp_throughput",
-        "devtool_opts": "-c 1-10 -m 0",
-    },
     "snapshot-latency": {
         "label": "📸 Snapshot Latency",
-        "test_path": "integration_tests/performance/test_snapshot_ab.py",
+        "tests": "integration_tests/performance/test_snapshot_ab.py::test_restore_latency integration_tests/performance/test_snapshot_ab.py::test_post_restore_latency",
+        "devtool_opts": "-c 1-12 -m 0",
+    },
+    "population-latency": {
+        "label": "📸 Memory Population Latency",
+        "tests": "integration_tests/performance/test_snapshot_ab.py::test_population_latency",
         "devtool_opts": "-c 1-12 -m 0",
     },
     "vsock-throughput": {
         "label": "🧦 Vsock Throughput",
-        "test_path": "integration_tests/performance/test_vsock_ab.py",
+        "tests": "integration_tests/performance/test_vsock_ab.py",
         "devtool_opts": "-c 1-10 -m 0",
     },
     "memory-overhead": {
         "label": "💾 Memory Overhead and 👢 Boottime",
-        "test_path": "integration_tests/performance/test_memory_overhead.py integration_tests/performance/test_boottime.py::test_boottime",
+        "tests": "integration_tests/performance/test_memory_overhead.py integration_tests/performance/test_boottime.py::test_boottime",
         "devtool_opts": "-c 1-10 -m 0",
     },
 }
@@ -92,21 +98,21 @@
 tests = [perf_test[test] for test in pipeline.args.test or perf_test.keys()]
 for test in tests:
     devtool_opts = test.pop("devtool_opts")
-    test_path = test.pop("test_path")
+    test_selector = test.pop("tests")
     ab_opts = test.pop("ab_opts", "")
     devtool_opts += " --performance"
-    pytest_opts = ""
+    test_script_opts = ""
     if REVISION_A:
         devtool_opts += " --ab"
-        pytest_opts = f"{ab_opts} run {REVISION_A} {REVISION_B} --test {test_path}"
+        test_script_opts = f'{ab_opts} run build/{REVISION_A}/ build/{REVISION_B} --pytest-opts "{test_selector}"'
     else:
         # Passing `-m ''` below instructs pytest to collect tests regardless of
         # their markers (e.g. it will collect both tests marked as nonci, and
         # tests without any markers).
-        pytest_opts += f" -m '' {test_path}"
+        test_script_opts += f" -m '' {test_selector}"
 
     pipeline.build_group(
-        command=pipeline.devtool_test(devtool_opts, pytest_opts),
+        command=pipeline.devtool_test(devtool_opts, test_script_opts),
         # and the rest can be command arguments
         **test,
     )
@@ -119,10 +125,7 @@
 # }
 # will pin steps running on instances "m6i.metal" with kernel version tagged "linux_6.1"
 # to a new kernel version tagged "linux_6.1-pinned"
-pins = {
-    # TODO: Unpin when performance instability on m6i/5.10 has gone.
-    "linux_5.10-pinned": {"instance": "m6i.metal", "kv": "linux_5.10"},
-}
+pins = {}
 
 
 def apply_pins(steps):

.buildkite/pipeline_pr.py

@@ -48,16 +48,18 @@
         "./tools/devtool -y make_release",
     )
 
-if not changed_files or any(
-    x.suffix in [".rs", ".toml", ".lock"] for x in changed_files
+if not pipeline.args.no_kani and (
+    not changed_files
+    or any(x.suffix in [".rs", ".toml", ".lock"] for x in changed_files)
+    or any(x.parent.name == "devctr" for x in changed_files)
 ):
     kani_grp = pipeline.build_group(
         "🔍 Kani",
         "./tools/devtool -y test --no-build -- ../tests/integration_tests/test_kani.py -n auto",
         # Kani step default
         # Kani runs fastest on m6a.metal
         instances=["m6a.metal", "m7g.metal"],
-        platforms=[("al2", "linux_5.10")],
+        platforms=[("al2023", "linux_6.1")],
         timeout_in_minutes=300,
         **DEFAULTS_PERF,
         depends_on_build=False,
@@ -76,7 +78,7 @@
     pipeline.build_group(
         "⚙ Functional and security 🔒",
         pipeline.devtool_test(
-            pytest_opts="-n 8 --dist worksteal integration_tests/{{functional,security}}",
+            pytest_opts="-n 16 --dist worksteal integration_tests/{{functional,security}}",
         ),
     )
 

.cargo/audit.toml

@@ -1 +1,7 @@
 [advisories]
+# The `paste` dependency is transitively included via `gdbstub`.
+# While the crate is archived/unmaintained, the author considers it feature-complete
+# and functionally stable. gdbstub will be update once they migrate 
+# to an alternative solution.
+# See https://github.com/daniel5151/gdbstub/issues/168
+ignore = ["RUSTSEC-2024-0436"]