From a7f649488116093ea18845c27951915304e39cba Mon Sep 17 00:00:00 2001
From: Valerie <valeriepieger@utexas.edu>
Date: Sat, 3 May 2025 22:24:02 +0000
Subject: [PATCH 01/27] tools/devtool: add PGO helper scripting to devtool

Added commands to build with instrumentation, profile, merge the
.profraw files, and optimize/rebuild

Signed-off-by: Valerie <valeriepieger@utexas.edu>
---
 .gitignore            |  3 +++
 resources/rootfs.ext4 |  0
 resources/vmlinux     |  0
 rootfs.ext4           |  2 ++
 tools/devtool         | 46 +++++++++++++++++++++++++++++++++++++++++++
 5 files changed, 51 insertions(+)
 create mode 100644 resources/rootfs.ext4
 create mode 100644 resources/vmlinux
 create mode 100644 rootfs.ext4

diff --git a/.gitignore b/.gitignore
index 155e4cbd8a8..072b49ae674 100644
--- a/.gitignore
+++ b/.gitignore
@@ -15,3 +15,6 @@ test_results/*
 /resources/linux
 /resources/x86_64
 /resources/aarch64
+llvm.sh
+vmlinux
+vmlinux.bin
diff --git a/resources/rootfs.ext4 b/resources/rootfs.ext4
new file mode 100644
index 00000000000..e69de29bb2d
diff --git a/resources/vmlinux b/resources/vmlinux
new file mode 100644
index 00000000000..e69de29bb2d
diff --git a/rootfs.ext4 b/rootfs.ext4
new file mode 100644
index 00000000000..abb1e353ba2
--- /dev/null
+++ b/rootfs.ext4
@@ -0,0 +1,2 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<Error><Code>NoSuchKey</Code><Message>The specified key does not exist.</Message><Key>img/quickstart_guide/rootfs.ext4</Key><RequestId>F23FN2VRXVEBN5NM</RequestId><HostId>RrqeuoufdOrTKnTLGPPYxjH2RiVaSJydFn55q2W4YypHv89V3TWTxzszOt7fMQTf0w86xQRdRUk=</HostId></Error>
\ No newline at end of file
diff --git a/tools/devtool b/tools/devtool
index f86263c4731..eda7a6d1e8a 100755
--- a/tools/devtool
+++ b/tools/devtool
@@ -411,6 +411,9 @@ cmd_help() {
     echo "        Downloads the CI artifacts used for testing from our S3 bucket. If --force is passed, purges any existing"
     echo "        artifacts first. Useful for refreshing local artifacts after an update, or if something got messed up."
     echo ""
+    echo "    pgo_build [instrument|profile|merge|optimize]"
+    echo "        Build Firecracker with Profile-Guided Optimization (PGO). See README-pgo.md for more information"
+    echo ""
 
     cat <<EOF
     test_debug [-- [<pytest args>]]
@@ -1083,6 +1086,49 @@ cmd_build_ci_artifacts() {
     cmd_fix_perms
 }
 
+cmd_pgo_build() {
+    # For error checking and detecting architecture type
+    set -e
+
+    PROFDATA_DIR=/tmp/firecracker-profdata
+    PROFDATA_FILE=/tmp/firecracker.profdata
+
+    arch=$(uname -m)
+    if [[ "$arch" != "x86_64" ]]; then
+        echo "Warning: Non-x86_64 architecture detected ($arch)"
+        echo "Some features like /dev/kvm and full microVM profiling may not work."
+    fi
+
+
+    case "$1" in
+        instrument)
+            echo "Building instrumented Firecracker binary"
+            RUSTFLAGS="-Cprofile-generate=/tmp/firecracker-profdata" cargo build --release
+	    echo "Instrumentation complete."
+            ;;
+        profile)
+            echo "Run workloads manually to generate .profraw files in /tmp/firecracker-profdata/"
+            echo "Please consult README-pgo.md for more information."
+	    ;;
+        merge)
+            echo "Merging .profraw files"
+              if ! llvm-profdata merge -output=${PROFDATA_FILE} ${PROFDATA_DIR}/*.profraw; then
+   	           echo "Error: Failed to merge profile data."
+                   echo "    Make sure .profraw files exist and are readable."
+                   exit 1
+              fi
+	    echo "Merging complete."
+            ;;
+        optimize)
+            echo "Building optimized Firecracker with profile data"
+            RUSTFLAGS="-Cprofile-use=/tmp/firecracker.profdata -Cllvm-args=-pgo-warn-missing-function" cargo build --release
+            ;;
+        *)
+            echo "Usage: $0 pgo_build [instrument|profile|merge|optimize]"
+            exit 1
+            ;;
+    esac
+}
 
 main() {
 

From 69e3eaede2e6b1dceff6c0f058a1af4de3467f0a Mon Sep 17 00:00:00 2001
From: Valerie <valeriepieger@utexas.edu>
Date: Sun, 27 Apr 2025 22:46:54 +0000
Subject: [PATCH 02/27] docs/pgo-getting-started.md: added a getting started
 guide for PGO

Added detailed PGO steps and benchmarking table in .md file

Signed-off-by: Valerie <valeriepieger@utexas.edu>
---
 docs/pgo-getting-started.md | 213 ++++++++++++++++++++++++++++++++++++
 rootfs.ext4                 |   2 -
 tools/devtool               |   6 +-
 3 files changed, 216 insertions(+), 5 deletions(-)
 create mode 100644 docs/pgo-getting-started.md
 delete mode 100644 rootfs.ext4

diff --git a/docs/pgo-getting-started.md b/docs/pgo-getting-started.md
new file mode 100644
index 00000000000..322e2733748
--- /dev/null
+++ b/docs/pgo-getting-started.md
@@ -0,0 +1,213 @@
+# Profile-Guided Optimization (PGO) for Firecracker
+
+This document provides a guide for building Firecracker using Profile-Guided
+Optimization (PGO) in an isolated manner.
+
+PGO can help improve performance by using runtime profiling data to guide
+compiler optimizations. This process is fully **optional** and does **not**
+affect the default Firecracker build system.
+
+## Overview
+
+PGO allows the Rust compiler (via LLVM) to use runtime profiling data to
+optimize the generated binary for actual workloads. This generally results in
+performance improvements for CPU-bound applications like Firecracker.
+
+We generate .profraw files at runtime and merge them into .profddata files to
+then be reused in an optimized build of Firecracker.
+
+The PGO build process involves three main phases:
+
+1. **Instrumentation**: Build Firecracker with instrumentation to collect
+   profiling data.
+1. **Profiling**: Run realistic workloads to generate `.profraw` profiling
+   files.
+1. **Optimize**: Rebuild Firecracker using the collected profiling data for
+   improved performance.
+
+## 1. Build with Instrumentation
+
+Build Firecracker with profiling instrumentation enabled. If starting in the
+`firecracker` directory, the command will be:
+
+```
+./tools/devtool pgo_build instrument
+```
+
+This produces a binary that, when executed, generates `.profraw` files
+containing runtime behavior data.
+
+______________________________________________________________________
+
+\*\* Note: the ideal environment for PGO is the same as the ideal environment
+for firecracker: x86_64 architecture, Ubuntu OS (24.04 currently), and bare
+metal (so that /dev/kvm is exposed). However, this step specifically can be done
+on non-x86_64 machines with
+`RUSTFLAGS="-Cprofile-generate=/tmp/firecracker-profdata" cargo build --release --package firecracker`.
+
+### Common Issue: Failed to run custom build command for `cpu-template-helper`
+
+Try: Ensuring the build directory exists and is writable with:
+
+```
+mkdir -p src/cpu-template-helper/build
+chmod -R u+rw src/cpu-template-helper/build
+```
+
+Also ensure all dependencies (e.g., aws-lc-sys, userfaultfd-sys) can be built by
+running:
+
+```
+cargo clean
+cargo build --release
+```
+
+### Common Issue: failed to run custom build command for userfaultfd-sys v0.5.0
+
+Try: `sudo apt install libclang-dev clang pkg-config`
+
+### Common Issue: failed to run custom build command for aws-lc-sys v0.28.1
+
+Try: `sudo apt install cmake ninja-build perl`
+
+### Common Issue: a bunch of errors like..
+
+```
+OUTPUT: Failed to compile memcmp_invalid_stripped_check
+note: run with RUST_BACKTRACE=1 environment variable to display a backtrace
+warning: build failed, waiting for other jobs to finish...
+```
+
+You might have an issue with global include-path overrides.
+
+## 2. Profiling
+
+Run realistic workloads to generate these `.profraw` files. Here are some
+examples of typical workloads:
+
+- Boot a microVM
+- Simulate network activity on a microVM
+- Simulate basic I/O on a microVM
+
+Try to touch all major systems you personally care about optimizing so that you
+can benchmark it against the base build later.
+
+Here's an example process of booting a minimal microVM:
+
+1. Download a test kernel and rootfs.
+1. Start Firecracker
+1. Use curl to configure in another terminal. E.g.,
+
+```
+# Configure boot source
+curl --unix-socket $API_SOCKET -i \
+  -X PUT 'http://localhost/boot-source' \
+  -H 'Content-Type: application/json' \
+  -d '{
+        "kernel_image_path": "vmlinux.bin",
+        "boot_args":   "console=ttyS0 reboot=k panic=1 pci=off"
+      }'
+
+# Configure rootfs
+curl --unix-socket $API_SOCKET -i \
+  -X PUT 'http://localhost/drives/rootfs' \
+  -H 'Content-Type: application/json' \
+  -d '{
+        "drive_id":       "rootfs",
+        "path_on_host":   "rootfs.ext4",
+        "is_root_device": true,
+        "is_read_only":   false
+      }'
+
+# (Optional) set machine config if you want custom vCPU/RAM:
+curl --unix-socket $API_SOCKET -i \
+  -X PUT 'http://localhost/machine-config' \
+  -H 'Content-Type: application/json' \
+  -d '{
+        "vcpu_count": 1,
+        "mem_size_mib": 128
+      }'
+
+# Start the VM
+curl --unix-socket $API_SOCKET -i \
+  -X PUT 'http://localhost/actions' \
+  -H 'Content-Type: application/json' \
+  -d '{"action_type":"InstanceStart"}'
+```
+
+Please refer to the Firecracker getting started guide
+[(link here)](https://github.com/firecracker-microvm/firecracker/blob/main/docs/getting-started.md)
+for a more in-depth look at how to do this.
+
+## 3. Optimize
+
+After running your desired workloads, the resulting `.profraw` files can be seen
+with:
+
+```
+ls /tmp/firecracker-profdata/
+```
+
+______________________________________________________________________
+
+#### Merging
+
+To merge these files into valid profile data use:
+
+```
+./tools/devtool pgo_build merge
+```
+
+#### Common Issue: version mismatch
+
+This will look something like: “raw profile format version = 10; expected
+version = 9”
+
+This is common and might even happen on an ideal environment due to the Rust
+toolchain producing v10 profile but Ubuntu 24.04 packages not shipping an
+llvm-profdata that works for v10. You may be able to install the matching LLVM
+on your host, but if it gives you trouble, using Rust's nightly toolchain can
+also work.
+
+To use nightly, try:
+
+```
+rustup toolchain install nightly
+rustup component add llvm-tools-preview --toolchain nightly
+
+export PATH="$HOME/.rustup/toolchains/nightly-x86_64-unknown-linux-gnu/lib/rustlib/x86_64-unknown-linux-gnu/bin:$PATH"
+```
+
+______________________________________________________________________
+
+#### Optimized Build
+
+Once the `.profraw` files are merged into `.profdata`, you can re-build with the
+merged profile:
+
+```
+./tools/devtool pgo_build optimize
+```
+
+Then, you can verify your optimized binary is in
+`build/cargo_target/release/firecracker` and run it with
+
+```
+./build/cargo_target/release/firecracker --api-sock /tmp/fc.socket
+```
+
+### 4. Verify/Benchmark
+
+Once you have this PGO build, you can run any of the repository's existing
+performance tests to observe the speed-ups.
+
+### Community Benchmark Results
+
+Please feel free to fork this repo, run your own benchmarks, and submit a PR
+updating the table below.
+
+| Machine (CPU/RAM)             | Firecracker (non-PGO) | Firecracker (PGO) | Δ (PGO vs. baseline) | Notes                                        |
+| ----------------------------- | --------------------: | ----------------: | -------------------: | -------------------------------------------- |
+| AMD Ryzen 7 7700X; 32 GiB RAM |               0.01275 |           0.01079 |              -15.37% | Ubuntu 24.04; used test_boottime.py for both |
+|                               |                       |                   |                      |                                              |
+|                               |                       |                   |                      |                                              |
diff --git a/rootfs.ext4 b/rootfs.ext4
deleted file mode 100644
index abb1e353ba2..00000000000
--- a/rootfs.ext4
+++ /dev/null
@@ -1,2 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<Error><Code>NoSuchKey</Code><Message>The specified key does not exist.</Message><Key>img/quickstart_guide/rootfs.ext4</Key><RequestId>F23FN2VRXVEBN5NM</RequestId><HostId>RrqeuoufdOrTKnTLGPPYxjH2RiVaSJydFn55q2W4YypHv89V3TWTxzszOt7fMQTf0w86xQRdRUk=</HostId></Error>
\ No newline at end of file
diff --git a/tools/devtool b/tools/devtool
index eda7a6d1e8a..8edc338a293 100755
--- a/tools/devtool
+++ b/tools/devtool
@@ -1104,12 +1104,12 @@ cmd_pgo_build() {
         instrument)
             echo "Building instrumented Firecracker binary"
             RUSTFLAGS="-Cprofile-generate=/tmp/firecracker-profdata" cargo build --release
-	    echo "Instrumentation complete."
+	        echo "Instrumentation complete."
             ;;
         profile)
             echo "Run workloads manually to generate .profraw files in /tmp/firecracker-profdata/"
             echo "Please consult README-pgo.md for more information."
-	    ;;
+	        ;;
         merge)
             echo "Merging .profraw files"
               if ! llvm-profdata merge -output=${PROFDATA_FILE} ${PROFDATA_DIR}/*.profraw; then
@@ -1117,7 +1117,7 @@ cmd_pgo_build() {
                    echo "    Make sure .profraw files exist and are readable."
                    exit 1
               fi
-	    echo "Merging complete."
+	        echo "Merging complete."
             ;;
         optimize)
             echo "Building optimized Firecracker with profile data"

From 41864da49041c347eba1d1977cd42ae6a116adda Mon Sep 17 00:00:00 2001
From: Dakshin Devanand <dakshind2005@gmail.com>
Date: Thu, 24 Apr 2025 21:36:54 +0000
Subject: [PATCH 03/27] feat: Add PVTime support for ARM

Adds functionality for pvtime, which displays steal time
to guest on ARM machines. PVTime is persisted across
snapshots as well (snapshot ver updated).

- Added ipa per vCPU for mem region storing steal time info.
- Persists this ipa per vCPU across snapshots.
- Shared steal time mem region is setup on boot and restore
  from snapshot in builder.rs.

Signed-off-by: Dakshin Devanand <dakshind2005@gmail.com>
---
 src/vmm/src/arch/aarch64/vcpu.rs | 49 ++++++++++++++++++++++++++
 src/vmm/src/builder.rs           | 60 ++++++++++++++++++++++++++++++++
 src/vmm/src/persist.rs           |  2 +-
 3 files changed, 110 insertions(+), 1 deletion(-)

diff --git a/src/vmm/src/arch/aarch64/vcpu.rs b/src/vmm/src/arch/aarch64/vcpu.rs
index 2c4c55375ed..59c00c3ff86 100644
--- a/src/vmm/src/arch/aarch64/vcpu.rs
+++ b/src/vmm/src/arch/aarch64/vcpu.rs
@@ -11,6 +11,7 @@ use std::mem::offset_of;
 use kvm_bindings::*;
 use kvm_ioctls::{VcpuExit, VcpuFd, VmFd};
 use serde::{Deserialize, Serialize};
+use vm_memory::GuestAddress;
 
 use super::get_fdt_addr;
 use super::regs::*;
@@ -42,6 +43,8 @@ pub enum VcpuArchError {
     Fam(vmm_sys_util::fam::Error),
     /// {0}
     GetMidrEl1(String),
+    /// Failed to set/get device attributes for vCPU: {0}
+    DeviceAttribute(kvm_ioctls::Error),
 }
 
 /// Extract the Manufacturer ID from the host.
@@ -115,6 +118,8 @@ pub struct KvmVcpu {
     /// Vcpu peripherals, such as buses
     pub peripherals: Peripherals,
     kvi: kvm_vcpu_init,
+    /// IPA of steal_time region
+    pub pvtime_ipa: Option<GuestAddress>,
 }
 
 /// Vcpu peripherals
@@ -148,6 +153,7 @@ impl KvmVcpu {
             fd: kvm_vcpu,
             peripherals: Default::default(),
             kvi,
+            pvtime_ipa: None,
         })
     }
 
@@ -243,6 +249,8 @@ impl KvmVcpu {
         // the boot state and turned secondary vcpus on.
         state.kvi.features[0] &= !(1 << KVM_ARM_VCPU_POWER_OFF);
 
+        state.pvtime_ipa = self.pvtime_ipa.map(|guest_addr| guest_addr.0);
+
         Ok(state)
     }
 
@@ -276,6 +284,13 @@ impl KvmVcpu {
         }
         self.set_mpstate(state.mp_state)
             .map_err(KvmVcpuError::RestoreState)?;
+
+        // Assumes that steal time memory region was set up already
+        if let Some(pvtime_ipa) = state.pvtime_ipa {
+            self.enable_pvtime(GuestAddress(pvtime_ipa))
+                .map_err(KvmVcpuError::RestoreState)?;
+        }
+
         Ok(())
     }
 
@@ -439,6 +454,38 @@ impl KvmVcpu {
     pub fn set_mpstate(&self, state: kvm_mp_state) -> Result<(), VcpuArchError> {
         self.fd.set_mp_state(state).map_err(VcpuArchError::SetMp)
     }
+
+    /// Check if pvtime (steal time on ARM) is supported for vcpu
+    pub fn supports_pvtime(&self) -> bool {
+        let pvtime_device_attr = kvm_bindings::kvm_device_attr {
+            group: kvm_bindings::KVM_ARM_VCPU_PVTIME_CTRL,
+            attr: kvm_bindings::KVM_ARM_VCPU_PVTIME_IPA as u64,
+            addr: 0,
+            flags: 0,
+        };
+
+        // Use kvm_has_device_attr to check if PVTime is supported
+        self.fd.has_device_attr(&pvtime_device_attr).is_ok()
+    }
+
+    /// Enables pvtime for vcpu
+    pub fn enable_pvtime(&mut self, ipa: GuestAddress) -> Result<(), VcpuArchError> {
+        self.pvtime_ipa = Some(ipa);
+
+        // Use KVM syscall (kvm_set_device_attr) to register the vCPU with the steal_time region
+        let vcpu_device_attr = kvm_bindings::kvm_device_attr {
+            group: KVM_ARM_VCPU_PVTIME_CTRL,
+            attr: KVM_ARM_VCPU_PVTIME_IPA as u64,
+            addr: &ipa.0 as *const u64 as u64, // userspace address of attr data
+            flags: 0,
+        };
+
+        self.fd
+            .set_device_attr(&vcpu_device_attr)
+            .map_err(VcpuArchError::DeviceAttribute)?;
+
+        Ok(())
+    }
 }
 
 impl Peripherals {
@@ -467,6 +514,8 @@ pub struct VcpuState {
     pub mpidr: u64,
     /// kvi states for vcpu initialization.
     pub kvi: kvm_vcpu_init,
+    /// ipa for steal_time region
+    pub pvtime_ipa: Option<u64>,
 }
 
 impl Debug for VcpuState {
diff --git a/src/vmm/src/builder.rs b/src/vmm/src/builder.rs
index 398c25ba056..4a810ee083a 100644
--- a/src/vmm/src/builder.rs
+++ b/src/vmm/src/builder.rs
@@ -15,6 +15,8 @@ use linux_loader::cmdline::Cmdline as LoaderKernelCmdline;
 use userfaultfd::Uffd;
 use utils::time::TimestampUs;
 #[cfg(target_arch = "aarch64")]
+use vm_memory::GuestAddress;
+#[cfg(target_arch = "aarch64")]
 use vm_superio::Rtc;
 use vm_superio::Serial;
 use vmm_sys_util::eventfd::EventFd;
@@ -82,6 +84,9 @@ pub enum StartMicrovmError {
     CreateLegacyDevice(device_manager::legacy::LegacyDeviceError),
     /// Error creating VMGenID device: {0}
     CreateVMGenID(VmGenIdError),
+    /// Error enabling pvtime on vcpu: {0}
+    #[cfg(target_arch = "aarch64")]
+    EnablePVTime(crate::arch::VcpuArchError),
     /// Invalid Memory Configuration: {0}
     GuestMemory(crate::vstate::memory::MemoryError),
     /// Error with initrd initialization: {0}.
@@ -289,6 +294,13 @@ pub fn build_microvm_for_boot(
 
     attach_vmgenid_device(&mut vmm)?;
 
+    #[cfg(target_arch = "aarch64")]
+    if vcpus[0].kvm_vcpu.supports_pvtime() {
+        setup_pvtime(&mut vmm, &mut vcpus)?;
+    } else {
+        log::warn!("Vcpus do not support pvtime, steal time will not be reported to guest");
+    }
+
     configure_system_for_boot(
         &mut vmm,
         vcpus.as_mut(),
@@ -449,6 +461,16 @@ pub fn build_microvm_from_snapshot(
         }
     }
 
+    // Restore allocator state
+    #[cfg(target_arch = "aarch64")]
+    if let Some(pvtime_ipa) = vcpus[0].kvm_vcpu.pvtime_ipa {
+        allocate_pvtime_region(
+            &mut vmm,
+            vcpus.len(),
+            vm_allocator::AllocPolicy::ExactMatch(pvtime_ipa.0),
+        )?;
+    }
+
     // Restore vcpus kvm state.
     for (vcpu, state) in vcpus.iter_mut().zip(microvm_state.vcpu_states.iter()) {
         vcpu.kvm_vcpu
@@ -552,6 +574,44 @@ pub fn setup_serial_device(
     Ok(serial)
 }
 
+/// 64 bytes due to alignment requirement in 3.1 of https://www.kernel.org/doc/html/v5.8/virt/kvm/devices/vcpu.html#attribute-kvm-arm-vcpu-pvtime-ipa
+#[cfg(target_arch = "aarch64")]
+const STEALTIME_STRUCT_MEM_SIZE: u64 = 64;
+
+/// Helper method to allocate steal time region
+#[cfg(target_arch = "aarch64")]
+fn allocate_pvtime_region(
+    vmm: &mut Vmm,
+    vcpu_count: usize,
+    policy: vm_allocator::AllocPolicy,
+) -> Result<GuestAddress, StartMicrovmError> {
+    let size = STEALTIME_STRUCT_MEM_SIZE * vcpu_count as u64;
+    let addr = vmm
+        .resource_allocator
+        .allocate_system_memory(size, STEALTIME_STRUCT_MEM_SIZE, policy)
+        .map_err(StartMicrovmError::AllocateResources)?;
+    Ok(GuestAddress(addr))
+}
+
+/// Sets up pvtime for all vcpus
+#[cfg(target_arch = "aarch64")]
+fn setup_pvtime(vmm: &mut Vmm, vcpus: &mut [Vcpu]) -> Result<(), StartMicrovmError> {
+    // Alloc sys mem for steal time region
+    let pvtime_mem: GuestAddress =
+        allocate_pvtime_region(vmm, vcpus.len(), vm_allocator::AllocPolicy::LastMatch)?;
+
+    // Register all vcpus with pvtime device
+    for (i, vcpu) in vcpus.iter_mut().enumerate() {
+        vcpu.kvm_vcpu
+            .enable_pvtime(GuestAddress(
+                pvtime_mem.0 + i as u64 * STEALTIME_STRUCT_MEM_SIZE,
+            ))
+            .map_err(StartMicrovmError::EnablePVTime)?;
+    }
+
+    Ok(())
+}
+
 #[cfg(target_arch = "aarch64")]
 fn attach_legacy_devices_aarch64(
     event_manager: &mut EventManager,
diff --git a/src/vmm/src/persist.rs b/src/vmm/src/persist.rs
index aeacadeb66e..4111d8d6c34 100644
--- a/src/vmm/src/persist.rs
+++ b/src/vmm/src/persist.rs
@@ -148,7 +148,7 @@ pub enum CreateSnapshotError {
 }
 
 /// Snapshot version
-pub const SNAPSHOT_VERSION: Version = Version::new(6, 0, 0);
+pub const SNAPSHOT_VERSION: Version = Version::new(7, 0, 0);
 
 /// Creates a Microvm snapshot.
 pub fn create_snapshot(

From 10429dcfb17a096008cf012c84c1e2b56c18d720 Mon Sep 17 00:00:00 2001
From: Dakshin Devanand <dakshind2005@gmail.com>
Date: Thu, 24 Apr 2025 21:42:47 +0000
Subject: [PATCH 04/27] test: Add steal time integration tests

Added integration tests checking:
- steal time increase
- steal time persistence across snapshots
- pvtime existence on ARM

Motivated by addition of PVTime functionality
for ARM.

Signed-off-by: Dakshin Devanand <dakshind2005@gmail.com>
---
 .../functional/test_steal_time.py             | 121 ++++++++++++++++++
 1 file changed, 121 insertions(+)
 create mode 100644 tests/integration_tests/functional/test_steal_time.py

diff --git a/tests/integration_tests/functional/test_steal_time.py b/tests/integration_tests/functional/test_steal_time.py
new file mode 100644
index 00000000000..5c607ae4201
--- /dev/null
+++ b/tests/integration_tests/functional/test_steal_time.py
@@ -0,0 +1,121 @@
+# Copyright 2025 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+"""Tests for verifying the PVTime device behavior under contention and across snapshots."""
+
+import time
+
+import pytest
+
+from framework.properties import global_props
+
+
+def get_steal_time_ms(vm):
+    """Returns total steal time of vCPUs in VM in milliseconds"""
+    _, out, _ = vm.ssh.run("grep -w '^cpu' /proc/stat")
+    steal_time_tck = int(out.strip().split()[8])
+    clk_tck = int(vm.ssh.run("getconf CLK_TCK").stdout)
+    return steal_time_tck / clk_tck * 1000
+
+
+@pytest.mark.skipif(
+    global_props.cpu_architecture != "aarch64", reason="Only run in aarch64"
+)
+def test_guest_has_pvtime_enabled(uvm_plain):
+    """
+    Check that the guest kernel has enabled PV steal time.
+    """
+    vm = uvm_plain
+    vm.spawn()
+    vm.basic_config()
+    vm.add_net_iface()
+    vm.start()
+
+    _, stdout, _ = vm.ssh.run("dmesg | grep 'stolen time PV'")
+    assert (
+        "stolen time PV" in stdout
+    ), "Guest kernel did not report PV steal time enabled"
+
+
+def test_pvtime_steal_time_increases(uvm_plain):
+    """
+    Test that PVTime steal time increases when both vCPUs are contended on the same pCPU.
+    """
+    vm = uvm_plain
+    vm.spawn()
+    vm.basic_config()
+    vm.add_net_iface()
+    vm.start()
+
+    # Pin both vCPUs to the same physical CPU to induce contention
+    vm.pin_vcpu(0, 0)
+    vm.pin_vcpu(1, 0)
+
+    # Start two infinite loops to hog CPU time
+    hog_cmd = "nohup bash -c 'while true; do :; done' >/dev/null 2>&1 &"
+    vm.ssh.run(hog_cmd)
+    vm.ssh.run(hog_cmd)
+
+    # Measure before and after steal time
+    steal_before = get_steal_time_ms(vm)
+    time.sleep(2)
+    steal_after = get_steal_time_ms(vm)
+
+    # Require increase in steal time
+    assert (
+        steal_after > steal_before
+    ), f"Steal time did not increase as expected. Before: {steal_before}, After: {steal_after}"
+
+
+def test_pvtime_snapshot(uvm_plain, microvm_factory):
+    """
+    Test that PVTime steal time is preserved across snapshot/restore
+    and continues increasing post-resume.
+    """
+    vm = uvm_plain
+    vm.spawn()
+    vm.basic_config()
+    vm.add_net_iface()
+    vm.start()
+
+    vm.pin_vcpu(0, 0)
+    vm.pin_vcpu(1, 0)
+
+    hog_cmd = "nohup bash -c 'while true; do :; done' >/dev/null 2>&1 &"
+    vm.ssh.run(hog_cmd)
+    vm.ssh.run(hog_cmd)
+
+    # Snapshot pre-steal time
+    steal_before = get_steal_time_ms(vm)
+
+    snapshot = vm.snapshot_full()
+    vm.kill()
+
+    # Restore microVM from snapshot and resume
+    restored_vm = microvm_factory.build()
+    restored_vm.spawn()
+    restored_vm.restore_from_snapshot(snapshot, resume=False)
+    snapshot.delete()
+
+    restored_vm.pin_vcpu(0, 0)
+    restored_vm.pin_vcpu(1, 0)
+    restored_vm.resume()
+
+    # Steal time just after restoring
+    steal_after_snap = get_steal_time_ms(restored_vm)
+
+    time.sleep(2)
+
+    # Steal time after running resumed VM
+    steal_after_resume = get_steal_time_ms(restored_vm)
+
+    # Ensure steal time persisted and continued increasing
+    tolerance = 2000  # 2.0 seconds tolerance for persistence check
+    persisted = (
+        steal_before < steal_after_snap and steal_after_snap - steal_before < tolerance
+    )
+    increased = steal_after_resume > steal_after_snap
+
+    assert (
+        persisted and increased
+    ), "Steal time did not persist through snapshot or failed to increase after resume"

From a2ae6bdd8a01b0f459d7cab48a782988d4cafb22 Mon Sep 17 00:00:00 2001
From: Dakshin Devanand <dakshind2005@gmail.com>
Date: Thu, 24 Apr 2025 21:46:26 +0000
Subject: [PATCH 05/27] doc: Add changelog entry for PVTime

Add a changelog entry to inform about addition
of PVTime (steal time) functionality on ARM.

Signed-off-by: Dakshin Devanand <dakshind2005@gmail.com>
---
 CHANGELOG.md | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index ead297b7b84..c05641e1780 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -10,6 +10,9 @@ and this project adheres to
 
 ### Added
 
+- [#5139](https://github.com/firecracker-microvm/firecracker/pull/5139): Added
+  support for [PVTime](https://docs.kernel.org/virt/kvm/arm/pvtime.html). This
+  is used to support steal time on ARM machines.
 - [#5048](https://github.com/firecracker-microvm/firecracker/pull/5048): Added
   support for [PVH boot mode](docs/pvh.md). This is used when an x86 kernel
   provides the appropriate ELF Note to indicate that PVH boot mode is supported.

From 83b3e773c8bd6b7de878d6c17675537ed0b05f11 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 28 Apr 2025 17:13:57 +0000
Subject: [PATCH 06/27] build(deps): Bump the firecracker group with 12 updates

Bumps the firecracker group with 12 updates:

| Package | From | To |
| --- | --- | --- |
| [zerocopy](https://github.com/google/zerocopy) | `0.8.24` | `0.8.25` |
| [syn](https://github.com/dtolnay/syn) | `2.0.100` | `2.0.101` |
| [micro_http](https://github.com/firecracker-microvm/micro-http) | ``e854e50`` | ``4f62153`` |
| [aws-lc-sys](https://github.com/aws/aws-lc-rs) | `0.28.1` | `0.28.2` |
| [cc](https://github.com/rust-lang/cc-rs) | `1.2.19` | `1.2.20` |
| [getrandom](https://github.com/rust-random/getrandom) | `0.2.15` | `0.2.16` |
| [jiff](https://github.com/BurntSushi/jiff) | `0.2.9` | `0.2.10` |
| [jiff-static](https://github.com/BurntSushi/jiff) | `0.2.9` | `0.2.10` |
| [toml](https://github.com/toml-rs/toml) | `0.8.20` | `0.8.21` |
| [toml_datetime](https://github.com/toml-rs/toml) | `0.6.8` | `0.6.9` |
| [toml_edit](https://github.com/toml-rs/toml) | `0.22.24` | `0.22.25` |
| [winnow](https://github.com/winnow-rs/winnow) | `0.7.6` | `0.7.7` |


Updates `zerocopy` from 0.8.24 to 0.8.25
- [Release notes](https://github.com/google/zerocopy/releases)
- [Changelog](https://github.com/google/zerocopy/blob/main/CHANGELOG.md)
- [Commits](https://github.com/google/zerocopy/compare/v0.8.24...v0.8.25)

Updates `syn` from 2.0.100 to 2.0.101
- [Release notes](https://github.com/dtolnay/syn/releases)
- [Commits](https://github.com/dtolnay/syn/compare/2.0.100...2.0.101)

Updates `micro_http` from `e854e50` to `4f62153`
- [Commits](https://github.com/firecracker-microvm/micro-http/compare/e854e50bc06a7e7bc0e5f5835d8f3f951e21f05f...4f621532e81ee2ad096a9c9592fdacc40d19de48)

Updates `aws-lc-sys` from 0.28.1 to 0.28.2
- [Release notes](https://github.com/aws/aws-lc-rs/releases)
- [Commits](https://github.com/aws/aws-lc-rs/compare/aws-lc-sys/v0.28.1...aws-lc-sys/v0.28.2)

Updates `cc` from 1.2.19 to 1.2.20
- [Release notes](https://github.com/rust-lang/cc-rs/releases)
- [Changelog](https://github.com/rust-lang/cc-rs/blob/main/CHANGELOG.md)
- [Commits](https://github.com/rust-lang/cc-rs/compare/cc-v1.2.19...cc-v1.2.20)

Updates `getrandom` from 0.2.15 to 0.2.16
- [Changelog](https://github.com/rust-random/getrandom/blob/master/CHANGELOG.md)
- [Commits](https://github.com/rust-random/getrandom/compare/v0.2.15...v0.2.16)

Updates `jiff` from 0.2.9 to 0.2.10
- [Release notes](https://github.com/BurntSushi/jiff/releases)
- [Changelog](https://github.com/BurntSushi/jiff/blob/master/CHANGELOG.md)
- [Commits](https://github.com/BurntSushi/jiff/compare/jiff-static-0.2.9...jiff-static-0.2.10)

Updates `jiff-static` from 0.2.9 to 0.2.10
- [Release notes](https://github.com/BurntSushi/jiff/releases)
- [Changelog](https://github.com/BurntSushi/jiff/blob/master/CHANGELOG.md)
- [Commits](https://github.com/BurntSushi/jiff/compare/jiff-static-0.2.9...jiff-static-0.2.10)

Updates `toml` from 0.8.20 to 0.8.21
- [Commits](https://github.com/toml-rs/toml/compare/toml-v0.8.20...toml-v0.8.21)

Updates `toml_datetime` from 0.6.8 to 0.6.9
- [Commits](https://github.com/toml-rs/toml/compare/toml_datetime-v0.6.8...toml_datetime-v0.6.9)

Updates `toml_edit` from 0.22.24 to 0.22.25
- [Commits](https://github.com/toml-rs/toml/compare/v0.22.24...v0.22.25)

Updates `winnow` from 0.7.6 to 0.7.7
- [Changelog](https://github.com/winnow-rs/winnow/blob/main/CHANGELOG.md)
- [Commits](https://github.com/winnow-rs/winnow/compare/v0.7.6...v0.7.7)

---
updated-dependencies:
- dependency-name: zerocopy
  dependency-version: 0.8.25
  dependency-type: direct:production
  update-type: version-update:semver-patch
  dependency-group: firecracker
- dependency-name: syn
  dependency-version: 2.0.101
  dependency-type: direct:production
  update-type: version-update:semver-patch
  dependency-group: firecracker
- dependency-name: micro_http
  dependency-version: 4f621532e81ee2ad096a9c9592fdacc40d19de48
  dependency-type: direct:production
  dependency-group: firecracker
- dependency-name: aws-lc-sys
  dependency-version: 0.28.2
  dependency-type: indirect
  update-type: version-update:semver-patch
  dependency-group: firecracker
- dependency-name: cc
  dependency-version: 1.2.20
  dependency-type: indirect
  update-type: version-update:semver-patch
  dependency-group: firecracker
- dependency-name: getrandom
  dependency-version: 0.2.16
  dependency-type: indirect
  update-type: version-update:semver-patch
  dependency-group: firecracker
- dependency-name: jiff
  dependency-version: 0.2.10
  dependency-type: indirect
  update-type: version-update:semver-patch
  dependency-group: firecracker
- dependency-name: jiff-static
  dependency-version: 0.2.10
  dependency-type: indirect
  update-type: version-update:semver-patch
  dependency-group: firecracker
- dependency-name: toml
  dependency-version: 0.8.21
  dependency-type: indirect
  update-type: version-update:semver-patch
  dependency-group: firecracker
- dependency-name: toml_datetime
  dependency-version: 0.6.9
  dependency-type: indirect
  update-type: version-update:semver-patch
  dependency-group: firecracker
- dependency-name: toml_edit
  dependency-version: 0.22.25
  dependency-type: indirect
  update-type: version-update:semver-patch
  dependency-group: firecracker
- dependency-name: winnow
  dependency-version: 0.7.7
  dependency-type: indirect
  update-type: version-update:semver-patch
  dependency-group: firecracker
...

Signed-off-by: dependabot[bot] <support@github.com>
---
 Cargo.lock                           | 73 +++++++++++++++-------------
 src/acpi-tables/Cargo.toml           |  2 +-
 src/clippy-tracing/Cargo.toml        |  2 +-
 src/log-instrument-macros/Cargo.toml |  2 +-
 src/seccompiler/Cargo.toml           |  2 +-
 src/vmm/Cargo.toml                   |  2 +-
 6 files changed, 45 insertions(+), 38 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index 39fd9998f4d..5a87079b2da 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -9,7 +9,7 @@ dependencies = [
  "displaydoc",
  "thiserror 2.0.12",
  "vm-memory",
- "zerocopy 0.8.24",
+ "zerocopy 0.8.25",
 ]
 
 [[package]]
@@ -152,9 +152,9 @@ dependencies = [
 
 [[package]]
 name = "aws-lc-sys"
-version = "0.28.1"
+version = "0.28.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0ddeb19ee86cb16ecfc871e5b0660aff6285760957aaedda6284cf0e790d3769"
+checksum = "bfa9b6986f250236c27e5a204062434a773a13243d2ffc2955f37bdba4c5c6a1"
 dependencies = [
  "bindgen 0.69.5",
  "cc",
@@ -268,9 +268,9 @@ checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5"
 
 [[package]]
 name = "cc"
-version = "1.2.19"
+version = "1.2.20"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8e3a13707ac958681c13b39b458c073d0d9bc8a22cb1b2f4c8e55eb72c13f362"
+checksum = "04da6a0d40b948dfc4fa8f5bbf402b0fc1a64a28dbf7d12ffd683550f2c1b63a"
 dependencies = [
  "jobserver",
  "libc",
@@ -670,9 +670,9 @@ dependencies = [
 
 [[package]]
 name = "getrandom"
-version = "0.2.15"
+version = "0.2.16"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c4567c8db10ae91089c99af84c68c38da3ec2f087c3f82960bcdbf3656b6f4d7"
+checksum = "335ff9f135e4384c8150d6f27c6daed433577f86b4750418338c01a1a2528592"
 dependencies = [
  "cfg-if",
  "libc",
@@ -818,9 +818,9 @@ dependencies = [
 
 [[package]]
 name = "jiff"
-version = "0.2.9"
+version = "0.2.10"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "59ec30f7142be6fe14e1b021f50b85db8df2d4324ea6e91ec3e5dcde092021d0"
+checksum = "5a064218214dc6a10fbae5ec5fa888d80c45d611aba169222fc272072bf7aef6"
 dependencies = [
  "jiff-static",
  "log",
@@ -831,9 +831,9 @@ dependencies = [
 
 [[package]]
 name = "jiff-static"
-version = "0.2.9"
+version = "0.2.10"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "526b834d727fd59d37b076b0c3236d9adde1b1729a4361e20b2026f738cc1dbe"
+checksum = "199b7932d97e325aff3a7030e141eafe7f2c6268e1d1b24859b753a627f45254"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -966,7 +966,7 @@ dependencies = [
 [[package]]
 name = "micro_http"
 version = "0.1.0"
-source = "git+https://github.com/firecracker-microvm/micro-http#e854e50bc06a7e7bc0e5f5835d8f3f951e21f05f"
+source = "git+https://github.com/firecracker-microvm/micro-http#4f621532e81ee2ad096a9c9592fdacc40d19de48"
 dependencies = [
  "libc",
  "vmm-sys-util",
@@ -1071,7 +1071,7 @@ version = "0.2.21"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9"
 dependencies = [
- "zerocopy 0.8.24",
+ "zerocopy 0.8.25",
 ]
 
 [[package]]
@@ -1137,7 +1137,7 @@ checksum = "3779b94aeb87e8bd4e834cee3650289ee9e0d5677f976ecdb6d219e5f4f6cd94"
 dependencies = [
  "rand_chacha 0.9.0",
  "rand_core 0.9.0",
- "zerocopy 0.8.24",
+ "zerocopy 0.8.25",
 ]
 
 [[package]]
@@ -1166,7 +1166,7 @@ version = "0.6.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c"
 dependencies = [
- "getrandom 0.2.15",
+ "getrandom 0.2.16",
 ]
 
 [[package]]
@@ -1176,7 +1176,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "b08f3c9802962f7e1b25113931d94f43ed9725bebc59db9d0c3e9a23b67e15ff"
 dependencies = [
  "getrandom 0.3.1",
- "zerocopy 0.8.24",
+ "zerocopy 0.8.25",
 ]
 
 [[package]]
@@ -1274,7 +1274,7 @@ dependencies = [
  "serde",
  "serde_json",
  "thiserror 2.0.12",
- "zerocopy 0.8.24",
+ "zerocopy 0.8.25",
 ]
 
 [[package]]
@@ -1372,9 +1372,9 @@ checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292"
 
 [[package]]
 name = "syn"
-version = "2.0.100"
+version = "2.0.101"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b09a44accad81e1ba1cd74a32461ba89dee89095ba17b32f5d03683b1b1fc2a0"
+checksum = "8ce2b7fc941b3a24138a0a7cf8e858bfc6a992e7978a068a5c760deb0ed43caf"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -1442,9 +1442,9 @@ dependencies = [
 
 [[package]]
 name = "toml"
-version = "0.8.20"
+version = "0.8.21"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "cd87a5cdd6ffab733b2f74bc4fd7ee5fff6634124999ac278c35fc78c6120148"
+checksum = "900f6c86a685850b1bc9f6223b20125115ee3f31e01207d81655bbcc0aea9231"
 dependencies = [
  "serde",
  "serde_spanned",
@@ -1454,26 +1454,33 @@ dependencies = [
 
 [[package]]
 name = "toml_datetime"
-version = "0.6.8"
+version = "0.6.9"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0dd7358ecb8fc2f8d014bf86f6f638ce72ba252a2c3a2572f2a795f1d23efb41"
+checksum = "3da5db5a963e24bc68be8b17b6fa82814bb22ee8660f192bb182771d498f09a3"
 dependencies = [
  "serde",
 ]
 
 [[package]]
 name = "toml_edit"
-version = "0.22.24"
+version = "0.22.25"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "17b4795ff5edd201c7cd6dca065ae59972ce77d1b80fa0a84d94950ece7d1474"
+checksum = "10558ed0bd2a1562e630926a2d1f0b98c827da99fabd3fe20920a59642504485"
 dependencies = [
  "indexmap",
  "serde",
  "serde_spanned",
  "toml_datetime",
+ "toml_write",
  "winnow",
 ]
 
+[[package]]
+name = "toml_write"
+version = "0.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "28391a4201ba7eb1984cfeb6862c0b3ea2cfe23332298967c749dddc0d6cd976"
+
 [[package]]
 name = "typenum"
 version = "1.18.0"
@@ -1684,7 +1691,7 @@ dependencies = [
  "vm-memory",
  "vm-superio",
  "vmm-sys-util",
- "zerocopy 0.8.24",
+ "zerocopy 0.8.25",
 ]
 
 [[package]]
@@ -1842,9 +1849,9 @@ checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec"
 
 [[package]]
 name = "winnow"
-version = "0.7.6"
+version = "0.7.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "63d3fcd9bba44b03821e7d699eeee959f3126dcc4aa8e4ae18ec617c2a5cea10"
+checksum = "6cb8234a863ea0e8cd7284fcdd4f145233eb00fee02bbdd9861aec44e6477bc5"
 dependencies = [
  "memchr",
 ]
@@ -1870,11 +1877,11 @@ dependencies = [
 
 [[package]]
 name = "zerocopy"
-version = "0.8.24"
+version = "0.8.25"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2586fea28e186957ef732a5f8b3be2da217d65c5969d4b1e17f973ebbe876879"
+checksum = "a1702d9583232ddb9174e01bb7c15a2ab8fb1bc6f227aa1233858c351a3ba0cb"
 dependencies = [
- "zerocopy-derive 0.8.24",
+ "zerocopy-derive 0.8.25",
 ]
 
 [[package]]
@@ -1890,9 +1897,9 @@ dependencies = [
 
 [[package]]
 name = "zerocopy-derive"
-version = "0.8.24"
+version = "0.8.25"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a996a8f63c5c4448cd959ac1bab0aaa3306ccfd060472f85943ee0750f0169be"
+checksum = "28a6e20d751156648aa063f3800b706ee209a32c0b4d9f24be3d980b01be55ef"
 dependencies = [
  "proc-macro2",
  "quote",
diff --git a/src/acpi-tables/Cargo.toml b/src/acpi-tables/Cargo.toml
index 141f09a47da..b31ebe4ac77 100644
--- a/src/acpi-tables/Cargo.toml
+++ b/src/acpi-tables/Cargo.toml
@@ -10,7 +10,7 @@ license = "Apache-2.0"
 displaydoc = "0.2.5"
 thiserror = "2.0.12"
 vm-memory = { version = "0.16.1", features = ["backend-mmap", "backend-bitmap"] }
-zerocopy = { version = "0.8.24", features = ["derive"] }
+zerocopy = { version = "0.8.25", features = ["derive"] }
 
 [lib]
 bench = false
diff --git a/src/clippy-tracing/Cargo.toml b/src/clippy-tracing/Cargo.toml
index d9883a03dc5..97f6a6c2275 100644
--- a/src/clippy-tracing/Cargo.toml
+++ b/src/clippy-tracing/Cargo.toml
@@ -14,7 +14,7 @@ clap = { version = "4.5.37", features = ["derive"] }
 itertools = "0.14.0"
 proc-macro2 = { version = "1.0.95", features = ["span-locations"] }
 quote = "1.0.40"
-syn = { version = "2.0.100", features = ["full", "extra-traits", "visit", "visit-mut", "printing"] } 
+syn = { version = "2.0.101", features = ["full", "extra-traits", "visit", "visit-mut", "printing"] } 
 walkdir = "2.5.0"
 
 [dev-dependencies]
diff --git a/src/log-instrument-macros/Cargo.toml b/src/log-instrument-macros/Cargo.toml
index 6f98f66721d..b41f836f0e8 100644
--- a/src/log-instrument-macros/Cargo.toml
+++ b/src/log-instrument-macros/Cargo.toml
@@ -13,7 +13,7 @@ bench = false
 [dependencies]
 proc-macro2 = "1.0.95"
 quote = "1.0.40"
-syn = { version = "2.0.100", features = ["full", "extra-traits"] }
+syn = { version = "2.0.101", features = ["full", "extra-traits"] }
 
 [lints]
 workspace = true
diff --git a/src/seccompiler/Cargo.toml b/src/seccompiler/Cargo.toml
index 152fe353ae2..4e152e82cfb 100644
--- a/src/seccompiler/Cargo.toml
+++ b/src/seccompiler/Cargo.toml
@@ -23,7 +23,7 @@ libc = "0.2.172"
 serde = { version = "1.0.219", features = ["derive"] }
 serde_json = "1.0.140"
 thiserror = "2.0.12"
-zerocopy = { version = "0.8.24" }
+zerocopy = { version = "0.8.25" }
 
 [lints]
 workspace = true
diff --git a/src/vmm/Cargo.toml b/src/vmm/Cargo.toml
index 39747ef1d53..86ba95e7768 100644
--- a/src/vmm/Cargo.toml
+++ b/src/vmm/Cargo.toml
@@ -43,7 +43,7 @@ vm-allocator = "0.1.0"
 vm-memory = { version = "0.16.1", features = ["backend-mmap", "backend-bitmap"] }
 vm-superio = "0.8.0"
 vmm-sys-util = { version = "0.12.1", features = ["with-serde"] }
-zerocopy = { version = "0.8.24" }
+zerocopy = { version = "0.8.25" }
 
 [target.'cfg(target_arch = "aarch64")'.dependencies]
 vm-fdt = "0.3.0"

From e8a6d6a282c867200b80aa540d89a6c2d264025f Mon Sep 17 00:00:00 2001
From: Riccardo Mancini <mancio@amazon.com>
Date: Thu, 1 May 2025 16:20:09 +0100
Subject: [PATCH 07/27] test(signal): remove flaky unit test

The signal handler unit test registers the signal handlers, spins up a
new thread, and sends signals to itself (using kill) to check that the
metrics get updated. This is a very complicated test for a unit test,
and it's already covered in the integration test test_signals.py.
As this test is seldomly failing in our CI, it's best to get rid of its
complexity and rely on the integration tests, which are better suited
for this kind of test.

Signed-off-by: Riccardo Mancini <mancio@amazon.com>
---
 src/vmm/src/signal_handler.rs | 72 -----------------------------------
 1 file changed, 72 deletions(-)

diff --git a/src/vmm/src/signal_handler.rs b/src/vmm/src/signal_handler.rs
index 3b6162c7e4c..0fdfa0d8b97 100644
--- a/src/vmm/src/signal_handler.rs
+++ b/src/vmm/src/signal_handler.rs
@@ -50,7 +50,6 @@ macro_rules! generate_handler {
 
             $body(si_code, info);
 
-            #[cfg(not(test))]
             match si_signo {
                 $signal_name => exit_with_code(crate::FcExitCode::$exit_code),
                 _ => exit_with_code(FcExitCode::UnexpectedError),
@@ -170,74 +169,3 @@ pub fn register_signal_handlers() -> vmm_sys_util::errno::Result<()> {
     register_signal_handler(SIGILL, sigill_handler)?;
     Ok(())
 }
-
-#[cfg(test)]
-mod tests {
-    #![allow(clippy::undocumented_unsafe_blocks)]
-    use std::{process, thread};
-
-    use libc::syscall;
-
-    use super::*;
-
-    #[test]
-    fn test_signal_handler() {
-        let child = thread::spawn(move || {
-            register_signal_handlers().unwrap();
-
-            // Call the forbidden `SYS_mkdirat`.
-            unsafe { libc::syscall(libc::SYS_mkdirat, "/foo/bar\0") };
-
-            // Call SIGBUS signal handler.
-            assert_eq!(METRICS.signals.sigbus.fetch(), 0);
-            unsafe {
-                syscall(libc::SYS_kill, process::id(), SIGBUS);
-            }
-
-            // Call SIGSEGV signal handler.
-            assert_eq!(METRICS.signals.sigsegv.fetch(), 0);
-            unsafe {
-                syscall(libc::SYS_kill, process::id(), SIGSEGV);
-            }
-
-            // Call SIGXFSZ signal handler.
-            assert_eq!(METRICS.signals.sigxfsz.fetch(), 0);
-            unsafe {
-                syscall(libc::SYS_kill, process::id(), SIGXFSZ);
-            }
-
-            // Call SIGXCPU signal handler.
-            assert_eq!(METRICS.signals.sigxcpu.fetch(), 0);
-            unsafe {
-                syscall(libc::SYS_kill, process::id(), SIGXCPU);
-            }
-
-            // Call SIGPIPE signal handler.
-            assert_eq!(METRICS.signals.sigpipe.count(), 0);
-            unsafe {
-                syscall(libc::SYS_kill, process::id(), SIGPIPE);
-            }
-
-            // Call SIGHUP signal handler.
-            assert_eq!(METRICS.signals.sighup.fetch(), 0);
-            unsafe {
-                syscall(libc::SYS_kill, process::id(), SIGHUP);
-            }
-
-            // Call SIGILL signal handler.
-            assert_eq!(METRICS.signals.sigill.fetch(), 0);
-            unsafe {
-                syscall(libc::SYS_kill, process::id(), SIGILL);
-            }
-        });
-        child.join().unwrap();
-
-        assert!(METRICS.signals.sigbus.fetch() >= 1);
-        assert!(METRICS.signals.sigsegv.fetch() >= 1);
-        assert!(METRICS.signals.sigxfsz.fetch() >= 1);
-        assert!(METRICS.signals.sigxcpu.fetch() >= 1);
-        assert!(METRICS.signals.sigpipe.count() >= 1);
-        assert!(METRICS.signals.sighup.fetch() >= 1);
-        assert!(METRICS.signals.sigill.fetch() >= 1);
-    }
-}

From 1ebf72323eeace567ecff49acc43f7b9f6e303a2 Mon Sep 17 00:00:00 2001
From: Riccardo Mancini <mancio@amazon.com>
Date: Tue, 29 Apr 2025 16:21:07 +0100
Subject: [PATCH 08/27] test(jailer): use tmp dir for mknod test

We seldom have failures in the CI where the test_mknod_and_own_dev fails
because a file already exists.
The test is using the actual /dev to create tmp devices. As there's no
reason to use the actual /dev, move it to use a random folder and clean
it up after the test.

Signed-off-by: Riccardo Mancini <mancio@amazon.com>
---
 src/jailer/src/env.rs | 35 ++++++++++++++++++++++-------------
 1 file changed, 22 insertions(+), 13 deletions(-)

diff --git a/src/jailer/src/env.rs b/src/jailer/src/env.rs
index e337ea95f90..8851adbf195 100644
--- a/src/jailer/src/env.rs
+++ b/src/jailer/src/env.rs
@@ -1138,26 +1138,35 @@ mod tests {
         mock_cgroups.add_v1_mounts().unwrap();
         let env = create_env(mock_cgroups.proc_mounts_path.as_str());
 
+        let mock_dev_dir = TempDir::new().unwrap();
+
         // Ensure device nodes are created with correct major/minor numbers and permissions.
-        let mut dev_infos: Vec<(&CStr, u32, u32)> = vec![
-            (c"/dev/net/tun-test", DEV_NET_TUN_MAJOR, DEV_NET_TUN_MINOR),
-            (c"/dev/kvm-test", DEV_KVM_MAJOR, DEV_KVM_MINOR),
+        let mut dev_infos: Vec<(PathBuf, u32, u32)> = vec![
+            (
+                mock_dev_dir.as_path().join("net/tun-test"),
+                DEV_NET_TUN_MAJOR,
+                DEV_NET_TUN_MINOR,
+            ),
+            (
+                mock_dev_dir.as_path().join("kvm-test"),
+                DEV_KVM_MAJOR,
+                DEV_KVM_MINOR,
+            ),
         ];
 
         if let Some(uffd_dev_minor) = env.uffd_dev_minor {
-            dev_infos.push((c"/dev/userfaultfd-test", DEV_UFFD_MAJOR, uffd_dev_minor));
+            dev_infos.push((
+                mock_dev_dir.as_path().join("userfaultfd-test"),
+                DEV_UFFD_MAJOR,
+                uffd_dev_minor,
+            ));
         }
 
         for (dev, major, minor) in dev_infos {
-            // Checking this just to be super sure there's no file at `dev_str` path (though
-            // it shouldn't be as we deleted it at the end of the previous test run).
-            if Path::new(dev.to_str().unwrap()).exists() {
-                fs::remove_file(dev.to_str().unwrap()).unwrap();
-            }
-
-            ensure_mknod_and_own_dev(&env, dev, major, minor);
-            // Remove the device node.
-            fs::remove_file(dev.to_str().unwrap()).expect("Could not remove file.");
+            // Ensure the folder where we are creating the node exists
+            fs::create_dir_all(dev.parent().unwrap()).unwrap();
+            let dev_path = dev.to_str().map(CString::new).unwrap().unwrap();
+            ensure_mknod_and_own_dev(&env, &dev_path, major, minor);
         }
     }
 

From ead75b76c15f83a659a8f7139b7df52253996d06 Mon Sep 17 00:00:00 2001
From: Riccardo Mancini <mancio@amazon.com>
Date: Thu, 1 May 2025 16:43:34 +0100
Subject: [PATCH 09/27] refactor(test/cgroup): use a TempDir instead of a
 manually created dir

The test was reimplementing the logic for creating a temporary directory
instead of using TempDir, so I've changed it to simplify it.
Also, save the PathBuf object instead of the String to be able to do
Path operations in a canonical way without formatting strings.

Signed-off-by: Riccardo Mancini <mancio@amazon.com>
---
 src/jailer/src/cgroup.rs | 94 ++++++++++++++++++++--------------------
 src/jailer/src/env.rs    | 56 ++++++++++++------------
 2 files changed, 76 insertions(+), 74 deletions(-)

diff --git a/src/jailer/src/cgroup.rs b/src/jailer/src/cgroup.rs
index 94098d3698f..8128a2b482f 100644
--- a/src/jailer/src/cgroup.rs
+++ b/src/jailer/src/cgroup.rs
@@ -504,13 +504,15 @@ pub mod test_util {
     use std::io::Write;
     use std::path::{Path, PathBuf};
 
-    use vmm_sys_util::rand;
+    use vmm_sys_util::tempdir::TempDir;
 
     #[derive(Debug)]
     pub struct MockCgroupFs {
         mounts_file: File,
-        pub proc_mounts_path: String,
-        pub sys_cgroups_path: String,
+        // kept to clean up on Drop
+        _mock_jailer_dir: TempDir,
+        pub proc_mounts_path: PathBuf,
+        pub sys_cgroups_path: PathBuf,
     }
 
     // Helper object that simulates the layout of the cgroup file system
@@ -533,17 +535,12 @@ pub mod test_util {
         }
 
         pub fn new() -> std::result::Result<MockCgroupFs, std::io::Error> {
-            let mock_jailer_dir = format!(
-                "/tmp/firecracker/test/{}/jailer",
-                rand::rand_alphanumerics(4).into_string().unwrap()
-            );
-            let mock_proc_mounts = format!("{}/{}", mock_jailer_dir, "proc/mounts",);
-            let mock_sys_cgroups = format!("{}/{}", mock_jailer_dir, "sys_cgroup",);
-
-            let mock_proc_dir = Path::new(&mock_proc_mounts).parent().unwrap();
+            let mock_jailer_dir = TempDir::new().unwrap();
+            let mock_proc_mounts = mock_jailer_dir.as_path().join("proc/mounts");
+            let mock_sys_cgroups = mock_jailer_dir.as_path().join("sys_cgroup");
 
             // create a mock /proc/mounts file in a temporary directory
-            fs::create_dir_all(mock_proc_dir)?;
+            fs::create_dir_all(mock_proc_mounts.parent().unwrap())?;
             let file = OpenOptions::new()
                 .read(true)
                 .write(true)
@@ -552,6 +549,7 @@ pub mod test_util {
                 .open(mock_proc_mounts.clone())?;
             Ok(MockCgroupFs {
                 mounts_file: file,
+                _mock_jailer_dir: mock_jailer_dir,
                 proc_mounts_path: mock_proc_mounts,
                 sys_cgroups_path: mock_sys_cgroups,
             })
@@ -563,9 +561,9 @@ pub mod test_util {
             writeln!(
                 self.mounts_file,
                 "cgroupv2 {}/unified cgroup2 rw,nosuid,nodev,noexec,relatime,nsdelegate 0 0",
-                self.sys_cgroups_path,
+                self.sys_cgroups_path.to_str().unwrap(),
             )?;
-            let cg_unified_path = PathBuf::from(format!("{}/unified", self.sys_cgroups_path));
+            let cg_unified_path = self.sys_cgroups_path.join("unified");
             fs::create_dir_all(&cg_unified_path)?;
             Self::create_file_with_contents(
                 cg_unified_path.join("cgroup.controllers"),
@@ -589,26 +587,14 @@ pub mod test_util {
                 writeln!(
                     self.mounts_file,
                     "cgroup {}/{} cgroup rw,nosuid,nodev,noexec,relatime,{} 0 0",
-                    self.sys_cgroups_path, c, c,
+                    self.sys_cgroups_path.to_str().unwrap(),
+                    c,
+                    c,
                 )?;
             }
             Ok(())
         }
     }
-
-    // Cleanup created files when object goes out of scope
-    impl Drop for MockCgroupFs {
-        fn drop(&mut self) {
-            let tmp_dir = Path::new(self.proc_mounts_path.as_str())
-                .parent()
-                .unwrap()
-                .parent()
-                .unwrap()
-                .parent()
-                .unwrap();
-            let _ = fs::remove_dir_all(tmp_dir);
-        }
-    }
 }
 
 #[cfg(test)]
@@ -639,14 +625,16 @@ mod tests {
     #[test]
     fn test_cgroup_conf_builder_invalid_version() {
         let mock_cgroups = MockCgroupFs::new().unwrap();
-        let builder = CgroupConfigurationBuilder::new(0, mock_cgroups.proc_mounts_path.as_str());
+        let builder =
+            CgroupConfigurationBuilder::new(0, mock_cgroups.proc_mounts_path.to_str().unwrap());
         builder.unwrap_err();
     }
 
     #[test]
     fn test_cgroup_conf_builder_no_mounts() {
         let mock_cgroups = MockCgroupFs::new().unwrap();
-        let builder = CgroupConfigurationBuilder::new(1, mock_cgroups.proc_mounts_path.as_str());
+        let builder =
+            CgroupConfigurationBuilder::new(1, mock_cgroups.proc_mounts_path.to_str().unwrap());
         builder.unwrap_err();
     }
 
@@ -654,7 +642,8 @@ mod tests {
     fn test_cgroup_conf_builder_v1() {
         let mut mock_cgroups = MockCgroupFs::new().unwrap();
         mock_cgroups.add_v1_mounts().unwrap();
-        let builder = CgroupConfigurationBuilder::new(1, mock_cgroups.proc_mounts_path.as_str());
+        let builder =
+            CgroupConfigurationBuilder::new(1, mock_cgroups.proc_mounts_path.to_str().unwrap());
         builder.unwrap();
     }
 
@@ -662,7 +651,8 @@ mod tests {
     fn test_cgroup_conf_builder_v2() {
         let mut mock_cgroups = MockCgroupFs::new().unwrap();
         mock_cgroups.add_v2_mounts().unwrap();
-        let builder = CgroupConfigurationBuilder::new(2, mock_cgroups.proc_mounts_path.as_str());
+        let builder =
+            CgroupConfigurationBuilder::new(2, mock_cgroups.proc_mounts_path.to_str().unwrap());
         builder.unwrap();
     }
 
@@ -670,14 +660,16 @@ mod tests {
     fn test_cgroup_conf_builder_v2_with_v1_mounts() {
         let mut mock_cgroups = MockCgroupFs::new().unwrap();
         mock_cgroups.add_v1_mounts().unwrap();
-        let builder = CgroupConfigurationBuilder::new(2, mock_cgroups.proc_mounts_path.as_str());
+        let builder =
+            CgroupConfigurationBuilder::new(2, mock_cgroups.proc_mounts_path.to_str().unwrap());
         builder.unwrap_err();
     }
 
     #[test]
     fn test_cgroup_conf_builder_v2_no_mounts() {
         let mock_cgroups = MockCgroupFs::new().unwrap();
-        let builder = CgroupConfigurationBuilder::new(2, mock_cgroups.proc_mounts_path.as_str());
+        let builder =
+            CgroupConfigurationBuilder::new(2, mock_cgroups.proc_mounts_path.to_str().unwrap());
         builder.unwrap_err();
     }
 
@@ -685,7 +677,8 @@ mod tests {
     fn test_cgroup_conf_builder_v1_with_v2_mounts() {
         let mut mock_cgroups = MockCgroupFs::new().unwrap();
         mock_cgroups.add_v2_mounts().unwrap();
-        let builder = CgroupConfigurationBuilder::new(1, mock_cgroups.proc_mounts_path.as_str());
+        let builder =
+            CgroupConfigurationBuilder::new(1, mock_cgroups.proc_mounts_path.to_str().unwrap());
         builder.unwrap_err();
     }
 
@@ -696,9 +689,11 @@ mod tests {
         mock_cgroups.add_v2_mounts().unwrap();
 
         for v in &[1, 2] {
-            let mut builder =
-                CgroupConfigurationBuilder::new(*v, mock_cgroups.proc_mounts_path.as_str())
-                    .unwrap();
+            let mut builder = CgroupConfigurationBuilder::new(
+                *v,
+                mock_cgroups.proc_mounts_path.to_str().unwrap(),
+            )
+            .unwrap();
 
             builder
                 .add_cgroup_property(
@@ -719,9 +714,11 @@ mod tests {
         mock_cgroups.add_v2_mounts().unwrap();
 
         for v in &[1, 2] {
-            let mut builder =
-                CgroupConfigurationBuilder::new(*v, mock_cgroups.proc_mounts_path.as_str())
-                    .unwrap();
+            let mut builder = CgroupConfigurationBuilder::new(
+                *v,
+                mock_cgroups.proc_mounts_path.to_str().unwrap(),
+            )
+            .unwrap();
             builder
                 .add_cgroup_property(
                     "invalid.cg".to_string(),
@@ -739,7 +736,8 @@ mod tests {
         mock_cgroups.add_v1_mounts().unwrap();
 
         let mut builder =
-            CgroupConfigurationBuilder::new(1, mock_cgroups.proc_mounts_path.as_str()).unwrap();
+            CgroupConfigurationBuilder::new(1, mock_cgroups.proc_mounts_path.to_str().unwrap())
+                .unwrap();
         builder
             .add_cgroup_property(
                 "cpuset.mems".to_string(),
@@ -750,7 +748,7 @@ mod tests {
             .unwrap();
         let cg_conf = builder.build();
 
-        let cg_root = PathBuf::from(format!("{}/cpuset", mock_cgroups.sys_cgroups_path));
+        let cg_root = mock_cgroups.sys_cgroups_path.join("cpuset");
 
         // with real cgroups these files are created automatically
         // since the mock will not do it automatically, we create it here
@@ -773,11 +771,13 @@ mod tests {
     fn test_cgroup_conf_v2_write_value() {
         let mut mock_cgroups = MockCgroupFs::new().unwrap();
         mock_cgroups.add_v2_mounts().unwrap();
-        let builder = CgroupConfigurationBuilder::new(2, mock_cgroups.proc_mounts_path.as_str());
+        let builder =
+            CgroupConfigurationBuilder::new(2, mock_cgroups.proc_mounts_path.to_str().unwrap());
         builder.unwrap();
 
         let mut builder =
-            CgroupConfigurationBuilder::new(2, mock_cgroups.proc_mounts_path.as_str()).unwrap();
+            CgroupConfigurationBuilder::new(2, mock_cgroups.proc_mounts_path.to_str().unwrap())
+                .unwrap();
         builder
             .add_cgroup_property(
                 "cpuset.mems".to_string(),
@@ -787,7 +787,7 @@ mod tests {
             )
             .unwrap();
 
-        let cg_root = PathBuf::from(format!("{}/unified", mock_cgroups.sys_cgroups_path));
+        let cg_root = mock_cgroups.sys_cgroups_path.join("unified");
 
         assert_eq!(builder.get_v2_hierarchy_path().unwrap(), &cg_root);
 
diff --git a/src/jailer/src/env.rs b/src/jailer/src/env.rs
index 8851adbf195..e2953584a32 100644
--- a/src/jailer/src/env.rs
+++ b/src/jailer/src/env.rs
@@ -854,7 +854,7 @@ mod tests {
         arg_vec
     }
 
-    fn create_env(mock_proc_mounts: &str) -> Env {
+    fn create_env(mock_proc_mounts: &Path) -> Env {
         // Create a standard environment.
         let arg_parser = build_arg_parser();
         let mut args = arg_parser.arguments().clone();
@@ -862,7 +862,7 @@ mod tests {
         let pseudo_exec_file_path = get_pseudo_exec_file_path();
         args.parse(&make_args(&ArgVals::new(pseudo_exec_file_path.as_str())))
             .unwrap();
-        Env::new(&args, 0, 0, mock_proc_mounts).unwrap()
+        Env::new(&args, 0, 0, mock_proc_mounts.to_str().unwrap()).unwrap()
     }
 
     #[test]
@@ -876,7 +876,7 @@ mod tests {
         let mut args = arg_parser.arguments().clone();
         args.parse(&make_args(&good_arg_vals)).unwrap();
         // This should be fine.
-        let good_env = Env::new(&args, 0, 0, mock_cgroups.proc_mounts_path.as_str())
+        let good_env = Env::new(&args, 0, 0, mock_cgroups.proc_mounts_path.to_str().unwrap())
             .expect("This new environment should be created successfully.");
 
         let mut chroot_dir = PathBuf::from(good_arg_vals.chroot_base);
@@ -902,8 +902,9 @@ mod tests {
         let arg_parser = build_arg_parser();
         args = arg_parser.arguments().clone();
         args.parse(&make_args(&another_good_arg_vals)).unwrap();
-        let another_good_env = Env::new(&args, 0, 0, mock_cgroups.proc_mounts_path.as_str())
-            .expect("This another new environment should be created successfully.");
+        let another_good_env =
+            Env::new(&args, 0, 0, mock_cgroups.proc_mounts_path.to_str().unwrap())
+                .expect("This another new environment should be created successfully.");
         assert!(!another_good_env.daemonize);
         assert!(!another_good_env.new_pid_ns);
 
@@ -920,7 +921,7 @@ mod tests {
         let arg_parser = build_arg_parser();
         args = arg_parser.arguments().clone();
         args.parse(&make_args(&invalid_cgroup_arg_vals)).unwrap();
-        Env::new(&args, 0, 0, mock_cgroups.proc_mounts_path.as_str()).unwrap_err();
+        Env::new(&args, 0, 0, mock_cgroups.proc_mounts_path.to_str().unwrap()).unwrap_err();
 
         let invalid_res_limit_arg_vals = ArgVals {
             resource_limits: vec!["zzz"],
@@ -930,7 +931,7 @@ mod tests {
         let arg_parser = build_arg_parser();
         args = arg_parser.arguments().clone();
         args.parse(&make_args(&invalid_res_limit_arg_vals)).unwrap();
-        Env::new(&args, 0, 0, mock_cgroups.proc_mounts_path.as_str()).unwrap_err();
+        Env::new(&args, 0, 0, mock_cgroups.proc_mounts_path.to_str().unwrap()).unwrap_err();
 
         let invalid_id_arg_vals = ArgVals {
             id: "/ad./sa12",
@@ -940,7 +941,7 @@ mod tests {
         let arg_parser = build_arg_parser();
         args = arg_parser.arguments().clone();
         args.parse(&make_args(&invalid_id_arg_vals)).unwrap();
-        Env::new(&args, 0, 0, mock_cgroups.proc_mounts_path.as_str()).unwrap_err();
+        Env::new(&args, 0, 0, mock_cgroups.proc_mounts_path.to_str().unwrap()).unwrap_err();
 
         let inexistent_exec_file_arg_vals = ArgVals {
             exec_file: "/this!/file!/should!/not!/exist!/",
@@ -951,7 +952,7 @@ mod tests {
         args = arg_parser.arguments().clone();
         args.parse(&make_args(&inexistent_exec_file_arg_vals))
             .unwrap();
-        Env::new(&args, 0, 0, mock_cgroups.proc_mounts_path.as_str()).unwrap_err();
+        Env::new(&args, 0, 0, mock_cgroups.proc_mounts_path.to_str().unwrap()).unwrap_err();
 
         let invalid_uid_arg_vals = ArgVals {
             uid: "zzz",
@@ -961,7 +962,7 @@ mod tests {
         let arg_parser = build_arg_parser();
         args = arg_parser.arguments().clone();
         args.parse(&make_args(&invalid_uid_arg_vals)).unwrap();
-        Env::new(&args, 0, 0, mock_cgroups.proc_mounts_path.as_str()).unwrap_err();
+        Env::new(&args, 0, 0, mock_cgroups.proc_mounts_path.to_str().unwrap()).unwrap_err();
 
         let invalid_gid_arg_vals = ArgVals {
             gid: "zzz",
@@ -971,7 +972,7 @@ mod tests {
         let arg_parser = build_arg_parser();
         args = arg_parser.arguments().clone();
         args.parse(&make_args(&invalid_gid_arg_vals)).unwrap();
-        Env::new(&args, 0, 0, mock_cgroups.proc_mounts_path.as_str()).unwrap_err();
+        Env::new(&args, 0, 0, mock_cgroups.proc_mounts_path.to_str().unwrap()).unwrap_err();
 
         let invalid_parent_cg_vals = ArgVals {
             parent_cgroup: Some("/root"),
@@ -981,7 +982,7 @@ mod tests {
         let arg_parser = build_arg_parser();
         args = arg_parser.arguments().clone();
         args.parse(&make_args(&invalid_parent_cg_vals)).unwrap();
-        Env::new(&args, 0, 0, mock_cgroups.proc_mounts_path.as_str()).unwrap_err();
+        Env::new(&args, 0, 0, mock_cgroups.proc_mounts_path.to_str().unwrap()).unwrap_err();
 
         let invalid_controller_pt = ArgVals {
             cgroups: vec!["../file_name=1", "./root=1", "/home=1"],
@@ -990,7 +991,7 @@ mod tests {
         let arg_parser = build_arg_parser();
         args = arg_parser.arguments().clone();
         args.parse(&make_args(&invalid_controller_pt)).unwrap();
-        Env::new(&args, 0, 0, mock_cgroups.proc_mounts_path.as_str()).unwrap_err();
+        Env::new(&args, 0, 0, mock_cgroups.proc_mounts_path.to_str().unwrap()).unwrap_err();
 
         let invalid_format = ArgVals {
             cgroups: vec!["./root/", "../root"],
@@ -999,7 +1000,7 @@ mod tests {
         let arg_parser = build_arg_parser();
         args = arg_parser.arguments().clone();
         args.parse(&make_args(&invalid_format)).unwrap();
-        Env::new(&args, 0, 0, mock_cgroups.proc_mounts_path.as_str()).unwrap_err();
+        Env::new(&args, 0, 0, mock_cgroups.proc_mounts_path.to_str().unwrap()).unwrap_err();
 
         // The chroot-base-dir param is not validated by Env::new, but rather in run, when we
         // actually attempt to create the folder structure (the same goes for netns).
@@ -1065,7 +1066,7 @@ mod tests {
     fn test_setup_jailed_folder() {
         let mut mock_cgroups = MockCgroupFs::new().unwrap();
         mock_cgroups.add_v1_mounts().unwrap();
-        let env = create_env(mock_cgroups.proc_mounts_path.as_str());
+        let env = create_env(&mock_cgroups.proc_mounts_path);
 
         // Error case: non UTF-8 paths.
         let bad_string_bytes: Vec<u8> = vec![0, 102, 111, 111, 0]; // A leading nul followed by 'f', 'o', 'o'
@@ -1136,7 +1137,7 @@ mod tests {
     fn test_mknod_and_own_dev() {
         let mut mock_cgroups = MockCgroupFs::new().unwrap();
         mock_cgroups.add_v1_mounts().unwrap();
-        let env = create_env(mock_cgroups.proc_mounts_path.as_str());
+        let env = create_env(&mock_cgroups.proc_mounts_path);
 
         let mock_dev_dir = TempDir::new().unwrap();
 
@@ -1174,7 +1175,7 @@ mod tests {
     fn test_userfaultfd_dev() {
         let mut mock_cgroups = MockCgroupFs::new().unwrap();
         mock_cgroups.add_v1_mounts().unwrap();
-        let env = create_env(mock_cgroups.proc_mounts_path.as_str());
+        let env = create_env(&mock_cgroups.proc_mounts_path);
 
         if !Path::new(DEV_UFFD_PATH.to_str().unwrap()).exists() {
             assert_eq!(env.uffd_dev_minor, None);
@@ -1216,7 +1217,8 @@ mod tests {
         let exec_file_name = Path::new(&some_arg_vals.exec_file).file_name().unwrap();
         fs::write(some_arg_vals.exec_file, "some_content").unwrap();
         args.parse(&make_args(&some_arg_vals)).unwrap();
-        let mut env = Env::new(&args, 0, 0, mock_cgroups.proc_mounts_path.as_str()).unwrap();
+        let mut env =
+            Env::new(&args, 0, 0, mock_cgroups.proc_mounts_path.to_str().unwrap()).unwrap();
 
         // Create the required chroot dir hierarchy.
         fs::create_dir_all(env.chroot_dir()).expect("Could not create dir hierarchy.");
@@ -1279,7 +1281,7 @@ mod tests {
             ..good_arg_vals.clone()
         };
         args.parse(&make_args(&invalid_cgroup_arg_vals)).unwrap();
-        Env::new(&args, 0, 0, mock_cgroups.proc_mounts_path.as_str()).unwrap_err();
+        Env::new(&args, 0, 0, mock_cgroups.proc_mounts_path.to_str().unwrap()).unwrap_err();
 
         // Check empty string
         let mut args = arg_parser.arguments().clone();
@@ -1288,7 +1290,7 @@ mod tests {
             ..good_arg_vals.clone()
         };
         args.parse(&make_args(&invalid_cgroup_arg_vals)).unwrap();
-        Env::new(&args, 0, 0, mock_cgroups.proc_mounts_path.as_str()).unwrap_err();
+        Env::new(&args, 0, 0, mock_cgroups.proc_mounts_path.to_str().unwrap()).unwrap_err();
 
         // Check valid file empty value
         let mut args = arg_parser.arguments().clone();
@@ -1297,7 +1299,7 @@ mod tests {
             ..good_arg_vals.clone()
         };
         args.parse(&make_args(&invalid_cgroup_arg_vals)).unwrap();
-        Env::new(&args, 0, 0, mock_cgroups.proc_mounts_path.as_str()).unwrap_err();
+        Env::new(&args, 0, 0, mock_cgroups.proc_mounts_path.to_str().unwrap()).unwrap_err();
 
         // Check valid file no value
         let mut args = arg_parser.arguments().clone();
@@ -1306,7 +1308,7 @@ mod tests {
             ..good_arg_vals.clone()
         };
         args.parse(&make_args(&invalid_cgroup_arg_vals)).unwrap();
-        Env::new(&args, 0, 0, mock_cgroups.proc_mounts_path.as_str()).unwrap_err();
+        Env::new(&args, 0, 0, mock_cgroups.proc_mounts_path.to_str().unwrap()).unwrap_err();
 
         // Cases that should succeed
 
@@ -1317,7 +1319,7 @@ mod tests {
             ..good_arg_vals.clone()
         };
         args.parse(&make_args(&invalid_cgroup_arg_vals)).unwrap();
-        Env::new(&args, 0, 0, mock_cgroups.proc_mounts_path.as_str()).unwrap();
+        Env::new(&args, 0, 0, mock_cgroups.proc_mounts_path.to_str().unwrap()).unwrap();
 
         // Check valid case
         let mut args = arg_parser.arguments().clone();
@@ -1326,7 +1328,7 @@ mod tests {
             ..good_arg_vals.clone()
         };
         args.parse(&make_args(&invalid_cgroup_arg_vals)).unwrap();
-        Env::new(&args, 0, 0, mock_cgroups.proc_mounts_path.as_str()).unwrap();
+        Env::new(&args, 0, 0, mock_cgroups.proc_mounts_path.to_str().unwrap()).unwrap();
 
         // Check file with multiple "."
         let mut args = arg_parser.arguments().clone();
@@ -1335,7 +1337,7 @@ mod tests {
             ..good_arg_vals.clone()
         };
         args.parse(&make_args(&invalid_cgroup_arg_vals)).unwrap();
-        Env::new(&args, 0, 0, mock_cgroups.proc_mounts_path.as_str()).unwrap();
+        Env::new(&args, 0, 0, mock_cgroups.proc_mounts_path.to_str().unwrap()).unwrap();
     }
 
     #[test]
@@ -1409,7 +1411,7 @@ mod tests {
         let mut mock_cgroups = MockCgroupFs::new().unwrap();
         mock_cgroups.add_v1_mounts().unwrap();
 
-        let env = create_env(mock_cgroups.proc_mounts_path.as_str());
+        let env = create_env(&mock_cgroups.proc_mounts_path);
 
         // Create the required chroot dir hierarchy.
         fs::create_dir_all(env.chroot_dir()).expect("Could not create dir hierarchy.");
@@ -1436,7 +1438,7 @@ mod tests {
         let mut mock_cgroups = MockCgroupFs::new().unwrap();
         mock_cgroups.add_v1_mounts().unwrap();
 
-        let mut env = create_env(mock_cgroups.proc_mounts_path.as_str());
+        let mut env = create_env(&mock_cgroups.proc_mounts_path);
         env.save_exec_file_pid(pid, PathBuf::from(exec_file_name))
             .unwrap();
 

From 377b7fadce0b7c5fa9ef7af1bd1498900b4cd716 Mon Sep 17 00:00:00 2001
From: Riccardo Mancini <mancio@amazon.com>
Date: Tue, 29 Apr 2025 16:29:45 +0100
Subject: [PATCH 10/27] test(net): add retry in test_tap_offload

This test has been flaky for a while, where sometimes the file is empty.
As we're just interested that the message got delivered, not that the
file was created in a timely manner, I'm adding a small retry.

Signed-off-by: Riccardo Mancini <mancio@amazon.com>
---
 tests/integration_tests/functional/test_net.py | 12 ++++++++++--
 1 file changed, 10 insertions(+), 2 deletions(-)

diff --git a/tests/integration_tests/functional/test_net.py b/tests/integration_tests/functional/test_net.py
index 20a40e677b0..21fd11d924a 100644
--- a/tests/integration_tests/functional/test_net.py
+++ b/tests/integration_tests/functional/test_net.py
@@ -6,6 +6,7 @@
 import time
 
 import pytest
+from tenacity import Retrying, stop_after_attempt, wait_fixed
 
 from framework import utils
 
@@ -118,5 +119,12 @@ def test_tap_offload(uvm_any):
     vm.netns.check_output(f"python3 ./host_tools/udp_offload.py {vm.ssh.host} {port}")
 
     # Check that the server received the message
-    ret = vm.ssh.run(f"sync ; cat {out_filename}")
-    assert ret.stdout == message, f"{ret.stdout=} {ret.stderr=}"
+    # Allow for some delay due to the asynchronous nature of the test
+    for attempt in Retrying(
+        stop=stop_after_attempt(10),
+        wait=wait_fixed(0.1),
+        reraise=True,
+    ):
+        with attempt:
+            ret = vm.ssh.check_output(f"sync; cat {out_filename}")
+            assert ret.stdout == message, f"{ret.stdout=} {ret.stderr=}"

From 482b2ecd8e3f05e284400697adf1f8edb4d1b7ac Mon Sep 17 00:00:00 2001
From: Riccardo Mancini <mancio@amazon.com>
Date: Tue, 29 Apr 2025 16:34:11 +0100
Subject: [PATCH 11/27] test(balloon): ensure we gave stats enough time to
 update before reading

test_balloon_snapshot started being flaky after we changed the logic on
how we wait for the RSS to become stable.
One theory is that we are not waiting enough time for the stats to
refresh, so this change adds a sleep to ensure we have waited enough for
the stats to be "fresh".

Failure:
```
assert 189022208 > 189022208
```

Signed-off-by: Riccardo Mancini <mancio@amazon.com>
---
 tests/integration_tests/functional/test_balloon.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/tests/integration_tests/functional/test_balloon.py b/tests/integration_tests/functional/test_balloon.py
index dafa84b6451..2bfa82f4f1c 100644
--- a/tests/integration_tests/functional/test_balloon.py
+++ b/tests/integration_tests/functional/test_balloon.py
@@ -522,6 +522,8 @@ def test_balloon_snapshot(microvm_factory, guest_kernel, rootfs):
 
     # Get the stats after we take a snapshot and dirty some memory,
     # then reclaim it.
+    # Ensure we gave enough time for the stats to update.
+    time.sleep(STATS_POLLING_INTERVAL_S)
     latest_stats = microvm.api.balloon_stats.get().json()
 
     # Ensure the stats are still working after restore and show

From 421b1553075d07423d3e734550f4e79bd2bbe4ae Mon Sep 17 00:00:00 2001
From: Riccardo Mancini <mancio@amazon.com>
Date: Wed, 30 Apr 2025 16:36:14 +0100
Subject: [PATCH 12/27] fix(test/pvtime): bump tolerance to 10s

We found a single failure for which the steal time between snapshot
and restore went slightly above 2s on an AMD instance.
As the purpose of this check is to ensure the value is "sane" (iow not
a completely random number), not that it's really accurate (that's a
kernel problem), I'm bumping it to 10s.

Signed-off-by: Riccardo Mancini <mancio@amazon.com>
---
 tests/integration_tests/functional/test_steal_time.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/integration_tests/functional/test_steal_time.py b/tests/integration_tests/functional/test_steal_time.py
index 5c607ae4201..d7c3c25cb92 100644
--- a/tests/integration_tests/functional/test_steal_time.py
+++ b/tests/integration_tests/functional/test_steal_time.py
@@ -110,7 +110,7 @@ def test_pvtime_snapshot(uvm_plain, microvm_factory):
     steal_after_resume = get_steal_time_ms(restored_vm)
 
     # Ensure steal time persisted and continued increasing
-    tolerance = 2000  # 2.0 seconds tolerance for persistence check
+    tolerance = 10000  # 10.0 seconds tolerance for persistence check
     persisted = (
         steal_before < steal_after_snap and steal_after_snap - steal_before < tolerance
     )

From 080c8763779b4f45ed8d245111a5581a5aa9b979 Mon Sep 17 00:00:00 2001
From: Riccardo Mancini <mancio@amazon.com>
Date: Wed, 30 Apr 2025 16:53:53 +0100
Subject: [PATCH 13/27] fix(test_reboot): do not assert thread count

An upstream patch was backported to ubuntu 24.04 6.8.0-58 kernel that
makes the nx hugepages recover thread a child of the firecracker
process, thus increasing process count to 7.
As we're not really interested in knowing how many threads we have in
this test, let's remove the assertion altogether.

Signed-off-by: Riccardo Mancini <mancio@amazon.com>
---
 tests/integration_tests/functional/test_shut_down.py | 9 ---------
 1 file changed, 9 deletions(-)

diff --git a/tests/integration_tests/functional/test_shut_down.py b/tests/integration_tests/functional/test_shut_down.py
index 591f04e4593..4b21aa3d2d5 100644
--- a/tests/integration_tests/functional/test_shut_down.py
+++ b/tests/integration_tests/functional/test_shut_down.py
@@ -27,15 +27,6 @@ def test_reboot(uvm_plain_any):
     vm.add_net_iface()
     vm.start()
 
-    # Get Firecracker PID so we can count the number of threads.
-    firecracker_pid = vm.firecracker_pid
-
-    # Get number of threads in Firecracker
-    cmd = "ps -o nlwp {} | tail -1 | awk '{{print $1}}'".format(firecracker_pid)
-    _, stdout, _ = utils.check_output(cmd)
-    nr_of_threads = stdout.rstrip()
-    assert int(nr_of_threads) == 6
-
     # Consume existing metrics
     lines = vm.get_all_metrics()
     assert len(lines) == 1

From ee6a2ccceb5c802798179074631961fcb41e8836 Mon Sep 17 00:00:00 2001
From: Egor Lazarchuk <yegorlz@amazon.co.uk>
Date: Tue, 29 Apr 2025 15:03:19 +0100
Subject: [PATCH 14/27] feat(tests): add systemd_analyze data as boot time
 metric

Currently we measure boottime by looking at the Firecracker
logs and waiting for the guest to trigger a special boot device
inside Firecracker.
There is an alternative way of measuring boot time from
a guest perspective by using systemd-analyze. Will help us to understand
boot times better.

Signed-off-by: Egor Lazarchuk <yegorlz@amazon.co.uk>
---
 .../performance/test_boottime.py              | 37 +++++++++++++++++++
 1 file changed, 37 insertions(+)

diff --git a/tests/integration_tests/performance/test_boottime.py b/tests/integration_tests/performance/test_boottime.py
index 3bf74e3607a..c8a8d6bf4d0 100644
--- a/tests/integration_tests/performance/test_boottime.py
+++ b/tests/integration_tests/performance/test_boottime.py
@@ -69,6 +69,37 @@ def find_events(log_data):
     return timestamps
 
 
+def get_systemd_analyze_times(microvm):
+    """
+    Parse systemd-analyze output
+    """
+    rc, stdout, stderr = microvm.ssh.run("systemd-analyze")
+    assert rc == 0, stderr
+    assert stderr == ""
+
+    boot_line = stdout.splitlines()[0]
+    # The line will look like this:
+    # Startup finished in 79ms (kernel) + 231ms (userspace) = 310ms
+    # In the regex we capture the time and the unit for kernel, userspace and total values
+    pattern = r"Startup finished in (\d*)(ms|s)\s+\(kernel\) \+ (\d*)(ms|s)\s+\(userspace\) = ([\d.]*)(ms|s)\s*"
+    kernel, kernel_unit, userspace, userspace_unit, total, total_unit = re.findall(
+        pattern, boot_line
+    )[0]
+
+    def to_ms(v, unit):
+        match unit:
+            case "ms":
+                return float(v)
+            case "s":
+                return float(v) * 1000
+
+    kernel = to_ms(kernel, kernel_unit)
+    userspace = to_ms(userspace, userspace_unit)
+    total = to_ms(total, total_unit)
+
+    return kernel, userspace, total
+
+
 @pytest.mark.parametrize(
     "vcpu_count,mem_size_mib",
     [(1, 128), (1, 1024), (2, 2048), (4, 4096)],
@@ -112,4 +143,10 @@ def test_boottime(
             boottime_us - build_time.microseconds,
             unit="Microseconds",
         )
+
+        kernel, userspace, total = get_systemd_analyze_times(vm)
+        metrics.put_metric("systemd_kernel", kernel, unit="Milliseconds")
+        metrics.put_metric("systemd_userspace", userspace, unit="Milliseconds")
+        metrics.put_metric("systemd_total", total, unit="Milliseconds")
+
         vm.kill()

From 7d164e6643a1fa7fbf873bbd75e6ae0bc4a1bf9a Mon Sep 17 00:00:00 2001
From: Egor Lazarchuk <yegorlz@amazon.co.uk>
Date: Tue, 29 Apr 2025 15:51:22 +0100
Subject: [PATCH 15/27] refactor(tests): update boot time device metrics

Emit both boot time metrics for boot device: clock time and cpu time.
Additionally, stop subtracting VM build time from a guest boot time at
the metric emit time. This can be done later at the visualization stage.

Signed-off-by: Egor Lazarchuk <yegorlz@amazon.co.uk>
---
 .../performance/test_boottime.py              | 34 ++++++++++++-------
 1 file changed, 22 insertions(+), 12 deletions(-)

diff --git a/tests/integration_tests/performance/test_boottime.py b/tests/integration_tests/performance/test_boottime.py
index c8a8d6bf4d0..67ac087ca0e 100644
--- a/tests/integration_tests/performance/test_boottime.py
+++ b/tests/integration_tests/performance/test_boottime.py
@@ -11,7 +11,6 @@
 from framework.properties import global_props
 
 # Regex for obtaining boot time from some string.
-TIMESTAMP_LOG_REGEX = r"Guest-boot-time\s+\=\s+(\d+)\s+us"
 
 DEFAULT_BOOT_ARGS = (
     "reboot=k panic=1 pci=off nomodule 8250.nr_uarts=0"
@@ -27,27 +26,32 @@
 }
 
 
-def _get_microvm_boottime(vm):
+def get_boottime_device_info(vm):
     """Auxiliary function for asserting the expected boot time."""
     boot_time_us = None
+    boot_time_cpu_us = None
     timestamps = []
 
+    timestamp_log_regex = (
+        r"Guest-boot-time =\s+(\d+) us\s+(\d+) ms,\s+(\d+) CPU us\s+(\d+) CPU ms"
+    )
+
     iterations = 50
     sleep_time_s = 0.1
     for _ in range(iterations):
-        timestamps = re.findall(TIMESTAMP_LOG_REGEX, vm.log_data)
+        timestamps = re.findall(timestamp_log_regex, vm.log_data)
         if timestamps:
             break
         time.sleep(sleep_time_s)
     if timestamps:
-        boot_time_us = int(timestamps[0])
+        boot_time_us, _, boot_time_cpu_us, _ = timestamps[0]
 
-    assert boot_time_us, (
+    assert boot_time_us and boot_time_cpu_us, (
         f"MicroVM did not boot within {sleep_time_s * iterations}s\n"
         f"Firecracker logs:\n{vm.log_data}\n"
         f"Thread backtraces:\n{vm.thread_backtraces}"
     )
-    return boot_time_us
+    return int(boot_time_us), int(boot_time_cpu_us)
 
 
 def find_events(log_data):
@@ -133,16 +137,22 @@ def test_boottime(
         vm.add_net_iface()
         vm.start()
         vm.pin_threads(0)
-        boottime_us = _get_microvm_boottime(vm)
-        metrics.put_metric("boot_time", boottime_us, unit="Microseconds")
-        timestamps = find_events(vm.log_data)
-        build_time = timestamps["build microvm for boot"]["duration"]
-        metrics.put_metric("build_time", build_time.microseconds, unit="Microseconds")
+
+        boot_time_us, cpu_boot_time_us = get_boottime_device_info(vm)
         metrics.put_metric(
             "guest_boot_time",
-            boottime_us - build_time.microseconds,
+            boot_time_us,
             unit="Microseconds",
         )
+        metrics.put_metric(
+            "guest_cpu_boot_time",
+            cpu_boot_time_us,
+            unit="Microseconds",
+        )
+
+        timestamps = find_events(vm.log_data)
+        build_time = timestamps["build microvm for boot"]["duration"]
+        metrics.put_metric("build_time", build_time.microseconds, unit="Microseconds")
 
         kernel, userspace, total = get_systemd_analyze_times(vm)
         metrics.put_metric("systemd_kernel", kernel, unit="Milliseconds")

From 65b8b33768091bc3419893caf7189c5d32a527ba Mon Sep 17 00:00:00 2001
From: Egor Lazarchuk <yegorlz@amazon.co.uk>
Date: Tue, 29 Apr 2025 16:04:55 +0100
Subject: [PATCH 16/27] feat(tests): add resume metric to boot time tests

Since we already collect this delta, put it into
metrics as well.

Signed-off-by: Egor Lazarchuk <yegorlz@amazon.co.uk>
---
 tests/integration_tests/performance/test_boottime.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/tests/integration_tests/performance/test_boottime.py b/tests/integration_tests/performance/test_boottime.py
index 67ac087ca0e..5732a7e56f8 100644
--- a/tests/integration_tests/performance/test_boottime.py
+++ b/tests/integration_tests/performance/test_boottime.py
@@ -150,9 +150,11 @@ def test_boottime(
             unit="Microseconds",
         )
 
-        timestamps = find_events(vm.log_data)
-        build_time = timestamps["build microvm for boot"]["duration"]
+        events = find_events(vm.log_data)
+        build_time = events["build microvm for boot"]["duration"]
         metrics.put_metric("build_time", build_time.microseconds, unit="Microseconds")
+        resume_time = events["boot microvm"]["duration"]
+        metrics.put_metric("resume_time", resume_time.microseconds, unit="Microseconds")
 
         kernel, userspace, total = get_systemd_analyze_times(vm)
         metrics.put_metric("systemd_kernel", kernel, unit="Milliseconds")

From 949ac4c6c403479f0f29e89abfe3702ad5e42738 Mon Sep 17 00:00:00 2001
From: Egor Lazarchuk <yegorlz@amazon.co.uk>
Date: Wed, 30 Apr 2025 11:03:47 +0100
Subject: [PATCH 17/27] chore: ignore boot test resume_time metric in A/B

This metrics has very small numbers which makes it too
volatile for A/B tests.

Signed-off-by: Egor Lazarchuk <yegorlz@amazon.co.uk>
---
 tools/ab_test.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/tools/ab_test.py b/tools/ab_test.py
index 3790ff4674c..063f795072a 100755
--- a/tools/ab_test.py
+++ b/tools/ab_test.py
@@ -49,6 +49,8 @@
     # block latencies if guest uses async request submission
     {"fio_engine": "libaio", "metric": "clat_read"},
     {"fio_engine": "libaio", "metric": "clat_write"},
+    # boot time metrics
+    {"performance_test": "test_boottime", "metric": "resume_time"},
 ]
 
 

From 783ca67945ceb7b55c7a11bde616f93708f3e1d4 Mon Sep 17 00:00:00 2001
From: Patrick Roy <roypat@amazon.co.uk>
Date: Fri, 2 May 2025 12:25:21 +0100
Subject: [PATCH 18/27] doc: fix arguments to ab_test.py in README

It's --pytest-opts, not --test anymore.

Fixes: 316d9554da08 ("test(ab): generalize --test to --pytest-opts")
Signed-off-by: Patrick Roy <roypat@amazon.co.uk>
---
 tests/README.md | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/tests/README.md b/tests/README.md
index c306566392f..677544889ad 100644
--- a/tests/README.md
+++ b/tests/README.md
@@ -184,7 +184,7 @@ function to [`.buildkite/pipeline_perf.py`](../.buildkite/pipeline_perf.py). To
 manually run an A/B-Test, use
 
 ```sh
-tools/devtool -y test --ab [optional arguments to ab_test.py] run <dir A> <dir B> --test <test specification>
+tools/devtool -y test --ab [optional arguments to ab_test.py] run <dir A> <dir B> --pytest-opts <test specification>
 ```
 
 Here, _dir A_ and _dir B_ are directories containing firecracker and jailer
@@ -198,7 +198,7 @@ branch and the `HEAD` of your current branch, run
 ```sh
 tools/devtool -y build --rev main --release
 tools/devtool -y build --rev HEAD --release
-tools/devtool -y test --no-build --ab -- run build/main build/HEAD --test integration_tests/performance/test_boottime.py::test_boottime
+tools/devtool -y test --no-build --ab -- run build/main build/HEAD --pytest-opts integration_tests/performance/test_boottime.py::test_boottime
 ```
 
 #### How to Write an A/B-Compatible Test and Common Pitfalls
@@ -213,9 +213,9 @@ dimension to match up data series between two test runs. It only matches up two
 data series with the same name if their dimensions match.
 
 Special care needs to be taken when pytest expands the argument passed to
-`tools/ab_test.py`'s `--test` option into multiple individual test cases. If two
-test cases use the same dimensions for different data series, the script will
-fail and print out the names of the violating data series. For this reason,
+`tools/ab_test.py`'s `--pytest-opts` option into multiple individual test cases.
+If two test cases use the same dimensions for different data series, the script
+will fail and print out the names of the violating data series. For this reason,
 **A/B-Compatible tests should include a `performance_test` key in their
 dimension set whose value is set to the name of the test**.
 

From 29517f6bd630313f14a10e5cf7d40d532b928b74 Mon Sep 17 00:00:00 2001
From: Patrick Roy <roypat@amazon.co.uk>
Date: Fri, 2 May 2025 12:26:26 +0100
Subject: [PATCH 19/27] fix(ab): only reduce dimension set for printing

By reducing the dimension set to eliminate all dimensions that only
every take on a single value straight during parsing, we broke the
ignore list, which does require all dimensions to be present. Fix this
by moving the "dimension reduction" to only happen during printing of
failure messages.

Fixes: fcb39a68b19c ("test(ab): do not print dimension that are the same across all metrics")
Signed-off-by: Patrick Roy <roypat@amazon.co.uk>
---
 tools/ab_test.py | 37 ++++++++++++++++++-------------------
 1 file changed, 18 insertions(+), 19 deletions(-)

diff --git a/tools/ab_test.py b/tools/ab_test.py
index 063f795072a..10857a35480 100755
--- a/tools/ab_test.py
+++ b/tools/ab_test.py
@@ -119,8 +119,6 @@ def load_data_series(report_path: Path, tag=None, *, reemit: bool = False):
     # Dictionary mapping EMF dimensions to A/B-testable metrics/properties
     processed_emf = {}
 
-    distinct_values_per_dimenson = defaultdict(set)
-
     report = json.loads(report_path.read_text("UTF-8"))
     for test in report["tests"]:
         for line in test["teardown"]["stdout"].splitlines():
@@ -140,9 +138,6 @@ def load_data_series(report_path: Path, tag=None, *, reemit: bool = False):
                 if not dimensions:
                     continue
 
-                for dimension, value in dimensions.items():
-                    distinct_values_per_dimenson[dimension].add(value)
-
                 dimension_set = frozenset(dimensions.items())
 
                 if dimension_set not in processed_emf:
@@ -159,24 +154,27 @@ def load_data_series(report_path: Path, tag=None, *, reemit: bool = False):
 
                         values.extend(result[metric][0])
 
-    irrelevant_dimensions = set()
+    return processed_emf
 
-    for dimension, distinct_values in distinct_values_per_dimenson.items():
-        if len(distinct_values) == 1:
-            irrelevant_dimensions.add(dimension)
 
-    post_processed_emf = {}
+def uninteresting_dimensions(processed_emf):
+    """
+    Computes the set of cloudwatch dimensions that only ever take on a
+    single value across the entire dataset.
+    """
+    values_per_dimension = defaultdict(set)
+
+    for dimension_set in processed_emf:
+        for dimension, value in dimension_set:
+            values_per_dimension[dimension].add(value)
 
-    for dimension_set, metrics in processed_emf.items():
-        processed_key = frozenset(
-            (dim, value)
-            for (dim, value) in dimension_set
-            if dim not in irrelevant_dimensions
-        )
+    uninteresting = set()
 
-        post_processed_emf[processed_key] = metrics
+    for dimension, distinct_values in values_per_dimension.items():
+        if len(distinct_values) == 1:
+            uninteresting.add(dimension)
 
-    return post_processed_emf
+    return uninteresting
 
 
 def collect_data(binary_dir: Path, pytest_opts: str):
@@ -304,6 +302,7 @@ def analyze_data(
             )
 
     messages = []
+    do_not_print_list = uninteresting_dimensions(processed_emf_a)
     for dimension_set, metric, result, unit in failures:
         # Sanity check as described above
         if abs(statistics.mean(relative_changes_by_metric[metric])) <= noise_threshold:
@@ -325,7 +324,7 @@ def analyze_data(
                 f"for metric \033[1m{metric}\033[0m with \033[0;31m\033[1mp={result.pvalue}\033[0m. "
                 f"This means that observing a change of this magnitude or worse, assuming that performance "
                 f"characteristics did not change across the tested commits, has a probability of {result.pvalue:.2%}. "
-                f"Tested Dimensions:\n{json.dumps(dict(dimension_set), indent=2, sort_keys=True)}"
+                f"Tested Dimensions:\n{json.dumps(dict({k: v for k,v in dimension_set.items() if k not in do_not_print_list}), indent=2, sort_keys=True)}"
             )
             messages.append(msg)
 

From 3a990f2600f029e1445b89ac68aee1fabb86a485 Mon Sep 17 00:00:00 2001
From: Patrick Roy <roypat@amazon.co.uk>
Date: Fri, 2 May 2025 12:29:16 +0100
Subject: [PATCH 20/27] ab: ignore some block throughput metrics on m8g

block throughput metrics on m8g.metal instances for test scenarios using
the async fio engine and more than 1 vcpu are volatile, so exclude them
from A/B-testing.

Suggested-by: Riccardo Mancini <mancio@amazon.com>
Signed-off-by: Patrick Roy <roypat@amazon.co.uk>
---
 tools/ab_test.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/tools/ab_test.py b/tools/ab_test.py
index 10857a35480..2ad3037e86d 100755
--- a/tools/ab_test.py
+++ b/tools/ab_test.py
@@ -51,6 +51,9 @@
     {"fio_engine": "libaio", "metric": "clat_write"},
     # boot time metrics
     {"performance_test": "test_boottime", "metric": "resume_time"},
+    # block throughput on m8g
+    {"fio_engine": "libaio", "vcpus": 2, "instance": "m8g.metal-24xl"},
+    {"fio_engine": "libaio", "vcpus": 2, "instance": "m8g.metal-48xl"},
 ]
 
 

From 7579beb91526988774b62629f8ece3c749b4f97c Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 5 May 2025 16:14:05 +0000
Subject: [PATCH 21/27] build(deps): Bump the firecracker group with 9 updates

Bumps the firecracker group with 9 updates:

| Package | From | To |
| --- | --- | --- |
| [aws-lc-fips-sys](https://github.com/aws/aws-lc-rs) | `0.13.5` | `0.13.6` |
| [cc](https://github.com/rust-lang/cc-rs) | `1.2.20` | `1.2.21` |
| [hashbrown](https://github.com/rust-lang/hashbrown) | `0.15.2` | `0.15.3` |
| [jiff](https://github.com/BurntSushi/jiff) | `0.2.10` | `0.2.12` |
| [jiff-static](https://github.com/BurntSushi/jiff) | `0.2.10` | `0.2.12` |
| [toml](https://github.com/toml-rs/toml) | `0.8.21` | `0.8.22` |
| [toml_edit](https://github.com/toml-rs/toml) | `0.22.25` | `0.22.26` |
| [toml_write](https://github.com/toml-rs/toml) | `0.1.0` | `0.1.1` |
| [winnow](https://github.com/winnow-rs/winnow) | `0.7.7` | `0.7.9` |


Updates `aws-lc-fips-sys` from 0.13.5 to 0.13.6
- [Release notes](https://github.com/aws/aws-lc-rs/releases)
- [Commits](https://github.com/aws/aws-lc-rs/compare/aws-lc-fips-sys/v0.13.5...aws-lc-fips-sys/v0.13.6)

Updates `cc` from 1.2.20 to 1.2.21
- [Release notes](https://github.com/rust-lang/cc-rs/releases)
- [Changelog](https://github.com/rust-lang/cc-rs/blob/main/CHANGELOG.md)
- [Commits](https://github.com/rust-lang/cc-rs/compare/cc-v1.2.20...cc-v1.2.21)

Updates `hashbrown` from 0.15.2 to 0.15.3
- [Release notes](https://github.com/rust-lang/hashbrown/releases)
- [Changelog](https://github.com/rust-lang/hashbrown/blob/master/CHANGELOG.md)
- [Commits](https://github.com/rust-lang/hashbrown/commits/v0.15.3)

Updates `jiff` from 0.2.10 to 0.2.12
- [Release notes](https://github.com/BurntSushi/jiff/releases)
- [Changelog](https://github.com/BurntSushi/jiff/blob/master/CHANGELOG.md)
- [Commits](https://github.com/BurntSushi/jiff/compare/jiff-static-0.2.10...jiff-static-0.2.12)

Updates `jiff-static` from 0.2.10 to 0.2.12
- [Release notes](https://github.com/BurntSushi/jiff/releases)
- [Changelog](https://github.com/BurntSushi/jiff/blob/master/CHANGELOG.md)
- [Commits](https://github.com/BurntSushi/jiff/compare/jiff-static-0.2.10...jiff-static-0.2.12)

Updates `toml` from 0.8.21 to 0.8.22
- [Commits](https://github.com/toml-rs/toml/compare/toml-v0.8.21...toml-v0.8.22)

Updates `toml_edit` from 0.22.25 to 0.22.26
- [Commits](https://github.com/toml-rs/toml/compare/v0.22.25...v0.22.26)

Updates `toml_write` from 0.1.0 to 0.1.1
- [Commits](https://github.com/toml-rs/toml/compare/toml_write-v0.1.0...toml_write-v0.1.1)

Updates `winnow` from 0.7.7 to 0.7.9
- [Changelog](https://github.com/winnow-rs/winnow/blob/main/CHANGELOG.md)
- [Commits](https://github.com/winnow-rs/winnow/compare/v0.7.7...v0.7.9)

---
updated-dependencies:
- dependency-name: aws-lc-fips-sys
  dependency-version: 0.13.6
  dependency-type: indirect
  update-type: version-update:semver-patch
  dependency-group: firecracker
- dependency-name: cc
  dependency-version: 1.2.21
  dependency-type: indirect
  update-type: version-update:semver-patch
  dependency-group: firecracker
- dependency-name: hashbrown
  dependency-version: 0.15.3
  dependency-type: indirect
  update-type: version-update:semver-patch
  dependency-group: firecracker
- dependency-name: jiff
  dependency-version: 0.2.12
  dependency-type: indirect
  update-type: version-update:semver-patch
  dependency-group: firecracker
- dependency-name: jiff-static
  dependency-version: 0.2.12
  dependency-type: indirect
  update-type: version-update:semver-patch
  dependency-group: firecracker
- dependency-name: toml
  dependency-version: 0.8.22
  dependency-type: indirect
  update-type: version-update:semver-patch
  dependency-group: firecracker
- dependency-name: toml_edit
  dependency-version: 0.22.26
  dependency-type: indirect
  update-type: version-update:semver-patch
  dependency-group: firecracker
- dependency-name: toml_write
  dependency-version: 0.1.1
  dependency-type: indirect
  update-type: version-update:semver-patch
  dependency-group: firecracker
- dependency-name: winnow
  dependency-version: 0.7.9
  dependency-type: indirect
  update-type: version-update:semver-patch
  dependency-group: firecracker
...

Signed-off-by: dependabot[bot] <support@github.com>
---
 Cargo.lock | 36 ++++++++++++++++++------------------
 1 file changed, 18 insertions(+), 18 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index 5a87079b2da..14bf01ee115 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -126,9 +126,9 @@ checksum = "ace50bade8e6234aa140d9a2f552bbee1db4d353f69b8217bc503490fc1a9f26"
 
 [[package]]
 name = "aws-lc-fips-sys"
-version = "0.13.5"
+version = "0.13.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2d9c2e952a1f57e8cbc78b058a968639e70c4ce8b9c0a5e6363d4e5670eed795"
+checksum = "e99d74bb793a19f542ae870a6edafbc5ecf0bc0ba01d4636b7f7e0aba9ee9bd3"
 dependencies = [
  "bindgen 0.69.5",
  "cc",
@@ -268,9 +268,9 @@ checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5"
 
 [[package]]
 name = "cc"
-version = "1.2.20"
+version = "1.2.21"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "04da6a0d40b948dfc4fa8f5bbf402b0fc1a64a28dbf7d12ffd683550f2c1b63a"
+checksum = "8691782945451c1c383942c4874dbe63814f61cb57ef773cda2972682b7bb3c0"
 dependencies = [
  "jobserver",
  "libc",
@@ -719,9 +719,9 @@ dependencies = [
 
 [[package]]
 name = "hashbrown"
-version = "0.15.2"
+version = "0.15.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bf151400ff0baff5465007dd2f3e717f3fe502074ca563069ce3a6629d07b289"
+checksum = "84b26c544d002229e640969970a2e74021aadf6e2f96372b9c58eff97de08eb3"
 
 [[package]]
 name = "heck"
@@ -818,9 +818,9 @@ dependencies = [
 
 [[package]]
 name = "jiff"
-version = "0.2.10"
+version = "0.2.12"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5a064218214dc6a10fbae5ec5fa888d80c45d611aba169222fc272072bf7aef6"
+checksum = "d07d8d955d798e7a4d6f9c58cd1f1916e790b42b092758a9ef6e16fef9f1b3fd"
 dependencies = [
  "jiff-static",
  "log",
@@ -831,9 +831,9 @@ dependencies = [
 
 [[package]]
 name = "jiff-static"
-version = "0.2.10"
+version = "0.2.12"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "199b7932d97e325aff3a7030e141eafe7f2c6268e1d1b24859b753a627f45254"
+checksum = "f244cfe006d98d26f859c7abd1318d85327e1882dc9cef80f62daeeb0adcf300"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -1442,9 +1442,9 @@ dependencies = [
 
 [[package]]
 name = "toml"
-version = "0.8.21"
+version = "0.8.22"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "900f6c86a685850b1bc9f6223b20125115ee3f31e01207d81655bbcc0aea9231"
+checksum = "05ae329d1f08c4d17a59bed7ff5b5a769d062e64a62d34a3261b219e62cd5aae"
 dependencies = [
  "serde",
  "serde_spanned",
@@ -1463,9 +1463,9 @@ dependencies = [
 
 [[package]]
 name = "toml_edit"
-version = "0.22.25"
+version = "0.22.26"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "10558ed0bd2a1562e630926a2d1f0b98c827da99fabd3fe20920a59642504485"
+checksum = "310068873db2c5b3e7659d2cc35d21855dbafa50d1ce336397c666e3cb08137e"
 dependencies = [
  "indexmap",
  "serde",
@@ -1477,9 +1477,9 @@ dependencies = [
 
 [[package]]
 name = "toml_write"
-version = "0.1.0"
+version = "0.1.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "28391a4201ba7eb1984cfeb6862c0b3ea2cfe23332298967c749dddc0d6cd976"
+checksum = "bfb942dfe1d8e29a7ee7fcbde5bd2b9a25fb89aa70caea2eba3bee836ff41076"
 
 [[package]]
 name = "typenum"
@@ -1849,9 +1849,9 @@ checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec"
 
 [[package]]
 name = "winnow"
-version = "0.7.7"
+version = "0.7.9"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6cb8234a863ea0e8cd7284fcdd4f145233eb00fee02bbdd9861aec44e6477bc5"
+checksum = "d9fb597c990f03753e08d3c29efbfcf2019a003b4bf4ba19225c158e1549f0f3"
 dependencies = [
  "memchr",
 ]

From 61a93c5bf18d1f1e09396a5edfb19d33f14b58e9 Mon Sep 17 00:00:00 2001
From: Patrick Roy <roypat@amazon.co.uk>
Date: Tue, 29 Apr 2025 14:34:37 +0100
Subject: [PATCH 22/27] allow specifying custom cpu template inline in config
 json

When starting firecracker without the API server and configuring it
through a config files, specifying a custom cpu template requires
writing this cpu template into a separate json file, and then passing
the path to this separate json file in the config json file. This is a
bit silly, since everything is just json. So additionally allow just
directly specifying the custom cpu template inside the config file.

Signed-off-by: Patrick Roy <roypat@amazon.co.uk>
---
 CHANGELOG.md             |  3 ++
 src/vmm/src/resources.rs | 64 ++++++++++++++++++++++++++++++++++++----
 2 files changed, 61 insertions(+), 6 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index c05641e1780..37af6a60291 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -24,6 +24,9 @@ and this project adheres to
   so users need to regenerate snapshots.
 - [#4731](https://github.com/firecracker-microvm/firecracker/pull/4731): Added
   support for modifying the host TAP device name during snapshot restore.
+- [#5175](https://github.com/firecracker-microvm/firecracker/pull/5175): Allow
+  including a custom cpu template directly in the json configuration file passed
+  to `--config-file` under the `cpu_config` key.
 
 ### Changed
 
diff --git a/src/vmm/src/resources.rs b/src/vmm/src/resources.rs
index 097e2041b55..70c317bb1e1 100644
--- a/src/vmm/src/resources.rs
+++ b/src/vmm/src/resources.rs
@@ -63,6 +63,13 @@ pub enum ResourcesError {
     EntropyDevice(#[from] EntropyDeviceError),
 }
 
+#[derive(Serialize, Deserialize, PartialEq, Eq, Debug)]
+#[serde(untagged)]
+enum CustomCpuTemplateOrPath {
+    Path(PathBuf),
+    Template(CustomCpuTemplate),
+}
+
 /// Used for configuring a vmm from one single json passed to the Firecracker process.
 #[derive(Debug, Default, PartialEq, Eq, Deserialize, Serialize)]
 #[serde(rename_all = "kebab-case")]
@@ -70,7 +77,7 @@ pub struct VmmConfig {
     balloon: Option<BalloonDeviceConfig>,
     drives: Vec<BlockDeviceConfig>,
     boot_source: BootSourceConfig,
-    cpu_config: Option<PathBuf>,
+    cpu_config: Option<CustomCpuTemplateOrPath>,
     logger: Option<crate::logger::LoggerConfig>,
     machine_config: Option<MachineConfig>,
     metrics: Option<MetricsConfig>,
@@ -136,11 +143,18 @@ impl VmResources {
             resources.update_machine_config(&machine_config)?;
         }
 
-        if let Some(cpu_config) = vmm_config.cpu_config {
-            let cpu_config_json =
-                std::fs::read_to_string(cpu_config).map_err(ResourcesError::File)?;
-            let cpu_template = CustomCpuTemplate::try_from(cpu_config_json.as_str())?;
-            resources.set_custom_cpu_template(cpu_template);
+        if let Some(either) = vmm_config.cpu_config {
+            match either {
+                CustomCpuTemplateOrPath::Path(path) => {
+                    let cpu_config_json =
+                        std::fs::read_to_string(path).map_err(ResourcesError::File)?;
+                    let cpu_template = CustomCpuTemplate::try_from(cpu_config_json.as_str())?;
+                    resources.set_custom_cpu_template(cpu_template);
+                }
+                CustomCpuTemplateOrPath::Template(template) => {
+                    resources.set_custom_cpu_template(template)
+                }
+            }
         }
 
         resources.build_boot_source(vmm_config.boot_source)?;
@@ -505,6 +519,7 @@ mod tests {
 
     use super::*;
     use crate::HTTP_MAX_PAYLOAD_SIZE;
+    use crate::cpu_config::templates::test_utils::TEST_TEMPLATE_JSON;
     use crate::cpu_config::templates::{CpuTemplateType, StaticCpuTemplate};
     use crate::devices::virtio::balloon::Balloon;
     use crate::devices::virtio::block::virtio::VirtioBlockError;
@@ -1051,6 +1066,43 @@ mod tests {
         assert!(matches!(error, ResourcesError::File(_)), "{:?}", error);
     }
 
+    #[test]
+    fn test_cpu_config_inline() {
+        // Include custom cpu template directly inline in config json
+        let kernel_file = TempFile::new().unwrap();
+        let rootfs_file = TempFile::new().unwrap();
+        let default_instance_info = InstanceInfo::default();
+
+        let json = format!(
+            r#"{{
+                    "boot-source": {{
+                        "kernel_image_path": "{}",
+                        "boot_args": "console=ttyS0 reboot=k panic=1 pci=off"
+                    }},
+                    "cpu-config": {},
+                    "drives": [
+                        {{
+                            "drive_id": "rootfs",
+                            "path_on_host": "{}",
+                            "is_root_device": true,
+                            "is_read_only": false
+                        }}
+                    ]
+            }}"#,
+            kernel_file.as_path().to_str().unwrap(),
+            TEST_TEMPLATE_JSON,
+            rootfs_file.as_path().to_str().unwrap(),
+        );
+
+        VmResources::from_json(
+            json.as_str(),
+            &default_instance_info,
+            HTTP_MAX_PAYLOAD_SIZE,
+            None,
+        )
+        .unwrap();
+    }
+
     #[test]
     fn test_cpu_config_from_valid_json() {
         // Valid cpu config file path.

From 3582c78c489db6952a6ba20ac3ebc1ce453b5d0f Mon Sep 17 00:00:00 2001
From: Takahiro Itazuri <itazur@amazon.com>
Date: Tue, 6 May 2025 12:55:01 +0000
Subject: [PATCH 23/27] chore: Update CHANGELOG in preparation of 1.12.0
 release

Creates a new [Unreleased] section and move changes to be included in
v1.12.0 to [1.12.0] section.

Signed-off-by: Takahiro Itazuri <itazur@amazon.com>
---
 CHANGELOG.md | 32 ++++++++++++++++++++++----------
 1 file changed, 22 insertions(+), 10 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 37af6a60291..df6d33805fd 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -13,6 +13,26 @@ and this project adheres to
 - [#5139](https://github.com/firecracker-microvm/firecracker/pull/5139): Added
   support for [PVTime](https://docs.kernel.org/virt/kvm/arm/pvtime.html). This
   is used to support steal time on ARM machines.
+- [#5175](https://github.com/firecracker-microvm/firecracker/pull/5175): Allow
+  including a custom cpu template directly in the json configuration file passed
+  to `--config-file` under the `cpu_config` key.
+
+### Changed
+
+- [#5165](https://github.com/firecracker-microvm/firecracker/pull/5165): Changed
+  Firecracker snapshot feature from developer preview to generally available.
+  Incremental snapshots remain in developer preview.
+
+### Deprecated
+
+### Removed
+
+### Fixed
+
+## [1.12.0]
+
+### Added
+
 - [#5048](https://github.com/firecracker-microvm/firecracker/pull/5048): Added
   support for [PVH boot mode](docs/pvh.md). This is used when an x86 kernel
   provides the appropriate ELF Note to indicate that PVH boot mode is supported.
@@ -24,9 +44,6 @@ and this project adheres to
   so users need to regenerate snapshots.
 - [#4731](https://github.com/firecracker-microvm/firecracker/pull/4731): Added
   support for modifying the host TAP device name during snapshot restore.
-- [#5175](https://github.com/firecracker-microvm/firecracker/pull/5175): Allow
-  including a custom cpu template directly in the json configuration file passed
-  to `--config-file` under the `cpu_config` key.
 
 ### Changed
 
@@ -38,9 +55,6 @@ and this project adheres to
   Clarified what CPU models are supported by each existing CPU template.
   Firecracker exits with an error if a CPU template is used on an unsupported
   CPU model.
-- [#5165](https://github.com/firecracker-microvm/firecracker/pull/5165): Changed
-  Firecracker snapshot feature from developer preview to generally available.
-  Incremental snapshots remain in developer preview.
 
 ### Deprecated
 
@@ -51,12 +65,10 @@ and this project adheres to
   misnamed, as the value Firecracker sets it to is actually the page size in
   _bytes_, not KiB. It will be removed in Firecracker 2.0.
 
-### Removed
-
 ### Fixed
 
-- #\[[5074](https://github.com/firecracker-microvm/firecracker/pull/5074)\] Fix
-  the `SendCtrlAltDel` command not working for ACPI-enabled guest kernels, by
+- [#5074](https://github.com/firecracker-microvm/firecracker/pull/5074) Fix the
+  `SendCtrlAltDel` command not working for ACPI-enabled guest kernels, by
   dropping the i8042.nopnp argument from the default kernel command line
   Firecracker constructs.
 - [#5122](https://github.com/firecracker-microvm/firecracker/pull/5122): Keep

From 85099fe253f782184fb9dfaf4502028186187627 Mon Sep 17 00:00:00 2001
From: Takahiro Itazuri <itazur@amazon.com>
Date: Tue, 6 May 2025 12:58:47 +0000
Subject: [PATCH 24/27] chore: Update release status in preparation of v1.12.0
 release

Adds v1.12 to the release status table and marks v1.10 as unsupported.

Signed-off-by: Takahiro Itazuri <itazur@amazon.com>
---
 docs/RELEASE_POLICY.md | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/docs/RELEASE_POLICY.md b/docs/RELEASE_POLICY.md
index c9bd2a51089..f8f7006b721 100644
--- a/docs/RELEASE_POLICY.md
+++ b/docs/RELEASE_POLICY.md
@@ -90,8 +90,9 @@ v3.1 will be patched since were the last two Firecracker releases and less than
 
 | Release | Release Date | Latest Patch | Min. end of support | Official end of Support         |
 | ------: | -----------: | -----------: | ------------------: | :------------------------------ |
+|   v1.12 |   2025-05-07 |      v1.12.0 |          2025-11-07 | Supported                       |
 |   v1.11 |   2025-03-18 |      v1.11.0 |          2025-09-18 | Supported                       |
-|   v1.10 |   2024-11-07 |      v1.10.1 |          2025-05-07 | Supported                       |
+|   v1.10 |   2024-11-07 |      v1.10.1 |          2025-05-07 | 2025-05-07 (v1.12 released)     |
 |    v1.9 |   2024-09-02 |       v1.9.1 |          2025-03-02 | 2025-03-18 (v1.11 released)     |
 |    v1.8 |   2024-07-10 |       v1.8.0 |          2025-01-10 | 2025-01-10 (end of 6mo support) |
 |    v1.7 |   2024-03-18 |       v1.7.0 |          2024-09-18 | 2024-09-18 (end of 6mo support) |

From 3827acda4749e6235d21e41964d5679de39b3ae6 Mon Sep 17 00:00:00 2001
From: Takahiro Itazuri <itazur@amazon.com>
Date: Tue, 6 May 2025 12:59:58 +0000
Subject: [PATCH 25/27] chore: Add CHANGELOG entries for newly supported
 instance types

Intel Sapphire Rapids and ARM Graviton4 are not supported.

Signed-off-by: Takahiro Itazuri <itazur@amazon.com>
---
 CHANGELOG.md | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index df6d33805fd..7416664a80f 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -44,6 +44,10 @@ and this project adheres to
   so users need to regenerate snapshots.
 - [#4731](https://github.com/firecracker-microvm/firecracker/pull/4731): Added
   support for modifying the host TAP device name during snapshot restore.
+- [#5146](https://github.com/firecracker-microvm/firecracker/pull/5146): Added
+  Intel Sapphire Rapids as a supported and tested platform for Firecracker.
+- [#5148](https://github.com/firecracker-microvm/firecracker/pull/5148): Added
+  ARM Graviton4 as a supported and tested platform for Firecracker.
 
 ### Changed
 

From bacff13ec4a545abde4bcb88576efc0857138447 Mon Sep 17 00:00:00 2001
From: Takahiro Itazuri <itazur@amazon.com>
Date: Tue, 6 May 2025 13:01:03 +0000
Subject: [PATCH 26/27] chore: Bump Rust dependencies

Runs `cargo update`.

Signed-off-by: Takahiro Itazuri <itazur@amazon.com>
---
 Cargo.lock | 70 ++++++++++++++++++++++++++++++++----------------------
 1 file changed, 42 insertions(+), 28 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index 14bf01ee115..24659f2f959 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -218,7 +218,7 @@ dependencies = [
  "bitflags 2.9.0",
  "cexpr",
  "clang-sys",
- "itertools 0.10.5",
+ "itertools 0.12.1",
  "lazy_static",
  "lazycell",
  "log",
@@ -681,14 +681,14 @@ dependencies = [
 
 [[package]]
 name = "getrandom"
-version = "0.3.1"
+version = "0.3.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "43a49c392881ce6d5c3b8cb70f98717b7c07aabbdff06687b9030dbfbe2725f8"
+checksum = "73fea8450eea4bac3940448fb7ae50d91f034f941199fcd9d909a5a07aa455f0"
 dependencies = [
  "cfg-if",
  "libc",
- "wasi 0.13.3+wasi-0.2.2",
- "windows-targets",
+ "r-efi",
+ "wasi 0.14.2+wasi-0.2.4",
 ]
 
 [[package]]
@@ -731,9 +731,9 @@ checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea"
 
 [[package]]
 name = "hermit-abi"
-version = "0.5.0"
+version = "0.5.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fbd780fe5cc30f81464441920d82ac8740e2e46b29a6fad543ddd075229ce37e"
+checksum = "f154ce46856750ed433c8649605bf7ed2de3bc35fd9d2a9f30cddd873c80cb08"
 
 [[package]]
 name = "home"
@@ -789,6 +789,15 @@ dependencies = [
  "either",
 ]
 
+[[package]]
+name = "itertools"
+version = "0.12.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ba291022dbbd398a455acf126c1e341954079855bc60dfdda641363bd6922569"
+dependencies = [
+ "either",
+]
+
 [[package]]
 name = "itertools"
 version = "0.14.0"
@@ -818,9 +827,9 @@ dependencies = [
 
 [[package]]
 name = "jiff"
-version = "0.2.12"
+version = "0.2.13"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d07d8d955d798e7a4d6f9c58cd1f1916e790b42b092758a9ef6e16fef9f1b3fd"
+checksum = "f02000660d30638906021176af16b17498bd0d12813dbfe7b276d8bc7f3c0806"
 dependencies = [
  "jiff-static",
  "log",
@@ -831,9 +840,9 @@ dependencies = [
 
 [[package]]
 name = "jiff-static"
-version = "0.2.12"
+version = "0.2.13"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f244cfe006d98d26f859c7abd1318d85327e1882dc9cef80f62daeeb0adcf300"
+checksum = "f3c30758ddd7188629c6713fc45d1188af4f44c90582311d0c8d8c9907f60c48"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -842,10 +851,11 @@ dependencies = [
 
 [[package]]
 name = "jobserver"
-version = "0.1.32"
+version = "0.1.33"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "48d1dbcbbeb6a7fec7e059840aa538bd62aaccf972c7346c4d9d2059312853d0"
+checksum = "38f262f097c174adebe41eb73d66ae9c06b2844fb0da69969647bbddd9b0538a"
 dependencies = [
+ "getrandom 0.3.2",
  "libc",
 ]
 
@@ -1118,6 +1128,12 @@ dependencies = [
  "proc-macro2",
 ]
 
+[[package]]
+name = "r-efi"
+version = "5.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "74765f6d916ee2faa39bc8e68e4f3ed8949b48cccdac59983d287a7cb71ce9c5"
+
 [[package]]
 name = "rand"
 version = "0.8.5"
@@ -1131,13 +1147,12 @@ dependencies = [
 
 [[package]]
 name = "rand"
-version = "0.9.0"
+version = "0.9.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3779b94aeb87e8bd4e834cee3650289ee9e0d5677f976ecdb6d219e5f4f6cd94"
+checksum = "9fbfd9d094a40bf3ae768db9361049ace4c0e04a4fd6b359518bd7b73a73dd97"
 dependencies = [
  "rand_chacha 0.9.0",
- "rand_core 0.9.0",
- "zerocopy 0.8.25",
+ "rand_core 0.9.3",
 ]
 
 [[package]]
@@ -1157,7 +1172,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb"
 dependencies = [
  "ppv-lite86",
- "rand_core 0.9.0",
+ "rand_core 0.9.3",
 ]
 
 [[package]]
@@ -1171,12 +1186,11 @@ dependencies = [
 
 [[package]]
 name = "rand_core"
-version = "0.9.0"
+version = "0.9.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b08f3c9802962f7e1b25113931d94f43ed9725bebc59db9d0c3e9a23b67e15ff"
+checksum = "99d9a13982dcf210057a8a78572b2217b667c3beacbf3a0d8b454f6f82837d38"
 dependencies = [
- "getrandom 0.3.1",
- "zerocopy 0.8.25",
+ "getrandom 0.3.2",
 ]
 
 [[package]]
@@ -1574,8 +1588,8 @@ version = "1.16.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "458f7a779bf54acc9f347480ac654f68407d3aab21269a6e3c9f922acd9e2da9"
 dependencies = [
- "getrandom 0.3.1",
- "rand 0.9.0",
+ "getrandom 0.3.2",
+ "rand 0.9.1",
  "uuid-macro-internal",
 ]
 
@@ -1724,9 +1738,9 @@ checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423"
 
 [[package]]
 name = "wasi"
-version = "0.13.3+wasi-0.2.2"
+version = "0.14.2+wasi-0.2.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "26816d2e1a4a36a2940b96c5296ce403917633dff8f3440e9b236ed6f6bacad2"
+checksum = "9683f9a5a998d873c0d21fcbe3c083009670149a8fab228644b8bd36b2c48cb3"
 dependencies = [
  "wit-bindgen-rt",
 ]
@@ -1858,9 +1872,9 @@ dependencies = [
 
 [[package]]
 name = "wit-bindgen-rt"
-version = "0.33.0"
+version = "0.39.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3268f3d866458b787f390cf61f4bbb563b922d091359f9608842999eaee3943c"
+checksum = "6f42320e61fe2cfd34354ecb597f86f413484a798ba44a8ca1165c58d42da6c1"
 dependencies = [
  "bitflags 2.9.0",
 ]

From 06b7ece8bf783c304b69a11f5bb06a819e8d2558 Mon Sep 17 00:00:00 2001
From: Takahiro Itazuri <itazur@amazon.com>
Date: Tue, 6 May 2025 13:04:11 +0000
Subject: [PATCH 27/27] chore: Bump version to 1.13.0-dev

We're going to release v1.12.0 tomorrow. Bumps the version to 1.13.0-dev
in main branch.

Signed-off-by: Takahiro Itazuri <itazur@amazon.com>
---
 Cargo.lock                               | 12 ++++++------
 src/cpu-template-helper/Cargo.toml       |  2 +-
 src/firecracker/Cargo.toml               |  2 +-
 src/firecracker/swagger/firecracker.yaml |  2 +-
 src/jailer/Cargo.toml                    |  2 +-
 src/rebase-snap/Cargo.toml               |  2 +-
 src/seccompiler/Cargo.toml               |  2 +-
 src/snapshot-editor/Cargo.toml           |  2 +-
 8 files changed, 13 insertions(+), 13 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index 24659f2f959..e35a1ac1bca 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -419,7 +419,7 @@ checksum = "5b63caa9aa9397e2d9480a9b13673856c78d8ac123288526c37d7839f2a86990"
 
 [[package]]
 name = "cpu-template-helper"
-version = "1.12.0-dev"
+version = "1.13.0-dev"
 dependencies = [
  "clap",
  "displaydoc",
@@ -607,7 +607,7 @@ dependencies = [
 
 [[package]]
 name = "firecracker"
-version = "1.12.0-dev"
+version = "1.13.0-dev"
 dependencies = [
  "cargo_toml",
  "displaydoc",
@@ -815,7 +815,7 @@ checksum = "4a5f13b858c8d314ee3e8f639011f7ccefe71f97f96e50151fb991f267928e2c"
 
 [[package]]
 name = "jailer"
-version = "1.12.0-dev"
+version = "1.13.0-dev"
 dependencies = [
  "libc",
  "log-instrument",
@@ -1204,7 +1204,7 @@ dependencies = [
 
 [[package]]
 name = "rebase-snap"
-version = "1.12.0-dev"
+version = "1.13.0-dev"
 dependencies = [
  "displaydoc",
  "libc",
@@ -1279,7 +1279,7 @@ dependencies = [
 
 [[package]]
 name = "seccompiler"
-version = "1.12.0-dev"
+version = "1.13.0-dev"
 dependencies = [
  "bincode",
  "clap",
@@ -1358,7 +1358,7 @@ dependencies = [
 
 [[package]]
 name = "snapshot-editor"
-version = "1.12.0-dev"
+version = "1.13.0-dev"
 dependencies = [
  "clap",
  "clap-num",
diff --git a/src/cpu-template-helper/Cargo.toml b/src/cpu-template-helper/Cargo.toml
index 4993c957e59..8328c18a7c0 100644
--- a/src/cpu-template-helper/Cargo.toml
+++ b/src/cpu-template-helper/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "cpu-template-helper"
-version = "1.12.0-dev"
+version = "1.13.0-dev"
 authors = ["Amazon Firecracker team <firecracker-devel@amazon.com>"]
 edition = "2024"
 license = "Apache-2.0"
diff --git a/src/firecracker/Cargo.toml b/src/firecracker/Cargo.toml
index 116c9e3ea2e..1a7cd3116b1 100644
--- a/src/firecracker/Cargo.toml
+++ b/src/firecracker/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "firecracker"
-version = "1.12.0-dev"
+version = "1.13.0-dev"
 authors = ["Amazon Firecracker team <firecracker-devel@amazon.com>"]
 edition = "2024"
 build = "build.rs"
diff --git a/src/firecracker/swagger/firecracker.yaml b/src/firecracker/swagger/firecracker.yaml
index dd834baa785..61a0057a1ad 100644
--- a/src/firecracker/swagger/firecracker.yaml
+++ b/src/firecracker/swagger/firecracker.yaml
@@ -5,7 +5,7 @@ info:
     The API is accessible through HTTP calls on specific URLs
     carrying JSON modeled data.
     The transport medium is a Unix Domain Socket.
-  version: 1.12.0-dev
+  version: 1.13.0-dev
   termsOfService: ""
   contact:
     email: "firecracker-maintainers@amazon.com"
diff --git a/src/jailer/Cargo.toml b/src/jailer/Cargo.toml
index 46893af00b5..9a25866a920 100644
--- a/src/jailer/Cargo.toml
+++ b/src/jailer/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "jailer"
-version = "1.12.0-dev"
+version = "1.13.0-dev"
 authors = ["Amazon Firecracker team <firecracker-devel@amazon.com>"]
 edition = "2024"
 description = "Process for starting Firecracker in production scenarios; applies a cgroup/namespace isolation barrier and then drops privileges."
diff --git a/src/rebase-snap/Cargo.toml b/src/rebase-snap/Cargo.toml
index ece545b9003..1a6474e212e 100644
--- a/src/rebase-snap/Cargo.toml
+++ b/src/rebase-snap/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "rebase-snap"
-version = "1.12.0-dev"
+version = "1.13.0-dev"
 authors = ["Amazon Firecracker team <firecracker-devel@amazon.com>"]
 edition = "2024"
 license = "Apache-2.0"
diff --git a/src/seccompiler/Cargo.toml b/src/seccompiler/Cargo.toml
index 4e152e82cfb..b487e258ae2 100644
--- a/src/seccompiler/Cargo.toml
+++ b/src/seccompiler/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "seccompiler"
-version = "1.12.0-dev"
+version = "1.13.0-dev"
 authors = ["Amazon Firecracker team <firecracker-devel@amazon.com>"]
 edition = "2024"
 description = "Program that compiles multi-threaded seccomp-bpf filters expressed as JSON into raw BPF programs, serializing them and outputting them to a file."
diff --git a/src/snapshot-editor/Cargo.toml b/src/snapshot-editor/Cargo.toml
index 489cbf46503..59967281716 100644
--- a/src/snapshot-editor/Cargo.toml
+++ b/src/snapshot-editor/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "snapshot-editor"
-version = "1.12.0-dev"
+version = "1.13.0-dev"
 authors = ["Amazon Firecracker team <firecracker-devel@amazon.com>"]
 edition = "2024"
 license = "Apache-2.0"