feat: Macro define_verify_openvm_stark (#1620)

- Add a RISC-V custom instruction `nativestorew` into
`Rv32IoTranspilerExtension`.
- Add macro `define_verify_openvm_stark`, which can define a function
for guest program to verify a stark proof.

closes INT-3896

Author: nyunyunyunyu

Cargo.lock

@@ -140,6 +140,7 @@ openvm-native-circuit = { path = "extensions/native/circuit", default-features =
 openvm-native-compiler = { path = "extensions/native/compiler", default-features = false }
 openvm-native-compiler-derive = { path = "extensions/native/compiler/derive", default-features = false }
 openvm-native-recursion = { path = "extensions/native/recursion", default-features = false }
+openvm-native-transpiler = { path = "extensions/native/transpiler", default-features = false }
 openvm-keccak256-circuit = { path = "extensions/keccak256/circuit", default-features = false }
 openvm-keccak256-transpiler = { path = "extensions/keccak256/transpiler", default-features = false }
 openvm-keccak256-guest = { path = "extensions/keccak256/guest", default-features = false }

Cargo.toml

@@ -25,6 +25,7 @@ openvm-pairing-transpiler = { workspace = true }
 openvm-native-circuit = { workspace = true }
 openvm-native-compiler = { workspace = true }
 openvm-native-recursion = { workspace = true, features = ["static-verifier"] }
+openvm-native-transpiler = { workspace = true }
 openvm-rv32im-circuit = { workspace = true }
 openvm-rv32im-transpiler = { workspace = true }
 openvm-transpiler = { workspace = true }
@@ -57,6 +58,10 @@ hex.workspace = true
 forge-fmt = { workspace = true, optional = true }
 rrs-lib = { workspace = true }
 
+[dev-dependencies]
+openvm-rv32im-guest.workspace = true
+
+
 [features]
 default = ["parallel", "jemalloc", "evm-verify"]
 evm-prove = ["openvm-native-recursion/evm-prove"]

crates/sdk/Cargo.toml

@@ -46,7 +46,7 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
     /// use std::path::PathBuf;
     ///
     /// let mut path = PathBuf::from(env!("CARGO_MANIFEST_DIR")).to_path_buf();
-    /// path.push("guest");
+    /// path.push("guest/fib");
     /// let target_path = path.to_str().unwrap();
     /// ```
     // ANCHOR: build

crates/sdk/examples/sdk_app.rs

@@ -46,7 +46,7 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
     /// use std::path::PathBuf;
     ///
     /// let mut path = PathBuf::from(env!("CARGO_MANIFEST_DIR")).to_path_buf();
-    /// path.push("guest");
+    /// path.push("guest/fib");
     /// let target_path = path.to_str().unwrap();
     /// ```
     // ANCHOR: build

crates/sdk/examples/sdk_evm.rs

@@ -5,4 +5,4 @@ version = "0.0.0"
 edition = "2021"
 
 [dependencies]
-openvm = { path = "../../toolchain/openvm" }
+openvm = { path = "../../../toolchain/openvm" }

crates/sdk/guest/Cargo.toml renamed to crates/sdk/guest/fib/Cargo.toml

@@ -0,0 +1,4 @@
+*.asm
+Cargo.lock
+target/
+openvm/

crates/sdk/guest/src/main.rs renamed to crates/sdk/guest/fib/src/main.rs

@@ -0,0 +1,18 @@
+[workspace]
+[package]
+name = "openvm-verify-stark-program"
+version = "0.0.0"
+edition = "2021"
+
+[dependencies]
+openvm = { path = "../../../toolchain/openvm", features = ["std"] }
+hex-literal = { version = "0.4.1", default-features = false }
+bytemuck = { version = "1.20.0", features = ["extern_crate_alloc"] }
+
+[features]
+default = []
+
+[profile.profiling]
+inherits = "release"
+debug = 2
+strip = false

crates/sdk/guest/verify_openvm_stark/.gitignore

@@ -0,0 +1,24 @@
+extern crate alloc;
+use alloc::vec::Vec;
+
+use openvm::{define_verify_openvm_stark, io::read};
+
+define_verify_openvm_stark!(
+    verify_openvm_stark,
+    env!("CARGO_MANIFEST_DIR"),
+    "root_verifier.asm"
+);
+
+// const APP_EXE_COMMIT: [u32; 8] = [
+//     343014587, 230645511, 1447462186, 773379336, 1182270030, 1497892484, 461820702, 353704350,
+// ];
+// const APP_VM_COMMIT: [u32; 8] = [
+//     445134834, 1133596793, 530952192, 425228715, 1806903712, 1362083369, 295028151, 482389308,
+// ];
+
+pub fn main() {
+    let app_exe_commit: [u32; 8] = read();
+    let app_vm_commit: [u32; 8] = read();
+    let pvs: Vec<u32> = read();
+    verify_openvm_stark(&app_exe_commit, &app_vm_commit, &pvs);
+}

crates/sdk/guest/verify_openvm_stark/Cargo.toml

@@ -24,6 +24,7 @@ use openvm_native_circuit::{
     CastFExtension, CastFExtensionExecutor, CastFExtensionPeriphery, Native, NativeExecutor,
     NativePeriphery,
 };
+use openvm_native_transpiler::LongFormTranspilerExtension;
 use openvm_pairing_circuit::{
     PairingExtension, PairingExtensionExecutor, PairingExtensionPeriphery,
 };
@@ -138,6 +139,9 @@ impl SdkVmConfig {
         if self.sha256.is_some() {
             transpiler = transpiler.with_extension(Sha256TranspilerExtension);
         }
+        if self.native.is_some() {
+            transpiler = transpiler.with_extension(LongFormTranspilerExtension);
+        }
         if self.rv32m.is_some() {
             transpiler = transpiler.with_extension(Rv32MTranspilerExtension);
         }

crates/sdk/guest/verify_openvm_stark/src/main.rs

@@ -1,4 +1,7 @@
-use std::collections::VecDeque;
+use std::{
+    collections::{HashMap, VecDeque},
+    sync::Arc,
+};
 
 use openvm_circuit::arch::Streams;
 use openvm_stark_backend::p3_field::FieldAlgebra;
@@ -9,6 +12,7 @@ use crate::F;
 #[derive(Clone, Default, Serialize, Deserialize)]
 pub struct StdIn {
     pub buffer: VecDeque<Vec<F>>,
+    pub kv_store: HashMap<Vec<u8>, Vec<u8>>,
 }
 
 impl StdIn {
@@ -36,6 +40,9 @@ impl StdIn {
     pub fn write_field(&mut self, data: &[F]) {
         self.buffer.push_back(data.to_vec());
     }
+    pub fn add_key_value(&mut self, key: Vec<u8>, value: Vec<u8>) {
+        self.kv_store.insert(key, value);
+    }
 }
 
 impl From<StdIn> for Streams<F> {
@@ -44,7 +51,9 @@ impl From<StdIn> for Streams<F> {
         while let Some(input) = std_in.read() {
             data.push(input);
         }
-        Streams::new(data)
+        let mut ret = Streams::new(data);
+        ret.kv_store = Arc::new(std_in.kv_store);
+        ret
     }
 }
 

crates/sdk/src/config/global.rs

@@ -6,6 +6,7 @@ use openvm_circuit::{
     arch::{
         hasher::poseidon2::vm_poseidon2_hasher, ContinuationVmProof, ExecutionError,
         GenerationError, SingleSegmentVmExecutor, SystemConfig, VmConfig, VmExecutor,
+        DEFAULT_MAX_NUM_PUBLIC_VALUES,
     },
     system::{memory::tree::public_values::UserPublicValuesProof, program::trace::VmCommittedExe},
 };
@@ -27,9 +28,11 @@ use openvm_native_recursion::{
         wrapper::Halo2WrapperProvingKey,
         RawEvmProof,
     },
+    hints::Hintable,
     types::InnerConfig,
     vars::StarkProofVariable,
 };
+use openvm_rv32im_guest::hint_load_by_key_encode;
 use openvm_rv32im_transpiler::{
     Rv32ITranspilerExtension, Rv32IoTranspilerExtension, Rv32MTranspilerExtension,
 };
@@ -41,7 +44,9 @@ use openvm_sdk::{
     types::{EvmHalo2Verifier, EvmProof},
     DefaultStaticVerifierPvHandler, Sdk, StdIn,
 };
-use openvm_stark_backend::{keygen::types::LinearConstraint, p3_matrix::Matrix};
+use openvm_stark_backend::{
+    keygen::types::LinearConstraint, p3_field::PrimeField32, p3_matrix::Matrix,
+};
 use openvm_stark_sdk::{
     config::{
         baby_bear_poseidon2::{BabyBearPoseidon2Config, BabyBearPoseidon2Engine},
@@ -393,7 +398,7 @@ fn test_static_verifier_custom_pv_handler() {
 #[test]
 fn test_e2e_proof_generation_and_verification_with_pvs() {
     let mut pkg_dir = PathBuf::from(env!("CARGO_MANIFEST_DIR")).to_path_buf();
-    pkg_dir.push("guest");
+    pkg_dir.push("guest/fib");
 
     let vm_config = SdkVmConfig::builder()
         .system(SdkSystemConfig {
@@ -463,7 +468,7 @@ fn test_sdk_guest_build_and_transpile() {
         // .with_options(vec!["--release"]);
         ;
     let mut pkg_dir = PathBuf::from(env!("CARGO_MANIFEST_DIR")).to_path_buf();
-    pkg_dir.push("guest");
+    pkg_dir.push("guest/fib");
     let one = sdk
         .build(guest_opts.clone(), &pkg_dir, &Default::default())
         .unwrap();
@@ -484,7 +489,7 @@ fn test_inner_proof_codec_roundtrip() -> eyre::Result<()> {
     // generate a proof
     let sdk = Sdk::new();
     let mut pkg_dir = PathBuf::from(env!("CARGO_MANIFEST_DIR")).to_path_buf();
-    pkg_dir.push("guest");
+    pkg_dir.push("guest/fib");
     let elf = sdk.build(Default::default(), pkg_dir, &Default::default())?;
 
     let vm_config = SdkVmConfig::builder()
@@ -574,11 +579,109 @@ fn test_segmentation_retry() {
         .sum();
     assert!(new_total_height < total_height);
 }
-
 #[test]
-fn test_root_verifier_asm_generate() {
-    let agg_stark_config = agg_stark_config_for_test();
-    let agg_pk = AggStarkProvingKey::keygen(agg_stark_config);
+fn test_verify_openvm_stark_e2e() -> Result<()> {
+    const ASM_FILENAME: &str = "root_verifier.asm";
     let sdk = Sdk::new();
-    sdk.generate_root_verifier_asm(&agg_pk);
+    let guest_opts = GuestOptions::default();
+    let mut pkg_dir = PathBuf::from(env!("CARGO_MANIFEST_DIR")).to_path_buf();
+    pkg_dir.push("guest/fib");
+    let elf = sdk.build(guest_opts.clone(), &pkg_dir, &Default::default())?;
+
+    let vm_config = SdkVmConfig::builder()
+        .system(SdkSystemConfig {
+            config: SystemConfig::default().with_continuations(),
+        })
+        .rv32i(Default::default())
+        .rv32m(Default::default())
+        .io(Default::default())
+        .native(Default::default())
+        .build();
+    assert!(vm_config.system.config.continuation_enabled);
+
+    let app_exe = sdk.transpile(elf, vm_config.transpiler())?;
+    let fri_params = FriParameters::new_for_testing(LEAF_LOG_BLOWUP);
+    let app_config = AppConfig::new_with_leaf_fri_params(fri_params, vm_config.clone(), fri_params);
+
+    let app_pk = sdk.app_keygen(app_config.clone())?;
+    let committed_app_exe = sdk.commit_app_exe(fri_params, app_exe.clone())?;
+
+    let commits =
+        AppExecutionCommit::compute(&vm_config, &committed_app_exe, &app_pk.leaf_committed_exe);
+
+    let agg_pk = AggStarkProvingKey::keygen(AggStarkConfig {
+        max_num_user_public_values: DEFAULT_MAX_NUM_PUBLIC_VALUES,
+        leaf_fri_params: FriParameters::new_for_testing(LEAF_LOG_BLOWUP),
+        internal_fri_params: FriParameters::new_for_testing(INTERNAL_LOG_BLOWUP),
+        root_fri_params: FriParameters::new_for_testing(ROOT_LOG_BLOWUP),
+        profiling: false,
+        compiler_options: CompilerOptions {
+            enable_cycle_tracker: true,
+            ..Default::default()
+        },
+        root_max_constraint_degree: (1 << ROOT_LOG_BLOWUP) + 1,
+    });
+    let asm = sdk.generate_root_verifier_asm(&agg_pk);
+    let asm_path = format!(
+        "{}/guest/verify_openvm_stark/{}",
+        env!("CARGO_MANIFEST_DIR"),
+        ASM_FILENAME
+    );
+    std::fs::write(asm_path, asm)?;
+
+    let e2e_stark_proof = sdk.generate_e2e_stark_proof(
+        Arc::new(app_pk),
+        committed_app_exe,
+        agg_pk,
+        StdIn::default(),
+    )?;
+
+    let verify_exe = {
+        let mut pkg_dir = PathBuf::from(env!("CARGO_MANIFEST_DIR")).to_path_buf();
+        pkg_dir.push("guest/verify_openvm_stark");
+        let elf = sdk.build(guest_opts.clone(), &pkg_dir, &Default::default())?;
+        sdk.transpile(elf, vm_config.transpiler())?
+    };
+
+    let pvs = [13u32, 21, 0, 0, 0, 0, 0, 0];
+
+    let exe_commit_u32: Vec<_> = commits
+        .exe_commit
+        .iter()
+        .map(|x| x.as_canonical_u32())
+        .collect();
+    let vm_commit_u32: Vec<_> = commits
+        .leaf_vm_verifier_commit
+        .iter()
+        .map(|x| x.as_canonical_u32())
+        .collect();
+    let pvs_u32: Vec<_> = pvs
+        .iter()
+        .flat_map(|x| x.to_le_bytes())
+        .map(|x| x as u32)
+        .collect();
+
+    let key = ASM_FILENAME
+        .as_bytes()
+        .iter()
+        .cloned()
+        .chain(exe_commit_u32.iter().flat_map(|x| x.to_le_bytes()))
+        .chain(vm_commit_u32.iter().flat_map(|x| x.to_le_bytes()))
+        .chain(pvs_u32.iter().flat_map(|x| x.to_le_bytes()))
+        .collect();
+    let mut stdin = StdIn::default();
+    let to_encode: Vec<Vec<F>> = e2e_stark_proof.proof.write();
+    let value = hint_load_by_key_encode(&to_encode);
+    stdin.add_key_value(key, value);
+
+    let exe_commit_u32_8: [u32; 8] = exe_commit_u32.try_into().unwrap();
+    let vm_commit_u32_8: [u32; 8] = vm_commit_u32.try_into().unwrap();
+
+    stdin.write(&exe_commit_u32_8);
+    stdin.write(&vm_commit_u32_8);
+    stdin.write(&pvs_u32);
+
+    sdk.execute(verify_exe, vm_config, stdin)?;
+
+    Ok(())
 }

crates/sdk/src/stdin.rs

@@ -126,6 +126,16 @@ pub fn reveal_u32(x: u32, index: usize) {
     println!("reveal {} at byte location {}", x, index * 4);
 }
 
+/// Store u32 `x` to the native address `native_addr` as 4 field element in byte.
+#[allow(unused_variables)]
+#[inline(always)]
+pub fn store_u32_to_native(native_addr: u32, x: u32) {
+    #[cfg(target_os = "zkvm")]
+    openvm_rv32im_guest::store_to_native!(native_addr, x);
+    #[cfg(not(target_os = "zkvm"))]
+    panic!("store_to_native_u32 cannot run on non-zkVM platforms");
+}
+
 /// A no-alloc writer to print to stdout on host machine for debugging purposes.
 pub struct Writer;
 

crates/sdk/tests/integration_test.rs

@@ -29,6 +29,8 @@ pub mod utils;
 
 #[cfg(not(target_os = "zkvm"))]
 pub mod host;
+#[cfg(target_os = "zkvm")]
+pub mod verify_stark;
 
 #[cfg(target_os = "zkvm")]
 core::arch::global_asm!(include_str!("memset.s"));