From 139e37e7549c02523ea187dfbdce9c82b343412d Mon Sep 17 00:00:00 2001 From: Xinding Wei Date: Fri, 9 May 2025 13:45:52 -0700 Subject: [PATCH 01/15] refactor: ELF and Program (#1638) - Remove `max_num_public_values` in `Program`/`Elf`. - Improve serialization/deserialization of `Program`. For a Program with 100,000 instructions: Previous: - Serialize: 3.1969ms - Deserialize: 3.2417ms - Size: 3625020 bytes Now: - Serialize: 1.3015ms - Deserialize: 1.9343ms - Size: 4000024 bytes closes INT-3131 closes INT-3771 --- Cargo.lock | 4 + benchmarks/prove/src/bin/fib_e2e.rs | 2 +- benchmarks/prove/src/bin/verify_fibair.rs | 2 +- crates/sdk/src/commit.rs | 3 - crates/sdk/src/config/mod.rs | 2 +- crates/sdk/src/keygen/dummy.rs | 6 +- crates/sdk/tests/integration_test.rs | 8 +- crates/toolchain/instructions/Cargo.toml | 8 ++ .../instructions/benches/program_serde.rs | 39 +++++++ crates/toolchain/instructions/src/program.rs | 105 ++++++++++++++---- crates/toolchain/transpiler/src/elf.rs | 6 - crates/toolchain/transpiler/src/lib.rs | 1 - crates/vm/src/arch/config.rs | 2 +- crates/vm/src/system/program/tests/mod.rs | 9 +- .../native/compiler/src/conversion/mod.rs | 4 +- 15 files changed, 148 insertions(+), 53 deletions(-) create mode 100644 crates/toolchain/instructions/benches/program_serde.rs diff --git a/Cargo.lock b/Cargo.lock index da6c6f773d..f45058ffab 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4356,12 +4356,16 @@ name = "openvm-instructions" version = "1.1.2" dependencies = [ "backtrace", + "bitcode", + "criterion", "derive-new 0.6.0", "itertools 0.14.0", "num-bigint 0.4.6", "num-traits", "openvm-instructions-derive", "openvm-stark-backend", + "p3-baby-bear", + "rand", "serde", "strum", "strum_macros", diff --git a/benchmarks/prove/src/bin/fib_e2e.rs b/benchmarks/prove/src/bin/fib_e2e.rs index 6e1cfa7a35..87b1a464ca 100644 --- a/benchmarks/prove/src/bin/fib_e2e.rs +++ b/benchmarks/prove/src/bin/fib_e2e.rs @@ -3,7 +3,7 @@ use std::{path::PathBuf, sync::Arc}; use clap::Parser; use eyre::Result; use openvm_benchmarks_prove::util::BenchmarkCli; -use openvm_circuit::arch::instructions::{exe::VmExe, program::DEFAULT_MAX_NUM_PUBLIC_VALUES}; +use openvm_circuit::arch::{instructions::exe::VmExe, DEFAULT_MAX_NUM_PUBLIC_VALUES}; use openvm_native_recursion::halo2::utils::{CacheHalo2ParamsReader, DEFAULT_PARAMS_DIR}; use openvm_rv32im_circuit::Rv32ImConfig; use openvm_rv32im_transpiler::{ diff --git a/benchmarks/prove/src/bin/verify_fibair.rs b/benchmarks/prove/src/bin/verify_fibair.rs index d2b75f7e71..1d8d6072da 100644 --- a/benchmarks/prove/src/bin/verify_fibair.rs +++ b/benchmarks/prove/src/bin/verify_fibair.rs @@ -1,7 +1,7 @@ use clap::Parser; use eyre::Result; use openvm_benchmarks_prove::util::BenchmarkCli; -use openvm_circuit::arch::instructions::program::DEFAULT_MAX_NUM_PUBLIC_VALUES; +use openvm_circuit::arch::DEFAULT_MAX_NUM_PUBLIC_VALUES; use openvm_native_circuit::NativeConfig; use openvm_native_compiler::conversion::CompilerOptions; use openvm_native_recursion::testing_utils::inner::build_verification_program; diff --git a/crates/sdk/src/commit.rs b/crates/sdk/src/commit.rs index 7045d96443..7e7bbcdcfd 100644 --- a/crates/sdk/src/commit.rs +++ b/crates/sdk/src/commit.rs @@ -42,9 +42,6 @@ impl AppExecutionCommit { app_exe: &NonRootCommittedExe, leaf_vm_verifier_exe: &NonRootCommittedExe, ) -> Self { - assert!( - app_exe.exe.program.max_num_public_values <= app_vm_config.system().num_public_values - ); let exe_commit = app_exe .compute_exe_commit(&app_vm_config.system().memory_config) .into(); diff --git a/crates/sdk/src/config/mod.rs b/crates/sdk/src/config/mod.rs index 035e3709ef..95581c6084 100644 --- a/crates/sdk/src/config/mod.rs +++ b/crates/sdk/src/config/mod.rs @@ -1,5 +1,5 @@ use clap::Args; -use openvm_circuit::arch::instructions::program::DEFAULT_MAX_NUM_PUBLIC_VALUES; +use openvm_circuit::arch::DEFAULT_MAX_NUM_PUBLIC_VALUES; use openvm_continuations::verifier::{ common::types::VmVerifierPvs, internal::types::InternalVmVerifierPvs, }; diff --git a/crates/sdk/src/keygen/dummy.rs b/crates/sdk/src/keygen/dummy.rs index 44b66454b9..3fe2bcd300 100644 --- a/crates/sdk/src/keygen/dummy.rs +++ b/crates/sdk/src/keygen/dummy.rs @@ -208,14 +208,12 @@ fn dummy_app_committed_exe(fri_params: FriParameters) -> Arc Program { - let mut ret = Program::from_instructions(&[Instruction::from_isize( + Program::from_instructions(&[Instruction::from_isize( TERMINATE.global_opcode(), 0, 0, 0, 0, 0, - )]); - ret.max_num_public_values = 0; - ret + )]) } diff --git a/crates/sdk/tests/integration_test.rs b/crates/sdk/tests/integration_test.rs index 1968f1c1da..7d70b8de67 100644 --- a/crates/sdk/tests/integration_test.rs +++ b/crates/sdk/tests/integration_test.rs @@ -115,9 +115,7 @@ fn app_committed_exe_for_test(app_log_blowup: usize) -> Arc> builder.assign(&b, c); }); builder.halt(); - let mut program = builder.compile_isa(); - program.max_num_public_values = NUM_PUB_VALUES; - program + builder.compile_isa() }; Sdk::new() .commit_app_exe( @@ -343,10 +341,6 @@ fn test_static_verifier_custom_pv_handler() { let app_pk = sdk.app_keygen(app_config.clone()).unwrap(); let app_committed_exe = app_committed_exe_for_test(app_log_blowup); println!("app_config: {:?}", app_config.app_vm_config); - println!( - "app_committed_exe max_num_public_values: {:?}", - app_committed_exe.exe.program.max_num_public_values - ); let params_reader = CacheHalo2ParamsReader::new_with_default_params_dir(); // Generate PK using custom PV handler diff --git a/crates/toolchain/instructions/Cargo.toml b/crates/toolchain/instructions/Cargo.toml index 5bf558da8d..99286c0854 100644 --- a/crates/toolchain/instructions/Cargo.toml +++ b/crates/toolchain/instructions/Cargo.toml @@ -21,3 +21,11 @@ num-bigint.workspace = true num-traits.workspace = true [dev-dependencies] +criterion = { version = "0.5", features = ["html_reports"] } +p3-baby-bear.workspace = true +bitcode.workspace = true +rand.workspace = true + +[[bench]] +name = "program_serde" +harness = false \ No newline at end of file diff --git a/crates/toolchain/instructions/benches/program_serde.rs b/crates/toolchain/instructions/benches/program_serde.rs new file mode 100644 index 0000000000..240d184ab4 --- /dev/null +++ b/crates/toolchain/instructions/benches/program_serde.rs @@ -0,0 +1,39 @@ +use std::hint::black_box; + +use criterion::{criterion_group, criterion_main, Criterion}; +use openvm_instructions::{instruction::Instruction, program::Program, VmOpcode}; +use p3_baby_bear::BabyBear; +use rand::prelude::*; + +type F = BabyBear; + +fn random_instruction(rng: &mut impl Rng) -> Instruction { + Instruction::new( + VmOpcode::from_usize(rng.gen()), + rng.gen(), + rng.gen(), + rng.gen(), + rng.gen(), + rng.gen(), + rng.gen(), + rng.gen(), + ) +} + +fn program_serde_bench(c: &mut Criterion) { + let mut rng = StdRng::from_seed([42; 32]); + let instructions: Vec<_> = (0..100_000).map(|_| random_instruction(&mut rng)).collect(); + let program: Program = Program::from_instructions(&instructions); + c.bench_function("bitcode serialize Program with 100000 instructions", |b| { + b.iter(|| bitcode::serialize(black_box(&program))) + }); + let bytes = bitcode::serialize(&program).unwrap(); + println!("Result length in bytes: {}", bytes.len()); + c.bench_function( + "bitcode deserialize Program with 100000 instructions", + |b| b.iter(|| bitcode::deserialize::<'_, Program>(black_box(&bytes))), + ); +} + +criterion_group!(benches, program_serde_bench); +criterion_main!(benches); diff --git a/crates/toolchain/instructions/src/program.rs b/crates/toolchain/instructions/src/program.rs index 08bbc51806..010b70514d 100644 --- a/crates/toolchain/instructions/src/program.rs +++ b/crates/toolchain/instructions/src/program.rs @@ -2,7 +2,7 @@ use std::{fmt, fmt::Display}; use itertools::Itertools; use openvm_stark_backend::p3_field::Field; -use serde::{Deserialize, Serialize}; +use serde::{de::Deserializer, Deserialize, Serialize, Serializer}; use crate::instruction::{DebugInfo, Instruction}; @@ -10,31 +10,30 @@ pub const PC_BITS: usize = 30; /// We use default PC step of 4 whenever possible for consistency with RISC-V, where 4 comes /// from the fact that each standard RISC-V instruction is 32-bits = 4 bytes. pub const DEFAULT_PC_STEP: u32 = 4; -pub const DEFAULT_MAX_NUM_PUBLIC_VALUES: usize = 32; pub const MAX_ALLOWED_PC: u32 = (1 << PC_BITS) - 1; #[derive(Clone, Debug, Default, Serialize, Deserialize)] +#[serde(bound(serialize = "F: Serialize", deserialize = "F: Deserialize<'de>"))] pub struct Program { /// A map from program counter to instruction. /// Sometimes the instructions are enumerated as 0, 4, 8, etc. /// Maybe at some point we will replace this with a struct that would have a `Vec` under the /// hood and divide the incoming `pc` by whatever given. + #[serde( + serialize_with = "serialize_instructions_and_debug_infos", + deserialize_with = "deserialize_instructions_and_debug_infos" + )] pub instructions_and_debug_infos: Vec, Option)>>, pub step: u32, pub pc_base: u32, - /// The upper bound of the number of public values the program would publish. - /// Currently, this won't result any constraint. But users should always be aware of the limit - /// of public values when they write programs. - pub max_num_public_values: usize, } impl Program { - pub fn new_empty(step: u32, pc_base: u32, max_num_public_values: usize) -> Self { + pub fn new_empty(step: u32, pc_base: u32) -> Self { Self { instructions_and_debug_infos: vec![], step, pc_base, - max_num_public_values, } } @@ -42,7 +41,6 @@ impl Program { instructions: &[Instruction], step: u32, pc_base: u32, - max_num_public_values: usize, ) -> Self { Self { instructions_and_debug_infos: instructions @@ -51,7 +49,6 @@ impl Program { .collect(), step, pc_base, - max_num_public_values, } } @@ -59,7 +56,6 @@ impl Program { instructions: &[Option>], step: u32, pc_base: u32, - max_num_public_values: usize, ) -> Self { Self { instructions_and_debug_infos: instructions @@ -68,7 +64,6 @@ impl Program { .collect(), step, pc_base, - max_num_public_values, } } @@ -86,7 +81,6 @@ impl Program { .collect(), step: DEFAULT_PC_STEP, pc_base: 0, - max_num_public_values: DEFAULT_MAX_NUM_PUBLIC_VALUES, } } @@ -102,12 +96,7 @@ impl Program { } pub fn from_instructions(instructions: &[Instruction]) -> Self { - Self::new_without_debug_infos( - instructions, - DEFAULT_PC_STEP, - 0, - DEFAULT_MAX_NUM_PUBLIC_VALUES, - ) + Self::new_without_debug_infos(instructions, DEFAULT_PC_STEP, 0) } pub fn len(&self) -> usize { @@ -224,3 +213,81 @@ pub fn display_program_with_pc(program: &Program) { ); } } + +// `debug_info` is based on the symbol table of the binary. Usually serializing `debug_info` is not +// meaningful because the program is executed by another binary. So here we only serialize +// instructions. +fn serialize_instructions_and_debug_infos( + data: &[Option<(Instruction, Option)>], + serializer: S, +) -> Result { + let mut ins_data = Vec::with_capacity(data.len()); + let total_len = data.len() as u32; + for (i, o) in data.iter().enumerate() { + if let Some(o) = o { + ins_data.push((&o.0, i as u32)); + } + } + (ins_data, total_len).serialize(serializer) +} + +#[allow(clippy::type_complexity)] +fn deserialize_instructions_and_debug_infos<'de, F: Deserialize<'de>, D: Deserializer<'de>>( + deserializer: D, +) -> Result, Option)>>, D::Error> { + let (inst_data, total_len): (Vec<(Instruction, u32)>, u32) = + Deserialize::deserialize(deserializer)?; + let mut ret: Vec, Option)>> = Vec::new(); + ret.resize_with(total_len as usize, || None); + for (inst, i) in inst_data { + ret[i as usize] = Some((inst, None)); + } + Ok(ret) +} + +#[cfg(test)] +mod tests { + use itertools::izip; + use p3_baby_bear::BabyBear; + + use super::*; + use crate::VmOpcode; + + type F = BabyBear; + + #[test] + fn test_program_serde() { + let mut program = Program::::new_empty(4, 0); + program.instructions_and_debug_infos.push(Some(( + Instruction::from_isize(VmOpcode::from_usize(113), 1, 2, 3, 4, 5), + None, + ))); + program.instructions_and_debug_infos.push(None); + program.instructions_and_debug_infos.push(None); + program.instructions_and_debug_infos.push(Some(( + Instruction::from_isize(VmOpcode::from_usize(145), 10, 20, 30, 40, 50), + None, + ))); + program.instructions_and_debug_infos.push(Some(( + Instruction::from_isize(VmOpcode::from_usize(145), 10, 20, 30, 40, 50), + None, + ))); + program.instructions_and_debug_infos.push(None); + let bytes = bitcode::serialize(&program).unwrap(); + let de_program: Program = bitcode::deserialize(&bytes).unwrap(); + for (expected_ins, ins) in izip!( + &program.instructions_and_debug_infos, + &de_program.instructions_and_debug_infos + ) { + match (expected_ins, ins) { + (Some(expected_ins), Some(ins)) => { + assert_eq!(expected_ins.0, ins.0); + } + (None, None) => {} + _ => { + panic!("Different instructions after serialization"); + } + } + } + } +} diff --git a/crates/toolchain/transpiler/src/elf.rs b/crates/toolchain/transpiler/src/elf.rs index 5059d90454..ab355683d5 100644 --- a/crates/toolchain/transpiler/src/elf.rs +++ b/crates/toolchain/transpiler/src/elf.rs @@ -19,8 +19,6 @@ use openvm_instructions::exe::FnBound; use openvm_instructions::{exe::FnBounds, program::MAX_ALLOWED_PC}; use openvm_platform::WORD_SIZE; -pub const ELF_DEFAULT_MAX_NUM_PUBLIC_VALUES: usize = 32; - /// RISC-V 32IM ELF (Executable and Linkable Format) File. /// /// This file represents a binary in the ELF format, specifically the RISC-V 32IM architecture @@ -40,9 +38,6 @@ pub struct Elf { pub(crate) pc_base: u32, /// The initial memory image, useful for global constants. pub(crate) memory_image: BTreeMap, - /// The upper bound of the number of public values the program would publish. - /// TODO: read from project config. - pub(crate) max_num_public_values: usize, /// Debug info for spanning benchmark metrics by function. pub(crate) fn_bounds: FnBounds, } @@ -61,7 +56,6 @@ impl Elf { pc_start, pc_base, memory_image, - max_num_public_values: ELF_DEFAULT_MAX_NUM_PUBLIC_VALUES, fn_bounds, } } diff --git a/crates/toolchain/transpiler/src/lib.rs b/crates/toolchain/transpiler/src/lib.rs index bf88d25a8e..367b028393 100644 --- a/crates/toolchain/transpiler/src/lib.rs +++ b/crates/toolchain/transpiler/src/lib.rs @@ -33,7 +33,6 @@ impl FromElf for VmExe { &instructions, DEFAULT_PC_STEP, elf.pc_base, - elf.max_num_public_values, ); let init_memory = elf_memory_image_to_openvm_memory_image(elf.memory_image); diff --git a/crates/vm/src/arch/config.rs b/crates/vm/src/arch/config.rs index 30d92130a1..62185779f4 100644 --- a/crates/vm/src/arch/config.rs +++ b/crates/vm/src/arch/config.rs @@ -2,7 +2,6 @@ use std::sync::Arc; use derive_new::new; use openvm_circuit::system::memory::MemoryTraceHeights; -use openvm_instructions::program::DEFAULT_MAX_NUM_PUBLIC_VALUES; use openvm_poseidon2_air::Poseidon2Config; use openvm_stark_backend::{p3_field::PrimeField32, ChipUsageGetter}; use serde::{de::DeserializeOwned, Deserialize, Serialize}; @@ -17,6 +16,7 @@ use crate::system::memory::BOUNDARY_AIR_OFFSET; // sbox is decomposed to have this max degree for Poseidon2. We set to 3 so quotient_degree = 2 // allows log_blowup = 1 const DEFAULT_POSEIDON2_MAX_CONSTRAINT_DEGREE: usize = 3; +pub const DEFAULT_MAX_NUM_PUBLIC_VALUES: usize = 32; /// Width of Poseidon2 VM uses. pub const POSEIDON2_WIDTH: usize = 16; /// Returns a Poseidon2 config for the VM. diff --git a/crates/vm/src/system/program/tests/mod.rs b/crates/vm/src/system/program/tests/mod.rs index 8a2726e150..4a0293b348 100644 --- a/crates/vm/src/system/program/tests/mod.rs +++ b/crates/vm/src/system/program/tests/mod.rs @@ -2,7 +2,7 @@ use std::iter; use openvm_instructions::{ instruction::Instruction, - program::{Program, DEFAULT_MAX_NUM_PUBLIC_VALUES, DEFAULT_PC_STEP}, + program::{Program, DEFAULT_PC_STEP}, LocalOpcode, }; use openvm_native_compiler::{ @@ -265,12 +265,7 @@ fn test_program_with_undefined_instructions() { )), ]; - let program = Program::new_without_debug_infos_with_option( - &instructions, - DEFAULT_PC_STEP, - 0, - DEFAULT_MAX_NUM_PUBLIC_VALUES, - ); + let program = Program::new_without_debug_infos_with_option(&instructions, DEFAULT_PC_STEP, 0); interaction_test(program, vec![0, 2, 5]); } diff --git a/extensions/native/compiler/src/conversion/mod.rs b/extensions/native/compiler/src/conversion/mod.rs index edf10467cf..9c3fc8d752 100644 --- a/extensions/native/compiler/src/conversion/mod.rs +++ b/extensions/native/compiler/src/conversion/mod.rs @@ -1,7 +1,7 @@ use openvm_circuit::arch::instructions::program::Program; use openvm_instructions::{ instruction::{DebugInfo, Instruction}, - program::{DEFAULT_MAX_NUM_PUBLIC_VALUES, DEFAULT_PC_STEP}, + program::DEFAULT_PC_STEP, LocalOpcode, PhantomDiscriminant, PublishOpcode, SysPhantom, SystemOpcode, VmOpcode, }; use openvm_rv32im_transpiler::BranchEqualOpcode; @@ -565,7 +565,7 @@ pub fn convert_program>( } } - let mut result = Program::new_empty(DEFAULT_PC_STEP, 0, DEFAULT_MAX_NUM_PUBLIC_VALUES); + let mut result = Program::new_empty(DEFAULT_PC_STEP, 0); result.push_instruction_and_debug_info(init_register_0, init_debug_info); for block in program.blocks.iter() { for (instruction, debug_info) in block.0.iter().zip(block.1.iter()) { From f86701ffdf710c2f4c508f4b3dea4648526979a4 Mon Sep 17 00:00:00 2001 From: Avaneesh-axiom Date: Fri, 25 Apr 2025 19:31:58 -0400 Subject: [PATCH 02/15] feat: Build script for calling init macros (#1596) Added a build script that automatically adds calls to `moduli_init, complex_init, sw_init` appropriately by reading the vm config. --- Cargo.lock | 5 +- benchmarks/prove/src/bin/ecrecover.rs | 12 ++- benchmarks/prove/src/bin/kitchen_sink.rs | 13 ++- benchmarks/prove/src/bin/pairing.rs | 7 +- book/src/custom-extensions/algebra.md | 39 ++++---- book/src/custom-extensions/ecc.md | 12 ++- book/src/custom-extensions/pairing.md | 2 +- crates/cli/Cargo.toml | 1 + crates/cli/src/commands/build.rs | 11 ++- crates/sdk/examples/sdk_app.rs | 8 +- crates/sdk/examples/sdk_evm.rs | 8 +- crates/sdk/src/config/global.rs | 41 +++++++- crates/sdk/src/lib.rs | 9 +- crates/sdk/tests/integration_test.rs | 46 ++++++++- crates/toolchain/openvm/src/lib.rs | 11 +++ crates/toolchain/tests/src/lib.rs | 49 ++++++++-- .../toolchain/tests/tests/transpiler_tests.rs | 16 +++- crates/vm/src/arch/config.rs | 38 +++++++- docs/crates/vm-extensions.md | 8 ++ examples/algebra/Cargo.toml | 11 +-- examples/algebra/openvm.toml | 2 +- examples/algebra/openvm_init.rs | 3 + examples/algebra/src/main.rs | 15 +-- examples/ecc/openvm.toml | 1 + examples/ecc/openvm_init.rs | 3 + examples/ecc/src/main.rs | 3 + examples/pairing/Cargo.toml | 17 ++-- examples/pairing/openvm.toml | 2 +- examples/pairing/openvm_init.rs | 3 + examples/pairing/src/main.rs | 3 + extensions/algebra/circuit/src/config.rs | 31 +++++- .../algebra/circuit/src/fp2_extension.rs | 40 +++++++- .../algebra/circuit/src/modular_extension.rs | 14 +++ extensions/algebra/complex-macros/README.md | 14 ++- extensions/algebra/moduli-macros/README.md | 13 ++- ...odulus.rs => complex_redundant_modulus.rs} | 7 +- ...plex-secp256k1.rs => complex_secp256k1.rs} | 7 +- ...x-two-modulos.rs => complex_two_moduli.rs} | 7 +- .../{invalid-setup.rs => invalid_setup.rs} | 0 .../algebra/tests/programs/examples/little.rs | 4 +- .../tests/programs/examples/moduli_setup.rs | 6 +- .../openvm_init_complex_redundant_modulus.rs | 3 + .../programs/openvm_init_complex_secp256k1.rs | 3 + .../openvm_init_complex_two_moduli.rs | 3 + .../tests/programs/openvm_init_little.rs | 2 + .../programs/openvm_init_moduli_setup.rs | 2 + extensions/algebra/tests/src/lib.rs | 84 ++++++++++------ extensions/bigint/circuit/src/extension.rs | 6 +- extensions/bigint/tests/src/lib.rs | 10 +- extensions/ecc/circuit/src/config.rs | 12 ++- .../ecc/circuit/src/weierstrass_extension.rs | 21 +++- extensions/ecc/guest/src/k256.rs | 4 + extensions/ecc/guest/src/p256.rs | 4 + extensions/ecc/sw-macros/README.md | 7 ++ .../ecc/tests/programs/examples/decompress.rs | 15 +-- .../examples/decompress_invalid_hint.rs | 15 +-- extensions/ecc/tests/programs/examples/ec.rs | 9 +- .../tests/programs/examples/ec_nonzero_a.rs | 9 +- .../tests/programs/examples/ec_two_curves.rs | 12 +-- .../ecc/tests/programs/examples/ecdsa.rs | 8 +- .../tests/programs/openvm_init_decompress.rs | 3 + .../openvm_init_decompress_invalid_hint.rs | 3 + .../ecc/tests/programs/openvm_init_ec.rs | 3 + .../programs/openvm_init_ec_nonzero_a.rs | 3 + .../programs/openvm_init_ec_two_curves.rs | 3 + .../ecc/tests/programs/openvm_init_ecdsa.rs | 3 + extensions/ecc/tests/src/lib.rs | 95 ++++++++++++------- extensions/keccak256/circuit/src/extension.rs | 6 +- extensions/keccak256/tests/src/lib.rs | 5 +- extensions/native/circuit/src/extension.rs | 10 +- extensions/pairing/circuit/src/config.rs | 29 ++++-- .../pairing/circuit/src/pairing_extension.rs | 8 +- extensions/pairing/guest/src/bls12_381/fp2.rs | 4 + extensions/pairing/guest/src/bls12_381/mod.rs | 4 + extensions/pairing/guest/src/bn254/fp2.rs | 4 + extensions/pairing/guest/src/bn254/mod.rs | 4 + .../pairing/tests/programs/examples/bls_ec.rs | 9 +- .../programs/examples/bls_final_exp_hint.rs | 5 +- .../programs/examples/bn_final_exp_hint.rs | 5 +- .../tests/programs/examples/fp12_mul.rs | 18 +--- .../tests/programs/examples/pairing_check.rs | 18 +--- .../examples/pairing_check_fallback.rs | 18 +--- .../tests/programs/examples/pairing_line.rs | 18 +--- .../programs/examples/pairing_miller_loop.rs | 18 +--- .../programs/examples/pairing_miller_step.rs | 18 +--- .../programs/openvm_init_bls_ec_bls12_381.rs | 3 + ...penvm_init_bls_final_exp_hint_bls12_381.rs | 4 + .../openvm_init_bn_final_exp_hint_bn254.rs | 4 + .../openvm_init_fp12_mul_bls12_381.rs | 4 + .../programs/openvm_init_fp12_mul_bn254.rs | 4 + .../openvm_init_pairing_check_bls12_381.rs | 4 + .../openvm_init_pairing_check_bn254.rs | 4 + ...m_init_pairing_check_fallback_bls12_381.rs | 4 + ...penvm_init_pairing_check_fallback_bn254.rs | 4 + .../openvm_init_pairing_line_bls12_381.rs | 4 + .../openvm_init_pairing_line_bn254.rs | 4 + ...envm_init_pairing_miller_loop_bls12_381.rs | 4 + .../openvm_init_pairing_miller_loop_bn254.rs | 4 + ...envm_init_pairing_miller_step_bls12_381.rs | 4 + .../openvm_init_pairing_miller_step_bn254.rs | 4 + extensions/pairing/tests/src/lib.rs | 87 ++++++++++++----- extensions/rv32im/circuit/src/extension.rs | 9 +- extensions/rv32im/tests/src/lib.rs | 48 ++++++---- extensions/sha256/circuit/src/extension.rs | 8 +- extensions/sha256/tests/src/lib.rs | 5 +- 105 files changed, 874 insertions(+), 427 deletions(-) create mode 100644 examples/algebra/openvm_init.rs create mode 100644 examples/ecc/openvm_init.rs create mode 100644 examples/pairing/openvm_init.rs rename extensions/algebra/tests/programs/examples/{complex-redundant-modulus.rs => complex_redundant_modulus.rs} (77%) rename extensions/algebra/tests/programs/examples/{complex-secp256k1.rs => complex_secp256k1.rs} (85%) rename extensions/algebra/tests/programs/examples/{complex-two-modulos.rs => complex_two_moduli.rs} (79%) rename extensions/algebra/tests/programs/examples/{invalid-setup.rs => invalid_setup.rs} (100%) create mode 100644 extensions/algebra/tests/programs/openvm_init_complex_redundant_modulus.rs create mode 100644 extensions/algebra/tests/programs/openvm_init_complex_secp256k1.rs create mode 100644 extensions/algebra/tests/programs/openvm_init_complex_two_moduli.rs create mode 100644 extensions/algebra/tests/programs/openvm_init_little.rs create mode 100644 extensions/algebra/tests/programs/openvm_init_moduli_setup.rs create mode 100644 extensions/ecc/tests/programs/openvm_init_decompress.rs create mode 100644 extensions/ecc/tests/programs/openvm_init_decompress_invalid_hint.rs create mode 100644 extensions/ecc/tests/programs/openvm_init_ec.rs create mode 100644 extensions/ecc/tests/programs/openvm_init_ec_nonzero_a.rs create mode 100644 extensions/ecc/tests/programs/openvm_init_ec_two_curves.rs create mode 100644 extensions/ecc/tests/programs/openvm_init_ecdsa.rs create mode 100644 extensions/pairing/tests/programs/openvm_init_bls_ec_bls12_381.rs create mode 100644 extensions/pairing/tests/programs/openvm_init_bls_final_exp_hint_bls12_381.rs create mode 100644 extensions/pairing/tests/programs/openvm_init_bn_final_exp_hint_bn254.rs create mode 100644 extensions/pairing/tests/programs/openvm_init_fp12_mul_bls12_381.rs create mode 100644 extensions/pairing/tests/programs/openvm_init_fp12_mul_bn254.rs create mode 100644 extensions/pairing/tests/programs/openvm_init_pairing_check_bls12_381.rs create mode 100644 extensions/pairing/tests/programs/openvm_init_pairing_check_bn254.rs create mode 100644 extensions/pairing/tests/programs/openvm_init_pairing_check_fallback_bls12_381.rs create mode 100644 extensions/pairing/tests/programs/openvm_init_pairing_check_fallback_bn254.rs create mode 100644 extensions/pairing/tests/programs/openvm_init_pairing_line_bls12_381.rs create mode 100644 extensions/pairing/tests/programs/openvm_init_pairing_line_bn254.rs create mode 100644 extensions/pairing/tests/programs/openvm_init_pairing_miller_loop_bls12_381.rs create mode 100644 extensions/pairing/tests/programs/openvm_init_pairing_miller_loop_bn254.rs create mode 100644 extensions/pairing/tests/programs/openvm_init_pairing_miller_step_bls12_381.rs create mode 100644 extensions/pairing/tests/programs/openvm_init_pairing_miller_step_bn254.rs diff --git a/Cargo.lock b/Cargo.lock index f45058ffab..d687670a60 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1292,6 +1292,7 @@ dependencies = [ "eyre", "hex", "openvm-build", + "openvm-circuit", "openvm-native-recursion", "openvm-sdk", "openvm-stark-backend", @@ -7285,9 +7286,9 @@ dependencies = [ [[package]] name = "tokio" -version = "1.44.2" +version = "1.43.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6b88822cbe49de4185e3a4cbf8321dd487cf5fe0c5c65695fef6346371e9c48" +checksum = "492a604e2fd7f814268a378409e6c92b5525d747d10db9a229723f55a417958c" dependencies = [ "backtrace", "bytes", diff --git a/benchmarks/prove/src/bin/ecrecover.rs b/benchmarks/prove/src/bin/ecrecover.rs index 2a2f146f4c..f0a3886f92 100644 --- a/benchmarks/prove/src/bin/ecrecover.rs +++ b/benchmarks/prove/src/bin/ecrecover.rs @@ -8,7 +8,7 @@ use openvm_algebra_circuit::{ use openvm_algebra_transpiler::ModularTranspilerExtension; use openvm_benchmarks_prove::util::BenchmarkCli; use openvm_circuit::{ - arch::{instructions::exe::VmExe, SystemConfig}, + arch::{instructions::exe::VmExe, InitFileGenerator, SystemConfig}, derive::VmConfig, }; use openvm_ecc_circuit::{ @@ -66,6 +66,16 @@ pub struct Rv32ImEcRecoverConfig { pub weierstrass: WeierstrassExtension, } +impl InitFileGenerator for Rv32ImEcRecoverConfig { + fn generate_init_file_contents(&self) -> Option { + Some(format!( + "// This file is automatically generated by cargo openvm. Do not rename or edit.\n{}\n{}\n", + self.modular.generate_moduli_init(), + self.weierstrass.generate_sw_init() + )) + } +} + impl Rv32ImEcRecoverConfig { pub fn for_curves(curves: Vec) -> Self { let primes: Vec = curves diff --git a/benchmarks/prove/src/bin/kitchen_sink.rs b/benchmarks/prove/src/bin/kitchen_sink.rs index c4b7791330..acf3a7f628 100644 --- a/benchmarks/prove/src/bin/kitchen_sink.rs +++ b/benchmarks/prove/src/bin/kitchen_sink.rs @@ -9,6 +9,9 @@ use openvm_circuit::arch::{instructions::exe::VmExe, SystemConfig}; use openvm_ecc_circuit::{WeierstrassExtension, P256_CONFIG, SECP256K1_CONFIG}; use openvm_native_recursion::halo2::utils::{CacheHalo2ParamsReader, DEFAULT_PARAMS_DIR}; use openvm_pairing_circuit::{PairingCurve, PairingExtension}; +use openvm_pairing_guest::{ + bls12_381::BLS12_381_COMPLEX_STRUCT_NAME, bn254::BN254_COMPLEX_STRUCT_NAME, +}; use openvm_sdk::{ commit::commit_app_exe, config::SdkVmConfig, prover::EvmHalo2Prover, DefaultStaticVerifierPvHandler, Sdk, StdIn, @@ -46,8 +49,14 @@ fn main() -> Result<()> { BigUint::from(7u32), ])) .fp2(Fp2Extension::new(vec![ - bn_config.modulus.clone(), - bls_config.modulus.clone(), + ( + BN254_COMPLEX_STRUCT_NAME.to_string(), + bn_config.modulus.clone(), + ), + ( + BLS12_381_COMPLEX_STRUCT_NAME.to_string(), + bls_config.modulus.clone(), + ), ])) .ecc(WeierstrassExtension::new(vec![ SECP256K1_CONFIG.clone(), diff --git a/benchmarks/prove/src/bin/pairing.rs b/benchmarks/prove/src/bin/pairing.rs index 6f200172a4..298169cb60 100644 --- a/benchmarks/prove/src/bin/pairing.rs +++ b/benchmarks/prove/src/bin/pairing.rs @@ -5,7 +5,7 @@ use openvm_benchmarks_prove::util::BenchmarkCli; use openvm_circuit::arch::SystemConfig; use openvm_ecc_circuit::WeierstrassExtension; use openvm_pairing_circuit::{PairingCurve, PairingExtension}; -use openvm_pairing_guest::bn254::{BN254_MODULUS, BN254_ORDER}; +use openvm_pairing_guest::bn254::{BN254_COMPLEX_STRUCT_NAME, BN254_MODULUS, BN254_ORDER}; use openvm_sdk::{config::SdkVmConfig, Sdk, StdIn}; use openvm_stark_sdk::bench::run_with_metric_collection; @@ -23,7 +23,10 @@ fn main() -> Result<()> { BN254_MODULUS.clone(), BN254_ORDER.clone(), ])) - .fp2(Fp2Extension::new(vec![BN254_MODULUS.clone()])) + .fp2(Fp2Extension::new(vec![( + BN254_COMPLEX_STRUCT_NAME.to_string(), + BN254_MODULUS.clone(), + )])) .ecc(WeierstrassExtension::new(vec![ PairingCurve::Bn254.curve_config() ])) diff --git a/book/src/custom-extensions/algebra.md b/book/src/custom-extensions/algebra.md index d80efbb1d9..7ce48698b3 100644 --- a/book/src/custom-extensions/algebra.md +++ b/book/src/custom-extensions/algebra.md @@ -32,13 +32,16 @@ moduli_declare! { This creates `Bls12_381Fp` and `Bn254Fp` structs, each implementing the `IntMod` trait. The modulus parameter must be a string literal in decimal or hexadecimal format. -2. **Init**: Use the `moduli_init!` macro exactly once in the final binary: +2. **Init**: Use the `init!` macro exactly once in the final binary: ```rust +init!(); +/* This expands to moduli_init! { "0x1a0111ea397fe69a4b1ba7b6434bacd764774b84f38512bf6730d2a0f6b0f6241eabfffeb153ffffb9feffffffffaaab", "21888242871839275222246405745257275088696311157297823662689037894645226208583" } +*/ ``` This step enumerates the declared moduli (e.g., `0` for the first one, `1` for the second one) and sets up internal linkage so the compiler can generate the appropriate RISC-V instructions associated with each modulus. @@ -48,7 +51,7 @@ This step enumerates the declared moduli (e.g., `0` for the first one, `1` for t **Summary**: - `moduli_declare!`: Declares modular arithmetic structures and can be done multiple times. -- `moduli_init!`: Called once in the final binary to assign and lock in the moduli. +- `init!`: Called once in the final binary to assign and lock in the moduli. - `setup_()`/`setup_all_moduli()`: Ensures at runtime that the correct modulus is in use, providing a security check and finalizing the environment for safe arithmetic operations. ## Complex field extension @@ -65,32 +68,21 @@ complex_declare! { This creates a `Bn254Fp2` struct, representing a complex extension field. The `mod_type` must implement `IntMod`. -2. **Init**: Called once, after `moduli_init!`, to enumerate these extensions and generate corresponding instructions: +2. **Init**: After calling `complex_declare!`, the `init!` macro will now expand to the appropriate call to `complex_init!`. ```rust -complex_init! { - Bn254Fp2 { mod_idx = 0 }, -} -``` - -Note that you need to use the same type name in `complex_declare!` and `complex_init!`. For example, the following code will **fail** to compile: - -```rust -// moduli related macros... - -complex_declare! { - Bn254Fp2 { mod_type = Bn254Fp }, +init!(); +/* This expands to: +moduli_init! { + "0x1a0111ea397fe69a4b1ba7b6434bacd764774b84f38512bf6730d2a0f6b0f6241eabfffeb153ffffb9feffffffffaaab", + "21888242871839275222246405745257275088696311157297823662689037894645226208583" } - -pub type Fp2 = Bn254Fp2; - complex_init! { - Fp2 { mod_idx = 0 }, + Bn254Fp2 { mod_idx = 0 }, } +*/ ``` -Here, `mod_idx` refers to the index of the underlying modulus as initialized by `moduli_init!` - 3. **Setup**: Similar to moduli, call `setup_complex_()` or `setup_all_complex_extensions()` at runtime to secure the environment. ### Config parameters @@ -102,12 +94,13 @@ For the guest program to build successfully, all used moduli must be declared in supported_modulus = ["115792089237316195423570985008687907853269984665640564039457584007908834671663"] [app_vm_config.fp2] -supported_modulus = ["115792089237316195423570985008687907853269984665640564039457584007908834671663"] +supported_modulus = [["Bn254Fp2", "115792089237316195423570985008687907853269984665640564039457584007908834671663"]] ``` The `supported_modulus` parameter is a list of moduli that the guest program will use. They must be provided in decimal format in the `.toml` file. The order of moduli in `[app_vm_config.modular]` must match the order in the `moduli_init!` macro. Similarly, the order of moduli in `[app_vm_config.fp2]` must match the order in the `complex_init!` macro. +Also, each modulus in `[app_vm_config.fp2]` must be paired with the name of the corresponding struct in `complex_declare!`. ### Example program @@ -138,5 +131,5 @@ Here is the full `openvm.toml` to accompany the above example: supported_modulus = ["998244353","1000000007"] [app_vm_config.fp2] -supported_modulus = ["998244353","1000000007"] +supported_modulus = [["Complex1", "998244353"], ["Complex2", "1000000007"]] ``` diff --git a/book/src/custom-extensions/ecc.md b/book/src/custom-extensions/ecc.md index 331430d6ad..25f98ba183 100644 --- a/book/src/custom-extensions/ecc.md +++ b/book/src/custom-extensions/ecc.md @@ -43,12 +43,15 @@ sw_declare! { Each declared curve must specify the `mod_type` (implementing `IntMod`) and a constant `b` for the Weierstrass curve equation \\(y^2 = x^3 + ax + b\\). `a` is optional and defaults to 0 for short Weierstrass curves. This creates `Bls12_381G1Affine` and `P256Affine` structs which implement the `Group` and `WeierstrassPoint` traits. The underlying memory layout of the structs uses the memory layout of the `Bls12_381Fp` and `P256Coord` structs, respectively. -2. **Init**: Called once, it enumerates these curves and allows the compiler to produce optimized instructions: +2. **Init**: Called once, the `init!` macro produces a call to `sw_init!` that enumerates these curves and allows the compiler to produce optimized instructions: ```rust +init!(); +/* This expands to sw_init! { Bls12_381G1Affine, P256Affine, } +*/ ``` 3. **Setup**: Similar to the moduli and complex extensions, runtime setup instructions ensure that the correct curve parameters are being used, guaranteeing secure operation. @@ -56,7 +59,7 @@ sw_init! { **Summary**: - `sw_declare!`: Declares elliptic curve structures. -- `sw_init!`: Initializes them once, linking them to the underlying moduli. +- `init!`: Initializes them once, linking them to the underlying moduli. - `setup_sw_()`/`setup_all_curves()`: Secures runtime correctness. To use elliptic curve operations on a struct defined with `sw_declare!`, it is expected that the struct for the curve's coordinate field was defined using `moduli_declare!`. In particular, the coordinate field needs to be initialized and set up as described in the [algebra extension](./algebra.md) chapter. @@ -86,7 +89,7 @@ One can define their own ECC structs but we will use the Secp256k1 struct from ` {{ #include ../../../examples/ecc/src/main.rs:init }} ``` -We `moduli_init!` both the coordinate and scalar field because they were declared in the `k256` module, although we will not be using the scalar field below. +`moduli_init!` is called for both the coordinate and scalar field because they were declared in the `k256` module, although we will not be using the scalar field below. With the above we can start doing elliptic curve operations like adding points: @@ -103,6 +106,7 @@ For the guest program to build successfully, all used moduli and curves must be supported_modulus = ["115792089237316195423570985008687907853269984665640564039457584007908834671663", "115792089237316195423570985008687907852837564279074904382605163141518161494337"] [[app_vm_config.ecc.supported_curves]] +struct_name = "Secp256k1Point" modulus = "115792089237316195423570985008687907853269984665640564039457584007908834671663" scalar = "115792089237316195423570985008687907852837564279074904382605163141518161494337" a = "0" @@ -112,3 +116,5 @@ b = "7" The `supported_modulus` parameter is a list of moduli that the guest program will use. As mentioned in the [algebra extension](./algebra.md) chapter, the order of moduli in `[app_vm_config.modular]` must match the order in the `moduli_init!` macro. The `ecc.supported_curves` parameter is a list of supported curves that the guest program will use. They must be provided in decimal format in the `.toml` file. For multiple curves create multiple `[[app_vm_config.ecc.supported_curves]]` sections. The order of curves in `[[app_vm_config.ecc.supported_curves]]` must match the order in the `sw_init!` macro. +Also, the `struct_name` field must be the name of the elliptic curve struct created by `sw_declare!`. +In this example, the `Secp256k1Point` struct is created in `openvm_ecc_guest::k256`. diff --git a/book/src/custom-extensions/pairing.md b/book/src/custom-extensions/pairing.md index f2ed43970c..807de50e20 100644 --- a/book/src/custom-extensions/pairing.md +++ b/book/src/custom-extensions/pairing.md @@ -101,7 +101,7 @@ supported_modulus = [ [app_vm_config.fp2] supported_modulus = [ - "4002409555221667393417789825735904156556882819939007885332058136124031650490837864442687629129015664037894272559787", + ["Bls12_381Fp2", "4002409555221667393417789825735904156556882819939007885332058136124031650490837864442687629129015664037894272559787"], ] ``` diff --git a/crates/cli/Cargo.toml b/crates/cli/Cargo.toml index a5d96e07a4..9e916c9641 100644 --- a/crates/cli/Cargo.toml +++ b/crates/cli/Cargo.toml @@ -23,6 +23,7 @@ openvm-native-recursion = { workspace = true } openvm-sdk = { workspace = true } openvm-stark-sdk.workspace = true openvm-stark-backend.workspace = true +openvm-circuit = { workspace = true } aws-sdk-s3 = "1.78" aws-config = "1.5" diff --git a/crates/cli/src/commands/build.rs b/crates/cli/src/commands/build.rs index 80fc9c0733..22665b0b41 100644 --- a/crates/cli/src/commands/build.rs +++ b/crates/cli/src/commands/build.rs @@ -10,6 +10,7 @@ use eyre::Result; use openvm_build::{ build_guest_package, find_unique_executable, get_package, GuestOptions, TargetFilter, }; +use openvm_circuit::arch::{InitFileGenerator, OPENVM_DEFAULT_INIT_FILE_NAME}; use openvm_sdk::{ commit::{commit_app_exe, committed_exe_as_bn254}, fs::write_exe_to_file, @@ -97,6 +98,9 @@ pub struct BuildArgs { #[arg(long, default_value = "false", help = "use --offline in cargo build")] pub offline: bool, + + #[arg(long, default_value = OPENVM_DEFAULT_INIT_FILE_NAME, help = "Name of the init file")] + pub init_file_name: String, } #[derive(Clone, Default, clap::Args)] @@ -136,6 +140,11 @@ pub(crate) fn build(build_args: &BuildArgs) -> Result> { guest_options.options = vec!["--offline".to_string()]; } + let app_config = read_config_toml_or_default(&build_args.config)?; + app_config + .app_vm_config + .write_to_init_file(&build_args.manifest_dir, Some(&build_args.init_file_name))?; + let pkg = get_package(&build_args.manifest_dir); // We support builds of libraries with 0 or >1 executables. let elf_path = match build_guest_package(&pkg, &guest_options, None, &target_filter) { @@ -154,7 +163,6 @@ pub(crate) fn build(build_args: &BuildArgs) -> Result> { let elf_path = elf_path?; println!("[openvm] Transpiling the package..."); let output_path = &build_args.exe_output; - let app_config = read_config_toml_or_default(&build_args.config)?; let transpiler = app_config.app_vm_config.transpiler(); let data = read(elf_path.clone())?; @@ -224,6 +232,7 @@ mod tests { profile: "dev".to_string(), target_dir: Some(target_dir.to_path_buf()), offline: false, + init_file_name: OPENVM_DEFAULT_INIT_FILE_NAME.to_string(), }; build(&build_args)?; assert!( diff --git a/crates/sdk/examples/sdk_app.rs b/crates/sdk/examples/sdk_app.rs index fb78a01a3a..0b3a500e2a 100644 --- a/crates/sdk/examples/sdk_app.rs +++ b/crates/sdk/examples/sdk_app.rs @@ -56,7 +56,13 @@ fn main() -> Result<(), Box> { // 2a. Build the ELF with guest options and a target filter. let guest_opts = GuestOptions::default(); let target_path = "your_path_project_root"; - let elf = sdk.build(guest_opts, target_path, &Default::default())?; + let elf = sdk.build( + guest_opts, + &vm_config, + target_path, + &Default::default(), + None, + )?; // ANCHOR_END: build // ANCHOR: transpilation diff --git a/crates/sdk/examples/sdk_evm.rs b/crates/sdk/examples/sdk_evm.rs index 7126870550..7d42dd6232 100644 --- a/crates/sdk/examples/sdk_evm.rs +++ b/crates/sdk/examples/sdk_evm.rs @@ -56,7 +56,13 @@ fn main() -> Result<(), Box> { // 2a. Build the ELF with guest options and a target filter. let guest_opts = GuestOptions::default(); let target_path = "your_path_project_root"; - let elf = sdk.build(guest_opts, target_path, &Default::default())?; + let elf = sdk.build( + guest_opts, + &vm_config, + target_path, + &Default::default(), + None, + )?; // ANCHOR_END: build // ANCHOR: transpilation diff --git a/crates/sdk/src/config/global.rs b/crates/sdk/src/config/global.rs index 532c9b8d1c..a28afd1918 100644 --- a/crates/sdk/src/config/global.rs +++ b/crates/sdk/src/config/global.rs @@ -9,7 +9,8 @@ use openvm_bigint_circuit::{Int256, Int256Executor, Int256Periphery}; use openvm_bigint_transpiler::Int256TranspilerExtension; use openvm_circuit::{ arch::{ - SystemConfig, SystemExecutor, SystemPeriphery, VmChipComplex, VmConfig, VmInventoryError, + InitFileGenerator, SystemConfig, SystemExecutor, SystemPeriphery, VmChipComplex, VmConfig, + VmInventoryError, }, circuit_derive::{Chip, ChipUsageGetter}, derive::{AnyEnum, InstructionExecutor}, @@ -233,11 +234,49 @@ impl VmConfig for SdkVmConfig { } } +impl InitFileGenerator for SdkVmConfig { + fn generate_init_file_contents(&self) -> Option { + if self.modular.is_some() || self.fp2.is_some() || self.ecc.is_some() { + let mut contents = String::new(); + contents.push_str( + "// This file is automatically generated by cargo openvm. Do not rename or edit.\n", + ); + + if let Some(modular_config) = &self.modular { + contents.push_str(&modular_config.generate_moduli_init()); + contents.push('\n'); + } + + if let Some(fp2_config) = &self.fp2 { + assert!( + self.modular.is_some(), + "ModularExtension is required for Fp2Extension" + ); + let modular_config = self.modular.as_ref().unwrap(); + contents.push_str(&fp2_config.generate_complex_init(modular_config)); + contents.push('\n'); + } + + if let Some(ecc_config) = &self.ecc { + contents.push_str(&ecc_config.generate_sw_init()); + contents.push('\n'); + } + + Some(contents) + } else { + None + } + } +} + #[derive(Clone, Debug, Serialize, Deserialize)] pub struct SdkSystemConfig { pub config: SystemConfig, } +// Default implementation uses no init file +impl InitFileGenerator for SdkSystemConfig {} + impl Default for SdkSystemConfig { fn default() -> Self { Self { diff --git a/crates/sdk/src/lib.rs b/crates/sdk/src/lib.rs index 193c161580..d63bcefdb4 100644 --- a/crates/sdk/src/lib.rs +++ b/crates/sdk/src/lib.rs @@ -12,8 +12,8 @@ use openvm_build::{ use openvm_circuit::{ arch::{ hasher::poseidon2::vm_poseidon2_hasher, instructions::exe::VmExe, verify_segments, - ContinuationVmProof, ExecutionError, VerifiedExecutionPayload, VmConfig, VmExecutor, - VmVerificationError, + ContinuationVmProof, ExecutionError, InitFileGenerator, VerifiedExecutionPayload, VmConfig, + VmExecutor, VmVerificationError, }, system::{ memory::{tree::public_values::extract_public_values, CHUNK}, @@ -42,7 +42,7 @@ use openvm_transpiler::{ use snark_verifier_sdk::{evm::gen_evm_verifier_sol_code, halo2::aggregation::AggregationCircuit}; use crate::{ - config::AggConfig, + config::{AggConfig, SdkVmConfig}, keygen::{AggProvingKey, AggStarkProvingKey}, prover::{AppProver, StarkProver}, }; @@ -125,9 +125,12 @@ impl> GenericSdk { pub fn build>( &self, guest_opts: GuestOptions, + vm_config: &SdkVmConfig, pkg_dir: P, target_filter: &Option, + init_file_name: Option<&str>, // If None, we use "openvm-init.rs" ) -> Result { + vm_config.write_to_init_file(pkg_dir.as_ref(), init_file_name)?; let pkg = get_package(pkg_dir.as_ref()); let target_dir = match build_guest_package(&pkg, &guest_opts, None, target_filter) { Ok(target_dir) => target_dir, diff --git a/crates/sdk/tests/integration_test.rs b/crates/sdk/tests/integration_test.rs index 7d70b8de67..d34bdabfb4 100644 --- a/crates/sdk/tests/integration_test.rs +++ b/crates/sdk/tests/integration_test.rs @@ -410,7 +410,13 @@ fn test_e2e_proof_generation_and_verification_with_pvs() { let sdk = Sdk::new(); let elf = sdk - .build(Default::default(), pkg_dir, &Default::default()) + .build( + Default::default(), + &vm_config, + pkg_dir, + &Default::default(), + None, + ) .unwrap(); let exe = sdk.transpile(elf, vm_config.transpiler()).unwrap(); @@ -464,11 +470,37 @@ fn test_sdk_guest_build_and_transpile() { ; let mut pkg_dir = PathBuf::from(env!("CARGO_MANIFEST_DIR")).to_path_buf(); pkg_dir.push("guest"); + + let vm_config = SdkVmConfig::builder() + .system(SdkSystemConfig { + config: SystemConfig::default() + .with_max_segment_len(200) + .with_continuations() + .with_public_values(NUM_PUB_VALUES), + }) + .rv32i(Default::default()) + .rv32m(Default::default()) + .io(Default::default()) + .native(Default::default()) + .build(); + let one = sdk - .build(guest_opts.clone(), &pkg_dir, &Default::default()) + .build( + guest_opts.clone(), + &vm_config, + &pkg_dir, + &Default::default(), + None, + ) .unwrap(); let two = sdk - .build(guest_opts.clone(), &pkg_dir, &Default::default()) + .build( + guest_opts.clone(), + &vm_config, + &pkg_dir, + &Default::default(), + None, + ) .unwrap(); assert_eq!(one.instructions, two.instructions); assert_eq!(one.instructions, two.instructions); @@ -485,7 +517,6 @@ fn test_inner_proof_codec_roundtrip() -> eyre::Result<()> { let sdk = Sdk::new(); let mut pkg_dir = PathBuf::from(env!("CARGO_MANIFEST_DIR")).to_path_buf(); pkg_dir.push("guest"); - let elf = sdk.build(Default::default(), pkg_dir, &Default::default())?; let vm_config = SdkVmConfig::builder() .system(SdkSystemConfig { @@ -499,6 +530,13 @@ fn test_inner_proof_codec_roundtrip() -> eyre::Result<()> { .io(Default::default()) .native(Default::default()) .build(); + let elf = sdk.build( + Default::default(), + &vm_config, + pkg_dir, + &Default::default(), + None, + )?; assert!(vm_config.system.config.continuation_enabled); let exe = sdk.transpile(elf, vm_config.transpiler())?; let fri_params = FriParameters::standard_fast(); diff --git a/crates/toolchain/openvm/src/lib.rs b/crates/toolchain/openvm/src/lib.rs index 134d10b559..00decf04d9 100644 --- a/crates/toolchain/openvm/src/lib.rs +++ b/crates/toolchain/openvm/src/lib.rs @@ -162,3 +162,14 @@ fn panic_impl(panic_info: &core::panic::PanicInfo) -> ! { openvm_platform::rust_rt::terminate::<1>(); unreachable!() } + +// Includes the openvm_init.rs file generated at build time +#[macro_export] +macro_rules! init { + () => { + include!(concat!(env!("CARGO_MANIFEST_DIR"), "/openvm_init.rs")); + }; + ($name:expr) => { + include!(concat!(env!("CARGO_MANIFEST_DIR"), concat!("/", $name))); + }; +} diff --git a/crates/toolchain/tests/src/lib.rs b/crates/toolchain/tests/src/lib.rs index 6fdae2b4d0..7c9c01f62e 100644 --- a/crates/toolchain/tests/src/lib.rs +++ b/crates/toolchain/tests/src/lib.rs @@ -7,6 +7,7 @@ use eyre::{Context, Result}; use openvm_build::{ build_guest_package, get_dir_with_profile, get_package, GuestOptions, TargetFilter, }; +use openvm_circuit::arch::{InitFileGenerator, OPENVM_DEFAULT_INIT_FILE_BASENAME}; use openvm_transpiler::{elf::Elf, openvm_platform::memory::MEM_SIZE}; use tempfile::tempdir; @@ -30,33 +31,63 @@ pub fn decode_elf(elf_path: impl AsRef) -> Result { Elf::decode(&data, MEM_SIZE as u32) } -pub fn build_example_program(example_name: &str) -> Result { - build_example_program_with_features::<&str>(example_name, []) +// Some tests need to manually override the init macro build script (e.g. to test invalid moduli), +// so we can use this struct to avoid generating an init file +pub struct NoInitFile; +impl InitFileGenerator for NoInitFile {} + +pub fn build_example_program( + example_name: &str, + init_config: &impl InitFileGenerator, +) -> Result { + build_example_program_with_features::<&str>(example_name, [], init_config) } pub fn build_example_program_with_features>( example_name: &str, - features: impl IntoIterator, + features: impl IntoIterator + Clone, + init_config: &impl InitFileGenerator, ) -> Result { let manifest_dir = get_programs_dir!(); - build_example_program_at_path_with_features(manifest_dir, example_name, features) + build_example_program_at_path_with_features(manifest_dir, example_name, features, init_config) } -pub fn build_example_program_at_path(manifest_dir: PathBuf, example_name: &str) -> Result { - build_example_program_at_path_with_features::<&str>(manifest_dir, example_name, []) +pub fn build_example_program_at_path( + manifest_dir: PathBuf, + example_name: &str, + init_config: &impl InitFileGenerator, +) -> Result { + build_example_program_at_path_with_features::<&str>(manifest_dir, example_name, [], init_config) } pub fn build_example_program_at_path_with_features>( manifest_dir: PathBuf, example_name: &str, - features: impl IntoIterator, + features: impl IntoIterator + Clone, + init_config: &impl InitFileGenerator, ) -> Result { - let pkg = get_package(manifest_dir); + let pkg = get_package(&manifest_dir); let target_dir = tempdir()?; // Build guest with default features let guest_opts = GuestOptions::default() - .with_features(features) + .with_features(features.clone()) .with_target_dir(target_dir.path()); + let features = features + .into_iter() + .map(|x| x.as_ref().to_string()) + .collect::>(); + let features_str = if !features.is_empty() { + format!("_{}", features.join("_")) + } else { + "".to_string() + }; + init_config.write_to_init_file( + &manifest_dir, + Some(&format!( + "{}_{}{}.rs", + OPENVM_DEFAULT_INIT_FILE_BASENAME, example_name, features_str + )), + )?; if let Err(Some(code)) = build_guest_package( &pkg, &guest_opts, diff --git a/crates/toolchain/tests/tests/transpiler_tests.rs b/crates/toolchain/tests/tests/transpiler_tests.rs index 82d74afcca..d0b8497240 100644 --- a/crates/toolchain/tests/tests/transpiler_tests.rs +++ b/crates/toolchain/tests/tests/transpiler_tests.rs @@ -12,7 +12,7 @@ use openvm_algebra_circuit::{ use openvm_algebra_transpiler::{Fp2TranspilerExtension, ModularTranspilerExtension}; use openvm_bigint_circuit::{Int256, Int256Executor, Int256Periphery}; use openvm_circuit::{ - arch::{SystemConfig, VmExecutor}, + arch::{InitFileGenerator, SystemConfig, VmExecutor}, derive::VmConfig, utils::air_test, }; @@ -104,7 +104,7 @@ pub struct Rv32ModularFp2Int256Config { } impl Rv32ModularFp2Int256Config { - pub fn new(modular_moduli: Vec, fp2_moduli: Vec) -> Self { + pub fn new(modular_moduli: Vec, fp2_moduli: Vec<(String, BigUint)>) -> Self { Self { system: SystemConfig::default().with_continuations(), base: Default::default(), @@ -117,11 +117,21 @@ impl Rv32ModularFp2Int256Config { } } +impl InitFileGenerator for Rv32ModularFp2Int256Config { + fn generate_init_file_contents(&self) -> Option { + Some(format!( + "{}\n{}\n", + self.modular.generate_moduli_init(), + self.fp2.generate_complex_init(&self.modular) + )) + } +} + #[test_case("tests/data/rv32im-intrin-from-as")] fn test_intrinsic_runtime(elf_path: &str) -> Result<()> { let config = Rv32ModularFp2Int256Config::new( vec![SECP256K1_MODULUS.clone(), SECP256K1_ORDER.clone()], - vec![SECP256K1_MODULUS.clone()], + vec![("Secp256k1Coord".to_string(), SECP256K1_MODULUS.clone())], ); let elf = get_elf(elf_path)?; let openvm_exe = VmExe::from_elf( diff --git a/crates/vm/src/arch/config.rs b/crates/vm/src/arch/config.rs index 62185779f4..d82b5f7cf0 100644 --- a/crates/vm/src/arch/config.rs +++ b/crates/vm/src/arch/config.rs @@ -1,4 +1,4 @@ -use std::sync::Arc; +use std::{fs::File, io::Write, path::Path, sync::Arc}; use derive_new::new; use openvm_circuit::system::memory::MemoryTraceHeights; @@ -24,7 +24,9 @@ pub fn vm_poseidon2_config() -> Poseidon2Config { Poseidon2Config::default() } -pub trait VmConfig: Clone + Serialize + DeserializeOwned { +pub trait VmConfig: + Clone + Serialize + DeserializeOwned + InitFileGenerator +{ type Executor: InstructionExecutor + AnyEnum + ChipUsageGetter; type Periphery: AnyEnum + ChipUsageGetter; @@ -37,6 +39,35 @@ pub trait VmConfig: Clone + Serialize + DeserializeOwned { ) -> Result, VmInventoryError>; } +pub const OPENVM_DEFAULT_INIT_FILE_BASENAME: &str = "openvm_init"; +pub const OPENVM_DEFAULT_INIT_FILE_NAME: &str = "openvm_init.rs"; + +/// Trait for generating a init.rs file that contains a call to moduli_init!, +/// complex_init!, sw_init! with the supported moduli and curves. +/// Should be implemented by all VM config structs. +pub trait InitFileGenerator { + // Default implementation is no init file. + fn generate_init_file_contents(&self) -> Option { + None + } + + // Do not override this method's default implementation. + // This method is called by cargo openvm and the SDK before building the guest package. + fn write_to_init_file( + &self, + manifest_dir: &Path, + init_file_name: Option<&str>, + ) -> eyre::Result<()> { + if let Some(contents) = self.generate_init_file_contents() { + let dest_path = Path::new(manifest_dir) + .join(init_file_name.unwrap_or(OPENVM_DEFAULT_INIT_FILE_NAME)); + let mut f = File::create(&dest_path)?; + write!(f, "{}", contents)?; + } + Ok(()) + } +} + #[derive(Debug, Serialize, Deserialize, Clone, new, Copy)] pub struct MemoryConfig { /// The maximum height of the address space. This means the trie has `as_height` layers for @@ -222,3 +253,6 @@ impl VmConfig for SystemConfig { Ok(complex) } } + +// Default implementation uses no init file +impl InitFileGenerator for SystemConfig {} diff --git a/docs/crates/vm-extensions.md b/docs/crates/vm-extensions.md index 70ddef0277..490bac08c5 100644 --- a/docs/crates/vm-extensions.md +++ b/docs/crates/vm-extensions.md @@ -152,6 +152,14 @@ function. What this does in words: For each extension's inventory generation, the `VmInventoryBuilder` is provided with a view of all current chips already inside the running chip complex. This means the inventory generation process is sequential in the order the extensions are specified, and each extension has borrow access to all chips constructed by any extension before it. +## Build hooks +Some of our extensions need to generate some code at build-time depending on the VM config (for example, the Algebra extension needs to call `moduli_init!` with the appropriate moduli). +To accommodate this, we support build hooks in both `cargo openvm` and the SDK. +To make use of this functionality, implement the `InitFileGenerator` trait. +The `String` returned by the `generate_init_file_contents` must be valid Rust code. +It will be written to a `openvm_init.rs` file in the package's manifest directory, and then (unhygenically) included in the guest code in place of the `openvm::init!` macro. +You can specify a custom file name at build time (by a `cargo openvm` option or an SDK method argument), in which case you must also pass it to `openvm::init!` as an argument. + ## Examples The [`extensions/`](../../extensions/) folder contains extensions implementing all non-system functionality via custom extensions. For example, the `Rv32I`, `Rv32M`, and `Rv32Io` extensions implement `VmExtension` in [`openvm-rv32im-circuit`](../../extensions/rv32im/circuit/) and correspond to the RISC-V 32-bit base and multiplication instruction sets and an extension for IO, respectively. diff --git a/examples/algebra/Cargo.toml b/examples/algebra/Cargo.toml index 58a7402e68..2ced09c910 100644 --- a/examples/algebra/Cargo.toml +++ b/examples/algebra/Cargo.toml @@ -7,12 +7,11 @@ edition = "2021" members = [] [dependencies] -openvm = { git = "https://github.com/openvm-org/openvm.git", features = [ - "std", -] } -openvm-platform = { git = "https://github.com/openvm-org/openvm.git" } -openvm-algebra-guest = { git = "https://github.com/openvm-org/openvm.git" } -openvm-algebra-complex-macros = { git = "https://github.com/openvm-org/openvm.git" } +openvm = { path = "../../crates/toolchain/openvm", features = ["std"] } +openvm-platform = { path = "../../crates/toolchain/platform" } +openvm-algebra-guest = { path = "../../extensions/algebra/guest" } +openvm-algebra-complex-macros = { path = "../../extensions/algebra/complex-macros" } + serde = { version = "1.0.216" } num-bigint = { version = "0.4.6", features = ["serde"] } diff --git a/examples/algebra/openvm.toml b/examples/algebra/openvm.toml index fcdce806e4..166a96c357 100644 --- a/examples/algebra/openvm.toml +++ b/examples/algebra/openvm.toml @@ -5,4 +5,4 @@ supported_modulus = ["998244353","1000000007"] [app_vm_config.fp2] -supported_modulus = ["998244353","1000000007"] \ No newline at end of file +supported_modulus = [["Complex1", "998244353"], ["Complex2", "1000000007"]] \ No newline at end of file diff --git a/examples/algebra/openvm_init.rs b/examples/algebra/openvm_init.rs new file mode 100644 index 0000000000..1de98b97a1 --- /dev/null +++ b/examples/algebra/openvm_init.rs @@ -0,0 +1,3 @@ +// This file is automatically generated by cargo openvm. Do not rename or edit. +openvm_algebra_guest::moduli_macros::moduli_init! { "998244353", "1000000007" } +openvm_algebra_guest::complex_macros::complex_init! { Complex1 { mod_idx = 0 }, Complex2 { mod_idx = 1 } } diff --git a/examples/algebra/src/main.rs b/examples/algebra/src/main.rs index e386041f9b..72c5d78ed7 100644 --- a/examples/algebra/src/main.rs +++ b/examples/algebra/src/main.rs @@ -7,12 +7,6 @@ moduli_declare! { Mod2 { modulus = "1000000007" } } -// This macro will initialize the moduli. -// Now, `Mod1` is the "zeroth" modular struct, and `Mod2` is the "first" one. -moduli_init! { - "998244353", "1000000007" -} - // This macro will create two structs, `Complex1` and `Complex2`, // one for arithmetic in the field $\mathbb{F}_{998244353}[x]/(x^2 + 1)$, // and the other for arithmetic in the field $\mathbb{F}_{1000000007}[x]/(x^2 + 1)$. @@ -21,11 +15,20 @@ openvm_algebra_complex_macros::complex_declare! { Complex2 { mod_type = Mod2 }, } +openvm::init!(); +/* The init! macro will expand to the following (excluding comments): +// This macro will initialize the moduli. +// Now, `Mod1` is the "zeroth" modular struct, and `Mod2` is the "first" one. +moduli_init! { + "998244353", "1000000007" +} + // The order of these structs does not matter, // given that we specify the `mod_idx` parameters properly. openvm_algebra_complex_macros::complex_init! { Complex1 { mod_idx = 0 }, Complex2 { mod_idx = 1 }, } +*/ pub fn main() { // Since we only use an arithmetic operation with `Mod1` and not `Mod2`, diff --git a/examples/ecc/openvm.toml b/examples/ecc/openvm.toml index bd968d85a2..47980b422d 100644 --- a/examples/ecc/openvm.toml +++ b/examples/ecc/openvm.toml @@ -5,6 +5,7 @@ supported_modulus = ["115792089237316195423570985008687907853269984665640564039457584007908834671663", "115792089237316195423570985008687907852837564279074904382605163141518161494337"] [[app_vm_config.ecc.supported_curves]] +struct_name = "Secp256k1Point" modulus = "115792089237316195423570985008687907853269984665640564039457584007908834671663" scalar = "115792089237316195423570985008687907852837564279074904382605163141518161494337" a = "0" diff --git a/examples/ecc/openvm_init.rs b/examples/ecc/openvm_init.rs new file mode 100644 index 0000000000..bec9f527e9 --- /dev/null +++ b/examples/ecc/openvm_init.rs @@ -0,0 +1,3 @@ +// This file is automatically generated by cargo openvm. Do not rename or edit. +openvm_algebra_guest::moduli_macros::moduli_init! { "115792089237316195423570985008687907853269984665640564039457584007908834671663", "115792089237316195423570985008687907852837564279074904382605163141518161494337" } +openvm_ecc_guest::sw_macros::sw_init! { Secp256k1Point } diff --git a/examples/ecc/src/main.rs b/examples/ecc/src/main.rs index 2fbb4ba45b..c8e3299dc5 100644 --- a/examples/ecc/src/main.rs +++ b/examples/ecc/src/main.rs @@ -8,6 +8,8 @@ use openvm_ecc_guest::{ // ANCHOR_END: imports // ANCHOR: init +openvm::init!(); +/* The init! macro will expand to the following openvm_algebra_guest::moduli_macros::moduli_init! { "0xFFFFFFFF FFFFFFFF FFFFFFFF FFFFFFFF FFFFFFFF FFFFFFFF FFFFFFFE FFFFFC2F", "0xFFFFFFFF FFFFFFFF FFFFFFFF FFFFFFFE BAAEDCE6 AF48A03B BFD25E8C D0364141" @@ -16,6 +18,7 @@ openvm_algebra_guest::moduli_macros::moduli_init! { openvm_ecc_guest::sw_macros::sw_init! { Secp256k1Point, } +*/ // ANCHOR_END: init // ANCHOR: main diff --git a/examples/pairing/Cargo.toml b/examples/pairing/Cargo.toml index dd64cfea53..a5812dd4ca 100644 --- a/examples/pairing/Cargo.toml +++ b/examples/pairing/Cargo.toml @@ -7,16 +7,17 @@ edition = "2021" members = [] [dependencies] -openvm = { git = "https://github.com/openvm-org/openvm.git", features = [ - "std", -] } -openvm-algebra-guest = { git = "https://github.com/openvm-org/openvm.git" } -openvm-algebra-moduli-macros = { git = "https://github.com/openvm-org/openvm.git" } -openvm-algebra-complex-macros = { git = "https://github.com/openvm-org/openvm.git" } -openvm-ecc-guest = { git = "https://github.com/openvm-org/openvm.git" } -openvm-pairing-guest = { git = "https://github.com/openvm-org/openvm.git", features = [ + +openvm = { path = "../../crates/toolchain/openvm", features = ["std"] } +openvm-algebra-guest = { path = "../../extensions/algebra/guest" } +openvm-algebra-moduli-macros = { path = "../../extensions/algebra/moduli-macros" } +openvm-algebra-complex-macros = { path = "../../extensions/algebra/complex-macros" } +openvm-ecc-guest = { path = "../../extensions/ecc/guest" } +openvm-pairing-guest = { path = "../../extensions/pairing/guest", features = [ "bls12_381", ] } +openvm-platform = { path = "../../crates/toolchain/platform" } + hex-literal = { version = "0.4.1", default-features = false } [features] diff --git a/examples/pairing/openvm.toml b/examples/pairing/openvm.toml index 94337448fe..3591f1dcd6 100644 --- a/examples/pairing/openvm.toml +++ b/examples/pairing/openvm.toml @@ -11,5 +11,5 @@ supported_modulus = [ [app_vm_config.fp2] supported_modulus = [ - "4002409555221667393417789825735904156556882819939007885332058136124031650490837864442687629129015664037894272559787", + ["Bls12_381Fp2", "4002409555221667393417789825735904156556882819939007885332058136124031650490837864442687629129015664037894272559787"], ] \ No newline at end of file diff --git a/examples/pairing/openvm_init.rs b/examples/pairing/openvm_init.rs new file mode 100644 index 0000000000..991d1237fc --- /dev/null +++ b/examples/pairing/openvm_init.rs @@ -0,0 +1,3 @@ +// This file is automatically generated by cargo openvm. Do not rename or edit. +openvm_algebra_guest::moduli_macros::moduli_init! { "4002409555221667393417789825735904156556882819939007885332058136124031650490837864442687629129015664037894272559787" } +openvm_algebra_guest::complex_macros::complex_init! { Bls12_381Fp2 { mod_idx = 0 } } diff --git a/examples/pairing/src/main.rs b/examples/pairing/src/main.rs index c1d19a1756..60a601efe3 100644 --- a/examples/pairing/src/main.rs +++ b/examples/pairing/src/main.rs @@ -11,6 +11,8 @@ use openvm_pairing_guest::{ // ANCHOR_END: imports // ANCHOR: init +openvm::init!(); +/* The init! macro will expand to the following openvm_algebra_moduli_macros::moduli_init! { "0x1a0111ea397fe69a4b1ba7b6434bacd764774b84f38512bf6730d2a0f6b0f6241eabfffeb153ffffb9feffffffffaaab", "0x73eda753299d7d483339d80809a1d80553bda402fffe5bfeffffffff00000001" @@ -19,6 +21,7 @@ openvm_algebra_moduli_macros::moduli_init! { openvm_algebra_complex_macros::complex_init! { Bls12_381Fp2 { mod_idx = 0 }, } +*/ // ANCHOR_END: init // ANCHOR: main diff --git a/extensions/algebra/circuit/src/config.rs b/extensions/algebra/circuit/src/config.rs index 3f2e83fef3..5b43163b77 100644 --- a/extensions/algebra/circuit/src/config.rs +++ b/extensions/algebra/circuit/src/config.rs @@ -1,5 +1,5 @@ use num_bigint::BigUint; -use openvm_circuit::arch::SystemConfig; +use openvm_circuit::arch::{InitFileGenerator, SystemConfig}; use openvm_circuit_derive::VmConfig; use openvm_rv32im_circuit::*; use openvm_stark_backend::p3_field::PrimeField32; @@ -21,6 +21,15 @@ pub struct Rv32ModularConfig { pub modular: ModularExtension, } +impl InitFileGenerator for Rv32ModularConfig { + fn generate_init_file_contents(&self) -> Option { + Some(format!( + "// This file is automatically generated by cargo openvm. Do not rename or edit.\n{}\n", + self.modular.generate_moduli_init() + )) + } +} + impl Rv32ModularConfig { pub fn new(moduli: Vec) -> Self { Self { @@ -50,14 +59,28 @@ pub struct Rv32ModularWithFp2Config { } impl Rv32ModularWithFp2Config { - pub fn new(moduli: Vec) -> Self { + pub fn new(moduli_with_names: Vec<(String, BigUint)>) -> Self { + let moduli = moduli_with_names + .iter() + .map(|(_, modulus)| modulus.clone()) + .collect(); Self { system: SystemConfig::default().with_continuations(), base: Default::default(), mul: Default::default(), io: Default::default(), - modular: ModularExtension::new(moduli.clone()), - fp2: Fp2Extension::new(moduli), + modular: ModularExtension::new(moduli), + fp2: Fp2Extension::new(moduli_with_names), } } } + +impl InitFileGenerator for Rv32ModularWithFp2Config { + fn generate_init_file_contents(&self) -> Option { + Some(format!( + "// This file is automatically generated by cargo openvm. Do not rename or edit.\n{}\n{}\n", + self.modular.generate_moduli_init(), + self.fp2.generate_complex_init(&self.modular) + )) + } +} diff --git a/extensions/algebra/circuit/src/fp2_extension.rs b/extensions/algebra/circuit/src/fp2_extension.rs index 940ec4c864..30a55cf015 100644 --- a/extensions/algebra/circuit/src/fp2_extension.rs +++ b/extensions/algebra/circuit/src/fp2_extension.rs @@ -18,13 +18,45 @@ use serde::{Deserialize, Serialize}; use serde_with::{serde_as, DisplayFromStr}; use strum::EnumCount; -use crate::fp2_chip::{Fp2AddSubChip, Fp2MulDivChip}; +use crate::{ + fp2_chip::{Fp2AddSubChip, Fp2MulDivChip}, + ModularExtension, +}; #[serde_as] #[derive(Clone, Debug, derive_new::new, Serialize, Deserialize)] pub struct Fp2Extension { - #[serde_as(as = "Vec")] - pub supported_modulus: Vec, + // (name, modulus) + // name must match the struct name defined by complex_declare + #[serde_as(as = "Vec<(_, DisplayFromStr)>")] + pub supported_modulus: Vec<(String, BigUint)>, +} + +impl Fp2Extension { + pub fn generate_complex_init(&self, modular_config: &ModularExtension) -> String { + fn get_index_of_modulus(modulus: &BigUint, modular_config: &ModularExtension) -> usize { + modular_config + .supported_modulus + .iter() + .position(|m| m == modulus) + .expect("Modulus used in Fp2Extension not found in ModularExtension") + } + + let supported_moduli = self + .supported_modulus + .iter() + .map(|(name, modulus)| { + format!( + "{} {{ mod_idx = {} }}", + name, + get_index_of_modulus(modulus, modular_config) + ) + }) + .collect::>() + .join(", "); + + format!("openvm_algebra_guest::complex_macros::complex_init! {{ {supported_moduli} }}") + } } #[derive(ChipUsageGetter, Chip, InstructionExecutor, AnyEnum, From)] @@ -76,7 +108,7 @@ impl VmExtension for Fp2Extension { let addsub_opcodes = (Fp2Opcode::ADD as usize)..=(Fp2Opcode::SETUP_ADDSUB as usize); let muldiv_opcodes = (Fp2Opcode::MUL as usize)..=(Fp2Opcode::SETUP_MULDIV as usize); - for (i, modulus) in self.supported_modulus.iter().enumerate() { + for (i, (_, modulus)) in self.supported_modulus.iter().enumerate() { // determine the number of bytes needed to represent a prime field element let bytes = modulus.bits().div_ceil(8); let start_offset = Fp2Opcode::CLASS_OFFSET + i * Fp2Opcode::COUNT; diff --git a/extensions/algebra/circuit/src/modular_extension.rs b/extensions/algebra/circuit/src/modular_extension.rs index 18a19becfa..6050045aed 100644 --- a/extensions/algebra/circuit/src/modular_extension.rs +++ b/extensions/algebra/circuit/src/modular_extension.rs @@ -30,6 +30,20 @@ pub struct ModularExtension { pub supported_modulus: Vec, } +impl ModularExtension { + // Generates a call to the moduli_init! macro with moduli in the correct order + pub fn generate_moduli_init(&self) -> String { + let supported_moduli = self + .supported_modulus + .iter() + .map(|modulus| format!("\"{}\"", modulus)) + .collect::>() + .join(", "); + + format!("openvm_algebra_guest::moduli_macros::moduli_init! {{ {supported_moduli} }}",) + } +} + #[derive(ChipUsageGetter, Chip, InstructionExecutor, AnyEnum, From)] pub enum ModularExtensionExecutor { // 32 limbs prime diff --git a/extensions/algebra/complex-macros/README.md b/extensions/algebra/complex-macros/README.md index 35e4dfa69a..9c786a73ca 100644 --- a/extensions/algebra/complex-macros/README.md +++ b/extensions/algebra/complex-macros/README.md @@ -10,17 +10,21 @@ The workflow of this macro is very similar to the [`openvm-algebra-moduli-macros openvm_algebra_moduli_macros::moduli_declare! { Secp256k1Coord { modulus = "0xFFFFFFFF FFFFFFFF FFFFFFFF FFFFFFFF FFFFFFFF FFFFFFFF FFFFFFFE FFFFFC2F" } } -openvm_algebra_moduli_macros::moduli_init!( - "0xFFFFFFFF FFFFFFFF FFFFFFFF FFFFFFFF FFFFFFFF FFFFFFFF FFFFFFFE FFFFFC2F" -); openvm_algebra_complex_macros::complex_declare! { Complex { mod_type = Secp256k1Coord } } +openvm::init!(); +/* The init! macro will expand to: +openvm_algebra_moduli_macros::moduli_init!( + "0xFFFFFFFF FFFFFFFF FFFFFFFF FFFFFFFF FFFFFFFF FFFFFFFF FFFFFFFE FFFFFC2F" +); + openvm_algebra_complex_macros::complex_init! { Complex { mod_idx = 0 }, } +*/ pub fn main() { setup_all_moduli(); @@ -100,3 +104,7 @@ complex_init! { ``` The reason is that, for example, the function `complex_add_extern_func_Bn254Fp2` remains unimplemented, but we implement `complex_add_extern_func_Fp2` instead. + +5. `cargo openvm build` will automatically generate a call to `complex_init!` based on `openvm.toml`. +Note that `openvm.toml` must list the supported moduli as pairs `(name, modulus)` where `name` is the name of the struct created by `complex_declare!` as a string (in the example at the top of this document, its `"Complex"`). +The SDK also supports this feature. \ No newline at end of file diff --git a/extensions/algebra/moduli-macros/README.md b/extensions/algebra/moduli-macros/README.md index 134209b0cc..61513d0603 100644 --- a/extensions/algebra/moduli-macros/README.md +++ b/extensions/algebra/moduli-macros/README.md @@ -14,11 +14,14 @@ openvm_algebra_moduli_macros::moduli_declare! { Mersenne61 { modulus = "0x1fffffffffffffff" }, } +openvm::init!(); +/* The init! macro will expand to: openvm_algebra_moduli_macros::moduli_init! { "4002409555221667393417789825735904156556882819939007885332058136124031650490837864442687629129015664037894272559787", "1000000000000000003", "0x1fffffffffffffff", } +*/ ``` ## Full story @@ -152,4 +155,12 @@ pub fn setup_all_moduli() { The setup operation (e.g., `setup_2`) consists of reading the value `OPENVM_SERIALIZED_MODULUS_2` from memory and constraining that the read value is equal to the modulus the chip has been configured with. For each used modulus, its corresponding setup instruction **must** be called before all other operations -- this currently must be checked by inspecting the program code; it is not enforced by the virtual machine. -5. It follows from the above that the `moduli_declare!` invocations may be in multiple places in various compilation units, but all the `declare!`d moduli must be specified at least once in `moduli_init!` so that there will be no linker errors due to missing function implementations. Correspondingly, the `moduli_init!` macro should only be called once in the entire program (in the guest crate as the topmost compilation unit). Finally, the order of the moduli in `moduli_init!` has nothing to do with the `moduli_declare!` invocations, but it **must match** the order of the moduli in the chip configuration -- more specifically, in the modular extension parameters (the order of numbers in `ModularExtension::supported_modulus`, which is usually defined with the whole `app_vm_config` in the `openvm.toml` file. The plan is to obtain this value from the specific section of the ELF file at some point). +5. It follows from the above that the `moduli_declare!` invocations may be in multiple places in various compilation units, but all the `declare!`d moduli must be specified at least once in `moduli_init!` so that there will be no linker errors due to missing function implementations. Correspondingly, the `moduli_init!` macro should only be called once in the entire program (in the guest crate as the topmost compilation unit). Finally, the order of the moduli in `moduli_init!` has nothing to do with the `moduli_declare!` invocations, but it **must match** the order of the moduli in the chip configuration -- more specifically, in the modular extension parameters (the order of numbers in `ModularExtension::supported_modulus`, which is usually defined with the whole `app_vm_config` in the `openvm.toml` file). + +6. For convenience, running `cargo openvm build` will automatically generate an appropriate call to `moduli_declare!` based on the order of the moduli in `openvm.toml`. +More specifically, `cargo openvm build` will read `openvm.toml`, then generate a file named `openvm_init.rs` in the project's manifest directory (where `Cargo.toml` is located) containing a call to `moduli_declare!`. +Then, you must call `openvm::init!();` in your code, and this will insert the contents of `openvm_init.rs` file into your code in place of the `init!` macro. +You may specify an alternate name for the init file using the `--init-file-name` option of `cargo openvm build`. +To include the generated file in your code, pass its name into the `init!` macro (for ex. `init!("my_init_file.rs")`). +The custom filename will be interpreted as relative to the manifest directory. +If you are using the SDK to build your code, it will also automatically generate the init file based on your config. diff --git a/extensions/algebra/tests/programs/examples/complex-redundant-modulus.rs b/extensions/algebra/tests/programs/examples/complex_redundant_modulus.rs similarity index 77% rename from extensions/algebra/tests/programs/examples/complex-redundant-modulus.rs rename to extensions/algebra/tests/programs/examples/complex_redundant_modulus.rs index e3fd151026..678b8b1838 100644 --- a/extensions/algebra/tests/programs/examples/complex-redundant-modulus.rs +++ b/extensions/algebra/tests/programs/examples/complex_redundant_modulus.rs @@ -11,17 +11,12 @@ openvm_algebra_moduli_macros::moduli_declare! { Mod3 { modulus = "1000000009" }, Mod4 { modulus = "987898789" }, } -openvm_algebra_moduli_macros::moduli_init! { - "998244353", "1000000007", "1000000009", "987898789" -} openvm_algebra_complex_macros::complex_declare! { Complex2 { mod_type = Mod3 }, } -openvm_algebra_complex_macros::complex_init! { - Complex2 { mod_idx = 2 }, -} +openvm::init!("openvm_init_complex_redundant_modulus.rs"); pub fn main() { setup_all_moduli(); diff --git a/extensions/algebra/tests/programs/examples/complex-secp256k1.rs b/extensions/algebra/tests/programs/examples/complex_secp256k1.rs similarity index 85% rename from extensions/algebra/tests/programs/examples/complex-secp256k1.rs rename to extensions/algebra/tests/programs/examples/complex_secp256k1.rs index 344d12ff6e..ab02878cfd 100644 --- a/extensions/algebra/tests/programs/examples/complex-secp256k1.rs +++ b/extensions/algebra/tests/programs/examples/complex_secp256k1.rs @@ -8,17 +8,12 @@ openvm::entry!(main); openvm_algebra_moduli_macros::moduli_declare! { Secp256k1Coord { modulus = "0xFFFFFFFF FFFFFFFF FFFFFFFF FFFFFFFF FFFFFFFF FFFFFFFF FFFFFFFE FFFFFC2F" } } -openvm_algebra_moduli_macros::moduli_init!( - "0xFFFFFFFF FFFFFFFF FFFFFFFF FFFFFFFF FFFFFFFF FFFFFFFF FFFFFFFE FFFFFC2F" -); openvm_algebra_complex_macros::complex_declare! { Complex { mod_type = Secp256k1Coord } } -openvm_algebra_complex_macros::complex_init! { - Complex { mod_idx = 0}, -} +openvm::init!("openvm_init_complex_secp256k1.rs"); pub fn main() { setup_all_moduli(); diff --git a/extensions/algebra/tests/programs/examples/complex-two-modulos.rs b/extensions/algebra/tests/programs/examples/complex_two_moduli.rs similarity index 79% rename from extensions/algebra/tests/programs/examples/complex-two-modulos.rs rename to extensions/algebra/tests/programs/examples/complex_two_moduli.rs index b8cd0e46c7..3e67028722 100644 --- a/extensions/algebra/tests/programs/examples/complex-two-modulos.rs +++ b/extensions/algebra/tests/programs/examples/complex_two_moduli.rs @@ -9,18 +9,13 @@ openvm_algebra_moduli_macros::moduli_declare! { Mod1 { modulus = "998244353" }, Mod2 { modulus = "1000000007" } } -openvm_algebra_moduli_macros::moduli_init! { - "998244353", "1000000007" -} openvm_algebra_complex_macros::complex_declare! { Complex1 { mod_type = Mod1 }, Complex2 { mod_type = Mod2 }, } -openvm_algebra_complex_macros::complex_init! { - Complex1 { mod_idx = 0 }, Complex2 { mod_idx = 1 }, -} +openvm::init!("openvm_init_complex_two_moduli.rs"); pub fn main() { setup_all_moduli(); diff --git a/extensions/algebra/tests/programs/examples/invalid-setup.rs b/extensions/algebra/tests/programs/examples/invalid_setup.rs similarity index 100% rename from extensions/algebra/tests/programs/examples/invalid-setup.rs rename to extensions/algebra/tests/programs/examples/invalid_setup.rs diff --git a/extensions/algebra/tests/programs/examples/little.rs b/extensions/algebra/tests/programs/examples/little.rs index dc1d2773a3..15203d98b7 100644 --- a/extensions/algebra/tests/programs/examples/little.rs +++ b/extensions/algebra/tests/programs/examples/little.rs @@ -9,9 +9,7 @@ openvm_algebra_moduli_macros::moduli_declare! { Secp256k1Coord { modulus = "0xFFFFFFFF FFFFFFFF FFFFFFFF FFFFFFFF FFFFFFFF FFFFFFFF FFFFFFFE FFFFFC2F" } } -openvm_algebra_moduli_macros::moduli_init!( - "0xFFFFFFFF FFFFFFFF FFFFFFFF FFFFFFFF FFFFFFFF FFFFFFFF FFFFFFFE FFFFFC2F" -); +openvm::init!("openvm_init_little.rs"); pub fn main() { setup_all_moduli(); diff --git a/extensions/algebra/tests/programs/examples/moduli_setup.rs b/extensions/algebra/tests/programs/examples/moduli_setup.rs index d5467a3f2a..cda883d133 100644 --- a/extensions/algebra/tests/programs/examples/moduli_setup.rs +++ b/extensions/algebra/tests/programs/examples/moduli_setup.rs @@ -15,11 +15,7 @@ openvm_algebra_moduli_macros::moduli_declare! { Mersenne61 { modulus = "0x1fffffffffffffff" }, } -openvm_algebra_moduli_macros::moduli_init! { - "4002409555221667393417789825735904156556882819939007885332058136124031650490837864442687629129015664037894272559787", - "1000000000000000003", - "0x1fffffffffffffff", -} +openvm::init!("openvm_init_moduli_setup.rs"); pub fn main() { setup_all_moduli(); diff --git a/extensions/algebra/tests/programs/openvm_init_complex_redundant_modulus.rs b/extensions/algebra/tests/programs/openvm_init_complex_redundant_modulus.rs new file mode 100644 index 0000000000..c32e692510 --- /dev/null +++ b/extensions/algebra/tests/programs/openvm_init_complex_redundant_modulus.rs @@ -0,0 +1,3 @@ +// This file is automatically generated by cargo openvm. Do not rename or edit. +openvm_algebra_guest::moduli_macros::moduli_init! { "998244353", "1000000007", "1000000009", "987898789" } +openvm_algebra_guest::complex_macros::complex_init! { Complex2 { mod_idx = 2 } } diff --git a/extensions/algebra/tests/programs/openvm_init_complex_secp256k1.rs b/extensions/algebra/tests/programs/openvm_init_complex_secp256k1.rs new file mode 100644 index 0000000000..af98350ae4 --- /dev/null +++ b/extensions/algebra/tests/programs/openvm_init_complex_secp256k1.rs @@ -0,0 +1,3 @@ +// This file is automatically generated by cargo openvm. Do not rename or edit. +openvm_algebra_guest::moduli_macros::moduli_init! { "115792089237316195423570985008687907853269984665640564039457584007908834671663" } +openvm_algebra_guest::complex_macros::complex_init! { Complex { mod_idx = 0 } } diff --git a/extensions/algebra/tests/programs/openvm_init_complex_two_moduli.rs b/extensions/algebra/tests/programs/openvm_init_complex_two_moduli.rs new file mode 100644 index 0000000000..1de98b97a1 --- /dev/null +++ b/extensions/algebra/tests/programs/openvm_init_complex_two_moduli.rs @@ -0,0 +1,3 @@ +// This file is automatically generated by cargo openvm. Do not rename or edit. +openvm_algebra_guest::moduli_macros::moduli_init! { "998244353", "1000000007" } +openvm_algebra_guest::complex_macros::complex_init! { Complex1 { mod_idx = 0 }, Complex2 { mod_idx = 1 } } diff --git a/extensions/algebra/tests/programs/openvm_init_little.rs b/extensions/algebra/tests/programs/openvm_init_little.rs new file mode 100644 index 0000000000..31acaf4433 --- /dev/null +++ b/extensions/algebra/tests/programs/openvm_init_little.rs @@ -0,0 +1,2 @@ +// This file is automatically generated by cargo openvm. Do not rename or edit. +openvm_algebra_guest::moduli_macros::moduli_init! { "115792089237316195423570985008687907853269984665640564039457584007908834671663" } diff --git a/extensions/algebra/tests/programs/openvm_init_moduli_setup.rs b/extensions/algebra/tests/programs/openvm_init_moduli_setup.rs new file mode 100644 index 0000000000..af99ca2e94 --- /dev/null +++ b/extensions/algebra/tests/programs/openvm_init_moduli_setup.rs @@ -0,0 +1,2 @@ +// This file is automatically generated by cargo openvm. Do not rename or edit. +openvm_algebra_guest::moduli_macros::moduli_init! { "4002409555221667393417789825735904156556882819939007885332058136124031650490837864442687629129015664037894272559787", "1000000000000000003", "2305843009213693951" } diff --git a/extensions/algebra/tests/src/lib.rs b/extensions/algebra/tests/src/lib.rs index 8a8051c4e3..3232de8ec1 100644 --- a/extensions/algebra/tests/src/lib.rs +++ b/extensions/algebra/tests/src/lib.rs @@ -15,14 +15,17 @@ mod tests { Rv32ITranspilerExtension, Rv32IoTranspilerExtension, Rv32MTranspilerExtension, }; use openvm_stark_sdk::p3_baby_bear::BabyBear; - use openvm_toolchain_tests::{build_example_program_at_path, get_programs_dir}; + use openvm_toolchain_tests::{build_example_program_at_path, get_programs_dir, NoInitFile}; use openvm_transpiler::{transpiler::Transpiler, FromElf}; type F = BabyBear; #[test] fn test_moduli_setup() -> Result<()> { - let elf = build_example_program_at_path(get_programs_dir!(), "moduli_setup")?; + let moduli = ["4002409555221667393417789825735904156556882819939007885332058136124031650490837864442687629129015664037894272559787", "1000000000000000003", "2305843009213693951"] + .map(|s| BigUint::from_str(s).unwrap()); + let config = Rv32ModularConfig::new(moduli.to_vec()); + let elf = build_example_program_at_path(get_programs_dir!(), "moduli_setup", &config)?; let openvm_exe = VmExe::from_elf( elf, Transpiler::::default() @@ -32,16 +35,14 @@ mod tests { .with_extension(ModularTranspilerExtension), )?; - let moduli = ["4002409555221667393417789825735904156556882819939007885332058136124031650490837864442687629129015664037894272559787", "1000000000000000003", "2305843009213693951"] - .map(|s| BigUint::from_str(s).unwrap()); - let config = Rv32ModularConfig::new(moduli.to_vec()); air_test(config, openvm_exe); Ok(()) } #[test] fn test_modular() -> Result<()> { - let elf = build_example_program_at_path(get_programs_dir!(), "little")?; + let config = Rv32ModularConfig::new(vec![SECP256K1_CONFIG.modulus.clone()]); + let elf = build_example_program_at_path(get_programs_dir!(), "little", &config)?; let openvm_exe = VmExe::from_elf( elf, Transpiler::::default() @@ -50,14 +51,24 @@ mod tests { .with_extension(Rv32IoTranspilerExtension) .with_extension(ModularTranspilerExtension), )?; - let config = Rv32ModularConfig::new(vec![SECP256K1_CONFIG.modulus.clone()]); air_test(config, openvm_exe); Ok(()) } #[test] fn test_complex_two_moduli() -> Result<()> { - let elf = build_example_program_at_path(get_programs_dir!(), "complex-two-modulos")?; + let config = Rv32ModularWithFp2Config::new(vec![ + ( + "Complex1".to_string(), + BigUint::from_str("998244353").unwrap(), + ), + ( + "Complex2".to_string(), + BigUint::from_str("1000000007").unwrap(), + ), + ]); + let elf = + build_example_program_at_path(get_programs_dir!(), "complex_two_moduli", &config)?; let openvm_exe = VmExe::from_elf( elf, Transpiler::::default() @@ -67,26 +78,12 @@ mod tests { .with_extension(Fp2TranspilerExtension) .with_extension(ModularTranspilerExtension), )?; - let config = Rv32ModularWithFp2Config::new(vec![ - BigUint::from_str("998244353").unwrap(), - BigUint::from_str("1000000007").unwrap(), - ]); air_test(config, openvm_exe); Ok(()) } #[test] fn test_complex_redundant_modulus() -> Result<()> { - let elf = build_example_program_at_path(get_programs_dir!(), "complex-redundant-modulus")?; - let openvm_exe = VmExe::from_elf( - elf, - Transpiler::::default() - .with_extension(Rv32ITranspilerExtension) - .with_extension(Rv32MTranspilerExtension) - .with_extension(Rv32IoTranspilerExtension) - .with_extension(Fp2TranspilerExtension) - .with_extension(ModularTranspilerExtension), - )?; let config = Rv32ModularWithFp2Config { system: SystemConfig::default().with_continuations(), base: Default::default(), @@ -98,15 +95,36 @@ mod tests { BigUint::from_str("1000000009").unwrap(), BigUint::from_str("987898789").unwrap(), ]), - fp2: Fp2Extension::new(vec![BigUint::from_str("1000000009").unwrap()]), + fp2: Fp2Extension::new(vec![( + "Complex2".to_string(), + BigUint::from_str("1000000009").unwrap(), + )]), }; + let elf = build_example_program_at_path( + get_programs_dir!(), + "complex_redundant_modulus", + &config, + )?; + let openvm_exe = VmExe::from_elf( + elf, + Transpiler::::default() + .with_extension(Rv32ITranspilerExtension) + .with_extension(Rv32MTranspilerExtension) + .with_extension(Rv32IoTranspilerExtension) + .with_extension(Fp2TranspilerExtension) + .with_extension(ModularTranspilerExtension), + )?; air_test(config, openvm_exe); Ok(()) } #[test] fn test_complex() -> Result<()> { - let elf = build_example_program_at_path(get_programs_dir!(), "complex-secp256k1")?; + let config = Rv32ModularWithFp2Config::new(vec![( + "Complex".to_string(), + SECP256K1_CONFIG.modulus.clone(), + )]); + let elf = build_example_program_at_path(get_programs_dir!(), "complex_secp256k1", &config)?; let openvm_exe = VmExe::from_elf( elf, Transpiler::::default() @@ -116,7 +134,6 @@ mod tests { .with_extension(Fp2TranspilerExtension) .with_extension(ModularTranspilerExtension), )?; - let config = Rv32ModularWithFp2Config::new(vec![SECP256K1_CONFIG.modulus.clone()]); air_test(config, openvm_exe); Ok(()) } @@ -124,7 +141,18 @@ mod tests { #[test] #[should_panic] fn test_invalid_setup() { - let elf = build_example_program_at_path(get_programs_dir!(), "invalid-setup").unwrap(); + let config = Rv32ModularConfig::new(vec![ + BigUint::from_str("998244353").unwrap(), + BigUint::from_str("1000000007").unwrap(), + ]); + let elf = build_example_program_at_path( + get_programs_dir!(), + "invalid_setup", + // We don't want init.rs to be generated for this test because we are testing an + // invalid moduli_init! call + &NoInitFile, + ) + .unwrap(); let openvm_exe = VmExe::from_elf( elf, Transpiler::::default() @@ -135,10 +163,6 @@ mod tests { .with_extension(ModularTranspilerExtension), ) .unwrap(); - let config = Rv32ModularConfig::new(vec![ - BigUint::from_str("998244353").unwrap(), - BigUint::from_str("1000000007").unwrap(), - ]); air_test(config, openvm_exe); } } diff --git a/extensions/bigint/circuit/src/extension.rs b/extensions/bigint/circuit/src/extension.rs index b9eeeafd99..390b79cc63 100644 --- a/extensions/bigint/circuit/src/extension.rs +++ b/extensions/bigint/circuit/src/extension.rs @@ -5,7 +5,8 @@ use openvm_bigint_transpiler::{ }; use openvm_circuit::{ arch::{ - SystemConfig, SystemPort, VmExtension, VmInventory, VmInventoryBuilder, VmInventoryError, + InitFileGenerator, SystemConfig, SystemPort, VmExtension, VmInventory, VmInventoryBuilder, + VmInventoryError, }, system::phantom::PhantomChip, }; @@ -39,6 +40,9 @@ pub struct Int256Rv32Config { pub bigint: Int256, } +// Default implementation uses no init file +impl InitFileGenerator for Int256Rv32Config {} + impl Default for Int256Rv32Config { fn default() -> Self { Self { diff --git a/extensions/bigint/tests/src/lib.rs b/extensions/bigint/tests/src/lib.rs index 866be2f672..25db54c2b7 100644 --- a/extensions/bigint/tests/src/lib.rs +++ b/extensions/bigint/tests/src/lib.rs @@ -16,7 +16,9 @@ mod tests { #[test] fn test_matrix_power() -> Result<()> { - let elf = build_example_program_at_path(get_programs_dir!(), "matrix-power-unsigned")?; + let config = Int256Rv32Config::default(); + let elf = + build_example_program_at_path(get_programs_dir!(), "matrix-power-unsigned", &config)?; let openvm_exe = VmExe::from_elf( elf, Transpiler::::default() @@ -25,14 +27,15 @@ mod tests { .with_extension(Rv32IoTranspilerExtension) .with_extension(Int256TranspilerExtension), )?; - let config = Int256Rv32Config::default(); air_test(config, openvm_exe); Ok(()) } #[test] fn test_matrix_power_signed() -> Result<()> { - let elf = build_example_program_at_path(get_programs_dir!(), "matrix-power-signed")?; + let config = Int256Rv32Config::default(); + let elf = + build_example_program_at_path(get_programs_dir!(), "matrix-power-signed", &config)?; let openvm_exe = VmExe::from_elf( elf, Transpiler::::default() @@ -41,7 +44,6 @@ mod tests { .with_extension(Rv32IoTranspilerExtension) .with_extension(Int256TranspilerExtension), )?; - let config = Int256Rv32Config::default(); air_test(config, openvm_exe); Ok(()) } diff --git a/extensions/ecc/circuit/src/config.rs b/extensions/ecc/circuit/src/config.rs index ab8809aa6d..a959938be9 100644 --- a/extensions/ecc/circuit/src/config.rs +++ b/extensions/ecc/circuit/src/config.rs @@ -1,5 +1,5 @@ use openvm_algebra_circuit::*; -use openvm_circuit::arch::SystemConfig; +use openvm_circuit::arch::{InitFileGenerator, SystemConfig}; use openvm_circuit_derive::VmConfig; use openvm_rv32im_circuit::*; use openvm_stark_backend::p3_field::PrimeField32; @@ -39,3 +39,13 @@ impl Rv32WeierstrassConfig { } } } + +impl InitFileGenerator for Rv32WeierstrassConfig { + fn generate_init_file_contents(&self) -> Option { + Some(format!( + "// This file is automatically generated by cargo openvm. Do not rename or edit.\n{}\n{}\n", + self.modular.generate_moduli_init(), + self.weierstrass.generate_sw_init() + )) + } +} diff --git a/extensions/ecc/circuit/src/weierstrass_extension.rs b/extensions/ecc/circuit/src/weierstrass_extension.rs index c5b23ccd0d..64d01a2711 100644 --- a/extensions/ecc/circuit/src/weierstrass_extension.rs +++ b/extensions/ecc/circuit/src/weierstrass_extension.rs @@ -13,8 +13,8 @@ use openvm_circuit_primitives::bitwise_op_lookup::{ }; use openvm_circuit_primitives_derive::{Chip, ChipUsageGetter}; use openvm_ecc_guest::{ - k256::{SECP256K1_MODULUS, SECP256K1_ORDER}, - p256::{CURVE_A as P256_A, CURVE_B as P256_B, P256_MODULUS, P256_ORDER}, + k256::{SECP256K1_ECC_STRUCT_NAME, SECP256K1_MODULUS, SECP256K1_ORDER}, + p256::{CURVE_A as P256_A, CURVE_B as P256_B, P256_ECC_STRUCT_NAME, P256_MODULUS, P256_ORDER}, }; use openvm_ecc_transpiler::{EccPhantom, Rv32WeierstrassOpcode}; use openvm_instructions::{LocalOpcode, PhantomDiscriminant, VmOpcode}; @@ -30,6 +30,8 @@ use super::{EcAddNeChip, EcDoubleChip}; #[serde_as] #[derive(Clone, Debug, derive_new::new, Serialize, Deserialize)] pub struct CurveConfig { + /// The name of the curve struct as defined by moduli_declare. + pub struct_name: String, /// The coordinate modulus of the curve. #[serde_as(as = "DisplayFromStr")] pub modulus: BigUint, @@ -45,6 +47,7 @@ pub struct CurveConfig { } pub static SECP256K1_CONFIG: Lazy = Lazy::new(|| CurveConfig { + struct_name: SECP256K1_ECC_STRUCT_NAME.to_string(), modulus: SECP256K1_MODULUS.clone(), scalar: SECP256K1_ORDER.clone(), a: BigUint::zero(), @@ -52,6 +55,7 @@ pub static SECP256K1_CONFIG: Lazy = Lazy::new(|| CurveConfig { }); pub static P256_CONFIG: Lazy = Lazy::new(|| CurveConfig { + struct_name: P256_ECC_STRUCT_NAME.to_string(), modulus: P256_MODULUS.clone(), scalar: P256_ORDER.clone(), a: BigUint::from_bytes_le(P256_A.as_le_bytes()), @@ -63,6 +67,19 @@ pub struct WeierstrassExtension { pub supported_curves: Vec, } +impl WeierstrassExtension { + pub fn generate_sw_init(&self) -> String { + let supported_curves = self + .supported_curves + .iter() + .map(|curve_config| curve_config.struct_name.to_string()) + .collect::>() + .join(", "); + + format!("openvm_ecc_guest::sw_macros::sw_init! {{ {supported_curves} }}") + } +} + #[derive(Chip, ChipUsageGetter, InstructionExecutor, AnyEnum)] pub enum WeierstrassExtensionExecutor { // 32 limbs prime diff --git a/extensions/ecc/guest/src/k256.rs b/extensions/ecc/guest/src/k256.rs index a3e63985ba..eed5f69be7 100644 --- a/extensions/ecc/guest/src/k256.rs +++ b/extensions/ecc/guest/src/k256.rs @@ -44,6 +44,10 @@ sw_declare! { Secp256k1Point { mod_type = Secp256k1Coord, b = CURVE_B }, } +#[cfg(not(target_os = "zkvm"))] +// Used in WeierstrassExtension config +pub const SECP256K1_ECC_STRUCT_NAME: &str = "Secp256k1Point"; + impl Field for Secp256k1Coord { const ZERO: Self = ::ZERO; const ONE: Self = ::ONE; diff --git a/extensions/ecc/guest/src/p256.rs b/extensions/ecc/guest/src/p256.rs index 6bbe4ab718..2892b9a200 100644 --- a/extensions/ecc/guest/src/p256.rs +++ b/extensions/ecc/guest/src/p256.rs @@ -41,6 +41,10 @@ openvm_ecc_sw_macros::sw_declare! { P256Point { mod_type = P256Coord, a = CURVE_A, b = CURVE_B }, } +#[cfg(not(target_os = "zkvm"))] +// Used in WeierstrassExtension config +pub const P256_ECC_STRUCT_NAME: &str = "P256Point"; + impl Field for P256Coord { const ZERO: Self = ::ZERO; const ONE: Self = ::ONE; diff --git a/extensions/ecc/sw-macros/README.md b/extensions/ecc/sw-macros/README.md index 5c247959f1..8b2da66e70 100644 --- a/extensions/ecc/sw-macros/README.md +++ b/extensions/ecc/sw-macros/README.md @@ -20,6 +20,8 @@ sw_declare! { Secp256k1Point { mod_type = Secp256k1Coord, b = CURVE_B }, } +openvm::init!(); +/* The init! macro will expand to: openvm_algebra_guest::moduli_macros::moduli_init! { "0xFFFFFFFF FFFFFFFF FFFFFFFF FFFFFFFF FFFFFFFF FFFFFFFF FFFFFFFE FFFFFC2F", "0xFFFFFFFF FFFFFFFF FFFFFFFF FFFFFFFE BAAEDCE6 AF48A03B BFD25E8C D0364141" @@ -28,6 +30,7 @@ openvm_algebra_guest::moduli_macros::moduli_init! { openvm_ecc_guest::sw_macros::sw_init! { Secp256k1Point, } +*/ pub fn main() { setup_all_moduli(); @@ -114,3 +117,7 @@ sw_init! { ``` The reason is that, for example, the function `sw_add_extern_func_Secp256k1Point` remains unimplemented, but we implement `sw_add_extern_func_Sw`. + +6. `cargo openvm build` will automatically generate a call to `sw_init!` based on `openvm.toml`. +Note that `openvm.toml` must contain the name of each struct created by `sw_declare!` as a string (in the example at the top of this document, its `"Secp256k1"`). +The SDK also supports this feature. \ No newline at end of file diff --git a/extensions/ecc/tests/programs/examples/decompress.rs b/extensions/ecc/tests/programs/examples/decompress.rs index 6f549d311c..8692b13675 100644 --- a/extensions/ecc/tests/programs/examples/decompress.rs +++ b/extensions/ecc/tests/programs/examples/decompress.rs @@ -22,15 +22,6 @@ openvm_algebra_moduli_macros::moduli_declare! { Fp1mod4 { modulus = "0xffffffffffffffffffffffffffffffff000000000000000000000001" }, } -openvm_algebra_moduli_macros::moduli_init! { - "0xFFFFFFFF FFFFFFFF FFFFFFFF FFFFFFFF FFFFFFFF FFFFFFFF FFFFFFFE FFFFFC2F", - "0xFFFFFFFF FFFFFFFF FFFFFFFF FFFFFFFE BAAEDCE6 AF48A03B BFD25E8C D0364141", - "115792089237316195423570985008687907853269984665640564039457584007913129639501", - "1000000007", - "0xffffffffffffffffffffffffffffffff000000000000000000000001", - "0xffffffffffffffffffffffffffff16a2e0b8f03e13dd29455c5c2a3d", -} - const CURVE_B_5MOD8: Fp5mod8 = Fp5mod8::from_const_u8(3); impl Field for Fp5mod8 { @@ -82,11 +73,7 @@ openvm_ecc_sw_macros::sw_declare! { }, } -openvm_ecc_sw_macros::sw_init! { - Secp256k1Point, - CurvePoint5mod8, - CurvePoint1mod4, -} +openvm::init!("openvm_init_decompress.rs"); // test decompression under an honest host pub fn main() { diff --git a/extensions/ecc/tests/programs/examples/decompress_invalid_hint.rs b/extensions/ecc/tests/programs/examples/decompress_invalid_hint.rs index b73e068132..c55a3e329d 100644 --- a/extensions/ecc/tests/programs/examples/decompress_invalid_hint.rs +++ b/extensions/ecc/tests/programs/examples/decompress_invalid_hint.rs @@ -22,15 +22,6 @@ openvm_algebra_moduli_macros::moduli_declare! { Fp1mod4 { modulus = "0xffffffffffffffffffffffffffffffff000000000000000000000001" }, } -openvm_algebra_moduli_macros::moduli_init! { - "0xFFFFFFFF FFFFFFFF FFFFFFFF FFFFFFFF FFFFFFFF FFFFFFFF FFFFFFFE FFFFFC2F", - "0xFFFFFFFF FFFFFFFF FFFFFFFF FFFFFFFE BAAEDCE6 AF48A03B BFD25E8C D0364141", - "115792089237316195423570985008687907853269984665640564039457584007913129639501", - "1000000007", - "0xffffffffffffffffffffffffffffffff000000000000000000000001", - "0xffffffffffffffffffffffffffff16a2e0b8f03e13dd29455c5c2a3d", -} - const CURVE_B_5MOD8: Fp5mod8 = Fp5mod8::from_const_u8(3); impl Field for Fp5mod8 { @@ -82,11 +73,7 @@ openvm_ecc_sw_macros::sw_declare! { }, } -openvm_ecc_sw_macros::sw_init! { - Secp256k1Point, - CurvePoint5mod8, - CurvePoint1mod4, -} +openvm::init!("openvm_init_decompress_invalid_hint.rs"); trait NonQr { fn get_non_qr() -> &'static P::Coordinate; diff --git a/extensions/ecc/tests/programs/examples/ec.rs b/extensions/ecc/tests/programs/examples/ec.rs index cb4f63e62a..056e12f41d 100644 --- a/extensions/ecc/tests/programs/examples/ec.rs +++ b/extensions/ecc/tests/programs/examples/ec.rs @@ -10,14 +10,7 @@ use openvm_ecc_guest::{ Group, }; -openvm_algebra_moduli_macros::moduli_init! { - "0xFFFFFFFF FFFFFFFF FFFFFFFF FFFFFFFF FFFFFFFF FFFFFFFF FFFFFFFE FFFFFC2F", - "0xFFFFFFFF FFFFFFFF FFFFFFFF FFFFFFFE BAAEDCE6 AF48A03B BFD25E8C D0364141" -} - -openvm_ecc_sw_macros::sw_init! { - Secp256k1Point, -} +openvm::init!("openvm_init_ec.rs"); openvm::entry!(main); diff --git a/extensions/ecc/tests/programs/examples/ec_nonzero_a.rs b/extensions/ecc/tests/programs/examples/ec_nonzero_a.rs index 18720e1ae1..a25277b4bb 100644 --- a/extensions/ecc/tests/programs/examples/ec_nonzero_a.rs +++ b/extensions/ecc/tests/programs/examples/ec_nonzero_a.rs @@ -11,14 +11,7 @@ use openvm_ecc_guest::{ openvm::entry!(main); -openvm_algebra_moduli_macros::moduli_init! { - "0xffffffff00000001000000000000000000000000ffffffffffffffffffffffff", - "0xffffffff00000000ffffffffffffffffbce6faada7179e84f3b9cac2fc632551" -} - -openvm_ecc_sw_macros::sw_init! { - P256Point, -} +openvm::init!("openvm_init_ec_nonzero_a.rs"); pub fn main() { setup_all_moduli(); diff --git a/extensions/ecc/tests/programs/examples/ec_two_curves.rs b/extensions/ecc/tests/programs/examples/ec_two_curves.rs index ab96e9d240..499d678593 100644 --- a/extensions/ecc/tests/programs/examples/ec_two_curves.rs +++ b/extensions/ecc/tests/programs/examples/ec_two_curves.rs @@ -11,17 +11,7 @@ use openvm_ecc_guest::{ Group, }; -openvm_algebra_moduli_macros::moduli_init! { - "0xFFFFFFFF FFFFFFFF FFFFFFFF FFFFFFFF FFFFFFFF FFFFFFFF FFFFFFFE FFFFFC2F", - "0xFFFFFFFF FFFFFFFF FFFFFFFF FFFFFFFE BAAEDCE6 AF48A03B BFD25E8C D0364141", - "0xffffffff00000001000000000000000000000000ffffffffffffffffffffffff", - "0xffffffff00000000ffffffffffffffffbce6faada7179e84f3b9cac2fc632551" -} - -openvm_ecc_sw_macros::sw_init! { - Secp256k1Point, - P256Point, -} +openvm::init!("openvm_init_ec_two_curves.rs"); openvm::entry!(main); diff --git a/extensions/ecc/tests/programs/examples/ecdsa.rs b/extensions/ecc/tests/programs/examples/ecdsa.rs index 3fc3c12f4e..80b0734941 100644 --- a/extensions/ecc/tests/programs/examples/ecdsa.rs +++ b/extensions/ecc/tests/programs/examples/ecdsa.rs @@ -15,13 +15,7 @@ use openvm_ecc_guest::{ use openvm_keccak256_guest::keccak256; openvm::entry!(main); -openvm_algebra_moduli_macros::moduli_init! { - "0xFFFFFFFF FFFFFFFF FFFFFFFF FFFFFFFF FFFFFFFF FFFFFFFF FFFFFFFE FFFFFC2F", - "0xFFFFFFFF FFFFFFFF FFFFFFFF FFFFFFFE BAAEDCE6 AF48A03B BFD25E8C D0364141" -} -openvm_ecc_sw_macros::sw_init! { - Secp256k1Point, -} +openvm::init!("openvm_init_ecdsa.rs"); // Ref: https://docs.rs/k256/latest/k256/ecdsa/index.html pub fn main() { diff --git a/extensions/ecc/tests/programs/openvm_init_decompress.rs b/extensions/ecc/tests/programs/openvm_init_decompress.rs new file mode 100644 index 0000000000..b6137ae9ee --- /dev/null +++ b/extensions/ecc/tests/programs/openvm_init_decompress.rs @@ -0,0 +1,3 @@ +// This file is automatically generated by cargo openvm. Do not rename or edit. +openvm_algebra_guest::moduli_macros::moduli_init! { "115792089237316195423570985008687907853269984665640564039457584007908834671663", "115792089237316195423570985008687907852837564279074904382605163141518161494337", "115792089237316195423570985008687907853269984665640564039457584007913129639501", "1000000007", "26959946667150639794667015087019630673557916260026308143510066298881", "26959946667150639794667015087019625940457807714424391721682722368061" } +openvm_ecc_guest::sw_macros::sw_init! { Secp256k1Point, CurvePoint5mod8, CurvePoint1mod4 } diff --git a/extensions/ecc/tests/programs/openvm_init_decompress_invalid_hint.rs b/extensions/ecc/tests/programs/openvm_init_decompress_invalid_hint.rs new file mode 100644 index 0000000000..b6137ae9ee --- /dev/null +++ b/extensions/ecc/tests/programs/openvm_init_decompress_invalid_hint.rs @@ -0,0 +1,3 @@ +// This file is automatically generated by cargo openvm. Do not rename or edit. +openvm_algebra_guest::moduli_macros::moduli_init! { "115792089237316195423570985008687907853269984665640564039457584007908834671663", "115792089237316195423570985008687907852837564279074904382605163141518161494337", "115792089237316195423570985008687907853269984665640564039457584007913129639501", "1000000007", "26959946667150639794667015087019630673557916260026308143510066298881", "26959946667150639794667015087019625940457807714424391721682722368061" } +openvm_ecc_guest::sw_macros::sw_init! { Secp256k1Point, CurvePoint5mod8, CurvePoint1mod4 } diff --git a/extensions/ecc/tests/programs/openvm_init_ec.rs b/extensions/ecc/tests/programs/openvm_init_ec.rs new file mode 100644 index 0000000000..bec9f527e9 --- /dev/null +++ b/extensions/ecc/tests/programs/openvm_init_ec.rs @@ -0,0 +1,3 @@ +// This file is automatically generated by cargo openvm. Do not rename or edit. +openvm_algebra_guest::moduli_macros::moduli_init! { "115792089237316195423570985008687907853269984665640564039457584007908834671663", "115792089237316195423570985008687907852837564279074904382605163141518161494337" } +openvm_ecc_guest::sw_macros::sw_init! { Secp256k1Point } diff --git a/extensions/ecc/tests/programs/openvm_init_ec_nonzero_a.rs b/extensions/ecc/tests/programs/openvm_init_ec_nonzero_a.rs new file mode 100644 index 0000000000..02f8b5c05d --- /dev/null +++ b/extensions/ecc/tests/programs/openvm_init_ec_nonzero_a.rs @@ -0,0 +1,3 @@ +// This file is automatically generated by cargo openvm. Do not rename or edit. +openvm_algebra_guest::moduli_macros::moduli_init! { "115792089210356248762697446949407573530086143415290314195533631308867097853951", "115792089210356248762697446949407573529996955224135760342422259061068512044369" } +openvm_ecc_guest::sw_macros::sw_init! { P256Point } diff --git a/extensions/ecc/tests/programs/openvm_init_ec_two_curves.rs b/extensions/ecc/tests/programs/openvm_init_ec_two_curves.rs new file mode 100644 index 0000000000..8689190544 --- /dev/null +++ b/extensions/ecc/tests/programs/openvm_init_ec_two_curves.rs @@ -0,0 +1,3 @@ +// This file is automatically generated by cargo openvm. Do not rename or edit. +openvm_algebra_guest::moduli_macros::moduli_init! { "115792089237316195423570985008687907853269984665640564039457584007908834671663", "115792089237316195423570985008687907852837564279074904382605163141518161494337", "115792089210356248762697446949407573530086143415290314195533631308867097853951", "115792089210356248762697446949407573529996955224135760342422259061068512044369" } +openvm_ecc_guest::sw_macros::sw_init! { Secp256k1Point, P256Point } diff --git a/extensions/ecc/tests/programs/openvm_init_ecdsa.rs b/extensions/ecc/tests/programs/openvm_init_ecdsa.rs new file mode 100644 index 0000000000..bec9f527e9 --- /dev/null +++ b/extensions/ecc/tests/programs/openvm_init_ecdsa.rs @@ -0,0 +1,3 @@ +// This file is automatically generated by cargo openvm. Do not rename or edit. +openvm_algebra_guest::moduli_macros::moduli_init! { "115792089237316195423570985008687907853269984665640564039457584007908834671663", "115792089237316195423570985008687907852837564279074904382605163141518161494337" } +openvm_ecc_guest::sw_macros::sw_init! { Secp256k1Point } diff --git a/extensions/ecc/tests/src/lib.rs b/extensions/ecc/tests/src/lib.rs index 6d90241156..dd3f885e90 100644 --- a/extensions/ecc/tests/src/lib.rs +++ b/extensions/ecc/tests/src/lib.rs @@ -22,13 +22,21 @@ mod tests { use openvm_sdk::config::SdkVmConfig; use openvm_stark_backend::p3_field::FieldAlgebra; use openvm_stark_sdk::{openvm_stark_backend, p3_baby_bear::BabyBear}; - use openvm_toolchain_tests::{build_example_program_at_path_with_features, get_programs_dir}; + use openvm_toolchain_tests::{ + build_example_program_at_path_with_features, get_programs_dir, NoInitFile, + }; use openvm_transpiler::{transpiler::Transpiler, FromElf}; type F = BabyBear; #[test] fn test_ec() -> Result<()> { - let elf = build_example_program_at_path_with_features(get_programs_dir!(), "ec", ["k256"])?; + let config = Rv32WeierstrassConfig::new(vec![SECP256K1_CONFIG.clone()]); + let elf = build_example_program_at_path_with_features( + get_programs_dir!(), + "ec", + ["k256"], + &config, + )?; let openvm_exe = VmExe::from_elf( elf, Transpiler::::default() @@ -38,17 +46,18 @@ mod tests { .with_extension(EccTranspilerExtension) .with_extension(ModularTranspilerExtension), )?; - let config = Rv32WeierstrassConfig::new(vec![SECP256K1_CONFIG.clone()]); air_test(config, openvm_exe); Ok(()) } #[test] fn test_ec_nonzero_a() -> Result<()> { + let config = Rv32WeierstrassConfig::new(vec![P256_CONFIG.clone()]); let elf = build_example_program_at_path_with_features( get_programs_dir!(), "ec_nonzero_a", ["p256"], + &config, )?; let openvm_exe = VmExe::from_elf( elf, @@ -59,17 +68,19 @@ mod tests { .with_extension(EccTranspilerExtension) .with_extension(ModularTranspilerExtension), )?; - let config = Rv32WeierstrassConfig::new(vec![P256_CONFIG.clone()]); air_test(config, openvm_exe); Ok(()) } #[test] fn test_ec_two_curves() -> Result<()> { + let config = + Rv32WeierstrassConfig::new(vec![SECP256K1_CONFIG.clone(), P256_CONFIG.clone()]); let elf = build_example_program_at_path_with_features( get_programs_dir!(), "ec_two_curves", ["k256", "p256"], + &config, )?; let openvm_exe = VmExe::from_elf( elf, @@ -80,8 +91,6 @@ mod tests { .with_extension(EccTranspilerExtension) .with_extension(ModularTranspilerExtension), )?; - let config = - Rv32WeierstrassConfig::new(vec![SECP256K1_CONFIG.clone(), P256_CONFIG.clone()]); air_test(config, openvm_exe); Ok(()) } @@ -90,23 +99,10 @@ mod tests { fn test_decompress() -> Result<()> { use openvm_ecc_guest::halo2curves::{group::Curve, secp256k1::Secp256k1Affine}; - let elf = build_example_program_at_path_with_features( - get_programs_dir!(), - "decompress", - ["k256"], - )?; - let openvm_exe = VmExe::from_elf( - elf, - Transpiler::::default() - .with_extension(Rv32ITranspilerExtension) - .with_extension(Rv32MTranspilerExtension) - .with_extension(Rv32IoTranspilerExtension) - .with_extension(EccTranspilerExtension) - .with_extension(ModularTranspilerExtension), - )?; let config = Rv32WeierstrassConfig::new(vec![SECP256K1_CONFIG.clone(), CurveConfig { + struct_name: "CurvePoint5mod8".to_string(), modulus: BigUint::from_str("115792089237316195423570985008687907853269984665640564039457584007913129639501") .unwrap(), // unused, set to 10e9 + 7 @@ -116,6 +112,7 @@ mod tests { b: BigUint::from_str("3").unwrap(), }, CurveConfig { + struct_name: "CurvePoint1mod4".to_string(), modulus: BigUint::from_radix_be(&hex!("ffffffffffffffffffffffffffffffff000000000000000000000001"), 256) .unwrap(), scalar: BigUint::from_radix_be(&hex!("ffffffffffffffffffffffffffff16a2e0b8f03e13dd29455c5c2a3d"), 256) @@ -127,6 +124,22 @@ mod tests { }, ]); + let elf = build_example_program_at_path_with_features( + get_programs_dir!(), + "decompress", + ["k256"], + &config, + )?; + let openvm_exe = VmExe::from_elf( + elf, + Transpiler::::default() + .with_extension(Rv32ITranspilerExtension) + .with_extension(Rv32MTranspilerExtension) + .with_extension(Rv32IoTranspilerExtension) + .with_extension(EccTranspilerExtension) + .with_extension(ModularTranspilerExtension), + )?; + let p = Secp256k1Affine::generator(); let p = (p + p + p).to_affine(); println!("decompressed: {:?}", p); @@ -151,23 +164,10 @@ mod tests { fn test_decompress_invalid_specific_test(test_type: &str) -> Result<()> { use openvm_ecc_guest::halo2curves::{group::Curve, secp256k1::Secp256k1Affine}; - let elf = build_example_program_at_path_with_features( - get_programs_dir!(), - "decompress_invalid_hint", - ["k256", test_type], - )?; - let openvm_exe = VmExe::from_elf( - elf, - Transpiler::::default() - .with_extension(Rv32ITranspilerExtension) - .with_extension(Rv32MTranspilerExtension) - .with_extension(Rv32IoTranspilerExtension) - .with_extension(EccTranspilerExtension) - .with_extension(ModularTranspilerExtension), - )?; let config = Rv32WeierstrassConfig::new(vec![SECP256K1_CONFIG.clone(), CurveConfig { + struct_name: "CurvePoint5mod8".to_string(), modulus: BigUint::from_str("115792089237316195423570985008687907853269984665640564039457584007913129639501") .unwrap(), // unused, set to 10e9 + 7 @@ -177,6 +177,7 @@ mod tests { b: BigUint::from_str("3").unwrap(), }, CurveConfig { + struct_name: "CurvePoint1mod4".to_string(), modulus: BigUint::from_radix_be(&hex!("ffffffffffffffffffffffffffffffff000000000000000000000001"), 256) .unwrap(), scalar: BigUint::from_radix_be(&hex!("ffffffffffffffffffffffffffff16a2e0b8f03e13dd29455c5c2a3d"), 256) @@ -188,6 +189,22 @@ mod tests { }, ]); + let elf = build_example_program_at_path_with_features( + get_programs_dir!(), + "decompress_invalid_hint", + ["k256", test_type], + &config, + )?; + let openvm_exe = VmExe::from_elf( + elf, + Transpiler::::default() + .with_extension(Rv32ITranspilerExtension) + .with_extension(Rv32MTranspilerExtension) + .with_extension(Rv32IoTranspilerExtension) + .with_extension(EccTranspilerExtension) + .with_extension(ModularTranspilerExtension), + )?; + let p = Secp256k1Affine::generator(); let p = (p + p + p).to_affine(); println!("decompressed: {:?}", p); @@ -247,8 +264,6 @@ mod tests { #[test] fn test_ecdsa() -> Result<()> { - let elf = - build_example_program_at_path_with_features(get_programs_dir!(), "ecdsa", ["k256"])?; let config = SdkVmConfig::builder() .system(SystemConfig::default().with_continuations().into()) .rv32i(Default::default()) @@ -261,6 +276,13 @@ mod tests { .keccak(Default::default()) .ecc(WeierstrassExtension::new(vec![SECP256K1_CONFIG.clone()])) .build(); + + let elf = build_example_program_at_path_with_features( + get_programs_dir!(), + "ecdsa", + ["k256"], + &config, + )?; let openvm_exe = VmExe::from_elf( elf, Transpiler::::default() @@ -282,6 +304,7 @@ mod tests { get_programs_dir!(), "invalid_setup", ["k256", "p256"], + &NoInitFile, // don't use build script since we are testing invalid setup ) .unwrap(); let openvm_exe = VmExe::from_elf( diff --git a/extensions/keccak256/circuit/src/extension.rs b/extensions/keccak256/circuit/src/extension.rs index d24681fb55..5993f69eda 100644 --- a/extensions/keccak256/circuit/src/extension.rs +++ b/extensions/keccak256/circuit/src/extension.rs @@ -1,7 +1,8 @@ use derive_more::derive::From; use openvm_circuit::{ arch::{ - SystemConfig, SystemPort, VmExtension, VmInventory, VmInventoryBuilder, VmInventoryError, + InitFileGenerator, SystemConfig, SystemPort, VmExtension, VmInventory, VmInventoryBuilder, + VmInventoryError, }, system::phantom::PhantomChip, }; @@ -45,6 +46,9 @@ impl Default for Keccak256Rv32Config { } } +// Default implementation uses no init file +impl InitFileGenerator for Keccak256Rv32Config {} + #[derive(Clone, Copy, Debug, Default, Serialize, Deserialize)] pub struct Keccak256; diff --git a/extensions/keccak256/tests/src/lib.rs b/extensions/keccak256/tests/src/lib.rs index ecb2f524ee..69b1ef65d3 100644 --- a/extensions/keccak256/tests/src/lib.rs +++ b/extensions/keccak256/tests/src/lib.rs @@ -16,7 +16,8 @@ mod tests { #[test] fn test_keccak256() -> Result<()> { - let elf = build_example_program_at_path(get_programs_dir!(), "keccak")?; + let config = Keccak256Rv32Config::default(); + let elf = build_example_program_at_path(get_programs_dir!(), "keccak", &config)?; let openvm_exe = VmExe::from_elf( elf, Transpiler::::default() @@ -25,7 +26,7 @@ mod tests { .with_extension(Rv32MTranspilerExtension) .with_extension(Rv32IoTranspilerExtension), )?; - air_test(Keccak256Rv32Config::default(), openvm_exe); + air_test(config, openvm_exe); Ok(()) } } diff --git a/extensions/native/circuit/src/extension.rs b/extensions/native/circuit/src/extension.rs index 8f73423e40..385c9392ac 100644 --- a/extensions/native/circuit/src/extension.rs +++ b/extensions/native/circuit/src/extension.rs @@ -6,8 +6,8 @@ use loadstore_native_adapter::NativeLoadStoreAdapterChip; use native_vectorized_adapter::NativeVectorizedAdapterChip; use openvm_circuit::{ arch::{ - ExecutionBridge, MemoryConfig, SystemConfig, SystemPort, VmExtension, VmInventory, - VmInventoryBuilder, VmInventoryError, + ExecutionBridge, InitFileGenerator, MemoryConfig, SystemConfig, SystemPort, VmExtension, + VmInventory, VmInventoryBuilder, VmInventoryError, }, system::phantom::PhantomChip, }; @@ -60,6 +60,9 @@ impl NativeConfig { } } +// Default implementation uses no init file +impl InitFileGenerator for NativeConfig {} + #[derive(Clone, Copy, Debug, Default, Serialize, Deserialize)] pub struct Native; @@ -433,3 +436,6 @@ impl Default for Rv32WithKernelsConfig { } } } + +// Default implementation uses no init file +impl InitFileGenerator for Rv32WithKernelsConfig {} diff --git a/extensions/pairing/circuit/src/config.rs b/extensions/pairing/circuit/src/config.rs index 4914fa433c..d63bac664e 100644 --- a/extensions/pairing/circuit/src/config.rs +++ b/extensions/pairing/circuit/src/config.rs @@ -1,5 +1,5 @@ use openvm_algebra_circuit::*; -use openvm_circuit::arch::SystemConfig; +use openvm_circuit::arch::{InitFileGenerator, SystemConfig}; use openvm_circuit_derive::VmConfig; use openvm_ecc_circuit::*; use openvm_rv32im_circuit::*; @@ -29,19 +29,25 @@ pub struct Rv32PairingConfig { } impl Rv32PairingConfig { - pub fn new(curves: Vec) -> Self { - let mut primes: Vec<_> = curves + pub fn new(curves: Vec, complex_struct_names: Vec) -> Self { + let modulus_primes: Vec<_> = curves .iter() .map(|c| c.curve_config().modulus.clone()) .collect(); - primes.extend(curves.iter().map(|c| c.curve_config().scalar.clone())); + let mut modulus_and_scalar_primes = modulus_primes.clone(); + modulus_and_scalar_primes.extend(curves.iter().map(|c| c.curve_config().scalar.clone())); Self { system: SystemConfig::default().with_continuations(), base: Default::default(), mul: Default::default(), io: Default::default(), - modular: ModularExtension::new(primes.to_vec()), - fp2: Fp2Extension::new(primes.to_vec()), + modular: ModularExtension::new(modulus_and_scalar_primes), + fp2: Fp2Extension::new( + complex_struct_names + .into_iter() + .zip(modulus_primes) + .collect(), + ), weierstrass: WeierstrassExtension::new( curves.iter().map(|c| c.curve_config()).collect(), ), @@ -49,3 +55,14 @@ impl Rv32PairingConfig { } } } + +impl InitFileGenerator for Rv32PairingConfig { + fn generate_init_file_contents(&self) -> Option { + Some(format!( + "// This file is automatically generated by cargo openvm. Do not rename or edit.\n{}\n{}\n{}\n", + self.modular.generate_moduli_init(), + self.fp2.generate_complex_init(&self.modular), + self.weierstrass.generate_sw_init() + )) + } +} diff --git a/extensions/pairing/circuit/src/pairing_extension.rs b/extensions/pairing/circuit/src/pairing_extension.rs index eca4cea8dd..b1e6afe38d 100644 --- a/extensions/pairing/circuit/src/pairing_extension.rs +++ b/extensions/pairing/circuit/src/pairing_extension.rs @@ -11,8 +11,10 @@ use openvm_circuit_primitives_derive::{Chip, ChipUsageGetter}; use openvm_ecc_circuit::CurveConfig; use openvm_instructions::PhantomDiscriminant; use openvm_pairing_guest::{ - bls12_381::{BLS12_381_MODULUS, BLS12_381_ORDER, BLS12_381_XI_ISIZE}, - bn254::{BN254_MODULUS, BN254_ORDER, BN254_XI_ISIZE}, + bls12_381::{ + BLS12_381_ECC_STRUCT_NAME, BLS12_381_MODULUS, BLS12_381_ORDER, BLS12_381_XI_ISIZE, + }, + bn254::{BN254_ECC_STRUCT_NAME, BN254_MODULUS, BN254_ORDER, BN254_XI_ISIZE}, }; use openvm_pairing_transpiler::PairingPhantom; use openvm_stark_backend::p3_field::PrimeField32; @@ -33,12 +35,14 @@ impl PairingCurve { pub fn curve_config(&self) -> CurveConfig { match self { PairingCurve::Bn254 => CurveConfig::new( + BN254_ECC_STRUCT_NAME.to_string(), BN254_MODULUS.clone(), BN254_ORDER.clone(), BigUint::zero(), BigUint::from_u8(3).unwrap(), ), PairingCurve::Bls12_381 => CurveConfig::new( + BLS12_381_ECC_STRUCT_NAME.to_string(), BLS12_381_MODULUS.clone(), BLS12_381_ORDER.clone(), BigUint::zero(), diff --git a/extensions/pairing/guest/src/bls12_381/fp2.rs b/extensions/pairing/guest/src/bls12_381/fp2.rs index 6e8672680e..20de223962 100644 --- a/extensions/pairing/guest/src/bls12_381/fp2.rs +++ b/extensions/pairing/guest/src/bls12_381/fp2.rs @@ -6,6 +6,10 @@ use openvm_algebra_guest::{field::FieldExtension, Field, IntMod}; use super::Fp; +#[cfg(not(target_os = "zkvm"))] +// Used in Fp2Extension config +pub const BLS12_381_COMPLEX_STRUCT_NAME: &str = "Bls12_381Fp2"; + // The struct name needs to be globally unique for linking purposes. // The mod_type is a path used only in the struct definition. complex_declare! { diff --git a/extensions/pairing/guest/src/bls12_381/mod.rs b/extensions/pairing/guest/src/bls12_381/mod.rs index a23d35d507..f1940ebe21 100644 --- a/extensions/pairing/guest/src/bls12_381/mod.rs +++ b/extensions/pairing/guest/src/bls12_381/mod.rs @@ -60,6 +60,10 @@ sw_declare! { Bls12_381G1Affine { mod_type = Bls12_381Fp, b = CURVE_B }, } +#[cfg(not(target_os = "zkvm"))] +// Used in WeierstrassExtension config +pub const BLS12_381_ECC_STRUCT_NAME: &str = "Bls12_381G1Affine"; + pub type Fp = Bls12_381Fp; pub type Scalar = Bls12_381Scalar; /// Affine point representation of `Fp` points of BLS12-381. diff --git a/extensions/pairing/guest/src/bn254/fp2.rs b/extensions/pairing/guest/src/bn254/fp2.rs index e05f6a0d9a..b086b7d6aa 100644 --- a/extensions/pairing/guest/src/bn254/fp2.rs +++ b/extensions/pairing/guest/src/bn254/fp2.rs @@ -6,6 +6,10 @@ use openvm_algebra_guest::{field::FieldExtension, Field, IntMod}; use super::Fp; +#[cfg(not(target_os = "zkvm"))] +// Used in Fp2Extension config +pub const BN254_COMPLEX_STRUCT_NAME: &str = "Bn254Fp2"; + // The struct name needs to be globally unique for linking purposes. // The mod_type is a path used only in the struct definition. complex_declare! { diff --git a/extensions/pairing/guest/src/bn254/mod.rs b/extensions/pairing/guest/src/bn254/mod.rs index 7a620e74e9..676a0adabd 100644 --- a/extensions/pairing/guest/src/bn254/mod.rs +++ b/extensions/pairing/guest/src/bn254/mod.rs @@ -66,6 +66,10 @@ sw_declare! { Bn254G1Affine { mod_type = Bn254Fp, b = CURVE_B }, } +#[cfg(not(target_os = "zkvm"))] +// Used in WeierstrassExtension config +pub const BN254_ECC_STRUCT_NAME: &str = "Bn254G1Affine"; + pub type Fp = Bn254Fp; pub type Scalar = Bn254Scalar; pub type G1Affine = Bn254G1Affine; diff --git a/extensions/pairing/tests/programs/examples/bls_ec.rs b/extensions/pairing/tests/programs/examples/bls_ec.rs index dc59c7d702..c7e13f626a 100644 --- a/extensions/pairing/tests/programs/examples/bls_ec.rs +++ b/extensions/pairing/tests/programs/examples/bls_ec.rs @@ -4,14 +4,7 @@ #[allow(unused_imports)] use openvm_pairing_guest::bls12_381::Bls12_381G1Affine; -openvm_algebra_moduli_macros::moduli_init! { - "0x1a0111ea397fe69a4b1ba7b6434bacd764774b84f38512bf6730d2a0f6b0f6241eabfffeb153ffffb9feffffffffaaab", - "0x73eda753299d7d483339d80809a1d80553bda402fffe5bfeffffffff00000001" -} - -openvm_ecc_sw_macros::sw_init! { - Bls12_381G1Affine, -} +openvm::init!("openvm_init_bls_ec_bls12_381.rs"); openvm::entry!(main); diff --git a/extensions/pairing/tests/programs/examples/bls_final_exp_hint.rs b/extensions/pairing/tests/programs/examples/bls_final_exp_hint.rs index dade1cd161..33660e0951 100644 --- a/extensions/pairing/tests/programs/examples/bls_final_exp_hint.rs +++ b/extensions/pairing/tests/programs/examples/bls_final_exp_hint.rs @@ -14,10 +14,7 @@ use openvm_pairing_guest::{ openvm::entry!(main); -openvm_algebra_moduli_macros::moduli_init! { - "0x1a0111ea397fe69a4b1ba7b6434bacd764774b84f38512bf6730d2a0f6b0f6241eabfffeb153ffffb9feffffffffaaab", - "0x73eda753299d7d483339d80809a1d80553bda402fffe5bfeffffffff00000001" -} +openvm::init!("openvm_init_bls_final_exp_hint_bls12_381.rs"); pub fn main() { #[allow(clippy::type_complexity)] diff --git a/extensions/pairing/tests/programs/examples/bn_final_exp_hint.rs b/extensions/pairing/tests/programs/examples/bn_final_exp_hint.rs index 3b71253b1e..347afa72bb 100644 --- a/extensions/pairing/tests/programs/examples/bn_final_exp_hint.rs +++ b/extensions/pairing/tests/programs/examples/bn_final_exp_hint.rs @@ -14,10 +14,7 @@ use openvm_pairing_guest::{ openvm::entry!(main); -openvm_algebra_moduli_macros::moduli_init! { - "21888242871839275222246405745257275088696311157297823662689037894645226208583", - "21888242871839275222246405745257275088548364400416034343698204186575808495617", -} +openvm::init!("openvm_init_bn_final_exp_hint_bn254.rs"); pub fn main() { #[allow(clippy::type_complexity)] diff --git a/extensions/pairing/tests/programs/examples/fp12_mul.rs b/extensions/pairing/tests/programs/examples/fp12_mul.rs index fddc5d9415..0ba32c336d 100644 --- a/extensions/pairing/tests/programs/examples/fp12_mul.rs +++ b/extensions/pairing/tests/programs/examples/fp12_mul.rs @@ -13,14 +13,7 @@ mod bn254 { use super::*; - openvm_algebra_moduli_macros::moduli_init! { - "0x30644e72e131a029b85045b68181585d97816a916871ca8d3c208c16d87cfd47", - "0x30644e72e131a029b85045b68181585d2833e84879b9709143e1f593f0000001" - } - - openvm_algebra_complex_macros::complex_init! { - Bn254Fp2 { mod_idx = 0 }, - } + openvm::init!("openvm_init_fp12_mul_bn254.rs"); pub fn test_fp12_mul(io: &[u8]) { setup_0(); @@ -52,14 +45,7 @@ mod bls12_381 { use super::*; - openvm_algebra_moduli_macros::moduli_init! { - "0x1a0111ea397fe69a4b1ba7b6434bacd764774b84f38512bf6730d2a0f6b0f6241eabfffeb153ffffb9feffffffffaaab", - "0x73eda753299d7d483339d80809a1d80553bda402fffe5bfeffffffff00000001" - } - - openvm_algebra_complex_macros::complex_init! { - Bls12_381Fp2 { mod_idx = 0 }, - } + openvm::init!("openvm_init_fp12_mul_bls12_381.rs"); pub fn test_fp12_mul(io: &[u8]) { setup_0(); diff --git a/extensions/pairing/tests/programs/examples/pairing_check.rs b/extensions/pairing/tests/programs/examples/pairing_check.rs index 88f17ceacb..e3be6a8a34 100644 --- a/extensions/pairing/tests/programs/examples/pairing_check.rs +++ b/extensions/pairing/tests/programs/examples/pairing_check.rs @@ -19,14 +19,7 @@ mod bn254 { use super::*; - openvm_algebra_moduli_macros::moduli_init! { - "0x30644e72e131a029b85045b68181585d97816a916871ca8d3c208c16d87cfd47", - "0x30644e72e131a029b85045b68181585d2833e84879b9709143e1f593f0000001" - } - - openvm_algebra_complex_macros::complex_init! { - Bn254Fp2 { mod_idx = 0 }, - } + openvm::init!("openvm_init_pairing_check_bn254.rs"); pub fn test_pairing_check(io: &[u8]) { setup_0(); @@ -61,14 +54,7 @@ mod bls12_381 { use super::*; - openvm_algebra_moduli_macros::moduli_init! { - "0x1a0111ea397fe69a4b1ba7b6434bacd764774b84f38512bf6730d2a0f6b0f6241eabfffeb153ffffb9feffffffffaaab", - "0x73eda753299d7d483339d80809a1d80553bda402fffe5bfeffffffff00000001" - } - - openvm_algebra_complex_macros::complex_init! { - Bls12_381Fp2 { mod_idx = 0 }, - } + openvm::init!("openvm_init_pairing_check_bls12_381.rs"); pub fn test_pairing_check(io: &[u8]) { setup_0(); diff --git a/extensions/pairing/tests/programs/examples/pairing_check_fallback.rs b/extensions/pairing/tests/programs/examples/pairing_check_fallback.rs index 7f47709097..f9b504bb71 100644 --- a/extensions/pairing/tests/programs/examples/pairing_check_fallback.rs +++ b/extensions/pairing/tests/programs/examples/pairing_check_fallback.rs @@ -24,14 +24,7 @@ mod bn254 { use super::*; - openvm_algebra_moduli_macros::moduli_init! { - "0x30644e72e131a029b85045b68181585d97816a916871ca8d3c208c16d87cfd47", - "0x30644e72e131a029b85045b68181585d2833e84879b9709143e1f593f0000001" - } - - openvm_algebra_complex_macros::complex_init! { - Bn254Fp2 { mod_idx = 0 }, - } + openvm::init!("openvm_init_pairing_check_fallback_bn254.rs"); // Wrapper so that we can override `pairing_check_hint` struct Bn254Wrapper(Bn254); @@ -138,14 +131,7 @@ mod bls12_381 { use super::*; - openvm_algebra_moduli_macros::moduli_init! { - "0x1a0111ea397fe69a4b1ba7b6434bacd764774b84f38512bf6730d2a0f6b0f6241eabfffeb153ffffb9feffffffffaaab", - "0x73eda753299d7d483339d80809a1d80553bda402fffe5bfeffffffff00000001" - } - - openvm_algebra_complex_macros::complex_init! { - Bls12_381Fp2 { mod_idx = 0 }, - } + openvm::init!("openvm_init_pairing_check_fallback_bls12_381.rs"); // Wrapper so that we can override `pairing_check_hint` struct Bls12_381Wrapper(Bls12_381); diff --git a/extensions/pairing/tests/programs/examples/pairing_line.rs b/extensions/pairing/tests/programs/examples/pairing_line.rs index 27969e15ea..36a9d1b4ef 100644 --- a/extensions/pairing/tests/programs/examples/pairing_line.rs +++ b/extensions/pairing/tests/programs/examples/pairing_line.rs @@ -14,14 +14,7 @@ mod bn254 { use super::*; - openvm_algebra_moduli_macros::moduli_init! { - "0x30644e72e131a029b85045b68181585d97816a916871ca8d3c208c16d87cfd47", - "0x30644e72e131a029b85045b68181585d2833e84879b9709143e1f593f0000001" - } - - openvm_algebra_complex_macros::complex_init! { - Bn254Fp2 { mod_idx = 0 }, - } + openvm::init!("openvm_init_pairing_line_bn254.rs"); pub fn test_mul_013_by_013(io: &[u8]) { assert_eq!(io.len(), 32 * 18); @@ -82,14 +75,7 @@ mod bls12_381 { use super::*; - openvm_algebra_moduli_macros::moduli_init! { - "0x1a0111ea397fe69a4b1ba7b6434bacd764774b84f38512bf6730d2a0f6b0f6241eabfffeb153ffffb9feffffffffaaab", - "0x73eda753299d7d483339d80809a1d80553bda402fffe5bfeffffffff00000001" - } - - openvm_algebra_complex_macros::complex_init! { - Bls12_381Fp2 { mod_idx = 0 }, - } + openvm::init!("openvm_init_pairing_line_bls12_381.rs"); pub fn test_mul_023_by_023(io: &[u8]) { assert_eq!(io.len(), 48 * 18); diff --git a/extensions/pairing/tests/programs/examples/pairing_miller_loop.rs b/extensions/pairing/tests/programs/examples/pairing_miller_loop.rs index eb732f68a2..afc1ae37a4 100644 --- a/extensions/pairing/tests/programs/examples/pairing_miller_loop.rs +++ b/extensions/pairing/tests/programs/examples/pairing_miller_loop.rs @@ -17,14 +17,7 @@ mod bn254 { use super::*; - openvm_algebra_moduli_macros::moduli_init! { - "0x30644e72e131a029b85045b68181585d97816a916871ca8d3c208c16d87cfd47", - "0x30644e72e131a029b85045b68181585d2833e84879b9709143e1f593f0000001" - } - - openvm_algebra_complex_macros::complex_init! { - Bn254Fp2 { mod_idx = 0 }, - } + openvm::init!("openvm_init_pairing_miller_loop_bn254.rs"); pub fn test_miller_loop(io: &[u8]) { setup_0(); @@ -60,14 +53,7 @@ mod bls12_381 { use super::*; - openvm_algebra_moduli_macros::moduli_init! { - "0x1a0111ea397fe69a4b1ba7b6434bacd764774b84f38512bf6730d2a0f6b0f6241eabfffeb153ffffb9feffffffffaaab", - "0x73eda753299d7d483339d80809a1d80553bda402fffe5bfeffffffff00000001" - } - - openvm_algebra_complex_macros::complex_init! { - Bls12_381Fp2 { mod_idx = 0 }, - } + openvm::init!("openvm_init_pairing_miller_loop_bls12_381.rs"); pub fn test_miller_loop(io: &[u8]) { setup_0(); diff --git a/extensions/pairing/tests/programs/examples/pairing_miller_step.rs b/extensions/pairing/tests/programs/examples/pairing_miller_step.rs index c32aefc7c1..c944298aa6 100644 --- a/extensions/pairing/tests/programs/examples/pairing_miller_step.rs +++ b/extensions/pairing/tests/programs/examples/pairing_miller_step.rs @@ -16,14 +16,7 @@ mod bn254 { use super::*; - openvm_algebra_moduli_macros::moduli_init! { - "0x30644e72e131a029b85045b68181585d97816a916871ca8d3c208c16d87cfd47", - "0x30644e72e131a029b85045b68181585d2833e84879b9709143e1f593f0000001" - } - - openvm_algebra_complex_macros::complex_init! { - Bn254Fp2 { mod_idx = 0 }, - } + openvm::init!("openvm_init_pairing_miller_step_bn254.rs"); pub fn test_miller_step(io: &[u8]) { assert_eq!(io.len(), 32 * 12); @@ -95,14 +88,7 @@ mod bls12_381 { use super::*; - openvm_algebra_moduli_macros::moduli_init! { - "0x1a0111ea397fe69a4b1ba7b6434bacd764774b84f38512bf6730d2a0f6b0f6241eabfffeb153ffffb9feffffffffaaab", - "0x73eda753299d7d483339d80809a1d80553bda402fffe5bfeffffffff00000001" - } - - openvm_algebra_complex_macros::complex_init! { - Bls12_381Fp2 { mod_idx = 0 }, - } + openvm::init!("openvm_init_pairing_miller_step_bls12_381.rs"); pub fn test_miller_step(io: &[u8]) { assert_eq!(io.len(), 48 * 12); diff --git a/extensions/pairing/tests/programs/openvm_init_bls_ec_bls12_381.rs b/extensions/pairing/tests/programs/openvm_init_bls_ec_bls12_381.rs new file mode 100644 index 0000000000..95a4e46fd3 --- /dev/null +++ b/extensions/pairing/tests/programs/openvm_init_bls_ec_bls12_381.rs @@ -0,0 +1,3 @@ +// This file is automatically generated by cargo openvm. Do not rename or edit. +openvm_algebra_guest::moduli_macros::moduli_init! { "4002409555221667393417789825735904156556882819939007885332058136124031650490837864442687629129015664037894272559787", "52435875175126190479447740508185965837690552500527637822603658699938581184513" } +openvm_ecc_guest::sw_macros::sw_init! { Bls12_381G1Affine } diff --git a/extensions/pairing/tests/programs/openvm_init_bls_final_exp_hint_bls12_381.rs b/extensions/pairing/tests/programs/openvm_init_bls_final_exp_hint_bls12_381.rs new file mode 100644 index 0000000000..00181b03ef --- /dev/null +++ b/extensions/pairing/tests/programs/openvm_init_bls_final_exp_hint_bls12_381.rs @@ -0,0 +1,4 @@ +// This file is automatically generated by cargo openvm. Do not rename or edit. +openvm_algebra_guest::moduli_macros::moduli_init! { "4002409555221667393417789825735904156556882819939007885332058136124031650490837864442687629129015664037894272559787" } +openvm_algebra_guest::complex_macros::complex_init! { Bls12_381Fp2 { mod_idx = 0 } } +openvm_ecc_guest::sw_macros::sw_init! { } diff --git a/extensions/pairing/tests/programs/openvm_init_bn_final_exp_hint_bn254.rs b/extensions/pairing/tests/programs/openvm_init_bn_final_exp_hint_bn254.rs new file mode 100644 index 0000000000..c130859ad8 --- /dev/null +++ b/extensions/pairing/tests/programs/openvm_init_bn_final_exp_hint_bn254.rs @@ -0,0 +1,4 @@ +// This file is automatically generated by cargo openvm. Do not rename or edit. +openvm_algebra_guest::moduli_macros::moduli_init! { "21888242871839275222246405745257275088696311157297823662689037894645226208583" } +openvm_algebra_guest::complex_macros::complex_init! { Bn254Fp2 { mod_idx = 0 } } +openvm_ecc_guest::sw_macros::sw_init! { } diff --git a/extensions/pairing/tests/programs/openvm_init_fp12_mul_bls12_381.rs b/extensions/pairing/tests/programs/openvm_init_fp12_mul_bls12_381.rs new file mode 100644 index 0000000000..00181b03ef --- /dev/null +++ b/extensions/pairing/tests/programs/openvm_init_fp12_mul_bls12_381.rs @@ -0,0 +1,4 @@ +// This file is automatically generated by cargo openvm. Do not rename or edit. +openvm_algebra_guest::moduli_macros::moduli_init! { "4002409555221667393417789825735904156556882819939007885332058136124031650490837864442687629129015664037894272559787" } +openvm_algebra_guest::complex_macros::complex_init! { Bls12_381Fp2 { mod_idx = 0 } } +openvm_ecc_guest::sw_macros::sw_init! { } diff --git a/extensions/pairing/tests/programs/openvm_init_fp12_mul_bn254.rs b/extensions/pairing/tests/programs/openvm_init_fp12_mul_bn254.rs new file mode 100644 index 0000000000..c130859ad8 --- /dev/null +++ b/extensions/pairing/tests/programs/openvm_init_fp12_mul_bn254.rs @@ -0,0 +1,4 @@ +// This file is automatically generated by cargo openvm. Do not rename or edit. +openvm_algebra_guest::moduli_macros::moduli_init! { "21888242871839275222246405745257275088696311157297823662689037894645226208583" } +openvm_algebra_guest::complex_macros::complex_init! { Bn254Fp2 { mod_idx = 0 } } +openvm_ecc_guest::sw_macros::sw_init! { } diff --git a/extensions/pairing/tests/programs/openvm_init_pairing_check_bls12_381.rs b/extensions/pairing/tests/programs/openvm_init_pairing_check_bls12_381.rs new file mode 100644 index 0000000000..00181b03ef --- /dev/null +++ b/extensions/pairing/tests/programs/openvm_init_pairing_check_bls12_381.rs @@ -0,0 +1,4 @@ +// This file is automatically generated by cargo openvm. Do not rename or edit. +openvm_algebra_guest::moduli_macros::moduli_init! { "4002409555221667393417789825735904156556882819939007885332058136124031650490837864442687629129015664037894272559787" } +openvm_algebra_guest::complex_macros::complex_init! { Bls12_381Fp2 { mod_idx = 0 } } +openvm_ecc_guest::sw_macros::sw_init! { } diff --git a/extensions/pairing/tests/programs/openvm_init_pairing_check_bn254.rs b/extensions/pairing/tests/programs/openvm_init_pairing_check_bn254.rs new file mode 100644 index 0000000000..c130859ad8 --- /dev/null +++ b/extensions/pairing/tests/programs/openvm_init_pairing_check_bn254.rs @@ -0,0 +1,4 @@ +// This file is automatically generated by cargo openvm. Do not rename or edit. +openvm_algebra_guest::moduli_macros::moduli_init! { "21888242871839275222246405745257275088696311157297823662689037894645226208583" } +openvm_algebra_guest::complex_macros::complex_init! { Bn254Fp2 { mod_idx = 0 } } +openvm_ecc_guest::sw_macros::sw_init! { } diff --git a/extensions/pairing/tests/programs/openvm_init_pairing_check_fallback_bls12_381.rs b/extensions/pairing/tests/programs/openvm_init_pairing_check_fallback_bls12_381.rs new file mode 100644 index 0000000000..00181b03ef --- /dev/null +++ b/extensions/pairing/tests/programs/openvm_init_pairing_check_fallback_bls12_381.rs @@ -0,0 +1,4 @@ +// This file is automatically generated by cargo openvm. Do not rename or edit. +openvm_algebra_guest::moduli_macros::moduli_init! { "4002409555221667393417789825735904156556882819939007885332058136124031650490837864442687629129015664037894272559787" } +openvm_algebra_guest::complex_macros::complex_init! { Bls12_381Fp2 { mod_idx = 0 } } +openvm_ecc_guest::sw_macros::sw_init! { } diff --git a/extensions/pairing/tests/programs/openvm_init_pairing_check_fallback_bn254.rs b/extensions/pairing/tests/programs/openvm_init_pairing_check_fallback_bn254.rs new file mode 100644 index 0000000000..c130859ad8 --- /dev/null +++ b/extensions/pairing/tests/programs/openvm_init_pairing_check_fallback_bn254.rs @@ -0,0 +1,4 @@ +// This file is automatically generated by cargo openvm. Do not rename or edit. +openvm_algebra_guest::moduli_macros::moduli_init! { "21888242871839275222246405745257275088696311157297823662689037894645226208583" } +openvm_algebra_guest::complex_macros::complex_init! { Bn254Fp2 { mod_idx = 0 } } +openvm_ecc_guest::sw_macros::sw_init! { } diff --git a/extensions/pairing/tests/programs/openvm_init_pairing_line_bls12_381.rs b/extensions/pairing/tests/programs/openvm_init_pairing_line_bls12_381.rs new file mode 100644 index 0000000000..00181b03ef --- /dev/null +++ b/extensions/pairing/tests/programs/openvm_init_pairing_line_bls12_381.rs @@ -0,0 +1,4 @@ +// This file is automatically generated by cargo openvm. Do not rename or edit. +openvm_algebra_guest::moduli_macros::moduli_init! { "4002409555221667393417789825735904156556882819939007885332058136124031650490837864442687629129015664037894272559787" } +openvm_algebra_guest::complex_macros::complex_init! { Bls12_381Fp2 { mod_idx = 0 } } +openvm_ecc_guest::sw_macros::sw_init! { } diff --git a/extensions/pairing/tests/programs/openvm_init_pairing_line_bn254.rs b/extensions/pairing/tests/programs/openvm_init_pairing_line_bn254.rs new file mode 100644 index 0000000000..c130859ad8 --- /dev/null +++ b/extensions/pairing/tests/programs/openvm_init_pairing_line_bn254.rs @@ -0,0 +1,4 @@ +// This file is automatically generated by cargo openvm. Do not rename or edit. +openvm_algebra_guest::moduli_macros::moduli_init! { "21888242871839275222246405745257275088696311157297823662689037894645226208583" } +openvm_algebra_guest::complex_macros::complex_init! { Bn254Fp2 { mod_idx = 0 } } +openvm_ecc_guest::sw_macros::sw_init! { } diff --git a/extensions/pairing/tests/programs/openvm_init_pairing_miller_loop_bls12_381.rs b/extensions/pairing/tests/programs/openvm_init_pairing_miller_loop_bls12_381.rs new file mode 100644 index 0000000000..00181b03ef --- /dev/null +++ b/extensions/pairing/tests/programs/openvm_init_pairing_miller_loop_bls12_381.rs @@ -0,0 +1,4 @@ +// This file is automatically generated by cargo openvm. Do not rename or edit. +openvm_algebra_guest::moduli_macros::moduli_init! { "4002409555221667393417789825735904156556882819939007885332058136124031650490837864442687629129015664037894272559787" } +openvm_algebra_guest::complex_macros::complex_init! { Bls12_381Fp2 { mod_idx = 0 } } +openvm_ecc_guest::sw_macros::sw_init! { } diff --git a/extensions/pairing/tests/programs/openvm_init_pairing_miller_loop_bn254.rs b/extensions/pairing/tests/programs/openvm_init_pairing_miller_loop_bn254.rs new file mode 100644 index 0000000000..c130859ad8 --- /dev/null +++ b/extensions/pairing/tests/programs/openvm_init_pairing_miller_loop_bn254.rs @@ -0,0 +1,4 @@ +// This file is automatically generated by cargo openvm. Do not rename or edit. +openvm_algebra_guest::moduli_macros::moduli_init! { "21888242871839275222246405745257275088696311157297823662689037894645226208583" } +openvm_algebra_guest::complex_macros::complex_init! { Bn254Fp2 { mod_idx = 0 } } +openvm_ecc_guest::sw_macros::sw_init! { } diff --git a/extensions/pairing/tests/programs/openvm_init_pairing_miller_step_bls12_381.rs b/extensions/pairing/tests/programs/openvm_init_pairing_miller_step_bls12_381.rs new file mode 100644 index 0000000000..00181b03ef --- /dev/null +++ b/extensions/pairing/tests/programs/openvm_init_pairing_miller_step_bls12_381.rs @@ -0,0 +1,4 @@ +// This file is automatically generated by cargo openvm. Do not rename or edit. +openvm_algebra_guest::moduli_macros::moduli_init! { "4002409555221667393417789825735904156556882819939007885332058136124031650490837864442687629129015664037894272559787" } +openvm_algebra_guest::complex_macros::complex_init! { Bls12_381Fp2 { mod_idx = 0 } } +openvm_ecc_guest::sw_macros::sw_init! { } diff --git a/extensions/pairing/tests/programs/openvm_init_pairing_miller_step_bn254.rs b/extensions/pairing/tests/programs/openvm_init_pairing_miller_step_bn254.rs new file mode 100644 index 0000000000..c130859ad8 --- /dev/null +++ b/extensions/pairing/tests/programs/openvm_init_pairing_miller_step_bn254.rs @@ -0,0 +1,4 @@ +// This file is automatically generated by cargo openvm. Do not rename or edit. +openvm_algebra_guest::moduli_macros::moduli_init! { "21888242871839275222246405745257275088696311157297823662689037894645226208583" } +openvm_algebra_guest::complex_macros::complex_init! { Bn254Fp2 { mod_idx = 0 } } +openvm_ecc_guest::sw_macros::sw_init! { } diff --git a/extensions/pairing/tests/src/lib.rs b/extensions/pairing/tests/src/lib.rs index 33890ecb8a..f14956f024 100644 --- a/extensions/pairing/tests/src/lib.rs +++ b/extensions/pairing/tests/src/lib.rs @@ -23,7 +23,7 @@ mod bn254 { use openvm_instructions::exe::VmExe; use openvm_pairing_circuit::{PairingCurve, PairingExtension, Rv32PairingConfig}; use openvm_pairing_guest::{ - bn254::BN254_MODULUS, + bn254::{BN254_COMPLEX_STRUCT_NAME, BN254_MODULUS}, halo2curves_shims::bn254::Bn254, pairing::{EvaluatedLine, FinalExp, LineMulDType, MillerStep, MultiMillerLoop}, }; @@ -40,13 +40,18 @@ mod bn254 { pub fn get_testing_config() -> Rv32PairingConfig { let primes = [BN254_MODULUS.clone()]; + let complex_struct_names = [BN254_COMPLEX_STRUCT_NAME.to_string()]; + let primes_with_names = complex_struct_names + .into_iter() + .zip(primes.clone()) + .collect::>(); Rv32PairingConfig { system: SystemConfig::default().with_continuations(), base: Default::default(), mul: Default::default(), io: Default::default(), modular: ModularExtension::new(primes.to_vec()), - fp2: Fp2Extension::new(primes.to_vec()), + fp2: Fp2Extension::new(primes_with_names), weierstrass: WeierstrassExtension::new(vec![]), pairing: PairingExtension::new(vec![PairingCurve::Bn254]), } @@ -54,10 +59,12 @@ mod bn254 { #[test] fn test_bn254_fp12_mul() -> Result<()> { + let config = get_testing_config(); let elf = build_example_program_at_path_with_features( get_programs_dir!(), "fp12_mul", ["bn254"], + &config, )?; let openvm_exe = VmExe::from_elf( elf, @@ -82,16 +89,18 @@ mod bn254 { .map(FieldAlgebra::from_canonical_u8) .collect::>(); - air_test_with_min_segments(get_testing_config(), openvm_exe, vec![io], 1); + air_test_with_min_segments(config, openvm_exe, vec![io], 1); Ok(()) } #[test] fn test_bn254_line_functions() -> Result<()> { + let config = get_testing_config(); let elf = build_example_program_at_path_with_features( get_programs_dir!(), "pairing_line", ["bn254"], + &config, )?; let openvm_exe = VmExe::from_elf( elf, @@ -138,16 +147,18 @@ mod bn254 { let io_all = io0.into_iter().chain(io1).collect::>(); - air_test_with_min_segments(get_testing_config(), openvm_exe, vec![io_all], 1); + air_test_with_min_segments(config, openvm_exe, vec![io_all], 1); Ok(()) } #[test] fn test_bn254_miller_step() -> Result<()> { + let config = get_testing_config(); let elf = build_example_program_at_path_with_features( get_programs_dir!(), "pairing_miller_step", ["bn254"], + &config, )?; let openvm_exe = VmExe::from_elf( elf, @@ -185,16 +196,18 @@ mod bn254 { let io_all = io0.into_iter().chain(io1).collect::>(); - air_test_with_min_segments(get_testing_config(), openvm_exe, vec![io_all], 1); + air_test_with_min_segments(config, openvm_exe, vec![io_all], 1); Ok(()) } #[test] fn test_bn254_miller_loop() -> Result<()> { + let config = get_testing_config(); let elf = build_example_program_at_path_with_features( get_programs_dir!(), "pairing_miller_loop", ["bn254"], + &config, )?; let openvm_exe = VmExe::from_elf( elf, @@ -236,16 +249,18 @@ mod bn254 { let io_all = io0.into_iter().chain(io1).collect::>(); - air_test_with_min_segments(get_testing_config(), openvm_exe, vec![io_all], 1); + air_test_with_min_segments(config, openvm_exe, vec![io_all], 1); Ok(()) } #[test] fn test_bn254_pairing_check() -> Result<()> { + let config = get_testing_config(); let elf = build_example_program_at_path_with_features( get_programs_dir!(), "pairing_check", ["bn254"], + &config, )?; let openvm_exe = VmExe::from_elf( elf, @@ -291,16 +306,18 @@ mod bn254 { let io_all = io0.into_iter().chain(io1).collect::>(); - air_test_with_min_segments(get_testing_config(), openvm_exe, vec![io_all], 1); + air_test_with_min_segments(config, openvm_exe, vec![io_all], 1); Ok(()) } #[test] fn test_bn254_pairing_check_fallback() -> Result<()> { + let config = get_testing_config(); let elf = build_example_program_at_path_with_features( get_programs_dir!(), "pairing_check_fallback", ["bn254"], + &config, )?; let openvm_exe = VmExe::from_elf( elf, @@ -352,10 +369,12 @@ mod bn254 { #[test] fn test_bn254_final_exp_hint() -> Result<()> { + let config = get_testing_config(); let elf = build_example_program_at_path_with_features( get_programs_dir!(), "bn_final_exp_hint", ["bn254"], + &config, )?; let openvm_exe = VmExe::from_elf( elf, @@ -398,7 +417,7 @@ mod bn254 { .flat_map(|w| w.to_le_bytes()) .map(F::from_canonical_u8) .collect(); - air_test_with_min_segments(get_testing_config(), openvm_exe, vec![io], 1); + air_test_with_min_segments(config, openvm_exe, vec![io], 1); Ok(()) } } @@ -426,7 +445,10 @@ mod bls12_381 { use openvm_ecc_transpiler::EccTranspilerExtension; use openvm_pairing_circuit::{PairingCurve, PairingExtension, Rv32PairingConfig}; use openvm_pairing_guest::{ - bls12_381::{BLS12_381_MODULUS, BLS12_381_ORDER}, + bls12_381::{ + BLS12_381_COMPLEX_STRUCT_NAME, BLS12_381_ECC_STRUCT_NAME, BLS12_381_MODULUS, + BLS12_381_ORDER, + }, halo2curves_shims::bls12_381::Bls12_381, pairing::{EvaluatedLine, FinalExp, LineMulMType, MillerStep, MultiMillerLoop}, }; @@ -443,13 +465,18 @@ mod bls12_381 { pub fn get_testing_config() -> Rv32PairingConfig { let primes = [BLS12_381_MODULUS.clone()]; + let complex_struct_names = [BLS12_381_COMPLEX_STRUCT_NAME.to_string()]; + let primes_with_names = complex_struct_names + .into_iter() + .zip(primes.clone()) + .collect::>(); Rv32PairingConfig { system: SystemConfig::default().with_continuations(), base: Default::default(), mul: Default::default(), io: Default::default(), modular: ModularExtension::new(primes.to_vec()), - fp2: Fp2Extension::new(primes.to_vec()), + fp2: Fp2Extension::new(primes_with_names), weierstrass: WeierstrassExtension::new(vec![]), pairing: PairingExtension::new(vec![PairingCurve::Bls12_381]), } @@ -457,17 +484,20 @@ mod bls12_381 { #[test] fn test_bls_ec() -> Result<()> { - let elf = build_example_program_at_path_with_features( - get_programs_dir!(), - "bls_ec", - ["bls12_381"], - )?; let curve = CurveConfig { + struct_name: BLS12_381_ECC_STRUCT_NAME.to_string(), modulus: BLS12_381_MODULUS.clone(), scalar: BLS12_381_ORDER.clone(), a: BigUint::ZERO, b: BigUint::from_u8(4).unwrap(), }; + let config = Rv32WeierstrassConfig::new(vec![curve]); + let elf = build_example_program_at_path_with_features( + get_programs_dir!(), + "bls_ec", + ["bls12_381"], + &config, + )?; let openvm_exe = VmExe::from_elf( elf, Transpiler::::default() @@ -477,17 +507,18 @@ mod bls12_381 { .with_extension(EccTranspilerExtension) .with_extension(ModularTranspilerExtension), )?; - let config = Rv32WeierstrassConfig::new(vec![curve]); air_test(config, openvm_exe); Ok(()) } #[test] fn test_bls12_381_fp12_mul() -> Result<()> { + let config = get_testing_config(); let elf = build_example_program_at_path_with_features( get_programs_dir!(), "fp12_mul", ["bls12_381"], + &config, )?; let openvm_exe = VmExe::from_elf( elf, @@ -512,16 +543,18 @@ mod bls12_381 { .map(FieldAlgebra::from_canonical_u8) .collect::>(); - air_test_with_min_segments(get_testing_config(), openvm_exe, vec![io], 1); + air_test_with_min_segments(config, openvm_exe, vec![io], 1); Ok(()) } #[test] fn test_bls12_381_line_functions() -> Result<()> { + let config = get_testing_config(); let elf = build_example_program_at_path_with_features( get_programs_dir!(), "pairing_line", ["bls12_381"], + &config, )?; let openvm_exe = VmExe::from_elf( elf, @@ -569,16 +602,18 @@ mod bls12_381 { let io_all = io0.into_iter().chain(io1).collect::>(); - air_test_with_min_segments(get_testing_config(), openvm_exe, vec![io_all], 1); + air_test_with_min_segments(config, openvm_exe, vec![io_all], 1); Ok(()) } #[test] fn test_bls12_381_miller_step() -> Result<()> { + let config = get_testing_config(); let elf = build_example_program_at_path_with_features( get_programs_dir!(), "pairing_miller_step", ["bls12_381"], + &config, )?; let openvm_exe = VmExe::from_elf( elf, @@ -616,16 +651,18 @@ mod bls12_381 { let io_all = io0.into_iter().chain(io1).collect::>(); - air_test_with_min_segments(get_testing_config(), openvm_exe, vec![io_all], 1); + air_test_with_min_segments(config, openvm_exe, vec![io_all], 1); Ok(()) } #[test] fn test_bls12_381_miller_loop() -> Result<()> { + let config = get_testing_config(); let elf = build_example_program_at_path_with_features( get_programs_dir!(), "pairing_miller_loop", ["bls12_381"], + &config, )?; let openvm_exe = VmExe::from_elf( elf, @@ -673,16 +710,18 @@ mod bls12_381 { let io_all = io0.into_iter().chain(io1).collect::>(); - air_test_with_min_segments(get_testing_config(), openvm_exe, vec![io_all], 1); + air_test_with_min_segments(config, openvm_exe, vec![io_all], 1); Ok(()) } #[test] fn test_bls12_381_pairing_check() -> Result<()> { + let config = get_testing_config(); let elf = build_example_program_at_path_with_features( get_programs_dir!(), "pairing_check", ["bls12_381"], + &config, )?; let openvm_exe = VmExe::from_elf( elf, @@ -727,17 +766,19 @@ mod bls12_381 { let io_all = io0.into_iter().chain(io1).collect::>(); - air_test_with_min_segments(get_testing_config(), openvm_exe, vec![io_all], 1); + air_test_with_min_segments(config, openvm_exe, vec![io_all], 1); Ok(()) } #[ignore] #[test] fn test_bls12_381_pairing_check_fallback() -> Result<()> { + let config = get_testing_config(); let elf = build_example_program_at_path_with_features( get_programs_dir!(), "pairing_check_fallback", ["bls12_381"], + &config, )?; let openvm_exe = VmExe::from_elf( elf, @@ -788,10 +829,12 @@ mod bls12_381 { #[test] fn test_bls12_381_final_exp_hint() -> Result<()> { + let config = get_testing_config(); let elf = build_example_program_at_path_with_features( get_programs_dir!(), "bls_final_exp_hint", ["bls12_381"], + &config, )?; let openvm_exe = VmExe::from_elf( elf, @@ -835,7 +878,7 @@ mod bls12_381 { .flat_map(|w| w.to_le_bytes()) .map(F::from_canonical_u8) .collect(); - air_test_with_min_segments(get_testing_config(), openvm_exe, vec![io], 1); + air_test_with_min_segments(config, openvm_exe, vec![io], 1); Ok(()) } } diff --git a/extensions/rv32im/circuit/src/extension.rs b/extensions/rv32im/circuit/src/extension.rs index f1f67d3994..6760df923b 100644 --- a/extensions/rv32im/circuit/src/extension.rs +++ b/extensions/rv32im/circuit/src/extension.rs @@ -1,7 +1,8 @@ use derive_more::derive::From; use openvm_circuit::{ arch::{ - SystemConfig, SystemPort, VmExtension, VmInventory, VmInventoryBuilder, VmInventoryError, + InitFileGenerator, SystemConfig, SystemPort, VmExtension, VmInventory, VmInventoryBuilder, + VmInventoryError, }, system::phantom::PhantomChip, }; @@ -34,6 +35,9 @@ pub struct Rv32IConfig { pub io: Rv32Io, } +// Default implementation uses no init file +impl InitFileGenerator for Rv32IConfig {} + /// Config for a VM with base extension, IO extension, and multiplication extension #[derive(Clone, Debug, Default, VmConfig, derive_new::new, Serialize, Deserialize)] pub struct Rv32ImConfig { @@ -43,6 +47,9 @@ pub struct Rv32ImConfig { pub mul: Rv32M, } +// Default implementation uses no init file +impl InitFileGenerator for Rv32ImConfig {} + impl Default for Rv32IConfig { fn default() -> Self { let system = SystemConfig::default().with_continuations(); diff --git a/extensions/rv32im/tests/src/lib.rs b/extensions/rv32im/tests/src/lib.rs index e59088eac8..cde90f6a82 100644 --- a/extensions/rv32im/tests/src/lib.rs +++ b/extensions/rv32im/tests/src/lib.rs @@ -23,7 +23,8 @@ mod tests { #[test_case("fibonacci", 1)] fn test_rv32i(example_name: &str, min_segments: usize) -> Result<()> { - let elf = build_example_program_at_path(get_programs_dir!(), example_name)?; + let config = Rv32IConfig::default(); + let elf = build_example_program_at_path(get_programs_dir!(), example_name, &config)?; let exe = VmExe::from_elf( elf, Transpiler::::default() @@ -31,14 +32,14 @@ mod tests { .with_extension(Rv32MTranspilerExtension) .with_extension(Rv32IoTranspilerExtension), )?; - let config = Rv32IConfig::default(); air_test_with_min_segments(config, exe, vec![], min_segments); Ok(()) } #[test_case("collatz", 1)] fn test_rv32im(example_name: &str, min_segments: usize) -> Result<()> { - let elf = build_example_program_at_path(get_programs_dir!(), example_name)?; + let config = Rv32ImConfig::default(); + let elf = build_example_program_at_path(get_programs_dir!(), example_name, &config)?; let exe = VmExe::from_elf( elf, Transpiler::::default() @@ -46,7 +47,6 @@ mod tests { .with_extension(Rv32IoTranspilerExtension) .with_extension(Rv32MTranspilerExtension), )?; - let config = Rv32ImConfig::default(); air_test_with_min_segments(config, exe, vec![], min_segments); Ok(()) } @@ -54,10 +54,12 @@ mod tests { // #[test_case("fibonacci", 1)] #[test_case("collatz", 1)] fn test_rv32im_std(example_name: &str, min_segments: usize) -> Result<()> { + let config = Rv32ImConfig::default(); let elf = build_example_program_at_path_with_features( get_programs_dir!(), example_name, ["std"], + &config, )?; let exe = VmExe::from_elf( elf, @@ -66,14 +68,14 @@ mod tests { .with_extension(Rv32IoTranspilerExtension) .with_extension(Rv32MTranspilerExtension), )?; - let config = Rv32ImConfig::default(); air_test_with_min_segments(config, exe, vec![], min_segments); Ok(()) } #[test] fn test_read_vec() -> Result<()> { - let elf = build_example_program_at_path(get_programs_dir!(), "hint")?; + let config = Rv32IConfig::default(); + let elf = build_example_program_at_path(get_programs_dir!(), "hint", &config)?; let exe = VmExe::from_elf( elf, Transpiler::::default() @@ -81,7 +83,6 @@ mod tests { .with_extension(Rv32MTranspilerExtension) .with_extension(Rv32IoTranspilerExtension), )?; - let config = Rv32IConfig::default(); let input = vec![[0, 1, 2, 3].map(F::from_canonical_u8).to_vec()]; air_test_with_min_segments(config, exe, input, 1); Ok(()) @@ -89,7 +90,8 @@ mod tests { #[test] fn test_read() -> Result<()> { - let elf = build_example_program_at_path(get_programs_dir!(), "read")?; + let config = Rv32IConfig::default(); + let elf = build_example_program_at_path(get_programs_dir!(), "read", &config)?; let exe = VmExe::from_elf( elf, Transpiler::::default() @@ -97,7 +99,6 @@ mod tests { .with_extension(Rv32MTranspilerExtension) .with_extension(Rv32IoTranspilerExtension), )?; - let config = Rv32IConfig::default(); #[derive(serde::Serialize)] struct Foo { @@ -120,7 +121,8 @@ mod tests { #[test] fn test_reveal() -> Result<()> { - let elf = build_example_program_at_path(get_programs_dir!(), "reveal")?; + let config = Rv32IConfig::default(); + let elf = build_example_program_at_path(get_programs_dir!(), "reveal", &config)?; let exe = VmExe::from_elf( elf, Transpiler::::default() @@ -128,7 +130,6 @@ mod tests { .with_extension(Rv32MTranspilerExtension) .with_extension(Rv32IoTranspilerExtension), )?; - let config = Rv32IConfig::default(); let executor = VmExecutor::::new(config.clone()); let final_memory = executor.execute(exe, vec![])?.unwrap(); let hasher = vm_poseidon2_hasher(); @@ -159,7 +160,8 @@ mod tests { #[test] fn test_print() -> Result<()> { - let elf = build_example_program_at_path(get_programs_dir!(), "print")?; + let config = Rv32IConfig::default(); + let elf = build_example_program_at_path(get_programs_dir!(), "print", &config)?; let exe = VmExe::from_elf( elf, Transpiler::::default() @@ -167,14 +169,14 @@ mod tests { .with_extension(Rv32MTranspilerExtension) .with_extension(Rv32IoTranspilerExtension), )?; - let config = Rv32IConfig::default(); air_test(config, exe); Ok(()) } #[test] fn test_heap_overflow() -> Result<()> { - let elf = build_example_program_at_path(get_programs_dir!(), "heap_overflow")?; + let config = Rv32ImConfig::default(); + let elf = build_example_program_at_path(get_programs_dir!(), "heap_overflow", &config)?; let exe = VmExe::from_elf( elf, Transpiler::::default() @@ -182,7 +184,6 @@ mod tests { .with_extension(Rv32MTranspilerExtension) .with_extension(Rv32IoTranspilerExtension), )?; - let config = Rv32ImConfig::default(); let executor = VmExecutor::::new(config.clone()); match executor.execute(exe, vec![[0, 0, 0, 1].map(F::from_canonical_u8).to_vec()]) { @@ -194,8 +195,13 @@ mod tests { #[test] fn test_hashmap() -> Result<()> { - let elf = - build_example_program_at_path_with_features(get_programs_dir!(), "hashmap", ["std"])?; + let config = Rv32ImConfig::default(); + let elf = build_example_program_at_path_with_features( + get_programs_dir!(), + "hashmap", + ["std"], + &config, + )?; let exe = VmExe::from_elf( elf, Transpiler::::default() @@ -203,17 +209,18 @@ mod tests { .with_extension(Rv32MTranspilerExtension) .with_extension(Rv32IoTranspilerExtension), )?; - let config = Rv32ImConfig::default(); air_test(config, exe); Ok(()) } #[test] fn test_tiny_mem_test() -> Result<()> { + let config = Rv32ImConfig::default(); let elf = build_example_program_at_path_with_features( get_programs_dir!(), "tiny-mem-test", ["heap-embedded-alloc"], + &config, )?; let exe = VmExe::from_elf( elf, @@ -222,7 +229,6 @@ mod tests { .with_extension(Rv32MTranspilerExtension) .with_extension(Rv32IoTranspilerExtension), )?; - let config = Rv32ImConfig::default(); air_test(config, exe); Ok(()) } @@ -230,7 +236,8 @@ mod tests { #[test] #[should_panic] fn test_load_x0() { - let elf = build_example_program_at_path(get_programs_dir!(), "load_x0").unwrap(); + let config = Rv32ImConfig::default(); + let elf = build_example_program_at_path(get_programs_dir!(), "load_x0", &config).unwrap(); let exe = VmExe::from_elf( elf, Transpiler::::default() @@ -239,7 +246,6 @@ mod tests { .with_extension(Rv32IoTranspilerExtension), ) .unwrap(); - let config = Rv32ImConfig::default(); let executor = VmExecutor::::new(config.clone()); executor.execute(exe, vec![]).unwrap(); } diff --git a/extensions/sha256/circuit/src/extension.rs b/extensions/sha256/circuit/src/extension.rs index 76a6c1ec0c..783bc54f63 100644 --- a/extensions/sha256/circuit/src/extension.rs +++ b/extensions/sha256/circuit/src/extension.rs @@ -1,6 +1,9 @@ use derive_more::derive::From; use openvm_circuit::{ - arch::{SystemConfig, VmExtension, VmInventory, VmInventoryBuilder, VmInventoryError}, + arch::{ + InitFileGenerator, SystemConfig, VmExtension, VmInventory, VmInventoryBuilder, + VmInventoryError, + }, system::phantom::PhantomChip, }; use openvm_circuit_derive::{AnyEnum, InstructionExecutor, VmConfig}; @@ -46,6 +49,9 @@ impl Default for Sha256Rv32Config { } } +// Default implementation uses no init file +impl InitFileGenerator for Sha256Rv32Config {} + #[derive(Clone, Copy, Debug, Default, Serialize, Deserialize)] pub struct Sha256; diff --git a/extensions/sha256/tests/src/lib.rs b/extensions/sha256/tests/src/lib.rs index 41ea37e8ae..7afaa08e56 100644 --- a/extensions/sha256/tests/src/lib.rs +++ b/extensions/sha256/tests/src/lib.rs @@ -16,7 +16,8 @@ mod tests { #[test] fn test_sha256() -> Result<()> { - let elf = build_example_program_at_path(get_programs_dir!(), "sha")?; + let config = Sha256Rv32Config::default(); + let elf = build_example_program_at_path(get_programs_dir!(), "sha", &config)?; let openvm_exe = VmExe::from_elf( elf, Transpiler::::default() @@ -25,7 +26,7 @@ mod tests { .with_extension(Rv32IoTranspilerExtension) .with_extension(Sha256TranspilerExtension), )?; - air_test(Sha256Rv32Config::default(), openvm_exe); + air_test(config, openvm_exe); Ok(()) } } From 26482eee7efe0b3d7c84878129607be16ef1ed98 Mon Sep 17 00:00:00 2001 From: Avaneesh-axiom Date: Mon, 28 Apr 2025 23:09:52 -0400 Subject: [PATCH 03/15] feat: Lazily call setup function for moduli and curves on first use (#1603) We no longer need to manually call `setup_*` at the start of main for the moduli and curves. Instead, setup is automatically done on first use of each modulus or curve. Depends on #1596 --------- Co-authored-by: Jonathan Wang <31040440+jonathanpwang@users.noreply.github.com> --- Cargo.lock | 1 + benchmarks/guest/ecrecover/src/main.rs | 3 -- benchmarks/guest/kitchen-sink/src/main.rs | 6 +-- benchmarks/guest/pairing/src/main.rs | 5 -- book/src/custom-extensions/algebra.md | 5 -- book/src/custom-extensions/ecc.md | 3 -- book/src/custom-extensions/pairing.md | 8 --- docs/specs/RISCV.md | 4 +- examples/algebra/src/main.rs | 4 -- examples/ecc/src/main.rs | 2 - examples/pairing/src/main.rs | 5 -- extensions/algebra/complex-macros/README.md | 23 ++++----- extensions/algebra/complex-macros/src/lib.rs | 46 +++++++++++------ extensions/algebra/guest/Cargo.toml | 1 + extensions/algebra/guest/src/lib.rs | 1 + extensions/algebra/moduli-macros/README.md | 41 +++++++-------- extensions/algebra/moduli-macros/src/lib.rs | 50 ++++++++++++------- .../examples/complex_redundant_modulus.rs | 2 - .../programs/examples/complex_secp256k1.rs | 2 - .../programs/examples/complex_two_moduli.rs | 2 - .../tests/programs/examples/invalid_setup.rs | 4 +- .../algebra/tests/programs/examples/little.rs | 1 - .../tests/programs/examples/moduli_setup.rs | 1 - extensions/ecc/sw-macros/README.md | 24 ++++----- extensions/ecc/sw-macros/src/lib.rs | 36 +++++++------ .../ecc/tests/programs/examples/decompress.rs | 5 -- .../examples/decompress_invalid_hint.rs | 5 -- extensions/ecc/tests/programs/examples/ec.rs | 3 -- .../tests/programs/examples/ec_nonzero_a.rs | 3 -- .../tests/programs/examples/ec_two_curves.rs | 3 -- .../ecc/tests/programs/examples/ecdsa.rs | 3 -- .../tests/programs/examples/invalid_setup.rs | 6 +-- .../pairing/tests/programs/examples/bls_ec.rs | 5 +- .../tests/programs/examples/fp12_mul.rs | 4 -- .../tests/programs/examples/pairing_check.rs | 4 -- .../examples/pairing_check_fallback.rs | 4 -- .../tests/programs/examples/pairing_line.rs | 4 -- .../programs/examples/pairing_miller_loop.rs | 4 -- .../programs/examples/pairing_miller_step.rs | 4 -- 39 files changed, 139 insertions(+), 198 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d687670a60..fe5d5d48c9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3969,6 +3969,7 @@ version = "1.1.2" dependencies = [ "halo2curves-axiom", "num-bigint 0.4.6", + "once_cell", "openvm-algebra-complex-macros", "openvm-algebra-moduli-macros", "serde-big-array", diff --git a/benchmarks/guest/ecrecover/src/main.rs b/benchmarks/guest/ecrecover/src/main.rs index fa29562fa6..db91ea1c22 100644 --- a/benchmarks/guest/ecrecover/src/main.rs +++ b/benchmarks/guest/ecrecover/src/main.rs @@ -24,9 +24,6 @@ openvm_ecc_guest::sw_macros::sw_init! { } pub fn main() { - setup_all_moduli(); - setup_all_curves(); - let expected_address = read_vec(); for _ in 0..5 { let input = read_vec(); diff --git a/benchmarks/guest/kitchen-sink/src/main.rs b/benchmarks/guest/kitchen-sink/src/main.rs index 6aa679eb3f..9bea64b283 100644 --- a/benchmarks/guest/kitchen-sink/src/main.rs +++ b/benchmarks/guest/kitchen-sink/src/main.rs @@ -48,10 +48,8 @@ openvm_algebra_guest::complex_macros::complex_init! { } pub fn main() { - // Setup will materialize every chip - setup_all_moduli(); - setup_all_complex_extensions(); - setup_all_curves(); + // TODO: Since we don't explicitly call setup functions anymore, we should rewrite this test + // to use every declared modulus and curve to ensure that every chip is materialized. let [one, six] = [1, 6].map(Seven::from_u32); assert_eq!(one + six, Seven::ZERO); diff --git a/benchmarks/guest/pairing/src/main.rs b/benchmarks/guest/pairing/src/main.rs index 807c5d2866..70e9c7683c 100644 --- a/benchmarks/guest/pairing/src/main.rs +++ b/benchmarks/guest/pairing/src/main.rs @@ -22,11 +22,6 @@ openvm_algebra_guest::complex_macros::complex_init! { const PAIR_ELEMENT_LEN: usize = 32 * (2 + 4); // 1 G1Affine (2 Fp), 1 G2Affine (4 Fp) pub fn main() { - setup_all_moduli(); - setup_all_complex_extensions(); - // Pairing doesn't need G1Affine intrinsics, but we trigger it anyways to test the chips - setup_all_curves(); - // copied from https://github.com/bluealloy/revm/blob/9e39df5dbc5fdc98779c644629b28b8bee75794a/crates/precompile/src/bn128.rs#L395 let input = hex::decode( "\ diff --git a/book/src/custom-extensions/algebra.md b/book/src/custom-extensions/algebra.md index 7ce48698b3..315cff2373 100644 --- a/book/src/custom-extensions/algebra.md +++ b/book/src/custom-extensions/algebra.md @@ -46,13 +46,10 @@ moduli_init! { This step enumerates the declared moduli (e.g., `0` for the first one, `1` for the second one) and sets up internal linkage so the compiler can generate the appropriate RISC-V instructions associated with each modulus. -3. **Setup**: At runtime, before performing arithmetic, a setup instruction must be sent to ensure security and correctness. For the \\(i\\)-th modulus, you call `setup_()` (e.g., `setup_0()` or `setup_1()`). Alternatively, `setup_all_moduli()` can be used to handle all declared moduli. - **Summary**: - `moduli_declare!`: Declares modular arithmetic structures and can be done multiple times. - `init!`: Called once in the final binary to assign and lock in the moduli. -- `setup_()`/`setup_all_moduli()`: Ensures at runtime that the correct modulus is in use, providing a security check and finalizing the environment for safe arithmetic operations. ## Complex field extension @@ -83,8 +80,6 @@ complex_init! { */ ``` -3. **Setup**: Similar to moduli, call `setup_complex_()` or `setup_all_complex_extensions()` at runtime to secure the environment. - ### Config parameters For the guest program to build successfully, all used moduli must be declared in the `.toml` config file in the following format: diff --git a/book/src/custom-extensions/ecc.md b/book/src/custom-extensions/ecc.md index 25f98ba183..4a1b76d6eb 100644 --- a/book/src/custom-extensions/ecc.md +++ b/book/src/custom-extensions/ecc.md @@ -54,13 +54,10 @@ sw_init! { */ ``` -3. **Setup**: Similar to the moduli and complex extensions, runtime setup instructions ensure that the correct curve parameters are being used, guaranteeing secure operation. - **Summary**: - `sw_declare!`: Declares elliptic curve structures. - `init!`: Initializes them once, linking them to the underlying moduli. -- `setup_sw_()`/`setup_all_curves()`: Secures runtime correctness. To use elliptic curve operations on a struct defined with `sw_declare!`, it is expected that the struct for the curve's coordinate field was defined using `moduli_declare!`. In particular, the coordinate field needs to be initialized and set up as described in the [algebra extension](./algebra.md) chapter. diff --git a/book/src/custom-extensions/pairing.md b/book/src/custom-extensions/pairing.md index 807de50e20..24d933dba1 100644 --- a/book/src/custom-extensions/pairing.md +++ b/book/src/custom-extensions/pairing.md @@ -36,14 +36,6 @@ Additionally, we'll need to initialize our moduli and `Fp2` struct via the follo {{ #include ../../../examples/pairing/src/main.rs:init }} ``` -And we'll run the required setup functions at the top of the guest program's `main()` function: - -```rust,no_run,noplayground -{{ #include ../../../examples/pairing/src/main.rs:setup }} -``` - -There are two moduli defined internally in the `Bls12_381` feature. The `moduli_init!` macro thus requires both of them to be initialized. However, we do not need the scalar field of BLS12-381 (which is at index 1), and thus we only initialize the modulus from index 0, thus we only use `setup_0()` (as opposed to `setup_all_moduli()`, which will save us some columns when generating the trace). - ## Input values The inputs to the pairing check are `AffinePoint`s in \\(\mathbb{F}\_p\\) and \\(\mathbb{F}\_{p^2}\\). They can be constructed via the `AffinePoint::new` function, with the inner `Fp` and `Fp2` values constructed via various `from_...` functions. diff --git a/docs/specs/RISCV.md b/docs/specs/RISCV.md index 7948272c66..427c7c16d5 100644 --- a/docs/specs/RISCV.md +++ b/docs/specs/RISCV.md @@ -132,7 +132,9 @@ generates classes `Bls12381` and `Bn254` that represent the elements of the corr ### Field Arithmetic -For each created modular class, one must call a corresponding `setup_*` function once at the beginning of the program. For example, for the structs above this would be `setup_0()` and `setup_1()`. This function generates the `setup` intrinsics which are distinguished by the `rs2` operand that specifies the chip this instruction is passed to.. +For each created modular class, one must call a corresponding `setup_*` function before using the intrinsics. +For example, for the structs above this would be `setup_0()` and `setup_1()`. This function generates the `setup` intrinsics which are distinguished by the `rs2` operand that specifies the chip this instruction is passed to. +For developer convenience, in the Rust function bindings for these intrinsics, each modulus's `setup_*` function is automatically called on the first use of any of its intrinsics. We use `config.mod_idx(N)` to denote the index of `N` in this list. In the list below, `idx` denotes `config.mod_idx(N)`. diff --git a/examples/algebra/src/main.rs b/examples/algebra/src/main.rs index 72c5d78ed7..de3d49697c 100644 --- a/examples/algebra/src/main.rs +++ b/examples/algebra/src/main.rs @@ -31,10 +31,6 @@ openvm_algebra_complex_macros::complex_init! { */ pub fn main() { - // Since we only use an arithmetic operation with `Mod1` and not `Mod2`, - // we only need to call `setup_0()` here. - setup_0(); - setup_all_complex_extensions(); let a = Complex1::new(Mod1::ZERO, Mod1::from_u32(0x3b8) * Mod1::from_u32(0x100000)); // a = -i in the corresponding field let b = Complex2::new(Mod2::ZERO, Mod2::from_u32(1000000006)); // b = -i in the corresponding field assert_eq!(a.clone() * &a * &a * &a * &a, a); // a^5 = a diff --git a/examples/ecc/src/main.rs b/examples/ecc/src/main.rs index c8e3299dc5..3468d6b26b 100644 --- a/examples/ecc/src/main.rs +++ b/examples/ecc/src/main.rs @@ -23,8 +23,6 @@ openvm_ecc_guest::sw_macros::sw_init! { // ANCHOR: main pub fn main() { - setup_all_moduli(); - setup_all_curves(); let x1 = Secp256k1Coord::from_u32(1); let y1 = Secp256k1Coord::from_le_bytes(&hex!( "EEA7767E580D75BC6FDD7F58D2A84C2614FB22586068DB63B346C6E60AF21842" diff --git a/examples/pairing/src/main.rs b/examples/pairing/src/main.rs index 60a601efe3..afb3828d0a 100644 --- a/examples/pairing/src/main.rs +++ b/examples/pairing/src/main.rs @@ -26,11 +26,6 @@ openvm_algebra_complex_macros::complex_init! { // ANCHOR: main pub fn main() { - // ANCHOR: setup - setup_0(); - setup_all_complex_extensions(); - // ANCHOR_END: setup - let p0 = AffinePoint::new( Fp::from_be_bytes(&hex!("17f1d3a73197d7942695638c4fa9ac0fc3688c4f9774b905a14e3a3f171bac586c55e83ff97a1aeffb3af00adb22c6bb")), Fp::from_be_bytes(&hex!("08b3f481e3aaa0f1a09e30ed741d8ae4fcf5e095d5d00af600db18cb2c04b3edd03cc744a2888ae40caa232946c5e7e1")) diff --git a/extensions/algebra/complex-macros/README.md b/extensions/algebra/complex-macros/README.md index 9c786a73ca..aba83ce9d6 100644 --- a/extensions/algebra/complex-macros/README.md +++ b/extensions/algebra/complex-macros/README.md @@ -27,8 +27,6 @@ openvm_algebra_complex_macros::complex_init! { */ pub fn main() { - setup_all_moduli(); - setup_all_complex_extensions(); // ... } ``` @@ -68,24 +66,23 @@ extern "C" { 2. Again, `complex_init!` macro implements these extern functions and defines the setup functions for the complex arithmetic struct. ```rust +#[allow(non_snake_case)] #[cfg(target_os = "zkvm")] mod openvm_intrinsics_ffi_complex { - fn complex_add_extern_func_Complex(rd: usize, rs1: usize, rs2: usize) { + #[no_mangle] + extern "C" fn complex_add_extern_func_Complex(rd: usize, rs1: usize, rs2: usize) { // send the instructions for the corresponding complex chip // If this struct was `init`ed k-th, these operations will be sent to the k-th complex chip } - // implement the other functions -} -pub fn setup_complex_0() { - // send the setup instructions -} -pub fn setup_all_complex_extensions() { - setup_complex_0(); - // call all other setup_complex_* for all the items in the moduli_init! macro + // .. implement the other functions + #[no_mangle] + extern "C" fn complex_setup_extern_func_Complex() { + // send the setup instructions + } } ``` -3. Obviously, `mod_idx` in the `complex_init!` must match the position of the corresponding modulus in the `moduli_init!` macro. The order of the items in `complex_init!` affects what `setup_complex_*` function will correspond to what complex class. Also, it **must match** the order of the moduli in the chip configuration -- more specifically, in the modular extension parameters (the order of numbers in `Fp2Extension::supported_modulus`, which is usually defined with the whole `app_vm_config` in the `openvm.toml` file). However, it again imposes the restriction that we only can invoke `complex_init!` once. Again analogous to the moduli setups, we must call `setup_complex_*` for each used complex extension before doing anything with entities of that class (or one can call `setup_all_complex_extensions` to setup all of them, if all are used). +3. Obviously, `mod_idx` in the `complex_init!` must match the position of the corresponding modulus in the `moduli_init!` macro. The order of the items in `complex_init!` affects what `setup_complex_*` function will correspond to what complex class. Also, it **must match** the order of the moduli in the chip configuration -- more specifically, in the modular extension parameters (the order of numbers in `Fp2Extension::supported_modulus`, which is usually defined with the whole `app_vm_config` in the `openvm.toml` file). However, it again imposes the restriction that we only can invoke `complex_init!` once. Again analogous to the moduli setups, the rust bindings will automatically call `complex_setup_extern_func_*` on each complex extension on first use of its intrinsics. 4. Note that, due to the nature of function names, the name of the struct used in `complex_init!` must be the same as in `complex_declare!`. To illustrate, the following code will **fail** to compile: @@ -107,4 +104,4 @@ The reason is that, for example, the function `complex_add_extern_func_Bn254Fp2` 5. `cargo openvm build` will automatically generate a call to `complex_init!` based on `openvm.toml`. Note that `openvm.toml` must list the supported moduli as pairs `(name, modulus)` where `name` is the name of the struct created by `complex_declare!` as a string (in the example at the top of this document, its `"Complex"`). -The SDK also supports this feature. \ No newline at end of file +The SDK also supports this feature. diff --git a/extensions/algebra/complex-macros/src/lib.rs b/extensions/algebra/complex-macros/src/lib.rs index d829abeeed..501587a7bb 100644 --- a/extensions/algebra/complex-macros/src/lib.rs +++ b/extensions/algebra/complex-macros/src/lib.rs @@ -58,6 +58,7 @@ pub fn complex_declare(input: TokenStream) -> TokenStream { create_extern_func!(complex_sub_extern_func); create_extern_func!(complex_mul_extern_func); create_extern_func!(complex_div_extern_func); + create_extern_func!(complex_setup_extern_func); let result = TokenStream::from(quote::quote_spanned! { span.into() => extern "C" { @@ -65,6 +66,7 @@ pub fn complex_declare(input: TokenStream) -> TokenStream { fn #complex_sub_extern_func(rd: usize, rs1: usize, rs2: usize); fn #complex_mul_extern_func(rd: usize, rs1: usize, rs2: usize); fn #complex_div_extern_func(rd: usize, rs1: usize, rs2: usize); + fn #complex_setup_extern_func(); } @@ -110,6 +112,7 @@ pub fn complex_declare(input: TokenStream) -> TokenStream { } #[cfg(target_os = "zkvm")] { + Self::assert_is_setup(); unsafe { #complex_add_extern_func( self as *mut Self as usize, @@ -130,6 +133,7 @@ pub fn complex_declare(input: TokenStream) -> TokenStream { } #[cfg(target_os = "zkvm")] { + Self::assert_is_setup(); unsafe { #complex_sub_extern_func( self as *mut Self as usize, @@ -154,6 +158,7 @@ pub fn complex_declare(input: TokenStream) -> TokenStream { } #[cfg(target_os = "zkvm")] { + Self::assert_is_setup(); unsafe { #complex_mul_extern_func( self as *mut Self as usize, @@ -179,6 +184,7 @@ pub fn complex_declare(input: TokenStream) -> TokenStream { } #[cfg(target_os = "zkvm")] { + Self::assert_is_setup(); unsafe { #complex_div_extern_func( self as *mut Self as usize, @@ -199,6 +205,7 @@ pub fn complex_declare(input: TokenStream) -> TokenStream { } #[cfg(target_os = "zkvm")] { + Self::assert_is_setup(); let mut uninit: core::mem::MaybeUninit = core::mem::MaybeUninit::uninit(); unsafe { #complex_add_extern_func( @@ -222,6 +229,7 @@ pub fn complex_declare(input: TokenStream) -> TokenStream { } #[cfg(target_os = "zkvm")] { + Self::assert_is_setup(); let mut uninit: core::mem::MaybeUninit = core::mem::MaybeUninit::uninit(); unsafe { #complex_sub_extern_func( @@ -249,6 +257,7 @@ pub fn complex_declare(input: TokenStream) -> TokenStream { } #[cfg(target_os = "zkvm")] { + Self::assert_is_setup(); unsafe { #complex_mul_extern_func( dst_ptr as usize, @@ -270,6 +279,7 @@ pub fn complex_declare(input: TokenStream) -> TokenStream { } #[cfg(target_os = "zkvm")] { + Self::assert_is_setup(); let mut uninit: core::mem::MaybeUninit = core::mem::MaybeUninit::uninit(); unsafe { #complex_div_extern_func( @@ -281,6 +291,15 @@ pub fn complex_declare(input: TokenStream) -> TokenStream { unsafe { uninit.assume_init() } } } + + // Helper function to call the setup instruction on first use + fn assert_is_setup() { + static is_setup: ::openvm_algebra_guest::once_cell::race::OnceBool = ::openvm_algebra_guest::once_cell::race::OnceBool::new(); + is_setup.get_or_init(|| { + unsafe { #complex_setup_extern_func(); } + true + }); + } } impl openvm_algebra_guest::field::ComplexConjugate for #struct_name { @@ -527,8 +546,6 @@ pub fn complex_init(input: TokenStream) -> TokenStream { let MacroArgs { items } = parse_macro_input!(input as MacroArgs); let mut externs = Vec::new(); - let mut setups = Vec::new(); - let mut setup_all_complex_extensions = Vec::new(); let span = proc_macro::Span::call_site(); @@ -587,22 +604,22 @@ pub fn complex_init(input: TokenStream) -> TokenStream { }); } - let setup_function = - syn::Ident::new(&format!("setup_complex_{}", complex_idx), span.into()); + let setup_extern_func = syn::Ident::new( + &format!("complex_setup_extern_func_{}", struct_name), + span.into(), + ); - setup_all_complex_extensions.push(quote::quote_spanned! { span.into() => - #setup_function(); - }); - setups.push(quote::quote_spanned! { span.into() => - #[allow(non_snake_case)] - pub fn #setup_function() { + externs.push(quote::quote_spanned! { span.into() => + #[no_mangle] + extern "C" fn #setup_extern_func() { #[cfg(target_os = "zkvm")] { - let two_modulus_bytes = &openvm_intrinsics_meta_do_not_type_this_by_yourself::two_modular_limbs_list[openvm_intrinsics_meta_do_not_type_this_by_yourself::limb_list_borders[#mod_idx]..openvm_intrinsics_meta_do_not_type_this_by_yourself::limb_list_borders[#mod_idx + 1]]; + use super::openvm_intrinsics_meta_do_not_type_this_by_yourself::{two_modular_limbs_list, limb_list_borders}; + let two_modulus_bytes = &two_modular_limbs_list[limb_list_borders[#mod_idx]..limb_list_borders[#mod_idx + 1]]; // We are going to use the numeric representation of the `rs2` register to distinguish the chip to setup. // The transpiler will transform this instruction, based on whether `rs2` is `x0` or `x1`, into a `SETUP_ADDSUB` or `SETUP_MULDIV` instruction. - let mut uninit: core::mem::MaybeUninit<[u8; openvm_intrinsics_meta_do_not_type_this_by_yourself::limb_list_borders[#mod_idx + 1] - openvm_intrinsics_meta_do_not_type_this_by_yourself::limb_list_borders[#mod_idx]]> = core::mem::MaybeUninit::uninit(); + let mut uninit: core::mem::MaybeUninit<[u8; limb_list_borders[#mod_idx + 1] - limb_list_borders[#mod_idx]]> = core::mem::MaybeUninit::uninit(); openvm::platform::custom_insn_r!( opcode = ::openvm_algebra_guest::OPCODE, funct3 = ::openvm_algebra_guest::COMPLEX_EXT_FIELD_FUNCT3, @@ -629,14 +646,11 @@ pub fn complex_init(input: TokenStream) -> TokenStream { } TokenStream::from(quote::quote_spanned! { span.into() => + #[allow(non_snake_case)] #[cfg(target_os = "zkvm")] mod openvm_intrinsics_ffi_complex { #(#externs)* } - #(#setups)* - pub fn setup_all_complex_extensions() { - #(#setup_all_complex_extensions)* - } }) } diff --git a/extensions/algebra/guest/Cargo.toml b/extensions/algebra/guest/Cargo.toml index d57c9117e3..64f4e61d51 100644 --- a/extensions/algebra/guest/Cargo.toml +++ b/extensions/algebra/guest/Cargo.toml @@ -12,6 +12,7 @@ openvm-algebra-moduli-macros = { workspace = true } openvm-algebra-complex-macros = { workspace = true } serde-big-array.workspace = true strum_macros.workspace = true +once_cell = { workspace = true, features = ["race", "alloc"] } [target.'cfg(not(target_os = "zkvm"))'.dependencies] num-bigint.workspace = true diff --git a/extensions/algebra/guest/src/lib.rs b/extensions/algebra/guest/src/lib.rs index 1d778f0013..c93bb8920b 100644 --- a/extensions/algebra/guest/src/lib.rs +++ b/extensions/algebra/guest/src/lib.rs @@ -68,6 +68,7 @@ mod halo2curves; /// Exponentiation by bytes mod exp_bytes; pub use exp_bytes::*; +pub use once_cell; /// Division operation that is undefined behavior when the denominator is not invertible. pub trait DivUnsafe: Sized { diff --git a/extensions/algebra/moduli-macros/README.md b/extensions/algebra/moduli-macros/README.md index 61513d0603..a7a3e27eba 100644 --- a/extensions/algebra/moduli-macros/README.md +++ b/extensions/algebra/moduli-macros/README.md @@ -51,6 +51,7 @@ extern "C" { fn mul_extern_func_de0b6b3a7640003(rd: usize, rs1: usize, rs2: usize); fn div_extern_func_de0b6b3a7640003(rd: usize, rs1: usize, rs2: usize); fn is_eq_extern_func_de0b6b3a7640003(rs1: usize, rs2: usize) -> bool; + fn moduli_setup_extern_func_de0b6b3a7640003(); } impl Mod1e18 { @@ -128,36 +129,36 @@ Here `add_extern_func_de0b6b3a7640003` is the name of the function that will be static OPENVM_SERIALIZED_MODULUS_2: [u8; 32] = [/* bytes of the modulus */]; #[cfg(target_os = "zkvm")] mod openvm_intrinsics_ffi { - fn add_extern_func_de0b6b3a7640003(rd: usize, rs1: usize, rs2: usize); - fn sub_extern_func_de0b6b3a7640003(rd: usize, rs1: usize, rs2: usize); - fn mul_extern_func_de0b6b3a7640003(rd: usize, rs1: usize, rs2: usize); - fn div_extern_func_de0b6b3a7640003(rd: usize, rs1: usize, rs2: usize); - fn is_eq_extern_func_de0b6b3a7640003(rs1: usize, rs2: usize) -> bool; + fn add_extern_func_de0b6b3a7640003(rd: usize, rs1: usize, rs2: usize) { + // Implementation here + } + fn sub_extern_func_de0b6b3a7640003(rd: usize, rs1: usize, rs2: usize) { + // Implementation here + } + fn mul_extern_func_de0b6b3a7640003(rd: usize, rs1: usize, rs2: usize) { + // Implementation here + } + fn div_extern_func_de0b6b3a7640003(rd: usize, rs1: usize, rs2: usize) { + // Implementation here + } + fn is_eq_extern_func_de0b6b3a7640003(rs1: usize, rs2: usize) -> bool { + // Implementation here + } + fn moduli_setup_extern_func_de0b6b3a7640003() { + // Implementation here + } } #[allow(non_snake_case, non_upper_case_globals)] pub mod openvm_intrinsics_meta_do_not_type_this_by_yourself { // information about the bytes of all moduli } -#[allow(non_snake_case)] -pub fn setup_2() { - #[cfg(target_os = "zkvm")] - { - // send the setup instruction designed for the chip number 2 - } -} -pub fn setup_all_moduli() { - setup_0(); - setup_1(); - setup_2(); - // setup functions for all the other moduli provided in the `moduli_init!` function -} ``` -The setup operation (e.g., `setup_2`) consists of reading the value `OPENVM_SERIALIZED_MODULUS_2` from memory and constraining that the read value is equal to the modulus the chip has been configured with. For each used modulus, its corresponding setup instruction **must** be called before all other operations -- this currently must be checked by inspecting the program code; it is not enforced by the virtual machine. +The setup operation (e.g., `moduli_setup_extern_func_de0b6b3a7640003`) consists of reading the value `OPENVM_SERIALIZED_MODULUS_2` from memory and constraining that the read value is equal to the modulus the chip has been configured with. For each used modulus, the Rust bindings for the non-setup intrinsic instructions will automatically call the corresponding setup instruction on first use of any of its intrinsics. 5. It follows from the above that the `moduli_declare!` invocations may be in multiple places in various compilation units, but all the `declare!`d moduli must be specified at least once in `moduli_init!` so that there will be no linker errors due to missing function implementations. Correspondingly, the `moduli_init!` macro should only be called once in the entire program (in the guest crate as the topmost compilation unit). Finally, the order of the moduli in `moduli_init!` has nothing to do with the `moduli_declare!` invocations, but it **must match** the order of the moduli in the chip configuration -- more specifically, in the modular extension parameters (the order of numbers in `ModularExtension::supported_modulus`, which is usually defined with the whole `app_vm_config` in the `openvm.toml` file). -6. For convenience, running `cargo openvm build` will automatically generate an appropriate call to `moduli_declare!` based on the order of the moduli in `openvm.toml`. +6. For convenience, running `cargo openvm build` will automatically generate an appropriate call to `moduli_declare!` based on the order of the moduli in `openvm.toml`. More specifically, `cargo openvm build` will read `openvm.toml`, then generate a file named `openvm_init.rs` in the project's manifest directory (where `Cargo.toml` is located) containing a call to `moduli_declare!`. Then, you must call `openvm::init!();` in your code, and this will insert the contents of `openvm_init.rs` file into your code in place of the `init!` macro. You may specify an alternate name for the init file using the `--init-file-name` option of `cargo openvm build`. diff --git a/extensions/algebra/moduli-macros/src/lib.rs b/extensions/algebra/moduli-macros/src/lib.rs index 5d8d921f2f..e0dd416fad 100644 --- a/extensions/algebra/moduli-macros/src/lib.rs +++ b/extensions/algebra/moduli-macros/src/lib.rs @@ -98,6 +98,7 @@ pub fn moduli_declare(input: TokenStream) -> TokenStream { create_extern_func!(mul_extern_func); create_extern_func!(div_extern_func); create_extern_func!(is_eq_extern_func); + create_extern_func!(moduli_setup_extern_func); let block_size = proc_macro::Literal::usize_unsuffixed(block_size); let block_size = syn::Lit::new(block_size.to_string().parse::<_>().unwrap()); @@ -126,6 +127,7 @@ pub fn moduli_declare(input: TokenStream) -> TokenStream { fn #mul_extern_func(rd: usize, rs1: usize, rs2: usize); fn #div_extern_func(rd: usize, rs1: usize, rs2: usize); fn #is_eq_extern_func(rs1: usize, rs2: usize) -> bool; + fn #moduli_setup_extern_func(); } impl #struct_name { @@ -152,6 +154,7 @@ pub fn moduli_declare(input: TokenStream) -> TokenStream { } #[cfg(target_os = "zkvm")] { + Self::assert_is_setup(); unsafe { #add_extern_func( self as *mut Self as usize, @@ -173,6 +176,7 @@ pub fn moduli_declare(input: TokenStream) -> TokenStream { } #[cfg(target_os = "zkvm")] { + Self::assert_is_setup(); unsafe { #sub_extern_func( self as *mut Self as usize, @@ -193,6 +197,7 @@ pub fn moduli_declare(input: TokenStream) -> TokenStream { } #[cfg(target_os = "zkvm")] { + Self::assert_is_setup(); unsafe { #mul_extern_func( self as *mut Self as usize, @@ -213,6 +218,7 @@ pub fn moduli_declare(input: TokenStream) -> TokenStream { } #[cfg(target_os = "zkvm")] { + Self::assert_is_setup(); unsafe { #div_extern_func( self as *mut Self as usize, @@ -237,6 +243,7 @@ pub fn moduli_declare(input: TokenStream) -> TokenStream { } #[cfg(target_os = "zkvm")] { + Self::assert_is_setup(); unsafe { #add_extern_func( dst_ptr as usize, @@ -261,6 +268,7 @@ pub fn moduli_declare(input: TokenStream) -> TokenStream { } #[cfg(target_os = "zkvm")] { + Self::assert_is_setup(); unsafe { #sub_extern_func( dst_ptr as usize, @@ -285,6 +293,7 @@ pub fn moduli_declare(input: TokenStream) -> TokenStream { } #[cfg(target_os = "zkvm")] { + Self::assert_is_setup(); unsafe { #mul_extern_func( dst_ptr as usize, @@ -305,6 +314,7 @@ pub fn moduli_declare(input: TokenStream) -> TokenStream { } #[cfg(target_os = "zkvm")] { + Self::assert_is_setup(); let mut uninit: core::mem::MaybeUninit<#struct_name> = core::mem::MaybeUninit::uninit(); unsafe { #div_extern_func( @@ -325,11 +335,21 @@ pub fn moduli_declare(input: TokenStream) -> TokenStream { } #[cfg(target_os = "zkvm")] { + Self::assert_is_setup(); unsafe { #is_eq_extern_func(self as *const #struct_name as usize, other as *const #struct_name as usize) } } } + + // Helper function to call the setup instruction on first use + fn assert_is_setup() { + static is_setup: ::openvm_algebra_guest::once_cell::race::OnceBool = ::openvm_algebra_guest::once_cell::race::OnceBool::new(); + is_setup.get_or_init(|| { + unsafe { #moduli_setup_extern_func(); } + true + }); + } } // Put trait implementations in a private module to avoid conflicts @@ -733,9 +753,7 @@ pub fn moduli_init(input: TokenStream) -> TokenStream { let ModuliDefine { items } = parse_macro_input!(input as ModuliDefine); let mut externs = Vec::new(); - let mut setups = Vec::new(); let mut openvm_section = Vec::new(); - let mut setup_all_moduli = Vec::new(); // List of all modular limbs in one (that is, with a compile-time known size) array. let mut two_modular_limbs_flattened_list = Vec::::new(); @@ -794,7 +812,10 @@ pub fn moduli_init(input: TokenStream) -> TokenStream { span.into(), ); let serialized_len = serialized_modulus.len(); - let setup_function = syn::Ident::new(&format!("setup_{}", mod_idx), span.into()); + let setup_extern_func = syn::Ident::new( + &format!("moduli_setup_extern_func_{}", modulus_hex), + span.into(), + ); openvm_section.push(quote::quote_spanned! { span.into() => #[cfg(target_os = "zkvm")] @@ -848,23 +869,19 @@ pub fn moduli_init(input: TokenStream) -> TokenStream { } }); - setup_all_moduli.push(quote::quote_spanned! { span.into() => - #setup_function(); - }); - - setups.push(quote::quote_spanned! { span.into() => - #[allow(non_snake_case)] - pub fn #setup_function() { + externs.push(quote::quote_spanned! { span.into() => + #[no_mangle] + extern "C" fn #setup_extern_func() { #[cfg(target_os = "zkvm")] { let mut ptr = 0; - assert_eq!(#serialized_name[ptr], 1); + assert_eq!(super::#serialized_name[ptr], 1); ptr += 1; - assert_eq!(#serialized_name[ptr], #mod_idx as u8); + assert_eq!(super::#serialized_name[ptr], #mod_idx as u8); ptr += 1; - assert_eq!(#serialized_name[ptr..ptr+4].iter().rev().fold(0, |acc, &x| acc * 256 + x as usize), #limbs); + assert_eq!(super::#serialized_name[ptr..ptr+4].iter().rev().fold(0, |acc, &x| acc * 256 + x as usize), #limbs); ptr += 4; - let remaining = &#serialized_name[ptr..]; + let remaining = &super::#serialized_name[ptr..]; // To avoid importing #struct_name, we create a placeholder struct with the same size and alignment. #[repr(C, align(#block_size))] @@ -917,6 +934,7 @@ pub fn moduli_init(input: TokenStream) -> TokenStream { let cnt_limbs_list_len = limb_list_borders.len(); TokenStream::from(quote::quote_spanned! { span.into() => #(#openvm_section)* + #[allow(non_snake_case)] #[cfg(target_os = "zkvm")] mod openvm_intrinsics_ffi { #(#externs)* @@ -926,9 +944,5 @@ pub fn moduli_init(input: TokenStream) -> TokenStream { pub const two_modular_limbs_list: [u8; #total_limbs_cnt] = [#(#two_modular_limbs_flattened_list),*]; pub const limb_list_borders: [usize; #cnt_limbs_list_len] = [#(#limb_list_borders),*]; } - #(#setups)* - pub fn setup_all_moduli() { - #(#setup_all_moduli)* - } }) } diff --git a/extensions/algebra/tests/programs/examples/complex_redundant_modulus.rs b/extensions/algebra/tests/programs/examples/complex_redundant_modulus.rs index 678b8b1838..950f83c1ef 100644 --- a/extensions/algebra/tests/programs/examples/complex_redundant_modulus.rs +++ b/extensions/algebra/tests/programs/examples/complex_redundant_modulus.rs @@ -19,8 +19,6 @@ openvm_algebra_complex_macros::complex_declare! { openvm::init!("openvm_init_complex_redundant_modulus.rs"); pub fn main() { - setup_all_moduli(); - setup_all_complex_extensions(); let b = Complex2::new(Mod3::ZERO, Mod3::from_u32(1000000008)); assert_eq!(b.clone() * &b * &b * &b * &b, b); } diff --git a/extensions/algebra/tests/programs/examples/complex_secp256k1.rs b/extensions/algebra/tests/programs/examples/complex_secp256k1.rs index ab02878cfd..9f47e79be1 100644 --- a/extensions/algebra/tests/programs/examples/complex_secp256k1.rs +++ b/extensions/algebra/tests/programs/examples/complex_secp256k1.rs @@ -16,8 +16,6 @@ openvm_algebra_complex_macros::complex_declare! { openvm::init!("openvm_init_complex_secp256k1.rs"); pub fn main() { - setup_all_moduli(); - setup_all_complex_extensions(); let mut a = Complex::new( Secp256k1Coord::from_repr(core::array::from_fn(|_| 10)), Secp256k1Coord::from_repr(core::array::from_fn(|_| 21)), diff --git a/extensions/algebra/tests/programs/examples/complex_two_moduli.rs b/extensions/algebra/tests/programs/examples/complex_two_moduli.rs index 3e67028722..01c1ba6a63 100644 --- a/extensions/algebra/tests/programs/examples/complex_two_moduli.rs +++ b/extensions/algebra/tests/programs/examples/complex_two_moduli.rs @@ -18,8 +18,6 @@ openvm_algebra_complex_macros::complex_declare! { openvm::init!("openvm_init_complex_two_moduli.rs"); pub fn main() { - setup_all_moduli(); - setup_all_complex_extensions(); let a = Complex1::new(Mod1::ZERO, Mod1::from_u32(998244352)); let b = Complex2::new(Mod2::ZERO, Mod2::from_u32(1000000006)); assert_eq!(a.clone() * &a * &a * &a * &a, a); diff --git a/extensions/algebra/tests/programs/examples/invalid_setup.rs b/extensions/algebra/tests/programs/examples/invalid_setup.rs index 3321dd3238..37d6946b1f 100644 --- a/extensions/algebra/tests/programs/examples/invalid_setup.rs +++ b/extensions/algebra/tests/programs/examples/invalid_setup.rs @@ -18,5 +18,7 @@ openvm_algebra_moduli_macros::moduli_init! { pub fn main() { // this should cause a debug assertion to fail - setup_all_moduli(); + let x = Mod1::from_u32(1); + let y = Mod1::from_u32(1); + let _z = x + y; } diff --git a/extensions/algebra/tests/programs/examples/little.rs b/extensions/algebra/tests/programs/examples/little.rs index 15203d98b7..caeb8de732 100644 --- a/extensions/algebra/tests/programs/examples/little.rs +++ b/extensions/algebra/tests/programs/examples/little.rs @@ -12,7 +12,6 @@ openvm_algebra_moduli_macros::moduli_declare! { openvm::init!("openvm_init_little.rs"); pub fn main() { - setup_all_moduli(); let mut pow = Secp256k1Coord::MODULUS; pow[0] -= 2; diff --git a/extensions/algebra/tests/programs/examples/moduli_setup.rs b/extensions/algebra/tests/programs/examples/moduli_setup.rs index cda883d133..b9abec0a9f 100644 --- a/extensions/algebra/tests/programs/examples/moduli_setup.rs +++ b/extensions/algebra/tests/programs/examples/moduli_setup.rs @@ -18,7 +18,6 @@ openvm_algebra_moduli_macros::moduli_declare! { openvm::init!("openvm_init_moduli_setup.rs"); pub fn main() { - setup_all_moduli(); let x = Bls12381::from_repr(core::array::from_fn(|i| i as u8)); assert_eq!(x.0.len(), 48); diff --git a/extensions/ecc/sw-macros/README.md b/extensions/ecc/sw-macros/README.md index 8b2da66e70..3fc0b64cf1 100644 --- a/extensions/ecc/sw-macros/README.md +++ b/extensions/ecc/sw-macros/README.md @@ -33,8 +33,6 @@ openvm_ecc_guest::sw_macros::sw_init! { */ pub fn main() { - setup_all_moduli(); - setup_all_curves(); // ... } ``` @@ -73,6 +71,7 @@ extern "C" { 2. Again, `sw_init!` macro implements these extern functions and defines the setup functions for the sw struct. ```rust +#[allow(non_snake_case)] #[cfg(target_os = "zkvm")] mod openvm_intrinsics_ffi_2 { use :openvm_ecc_guest::{OPCODE, SW_FUNCT3, SwBaseFunct7}; @@ -82,21 +81,18 @@ mod openvm_intrinsics_ffi_2 { // ... } // other externs -} -#[allow(non_snake_case)] -pub fn setup_sw_Secp256k1Point() { - #[cfg(target_os = "zkvm")] - { - // ... + + #[no_mangle] + extern "C" fn sw_setup_extern_func_Secp256k1Point() { + #[cfg(target_os = "zkvm")] + { + // ... + } } } -pub fn setup_all_curves() { - setup_sw_Secp256k1Point(); - // other setups -} ``` -3. Again, the `setup` function for every used curve must be called before any other instructions for that curve. If all curves are used, one can call `setup_all_curves()` to setup all of them. +3. Again, if using the Rust bindings, then the `sw_setup_extern_func_*` function for every curve is automatically called on first use of any of the curve's intrinsics. 4. The order of the items in `sw_init!` **must match** the order of the moduli in the chip configuration -- more specifically, in the modular extension parameters (the order of `CurveConfig`s in `WeierstrassExtension::supported_curves`, which is usually defined with the whole `app_vm_config` in the `openvm.toml` file). @@ -120,4 +116,4 @@ The reason is that, for example, the function `sw_add_extern_func_Secp256k1Point 6. `cargo openvm build` will automatically generate a call to `sw_init!` based on `openvm.toml`. Note that `openvm.toml` must contain the name of each struct created by `sw_declare!` as a string (in the example at the top of this document, its `"Secp256k1"`). -The SDK also supports this feature. \ No newline at end of file +The SDK also supports this feature. diff --git a/extensions/ecc/sw-macros/src/lib.rs b/extensions/ecc/sw-macros/src/lib.rs index c739b1965c..77d328870b 100644 --- a/extensions/ecc/sw-macros/src/lib.rs +++ b/extensions/ecc/sw-macros/src/lib.rs @@ -89,6 +89,7 @@ pub fn sw_declare(input: TokenStream) -> TokenStream { create_extern_func!(sw_double_extern_func); create_extern_func!(hint_decompress_extern_func); create_extern_func!(hint_non_qr_extern_func); + create_extern_func!(sw_setup_extern_func); let group_ops_mod_name = format_ident!("{}_ops", struct_name.to_string().to_lowercase()); @@ -98,6 +99,7 @@ pub fn sw_declare(input: TokenStream) -> TokenStream { fn #sw_double_extern_func(rd: usize, rs1: usize); fn #hint_decompress_extern_func(rs1: usize, rs2: usize); fn #hint_non_qr_extern_func(); + fn #sw_setup_extern_func(); } #[derive(Eq, PartialEq, Clone, Debug, serde::Serialize, serde::Deserialize)] @@ -129,6 +131,7 @@ pub fn sw_declare(input: TokenStream) -> TokenStream { } #[cfg(target_os = "zkvm")] { + Self::assert_is_setup(); let mut uninit: core::mem::MaybeUninit<#struct_name> = core::mem::MaybeUninit::uninit(); unsafe { #sw_add_ne_extern_func( @@ -154,6 +157,7 @@ pub fn sw_declare(input: TokenStream) -> TokenStream { } #[cfg(target_os = "zkvm")] { + Self::assert_is_setup(); unsafe { #sw_add_ne_extern_func( self as *mut #struct_name as usize, @@ -179,6 +183,7 @@ pub fn sw_declare(input: TokenStream) -> TokenStream { } #[cfg(target_os = "zkvm")] { + Self::assert_is_setup(); let mut uninit: core::mem::MaybeUninit<#struct_name> = core::mem::MaybeUninit::uninit(); unsafe { #sw_double_extern_func( @@ -198,6 +203,7 @@ pub fn sw_declare(input: TokenStream) -> TokenStream { } #[cfg(target_os = "zkvm")] { + Self::assert_is_setup(); unsafe { #sw_double_extern_func( self as *mut #struct_name as usize, @@ -207,6 +213,14 @@ pub fn sw_declare(input: TokenStream) -> TokenStream { } } + // Helper function to call the setup instruction on first use + fn assert_is_setup() { + static is_setup: ::openvm_ecc_guest::once_cell::race::OnceBool = ::openvm_ecc_guest::once_cell::race::OnceBool::new(); + is_setup.get_or_init(|| { + unsafe { #sw_setup_extern_func(); } + true + }); + } } impl ::openvm_ecc_guest::weierstrass::WeierstrassPoint for #struct_name { @@ -446,8 +460,6 @@ pub fn sw_init(input: TokenStream) -> TokenStream { let SwDefine { items } = parse_macro_input!(input as SwDefine); let mut externs = Vec::new(); - let mut setups = Vec::new(); - let mut setup_all_curves = Vec::new(); let span = proc_macro::Span::call_site(); @@ -470,6 +482,9 @@ pub fn sw_init(input: TokenStream) -> TokenStream { &format!("hint_non_qr_extern_func_{}", str_path), span.into(), ); + let setup_extern_func = + syn::Ident::new(&format!("sw_setup_extern_func_{}", str_path), span.into()); + externs.push(quote::quote_spanned! { span.into() => #[no_mangle] extern "C" fn #add_ne_extern_func(rd: usize, rs1: usize, rs2: usize) { @@ -522,14 +537,12 @@ pub fn sw_init(input: TokenStream) -> TokenStream { rs2 = Const "x0" ); } - }); - let setup_function = syn::Ident::new(&format!("setup_sw_{}", str_path), span.into()); - setups.push(quote::quote_spanned! { span.into() => - #[allow(non_snake_case)] - pub fn #setup_function() { + #[no_mangle] + extern "C" fn #setup_extern_func() { #[cfg(target_os = "zkvm")] { + use super::#item; // p1 is (x1, y1), and x1 must be the modulus. // y1 can be anything for SetupEcAdd, but must equal `a` for SetupEcDouble let modulus_bytes = <<#item as openvm_ecc_guest::weierstrass::WeierstrassPoint>::Coordinate as openvm_algebra_guest::IntMod>::MODULUS; @@ -564,22 +577,15 @@ pub fn sw_init(input: TokenStream) -> TokenStream { } } }); - - setup_all_curves.push(quote::quote_spanned! { span.into() => - #setup_function(); - }); } TokenStream::from(quote::quote_spanned! { span.into() => + #[allow(non_snake_case)] #[cfg(target_os = "zkvm")] mod openvm_intrinsics_ffi_2 { use ::openvm_ecc_guest::{OPCODE, SW_FUNCT3, SwBaseFunct7}; #(#externs)* } - #(#setups)* - pub fn setup_all_curves() { - #(#setup_all_curves)* - } }) } diff --git a/extensions/ecc/tests/programs/examples/decompress.rs b/extensions/ecc/tests/programs/examples/decompress.rs index 8692b13675..206562ef90 100644 --- a/extensions/ecc/tests/programs/examples/decompress.rs +++ b/extensions/ecc/tests/programs/examples/decompress.rs @@ -77,11 +77,6 @@ openvm::init!("openvm_init_decompress.rs"); // test decompression under an honest host pub fn main() { - setup_0(); - setup_2(); - setup_4(); - setup_all_curves(); - let bytes = read_vec(); let x = Secp256k1Coord::from_le_bytes(&bytes[..32]); let y = Secp256k1Coord::from_le_bytes(&bytes[32..64]); diff --git a/extensions/ecc/tests/programs/examples/decompress_invalid_hint.rs b/extensions/ecc/tests/programs/examples/decompress_invalid_hint.rs index c55a3e329d..c341ef6311 100644 --- a/extensions/ecc/tests/programs/examples/decompress_invalid_hint.rs +++ b/extensions/ecc/tests/programs/examples/decompress_invalid_hint.rs @@ -192,11 +192,6 @@ type CurvePoint1mod4Wrapper = CurvePointWrapper; // Check that decompress enters an infinite loop when hint_decompress returns an incorrect value. pub fn main() { - setup_0(); - setup_2(); - setup_4(); - setup_all_curves(); - let bytes = read_vec(); test_p_3_mod_4(&bytes[..32], &bytes[32..64]); diff --git a/extensions/ecc/tests/programs/examples/ec.rs b/extensions/ecc/tests/programs/examples/ec.rs index 056e12f41d..b4a03bea6c 100644 --- a/extensions/ecc/tests/programs/examples/ec.rs +++ b/extensions/ecc/tests/programs/examples/ec.rs @@ -15,9 +15,6 @@ openvm::init!("openvm_init_ec.rs"); openvm::entry!(main); pub fn main() { - setup_all_moduli(); - setup_all_curves(); - // Sample points got from https://asecuritysite.com/ecc/ecc_points2 and // https://learnmeabitcoin.com/technical/cryptography/elliptic-curve/#add let x1 = Secp256k1Coord::from_u32(1); diff --git a/extensions/ecc/tests/programs/examples/ec_nonzero_a.rs b/extensions/ecc/tests/programs/examples/ec_nonzero_a.rs index a25277b4bb..2402151bdf 100644 --- a/extensions/ecc/tests/programs/examples/ec_nonzero_a.rs +++ b/extensions/ecc/tests/programs/examples/ec_nonzero_a.rs @@ -14,9 +14,6 @@ openvm::entry!(main); openvm::init!("openvm_init_ec_nonzero_a.rs"); pub fn main() { - setup_all_moduli(); - setup_all_curves(); - // Sample points got from https://asecuritysite.com/ecc/p256p let x1 = P256Coord::from_u32(5); let y1 = P256Coord::from_le_bytes(&hex!( diff --git a/extensions/ecc/tests/programs/examples/ec_two_curves.rs b/extensions/ecc/tests/programs/examples/ec_two_curves.rs index 499d678593..21cdb2eb51 100644 --- a/extensions/ecc/tests/programs/examples/ec_two_curves.rs +++ b/extensions/ecc/tests/programs/examples/ec_two_curves.rs @@ -16,9 +16,6 @@ openvm::init!("openvm_init_ec_two_curves.rs"); openvm::entry!(main); pub fn main() { - setup_all_moduli(); - setup_all_curves(); - // Sample points got from https://asecuritysite.com/ecc/ecc_points2 and // https://learnmeabitcoin.com/technical/cryptography/elliptic-curve/#add let x1 = Secp256k1Coord::from_u32(1); diff --git a/extensions/ecc/tests/programs/examples/ecdsa.rs b/extensions/ecc/tests/programs/examples/ecdsa.rs index 80b0734941..da556cd096 100644 --- a/extensions/ecc/tests/programs/examples/ecdsa.rs +++ b/extensions/ecc/tests/programs/examples/ecdsa.rs @@ -19,9 +19,6 @@ openvm::init!("openvm_init_ecdsa.rs"); // Ref: https://docs.rs/k256/latest/k256/ecdsa/index.html pub fn main() { - setup_all_moduli(); - setup_all_curves(); - let msg = b"example message"; let signature = hex!( diff --git a/extensions/ecc/tests/programs/examples/invalid_setup.rs b/extensions/ecc/tests/programs/examples/invalid_setup.rs index 9c6e8a3a51..44b70f1d59 100644 --- a/extensions/ecc/tests/programs/examples/invalid_setup.rs +++ b/extensions/ecc/tests/programs/examples/invalid_setup.rs @@ -2,7 +2,7 @@ #![cfg_attr(not(feature = "std"), no_std)] #[allow(unused_imports)] -use openvm_ecc_guest::{k256::Secp256k1Point, p256::P256Point}; +use openvm_ecc_guest::{k256::Secp256k1Point, p256::P256Point, CyclicGroup}; openvm_algebra_moduli_macros::moduli_init! { "0xFFFFFFFF FFFFFFFF FFFFFFFF FFFFFFFF FFFFFFFF FFFFFFFF FFFFFFFE FFFFFC2F", @@ -20,7 +20,7 @@ openvm_ecc_sw_macros::sw_init! { openvm::entry!(main); pub fn main() { - setup_all_moduli(); // this should cause a debug assertion to fail - setup_all_curves(); + let p1 = Secp256k1Point::GENERATOR; + let _p2 = &p1 + &p1; } diff --git a/extensions/pairing/tests/programs/examples/bls_ec.rs b/extensions/pairing/tests/programs/examples/bls_ec.rs index c7e13f626a..93f72115cf 100644 --- a/extensions/pairing/tests/programs/examples/bls_ec.rs +++ b/extensions/pairing/tests/programs/examples/bls_ec.rs @@ -8,7 +8,4 @@ openvm::init!("openvm_init_bls_ec_bls12_381.rs"); openvm::entry!(main); -pub fn main() { - setup_all_moduli(); - setup_all_curves(); -} +pub fn main() {} diff --git a/extensions/pairing/tests/programs/examples/fp12_mul.rs b/extensions/pairing/tests/programs/examples/fp12_mul.rs index 0ba32c336d..aeec2fb63d 100644 --- a/extensions/pairing/tests/programs/examples/fp12_mul.rs +++ b/extensions/pairing/tests/programs/examples/fp12_mul.rs @@ -16,8 +16,6 @@ mod bn254 { openvm::init!("openvm_init_fp12_mul_bn254.rs"); pub fn test_fp12_mul(io: &[u8]) { - setup_0(); - setup_all_complex_extensions(); assert_eq!(io.len(), 32 * 36); let f0 = &io[0..32 * 12]; @@ -48,8 +46,6 @@ mod bls12_381 { openvm::init!("openvm_init_fp12_mul_bls12_381.rs"); pub fn test_fp12_mul(io: &[u8]) { - setup_0(); - setup_all_complex_extensions(); assert_eq!(io.len(), 48 * 36); let f0 = &io[0..48 * 12]; diff --git a/extensions/pairing/tests/programs/examples/pairing_check.rs b/extensions/pairing/tests/programs/examples/pairing_check.rs index e3be6a8a34..c01caded79 100644 --- a/extensions/pairing/tests/programs/examples/pairing_check.rs +++ b/extensions/pairing/tests/programs/examples/pairing_check.rs @@ -22,8 +22,6 @@ mod bn254 { openvm::init!("openvm_init_pairing_check_bn254.rs"); pub fn test_pairing_check(io: &[u8]) { - setup_0(); - setup_all_complex_extensions(); let s0 = &io[0..32 * 2]; let s1 = &io[32 * 2..32 * 4]; let q0 = &io[32 * 4..32 * 8]; @@ -57,8 +55,6 @@ mod bls12_381 { openvm::init!("openvm_init_pairing_check_bls12_381.rs"); pub fn test_pairing_check(io: &[u8]) { - setup_0(); - setup_all_complex_extensions(); let s0 = &io[0..48 * 2]; let s1 = &io[48 * 2..48 * 4]; let q0 = &io[48 * 4..48 * 8]; diff --git a/extensions/pairing/tests/programs/examples/pairing_check_fallback.rs b/extensions/pairing/tests/programs/examples/pairing_check_fallback.rs index f9b504bb71..da3bcbb16f 100644 --- a/extensions/pairing/tests/programs/examples/pairing_check_fallback.rs +++ b/extensions/pairing/tests/programs/examples/pairing_check_fallback.rs @@ -94,8 +94,6 @@ mod bn254 { } pub fn test_pairing_check(io: &[u8]) { - setup_0(); - setup_all_complex_extensions(); let s0 = &io[0..32 * 2]; let s1 = &io[32 * 2..32 * 4]; let q0 = &io[32 * 4..32 * 8]; @@ -202,8 +200,6 @@ mod bls12_381 { } pub fn test_pairing_check(io: &[u8]) { - setup_0(); - setup_all_complex_extensions(); let s0 = &io[0..48 * 2]; let s1 = &io[48 * 2..48 * 4]; let q0 = &io[48 * 4..48 * 8]; diff --git a/extensions/pairing/tests/programs/examples/pairing_line.rs b/extensions/pairing/tests/programs/examples/pairing_line.rs index 36a9d1b4ef..b36c200391 100644 --- a/extensions/pairing/tests/programs/examples/pairing_line.rs +++ b/extensions/pairing/tests/programs/examples/pairing_line.rs @@ -136,15 +136,11 @@ pub fn main() { #[cfg(feature = "bn254")] { - bn254::setup_0(); - bn254::setup_all_complex_extensions(); bn254::test_mul_013_by_013(&io[..32 * 18]); bn254::test_mul_by_01234(&io[32 * 18..32 * 52]); } #[cfg(feature = "bls12_381")] { - bls12_381::setup_0(); - bls12_381::setup_all_complex_extensions(); bls12_381::test_mul_023_by_023(&io[..48 * 18]); bls12_381::test_mul_by_02345(&io[48 * 18..48 * 52]); } diff --git a/extensions/pairing/tests/programs/examples/pairing_miller_loop.rs b/extensions/pairing/tests/programs/examples/pairing_miller_loop.rs index afc1ae37a4..a9d8f09dbd 100644 --- a/extensions/pairing/tests/programs/examples/pairing_miller_loop.rs +++ b/extensions/pairing/tests/programs/examples/pairing_miller_loop.rs @@ -20,8 +20,6 @@ mod bn254 { openvm::init!("openvm_init_pairing_miller_loop_bn254.rs"); pub fn test_miller_loop(io: &[u8]) { - setup_0(); - setup_all_complex_extensions(); let s0 = &io[0..32 * 2]; let s1 = &io[32 * 2..32 * 4]; let q0 = &io[32 * 4..32 * 8]; @@ -56,8 +54,6 @@ mod bls12_381 { openvm::init!("openvm_init_pairing_miller_loop_bls12_381.rs"); pub fn test_miller_loop(io: &[u8]) { - setup_0(); - setup_all_complex_extensions(); let s0 = &io[0..48 * 2]; let s1 = &io[48 * 2..48 * 4]; let q0 = &io[48 * 4..48 * 8]; diff --git a/extensions/pairing/tests/programs/examples/pairing_miller_step.rs b/extensions/pairing/tests/programs/examples/pairing_miller_step.rs index c944298aa6..a2d4662cf4 100644 --- a/extensions/pairing/tests/programs/examples/pairing_miller_step.rs +++ b/extensions/pairing/tests/programs/examples/pairing_miller_step.rs @@ -155,15 +155,11 @@ pub fn main() { #[cfg(feature = "bn254")] { - bn254::setup_0(); - bn254::setup_all_complex_extensions(); bn254::test_miller_step(&io[..32 * 12]); bn254::test_miller_double_and_add_step(&io[32 * 12..]); } #[cfg(feature = "bls12_381")] { - bls12_381::setup_0(); - bls12_381::setup_all_complex_extensions(); bls12_381::test_miller_step(&io[..48 * 12]); bls12_381::test_miller_double_and_add_step(&io[48 * 12..]); } From 2b220f3e0dede6fa6fe3c67a591e7e3f0d93202a Mon Sep 17 00:00:00 2001 From: Jonathan Wang <31040440+jonathanpwang@users.noreply.github.com> Date: Fri, 9 May 2025 00:39:38 -0400 Subject: [PATCH 04/15] chore: update `getrandom` to v0.3 in `openvm` lib (#1635) delete getrandom dependency from platform since getrandom now makes enabling of the custom backend opt-in via extern function and --cfg flag closes INT-3934 --- .github/workflows/lints.yml | 7 ++-- Cargo.lock | 2 +- book/src/writing-apps/write-program.md | 8 ++++ crates/toolchain/build/src/lib.rs | 3 ++ crates/toolchain/openvm/Cargo.toml | 15 ++++--- crates/toolchain/openvm/src/getrandom.rs | 14 +++++++ crates/toolchain/openvm/src/lib.rs | 2 + crates/toolchain/platform/Cargo.toml | 16 +------- crates/toolchain/platform/src/getrandom.rs | 18 --------- crates/toolchain/platform/src/lib.rs | 2 - extensions/rv32im/tests/programs/Cargo.toml | 7 ++++ .../tests/programs/examples/getrandom.rs | 39 +++++++++++++++++++ extensions/rv32im/tests/src/lib.rs | 22 +++++++++++ 13 files changed, 108 insertions(+), 47 deletions(-) create mode 100644 crates/toolchain/openvm/src/getrandom.rs delete mode 100644 crates/toolchain/platform/src/getrandom.rs create mode 100644 extensions/rv32im/tests/programs/examples/getrandom.rs diff --git a/.github/workflows/lints.yml b/.github/workflows/lints.yml index bb99d4ff4e..e26cecf7bf 100644 --- a/.github/workflows/lints.yml +++ b/.github/workflows/lints.yml @@ -43,9 +43,10 @@ jobs: - name: Run clippy run: | - # list of features generated using: - # echo -e "\033[1;32mAll unique features across workspace:\033[0m" && cargo metadata --format-version=1 --no-deps | jq -r '.packages[].features | to_entries[] | .key' | sort -u | sed 's/^/• /' - cargo clippy --all-targets --all --tests --features "aggregation bench-metrics bls12_381 bn254 default entrypoint evm-prove evm-verify export-getrandom export-libm function-span getrandom halo2-compiler halo2curves heap-embedded-alloc k256 jemalloc jemalloc-prof nightly-features panic-handler parallel rust-runtime static-verifier std test-utils unstable" -- -D warnings + # list of all unique features across workspace generated using: + # cargo metadata --format-version=1 --no-deps | jq -r '.packages[].features | to_entries[] | .key' | sort -u | tr '\n' ' ' && echo "" + # (exclude mimalloc since it conflicts with jemalloc) + cargo clippy --all-targets --all --tests --features "aggregation bench-metrics bls12_381 bn254 build-binaries default entrypoint evm-prove evm-verify export-intrinsics export-libm function-span getrandom-unsupported halo2-compiler halo2curves heap-embedded-alloc jemalloc jemalloc-prof k256 nightly-features p256 panic-handler parallel profiling rust-runtime static-verifier std test-utils" -- -D warnings cargo clippy --all-targets --all --tests --no-default-features --features "mimalloc" -- -D warnings - name: Run fmt, clippy for guest diff --git a/Cargo.lock b/Cargo.lock index fe5d5d48c9..518f8fde11 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3918,6 +3918,7 @@ version = "1.1.2" dependencies = [ "bytemuck", "chrono", + "getrandom 0.3.1", "num-bigint 0.4.6", "openvm-custom-insn", "openvm-platform", @@ -4673,7 +4674,6 @@ version = "1.1.2" dependencies = [ "critical-section", "embedded-alloc", - "getrandom 0.2.15", "libm", "openvm-custom-insn", "openvm-rv32im-guest", diff --git a/book/src/writing-apps/write-program.md b/book/src/writing-apps/write-program.md index 8f2fb7d9d9..b4005953af 100644 --- a/book/src/writing-apps/write-program.md +++ b/book/src/writing-apps/write-program.md @@ -122,3 +122,11 @@ This tells Rust to use the custom `main` handler when the environment is `no_std ## Building and running See the [overview](./overview.md) on how to build and run the program. + +## Using crates that depend on `getrandom` + +OpenVM is compatible with [getrandom](https://crates.io/crates/getrandom) `v0.3`. The `cargo openvm` CLI will always compile with the [custom](https://docs.rs/getrandom/latest/getrandom/#opt-in-backends) `getrandom` backend. + +By default the `openvm` crate has a default feature `"getrandom-unsupported"` which exports a `__getrandom_v03_custom` function that always returns `Err(Error::UNSUPPORTED)`. This is enabled by default to allow compilation of guest programs that pull in dependencies which require `getrandom` but where the executed code does not actually use `getrandom` functions. + +To override the default behavior and provide a custom implementation, turn off the `"getrandom-unsupported"` feature in the `openvm` crate and supply your own `__getrandom_v03_custom` function as specified in the [getrandom docs](https://docs.rs/getrandom/latest/getrandom/#custom-backend). diff --git a/crates/toolchain/build/src/lib.rs b/crates/toolchain/build/src/lib.rs index b6cffb3164..79cf5b2354 100644 --- a/crates/toolchain/build/src/lib.rs +++ b/crates/toolchain/build/src/lib.rs @@ -233,6 +233,9 @@ pub(crate) fn encode_rust_flags(rustc_flags: &[&str]) -> String { "link-arg=--fatal-warnings", "-C", "panic=abort", + // https://docs.rs/getrandom/0.3.2/getrandom/index.html#opt-in-backends + "--cfg", + "getrandom_backend=\"custom\"", ], ] .concat() diff --git a/crates/toolchain/openvm/Cargo.toml b/crates/toolchain/openvm/Cargo.toml index 08a7996a3d..1692e30f79 100644 --- a/crates/toolchain/openvm/Cargo.toml +++ b/crates/toolchain/openvm/Cargo.toml @@ -9,15 +9,15 @@ repository.workspace = true license.workspace = true [dependencies] -openvm-platform = { workspace = true, features = [ - "rust-runtime", - "export-getrandom", -] } +openvm-platform = { workspace = true, features = ["rust-runtime"] } openvm-custom-insn = { workspace = true } openvm-rv32im-guest = { workspace = true } serde = { workspace = true, features = ["alloc"] } bytemuck = { workspace = true, features = ["extern_crate_alloc"] } +[target.'cfg(target_os = "zkvm")'.dependencies] +getrandom = { version = "0.3", optional = true } + [target.'cfg(not(target_os = "zkvm"))'.dependencies] num-bigint.workspace = true @@ -25,10 +25,9 @@ num-bigint.workspace = true chrono = { version = "0.4", default-features = false, features = ["serde"] } [features] -default = [] -# The zkVM exposes a getrandom implementation that panics by default. -# This (currently unimplemented) feature expose a getrandom implementation that uses host randomness. -getrandom = ["openvm-platform/getrandom"] +default = ["getrandom-unsupported"] +# Defines a custom getrandom backend that always errors. This feature should be enabled if you are sure getrandom is never used but it is pulled in as a compilation dependency. +getrandom-unsupported = ["dep:getrandom"] # The zkVM uses a bump-pointer heap allocator by default which does not free # memory. This will use a slower linked-list heap allocator to reclaim memory. heap-embedded-alloc = ["openvm-platform/heap-embedded-alloc"] diff --git a/crates/toolchain/openvm/src/getrandom.rs b/crates/toolchain/openvm/src/getrandom.rs new file mode 100644 index 0000000000..429c2c9b76 --- /dev/null +++ b/crates/toolchain/openvm/src/getrandom.rs @@ -0,0 +1,14 @@ +//! [getrandom] custom backend implementations. The implementations are feature-gated. The default +//! feature enables "getrandom-unsupported", which is a backend that always errors. This should be +//! used when `getrandom` is never called but pulled in as a dependency unavoidably. If no feature +//! is enabled, then no custom implementation is registered, and the user must supply their own as +//! described in the [getrandom] documentation. + +#[cfg(feature = "getrandom-unsupported")] +#[no_mangle] +unsafe extern "Rust" fn __getrandom_v03_custom( + _dest: *mut u8, + _len: usize, +) -> Result<(), getrandom::Error> { + Err(getrandom::Error::UNSUPPORTED) +} diff --git a/crates/toolchain/openvm/src/lib.rs b/crates/toolchain/openvm/src/lib.rs index 00decf04d9..d0507db84c 100644 --- a/crates/toolchain/openvm/src/lib.rs +++ b/crates/toolchain/openvm/src/lib.rs @@ -18,6 +18,8 @@ use openvm_platform::rust_rt; #[cfg(target_os = "zkvm")] pub use openvm_rv32im_guest::*; +#[cfg(target_os = "zkvm")] +mod getrandom; pub mod io; #[cfg(all(feature = "std", target_os = "zkvm"))] pub mod pal_abi; diff --git a/crates/toolchain/platform/Cargo.toml b/crates/toolchain/platform/Cargo.toml index 9f6bb6d47a..26d935dc41 100644 --- a/crates/toolchain/platform/Cargo.toml +++ b/crates/toolchain/platform/Cargo.toml @@ -18,29 +18,16 @@ critical-section = { version = "1.1.2", optional = true } embedded-alloc = { version = "0.6.0", features = [ "allocator_api", ], optional = true } -getrandom = { version = "0.2", features = ["custom"], optional = true } libm = { version = "0.2", optional = true } [package.metadata.docs.rs] rustdoc-args = ["--cfg", "docsrs"] -features = [ - "rust-runtime", - "panic-handler", - "export-libm", - "export-getrandom", - "export-pal-abi", - "getrandom", - "unstable", -] +features = ["rust-runtime", "panic-handler", "export-libm", "export-pal-abi"] [features] default = [] entrypoint = [] -# exports a `getrandom` implementation that panics -export-getrandom = ["dep:getrandom"] export-libm = ["dep:libm"] -# Currently unimplemented: exports a `getrandom` implementation that uses sys_random -getrandom = ["export-getrandom"] heap-embedded-alloc = [ "dep:critical-section", "dep:embedded-alloc", @@ -50,4 +37,3 @@ panic-handler = [] # Build a rust runtime rust-runtime = ["export-libm"] std = [] -unstable = [] diff --git a/crates/toolchain/platform/src/getrandom.rs b/crates/toolchain/platform/src/getrandom.rs deleted file mode 100644 index 1edb4578ee..0000000000 --- a/crates/toolchain/platform/src/getrandom.rs +++ /dev/null @@ -1,18 +0,0 @@ -//! We need to export a custom getrandom implementation just to get crates that import getrandom to -//! compile. -use getrandom::{register_custom_getrandom, Error}; - -/// This is a getrandom handler for the zkvm. It's intended to hook into a -/// getrandom crate or a dependent of the getrandom crate used by the guest code. -#[cfg(feature = "getrandom")] -pub fn zkvm_getrandom(dest: &mut [u8]) -> Result<(), Error> { - todo!() - // Randomness would come from the host -} - -#[cfg(not(feature = "getrandom"))] -pub fn zkvm_getrandom(dest: &mut [u8]) -> Result<(), Error> { - panic!("getrandom is not enabled in the current build"); -} - -register_custom_getrandom!(zkvm_getrandom); diff --git a/crates/toolchain/platform/src/lib.rs b/crates/toolchain/platform/src/lib.rs index 901666d530..1ace328a66 100644 --- a/crates/toolchain/platform/src/lib.rs +++ b/crates/toolchain/platform/src/lib.rs @@ -6,8 +6,6 @@ #[cfg(all(feature = "rust-runtime", target_os = "zkvm"))] pub use openvm_custom_insn::{custom_insn_i, custom_insn_r}; -#[cfg(all(feature = "export-getrandom", target_os = "zkvm"))] -mod getrandom; #[cfg(all(feature = "rust-runtime", target_os = "zkvm"))] pub mod heap; #[cfg(all(feature = "export-libm", target_os = "zkvm"))] diff --git a/extensions/rv32im/tests/programs/Cargo.toml b/extensions/rv32im/tests/programs/Cargo.toml index b052b7be9a..12915c0da5 100644 --- a/extensions/rv32im/tests/programs/Cargo.toml +++ b/extensions/rv32im/tests/programs/Cargo.toml @@ -13,13 +13,20 @@ serde = { version = "1.0", default-features = false, features = [ "alloc", "derive", ] } +getrandom = { version = "0.3", optional = true } [features] default = [] std = ["serde/std", "openvm/std"] heap-embedded-alloc = ["openvm/heap-embedded-alloc"] +getrandom-unsupported = ["openvm/getrandom-unsupported"] +getrandom = ["dep:getrandom"] [profile.release] panic = "abort" lto = "thin" # turn on lto = fat to decrease binary size, but this optimizes out some missing extern links so we shouldn't use it for testing # strip = "symbols" + +[[example]] +name = "getrandom" +required-features = ["getrandom"] diff --git a/extensions/rv32im/tests/programs/examples/getrandom.rs b/extensions/rv32im/tests/programs/examples/getrandom.rs new file mode 100644 index 0000000000..353bf47355 --- /dev/null +++ b/extensions/rv32im/tests/programs/examples/getrandom.rs @@ -0,0 +1,39 @@ +#![cfg_attr(not(feature = "std"), no_main)] +#![cfg_attr(not(feature = "std"), no_std)] +use getrandom::Error; + +fn get_random_u128() -> Result { + let mut buf = [0u8; 16]; + getrandom::fill(&mut buf)?; + Ok(u128::from_ne_bytes(buf)) +} + +openvm::entry!(main); + +pub fn main() { + // do unrelated stuff + let mut c = core::hint::black_box(0); + for _ in 0..10 { + c += 1; + } + + #[cfg(not(feature = "getrandom-unsupported"))] + { + // not a good random function! + assert_eq!(get_random_u128(), Ok(0)); + } + #[cfg(feature = "getrandom-unsupported")] + { + assert!(get_random_u128().is_err()); + } +} + +// custom user-specified getrandom +#[cfg(all(feature = "getrandom", not(feature = "getrandom-unsupported")))] +#[no_mangle] +unsafe extern "Rust" fn __getrandom_v03_custom(dest: *mut u8, len: usize) -> Result<(), Error> { + for i in 0..len { + *dest.add(i) = 0u8; + } + Ok(()) +} diff --git a/extensions/rv32im/tests/src/lib.rs b/extensions/rv32im/tests/src/lib.rs index cde90f6a82..7799c7d971 100644 --- a/extensions/rv32im/tests/src/lib.rs +++ b/extensions/rv32im/tests/src/lib.rs @@ -249,4 +249,26 @@ mod tests { let executor = VmExecutor::::new(config.clone()); executor.execute(exe, vec![]).unwrap(); } + + #[test_case(vec!["getrandom", "getrandom-unsupported"])] + #[test_case(vec!["getrandom"])] + fn test_getrandom_unsupported(features: Vec<&str>) { + let config = Rv32ImConfig::default(); + let elf = build_example_program_at_path_with_features( + get_programs_dir!(), + "getrandom", + &features, + &config, + ) + .unwrap(); + let exe = VmExe::from_elf( + elf, + Transpiler::::default() + .with_extension(Rv32ITranspilerExtension) + .with_extension(Rv32MTranspilerExtension) + .with_extension(Rv32IoTranspilerExtension), + ) + .unwrap(); + air_test(config, exe); + } } From dd6562e77818721f53fc3c95968ab0ad11068529 Mon Sep 17 00:00:00 2001 From: stephenh-axiom-xyz Date: Wed, 14 May 2025 15:10:04 -0400 Subject: [PATCH 05/15] feat: openvm build can build multiple targets + additional cargo options (#1647) Resolves INT-4000, INT-3883, and INT-3662. In general, we would like `cargo openvm build` to align as closely with `cargo build` as possible. OpenVM users should be able to almost completely replace the latter with the former in their development workflows. Implements the following: - Parity between `cargo openvm build` and `cargo build` options - Ability to build multiple targets when building at the workspace level - `cargo openvm build` works outside of project root - Default paths in OpenVM CLI evaluate environment variable `HOME` at runtime OpenVM-generated artifacts being stored in `${CARGO_TARGET_DIR}/.openvm` is still to be implemented, but in order to maintain coherency between `cargo openvm build` and `cargo openvm run/prove` this change will be added later. --- crates/cli/src/bin/cargo-openvm.rs | 1 + crates/cli/src/commands/build.rs | 507 +++++++++++++----- crates/cli/src/commands/prove.rs | 4 +- crates/cli/src/commands/setup.rs | 26 +- crates/cli/src/commands/verify.rs | 2 +- crates/cli/src/default.rs | 19 +- crates/cli/src/util.rs | 11 + crates/cli/tests/app_e2e.rs | 28 +- crates/cli/tests/build.rs | 142 +++++ .../programs/fibonacci}/Cargo.toml | 2 +- .../programs/fibonacci}/openvm.toml | 0 .../programs/fibonacci}/src/main.rs | 0 crates/cli/tests/programs/multi/Cargo.toml | 9 + .../programs/multi/calculator/Cargo.toml | 19 + .../multi/calculator/examples/fibonacci.rs | 18 + .../programs/multi/calculator/src/lib.rs | 33 ++ .../programs/multi/calculator/src/main.rs | 13 + crates/cli/tests/programs/multi/openvm.toml | 3 + .../programs/multi/string-utils/Cargo.toml | 15 + .../multi/string-utils/examples/palindrome.rs | 25 + .../programs/multi/string-utils/src/main.rs | 21 + crates/toolchain/build/src/lib.rs | 123 +++-- 22 files changed, 800 insertions(+), 221 deletions(-) create mode 100644 crates/cli/tests/build.rs rename crates/cli/{example => tests/programs/fibonacci}/Cargo.toml (67%) rename crates/cli/{example => tests/programs/fibonacci}/openvm.toml (100%) rename crates/cli/{example => tests/programs/fibonacci}/src/main.rs (100%) create mode 100644 crates/cli/tests/programs/multi/Cargo.toml create mode 100644 crates/cli/tests/programs/multi/calculator/Cargo.toml create mode 100644 crates/cli/tests/programs/multi/calculator/examples/fibonacci.rs create mode 100644 crates/cli/tests/programs/multi/calculator/src/lib.rs create mode 100644 crates/cli/tests/programs/multi/calculator/src/main.rs create mode 100644 crates/cli/tests/programs/multi/openvm.toml create mode 100644 crates/cli/tests/programs/multi/string-utils/Cargo.toml create mode 100644 crates/cli/tests/programs/multi/string-utils/examples/palindrome.rs create mode 100644 crates/cli/tests/programs/multi/string-utils/src/main.rs diff --git a/crates/cli/src/bin/cargo-openvm.rs b/crates/cli/src/bin/cargo-openvm.rs index d3002fdfdf..1a5e9f5c58 100644 --- a/crates/cli/src/bin/cargo-openvm.rs +++ b/crates/cli/src/bin/cargo-openvm.rs @@ -19,6 +19,7 @@ pub struct VmCli { } #[derive(Subcommand)] +#[allow(clippy::large_enum_variant)] pub enum VmCliCommands { Build(BuildCmd), Keygen(KeygenCmd), diff --git a/crates/cli/src/commands/build.rs b/crates/cli/src/commands/build.rs index 22665b0b41..576a9e044d 100644 --- a/crates/cli/src/commands/build.rs +++ b/crates/cli/src/commands/build.rs @@ -8,7 +8,8 @@ use std::{ use clap::Parser; use eyre::Result; use openvm_build::{ - build_guest_package, find_unique_executable, get_package, GuestOptions, TargetFilter, + build_generic, get_package, get_target_dir, get_workspace_packages, get_workspace_root, + GuestOptions, }; use openvm_circuit::arch::{InitFileGenerator, OPENVM_DEFAULT_INIT_FILE_NAME}; use openvm_sdk::{ @@ -21,9 +22,9 @@ use openvm_transpiler::{elf::Elf, openvm_platform::memory::MEM_SIZE}; use crate::{ default::{ DEFAULT_APP_CONFIG_PATH, DEFAULT_APP_EXE_PATH, DEFAULT_COMMITTED_APP_EXE_PATH, - DEFAULT_EXE_COMMIT_PATH, DEFAULT_MANIFEST_DIR, + DEFAULT_EXE_COMMIT_PATH, }, - util::read_config_toml_or_default, + util::{find_manifest_dir, read_config_toml_or_default}, }; #[derive(Parser)] @@ -31,126 +32,326 @@ use crate::{ pub struct BuildCmd { #[clap(flatten)] build_args: BuildArgs, + + #[clap(flatten)] + cargo_args: BuildCargoArgs, } impl BuildCmd { pub fn run(&self) -> Result<()> { - build(&self.build_args)?; + build(&self.build_args, &self.cargo_args)?; Ok(()) } } #[derive(Clone, Parser)] pub struct BuildArgs { - #[arg( - long, - help = "Path to the directory containing the Cargo.toml file for the guest code (relative to the current directory)", - default_value = DEFAULT_MANIFEST_DIR - )] - pub manifest_dir: PathBuf, - - #[arg(long, help = "Path to the target directory")] - pub target_dir: Option, - - #[arg(long, value_delimiter = ',', help = "Feature flags passed to cargo")] - pub features: Vec, - - #[clap(flatten, help = "Filter the target to build")] - pub bin_type_filter: BinTypeFilter, - #[arg( long, default_value = "false", - help = "Skips transpilation into exe when set" + help = "Skips transpilation into exe when set", + help_heading = "OpenVM Options" )] pub no_transpile: bool, #[arg( long, default_value = DEFAULT_APP_CONFIG_PATH, - help = "Path to the SDK config .toml file that specifies the transpiler extensions" + help = "Path to the OpenVM config .toml file that specifies the VM extensions", + help_heading = "OpenVM Options" )] pub config: PathBuf, #[arg( long, default_value = DEFAULT_APP_EXE_PATH, - help = "Output path for the transpiled program" + help = "Output path for the transpiled program", + help_heading = "OpenVM Options" )] pub exe_output: PathBuf, #[arg( long, default_value = DEFAULT_COMMITTED_APP_EXE_PATH, - help = "Output path for the committed program" + help = "Output path for the committed program", + help_heading = "OpenVM Options" )] pub committed_exe_output: PathBuf, #[arg( long, default_value = DEFAULT_EXE_COMMIT_PATH, - help = "Output path for the exe commit (bn254 commit of committed program)" + help = "Output path for the exe commit (bn254 commit of committed program)", + help_heading = "OpenVM Options" )] pub exe_commit_output: PathBuf, - #[arg(long, default_value = "release", help = "Build profile")] + #[arg( + long, + default_value = OPENVM_DEFAULT_INIT_FILE_NAME, + help = "Name of the init file", + help_heading = "OpenVM Options" + )] + pub init_file_name: String, +} + +#[derive(Clone, Parser)] +pub struct BuildCargoArgs { + #[arg( + long, + short = 'p', + value_name = "PACKAGES", + help = "Build only specified packages", + help_heading = "Package Selection" + )] + pub package: Vec, + + #[arg( + long, + alias = "all", + help = "Build all members of the workspace", + help_heading = "Package Selection" + )] + pub workspace: bool, + + #[arg( + long, + value_name = "PACKAGES", + help = "Exclude specified packages", + help_heading = "Package Selection" + )] + pub exclude: Vec, + + #[arg( + long, + help = "Build the package library", + help_heading = "Target Selection" + )] + pub lib: bool, + + #[arg( + long, + value_name = "BIN", + help = "Build the specified binary", + help_heading = "Target Selection" + )] + pub bin: Vec, + + #[arg( + long, + help = "Build all binary targets", + help_heading = "Target Selection" + )] + pub bins: bool, + + #[arg( + long, + value_name = "EXAMPLE", + help = "Build the specified example", + help_heading = "Target Selection" + )] + pub example: Vec, + + #[arg( + long, + help = "Build all example targets", + help_heading = "Target Selection" + )] + pub examples: bool, + + #[arg( + long, + help = "Build all package targets", + help_heading = "Target Selection" + )] + pub all_targets: bool, + + #[arg( + long, + short = 'F', + value_name = "FEATURES", + value_delimiter = ',', + help = "Space/comma separated list of features to activate", + help_heading = "Feature Selection" + )] + pub features: Vec, + + #[arg( + long, + help = "Activate all available features of all selected packages", + help_heading = "Feature Selection" + )] + pub all_features: bool, + + #[arg( + long, + help = "Do not activate the `default` feature of the selected packages", + help_heading = "Feature Selection" + )] + pub no_default_features: bool, + + #[arg( + long, + value_name = "NAME", + default_value = "release", + help = "Build with the given profile", + help_heading = "Compilation Options" + )] pub profile: String, - #[arg(long, default_value = "false", help = "use --offline in cargo build")] - pub offline: bool, + #[arg( + long, + value_name = "DIR", + help = "Directory for all generated artifacts and intermediate files", + help_heading = "Output Options" + )] + pub target_dir: Option, - #[arg(long, default_value = OPENVM_DEFAULT_INIT_FILE_NAME, help = "Name of the init file")] - pub init_file_name: String, -} + #[arg( + long, + short = 'v', + help = "Use verbose output", + help_heading = "Display Options" + )] + pub verbose: bool, + + #[arg( + long, + short = 'q', + help = "Do not print cargo log messages", + help_heading = "Display Options" + )] + pub quiet: bool, + + #[arg( + long, + value_name = "WHEN", + default_value = "always", + help = "Control when colored output is used", + help_heading = "Display Options" + )] + pub color: String, + + #[arg( + long, + value_name = "PATH", + help = "Path to the Cargo.toml file, by default searches for the file in the current or any parent directory", + help_heading = "Manifest Options" + )] + pub manifest_path: Option, + + #[arg( + long, + help = "Ignore rust-version specification in packages", + help_heading = "Manifest Options" + )] + pub ignore_rust_version: bool, + + #[arg( + long, + help = "Asserts same dependencies and versions are used as when the existing Cargo.lock file was originally generated", + help_heading = "Manifest Options" + )] + pub locked: bool, -#[derive(Clone, Default, clap::Args)] -#[group(required = false, multiple = false)] -pub struct BinTypeFilter { - #[arg(long, help = "Specifies the bin target to build")] - pub bin: Option, + #[arg( + long, + help = "Prevents Cargo from accessing the network for any reason", + help_heading = "Manifest Options" + )] + pub offline: bool, - #[arg(long, help = "Specifies the example target to build")] - pub example: Option, + #[arg( + long, + help = "Equivalent to specifying both --locked and --offline", + help_heading = "Manifest Options" + )] + pub frozen: bool, } -// Returns the path to the ELF file if it is unique. -pub(crate) fn build(build_args: &BuildArgs) -> Result> { +// Returns the paths to the ELF file for each built executable target. +pub fn build(build_args: &BuildArgs, cargo_args: &BuildCargoArgs) -> Result> { println!("[openvm] Building the package..."); - let target_filter = if let Some(bin) = &build_args.bin_type_filter.bin { - Some(TargetFilter { - name: bin.clone(), - kind: "bin".to_string(), - }) + + // Find manifest directory using either manifest_path or find_manifest_dir + let manifest_dir = if let Some(manifest_path) = &cargo_args.manifest_path { + if !manifest_path.ends_with("Cargo.toml") { + return Err(eyre::eyre!( + "manifest_path must be a path to a Cargo.toml file" + )); + } + manifest_path.parent().unwrap().to_path_buf() } else { - build_args - .bin_type_filter - .example - .as_ref() - .map(|example| TargetFilter { - name: example.clone(), - kind: "example".to_string(), - }) + find_manifest_dir(PathBuf::from("."))? }; + let manifest_path = manifest_dir.join("Cargo.toml"); + println!("[openvm] Manifest directory: {}", manifest_dir.display()); + + // Get target path + let target_path = if let Some(target_path) = &cargo_args.target_dir { + target_path.to_path_buf() + } else { + get_target_dir(&manifest_path) + }; + + // Set guest options using build arguments; use found manifest directory for consistency let mut guest_options = GuestOptions::default() - .with_features(build_args.features.clone()) - .with_profile(build_args.profile.clone()) + .with_features(cargo_args.features.clone()) + .with_profile(cargo_args.profile.clone()) .with_rustc_flags(var("RUSTFLAGS").unwrap_or_default().split_whitespace()); - guest_options.target_dir = build_args.target_dir.clone(); - if build_args.offline { - guest_options.options = vec!["--offline".to_string()]; + + guest_options.target_dir = Some(target_path); + guest_options + .options + .push(format!("--color={}", cargo_args.color)); + guest_options.options.push("--manifest-path".to_string()); + guest_options + .options + .push(manifest_path.to_string_lossy().to_string()); + + for pkg in &cargo_args.package { + guest_options.options.push("--package".to_string()); + guest_options.options.push(pkg.clone()); + } + for pkg in &cargo_args.exclude { + guest_options.options.push("--exclude".to_string()); + guest_options.options.push(pkg.clone()); + } + for target in &cargo_args.bin { + guest_options.options.push("--bin".to_string()); + guest_options.options.push(target.clone()); + } + for example in &cargo_args.example { + guest_options.options.push("--example".to_string()); + guest_options.options.push(example.clone()); } - let app_config = read_config_toml_or_default(&build_args.config)?; - app_config - .app_vm_config - .write_to_init_file(&build_args.manifest_dir, Some(&build_args.init_file_name))?; + let all_bins = cargo_args.bins || cargo_args.all_targets; + let all_examples = cargo_args.examples || cargo_args.all_targets; - let pkg = get_package(&build_args.manifest_dir); - // We support builds of libraries with 0 or >1 executables. - let elf_path = match build_guest_package(&pkg, &guest_options, None, &target_filter) { - Ok(target_dir) => { - find_unique_executable(&build_args.manifest_dir, &target_dir, &target_filter) + let boolean_flags = [ + ("--workspace", cargo_args.workspace), + ("--lib", cargo_args.lib || cargo_args.all_targets), + ("--bins", all_bins), + ("--examples", all_examples), + ("--all-features", cargo_args.all_features), + ("--no-default-features", cargo_args.no_default_features), + ("--verbose", cargo_args.verbose), + ("--quiet", cargo_args.quiet), + ("--ignore-rust-version", cargo_args.ignore_rust_version), + ("--locked", cargo_args.locked), + ("--offline", cargo_args.offline), + ("--frozen", cargo_args.frozen), + ]; + for (flag, enabled) in boolean_flags { + if enabled { + guest_options.options.push(flag.to_string()); } + } + + // Build (allowing passed options to decide what gets built) + let target_dir = match build_generic(&guest_options) { + Ok(target_dir) => target_dir, Err(None) => { return Err(eyre::eyre!("Failed to build guest")); } @@ -159,87 +360,101 @@ pub(crate) fn build(build_args: &BuildArgs) -> Result> { } }; + // Write to init file + let app_config = read_config_toml_or_default(&build_args.config)?; + app_config + .app_vm_config + .write_to_init_file(&manifest_dir, Some(&build_args.init_file_name))?; + + // Get all built packages + let workspace_root = get_workspace_root(&manifest_path); + let packages = if cargo_args.workspace || manifest_dir == workspace_root { + get_workspace_packages(manifest_dir) + .into_iter() + .filter(|pkg| { + (cargo_args.package.is_empty() || cargo_args.package.contains(&pkg.name)) + && !cargo_args.exclude.contains(&pkg.name) + }) + .collect() + } else { + vec![get_package(manifest_dir)] + }; + + // Find elf paths of all targets for all built packages + let elf_paths: Vec = packages + .iter() + .flat_map(|pkg| { + pkg.targets + .iter() + .filter(move |target| { + // We only build bin and example targets (note they are mutually exclusive + // types). If no target selection flags are set, then all bin targets are + // built by default. + if target.is_example() { + return all_examples || cargo_args.example.contains(&target.name); + } else if target.is_bin() { + return all_bins + || cargo_args.bin.contains(&target.name) + || (!cargo_args.examples + && !cargo_args.lib + && cargo_args.bin.is_empty() + && cargo_args.example.is_empty()); + } + false + }) + .map(|target| { + if target.is_example() { + target_dir.join("examples") + } else { + target_dir.clone() + } + .join(&target.name) + }) + .collect::>() + }) + .collect::>(); + + // Transpile and commit, storing in target_dir/openvm/${profile} by default + // TODO[stephen]: actually implement target_dir change if !build_args.no_transpile { - let elf_path = elf_path?; - println!("[openvm] Transpiling the package..."); - let output_path = &build_args.exe_output; - let transpiler = app_config.app_vm_config.transpiler(); - - let data = read(elf_path.clone())?; - let elf = Elf::decode(&data, MEM_SIZE as u32)?; - let exe = Sdk::new().transpile(elf, transpiler)?; - let committed_exe = commit_app_exe(app_config.app_fri_params.fri_params, exe.clone()); - write_exe_to_file(exe, output_path)?; - - if let Some(parent) = build_args.exe_commit_output.parent() { - create_dir_all(parent)?; - } - write( - &build_args.exe_commit_output, - committed_exe_as_bn254(&committed_exe).value.to_bytes(), - )?; - if let Some(parent) = build_args.committed_exe_output.parent() { - create_dir_all(parent)?; + for elf_path in &elf_paths { + println!("[openvm] Transpiling the package..."); + let output_path = &build_args.exe_output; + let transpiler = app_config.app_vm_config.transpiler(); + + let data = read(elf_path.clone())?; + let elf = Elf::decode(&data, MEM_SIZE as u32)?; + let exe = Sdk::new().transpile(elf, transpiler)?; + let committed_exe = commit_app_exe(app_config.app_fri_params.fri_params, exe.clone()); + write_exe_to_file(exe, output_path)?; + + if let Some(parent) = build_args.exe_commit_output.parent() { + create_dir_all(parent)?; + } + write( + &build_args.exe_commit_output, + committed_exe_as_bn254(&committed_exe).value.to_bytes(), + )?; + if let Some(parent) = build_args.committed_exe_output.parent() { + create_dir_all(parent)?; + } + let committed_exe = match Arc::try_unwrap(committed_exe) { + Ok(exe) => exe, + Err(_) => return Err(eyre::eyre!("Failed to unwrap committed_exe Arc")), + }; + write( + &build_args.committed_exe_output, + bitcode::serialize(&committed_exe)?, + )?; + + println!( + "[openvm] Successfully transpiled to {}", + output_path.display() + ); } - let committed_exe = match Arc::try_unwrap(committed_exe) { - Ok(exe) => exe, - Err(_) => return Err(eyre::eyre!("Failed to unwrap committed_exe Arc")), - }; - write( - &build_args.committed_exe_output, - bitcode::serialize(&committed_exe)?, - )?; - - println!( - "[openvm] Successfully transpiled to {}", - output_path.display() - ); - Ok(Some(elf_path)) - } else if let Ok(elf_path) = elf_path { - println!( - "[openvm] Successfully built the package: {}", - elf_path.display() - ); - Ok(Some(elf_path)) - } else { - println!("[openvm] Successfully built the package"); - Ok(None) } -} -#[cfg(test)] -mod tests { - use std::path::PathBuf; - - use eyre::Result; - use openvm_build::RUSTC_TARGET; - - use super::*; - - #[test] - fn test_build_with_profile() -> Result<()> { - let temp_dir = tempfile::tempdir()?; - let target_dir = temp_dir.path(); - let build_args = BuildArgs { - manifest_dir: PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("example"), - features: vec![], - bin_type_filter: Default::default(), - no_transpile: true, - config: PathBuf::from(DEFAULT_APP_CONFIG_PATH), - exe_output: PathBuf::from(DEFAULT_APP_EXE_PATH), - committed_exe_output: PathBuf::from(DEFAULT_COMMITTED_APP_EXE_PATH), - exe_commit_output: PathBuf::from(DEFAULT_EXE_COMMIT_PATH), - profile: "dev".to_string(), - target_dir: Some(target_dir.to_path_buf()), - offline: false, - init_file_name: OPENVM_DEFAULT_INIT_FILE_NAME.to_string(), - }; - build(&build_args)?; - assert!( - target_dir.join(RUSTC_TARGET).join("debug").exists(), - "did not build with dev profile" - ); - temp_dir.close()?; - Ok(()) - } + // Return elf paths of all targets for all built packages + println!("[openvm] Successfully built the packages"); + Ok(elf_paths) } diff --git a/crates/cli/src/commands/prove.rs b/crates/cli/src/commands/prove.rs index 2dbc07add8..d4b0ac50f7 100644 --- a/crates/cli/src/commands/prove.rs +++ b/crates/cli/src/commands/prove.rs @@ -88,11 +88,11 @@ impl ProveCmd { let mut sdk = sdk; sdk.set_agg_tree_config(*agg_tree_config); - let params_reader = CacheHalo2ParamsReader::new(DEFAULT_PARAMS_DIR); + let params_reader = CacheHalo2ParamsReader::new(default_params_dir()); let (app_pk, committed_exe, input) = Self::prepare_execution(&sdk, app_pk, exe, input)?; println!("Generating EVM proof, this may take a lot of compute and memory..."); - let agg_pk = read_agg_pk_from_file(DEFAULT_AGG_PK_PATH).map_err(|e| { + let agg_pk = read_agg_pk_from_file(default_agg_pk_path()).map_err(|e| { eyre::eyre!("Failed to read aggregation proving key: {}\nPlease run 'cargo openvm setup' first", e) })?; let evm_proof = diff --git a/crates/cli/src/commands/setup.rs b/crates/cli/src/commands/setup.rs index a62faaa0df..079d4b65e3 100644 --- a/crates/cli/src/commands/setup.rs +++ b/crates/cli/src/commands/setup.rs @@ -17,7 +17,7 @@ use openvm_sdk::{ DefaultStaticVerifierPvHandler, Sdk, }; -use crate::default::{DEFAULT_AGG_PK_PATH, DEFAULT_EVM_HALO2_VERIFIER_PATH, DEFAULT_PARAMS_DIR}; +use crate::default::{default_agg_pk_path, default_evm_halo2_verifier_path, default_params_dir}; #[derive(Parser)] #[command( @@ -28,14 +28,18 @@ pub struct EvmProvingSetupCmd {} impl EvmProvingSetupCmd { pub async fn run(&self) -> Result<()> { - if PathBuf::from(DEFAULT_AGG_PK_PATH).exists() - && PathBuf::from(DEFAULT_EVM_HALO2_VERIFIER_PATH) + let default_agg_pk_path = default_agg_pk_path(); + let default_params_dir = default_params_dir(); + let default_evm_halo2_verifier_path = default_evm_halo2_verifier_path(); + + if PathBuf::from(&default_agg_pk_path).exists() + && PathBuf::from(&default_evm_halo2_verifier_path) .join(EVM_HALO2_VERIFIER_PARENT_NAME) .exists() - && PathBuf::from(DEFAULT_EVM_HALO2_VERIFIER_PATH) + && PathBuf::from(&default_evm_halo2_verifier_path) .join(EVM_HALO2_VERIFIER_BASE_NAME) .exists() - && PathBuf::from(DEFAULT_EVM_HALO2_VERIFIER_PATH) + && PathBuf::from(&default_evm_halo2_verifier_path) .join("interfaces") .join(EVM_HALO2_VERIFIER_INTERFACE_NAME) .exists() @@ -49,7 +53,7 @@ impl EvmProvingSetupCmd { } Self::download_params(10, 24).await?; - let params_reader = CacheHalo2ParamsReader::new(DEFAULT_PARAMS_DIR); + let params_reader = CacheHalo2ParamsReader::new(default_params_dir); let agg_config = AggConfig::default(); let sdk = Sdk::new(); @@ -60,10 +64,10 @@ impl EvmProvingSetupCmd { let verifier = sdk.generate_halo2_verifier_solidity(¶ms_reader, &agg_pk)?; println!("Writing proving key to file..."); - write_agg_pk_to_file(agg_pk, DEFAULT_AGG_PK_PATH)?; + write_agg_pk_to_file(agg_pk, &default_agg_pk_path)?; println!("Writing verifier contract to file..."); - write_evm_halo2_verifier_to_folder(verifier, DEFAULT_EVM_HALO2_VERIFIER_PATH)?; + write_evm_halo2_verifier_to_folder(verifier, &default_evm_halo2_verifier_path)?; Ok(()) } @@ -76,7 +80,9 @@ impl EvmProvingSetupCmd { } async fn download_params(min_k: u32, max_k: u32) -> Result<()> { - create_dir_all(DEFAULT_PARAMS_DIR)?; + let default_params_dir = default_params_dir(); + create_dir_all(&default_params_dir)?; + let config = defaults(BehaviorVersion::latest()) .region(Region::new("us-east-1")) .no_credentials() @@ -86,7 +92,7 @@ impl EvmProvingSetupCmd { for k in min_k..=max_k { let file_name = format!("kzg_bn254_{}.srs", k); - let local_file_path = PathBuf::from(DEFAULT_PARAMS_DIR).join(&file_name); + let local_file_path = PathBuf::from(&default_params_dir).join(&file_name); if !local_file_path.exists() { println!("Downloading {}", file_name); let key = format!("challenge_0085/{}", file_name); diff --git a/crates/cli/src/commands/verify.rs b/crates/cli/src/commands/verify.rs index 41a1f397cc..94c7f9e355 100644 --- a/crates/cli/src/commands/verify.rs +++ b/crates/cli/src/commands/verify.rs @@ -47,7 +47,7 @@ impl VerifyCmd { read_evm_halo2_verifier_from_folder, read_evm_proof_from_file, }; - let evm_verifier = read_evm_halo2_verifier_from_folder(DEFAULT_EVM_HALO2_VERIFIER_PATH).map_err(|e| { + let evm_verifier = read_evm_halo2_verifier_from_folder(default_evm_halo2_verifier_path()).map_err(|e| { eyre::eyre!("Failed to read EVM verifier: {}\nPlease run 'cargo openvm evm-proving-setup' first", e) })?; let evm_proof = read_evm_proof_from_file(proof)?; diff --git a/crates/cli/src/default.rs b/crates/cli/src/default.rs index 3d1e9156ed..809427e846 100644 --- a/crates/cli/src/default.rs +++ b/crates/cli/src/default.rs @@ -1,13 +1,10 @@ +use std::env; + use openvm_sdk::config::{AppConfig, SdkVmConfig, DEFAULT_APP_LOG_BLOWUP, DEFAULT_LEAF_LOG_BLOWUP}; use openvm_stark_sdk::config::FriParameters; pub const DEFAULT_MANIFEST_DIR: &str = "."; -pub const DEFAULT_AGG_PK_PATH: &str = concat!(env!("HOME"), "/.openvm/agg.pk"); -pub const DEFAULT_PARAMS_DIR: &str = concat!(env!("HOME"), "/.openvm/params/"); - -pub const DEFAULT_EVM_HALO2_VERIFIER_PATH: &str = concat!(env!("HOME"), "/.openvm/halo2/"); - pub const DEFAULT_APP_CONFIG_PATH: &str = "./openvm.toml"; pub const DEFAULT_APP_EXE_PATH: &str = "./openvm/app.vmexe"; pub const DEFAULT_EXE_COMMIT_PATH: &str = "./openvm/exe_commit.bytes"; @@ -17,6 +14,18 @@ pub const DEFAULT_APP_VK_PATH: &str = "./openvm/app.vk"; pub const DEFAULT_APP_PROOF_PATH: &str = "./openvm/app.proof"; pub const DEFAULT_EVM_PROOF_PATH: &str = "./openvm/evm.proof"; +pub fn default_agg_pk_path() -> String { + env::var("HOME").unwrap() + "/.openvm/agg.pk" +} + +pub fn default_params_dir() -> String { + env::var("HOME").unwrap() + "/.openvm/params/" +} + +pub fn default_evm_halo2_verifier_path() -> String { + env::var("HOME").unwrap() + "/.openvm/halo2/" +} + pub fn default_app_config() -> AppConfig { AppConfig { app_fri_params: FriParameters::standard_with_100_bits_conjectured_security( diff --git a/crates/cli/src/util.rs b/crates/cli/src/util.rs index 19938c64bb..8db24aa89e 100644 --- a/crates/cli/src/util.rs +++ b/crates/cli/src/util.rs @@ -26,3 +26,14 @@ pub fn read_config_toml_or_default(config: &PathBuf) -> Result Result { + current_dir = current_dir.canonicalize()?; + while !current_dir.join("Cargo.toml").exists() { + current_dir = current_dir + .parent() + .expect("Could not find Cargo.toml in current directory or any parent directory") + .to_path_buf(); + } + Ok(current_dir) +} diff --git a/crates/cli/tests/app_e2e.rs b/crates/cli/tests/app_e2e.rs index a2af9cb08f..60bd5b2d68 100644 --- a/crates/cli/tests/app_e2e.rs +++ b/crates/cli/tests/app_e2e.rs @@ -7,20 +7,20 @@ use tempfile::tempdir; fn test_cli_app_e2e() -> Result<()> { let temp_dir = tempdir()?; run_cmd("cargo", &["install", "--path", ".", "--force"])?; - let temp_exe = temp_dir.path().join("example.vmexe"); - let temp_pk = temp_dir.path().join("example.pk"); - let temp_vk = temp_dir.path().join("example.vk"); - let temp_proof = temp_dir.path().join("example.apppf"); + let temp_exe = temp_dir.path().join("fibonacci.vmexe"); + let temp_pk = temp_dir.path().join("fibonacci.pk"); + let temp_vk = temp_dir.path().join("fibonacci.vk"); + let temp_proof = temp_dir.path().join("fibonacci.apppf"); run_cmd( "cargo", &[ "openvm", "build", - "--manifest-dir", - "example", + "--manifest-path", + "tests/programs/fibonacci/Cargo.toml", "--config", - "example/openvm.toml", + "tests/programs/fibonacci/openvm.toml", "--exe-output", temp_exe.to_str().unwrap(), ], @@ -32,7 +32,7 @@ fn test_cli_app_e2e() -> Result<()> { "openvm", "keygen", "--config", - "example/openvm.toml", + "tests/programs/fibonacci/openvm.toml", "--output", temp_pk.to_str().unwrap(), "--vk-output", @@ -48,7 +48,7 @@ fn test_cli_app_e2e() -> Result<()> { "--exe", temp_exe.to_str().unwrap(), "--config", - "example/openvm.toml", + "tests/programs/fibonacci/openvm.toml", ], )?; @@ -86,7 +86,15 @@ fn test_cli_app_e2e() -> Result<()> { #[test] fn test_cli_app_e2e_default_paths() -> Result<()> { run_cmd("cargo", &["install", "--path", ".", "--force"])?; - run_cmd("cargo", &["openvm", "build", "--manifest-dir", "example"])?; + run_cmd( + "cargo", + &[ + "openvm", + "build", + "--manifest-path", + "tests/programs/fibonacci/Cargo.toml", + ], + )?; run_cmd("cargo", &["openvm", "keygen"])?; run_cmd("cargo", &["openvm", "run"])?; run_cmd("cargo", &["openvm", "prove", "app"])?; diff --git a/crates/cli/tests/build.rs b/crates/cli/tests/build.rs new file mode 100644 index 0000000000..1037d07e2d --- /dev/null +++ b/crates/cli/tests/build.rs @@ -0,0 +1,142 @@ +use std::path::PathBuf; + +use cargo_openvm::{ + commands::{build, BuildArgs, BuildCargoArgs}, + default::{DEFAULT_APP_EXE_PATH, DEFAULT_COMMITTED_APP_EXE_PATH, DEFAULT_EXE_COMMIT_PATH}, +}; +use eyre::Result; +use openvm_build::RUSTC_TARGET; +use openvm_circuit::arch::OPENVM_DEFAULT_INIT_FILE_NAME; + +fn default_build_args(example: &str) -> BuildArgs { + BuildArgs { + no_transpile: true, + config: PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("tests") + .join("programs") + .join(example) + .join("openvm.toml"), + exe_output: PathBuf::from(DEFAULT_APP_EXE_PATH), + committed_exe_output: PathBuf::from(DEFAULT_COMMITTED_APP_EXE_PATH), + exe_commit_output: PathBuf::from(DEFAULT_EXE_COMMIT_PATH), + init_file_name: OPENVM_DEFAULT_INIT_FILE_NAME.to_string(), + } +} + +fn default_cargo_args(example: &str) -> BuildCargoArgs { + BuildCargoArgs { + package: vec![], + workspace: false, + exclude: vec![], + lib: false, + bin: vec![], + bins: false, + example: vec![], + examples: false, + all_targets: false, + all_features: false, + no_default_features: false, + features: vec![], + profile: "release".to_string(), + target_dir: None, + verbose: false, + quiet: false, + color: "always".to_string(), + manifest_path: Some( + PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("tests") + .join("programs") + .join(example) + .join("Cargo.toml"), + ), + ignore_rust_version: false, + locked: false, + offline: false, + frozen: false, + } +} + +#[test] +fn test_build_with_profile() -> Result<()> { + let temp_dir = tempfile::tempdir()?; + let target_dir = temp_dir.path(); + + let build_args = default_build_args("fibonacci"); + let mut cargo_args = default_cargo_args("fibonacci"); + cargo_args.target_dir = Some(target_dir.to_path_buf()); + cargo_args.profile = "dev".to_string(); + + build(&build_args, &cargo_args)?; + assert!( + target_dir.join(RUSTC_TARGET).join("debug").exists(), + "did not build with dev profile" + ); + temp_dir.close()?; + Ok(()) +} + +#[test] +fn test_multi_target_build() -> Result<()> { + let temp_dir = tempfile::tempdir()?; + let target_dir = temp_dir.path(); + + let build_args = default_build_args("multi"); + let mut cargo_args = default_cargo_args("multi"); + cargo_args.target_dir = Some(target_dir.to_path_buf()); + + // Build lib + cargo_args.lib = true; + let build_result = build(&build_args, &cargo_args)?; + assert!(build_result.is_empty()); + + // Build bins + cargo_args.lib = false; + let build_result = build(&build_args, &cargo_args)?; + let binary_names: Vec = build_result + .iter() + .map(|path| path.file_stem().unwrap().to_string_lossy().to_string()) + .collect(); + assert!(binary_names.len() == 2); + assert!(binary_names.contains(&"calculator".to_string())); + assert!(binary_names.contains(&"string-utils".to_string())); + + // Build examples + cargo_args.examples = true; + let build_result = build(&build_args, &cargo_args)?; + let example_names: Vec = build_result + .iter() + .map(|path| path.file_stem().unwrap().to_string_lossy().to_string()) + .collect(); + assert!(example_names.len() == 2); + assert!(example_names.contains(&"fibonacci".to_string())); + assert!(example_names.contains(&"palindrome".to_string())); + + // Build examples and a single bin + cargo_args.bin = vec!["calculator".to_string()]; + let build_result = build(&build_args, &cargo_args)?; + let exe_names: Vec = build_result + .iter() + .map(|path| path.file_stem().unwrap().to_string_lossy().to_string()) + .collect(); + assert!(exe_names.len() == 3); + assert!(exe_names.contains(&"calculator".to_string())); + assert!(exe_names.contains(&"fibonacci".to_string())); + assert!(exe_names.contains(&"palindrome".to_string())); + + // Build all targets + cargo_args.bin = vec![]; + cargo_args.examples = false; + cargo_args.all_targets = true; + let build_result = build(&build_args, &cargo_args)?; + let all_names: Vec = build_result + .iter() + .map(|path| path.file_stem().unwrap().to_string_lossy().to_string()) + .collect(); + assert!(all_names.len() == 4); + assert!(all_names.contains(&"calculator".to_string())); + assert!(all_names.contains(&"string-utils".to_string())); + assert!(all_names.contains(&"fibonacci".to_string())); + assert!(all_names.contains(&"palindrome".to_string())); + + Ok(()) +} diff --git a/crates/cli/example/Cargo.toml b/crates/cli/tests/programs/fibonacci/Cargo.toml similarity index 67% rename from crates/cli/example/Cargo.toml rename to crates/cli/tests/programs/fibonacci/Cargo.toml index 140901f1d0..9cbe05c99b 100644 --- a/crates/cli/example/Cargo.toml +++ b/crates/cli/tests/programs/fibonacci/Cargo.toml @@ -5,4 +5,4 @@ version = "0.0.0" edition = "2021" [dependencies] -openvm = { path = "../../toolchain/openvm" } +openvm = { path = "../../../../toolchain/openvm" } diff --git a/crates/cli/example/openvm.toml b/crates/cli/tests/programs/fibonacci/openvm.toml similarity index 100% rename from crates/cli/example/openvm.toml rename to crates/cli/tests/programs/fibonacci/openvm.toml diff --git a/crates/cli/example/src/main.rs b/crates/cli/tests/programs/fibonacci/src/main.rs similarity index 100% rename from crates/cli/example/src/main.rs rename to crates/cli/tests/programs/fibonacci/src/main.rs diff --git a/crates/cli/tests/programs/multi/Cargo.toml b/crates/cli/tests/programs/multi/Cargo.toml new file mode 100644 index 0000000000..c420c46bc2 --- /dev/null +++ b/crates/cli/tests/programs/multi/Cargo.toml @@ -0,0 +1,9 @@ +[workspace] +members = [ + "calculator", + "string-utils" +] + +[workspace.package] +version = "0.0.0" +edition = "2021" diff --git a/crates/cli/tests/programs/multi/calculator/Cargo.toml b/crates/cli/tests/programs/multi/calculator/Cargo.toml new file mode 100644 index 0000000000..1f8543e2ab --- /dev/null +++ b/crates/cli/tests/programs/multi/calculator/Cargo.toml @@ -0,0 +1,19 @@ +[package] +name = "openvm-calculator" +version = "0.0.0" +edition = "2021" + +[dependencies] +openvm = { path = "../../../../../toolchain/openvm" } + +[lib] +name = "calculator" +path = "src/lib.rs" + +[[bin]] +name = "calculator" +path = "src/main.rs" + +[[example]] +name = "fibonacci" +path = "examples/fibonacci.rs" diff --git a/crates/cli/tests/programs/multi/calculator/examples/fibonacci.rs b/crates/cli/tests/programs/multi/calculator/examples/fibonacci.rs new file mode 100644 index 0000000000..5009081f3b --- /dev/null +++ b/crates/cli/tests/programs/multi/calculator/examples/fibonacci.rs @@ -0,0 +1,18 @@ +#![cfg_attr(target_os = "zkvm", no_main)] +#![cfg_attr(target_os = "zkvm", no_std)] + +openvm::entry!(main); + +pub fn main() { + let n = core::hint::black_box(10); + let mut a: u32 = 0; + let mut b: u32 = 1; + for _ in 1..n { + let sum = a + b; + a = b; + b = sum; + } + if a == 0 { + panic!(); + } +} diff --git a/crates/cli/tests/programs/multi/calculator/src/lib.rs b/crates/cli/tests/programs/multi/calculator/src/lib.rs new file mode 100644 index 0000000000..e6ceb3f555 --- /dev/null +++ b/crates/cli/tests/programs/multi/calculator/src/lib.rs @@ -0,0 +1,33 @@ +#![cfg_attr(target_os = "zkvm", no_std)] + +/// Checks if a number is prime. +pub fn is_prime(n: u32) -> bool { + if n <= 1 { + return false; + } + if n <= 3 { + return true; + } + if n % 2 == 0 || n % 3 == 0 { + return false; + } + let mut i = 5; + while i * i <= n { + if n % i == 0 || n % (i + 2) == 0 { + return false; + } + i += 6; + } + true +} + +/// Counts the number of prime numbers up to a given limit. +pub fn count_primes(limit: u32) -> u32 { + let mut count = 0; + for i in 2..limit { + if is_prime(i) { + count += 1; + } + } + count +} diff --git a/crates/cli/tests/programs/multi/calculator/src/main.rs b/crates/cli/tests/programs/multi/calculator/src/main.rs new file mode 100644 index 0000000000..42438497c2 --- /dev/null +++ b/crates/cli/tests/programs/multi/calculator/src/main.rs @@ -0,0 +1,13 @@ +#![cfg_attr(target_os = "zkvm", no_main)] +#![cfg_attr(target_os = "zkvm", no_std)] + +use calculator::count_primes; +openvm::entry!(main); + +pub fn main() { + let n = core::hint::black_box(100); + let count = count_primes(n); + if count == 0 { + panic!(); + } +} diff --git a/crates/cli/tests/programs/multi/openvm.toml b/crates/cli/tests/programs/multi/openvm.toml new file mode 100644 index 0000000000..8ac6a25d95 --- /dev/null +++ b/crates/cli/tests/programs/multi/openvm.toml @@ -0,0 +1,3 @@ +[app_vm_config.rv32i] +[app_vm_config.rv32m] +range_tuple_checker_sizes = [256, 2048] diff --git a/crates/cli/tests/programs/multi/string-utils/Cargo.toml b/crates/cli/tests/programs/multi/string-utils/Cargo.toml new file mode 100644 index 0000000000..245b777912 --- /dev/null +++ b/crates/cli/tests/programs/multi/string-utils/Cargo.toml @@ -0,0 +1,15 @@ +[package] +name = "openvm-string-utils" +version = "0.0.0" +edition = "2021" + +[dependencies] +openvm = { path = "../../../../../toolchain/openvm" } + +[[bin]] +name = "string-utils" +path = "src/main.rs" + +[[example]] +name = "palindrome" +path = "examples/palindrome.rs" diff --git a/crates/cli/tests/programs/multi/string-utils/examples/palindrome.rs b/crates/cli/tests/programs/multi/string-utils/examples/palindrome.rs new file mode 100644 index 0000000000..030d7c76d9 --- /dev/null +++ b/crates/cli/tests/programs/multi/string-utils/examples/palindrome.rs @@ -0,0 +1,25 @@ +#![cfg_attr(target_os = "zkvm", no_main)] +#![cfg_attr(target_os = "zkvm", no_std)] + +extern crate alloc; +use alloc::vec::Vec; + +openvm::entry!(main); + +fn is_palindrome(s: &str) -> bool { + let chars: Vec = s.chars().collect(); + let len = chars.len(); + for i in 0..len / 2 { + if chars[i] != chars[len - 1 - i] { + return false; + } + } + true +} + +pub fn main() { + let text = core::hint::black_box("racecar"); + if !is_palindrome(text) { + panic!(); + } +} diff --git a/crates/cli/tests/programs/multi/string-utils/src/main.rs b/crates/cli/tests/programs/multi/string-utils/src/main.rs new file mode 100644 index 0000000000..fa903446c0 --- /dev/null +++ b/crates/cli/tests/programs/multi/string-utils/src/main.rs @@ -0,0 +1,21 @@ +#![cfg_attr(target_os = "zkvm", no_main)] +#![cfg_attr(target_os = "zkvm", no_std)] + +openvm::entry!(main); + +pub fn main() { + let text = core::hint::black_box("hello world"); + let mut counts = [0u32; 26]; + + for c in text.chars() { + if c.is_ascii_alphabetic() { + let idx = (c.to_ascii_lowercase() as u8 - b'a') as usize; + counts[idx] += 1; + } + } + + let total: u32 = counts.iter().sum(); + if total == 0 { + panic!(); + } +} diff --git a/crates/toolchain/build/src/lib.rs b/crates/toolchain/build/src/lib.rs index 79cf5b2354..3f83430d09 100644 --- a/crates/toolchain/build/src/lib.rs +++ b/crates/toolchain/build/src/lib.rs @@ -66,6 +66,27 @@ pub fn get_package(manifest_dir: impl AsRef) -> Package { matching.pop().unwrap() } +/// Returns all packages from the Cargo.toml manifest at the given `manifest_dir`. +pub fn get_workspace_packages(manifest_dir: impl AsRef) -> Vec { + let manifest_path = fs::canonicalize(manifest_dir.as_ref().join("Cargo.toml")).unwrap(); + let manifest_meta = MetadataCommand::new() + .manifest_path(&manifest_path) + .no_deps() + .exec() + .unwrap_or_else(|e| { + panic!( + "cargo metadata command failed for manifest path: {}: {e:?}", + manifest_path.display() + ) + }); + let packages: Vec = manifest_meta + .packages + .into_iter() + .filter(|pkg: &Package| manifest_meta.workspace_members.contains(&pkg.id)) + .collect(); + packages +} + /// Determines and returns the build target directory from the Cargo manifest at /// the given `manifest_path`. pub fn get_target_dir(manifest_path: impl AsRef) -> PathBuf { @@ -78,6 +99,18 @@ pub fn get_target_dir(manifest_path: impl AsRef) -> PathBuf { .into() } +/// Returns the workspace root directory from the Cargo manifest at +/// the given `manifest_path`. +pub fn get_workspace_root(manifest_path: impl AsRef) -> PathBuf { + MetadataCommand::new() + .manifest_path(manifest_path.as_ref()) + .no_deps() + .exec() + .expect("cargo metadata command failed") + .workspace_root + .into() +} + /// Returns the target executable directory given `target_dir` and `profile`. pub fn get_dir_with_profile( target_dir: impl AsRef, @@ -272,7 +305,41 @@ pub fn build_guest_package( runtime_lib: Option<&str>, target_filter: &Option, ) -> Result> { - if is_skip_build() { + let mut new_opts = guest_opts.clone(); + + if new_opts.target_dir.is_none() { + new_opts.target_dir = Some(get_target_dir(&pkg.manifest_path)); + } + + new_opts.options.extend(vec![ + "--manifest-path".into(), + pkg.manifest_path.to_string(), + ]); + + if let Some(runtime_lib) = runtime_lib { + new_opts.rustc_flags.extend(vec![ + String::from("-C"), + format!("link_arg={}", runtime_lib), + ]); + } + + let mut example = false; + if let Some(target_filter) = target_filter { + new_opts.options.extend(vec![ + format!("--{}", target_filter.kind), + target_filter.name.clone(), + ]); + example = target_filter.kind == "example"; + } + + let res = build_generic(&new_opts); + res.map(|path| if example { path.join("examples") } else { path }) +} + +/// Generic wrapper call to cargo build +pub fn build_generic(guest_opts: &GuestOptions) -> Result> { + if is_skip_build() || guest_opts.target_dir.is_none() { + eprintln!("Skipping build"); return Err(None); } @@ -283,45 +350,16 @@ pub fn build_guest_package( return Err(Some(code)); } - let target_dir = guest_opts - .target_dir - .clone() - .unwrap_or_else(|| get_target_dir(pkg.manifest_path.clone())); - - fs::create_dir_all(&target_dir).unwrap(); - - let runtime_rust_flags = runtime_lib - .map(|lib| vec![String::from("-C"), format!("link_arg={}", lib)]) - .unwrap_or_default(); - let rust_flags: Vec<_> = [ - runtime_rust_flags - .iter() - .map(|s| s.as_str()) - .collect::>(), - guest_opts.rustc_flags.iter().map(|s| s.as_str()).collect(), - ] - .concat(); + let target_dir = guest_opts.target_dir.as_ref().unwrap(); + fs::create_dir_all(target_dir).unwrap(); + let rust_flags: Vec<_> = guest_opts.rustc_flags.iter().map(|s| s.as_str()).collect(); let mut cmd = cargo_command("build", &rust_flags); - let features_str = guest_opts.features.join(","); - if !features_str.is_empty() { - cmd.args(["--features", &features_str]); - } - - cmd.args([ - "--manifest-path", - pkg.manifest_path.as_str(), - "--target-dir", - target_dir.to_str().unwrap(), - ]); - - if let Some(target_filter) = target_filter { - cmd.args([ - format!("--{}", target_filter.kind).as_str(), - target_filter.name.as_str(), - ]); + if !guest_opts.features.is_empty() { + cmd.args(["--features", guest_opts.features.join(",").as_str()]); } + cmd.args(["--target-dir", target_dir.to_str().unwrap()]); let profile = if let Some(profile) = &guest_opts.profile { profile @@ -349,24 +387,17 @@ pub fn build_guest_package( .expect("cargo build failed"); let stderr = child.stderr.take().unwrap(); - tty_println(&format!("{}: Starting build for {RUSTC_TARGET}", pkg.name)); + tty_println(&format!("openvm build: Starting build for {RUSTC_TARGET}")); for line in BufReader::new(stderr).lines() { - tty_println(&format!("{}: {}", pkg.name, line.unwrap())); + tty_println(&format!("openvm build: {}", line.unwrap())); } let res = child.wait().expect("Guest 'cargo build' failed"); if !res.success() { Err(res.code()) } else { - Ok(get_dir_with_profile( - &target_dir, - profile, - target_filter - .as_ref() - .map(|t| t.kind == "example") - .unwrap_or(false), - )) + Ok(get_dir_with_profile(target_dir, profile, false)) } } From 3e34e5d79565dae45a4912c5a9d40f6b03c850b9 Mon Sep 17 00:00:00 2001 From: Xinding Wei Date: Wed, 30 Apr 2025 14:31:39 -0700 Subject: [PATCH 06/15] feat: Add e2e stark proof support (#1597) - Add new type `E2eStarkProof` as the format of the final internal proof. `E2eStarkProof` supports `Encode` so it can be deserialized in other languages. - Add function `wrap_e2e_stark_proof` to wrap the final internal proof to `RootVerifierInput`. - Add `cargo openvm prove stark` to generate e2e stark proof. closes INT-3784 --- crates/cli/src/commands/prove.rs | 52 +++++++- .../src/verifier/internal/types.rs | 49 ++++--- crates/sdk/src/codec.rs | 24 +++- crates/sdk/src/lib.rs | 21 ++- crates/sdk/src/prover/agg.rs | 124 +++++++++++++----- crates/sdk/src/prover/stark.rs | 16 ++- 6 files changed, 222 insertions(+), 64 deletions(-) diff --git a/crates/cli/src/commands/prove.rs b/crates/cli/src/commands/prove.rs index d4b0ac50f7..915830e5b7 100644 --- a/crates/cli/src/commands/prove.rs +++ b/crates/cli/src/commands/prove.rs @@ -2,18 +2,18 @@ use std::{path::PathBuf, sync::Arc}; use clap::Parser; use eyre::Result; +#[cfg(feature = "evm-prove")] +use openvm_sdk::fs::write_evm_proof_to_file; use openvm_sdk::{ commit::AppExecutionCommit, - config::SdkVmConfig, - fs::{read_app_pk_from_file, read_exe_from_file, write_app_proof_to_file}, + config::{AggregationTreeConfig, SdkVmConfig}, + fs::{ + encode_to_file, read_agg_pk_from_file, read_app_pk_from_file, read_exe_from_file, + write_app_proof_to_file, + }, keygen::AppProvingKey, NonRootCommittedExe, Sdk, StdIn, }; -#[cfg(feature = "evm-prove")] -use openvm_sdk::{ - config::AggregationTreeConfig, - fs::{read_agg_pk_from_file, write_evm_proof_to_file}, -}; use crate::{ default::*, @@ -42,6 +42,22 @@ enum ProveSubCommand { #[arg(long, action, help = "Path to output proof", default_value = DEFAULT_APP_PROOF_PATH)] output: PathBuf, }, + Stark { + #[arg(long, action, help = "Path to app proving key", default_value = DEFAULT_APP_PK_PATH)] + app_pk: PathBuf, + + #[arg(long, action, help = "Path to OpenVM executable", default_value = DEFAULT_APP_EXE_PATH)] + exe: PathBuf, + + #[arg(long, value_parser, help = "Input to OpenVM program")] + input: Option, + + #[arg(long, action, help = "Path to output proof", default_value = DEFAULT_APP_PROOF_PATH)] + output: PathBuf, + + #[command(flatten)] + agg_tree_config: AggregationTreeConfig, + }, #[cfg(feature = "evm-prove")] Evm { #[arg(long, action, help = "Path to app proving key", default_value = DEFAULT_APP_PK_PATH)] @@ -76,6 +92,28 @@ impl ProveCmd { let app_proof = sdk.generate_app_proof(app_pk, committed_exe, input)?; write_app_proof_to_file(app_proof, output)?; } + ProveSubCommand::Stark { + app_pk, + exe, + input, + output, + agg_tree_config, + } => { + let mut sdk = sdk; + sdk.set_agg_tree_config(*agg_tree_config); + let (app_pk, committed_exe, input) = + Self::prepare_execution(&sdk, app_pk, exe, input)?; + let agg_pk = read_agg_pk_from_file(DEFAULT_AGG_PK_PATH).map_err(|e| { + eyre::eyre!("Failed to read aggregation proving key: {}\nPlease run 'cargo openvm setup' first", e) + })?; + let stark_proof = sdk.generate_e2e_stark_proof( + app_pk, + committed_exe, + agg_pk.agg_stark_pk, + input, + )?; + encode_to_file(output, stark_proof)?; + } #[cfg(feature = "evm-prove")] ProveSubCommand::Evm { app_pk, diff --git a/crates/continuations/src/verifier/internal/types.rs b/crates/continuations/src/verifier/internal/types.rs index 9512518843..400100f9ae 100644 --- a/crates/continuations/src/verifier/internal/types.rs +++ b/crates/continuations/src/verifier/internal/types.rs @@ -30,6 +30,36 @@ pub struct InternalVmVerifierInput { } assert_impl_all!(InternalVmVerifierInput: Serialize, DeserializeOwned); +/// The final output of the internal VM verifier. +#[derive(Deserialize, Serialize, Derivative)] +#[serde(bound = "")] +#[derivative(Clone(bound = "Com: Clone"))] +pub struct E2eStarkProof { + pub proof: Proof, + pub user_public_values: Vec>, +} +assert_impl_all!(E2eStarkProof: Serialize, DeserializeOwned); + +/// Aggregated state of all segments +#[derive(Debug, Clone, Copy, AlignedBorrow)] +#[repr(C)] +pub struct InternalVmVerifierPvs { + pub vm_verifier_pvs: VmVerifierPvs, + pub extra_pvs: InternalVmVerifierExtraPvs, +} + +/// Extra PVs for internal VM verifier except VmVerifierPvs. +#[derive(Debug, Clone, Copy, AlignedBorrow)] +#[repr(C)] +pub struct InternalVmVerifierExtraPvs { + /// The commitment of the leaf verifier program. + pub leaf_verifier_commit: [T; DIGEST_SIZE], + /// For recursion verification, a program need its own commitment, but its own commitment + /// cannot be hardcoded inside the program itself. So the commitment has to be read from + /// external and be committed. + pub internal_program_commit: [T; DIGEST_SIZE], +} + impl InternalVmVerifierInput { pub fn chunk_leaf_or_internal_proofs( self_program_commit: [Val; DIGEST_SIZE], @@ -45,13 +75,6 @@ impl InternalVmVerifierInput { .collect() } } -/// Aggregated state of all segments -#[derive(Debug, Clone, Copy, AlignedBorrow)] -#[repr(C)] -pub struct InternalVmVerifierPvs { - pub vm_verifier_pvs: VmVerifierPvs, - pub extra_pvs: InternalVmVerifierExtraPvs, -} impl InternalVmVerifierPvs> { pub fn uninit>(builder: &mut Builder) -> Self { @@ -70,18 +93,6 @@ impl InternalVmVerifierPvs> { } } -/// Extra PVs for internal VM verifier except VmVerifierPvs. -#[derive(Debug, Clone, Copy, AlignedBorrow)] -#[repr(C)] -pub struct InternalVmVerifierExtraPvs { - /// The commitment of the leaf verifier program. - pub leaf_verifier_commit: [T; DIGEST_SIZE], - /// For recursion verification, a program need its own commitment, but its own commitment - /// cannot be hardcoded inside the program itself. So the commitment has to be read from - /// external and be committed. - pub internal_program_commit: [T; DIGEST_SIZE], -} - impl InternalVmVerifierExtraPvs> { pub fn uninit>(builder: &mut Builder) -> Self { Self { diff --git a/crates/sdk/src/codec.rs b/crates/sdk/src/codec.rs index c66f254625..5bfabddce9 100644 --- a/crates/sdk/src/codec.rs +++ b/crates/sdk/src/codec.rs @@ -3,7 +3,9 @@ use std::io::{self, Cursor, Read, Result, Write}; use openvm_circuit::{ arch::ContinuationVmProof, system::memory::tree::public_values::UserPublicValuesProof, }; -use openvm_continuations::verifier::root::types::RootVmVerifierInput; +use openvm_continuations::verifier::{ + internal::types::E2eStarkProof, root::types::RootVmVerifierInput, +}; use openvm_native_compiler::ir::DIGEST_SIZE; use openvm_native_recursion::hints::{InnerBatchOpening, InnerFriProof, InnerQueryProof}; use openvm_stark_backend::{ @@ -16,7 +18,7 @@ use openvm_stark_backend::{ }; use p3_fri::CommitPhaseProofStep; -use super::{F, SC}; // BabyBearPoseidon2Config +use super::{F, SC}; type Challenge = BinomialExtensionField; @@ -59,6 +61,13 @@ impl Encode for ContinuationVmProof { } } +impl Encode for E2eStarkProof { + fn encode(&self, writer: &mut W) -> Result<()> { + self.proof.encode(writer)?; + encode_slice(&self.user_public_values, writer) + } +} + impl Encode for UserPublicValuesProof { fn encode(&self, writer: &mut W) -> Result<()> { encode_slice(&self.proof, writer)?; @@ -323,6 +332,17 @@ impl Decode for ContinuationVmProof { } } +impl Decode for E2eStarkProof { + fn decode(reader: &mut R) -> Result { + let proof = Proof::decode(reader)?; + let user_public_values = decode_vec(reader)?; + Ok(Self { + proof, + user_public_values, + }) + } +} + impl Decode for UserPublicValuesProof { fn decode(reader: &mut R) -> Result { let proof = decode_vec(reader)?; diff --git a/crates/sdk/src/lib.rs b/crates/sdk/src/lib.rs index d63bcefdb4..31362ef911 100644 --- a/crates/sdk/src/lib.rs +++ b/crates/sdk/src/lib.rs @@ -20,7 +20,9 @@ use openvm_circuit::{ program::trace::VmCommittedExe, }, }; -use openvm_continuations::verifier::root::types::RootVmVerifierInput; +use openvm_continuations::verifier::{ + internal::types::E2eStarkProof, root::types::RootVmVerifierInput, +}; pub use openvm_continuations::{ static_verifier::{DefaultStaticVerifierPvHandler, StaticVerifierPvHandler}, RootSC, C, F, SC, @@ -276,6 +278,23 @@ impl> GenericSdk { Ok(proof) } + pub fn generate_e2e_stark_proof>( + &self, + app_pk: Arc>, + app_exe: Arc, + agg_stark_pk: AggStarkProvingKey, + inputs: StdIn, + ) -> Result> + where + VC::Executor: Chip, + VC::Periphery: Chip, + { + let stark_prover = + StarkProver::::new(app_pk, app_exe, agg_stark_pk, self.agg_tree_config); + let proof = stark_prover.generate_e2e_stark_proof(inputs); + Ok(proof) + } + #[cfg(feature = "evm-prove")] pub fn generate_evm_proof>( &self, diff --git a/crates/sdk/src/prover/agg.rs b/crates/sdk/src/prover/agg.rs index 42f0d85d58..5bdd3f8d5f 100644 --- a/crates/sdk/src/prover/agg.rs +++ b/crates/sdk/src/prover/agg.rs @@ -2,10 +2,12 @@ use std::sync::Arc; use openvm_circuit::arch::ContinuationVmProof; use openvm_continuations::verifier::{ - internal::types::InternalVmVerifierInput, leaf::types::LeafVmVerifierInput, + internal::types::{E2eStarkProof, InternalVmVerifierInput}, + leaf::types::LeafVmVerifierInput, root::types::RootVmVerifierInput, }; use openvm_native_circuit::NativeConfig; +use openvm_native_compiler::ir::DIGEST_SIZE; use openvm_native_recursion::hints::Hintable; use openvm_stark_sdk::{engine::StarkFriEngine, openvm_stark_backend::proof::Proof}; use tracing::info_span; @@ -83,50 +85,30 @@ impl> AggStarkProver { self.generate_root_proof_impl(root_verifier_input) } + pub fn generate_leaf_proofs(&self, app_proofs: &ContinuationVmProof) -> Vec> { + self.leaf_controller + .generate_proof(&self.leaf_prover, app_proofs) + } + pub fn generate_root_verifier_input( &self, app_proofs: ContinuationVmProof, ) -> RootVmVerifierInput { - let leaf_proofs = self - .leaf_controller - .generate_proof(&self.leaf_prover, &app_proofs); + let leaf_proofs = self.generate_leaf_proofs(&app_proofs); let public_values = app_proofs.user_public_values.public_values; - let internal_proof = self.generate_internal_proof_impl(leaf_proofs, &public_values); - RootVmVerifierInput { - proofs: vec![internal_proof], - public_values, - } + let e2e_stark_proof = self.generate_e2e_stark_proof(leaf_proofs, public_values); + self.wrap_e2e_stark_proof(e2e_stark_proof) } - fn generate_internal_proof_impl( + pub fn generate_e2e_stark_proof( &self, leaf_proofs: Vec>, - public_values: &[F], - ) -> Proof { + public_values: Vec, + ) -> E2eStarkProof { let mut internal_node_idx = -1; let mut internal_node_height = 0; let mut proofs = leaf_proofs; - let mut wrapper_layers = 0; - loop { - if proofs.len() == 1 { - let actual_air_heights = - self.root_prover - .execute_for_air_heights(RootVmVerifierInput { - proofs: vec![proofs[0].clone()], - public_values: public_values.to_vec(), - }); - // Root verifier can handle the internal proof. We can stop here. - if heights_le( - &actual_air_heights, - &self.root_prover.root_verifier_pk.air_heights, - ) { - break; - } - if wrapper_layers >= self.max_internal_wrapper_layers { - panic!("The heights of the root verifier still exceed the required heights after {} wrapper layers", self.max_internal_wrapper_layers); - } - wrapper_layers += 1; - } + while proofs.len() > 1 { let internal_inputs = InternalVmVerifierInput::chunk_leaf_or_internal_proofs( self.internal_prover .committed_exe @@ -158,7 +140,29 @@ impl> AggStarkProver { }); internal_node_height += 1; } - proofs.pop().unwrap() + E2eStarkProof { + proof: proofs.pop().unwrap(), + user_public_values: public_values, + } + } + + /// Wrap the e2e stark proof until its heights meet the requirements of the root verifier. + pub fn wrap_e2e_stark_proof( + &self, + e2e_stark_proof: E2eStarkProof, + ) -> RootVmVerifierInput { + let internal_commit = self + .internal_prover + .committed_exe + .get_program_commit() + .into(); + wrap_e2e_stark_proof( + &self.internal_prover, + &self.root_prover, + internal_commit, + self.max_internal_wrapper_layers, + e2e_stark_proof, + ) } fn generate_root_proof_impl(&self, root_input: RootVmVerifierInput) -> Proof { @@ -204,6 +208,58 @@ impl LeafProvingController { } } +/// Wrap the e2e stark proof until its heights meet the requirements of the root verifier. +pub fn wrap_e2e_stark_proof>( + internal_prover: &VmLocalProver, + root_prover: &RootVerifierLocalProver, + internal_commit: [F; DIGEST_SIZE], + max_internal_wrapper_layers: usize, + e2e_stark_proof: E2eStarkProof, +) -> RootVmVerifierInput { + let E2eStarkProof { + mut proof, + user_public_values, + } = e2e_stark_proof; + let mut wrapper_layers = 0; + loop { + let actual_air_heights = root_prover.execute_for_air_heights(RootVmVerifierInput { + proofs: vec![proof.clone()], + public_values: user_public_values.clone(), + }); + // Root verifier can handle the internal proof. We can stop here. + if heights_le( + &actual_air_heights, + &root_prover.root_verifier_pk.air_heights, + ) { + break; + } + if wrapper_layers >= max_internal_wrapper_layers { + panic!("The heights of the root verifier still exceed the required heights after {} wrapper layers", max_internal_wrapper_layers); + } + wrapper_layers += 1; + let input = InternalVmVerifierInput { + self_program_commit: internal_commit, + proofs: vec![proof.clone()], + }; + proof = info_span!( + "wrapper_layer", + group = format!("internal_wrapper.{wrapper_layers}") + ) + .in_scope(|| { + #[cfg(feature = "bench-metrics")] + { + metrics::counter!("fri.log_blowup") + .absolute(internal_prover.fri_params().log_blowup as u64); + } + SingleSegmentVmProver::prove(internal_prover, input.write()) + }); + } + RootVmVerifierInput { + proofs: vec![proof], + public_values: user_public_values, + } +} + fn heights_le(a: &[usize], b: &[usize]) -> bool { assert_eq!(a.len(), b.len()); a.iter().zip(b.iter()).all(|(a, b)| a <= b) diff --git a/crates/sdk/src/prover/stark.rs b/crates/sdk/src/prover/stark.rs index c95bdc0655..342fe68913 100644 --- a/crates/sdk/src/prover/stark.rs +++ b/crates/sdk/src/prover/stark.rs @@ -1,7 +1,9 @@ use std::sync::Arc; use openvm_circuit::arch::VmConfig; -use openvm_continuations::verifier::root::types::RootVmVerifierInput; +use openvm_continuations::verifier::{ + internal::types::E2eStarkProof, root::types::RootVmVerifierInput, +}; use openvm_stark_backend::{proof::Proof, Chip}; use openvm_stark_sdk::engine::StarkFriEngine; @@ -68,4 +70,16 @@ impl> StarkProver { let app_proof = self.app_prover.generate_app_proof(input); self.agg_prover.generate_root_verifier_input(app_proof) } + + pub fn generate_e2e_stark_proof(&self, input: StdIn) -> E2eStarkProof + where + VC: VmConfig, + VC::Executor: Chip, + VC::Periphery: Chip, + { + let app_proof = self.app_prover.generate_app_proof(input); + let leaf_proofs = self.agg_prover.generate_leaf_proofs(&app_proof); + self.agg_prover + .generate_e2e_stark_proof(leaf_proofs, app_proof.user_public_values.public_values) + } } From a9758582063ccaa3138c049516351addd7301b9a Mon Sep 17 00:00:00 2001 From: Xinding Wei Date: Wed, 30 Apr 2025 14:31:46 -0700 Subject: [PATCH 07/15] feat: Add Rv32HintLoadByKey (#1606) - Add `kv_store` into `Streams`. More details at `docs/specs/ISA.md`. - Add a new phantom instruction `Rv32HintLoadByKey` which can hint data based on a key at runtime. More details at `docs/specs/ISA.md`. - SDK support will be added in the future PRs. close INT-3893 --- Cargo.lock | 2 + crates/toolchain/openvm/src/io/mod.rs | 10 ++++ crates/vm/src/arch/vm.rs | 31 +++++++++- docs/specs/ISA.md | 17 +++--- docs/specs/isa-table.md | 11 ++-- .../native/circuit/src/loadstore/core.rs | 6 +- extensions/rv32im/circuit/src/extension.rs | 60 +++++++++++++++++++ .../rv32im/circuit/src/hintstore/mod.rs | 5 +- extensions/rv32im/guest/Cargo.toml | 3 + extensions/rv32im/guest/src/io.rs | 12 ++++ extensions/rv32im/guest/src/lib.rs | 17 ++++++ extensions/rv32im/tests/Cargo.toml | 1 + .../programs/examples/hint_load_by_key.rs | 32 ++++++++++ extensions/rv32im/tests/src/lib.rs | 28 ++++++++- .../rv32im/transpiler/src/instructions.rs | 2 + extensions/rv32im/transpiler/src/lib.rs | 6 ++ 16 files changed, 225 insertions(+), 18 deletions(-) create mode 100644 extensions/rv32im/tests/programs/examples/hint_load_by_key.rs diff --git a/Cargo.lock b/Cargo.lock index 518f8fde11..fc33da2225 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4755,6 +4755,7 @@ name = "openvm-rv32im-guest" version = "1.1.2" dependencies = [ "openvm-custom-insn", + "p3-field", "strum_macros", ] @@ -4767,6 +4768,7 @@ dependencies = [ "openvm-circuit", "openvm-instructions", "openvm-rv32im-circuit", + "openvm-rv32im-guest", "openvm-rv32im-transpiler", "openvm-stark-sdk", "openvm-toolchain-tests", diff --git a/crates/toolchain/openvm/src/io/mod.rs b/crates/toolchain/openvm/src/io/mod.rs index 0c3b80ccca..1bde9e886f 100644 --- a/crates/toolchain/openvm/src/io/mod.rs +++ b/crates/toolchain/openvm/src/io/mod.rs @@ -56,6 +56,16 @@ fn hint_store_word(ptr: *mut u32) { } } +/// Load hints by key and append into the input stream. +#[allow(unused_variables)] +#[inline(always)] +pub fn hint_load_by_key(key: &[u8]) { + #[cfg(target_os = "zkvm")] + openvm_rv32im_guest::hint_load_by_key(key.as_ptr(), key.len() as u32); + #[cfg(not(target_os = "zkvm"))] + panic!("hint_load_by_key cannot run on non-zkVM platforms"); +} + /// Read the next `len` bytes from the hint stream into a vector. pub(crate) fn read_vec_by_len(len: usize) -> Vec { let num_words = len.div_ceil(4); diff --git a/crates/vm/src/arch/vm.rs b/crates/vm/src/arch/vm.rs index a826fb4137..c9d5cb2ffc 100644 --- a/crates/vm/src/arch/vm.rs +++ b/crates/vm/src/arch/vm.rs @@ -1,4 +1,10 @@ -use std::{borrow::Borrow, collections::VecDeque, marker::PhantomData, mem, sync::Arc}; +use std::{ + borrow::Borrow, + collections::{HashMap, VecDeque}, + marker::PhantomData, + mem, + sync::Arc, +}; use openvm_circuit::system::program::trace::compute_exe_commit; use openvm_instructions::exe::VmExe; @@ -49,11 +55,25 @@ pub enum GenerationError { /// VM memory state for continuations. pub type VmMemoryState = MemoryImage; -#[derive(Clone, Default, Debug)] +/// A trait for key-value store for `Streams`. +pub trait KvStore: Send + Sync { + fn get(&self, key: &[u8]) -> Option<&[u8]>; +} + +impl KvStore for HashMap, Vec> { + fn get(&self, key: &[u8]) -> Option<&[u8]> { + self.get(key).map(|v| v.as_slice()) + } +} + +#[derive(Clone)] pub struct Streams { pub input_stream: VecDeque>, pub hint_stream: VecDeque, pub hint_space: Vec>, + /// The key-value store for hints. Both key and value are byte arrays. Executors which + /// read `kv_store` need to encode the key and decode the value. + pub kv_store: Arc, } impl Streams { @@ -62,10 +82,17 @@ impl Streams { input_stream: input_stream.into(), hint_stream: VecDeque::default(), hint_space: Vec::default(), + kv_store: Arc::new(HashMap::new()), } } } +impl Default for Streams { + fn default() -> Self { + Self::new(VecDeque::default()) + } +} + impl From>> for Streams { fn from(value: VecDeque>) -> Self { Streams::new(value) diff --git a/docs/specs/ISA.md b/docs/specs/ISA.md index 1bc3d0c4a0..d901c6e320 100644 --- a/docs/specs/ISA.md +++ b/docs/specs/ISA.md @@ -171,6 +171,9 @@ structures during runtime execution: - `hint_space`: a vector of vectors of field elements used to store hints during runtime execution via [phantom sub-instructions](#phantom-sub-instructions) such as `NativeHintLoad`. The outer `hint_space` vector is append-only, but each internal `hint_space[hint_id]` vector may be mutated, including deletions, by the host. +- `kv_store`: a read-only key-value store for hints. Executors(e.g. `Rv32HintLoadByKey`) can read data from `kv_store` + at runtime. `kv_store` is designed for general purposes so both key and value are byte arrays. Encoding of key/value + are decided by each executor. Users need to use the corresponding encoding when adding data to `kv_store`. These data structures are **not** part of the guest state, and their state depends on host behavior that cannot be determined by the guest. @@ -204,7 +207,7 @@ which must satisfy the following conditions: - The execution has full read/write access to the data memory, except address space `0` must be read-only. - User public outputs can be set at any index in `[0, num_public_values)`. If continuations are disabled, a public value cannot be overwritten with a different value once it is set. -- Input stream can only be popped from the front as a queue. Appends are not allowed. +- Input stream can only be popped from the front as a queue. - Full read/write access to the hint stream. - Hint spaces can be read from at any index. Hint spaces may be mutated only by append. - The program counter is set to a new `to_pc` at the end of the instruction execution. @@ -426,12 +429,12 @@ with user input-output. The RV32IM extension defines the following phantom sub-instructions. -| Name | Discriminant | Operands | Description | -| -------------- | ------------ | -------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| Rv32HintInput | 0x20 | `_` | Pops a vector `hint` of field elements from the input stream and resets the hint stream to equal the vector `[(hint.len() as u32).to_le_bytes()), hint].concat()`. | -| Rv32PrintStr | 0x21 | `a,b,_` | Peeks at `[r32{0}(a)..r32{0}(a) + r32{0}(b)]_2`, tries to convert to byte array and then UTF-8 string and prints to host stdout. Prints error message if conversion fails. Does not change any VM state. | -| Rv32HintRandom | 0x22 | `a,_,_` | Resets the hint stream to `4 * r32{0}(a)` random bytes. The source of randomness is the host operating system (`rand::rngs::OsRng`). Its result is not constrained in any way. | - +| Name | Discriminant | Operands | Description | +|-------------------| ------------ | -------- |--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| Rv32HintInput | 0x20 | `_` | Pops a vector `hint` of field elements from the input stream and resets the hint stream to equal the vector `[(hint.len() as u32).to_le_bytes()), hint].concat()`. | +| Rv32PrintStr | 0x21 | `a,b,_` | Peeks at `[r32{0}(a)..r32{0}(a) + r32{0}(b)]_2`, tries to convert to byte array and then UTF-8 string and prints to host stdout. Prints error message if conversion fails. Does not change any VM state. | +| Rv32HintRandom | 0x22 | `a,_,_` | Resets the hint stream to `4 * r32{0}(a)` random bytes. The source of randomness is the host operating system (`rand::rngs::OsRng`). Its result is not constrained in any way. | +| Rv32HintLoadByKey | 0x23 | `a,b,_` | Look up the value by key `[r32{0}{a}:r32{0}{b}]_2` and prepend the value into `input_stream`. Users should use `openvm-rv32im-guest::hint_load_by_key_encode` to encode the value when constructing inputs. | ### Native Extension The native extension operates over native field elements and has instructions tailored for STARK proof recursion. It diff --git a/docs/specs/isa-table.md b/docs/specs/isa-table.md index 75cf1bad2c..940d90ed0e 100644 --- a/docs/specs/isa-table.md +++ b/docs/specs/isa-table.md @@ -75,11 +75,12 @@ In the tables below, we provide the mapping between the `LocalOpcode` and `Phant #### Phantom Sub-Instructions -| VM Extension | `PhantomDiscriminant` | ISA Phantom Sub-Instruction | -| ------------- | ---------- | ------------- | -| RV32IM | `Rv32Phantom::HintInput` | Rv32HintInput | -| RV32IM | `Rv32Phantom::PrintStr` | Rv32PrintStr | -| RV32IM | `Rv32Phantom::HintRandom` | Rv32HintRandom | +| VM Extension | `PhantomDiscriminant` | ISA Phantom Sub-Instruction | +| ------------- |-------------------------------| ------------- | +| RV32IM | `Rv32Phantom::HintInput` | Rv32HintInput | +| RV32IM | `Rv32Phantom::PrintStr` | Rv32PrintStr | +| RV32IM | `Rv32Phantom::HintRandom` | Rv32HintRandom | +| RV32IM | `Rv32Phantom::HintLoadByKey` | Rv32HintLoadByKey | ## Native Extension diff --git a/extensions/native/circuit/src/loadstore/core.rs b/extensions/native/circuit/src/loadstore/core.rs index 60c7bbdbdb..094a57dccc 100644 --- a/extensions/native/circuit/src/loadstore/core.rs +++ b/extensions/native/circuit/src/loadstore/core.rs @@ -113,7 +113,6 @@ where } } -#[derive(Debug)] pub struct NativeLoadStoreCoreChip { pub air: NativeLoadStoreCoreAir, pub streams: OnceLock>>>, @@ -127,7 +126,10 @@ impl NativeLoadStoreCoreChip { } } pub fn set_streams(&mut self, streams: Arc>>) { - self.streams.set(streams).unwrap(); + self.streams + .set(streams) + .map_err(|_| "streams have already been set.") + .unwrap(); } } diff --git a/extensions/rv32im/circuit/src/extension.rs b/extensions/rv32im/circuit/src/extension.rs index 6760df923b..f8dd2fbf54 100644 --- a/extensions/rv32im/circuit/src/extension.rs +++ b/extensions/rv32im/circuit/src/extension.rs @@ -359,6 +359,10 @@ impl VmExtension for Rv32I { phantom::Rv32PrintStrSubEx, PhantomDiscriminant(Rv32Phantom::PrintStr as u16), )?; + builder.add_phantom_sub_executor( + phantom::Rv32HintLoadByKeySubEx, + PhantomDiscriminant(Rv32Phantom::HintLoadByKey as u16), + )?; Ok(inventory) } @@ -511,6 +515,7 @@ mod phantom { } } pub struct Rv32PrintStrSubEx; + pub struct Rv32HintLoadByKeySubEx; impl PhantomSubExecutor for Rv32HintInputSubEx { fn phantom_execute( @@ -586,4 +591,59 @@ mod phantom { Ok(()) } } + + impl PhantomSubExecutor for Rv32HintLoadByKeySubEx { + fn phantom_execute( + &mut self, + memory: &MemoryController, + streams: &mut Streams, + _: PhantomDiscriminant, + a: F, + b: F, + _: u16, + ) -> eyre::Result<()> { + let ptr = unsafe_read_rv32_register(memory, a); + let len = unsafe_read_rv32_register(memory, b); + let key: Vec = (0..len) + .map(|i| { + memory + .unsafe_read_cell(F::TWO, F::from_canonical_u32(ptr + i)) + .as_canonical_u32() as u8 + }) + .collect(); + if let Some(val) = streams.kv_store.get(&key) { + let to_push = hint_load_by_key_decode::(val); + for input in to_push.into_iter().rev() { + streams.input_stream.push_front(input); + } + } else { + bail!("Rv32HintLoadByKey: key not found"); + } + Ok(()) + } + } + + pub fn hint_load_by_key_decode(value: &[u8]) -> Vec> { + let mut offset = 0; + let len = extract_u32(value, offset) as usize; + offset += 4; + let mut ret = Vec::with_capacity(len); + for _ in 0..len { + let v_len = extract_u32(value, offset) as usize; + offset += 4; + let v = (0..v_len) + .map(|_| { + let ret = F::from_canonical_u32(extract_u32(value, offset)); + offset += 4; + ret + }) + .collect(); + ret.push(v); + } + ret + } + + fn extract_u32(value: &[u8], offset: usize) -> u32 { + u32::from_le_bytes(value[offset..offset + 4].try_into().unwrap()) + } } diff --git a/extensions/rv32im/circuit/src/hintstore/mod.rs b/extensions/rv32im/circuit/src/hintstore/mod.rs index 6f70a584d0..d566292207 100644 --- a/extensions/rv32im/circuit/src/hintstore/mod.rs +++ b/extensions/rv32im/circuit/src/hintstore/mod.rs @@ -313,7 +313,10 @@ impl Rv32HintStoreChip { } } pub fn set_streams(&mut self, streams: Arc>>) { - self.streams.set(streams).unwrap(); + self.streams + .set(streams) + .map_err(|_| "streams have already been set.") + .unwrap(); } } diff --git a/extensions/rv32im/guest/Cargo.toml b/extensions/rv32im/guest/Cargo.toml index 75ed32ec1e..d98173bcdf 100644 --- a/extensions/rv32im/guest/Cargo.toml +++ b/extensions/rv32im/guest/Cargo.toml @@ -11,5 +11,8 @@ repository.workspace = true openvm-custom-insn = { workspace = true } strum_macros = { workspace = true } +[target.'cfg(not(target_os = "zkvm"))'.dependencies] +p3-field = { workspace = true } + [features] default = [] diff --git a/extensions/rv32im/guest/src/io.rs b/extensions/rv32im/guest/src/io.rs index 5f70533f1b..ee123c0577 100644 --- a/extensions/rv32im/guest/src/io.rs +++ b/extensions/rv32im/guest/src/io.rs @@ -55,6 +55,18 @@ pub fn hint_random(len: usize) { ); } +/// Hint the VM to load values with key = [ptr: len] into input streams. +#[inline(always)] +pub fn hint_load_by_key(ptr: *const u8, len: u32) { + openvm_custom_insn::custom_insn_i!( + opcode = SYSTEM_OPCODE, + funct3 = PHANTOM_FUNCT3, + rd = In ptr, + rs1 = In len, + imm = Const PhantomImm::HintLoadByKey as u16, + ); +} + /// Store rs1 to [[rd] + imm]_3. #[macro_export] macro_rules! reveal { diff --git a/extensions/rv32im/guest/src/lib.rs b/extensions/rv32im/guest/src/lib.rs index d4c860b822..5a6b7087e4 100644 --- a/extensions/rv32im/guest/src/lib.rs +++ b/extensions/rv32im/guest/src/lib.rs @@ -1,8 +1,10 @@ #![no_std] +extern crate alloc; /// Library functions for user input/output. #[cfg(target_os = "zkvm")] mod io; + #[cfg(target_os = "zkvm")] pub use io::*; use strum_macros::FromRepr; @@ -28,4 +30,19 @@ pub enum PhantomImm { HintInput = 0, PrintStr, HintRandom, + HintLoadByKey, +} + +/// Encode a 2d-array of field elements into bytes for `hint_load_by_key` +#[cfg(not(target_os = "zkvm"))] +pub fn hint_load_by_key_encode( + value: &[alloc::vec::Vec], +) -> alloc::vec::Vec { + let len = value.len(); + let mut ret = (len as u32).to_le_bytes().to_vec(); + for v in value { + ret.extend((v.len() as u32).to_le_bytes()); + ret.extend(v.iter().flat_map(|x| x.as_canonical_u32().to_le_bytes())); + } + ret } diff --git a/extensions/rv32im/tests/Cargo.toml b/extensions/rv32im/tests/Cargo.toml index 21282f9b2f..2e68359532 100644 --- a/extensions/rv32im/tests/Cargo.toml +++ b/extensions/rv32im/tests/Cargo.toml @@ -13,6 +13,7 @@ openvm-stark-sdk.workspace = true openvm-circuit = { workspace = true, features = ["test-utils"] } openvm-transpiler.workspace = true openvm-rv32im-circuit.workspace = true +openvm-rv32im-guest.workspace = true openvm-rv32im-transpiler.workspace = true openvm = { workspace = true } openvm-toolchain-tests = { path = "../../../crates/toolchain/tests" } diff --git a/extensions/rv32im/tests/programs/examples/hint_load_by_key.rs b/extensions/rv32im/tests/programs/examples/hint_load_by_key.rs new file mode 100644 index 0000000000..c770ac2268 --- /dev/null +++ b/extensions/rv32im/tests/programs/examples/hint_load_by_key.rs @@ -0,0 +1,32 @@ +#![cfg_attr(not(feature = "std"), no_main)] +#![cfg_attr(not(feature = "std"), no_std)] +use openvm::io::{hint_load_by_key, read_vec}; + +openvm::entry!(main); + +pub fn main() { + const KEY: &str = "key"; + hint_load_by_key(KEY.as_bytes()); + + let vec = read_vec(); + // assert_eq!(vec.len(), 4); + if vec.len() != 4 { + openvm::process::panic(); + } + #[allow(clippy::needless_range_loop)] + for i in 0..4 { + if vec[i] != i as u8 { + openvm::process::panic(); + } + } + let vec = read_vec(); + if vec.len() != 3 { + openvm::process::panic(); + } + #[allow(clippy::needless_range_loop)] + for i in 0..3 { + if vec[i] != i as u8 { + openvm::process::panic(); + } + } +} diff --git a/extensions/rv32im/tests/src/lib.rs b/extensions/rv32im/tests/src/lib.rs index 7799c7d971..f07a6c43a2 100644 --- a/extensions/rv32im/tests/src/lib.rs +++ b/extensions/rv32im/tests/src/lib.rs @@ -1,13 +1,16 @@ #[cfg(test)] mod tests { + use std::{collections::HashMap, sync::Arc}; + use eyre::Result; use openvm_circuit::{ - arch::{hasher::poseidon2::vm_poseidon2_hasher, ExecutionError, VmExecutor}, + arch::{hasher::poseidon2::vm_poseidon2_hasher, ExecutionError, Streams, VmExecutor}, system::memory::tree::public_values::UserPublicValuesProof, utils::{air_test, air_test_with_min_segments}, }; use openvm_instructions::exe::VmExe; use openvm_rv32im_circuit::{Rv32IConfig, Rv32ImConfig}; + use openvm_rv32im_guest::hint_load_by_key_encode; use openvm_rv32im_transpiler::{ Rv32ITranspilerExtension, Rv32IoTranspilerExtension, Rv32MTranspilerExtension, }; @@ -88,6 +91,29 @@ mod tests { Ok(()) } + #[test] + fn test_hint_load_by_key() -> Result<()> { + let elf = build_example_program_at_path(get_programs_dir!(), "hint_load_by_key")?; + let exe = VmExe::from_elf( + elf, + Transpiler::::default() + .with_extension(Rv32ITranspilerExtension) + .with_extension(Rv32MTranspilerExtension) + .with_extension(Rv32IoTranspilerExtension), + )?; + let config = Rv32IConfig::default(); + // stdin will be read after reading kv_store + let stdin = vec![[0, 1, 2].map(F::from_canonical_u8).to_vec()]; + let mut streams: Streams = stdin.into(); + let input = vec![[0, 1, 2, 3].map(F::from_canonical_u8).to_vec()]; + streams.kv_store = Arc::new(HashMap::from([( + "key".as_bytes().to_vec(), + hint_load_by_key_encode(&input), + )])); + air_test_with_min_segments(config, exe, streams, 1); + Ok(()) + } + #[test] fn test_read() -> Result<()> { let config = Rv32IConfig::default(); diff --git a/extensions/rv32im/transpiler/src/instructions.rs b/extensions/rv32im/transpiler/src/instructions.rs index 0cd013a8ba..38c3b42419 100644 --- a/extensions/rv32im/transpiler/src/instructions.rs +++ b/extensions/rv32im/transpiler/src/instructions.rs @@ -279,4 +279,6 @@ pub enum Rv32Phantom { PrintStr, /// Prepare given amount of random numbers for hinting. HintRandom, + /// Hint the VM to load values from the stream KV store into input streams. + HintLoadByKey, } diff --git a/extensions/rv32im/transpiler/src/lib.rs b/extensions/rv32im/transpiler/src/lib.rs index 445ef9f43e..42b5f9d2e2 100644 --- a/extensions/rv32im/transpiler/src/lib.rs +++ b/extensions/rv32im/transpiler/src/lib.rs @@ -91,6 +91,12 @@ impl TranspilerExtension for Rv32ITranspilerExtension { F::from_canonical_usize(RV32_REGISTER_NUM_LIMBS * dec_insn.rs1), 0, ), + PhantomImm::HintLoadByKey => Instruction::phantom( + PhantomDiscriminant(Rv32Phantom::HintLoadByKey as u16), + F::from_canonical_usize(RV32_REGISTER_NUM_LIMBS * dec_insn.rd), + F::from_canonical_usize(RV32_REGISTER_NUM_LIMBS * dec_insn.rs1), + 0, + ), }) } (RV32_ALU_OPCODE, _) => { From c8fc3471fc23fb724616207fb012ae62e4db1b58 Mon Sep 17 00:00:00 2001 From: Xinding Wei Date: Mon, 5 May 2025 14:18:37 -0700 Subject: [PATCH 08/15] feat: Root Verifier ASM (#1615) - Add a function `build_kernel_asm` into `RootVmVerifierConfig`. The generated ASM can be used in kernel functions for guest program to verify starks. More details could be found in the doc comments of the function. - Fix a confusing naming. `num_public_values` is actually the number of user public values instead of the number of public values of the proof. Use `num_user_public_values` instead. closes INT-3894 --- Cargo.lock | 1 + crates/cli/src/commands/prove.rs | 2 +- crates/cli/src/commands/setup.rs | 10 +- crates/cli/src/default.rs | 4 + .../src/verifier/internal/mod.rs | 2 +- .../src/verifier/internal/vars.rs | 32 ++- crates/continuations/src/verifier/root/mod.rs | 213 +++++++++++++----- crates/sdk/Cargo.toml | 1 + crates/sdk/src/keygen/asm.rs | 106 +++++++++ crates/sdk/src/keygen/mod.rs | 5 +- crates/sdk/src/lib.rs | 23 +- crates/sdk/src/prover/stark.rs | 2 +- crates/sdk/tests/integration_test.rs | 10 +- .../native/compiler/src/asm/compiler.rs | 2 +- extensions/rv32im/tests/src/lib.rs | 4 +- 15 files changed, 343 insertions(+), 74 deletions(-) create mode 100644 crates/sdk/src/keygen/asm.rs diff --git a/Cargo.lock b/Cargo.lock index fc33da2225..3cb6316d13 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4834,6 +4834,7 @@ dependencies = [ "openvm-stark-sdk", "openvm-transpiler", "p3-fri", + "rrs-lib", "serde", "serde_json", "serde_with", diff --git a/crates/cli/src/commands/prove.rs b/crates/cli/src/commands/prove.rs index 915830e5b7..143d34b779 100644 --- a/crates/cli/src/commands/prove.rs +++ b/crates/cli/src/commands/prove.rs @@ -103,7 +103,7 @@ impl ProveCmd { sdk.set_agg_tree_config(*agg_tree_config); let (app_pk, committed_exe, input) = Self::prepare_execution(&sdk, app_pk, exe, input)?; - let agg_pk = read_agg_pk_from_file(DEFAULT_AGG_PK_PATH).map_err(|e| { + let agg_pk = read_agg_pk_from_file(default_agg_pk_path()).map_err(|e| { eyre::eyre!("Failed to read aggregation proving key: {}\nPlease run 'cargo openvm setup' first", e) })?; let stark_proof = sdk.generate_e2e_stark_proof( diff --git a/crates/cli/src/commands/setup.rs b/crates/cli/src/commands/setup.rs index 079d4b65e3..c617112707 100644 --- a/crates/cli/src/commands/setup.rs +++ b/crates/cli/src/commands/setup.rs @@ -17,7 +17,9 @@ use openvm_sdk::{ DefaultStaticVerifierPvHandler, Sdk, }; -use crate::default::{default_agg_pk_path, default_evm_halo2_verifier_path, default_params_dir}; +use crate::default::{ + default_agg_pk_path, default_asm_path, default_evm_halo2_verifier_path, default_params_dir, +}; #[derive(Parser)] #[command( @@ -60,12 +62,18 @@ impl EvmProvingSetupCmd { println!("Generating proving key..."); let agg_pk = sdk.agg_keygen(agg_config, ¶ms_reader, &DefaultStaticVerifierPvHandler)?; + println!("Generating root verifier ASM..."); + let root_verifier_asm = sdk.generate_root_verifier_asm(&agg_pk.agg_stark_pk); + println!("Generating verifier contract..."); let verifier = sdk.generate_halo2_verifier_solidity(¶ms_reader, &agg_pk)?; println!("Writing proving key to file..."); write_agg_pk_to_file(agg_pk, &default_agg_pk_path)?; + println!("Writing root verifier ASM to file..."); + write(default_asm_path(), root_verifier_asm)?; + println!("Writing verifier contract to file..."); write_evm_halo2_verifier_to_folder(verifier, &default_evm_halo2_verifier_path)?; diff --git a/crates/cli/src/default.rs b/crates/cli/src/default.rs index 809427e846..224d3dc219 100644 --- a/crates/cli/src/default.rs +++ b/crates/cli/src/default.rs @@ -18,6 +18,10 @@ pub fn default_agg_pk_path() -> String { env::var("HOME").unwrap() + "/.openvm/agg.pk" } +pub fn default_asm_path() -> String { + env::var("HOME").unwrap() + "/.openvm/root.asm" +} + pub fn default_params_dir() -> String { env::var("HOME").unwrap() + "/.openvm/params/" } diff --git a/crates/continuations/src/verifier/internal/mod.rs b/crates/continuations/src/verifier/internal/mod.rs index 8775e9e89c..eb444d0291 100644 --- a/crates/continuations/src/verifier/internal/mod.rs +++ b/crates/continuations/src/verifier/internal/mod.rs @@ -21,7 +21,7 @@ use crate::{ }; pub mod types; -mod vars; +pub mod vars; /// Config to generate internal VM verifier program. pub struct InternalVmVerifierConfig { diff --git a/crates/continuations/src/verifier/internal/vars.rs b/crates/continuations/src/verifier/internal/vars.rs index e700d92248..0e6050591d 100644 --- a/crates/continuations/src/verifier/internal/vars.rs +++ b/crates/continuations/src/verifier/internal/vars.rs @@ -5,8 +5,11 @@ use openvm_native_recursion::{hints::Hintable, vars::StarkProofVariable}; use openvm_stark_sdk::openvm_stark_backend::proof::Proof; use crate::{ - verifier::{internal::types::InternalVmVerifierInput, utils::write_field_slice}, - C, SC, + verifier::{ + internal::types::{E2eStarkProof, InternalVmVerifierInput}, + utils::write_field_slice, + }, + C, F, SC, }; #[derive(DslVariable, Clone)] @@ -16,6 +19,12 @@ pub struct InternalVmVerifierInputVariable { pub proofs: Array>, } +#[derive(DslVariable, Clone)] +pub struct E2eStarkProofVariable { + pub proof: StarkProofVariable, + pub user_public_values: Array>, +} + impl Hintable for InternalVmVerifierInput { type HintVariable = InternalVmVerifierInputVariable; @@ -34,3 +43,22 @@ impl Hintable for InternalVmVerifierInput { stream } } + +impl Hintable for E2eStarkProof { + type HintVariable = E2eStarkProofVariable; + + fn read(builder: &mut Builder) -> Self::HintVariable { + let proof = Proof::::read(builder); + let user_public_values = Vec::::read(builder); + Self::HintVariable { + proof, + user_public_values, + } + } + + fn write(&self) -> Vec::N>> { + let mut stream = self.proof.write(); + stream.extend(self.user_public_values.write()); + stream + } +} diff --git a/crates/continuations/src/verifier/root/mod.rs b/crates/continuations/src/verifier/root/mod.rs index 666f2ca969..b6f2816b1e 100644 --- a/crates/continuations/src/verifier/root/mod.rs +++ b/crates/continuations/src/verifier/root/mod.rs @@ -1,11 +1,12 @@ use std::array; use openvm_circuit::arch::instructions::program::Program; -use openvm_native_compiler::{conversion::CompilerOptions, prelude::*}; +use openvm_native_compiler::{asm::HEAP_START_ADDRESS, conversion::CompilerOptions, prelude::*}; use openvm_native_recursion::{ fri::TwoAdicFriPcsVariable, hints::Hintable, types::new_from_inner_multi_vk, utils::const_fri_config, }; +use openvm_stark_backend::proof::Proof; use openvm_stark_sdk::{ config::FriParameters, openvm_stark_backend::{keygen::types::MultiStarkVerifyingKey, p3_field::FieldAlgebra}, @@ -30,7 +31,7 @@ mod vars; pub struct RootVmVerifierConfig { pub leaf_fri_params: FriParameters, pub internal_fri_params: FriParameters, - pub num_public_values: usize, + pub num_user_public_values: usize, pub internal_vm_verifier_commit: [F; DIGEST_SIZE], pub compiler_options: CompilerOptions, } @@ -40,73 +41,163 @@ impl RootVmVerifierConfig { leaf_vm_vk: &MultiStarkVerifyingKey, internal_vm_vk: &MultiStarkVerifyingKey, ) -> Program { - let leaf_advice = new_from_inner_multi_vk(leaf_vm_vk); - let internal_advice = new_from_inner_multi_vk(internal_vm_vk); let mut builder = Builder::::default(); - { - builder.cycle_tracker_start("ReadProofsFromInput"); - let RootVmVerifierInputVariable { + builder.cycle_tracker_start("ReadProofsFromInput"); + let root_verifier_input = RootVmVerifierInput::::read(&mut builder); + builder.cycle_tracker_end("ReadProofsFromInput"); + let pvs = self.verifier_impl( + &mut builder, + leaf_vm_vk, + internal_vm_vk, + root_verifier_input, + ); + pvs.flatten() + .into_iter() + .for_each(|v| builder.commit_public_value(v)); + builder.halt(); + builder.compile_isa_with_options(self.compiler_options) + } + + /// Build instructions which can be called as a kernel function in RISC-V guest programs. + /// Inputs for generated instructions: + /// - expected `app_exe_commit`, `app_vm_commit` and user public values should be stored from + /// `HEAP_START_ADDRESS` in the native address space . + /// + /// These instructions take a proof from the input stream and verify the proof. Then these + /// instructions check if the public values are consistent with the expected public values + /// from RISC-V guest programs. + pub fn build_kernel_asm( + &self, + leaf_vm_vk: &MultiStarkVerifyingKey, + internal_vm_vk: &MultiStarkVerifyingKey, + ) -> Program { + let mut builder = Builder::::default(); + + const BYTE_PER_WORD: usize = 4; + let num_public_values = self.num_user_public_values + DIGEST_SIZE * 2; + let num_bytes = num_public_values * BYTE_PER_WORD; + // Move heap pointer in order to keep input arguments from address space 2. + let heap_addr: Var = builder.eval(F::from_canonical_u32( + HEAP_START_ADDRESS as u32 + num_bytes as u32, + )); + builder.store_heap_ptr(Ptr { address: heap_addr }); + let expected_pvs: Vec> = (0..num_public_values) + .map(|i| { + let fs: [Felt<_>; BYTE_PER_WORD] = array::from_fn(|j| { + let ptr = Ptr { + address: builder.eval(F::from_canonical_u32( + HEAP_START_ADDRESS as u32 + (i * 4) as u32, + )), + }; + let idx = MemIndex { + index: RVar::from(j), + offset: 0, + size: 1, + }; + let f = Felt::uninit(&mut builder); + f.load(ptr, idx, &mut builder); + f + }); + builder.eval( + fs[0] + + fs[1] * F::from_canonical_u32(1 << 8) + + fs[2] * F::from_canonical_u32(1 << 16) + + fs[3] * F::from_canonical_u32(1 << 24), + ) + }) + .collect(); + let expected_pvs = RootVmVerifierPvs::>::from_flatten(expected_pvs); + let user_pvs = builder.array(self.num_user_public_values); + for (i, &pv) in expected_pvs.public_values.iter().enumerate() { + builder.set(&user_pvs, i, pv); + } + + builder.cycle_tracker_start("ReadFromStdin"); + let proof = Proof::::read(&mut builder); + builder.cycle_tracker_end("ReadFromStdin"); + let proofs = builder.array(1); + builder.set(&proofs, 0, proof); + let pvs = self.verifier_impl( + &mut builder, + leaf_vm_vk, + internal_vm_vk, + RootVmVerifierInputVariable { proofs, - public_values, - } = RootVmVerifierInput::::read(&mut builder); - builder.cycle_tracker_end("ReadProofsFromInput"); - builder.cycle_tracker_start("InitializePcsConst"); - let leaf_pcs = TwoAdicFriPcsVariable { - config: const_fri_config(&mut builder, &self.leaf_fri_params), - }; - let internal_pcs = TwoAdicFriPcsVariable { - config: const_fri_config(&mut builder, &self.internal_fri_params), - }; - builder.cycle_tracker_end("InitializePcsConst"); - builder.cycle_tracker_start("VerifyProofs"); - let internal_program_commit = - array::from_fn(|i| builder.eval(self.internal_vm_verifier_commit[i])); - let non_leaf_verifier = NonLeafVerifierVariables { - internal_program_commit, - leaf_pcs, - leaf_advice, - internal_pcs, - internal_advice, - }; - let (merged_pvs, expected_leaf_commit) = - non_leaf_verifier.verify_internal_or_leaf_verifier_proofs(&mut builder, &proofs); - builder.cycle_tracker_end("VerifyProofs"); + public_values: user_pvs, + }, + ); + builder.assert_eq::<[Felt<_>; DIGEST_SIZE]>(pvs.exe_commit, expected_pvs.exe_commit); + builder.assert_eq::<[Felt<_>; DIGEST_SIZE]>( + pvs.leaf_verifier_commit, + expected_pvs.leaf_verifier_commit, + ); + + builder.compile_isa_with_options(self.compiler_options) + } + + fn verifier_impl( + &self, + builder: &mut Builder, + leaf_vm_vk: &MultiStarkVerifyingKey, + internal_vm_vk: &MultiStarkVerifyingKey, + root_verifier_input: RootVmVerifierInputVariable, + ) -> RootVmVerifierPvs> { + let leaf_advice = new_from_inner_multi_vk(leaf_vm_vk); + let internal_advice = new_from_inner_multi_vk(internal_vm_vk); + let RootVmVerifierInputVariable { + proofs, + public_values, + } = root_verifier_input; - // App Program should terminate - builder.assert_felt_eq(merged_pvs.connector.is_terminate, F::ONE); - // App Program should exit successfully - builder.assert_felt_eq(merged_pvs.connector.exit_code, F::ZERO); + builder.cycle_tracker_start("InitializePcsConst"); + let leaf_pcs = TwoAdicFriPcsVariable { + config: const_fri_config(builder, &self.leaf_fri_params), + }; + let internal_pcs = TwoAdicFriPcsVariable { + config: const_fri_config(builder, &self.internal_fri_params), + }; + builder.cycle_tracker_end("InitializePcsConst"); + builder.cycle_tracker_start("VerifyProofs"); + let internal_program_commit = + array::from_fn(|i| builder.eval(self.internal_vm_verifier_commit[i])); + let non_leaf_verifier = NonLeafVerifierVariables { + internal_program_commit, + leaf_pcs, + leaf_advice, + internal_pcs, + internal_advice, + }; + let (merged_pvs, expected_leaf_commit) = + non_leaf_verifier.verify_internal_or_leaf_verifier_proofs(builder, &proofs); + builder.cycle_tracker_end("VerifyProofs"); - builder.cycle_tracker_start("ExtractPublicValues"); - builder.assert_usize_eq(public_values.len(), RVar::from(self.num_public_values)); - let public_values_vec: Vec> = (0..self.num_public_values) - .map(|i| builder.get(&public_values, i)) - .collect(); - let hasher = VariableP2Hasher::new(&mut builder); - let pv_commit = hasher.merkle_root(&mut builder, &public_values_vec); - builder.assert_eq::<[_; DIGEST_SIZE]>(merged_pvs.public_values_commit, pv_commit); - builder.cycle_tracker_end("ExtractPublicValues"); + // App Program should terminate + builder.assert_felt_eq(merged_pvs.connector.is_terminate, F::ONE); + // App Program should exit successfully + builder.assert_felt_eq(merged_pvs.connector.exit_code, F::ZERO); - let pvs = RootVmVerifierPvs { - exe_commit: compute_exe_commit( - &mut builder, - &hasher, - merged_pvs.app_commit, - merged_pvs.memory.initial_root, - merged_pvs.connector.initial_pc, - ), - leaf_verifier_commit: expected_leaf_commit, - public_values: public_values_vec, - }; - pvs.flatten() - .into_iter() - .for_each(|v| builder.commit_public_value(v)); + builder.cycle_tracker_start("ExtractPublicValues"); + builder.assert_usize_eq(public_values.len(), RVar::from(self.num_user_public_values)); + let public_values_vec: Vec> = (0..self.num_user_public_values) + .map(|i| builder.get(&public_values, i)) + .collect(); + let hasher = VariableP2Hasher::new(builder); + let pv_commit = hasher.merkle_root(builder, &public_values_vec); + builder.assert_eq::<[_; DIGEST_SIZE]>(merged_pvs.public_values_commit, pv_commit); + builder.cycle_tracker_end("ExtractPublicValues"); - builder.halt(); + RootVmVerifierPvs { + exe_commit: compute_exe_commit( + builder, + &hasher, + merged_pvs.app_commit, + merged_pvs.memory.initial_root, + merged_pvs.connector.initial_pc, + ), + leaf_verifier_commit: expected_leaf_commit, + public_values: public_values_vec, } - - builder.compile_isa_with_options(self.compiler_options) } } diff --git a/crates/sdk/Cargo.toml b/crates/sdk/Cargo.toml index 4d69470a55..e0727b6202 100644 --- a/crates/sdk/Cargo.toml +++ b/crates/sdk/Cargo.toml @@ -55,6 +55,7 @@ snark-verifier-sdk.workspace = true tempfile.workspace = true hex.workspace = true forge-fmt = { workspace = true, optional = true } +rrs-lib = { workspace = true } [features] default = ["parallel", "jemalloc", "evm-verify"] diff --git a/crates/sdk/src/keygen/asm.rs b/crates/sdk/src/keygen/asm.rs new file mode 100644 index 0000000000..5c8779abbe --- /dev/null +++ b/crates/sdk/src/keygen/asm.rs @@ -0,0 +1,106 @@ +use openvm_circuit::arch::instructions::{ + instruction::{Instruction, NUM_OPERANDS}, + program::Program, + LocalOpcode, +}; +use openvm_continuations::F; +use openvm_native_compiler::{asm::A0, conversion::AS, NativeJalOpcode}; +use openvm_stark_backend::p3_field::{FieldAlgebra, PrimeField32}; +use rrs_lib::instruction_formats::IType; + +const OPCODE: u32 = 0x0b; +const FUNCT3: u32 = 0b111; +const LONG_FORM_INSTRUCTION_INDICATOR: u32 = (FUNCT3 << 12) + OPCODE; +const GAP_INDICATOR: u32 = (1 << 25) + (FUNCT3 << 12) + OPCODE; + +pub fn program_to_asm(mut program: Program) -> String { + let pc_diff = handle_pc_diff(&mut program); + let assembly_and_comments = convert_program_to_u32s_and_comments(&program, pc_diff); + let mut asm_output = String::new(); + for (u32s, comment) in &assembly_and_comments { + for (idx, x) in u32s.iter().enumerate() { + asm_output.push_str(&u32_to_directive(*x)); + if idx == 0 { + asm_output.push_str(" // "); + asm_output.push_str(comment); + } + asm_output.push('\n'); + } + } + asm_output +} + +fn u32_to_directive(x: u32) -> String { + let opcode = x & 0b1111111; + let dec_insn = IType::new(x); + format!( + ".insn i {}, {}, x{}, x{}, {}", + opcode, dec_insn.funct3, dec_insn.rd, dec_insn.rs1, dec_insn.imm + ) +} + +/// In order to use native instructions in kernel functions, native instructions need to be +/// converted to RISC-V machine code(long form instructions) first. Then Rust compiler compiles the +/// whole program into an ELF. Finally, the ELF is transpiled into an OpenVm Exe. +/// In the perspective of the native compiler and the transpiler, the PC step between 2 native +/// instructions is 4. However, in the ELF, each native instruction takes longer than 4 bytes, so +/// the instructions after the code blocks use the actual lengths of the native instructions to +/// compute PC offsets. To solve this problem, we need the gap indicator to pad the native code +/// block in order to align the PC of the following instructions. +/// More details about long form instructions and gap indicators can be found in +/// `docs/specs/transpiler.md`. +fn handle_pc_diff(program: &mut Program) -> usize { + const GAP_INDICATOR_WIDTH: usize = 2; + const LONG_FORM_NATIVE_INSTRUCTION_WIDTH: usize = 10; + const PC_STEP: usize = 4; + // For GAP_INDICATOR, whose width is 2. + let mut pc_diff = GAP_INDICATOR_WIDTH; + // For each native instruction + pc_diff += program.num_defined_instructions() * (LONG_FORM_NATIVE_INSTRUCTION_WIDTH - 1); + // For next jal + pc_diff += LONG_FORM_NATIVE_INSTRUCTION_WIDTH - 1; + let jal = Instruction:: { + opcode: NativeJalOpcode::JAL.global_opcode(), + a: F::from_canonical_usize(A0 as usize), // A0 + // +1 means the next instruction after the gap + b: F::from_canonical_usize(PC_STEP * (pc_diff + 1)), + c: F::from_canonical_usize(0), + d: F::from_canonical_u32(AS::Native as u32), + e: F::from_canonical_usize(0), + f: F::from_canonical_usize(0), + g: F::from_canonical_usize(0), + }; + program.push_instruction(jal); + pc_diff +} + +fn convert_program_to_u32s_and_comments( + program: &Program, + pc_diff: usize, +) -> Vec<(Vec, String)> { + program + .defined_instructions() + .iter() + .map(|ins| { + ( + vec![ + LONG_FORM_INSTRUCTION_INDICATOR, + NUM_OPERANDS as u32, + ins.opcode.as_usize() as u32, + ins.a.as_canonical_u32(), + ins.b.as_canonical_u32(), + ins.c.as_canonical_u32(), + ins.d.as_canonical_u32(), + ins.e.as_canonical_u32(), + ins.f.as_canonical_u32(), + ins.g.as_canonical_u32(), + ], + format!("{:?}", ins.opcode), + ) + }) + .chain(std::iter::once(( + vec![GAP_INDICATOR, pc_diff as u32], + "GAP_INDICATOR".to_string(), + ))) + .collect() +} diff --git a/crates/sdk/src/keygen/mod.rs b/crates/sdk/src/keygen/mod.rs index bbd6ec269e..182926a282 100644 --- a/crates/sdk/src/keygen/mod.rs +++ b/crates/sdk/src/keygen/mod.rs @@ -46,6 +46,7 @@ use crate::{ NonRootCommittedExe, RootSC, F, SC, }; +pub mod asm; pub(crate) mod dummy; pub mod perm; pub mod static_verifier; @@ -321,7 +322,7 @@ impl AggStarkProvingKey { let root_program = RootVmVerifierConfig { leaf_fri_params: config.leaf_fri_params, internal_fri_params: config.internal_fri_params, - num_public_values: config.max_num_user_public_values, + num_user_public_values: config.max_num_user_public_values, internal_vm_verifier_commit: internal_committed_exe.get_program_commit().into(), compiler_options: config.compiler_options, } @@ -369,7 +370,7 @@ impl AggStarkProvingKey { self.internal_committed_exe.get_program_commit().into() } - pub fn num_public_values(&self) -> usize { + pub fn num_user_public_values(&self) -> usize { self.root_verifier_pk .vm_pk .vm_config diff --git a/crates/sdk/src/lib.rs b/crates/sdk/src/lib.rs index 31362ef911..e8e24454e1 100644 --- a/crates/sdk/src/lib.rs +++ b/crates/sdk/src/lib.rs @@ -21,7 +21,8 @@ use openvm_circuit::{ }, }; use openvm_continuations::verifier::{ - internal::types::E2eStarkProof, root::types::RootVmVerifierInput, + internal::types::E2eStarkProof, + root::{types::RootVmVerifierInput, RootVmVerifierConfig}, }; pub use openvm_continuations::{ static_verifier::{DefaultStaticVerifierPvHandler, StaticVerifierPvHandler}, @@ -60,6 +61,8 @@ pub mod prover; mod stdin; pub use stdin::*; +use crate::keygen::asm::program_to_asm; + pub mod fs; pub mod types; @@ -261,6 +264,24 @@ impl> GenericSdk { Ok(agg_pk) } + pub fn generate_root_verifier_asm(&self, agg_stark_pk: &AggStarkProvingKey) -> String { + let kernel_asm = RootVmVerifierConfig { + leaf_fri_params: agg_stark_pk.leaf_vm_pk.fri_params, + internal_fri_params: agg_stark_pk.internal_vm_pk.fri_params, + num_user_public_values: agg_stark_pk.num_user_public_values(), + internal_vm_verifier_commit: agg_stark_pk + .internal_committed_exe + .get_program_commit() + .into(), + compiler_options: Default::default(), + } + .build_kernel_asm( + &agg_stark_pk.leaf_vm_pk.vm_pk.get_vk(), + &agg_stark_pk.internal_vm_pk.vm_pk.get_vk(), + ); + program_to_asm(kernel_asm) + } + pub fn generate_root_verifier_input>( &self, app_pk: Arc>, diff --git a/crates/sdk/src/prover/stark.rs b/crates/sdk/src/prover/stark.rs index 342fe68913..5b4ad96e30 100644 --- a/crates/sdk/src/prover/stark.rs +++ b/crates/sdk/src/prover/stark.rs @@ -34,7 +34,7 @@ impl> StarkProver { ); assert_eq!( app_pk.app_vm_pk.vm_config.system().num_public_values, - agg_stark_pk.num_public_values(), + agg_stark_pk.num_user_public_values(), "App VM is incompatible with Agg VM because of the number of public values" ); diff --git a/crates/sdk/tests/integration_test.rs b/crates/sdk/tests/integration_test.rs index d34bdabfb4..091f2e1126 100644 --- a/crates/sdk/tests/integration_test.rs +++ b/crates/sdk/tests/integration_test.rs @@ -37,7 +37,7 @@ use openvm_sdk::{ codec::{Decode, Encode}, commit::AppExecutionCommit, config::{AggConfig, AggStarkConfig, AppConfig, Halo2Config, SdkSystemConfig, SdkVmConfig}, - keygen::AppProvingKey, + keygen::{AggStarkProvingKey, AppProvingKey}, types::{EvmHalo2Verifier, EvmProof}, DefaultStaticVerifierPvHandler, Sdk, StdIn, }; @@ -612,3 +612,11 @@ fn test_segmentation_retry() { .sum(); assert!(new_total_height < total_height); } + +#[test] +fn test_root_verifier_asm_generate() { + let agg_stark_config = agg_stark_config_for_test(); + let agg_pk = AggStarkProvingKey::keygen(agg_stark_config); + let sdk = Sdk::new(); + sdk.generate_root_verifier_asm(&agg_pk); +} diff --git a/extensions/native/compiler/src/asm/compiler.rs b/extensions/native/compiler/src/asm/compiler.rs index 7d32358a94..a09c8a217e 100644 --- a/extensions/native/compiler/src/asm/compiler.rs +++ b/extensions/native/compiler/src/asm/compiler.rs @@ -15,7 +15,7 @@ pub const MEMORY_BITS: usize = 29; pub const MEMORY_TOP: u32 = (1 << MEMORY_BITS) - 4; // The memory location for the start of the heap. -pub(crate) const HEAP_START_ADDRESS: i32 = 1 << 24; +pub const HEAP_START_ADDRESS: i32 = 1 << 24; /// The heap pointer address. pub(crate) const HEAP_PTR: i32 = HEAP_START_ADDRESS - 4; diff --git a/extensions/rv32im/tests/src/lib.rs b/extensions/rv32im/tests/src/lib.rs index f07a6c43a2..8bd4b42763 100644 --- a/extensions/rv32im/tests/src/lib.rs +++ b/extensions/rv32im/tests/src/lib.rs @@ -93,7 +93,8 @@ mod tests { #[test] fn test_hint_load_by_key() -> Result<()> { - let elf = build_example_program_at_path(get_programs_dir!(), "hint_load_by_key")?; + let config = Rv32IConfig::default(); + let elf = build_example_program_at_path(get_programs_dir!(), "hint_load_by_key", &config)?; let exe = VmExe::from_elf( elf, Transpiler::::default() @@ -101,7 +102,6 @@ mod tests { .with_extension(Rv32MTranspilerExtension) .with_extension(Rv32IoTranspilerExtension), )?; - let config = Rv32IConfig::default(); // stdin will be read after reading kv_store let stdin = vec![[0, 1, 2].map(F::from_canonical_u8).to_vec()]; let mut streams: Streams = stdin.into(); From 0dff6bd8c9f5cc69fe1944a1146797f9817b19d3 Mon Sep 17 00:00:00 2001 From: Xinding Wei Date: Mon, 12 May 2025 15:18:03 -0700 Subject: [PATCH 09/15] feat: Macro define_verify_openvm_stark (#1620) - Add a RISC-V custom instruction `nativestorew` into `Rv32IoTranspilerExtension`. - Add macro `define_verify_openvm_stark`, which can define a function for guest program to verify a stark proof. closes INT-3896 --- Cargo.lock | 2 + Cargo.toml | 1 + crates/sdk/Cargo.toml | 5 + crates/sdk/examples/sdk_app.rs | 2 +- crates/sdk/examples/sdk_evm.rs | 2 +- crates/sdk/guest/{ => fib}/Cargo.toml | 2 +- crates/sdk/guest/{ => fib}/src/main.rs | 0 .../sdk/guest/verify_openvm_stark/.gitignore | 4 + .../sdk/guest/verify_openvm_stark/Cargo.toml | 18 +++ .../sdk/guest/verify_openvm_stark/src/main.rs | 24 ++++ crates/sdk/src/config/global.rs | 4 + crates/sdk/src/stdin.rs | 13 +- crates/sdk/tests/integration_test.rs | 131 ++++++++++++++++-- crates/toolchain/openvm/src/io/mod.rs | 10 ++ crates/toolchain/openvm/src/lib.rs | 2 + crates/toolchain/openvm/src/verify_stark.rs | 56 ++++++++ docs/specs/RISCV.md | 4 + docs/specs/isa-table.md | 79 +++++------ extensions/rv32im/guest/src/io.rs | 15 ++ extensions/rv32im/guest/src/lib.rs | 2 + extensions/rv32im/transpiler/src/lib.rs | 23 ++- 21 files changed, 343 insertions(+), 56 deletions(-) rename crates/sdk/guest/{ => fib}/Cargo.toml (68%) rename crates/sdk/guest/{ => fib}/src/main.rs (100%) create mode 100644 crates/sdk/guest/verify_openvm_stark/.gitignore create mode 100644 crates/sdk/guest/verify_openvm_stark/Cargo.toml create mode 100644 crates/sdk/guest/verify_openvm_stark/src/main.rs create mode 100644 crates/toolchain/openvm/src/verify_stark.rs diff --git a/Cargo.lock b/Cargo.lock index 3cb6316d13..811f9583b6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4824,9 +4824,11 @@ dependencies = [ "openvm-native-circuit", "openvm-native-compiler", "openvm-native-recursion", + "openvm-native-transpiler", "openvm-pairing-circuit", "openvm-pairing-transpiler", "openvm-rv32im-circuit", + "openvm-rv32im-guest", "openvm-rv32im-transpiler", "openvm-sha256-circuit", "openvm-sha256-transpiler", diff --git a/Cargo.toml b/Cargo.toml index 3948dfc709..a786fe28a9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -140,6 +140,7 @@ openvm-native-circuit = { path = "extensions/native/circuit", default-features = openvm-native-compiler = { path = "extensions/native/compiler", default-features = false } openvm-native-compiler-derive = { path = "extensions/native/compiler/derive", default-features = false } openvm-native-recursion = { path = "extensions/native/recursion", default-features = false } +openvm-native-transpiler = { path = "extensions/native/transpiler", default-features = false } openvm-keccak256-circuit = { path = "extensions/keccak256/circuit", default-features = false } openvm-keccak256-transpiler = { path = "extensions/keccak256/transpiler", default-features = false } openvm-keccak256-guest = { path = "extensions/keccak256/guest", default-features = false } diff --git a/crates/sdk/Cargo.toml b/crates/sdk/Cargo.toml index e0727b6202..3996ef8c50 100644 --- a/crates/sdk/Cargo.toml +++ b/crates/sdk/Cargo.toml @@ -25,6 +25,7 @@ openvm-pairing-transpiler = { workspace = true } openvm-native-circuit = { workspace = true } openvm-native-compiler = { workspace = true } openvm-native-recursion = { workspace = true, features = ["static-verifier"] } +openvm-native-transpiler = { workspace = true } openvm-rv32im-circuit = { workspace = true } openvm-rv32im-transpiler = { workspace = true } openvm-transpiler = { workspace = true } @@ -57,6 +58,10 @@ hex.workspace = true forge-fmt = { workspace = true, optional = true } rrs-lib = { workspace = true } +[dev-dependencies] +openvm-rv32im-guest.workspace = true + + [features] default = ["parallel", "jemalloc", "evm-verify"] evm-prove = ["openvm-native-recursion/evm-prove"] diff --git a/crates/sdk/examples/sdk_app.rs b/crates/sdk/examples/sdk_app.rs index 0b3a500e2a..31ba0ab264 100644 --- a/crates/sdk/examples/sdk_app.rs +++ b/crates/sdk/examples/sdk_app.rs @@ -46,7 +46,7 @@ fn main() -> Result<(), Box> { /// use std::path::PathBuf; /// /// let mut path = PathBuf::from(env!("CARGO_MANIFEST_DIR")).to_path_buf(); - /// path.push("guest"); + /// path.push("guest/fib"); /// let target_path = path.to_str().unwrap(); /// ``` // ANCHOR: build diff --git a/crates/sdk/examples/sdk_evm.rs b/crates/sdk/examples/sdk_evm.rs index 7d42dd6232..8833542b73 100644 --- a/crates/sdk/examples/sdk_evm.rs +++ b/crates/sdk/examples/sdk_evm.rs @@ -46,7 +46,7 @@ fn main() -> Result<(), Box> { /// use std::path::PathBuf; /// /// let mut path = PathBuf::from(env!("CARGO_MANIFEST_DIR")).to_path_buf(); - /// path.push("guest"); + /// path.push("guest/fib"); /// let target_path = path.to_str().unwrap(); /// ``` // ANCHOR: build diff --git a/crates/sdk/guest/Cargo.toml b/crates/sdk/guest/fib/Cargo.toml similarity index 68% rename from crates/sdk/guest/Cargo.toml rename to crates/sdk/guest/fib/Cargo.toml index 1ca1917784..8bb44b44b6 100644 --- a/crates/sdk/guest/Cargo.toml +++ b/crates/sdk/guest/fib/Cargo.toml @@ -5,4 +5,4 @@ version = "0.0.0" edition = "2021" [dependencies] -openvm = { path = "../../toolchain/openvm" } +openvm = { path = "../../../toolchain/openvm" } diff --git a/crates/sdk/guest/src/main.rs b/crates/sdk/guest/fib/src/main.rs similarity index 100% rename from crates/sdk/guest/src/main.rs rename to crates/sdk/guest/fib/src/main.rs diff --git a/crates/sdk/guest/verify_openvm_stark/.gitignore b/crates/sdk/guest/verify_openvm_stark/.gitignore new file mode 100644 index 0000000000..0e52c45d5d --- /dev/null +++ b/crates/sdk/guest/verify_openvm_stark/.gitignore @@ -0,0 +1,4 @@ +*.asm +Cargo.lock +target/ +openvm/ \ No newline at end of file diff --git a/crates/sdk/guest/verify_openvm_stark/Cargo.toml b/crates/sdk/guest/verify_openvm_stark/Cargo.toml new file mode 100644 index 0000000000..f788fc95b1 --- /dev/null +++ b/crates/sdk/guest/verify_openvm_stark/Cargo.toml @@ -0,0 +1,18 @@ +[workspace] +[package] +name = "openvm-verify-stark-program" +version = "0.0.0" +edition = "2021" + +[dependencies] +openvm = { path = "../../../toolchain/openvm", features = ["std"] } +hex-literal = { version = "0.4.1", default-features = false } +bytemuck = { version = "1.20.0", features = ["extern_crate_alloc"] } + +[features] +default = [] + +[profile.profiling] +inherits = "release" +debug = 2 +strip = false diff --git a/crates/sdk/guest/verify_openvm_stark/src/main.rs b/crates/sdk/guest/verify_openvm_stark/src/main.rs new file mode 100644 index 0000000000..bb9011f328 --- /dev/null +++ b/crates/sdk/guest/verify_openvm_stark/src/main.rs @@ -0,0 +1,24 @@ +extern crate alloc; +use alloc::vec::Vec; + +use openvm::{define_verify_openvm_stark, io::read}; + +define_verify_openvm_stark!( + verify_openvm_stark, + env!("CARGO_MANIFEST_DIR"), + "root_verifier.asm" +); + +// const APP_EXE_COMMIT: [u32; 8] = [ +// 343014587, 230645511, 1447462186, 773379336, 1182270030, 1497892484, 461820702, 353704350, +// ]; +// const APP_VM_COMMIT: [u32; 8] = [ +// 445134834, 1133596793, 530952192, 425228715, 1806903712, 1362083369, 295028151, 482389308, +// ]; + +pub fn main() { + let app_exe_commit: [u32; 8] = read(); + let app_vm_commit: [u32; 8] = read(); + let pvs: Vec = read(); + verify_openvm_stark(&app_exe_commit, &app_vm_commit, &pvs); +} diff --git a/crates/sdk/src/config/global.rs b/crates/sdk/src/config/global.rs index a28afd1918..faf8182246 100644 --- a/crates/sdk/src/config/global.rs +++ b/crates/sdk/src/config/global.rs @@ -25,6 +25,7 @@ use openvm_native_circuit::{ CastFExtension, CastFExtensionExecutor, CastFExtensionPeriphery, Native, NativeExecutor, NativePeriphery, }; +use openvm_native_transpiler::LongFormTranspilerExtension; use openvm_pairing_circuit::{ PairingExtension, PairingExtensionExecutor, PairingExtensionPeriphery, }; @@ -139,6 +140,9 @@ impl SdkVmConfig { if self.sha256.is_some() { transpiler = transpiler.with_extension(Sha256TranspilerExtension); } + if self.native.is_some() { + transpiler = transpiler.with_extension(LongFormTranspilerExtension); + } if self.rv32m.is_some() { transpiler = transpiler.with_extension(Rv32MTranspilerExtension); } diff --git a/crates/sdk/src/stdin.rs b/crates/sdk/src/stdin.rs index eaa7bebeef..9101e8d4de 100644 --- a/crates/sdk/src/stdin.rs +++ b/crates/sdk/src/stdin.rs @@ -1,4 +1,7 @@ -use std::collections::VecDeque; +use std::{ + collections::{HashMap, VecDeque}, + sync::Arc, +}; use openvm_circuit::arch::Streams; use openvm_stark_backend::p3_field::FieldAlgebra; @@ -9,6 +12,7 @@ use crate::F; #[derive(Clone, Default, Serialize, Deserialize)] pub struct StdIn { pub buffer: VecDeque>, + pub kv_store: HashMap, Vec>, } impl StdIn { @@ -36,6 +40,9 @@ impl StdIn { pub fn write_field(&mut self, data: &[F]) { self.buffer.push_back(data.to_vec()); } + pub fn add_key_value(&mut self, key: Vec, value: Vec) { + self.kv_store.insert(key, value); + } } impl From for Streams { @@ -44,7 +51,9 @@ impl From for Streams { while let Some(input) = std_in.read() { data.push(input); } - Streams::new(data) + let mut ret = Streams::new(data); + ret.kv_store = Arc::new(std_in.kv_store); + ret } } diff --git a/crates/sdk/tests/integration_test.rs b/crates/sdk/tests/integration_test.rs index 091f2e1126..128ff3fdc2 100644 --- a/crates/sdk/tests/integration_test.rs +++ b/crates/sdk/tests/integration_test.rs @@ -6,6 +6,7 @@ use openvm_circuit::{ arch::{ hasher::poseidon2::vm_poseidon2_hasher, ContinuationVmProof, ExecutionError, GenerationError, SingleSegmentVmExecutor, SystemConfig, VmConfig, VmExecutor, + DEFAULT_MAX_NUM_PUBLIC_VALUES, }, system::{memory::tree::public_values::UserPublicValuesProof, program::trace::VmCommittedExe}, }; @@ -27,9 +28,11 @@ use openvm_native_recursion::{ wrapper::Halo2WrapperProvingKey, RawEvmProof, }, + hints::Hintable, types::InnerConfig, vars::StarkProofVariable, }; +use openvm_rv32im_guest::hint_load_by_key_encode; use openvm_rv32im_transpiler::{ Rv32ITranspilerExtension, Rv32IoTranspilerExtension, Rv32MTranspilerExtension, }; @@ -41,7 +44,9 @@ use openvm_sdk::{ types::{EvmHalo2Verifier, EvmProof}, DefaultStaticVerifierPvHandler, Sdk, StdIn, }; -use openvm_stark_backend::{keygen::types::LinearConstraint, p3_matrix::Matrix}; +use openvm_stark_backend::{ + keygen::types::LinearConstraint, p3_field::PrimeField32, p3_matrix::Matrix, +}; use openvm_stark_sdk::{ config::{ baby_bear_poseidon2::{BabyBearPoseidon2Config, BabyBearPoseidon2Engine}, @@ -393,7 +398,7 @@ fn test_static_verifier_custom_pv_handler() { #[test] fn test_e2e_proof_generation_and_verification_with_pvs() { let mut pkg_dir = PathBuf::from(env!("CARGO_MANIFEST_DIR")).to_path_buf(); - pkg_dir.push("guest"); + pkg_dir.push("guest/fib"); let vm_config = SdkVmConfig::builder() .system(SdkSystemConfig { @@ -469,7 +474,7 @@ fn test_sdk_guest_build_and_transpile() { // .with_options(vec!["--release"]); ; let mut pkg_dir = PathBuf::from(env!("CARGO_MANIFEST_DIR")).to_path_buf(); - pkg_dir.push("guest"); + pkg_dir.push("guest/fib"); let vm_config = SdkVmConfig::builder() .system(SdkSystemConfig { @@ -516,7 +521,7 @@ fn test_inner_proof_codec_roundtrip() -> eyre::Result<()> { // generate a proof let sdk = Sdk::new(); let mut pkg_dir = PathBuf::from(env!("CARGO_MANIFEST_DIR")).to_path_buf(); - pkg_dir.push("guest"); + pkg_dir.push("guest/fib"); let vm_config = SdkVmConfig::builder() .system(SdkSystemConfig { @@ -614,9 +619,119 @@ fn test_segmentation_retry() { } #[test] -fn test_root_verifier_asm_generate() { - let agg_stark_config = agg_stark_config_for_test(); - let agg_pk = AggStarkProvingKey::keygen(agg_stark_config); +fn test_verify_openvm_stark_e2e() -> Result<()> { + const ASM_FILENAME: &str = "root_verifier.asm"; let sdk = Sdk::new(); - sdk.generate_root_verifier_asm(&agg_pk); + let mut pkg_dir = PathBuf::from(env!("CARGO_MANIFEST_DIR")).to_path_buf(); + pkg_dir.push("guest/fib"); + + let vm_config = SdkVmConfig::builder() + .system(SdkSystemConfig { + config: SystemConfig::default().with_continuations(), + }) + .rv32i(Default::default()) + .rv32m(Default::default()) + .io(Default::default()) + .native(Default::default()) + .build(); + assert!(vm_config.system.config.continuation_enabled); + let elf = sdk.build( + Default::default(), + &vm_config, + pkg_dir, + &Default::default(), + None, + )?; + + let app_exe = sdk.transpile(elf, vm_config.transpiler())?; + let fri_params = FriParameters::new_for_testing(LEAF_LOG_BLOWUP); + let app_config = AppConfig::new_with_leaf_fri_params(fri_params, vm_config.clone(), fri_params); + + let app_pk = sdk.app_keygen(app_config.clone())?; + let committed_app_exe = sdk.commit_app_exe(fri_params, app_exe.clone())?; + + let commits = + AppExecutionCommit::compute(&vm_config, &committed_app_exe, &app_pk.leaf_committed_exe); + + let agg_pk = AggStarkProvingKey::keygen(AggStarkConfig { + max_num_user_public_values: DEFAULT_MAX_NUM_PUBLIC_VALUES, + leaf_fri_params: FriParameters::new_for_testing(LEAF_LOG_BLOWUP), + internal_fri_params: FriParameters::new_for_testing(INTERNAL_LOG_BLOWUP), + root_fri_params: FriParameters::new_for_testing(ROOT_LOG_BLOWUP), + profiling: false, + compiler_options: CompilerOptions { + enable_cycle_tracker: true, + ..Default::default() + }, + root_max_constraint_degree: (1 << ROOT_LOG_BLOWUP) + 1, + }); + let asm = sdk.generate_root_verifier_asm(&agg_pk); + let asm_path = format!( + "{}/guest/verify_openvm_stark/{}", + env!("CARGO_MANIFEST_DIR"), + ASM_FILENAME + ); + std::fs::write(asm_path, asm)?; + + let e2e_stark_proof = sdk.generate_e2e_stark_proof( + Arc::new(app_pk), + committed_app_exe, + agg_pk, + StdIn::default(), + )?; + + let verify_exe = { + let mut pkg_dir = PathBuf::from(env!("CARGO_MANIFEST_DIR")).to_path_buf(); + pkg_dir.push("guest/verify_openvm_stark"); + let elf = sdk.build( + Default::default(), + &vm_config, + pkg_dir, + &Default::default(), + None, + )?; + sdk.transpile(elf, vm_config.transpiler())? + }; + + let pvs = [13u32, 21, 0, 0, 0, 0, 0, 0]; + + let exe_commit_u32: Vec<_> = commits + .exe_commit + .iter() + .map(|x| x.as_canonical_u32()) + .collect(); + let vm_commit_u32: Vec<_> = commits + .leaf_vm_verifier_commit + .iter() + .map(|x| x.as_canonical_u32()) + .collect(); + let pvs_u32: Vec<_> = pvs + .iter() + .flat_map(|x| x.to_le_bytes()) + .map(|x| x as u32) + .collect(); + + let key = ASM_FILENAME + .as_bytes() + .iter() + .cloned() + .chain(exe_commit_u32.iter().flat_map(|x| x.to_le_bytes())) + .chain(vm_commit_u32.iter().flat_map(|x| x.to_le_bytes())) + .chain(pvs_u32.iter().flat_map(|x| x.to_le_bytes())) + .collect(); + let mut stdin = StdIn::default(); + let to_encode: Vec> = e2e_stark_proof.proof.write(); + let value = hint_load_by_key_encode(&to_encode); + stdin.add_key_value(key, value); + + let exe_commit_u32_8: [u32; 8] = exe_commit_u32.try_into().unwrap(); + let vm_commit_u32_8: [u32; 8] = vm_commit_u32.try_into().unwrap(); + + stdin.write(&exe_commit_u32_8); + stdin.write(&vm_commit_u32_8); + stdin.write(&pvs_u32); + + sdk.execute(verify_exe, vm_config, stdin)?; + + Ok(()) } diff --git a/crates/toolchain/openvm/src/io/mod.rs b/crates/toolchain/openvm/src/io/mod.rs index 1bde9e886f..eb00a9d3cd 100644 --- a/crates/toolchain/openvm/src/io/mod.rs +++ b/crates/toolchain/openvm/src/io/mod.rs @@ -126,6 +126,16 @@ pub fn reveal_u32(x: u32, index: usize) { println!("reveal {} at byte location {}", x, index * 4); } +/// Store u32 `x` to the native address `native_addr` as 4 field element in byte. +#[allow(unused_variables)] +#[inline(always)] +pub fn store_u32_to_native(native_addr: u32, x: u32) { + #[cfg(target_os = "zkvm")] + openvm_rv32im_guest::store_to_native!(native_addr, x); + #[cfg(not(target_os = "zkvm"))] + panic!("store_to_native_u32 cannot run on non-zkVM platforms"); +} + /// A no-alloc writer to print to stdout on host machine for debugging purposes. pub struct Writer; diff --git a/crates/toolchain/openvm/src/lib.rs b/crates/toolchain/openvm/src/lib.rs index d0507db84c..898676da20 100644 --- a/crates/toolchain/openvm/src/lib.rs +++ b/crates/toolchain/openvm/src/lib.rs @@ -31,6 +31,8 @@ pub mod utils; #[cfg(not(target_os = "zkvm"))] pub mod host; +#[cfg(target_os = "zkvm")] +pub mod verify_stark; #[cfg(target_os = "zkvm")] core::arch::global_asm!(include_str!("memset.s")); diff --git a/crates/toolchain/openvm/src/verify_stark.rs b/crates/toolchain/openvm/src/verify_stark.rs new file mode 100644 index 0000000000..1382dcd9d5 --- /dev/null +++ b/crates/toolchain/openvm/src/verify_stark.rs @@ -0,0 +1,56 @@ +/// Define a function that verifies an OpenVM Stark proof. +/// To define this function, users need to specify the function name and an ASM file containing the +/// assembly code for the verification(this ASM file can be generated by +/// `Sdk.generate_root_verifier_asm` function). To specify the ASM file, users need to provide the +/// parent folder and filename of the ASM file. +/// To call this function: +/// 1. users need to provide `app_exe_commit`/`app_vm_commit`/`user_pvs`(user public values) as the +/// arguments. CAREFUL: `app_exe_commit`/`app_vm_commit`/`user_pvs` are in u32 and are interpreted +/// as native fields. `user_pvs` are the exact public values of that proof. Which means, if a guest +/// program calls `reveal_u32` to publish an u32(e.g. `258`) in the proof, that u32 is decomposed +/// into 4 field elements in byte. In `user_pvs`, it should be an u32 array, `[2, 1, 0, 0]`. +/// 2. Provide the corresponding stark proof in the key-value store in OpenVM streams. The key +/// should +/// be the concatenation of the filename, `app_exe_commit`, `app_vm_commit`, and `user_pvs` in +/// little-endian bytes. +#[macro_export] +macro_rules! define_verify_openvm_stark { + ($fn_name: ident, $asm_folder: expr, $asm_filename: literal) => { + pub fn $fn_name(app_exe_commit: &[u32; 8], app_vm_commit: &[u32; 8], user_pvs: &[u32]) { + // The memory location for the start of the heap. + const HEAP_START_ADDRESS: u32 = 1 << 24; + const FIELDS_PER_U32: u32 = 4; + const FILENAME: &str = $asm_filename; + // Construct the hint key + let hint_key: alloc::vec::Vec = FILENAME + .as_bytes() + .iter() + .cloned() + .chain(app_exe_commit.iter().flat_map(|x| x.to_le_bytes())) + .chain(app_vm_commit.iter().flat_map(|x| x.to_le_bytes())) + .chain(user_pvs.iter().flat_map(|x| x.to_le_bytes())) + .collect(); + openvm::io::hint_load_by_key(&hint_key); + // Store the expected public values into the beginning of the native heap. + let mut native_addr = HEAP_START_ADDRESS; + for &x in app_exe_commit { + openvm::io::store_u32_to_native(native_addr, x); + native_addr += FIELDS_PER_U32; + } + for &x in app_vm_commit { + openvm::io::store_u32_to_native(native_addr, x); + native_addr += FIELDS_PER_U32; + } + for &x in user_pvs { + openvm::io::store_u32_to_native(native_addr, x); + native_addr += FIELDS_PER_U32; + } + // Assumption: the asm file should be generated by SDK. The code block should: + // 1. Increase the heap pointer in order to avoid overwriting expected public values. + // 2. Hint a stark proof from the input stream + // 3. Verify the proof + // 4. Compare the public values with the expected ones. Panic if not equal. + unsafe { core::arch::asm!(include_str!(concat!($asm_folder, "/", $asm_filename)),) } + } + }; +} diff --git a/docs/specs/RISCV.md b/docs/specs/RISCV.md index 427c7c16d5..904c675167 100644 --- a/docs/specs/RISCV.md +++ b/docs/specs/RISCV.md @@ -72,6 +72,10 @@ the guest must take care to validate all data and account for behavior in cases | printstr | I | 0001011 | 011 | 0x1 | Tries to convert `[rd..rd + rs1]_2` to UTF-8 string and print to host stdout. Will print error message if conversion fails. | | hintrandom | I | 0001011 | 011 | 0x2 | Resets the hint stream to `4 * rd` random bytes from `rand::rngs::OsRng` on the host. | +| RISC-V Inst | FMT | opcode[6:0] | funct3 | funct7 | RISC-V description and notes | +|--------------|-----|-------------|---------|--------|------------------------------------------------------------------------------------------------------------------------------| +| nativestorew | R | 0001011 | 111 | 0x2 | Stores the 4-byte word `rs1` at address `rd` in native address space. The address `rd` must be aligned to a 4-byte boundary. | + ## Keccak Extension | RISC-V Inst | FMT | opcode[6:0] | funct3 | funct7 | RISC-V description and notes | diff --git a/docs/specs/isa-table.md b/docs/specs/isa-table.md index 940d90ed0e..7b7f374065 100644 --- a/docs/specs/isa-table.md +++ b/docs/specs/isa-table.md @@ -31,47 +31,48 @@ In the tables below, we provide the mapping between the `LocalOpcode` and `Phant #### Instructions -| VM Extension | `LocalOpcode` | ISA Instruction | -| ------------- | ---------- | ------------- | -| RV32IM | `BaseAluOpcode::ADD` | ADD_RV32 | -| RV32IM | `BaseAluOpcode::SUB` | SUB_RV32 | -| RV32IM | `BaseAluOpcode::XOR` | XOR_RV32 | -| RV32IM | `BaseAluOpcode::OR` | OR_RV32 | -| RV32IM | `BaseAluOpcode::AND` | AND_RV32 | -| RV32IM | `ShiftOpcode::SLL` | SLL_RV32 | -| RV32IM | `ShiftOpcode::SRL` | SRL_RV32 | -| RV32IM | `ShiftOpcode::SRA` | SRA_RV32 | -| RV32IM | `LessThanOpcode::SLT` | SLT_RV32 | -| RV32IM | `LessThanOpcode::SLTU` | SLTU_RV32 | -| RV32IM | `Rv32LoadStoreOpcode::LOADB` | LOADB_RV32 | -| RV32IM | `Rv32LoadStoreOpcode::LOADH` | LOADH_RV32 | -| RV32IM | `Rv32LoadStoreOpcode::LOADW` | LOADW_RV32 | -| RV32IM | `Rv32LoadStoreOpcode::LOADBU` | LOADBU_RV32 | -| RV32IM | `Rv32LoadStoreOpcode::LOADHU` | LOADHU_RV32 | -| RV32IM | `Rv32LoadStoreOpcode::STOREB` | STOREB_RV32 | -| RV32IM | `Rv32LoadStoreOpcode::STOREH` | STOREH_RV32 | -| RV32IM | `Rv32LoadStoreOpcode::STOREW` | STOREW_RV32 | -| RV32IM | `BranchEqualOpcode::BEQ` | BEQ_RV32 | -| RV32IM | `BranchEqualOpcode::BNE` | BNE_RV32 | -| RV32IM | `BranchLessThanOpcode::BLT` | BLT_RV32 | -| RV32IM | `BranchLessThanOpcode::BGE` | BGE_RV32 | -| RV32IM | `BranchLessThanOpcode::BLTU` | BLTU_RV32 | -| RV32IM | `BranchLessThanOpcode::BGEU` | BGEU_RV32 | -| RV32IM | `Rv32JalLuiOpcode::JAL` | JAL_RV32 | -| RV32IM | `Rv32JalrOpcode::JALR` | JALR_RV32 | -| RV32IM | `Rv32JalLuiOpcode::LUI` | LUI_RV32 | -| RV32IM | `Rv32AuipcOpcode::AUIPC` | AUIPC_RV32 | -| RV32IM | `MulOpcode::MUL` | MUL_RV32 | -| RV32IM | `MulHOpcode::MULH` | MULH_RV32 | -| RV32IM | `MulHOpcode::MULHSU` | MULHSU_RV32 | -| RV32IM | `MulHOpcode::MULHU` | MULHU_RV32 | -| RV32IM | `DivRemOpcode::DIV` | DIV_RV32 | -| RV32IM | `DivRemOpcode::DIVU` | DIVU_RV32 | -| RV32IM | `DivRemOpcode::REM` | REM_RV32 | -| RV32IM | `DivRemOpcode::REMU` | REMU_RV32 | +| VM Extension | `LocalOpcode` | ISA Instruction | +| ------------- | ---------- |------------------| +| RV32IM | `BaseAluOpcode::ADD` | ADD_RV32 | +| RV32IM | `BaseAluOpcode::SUB` | SUB_RV32 | +| RV32IM | `BaseAluOpcode::XOR` | XOR_RV32 | +| RV32IM | `BaseAluOpcode::OR` | OR_RV32 | +| RV32IM | `BaseAluOpcode::AND` | AND_RV32 | +| RV32IM | `ShiftOpcode::SLL` | SLL_RV32 | +| RV32IM | `ShiftOpcode::SRL` | SRL_RV32 | +| RV32IM | `ShiftOpcode::SRA` | SRA_RV32 | +| RV32IM | `LessThanOpcode::SLT` | SLT_RV32 | +| RV32IM | `LessThanOpcode::SLTU` | SLTU_RV32 | +| RV32IM | `Rv32LoadStoreOpcode::LOADB` | LOADB_RV32 | +| RV32IM | `Rv32LoadStoreOpcode::LOADH` | LOADH_RV32 | +| RV32IM | `Rv32LoadStoreOpcode::LOADW` | LOADW_RV32 | +| RV32IM | `Rv32LoadStoreOpcode::LOADBU` | LOADBU_RV32 | +| RV32IM | `Rv32LoadStoreOpcode::LOADHU` | LOADHU_RV32 | +| RV32IM | `Rv32LoadStoreOpcode::STOREB` | STOREB_RV32 | +| RV32IM | `Rv32LoadStoreOpcode::STOREH` | STOREH_RV32 | +| RV32IM | `Rv32LoadStoreOpcode::STOREW` | STOREW_RV32 | +| RV32IM | `BranchEqualOpcode::BEQ` | BEQ_RV32 | +| RV32IM | `BranchEqualOpcode::BNE` | BNE_RV32 | +| RV32IM | `BranchLessThanOpcode::BLT` | BLT_RV32 | +| RV32IM | `BranchLessThanOpcode::BGE` | BGE_RV32 | +| RV32IM | `BranchLessThanOpcode::BLTU` | BLTU_RV32 | +| RV32IM | `BranchLessThanOpcode::BGEU` | BGEU_RV32 | +| RV32IM | `Rv32JalLuiOpcode::JAL` | JAL_RV32 | +| RV32IM | `Rv32JalrOpcode::JALR` | JALR_RV32 | +| RV32IM | `Rv32JalLuiOpcode::LUI` | LUI_RV32 | +| RV32IM | `Rv32AuipcOpcode::AUIPC` | AUIPC_RV32 | +| RV32IM | `MulOpcode::MUL` | MUL_RV32 | +| RV32IM | `MulHOpcode::MULH` | MULH_RV32 | +| RV32IM | `MulHOpcode::MULHSU` | MULHSU_RV32 | +| RV32IM | `MulHOpcode::MULHU` | MULHU_RV32 | +| RV32IM | `DivRemOpcode::DIV` | DIV_RV32 | +| RV32IM | `DivRemOpcode::DIVU` | DIVU_RV32 | +| RV32IM | `DivRemOpcode::REM` | REM_RV32 | +| RV32IM | `DivRemOpcode::REMU` | REMU_RV32 | | RV32IM | `Rv32HintStoreOpcode::HINT_STOREW` | HINT_STOREW_RV32 | | RV32IM | `Rv32HintStoreOpcode::HINT_BUFFER` | HINT_BUFFER_RV32 | -| RV32IM | Pseudo-instruction for `STOREW_RV32` | REVEAL_RV32 | +| RV32IM | Pseudo-instruction for `STOREW_RV32` | REVEAL_RV32 | +| RV32IM | Pseudo-instruction for `STOREW_RV32` | NATIVE_STOREW | | #### Phantom Sub-Instructions diff --git a/extensions/rv32im/guest/src/io.rs b/extensions/rv32im/guest/src/io.rs index ee123c0577..664b9b1117 100644 --- a/extensions/rv32im/guest/src/io.rs +++ b/extensions/rv32im/guest/src/io.rs @@ -81,6 +81,21 @@ macro_rules! reveal { }; } +/// Store rs1 to [[rd]]_4. +#[macro_export] +macro_rules! store_to_native { + ($rd:ident, $rs1:ident) => { + openvm_custom_insn::custom_insn_r!( + opcode = openvm_rv32im_guest::SYSTEM_OPCODE, + funct3 = openvm_rv32im_guest::NATIVE_STOREW_FUNCT3, + funct7 = openvm_rv32im_guest::NATIVE_STOREW_FUNCT7, + rd = In $rd, + rs1 = In $rs1, + rs2 = In $rs1, + ) + }; +} + /// Print UTF-8 string encoded as bytes to host stdout for debugging purposes. #[inline(always)] pub fn print_str_from_bytes(str_as_bytes: &[u8]) { diff --git a/extensions/rv32im/guest/src/lib.rs b/extensions/rv32im/guest/src/lib.rs index 5a6b7087e4..99f1a6f97f 100644 --- a/extensions/rv32im/guest/src/lib.rs +++ b/extensions/rv32im/guest/src/lib.rs @@ -14,6 +14,8 @@ pub const SYSTEM_OPCODE: u8 = 0x0b; pub const CSR_OPCODE: u8 = 0b1110011; pub const RV32_ALU_OPCODE: u8 = 0b0110011; pub const RV32M_FUNCT7: u8 = 0x01; +pub const NATIVE_STOREW_FUNCT3: u8 = 0b111; +pub const NATIVE_STOREW_FUNCT7: u32 = 2; pub const TERMINATE_FUNCT3: u8 = 0b000; pub const HINT_FUNCT3: u8 = 0b001; diff --git a/extensions/rv32im/transpiler/src/lib.rs b/extensions/rv32im/transpiler/src/lib.rs index 42b5f9d2e2..abd4413022 100644 --- a/extensions/rv32im/transpiler/src/lib.rs +++ b/extensions/rv32im/transpiler/src/lib.rs @@ -6,7 +6,8 @@ use openvm_instructions::{ }; use openvm_rv32im_guest::{ PhantomImm, CSRRW_FUNCT3, CSR_OPCODE, HINT_BUFFER_IMM, HINT_FUNCT3, HINT_STOREW_IMM, - PHANTOM_FUNCT3, REVEAL_FUNCT3, RV32M_FUNCT7, RV32_ALU_OPCODE, SYSTEM_OPCODE, TERMINATE_FUNCT3, + NATIVE_STOREW_FUNCT3, NATIVE_STOREW_FUNCT7, PHANTOM_FUNCT3, REVEAL_FUNCT3, RV32M_FUNCT7, + RV32_ALU_OPCODE, SYSTEM_OPCODE, TERMINATE_FUNCT3, }; use openvm_stark_backend::p3_field::PrimeField32; use openvm_transpiler::{ @@ -155,9 +156,6 @@ impl TranspilerExtension for Rv32IoTranspilerExtension { if opcode != SYSTEM_OPCODE { return None; } - if funct3 != HINT_FUNCT3 && funct3 != REVEAL_FUNCT3 { - return None; - } let instruction = match funct3 { HINT_FUNCT3 => { @@ -198,6 +196,23 @@ impl TranspilerExtension for Rv32IoTranspilerExtension { (dec_insn.imm < 0) as isize, )) } + NATIVE_STOREW_FUNCT3 => { + // NATIVE_STOREW is a pseudo-instruction for STOREW_RV32 a,b,0,1,4 + let dec_insn = RType::new(instruction_u32); + if dec_insn.funct7 != NATIVE_STOREW_FUNCT7 { + return None; + } + Some(Instruction::large_from_isize( + Rv32LoadStoreOpcode::STOREW.global_opcode(), + (RV32_REGISTER_NUM_LIMBS * dec_insn.rs1) as isize, + (RV32_REGISTER_NUM_LIMBS * dec_insn.rd) as isize, + 0, + 1, + 4, + 1, + 0, + )) + } _ => return None, }; From 2e807c3b77b1c0b75b283b0f5dff045ec8bf6c2c Mon Sep 17 00:00:00 2001 From: Xinding Wei Date: Thu, 15 May 2025 14:08:18 -0700 Subject: [PATCH 10/15] feat: helper functions for verify_openvm_stark (#1631) - `AppExecutionCommit` stores `u32` instead of `BabyBear`. - Rename `E2eStarkProof` to `VmStarkProof`. - Add `compute_hint_key_for_verify_openvm_stark`/`encode_rv32_public_values` for users to prepare inputs for `verify_openvm_stark`. But it still seems hard to understand what happens. - Improve docs about `Rv32HintLoadByKey`. --- Cargo.lock | 16 ++- Cargo.toml | 2 + crates/cli/src/commands/prove.rs | 24 ++-- .../src/verifier/internal/types.rs | 6 +- .../src/verifier/internal/vars.rs | 4 +- crates/sdk/Cargo.toml | 4 - .../sdk/guest/verify_openvm_stark/src/main.rs | 24 ---- crates/sdk/src/codec.rs | 6 +- crates/sdk/src/commit.rs | 52 +++---- crates/sdk/src/lib.rs | 11 +- crates/sdk/src/prover/agg.rs | 20 +-- crates/sdk/src/prover/stark.rs | 8 +- crates/sdk/tests/integration_test.rs | 129 +----------------- crates/toolchain/openvm/src/lib.rs | 2 - docs/specs/ISA.md | 12 +- docs/specs/RISCV.md | 5 + guest-libs/verify_stark/guest/Cargo.toml | 25 ++++ .../examples}/verify_openvm_stark/.gitignore | 0 .../examples}/verify_openvm_stark/Cargo.toml | 5 +- .../examples/verify_openvm_stark/openvm.toml | 4 + .../examples/verify_openvm_stark/src/main.rs | 18 +++ guest-libs/verify_stark/guest/src/host.rs | 28 ++++ .../verify_stark/guest/src/lib.rs | 22 +-- .../guest/tests/integration_test.rs | 126 +++++++++++++++++ 24 files changed, 310 insertions(+), 243 deletions(-) delete mode 100644 crates/sdk/guest/verify_openvm_stark/src/main.rs create mode 100644 guest-libs/verify_stark/guest/Cargo.toml rename {crates/sdk/guest => guest-libs/verify_stark/guest/examples}/verify_openvm_stark/.gitignore (100%) rename {crates/sdk/guest => guest-libs/verify_stark/guest/examples}/verify_openvm_stark/Cargo.toml (50%) create mode 100644 guest-libs/verify_stark/guest/examples/verify_openvm_stark/openvm.toml create mode 100644 guest-libs/verify_stark/guest/examples/verify_openvm_stark/src/main.rs create mode 100644 guest-libs/verify_stark/guest/src/host.rs rename crates/toolchain/openvm/src/verify_stark.rs => guest-libs/verify_stark/guest/src/lib.rs (78%) create mode 100644 guest-libs/verify_stark/guest/tests/integration_test.rs diff --git a/Cargo.lock b/Cargo.lock index 811f9583b6..a726c28bde 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4828,7 +4828,6 @@ dependencies = [ "openvm-pairing-circuit", "openvm-pairing-transpiler", "openvm-rv32im-circuit", - "openvm-rv32im-guest", "openvm-rv32im-transpiler", "openvm-sha256-circuit", "openvm-sha256-transpiler", @@ -5020,6 +5019,21 @@ dependencies = [ "thiserror 1.0.69", ] +[[package]] +name = "openvm-verify-stark" +version = "1.1.2" +dependencies = [ + "eyre", + "openvm-build", + "openvm-circuit", + "openvm-native-compiler", + "openvm-native-recursion", + "openvm-rv32im-guest", + "openvm-sdk", + "openvm-stark-sdk", + "openvm-verify-stark", +] + [[package]] name = "option-ext" version = "0.2.0" diff --git a/Cargo.toml b/Cargo.toml index a786fe28a9..bded0ba140 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -66,6 +66,7 @@ members = [ "extensions/pairing/transpiler", "extensions/pairing/guest", "extensions/pairing/tests", + "guest-libs/verify_stark/guest", ] exclude = ["crates/sdk/example"] resolver = "2" @@ -162,6 +163,7 @@ openvm-ecc-sw-macros = { path = "extensions/ecc/sw-macros", default-features = f openvm-pairing-circuit = { path = "extensions/pairing/circuit", default-features = false } openvm-pairing-transpiler = { path = "extensions/pairing/transpiler", default-features = false } openvm-pairing-guest = { path = "extensions/pairing/guest", default-features = false } +openvm-verify-stark = { path = "guest-libs/verify_stark/guest", default-features = false } # Benchmarking openvm-benchmarks-utils = { path = "benchmarks/utils", default-features = false } diff --git a/crates/cli/src/commands/prove.rs b/crates/cli/src/commands/prove.rs index 143d34b779..db33cf766a 100644 --- a/crates/cli/src/commands/prove.rs +++ b/crates/cli/src/commands/prove.rs @@ -103,6 +103,14 @@ impl ProveCmd { sdk.set_agg_tree_config(*agg_tree_config); let (app_pk, committed_exe, input) = Self::prepare_execution(&sdk, app_pk, exe, input)?; + let commits = AppExecutionCommit::compute( + &app_pk.app_vm_pk.vm_config, + &committed_exe, + &app_pk.leaf_committed_exe, + ); + println!("exe commit: {:?}", commits.exe_commit); + println!("vm commit: {:?}", commits.vm_commit); + let agg_pk = read_agg_pk_from_file(default_agg_pk_path()).map_err(|e| { eyre::eyre!("Failed to read aggregation proving key: {}\nPlease run 'cargo openvm setup' first", e) })?; @@ -129,6 +137,14 @@ impl ProveCmd { let params_reader = CacheHalo2ParamsReader::new(default_params_dir()); let (app_pk, committed_exe, input) = Self::prepare_execution(&sdk, app_pk, exe, input)?; + let commits = AppExecutionCommit::compute( + &app_pk.app_vm_pk.vm_config, + &committed_exe, + &app_pk.leaf_committed_exe, + ); + println!("exe commit: {:?}", commits.exe_commit_to_bn254()); + println!("vm commit: {:?}", commits.vm_commit_to_bn254()); + println!("Generating EVM proof, this may take a lot of compute and memory..."); let agg_pk = read_agg_pk_from_file(default_agg_pk_path()).map_err(|e| { eyre::eyre!("Failed to read aggregation proving key: {}\nPlease run 'cargo openvm setup' first", e) @@ -155,14 +171,6 @@ impl ProveCmd { let app_exe = read_exe_from_file(exe)?; let committed_exe = sdk.commit_app_exe(app_pk.app_fri_params(), app_exe)?; - let commits = AppExecutionCommit::compute( - &app_pk.app_vm_pk.vm_config, - &committed_exe, - &app_pk.leaf_committed_exe, - ); - println!("app_pk commit: {:?}", commits.app_config_commit_to_bn254()); - println!("exe commit: {:?}", commits.exe_commit_to_bn254()); - let input = read_to_stdin(input)?; Ok((app_pk, committed_exe, input)) } diff --git a/crates/continuations/src/verifier/internal/types.rs b/crates/continuations/src/verifier/internal/types.rs index 400100f9ae..14560053c0 100644 --- a/crates/continuations/src/verifier/internal/types.rs +++ b/crates/continuations/src/verifier/internal/types.rs @@ -30,15 +30,15 @@ pub struct InternalVmVerifierInput { } assert_impl_all!(InternalVmVerifierInput: Serialize, DeserializeOwned); -/// The final output of the internal VM verifier. +/// A proof which can prove OpenVM program execution. #[derive(Deserialize, Serialize, Derivative)] #[serde(bound = "")] #[derivative(Clone(bound = "Com: Clone"))] -pub struct E2eStarkProof { +pub struct VmStarkProof { pub proof: Proof, pub user_public_values: Vec>, } -assert_impl_all!(E2eStarkProof: Serialize, DeserializeOwned); +assert_impl_all!(VmStarkProof: Serialize, DeserializeOwned); /// Aggregated state of all segments #[derive(Debug, Clone, Copy, AlignedBorrow)] diff --git a/crates/continuations/src/verifier/internal/vars.rs b/crates/continuations/src/verifier/internal/vars.rs index 0e6050591d..4fa00c004b 100644 --- a/crates/continuations/src/verifier/internal/vars.rs +++ b/crates/continuations/src/verifier/internal/vars.rs @@ -6,7 +6,7 @@ use openvm_stark_sdk::openvm_stark_backend::proof::Proof; use crate::{ verifier::{ - internal::types::{E2eStarkProof, InternalVmVerifierInput}, + internal::types::{InternalVmVerifierInput, VmStarkProof}, utils::write_field_slice, }, C, F, SC, @@ -44,7 +44,7 @@ impl Hintable for InternalVmVerifierInput { } } -impl Hintable for E2eStarkProof { +impl Hintable for VmStarkProof { type HintVariable = E2eStarkProofVariable; fn read(builder: &mut Builder) -> Self::HintVariable { diff --git a/crates/sdk/Cargo.toml b/crates/sdk/Cargo.toml index 3996ef8c50..777662106d 100644 --- a/crates/sdk/Cargo.toml +++ b/crates/sdk/Cargo.toml @@ -58,10 +58,6 @@ hex.workspace = true forge-fmt = { workspace = true, optional = true } rrs-lib = { workspace = true } -[dev-dependencies] -openvm-rv32im-guest.workspace = true - - [features] default = ["parallel", "jemalloc", "evm-verify"] evm-prove = ["openvm-native-recursion/evm-prove"] diff --git a/crates/sdk/guest/verify_openvm_stark/src/main.rs b/crates/sdk/guest/verify_openvm_stark/src/main.rs deleted file mode 100644 index bb9011f328..0000000000 --- a/crates/sdk/guest/verify_openvm_stark/src/main.rs +++ /dev/null @@ -1,24 +0,0 @@ -extern crate alloc; -use alloc::vec::Vec; - -use openvm::{define_verify_openvm_stark, io::read}; - -define_verify_openvm_stark!( - verify_openvm_stark, - env!("CARGO_MANIFEST_DIR"), - "root_verifier.asm" -); - -// const APP_EXE_COMMIT: [u32; 8] = [ -// 343014587, 230645511, 1447462186, 773379336, 1182270030, 1497892484, 461820702, 353704350, -// ]; -// const APP_VM_COMMIT: [u32; 8] = [ -// 445134834, 1133596793, 530952192, 425228715, 1806903712, 1362083369, 295028151, 482389308, -// ]; - -pub fn main() { - let app_exe_commit: [u32; 8] = read(); - let app_vm_commit: [u32; 8] = read(); - let pvs: Vec = read(); - verify_openvm_stark(&app_exe_commit, &app_vm_commit, &pvs); -} diff --git a/crates/sdk/src/codec.rs b/crates/sdk/src/codec.rs index 5bfabddce9..29c7f6c01d 100644 --- a/crates/sdk/src/codec.rs +++ b/crates/sdk/src/codec.rs @@ -4,7 +4,7 @@ use openvm_circuit::{ arch::ContinuationVmProof, system::memory::tree::public_values::UserPublicValuesProof, }; use openvm_continuations::verifier::{ - internal::types::E2eStarkProof, root::types::RootVmVerifierInput, + internal::types::VmStarkProof, root::types::RootVmVerifierInput, }; use openvm_native_compiler::ir::DIGEST_SIZE; use openvm_native_recursion::hints::{InnerBatchOpening, InnerFriProof, InnerQueryProof}; @@ -61,7 +61,7 @@ impl Encode for ContinuationVmProof { } } -impl Encode for E2eStarkProof { +impl Encode for VmStarkProof { fn encode(&self, writer: &mut W) -> Result<()> { self.proof.encode(writer)?; encode_slice(&self.user_public_values, writer) @@ -332,7 +332,7 @@ impl Decode for ContinuationVmProof { } } -impl Decode for E2eStarkProof { +impl Decode for VmStarkProof { fn decode(reader: &mut R) -> Result { let proof = Proof::decode(reader)?; let user_public_values = decode_vec(reader)?; diff --git a/crates/sdk/src/commit.rs b/crates/sdk/src/commit.rs index 7e7bbcdcfd..931d44e4b3 100644 --- a/crates/sdk/src/commit.rs +++ b/crates/sdk/src/commit.rs @@ -4,8 +4,7 @@ use openvm_circuit::{ arch::{instructions::exe::VmExe, VmConfig}, system::program::trace::VmCommittedExe, }; -use openvm_continuations::verifier::leaf::LeafVmVerifierConfig; -use openvm_native_compiler::{conversion::CompilerOptions, ir::DIGEST_SIZE}; +use openvm_native_compiler::ir::DIGEST_SIZE; use openvm_stark_backend::{config::StarkGenericConfig, p3_field::PrimeField32}; use openvm_stark_sdk::{ config::{baby_bear_poseidon2::BabyBearPoseidon2Engine, FriParameters}, @@ -14,14 +13,16 @@ use openvm_stark_sdk::{ p3_baby_bear::BabyBear, p3_bn254_fr::Bn254Fr, }; +use serde::{Deserialize, Serialize}; -use crate::{keygen::AppProvingKey, NonRootCommittedExe, F, SC}; +use crate::{NonRootCommittedExe, F, SC}; /// `AppExecutionCommit` has all the commitments users should check against the final proof. -pub struct AppExecutionCommit { +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct AppExecutionCommit { /// Commitment of the leaf VM verifier program which commits the VmConfig of App VM. /// Internal verifier will verify `leaf_vm_verifier_commit`. - pub leaf_vm_verifier_commit: [T; DIGEST_SIZE], + pub vm_commit: [u32; DIGEST_SIZE], /// Commitment of the executable. It's computed as /// compress( /// compress( @@ -31,10 +32,10 @@ pub struct AppExecutionCommit { /// hash(right_pad(pc_start, 0)) /// ) /// `right_pad` example, if pc_start = 123, right_pad(pc_start, 0) = \[123,0,0,0,0,0,0,0\] - pub exe_commit: [T; DIGEST_SIZE], + pub exe_commit: [u32; DIGEST_SIZE], } -impl AppExecutionCommit { +impl AppExecutionCommit { /// Users should use this function to compute `AppExecutionCommit` and check it against the /// final proof. pub fn compute>( @@ -42,26 +43,28 @@ impl AppExecutionCommit { app_exe: &NonRootCommittedExe, leaf_vm_verifier_exe: &NonRootCommittedExe, ) -> Self { - let exe_commit = app_exe + let exe_commit: [F; DIGEST_SIZE] = app_exe .compute_exe_commit(&app_vm_config.system().memory_config) .into(); - let leaf_vm_verifier_commit: [F; DIGEST_SIZE] = - leaf_vm_verifier_exe.committed_program.commitment.into(); + let vm_commit: [F; DIGEST_SIZE] = leaf_vm_verifier_exe.committed_program.commitment.into(); Self { - leaf_vm_verifier_commit, - exe_commit, + vm_commit: vm_commit.map(|x| x.as_canonical_u32()), + exe_commit: exe_commit.map(|x| x.as_canonical_u32()), } } - pub fn app_config_commit_to_bn254(&self) -> Bn254Fr { - babybear_digest_to_bn254(&self.leaf_vm_verifier_commit) + pub fn vm_commit_to_bn254(&self) -> Bn254Fr { + babybear_u32_digest_to_bn254(&self.vm_commit) } pub fn exe_commit_to_bn254(&self) -> Bn254Fr { - babybear_digest_to_bn254(&self.exe_commit) + babybear_u32_digest_to_bn254(&self.exe_commit) } } +fn babybear_u32_digest_to_bn254(digest: &[u32; DIGEST_SIZE]) -> Bn254Fr { + babybear_digest_to_bn254(&digest.map(F::from_canonical_u32)) +} pub(crate) fn babybear_digest_to_bn254(digest: &[F; DIGEST_SIZE]) -> Bn254Fr { let mut ret = Bn254Fr::ZERO; @@ -74,25 +77,6 @@ pub(crate) fn babybear_digest_to_bn254(digest: &[F; DIGEST_SIZE]) -> Bn254Fr { ret } -pub fn generate_leaf_committed_exe>( - leaf_fri_params: FriParameters, - compiler_options: CompilerOptions, - app_pk: &AppProvingKey, -) -> Arc { - let app_vm_vk = app_pk.app_vm_pk.vm_pk.get_vk(); - let leaf_engine = BabyBearPoseidon2Engine::new(leaf_fri_params); - let leaf_program = LeafVmVerifierConfig { - app_fri_params: app_pk.app_vm_pk.fri_params, - app_system_config: app_pk.app_vm_pk.vm_config.system().clone(), - compiler_options, - } - .build_program(&app_vm_vk); - Arc::new(VmCommittedExe::commit( - leaf_program.into(), - leaf_engine.config.pcs(), - )) -} - pub fn commit_app_exe( app_fri_params: FriParameters, app_exe: impl Into>, diff --git a/crates/sdk/src/lib.rs b/crates/sdk/src/lib.rs index e8e24454e1..91b2db0cc0 100644 --- a/crates/sdk/src/lib.rs +++ b/crates/sdk/src/lib.rs @@ -21,7 +21,7 @@ use openvm_circuit::{ }, }; use openvm_continuations::verifier::{ - internal::types::E2eStarkProof, + internal::types::VmStarkProof, root::{types::RootVmVerifierInput, RootVmVerifierConfig}, }; pub use openvm_continuations::{ @@ -61,7 +61,7 @@ pub mod prover; mod stdin; pub use stdin::*; -use crate::keygen::asm::program_to_asm; +use crate::{config::AggStarkConfig, keygen::asm::program_to_asm}; pub mod fs; pub mod types; @@ -264,6 +264,11 @@ impl> GenericSdk { Ok(agg_pk) } + pub fn agg_stark_keygen(&self, config: AggStarkConfig) -> Result { + let agg_pk = AggStarkProvingKey::keygen(config); + Ok(agg_pk) + } + pub fn generate_root_verifier_asm(&self, agg_stark_pk: &AggStarkProvingKey) -> String { let kernel_asm = RootVmVerifierConfig { leaf_fri_params: agg_stark_pk.leaf_vm_pk.fri_params, @@ -305,7 +310,7 @@ impl> GenericSdk { app_exe: Arc, agg_stark_pk: AggStarkProvingKey, inputs: StdIn, - ) -> Result> + ) -> Result> where VC::Executor: Chip, VC::Periphery: Chip, diff --git a/crates/sdk/src/prover/agg.rs b/crates/sdk/src/prover/agg.rs index 5bdd3f8d5f..d3c5fd29c1 100644 --- a/crates/sdk/src/prover/agg.rs +++ b/crates/sdk/src/prover/agg.rs @@ -2,7 +2,7 @@ use std::sync::Arc; use openvm_circuit::arch::ContinuationVmProof; use openvm_continuations::verifier::{ - internal::types::{E2eStarkProof, InternalVmVerifierInput}, + internal::types::{InternalVmVerifierInput, VmStarkProof}, leaf::types::LeafVmVerifierInput, root::types::RootVmVerifierInput, }; @@ -79,8 +79,8 @@ impl> AggStarkProver { self } - /// Generate a proof to aggregate app proofs. - pub fn generate_agg_proof(&self, app_proofs: ContinuationVmProof) -> Proof { + /// Generate the root proof for outer recursion. + pub fn generate_root_proof(&self, app_proofs: ContinuationVmProof) -> Proof { let root_verifier_input = self.generate_root_verifier_input(app_proofs); self.generate_root_proof_impl(root_verifier_input) } @@ -96,15 +96,15 @@ impl> AggStarkProver { ) -> RootVmVerifierInput { let leaf_proofs = self.generate_leaf_proofs(&app_proofs); let public_values = app_proofs.user_public_values.public_values; - let e2e_stark_proof = self.generate_e2e_stark_proof(leaf_proofs, public_values); + let e2e_stark_proof = self.aggregate_leaf_proofs(leaf_proofs, public_values); self.wrap_e2e_stark_proof(e2e_stark_proof) } - pub fn generate_e2e_stark_proof( + pub fn aggregate_leaf_proofs( &self, leaf_proofs: Vec>, public_values: Vec, - ) -> E2eStarkProof { + ) -> VmStarkProof { let mut internal_node_idx = -1; let mut internal_node_height = 0; let mut proofs = leaf_proofs; @@ -140,7 +140,7 @@ impl> AggStarkProver { }); internal_node_height += 1; } - E2eStarkProof { + VmStarkProof { proof: proofs.pop().unwrap(), user_public_values: public_values, } @@ -149,7 +149,7 @@ impl> AggStarkProver { /// Wrap the e2e stark proof until its heights meet the requirements of the root verifier. pub fn wrap_e2e_stark_proof( &self, - e2e_stark_proof: E2eStarkProof, + e2e_stark_proof: VmStarkProof, ) -> RootVmVerifierInput { let internal_commit = self .internal_prover @@ -214,9 +214,9 @@ pub fn wrap_e2e_stark_proof>( root_prover: &RootVerifierLocalProver, internal_commit: [F; DIGEST_SIZE], max_internal_wrapper_layers: usize, - e2e_stark_proof: E2eStarkProof, + e2e_stark_proof: VmStarkProof, ) -> RootVmVerifierInput { - let E2eStarkProof { + let VmStarkProof { mut proof, user_public_values, } = e2e_stark_proof; diff --git a/crates/sdk/src/prover/stark.rs b/crates/sdk/src/prover/stark.rs index 5b4ad96e30..fdec583f0f 100644 --- a/crates/sdk/src/prover/stark.rs +++ b/crates/sdk/src/prover/stark.rs @@ -2,7 +2,7 @@ use std::sync::Arc; use openvm_circuit::arch::VmConfig; use openvm_continuations::verifier::{ - internal::types::E2eStarkProof, root::types::RootVmVerifierInput, + internal::types::VmStarkProof, root::types::RootVmVerifierInput, }; use openvm_stark_backend::{proof::Proof, Chip}; use openvm_stark_sdk::engine::StarkFriEngine; @@ -58,7 +58,7 @@ impl> StarkProver { VC::Periphery: Chip, { let app_proof = self.app_prover.generate_app_proof(input); - self.agg_prover.generate_agg_proof(app_proof) + self.agg_prover.generate_root_proof(app_proof) } pub fn generate_root_verifier_input(&self, input: StdIn) -> RootVmVerifierInput @@ -71,7 +71,7 @@ impl> StarkProver { self.agg_prover.generate_root_verifier_input(app_proof) } - pub fn generate_e2e_stark_proof(&self, input: StdIn) -> E2eStarkProof + pub fn generate_e2e_stark_proof(&self, input: StdIn) -> VmStarkProof where VC: VmConfig, VC::Executor: Chip, @@ -80,6 +80,6 @@ impl> StarkProver { let app_proof = self.app_prover.generate_app_proof(input); let leaf_proofs = self.agg_prover.generate_leaf_proofs(&app_proof); self.agg_prover - .generate_e2e_stark_proof(leaf_proofs, app_proof.user_public_values.public_values) + .aggregate_leaf_proofs(leaf_proofs, app_proof.user_public_values.public_values) } } diff --git a/crates/sdk/tests/integration_test.rs b/crates/sdk/tests/integration_test.rs index 128ff3fdc2..a1cc993c14 100644 --- a/crates/sdk/tests/integration_test.rs +++ b/crates/sdk/tests/integration_test.rs @@ -6,7 +6,6 @@ use openvm_circuit::{ arch::{ hasher::poseidon2::vm_poseidon2_hasher, ContinuationVmProof, ExecutionError, GenerationError, SingleSegmentVmExecutor, SystemConfig, VmConfig, VmExecutor, - DEFAULT_MAX_NUM_PUBLIC_VALUES, }, system::{memory::tree::public_values::UserPublicValuesProof, program::trace::VmCommittedExe}, }; @@ -28,11 +27,9 @@ use openvm_native_recursion::{ wrapper::Halo2WrapperProvingKey, RawEvmProof, }, - hints::Hintable, types::InnerConfig, vars::StarkProofVariable, }; -use openvm_rv32im_guest::hint_load_by_key_encode; use openvm_rv32im_transpiler::{ Rv32ITranspilerExtension, Rv32IoTranspilerExtension, Rv32MTranspilerExtension, }; @@ -40,13 +37,11 @@ use openvm_sdk::{ codec::{Decode, Encode}, commit::AppExecutionCommit, config::{AggConfig, AggStarkConfig, AppConfig, Halo2Config, SdkSystemConfig, SdkVmConfig}, - keygen::{AggStarkProvingKey, AppProvingKey}, + keygen::AppProvingKey, types::{EvmHalo2Verifier, EvmProof}, DefaultStaticVerifierPvHandler, Sdk, StdIn, }; -use openvm_stark_backend::{ - keygen::types::LinearConstraint, p3_field::PrimeField32, p3_matrix::Matrix, -}; +use openvm_stark_backend::{keygen::types::LinearConstraint, p3_matrix::Matrix}; use openvm_stark_sdk::{ config::{ baby_bear_poseidon2::{BabyBearPoseidon2Config, BabyBearPoseidon2Engine}, @@ -356,7 +351,7 @@ fn test_static_verifier_custom_pv_handler() { &app_pk.leaf_committed_exe, ); let exe_commit = commits.exe_commit_to_bn254(); - let leaf_verifier_commit = commits.app_config_commit_to_bn254(); + let leaf_verifier_commit = commits.vm_commit_to_bn254(); let pv_handler = CustomPvHandler { exe_commit, @@ -617,121 +612,3 @@ fn test_segmentation_retry() { .sum(); assert!(new_total_height < total_height); } - -#[test] -fn test_verify_openvm_stark_e2e() -> Result<()> { - const ASM_FILENAME: &str = "root_verifier.asm"; - let sdk = Sdk::new(); - let mut pkg_dir = PathBuf::from(env!("CARGO_MANIFEST_DIR")).to_path_buf(); - pkg_dir.push("guest/fib"); - - let vm_config = SdkVmConfig::builder() - .system(SdkSystemConfig { - config: SystemConfig::default().with_continuations(), - }) - .rv32i(Default::default()) - .rv32m(Default::default()) - .io(Default::default()) - .native(Default::default()) - .build(); - assert!(vm_config.system.config.continuation_enabled); - let elf = sdk.build( - Default::default(), - &vm_config, - pkg_dir, - &Default::default(), - None, - )?; - - let app_exe = sdk.transpile(elf, vm_config.transpiler())?; - let fri_params = FriParameters::new_for_testing(LEAF_LOG_BLOWUP); - let app_config = AppConfig::new_with_leaf_fri_params(fri_params, vm_config.clone(), fri_params); - - let app_pk = sdk.app_keygen(app_config.clone())?; - let committed_app_exe = sdk.commit_app_exe(fri_params, app_exe.clone())?; - - let commits = - AppExecutionCommit::compute(&vm_config, &committed_app_exe, &app_pk.leaf_committed_exe); - - let agg_pk = AggStarkProvingKey::keygen(AggStarkConfig { - max_num_user_public_values: DEFAULT_MAX_NUM_PUBLIC_VALUES, - leaf_fri_params: FriParameters::new_for_testing(LEAF_LOG_BLOWUP), - internal_fri_params: FriParameters::new_for_testing(INTERNAL_LOG_BLOWUP), - root_fri_params: FriParameters::new_for_testing(ROOT_LOG_BLOWUP), - profiling: false, - compiler_options: CompilerOptions { - enable_cycle_tracker: true, - ..Default::default() - }, - root_max_constraint_degree: (1 << ROOT_LOG_BLOWUP) + 1, - }); - let asm = sdk.generate_root_verifier_asm(&agg_pk); - let asm_path = format!( - "{}/guest/verify_openvm_stark/{}", - env!("CARGO_MANIFEST_DIR"), - ASM_FILENAME - ); - std::fs::write(asm_path, asm)?; - - let e2e_stark_proof = sdk.generate_e2e_stark_proof( - Arc::new(app_pk), - committed_app_exe, - agg_pk, - StdIn::default(), - )?; - - let verify_exe = { - let mut pkg_dir = PathBuf::from(env!("CARGO_MANIFEST_DIR")).to_path_buf(); - pkg_dir.push("guest/verify_openvm_stark"); - let elf = sdk.build( - Default::default(), - &vm_config, - pkg_dir, - &Default::default(), - None, - )?; - sdk.transpile(elf, vm_config.transpiler())? - }; - - let pvs = [13u32, 21, 0, 0, 0, 0, 0, 0]; - - let exe_commit_u32: Vec<_> = commits - .exe_commit - .iter() - .map(|x| x.as_canonical_u32()) - .collect(); - let vm_commit_u32: Vec<_> = commits - .leaf_vm_verifier_commit - .iter() - .map(|x| x.as_canonical_u32()) - .collect(); - let pvs_u32: Vec<_> = pvs - .iter() - .flat_map(|x| x.to_le_bytes()) - .map(|x| x as u32) - .collect(); - - let key = ASM_FILENAME - .as_bytes() - .iter() - .cloned() - .chain(exe_commit_u32.iter().flat_map(|x| x.to_le_bytes())) - .chain(vm_commit_u32.iter().flat_map(|x| x.to_le_bytes())) - .chain(pvs_u32.iter().flat_map(|x| x.to_le_bytes())) - .collect(); - let mut stdin = StdIn::default(); - let to_encode: Vec> = e2e_stark_proof.proof.write(); - let value = hint_load_by_key_encode(&to_encode); - stdin.add_key_value(key, value); - - let exe_commit_u32_8: [u32; 8] = exe_commit_u32.try_into().unwrap(); - let vm_commit_u32_8: [u32; 8] = vm_commit_u32.try_into().unwrap(); - - stdin.write(&exe_commit_u32_8); - stdin.write(&vm_commit_u32_8); - stdin.write(&pvs_u32); - - sdk.execute(verify_exe, vm_config, stdin)?; - - Ok(()) -} diff --git a/crates/toolchain/openvm/src/lib.rs b/crates/toolchain/openvm/src/lib.rs index 898676da20..d0507db84c 100644 --- a/crates/toolchain/openvm/src/lib.rs +++ b/crates/toolchain/openvm/src/lib.rs @@ -31,8 +31,6 @@ pub mod utils; #[cfg(not(target_os = "zkvm"))] pub mod host; -#[cfg(target_os = "zkvm")] -pub mod verify_stark; #[cfg(target_os = "zkvm")] core::arch::global_asm!(include_str!("memset.s")); diff --git a/docs/specs/ISA.md b/docs/specs/ISA.md index d901c6e320..06c8d1bb5c 100644 --- a/docs/specs/ISA.md +++ b/docs/specs/ISA.md @@ -429,12 +429,12 @@ with user input-output. The RV32IM extension defines the following phantom sub-instructions. -| Name | Discriminant | Operands | Description | -|-------------------| ------------ | -------- |--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| Rv32HintInput | 0x20 | `_` | Pops a vector `hint` of field elements from the input stream and resets the hint stream to equal the vector `[(hint.len() as u32).to_le_bytes()), hint].concat()`. | -| Rv32PrintStr | 0x21 | `a,b,_` | Peeks at `[r32{0}(a)..r32{0}(a) + r32{0}(b)]_2`, tries to convert to byte array and then UTF-8 string and prints to host stdout. Prints error message if conversion fails. Does not change any VM state. | -| Rv32HintRandom | 0x22 | `a,_,_` | Resets the hint stream to `4 * r32{0}(a)` random bytes. The source of randomness is the host operating system (`rand::rngs::OsRng`). Its result is not constrained in any way. | -| Rv32HintLoadByKey | 0x23 | `a,b,_` | Look up the value by key `[r32{0}{a}:r32{0}{b}]_2` and prepend the value into `input_stream`. Users should use `openvm-rv32im-guest::hint_load_by_key_encode` to encode the value when constructing inputs. | +| Name | Discriminant | Operands | Description | +|-------------------| ------------ | -------- |----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| Rv32HintInput | 0x20 | `_` | Pops a vector `hint` of field elements from the input stream and resets the hint stream to equal the vector `[(hint.len() as u32).to_le_bytes()), hint].concat()`. | +| Rv32PrintStr | 0x21 | `a,b,_` | Peeks at `[r32{0}(a)..r32{0}(a) + r32{0}(b)]_2`, tries to convert to byte array and then UTF-8 string and prints to host stdout. Prints error message if conversion fails. Does not change any VM state. | +| Rv32HintRandom | 0x22 | `a,_,_` | Resets the hint stream to `4 * r32{0}(a)` random bytes. The source of randomness is the host operating system (`rand::rngs::OsRng`). Its result is not constrained in any way. | +| Rv32HintLoadByKey | 0x23 | `a,b,_` | Look up the value by key `[r32{0}{a}:r32{0}{b}]_2` and prepend the value into `input_stream`. The logical value is `Vec>`. The serialization of `Vec` follows the format `[length, ]`. Both length and content encoded as little-endian bytes. | ### Native Extension The native extension operates over native field elements and has instructions tailored for STARK proof recursion. It diff --git a/docs/specs/RISCV.md b/docs/specs/RISCV.md index 904c675167..d921872497 100644 --- a/docs/specs/RISCV.md +++ b/docs/specs/RISCV.md @@ -76,6 +76,9 @@ the guest must take care to validate all data and account for behavior in cases |--------------|-----|-------------|---------|--------|------------------------------------------------------------------------------------------------------------------------------| | nativestorew | R | 0001011 | 111 | 0x2 | Stores the 4-byte word `rs1` at address `rd` in native address space. The address `rd` must be aligned to a 4-byte boundary. | +`nativestorew` connects RV32 address space and native address space. We put it in RV32 extension because its +implementation is here. But we use `funct3 = 111` because the native extension has an available slot. + ## Keccak Extension | RISC-V Inst | FMT | opcode[6:0] | funct3 | funct7 | RISC-V description and notes | @@ -121,6 +124,8 @@ These use the _custom-0_ opcode prefix and funct3 = 0b111. | lfii | R | 0001011 | 111 | 0 | Long Form Instruction Indicator. `rd = rs1 = rs2 = 0` | | gi | R | 0001011 | 111 | 1 | Gap Indicator. `rd = rs1 = rs2 = 0` | +`nativestorew` also uses `funct3 = 111`. It's listed in the RV32 extension. + ## Algebra Extension Modular arithmetic instructions depend on the modulus `N`. The ordered list of supported moduli should be saved in the `.openvm` section of the ELF file in the serialized format. This is achieved by the `moduli_declare!` macro; for example, the following code diff --git a/guest-libs/verify_stark/guest/Cargo.toml b/guest-libs/verify_stark/guest/Cargo.toml new file mode 100644 index 0000000000..794bcdc1da --- /dev/null +++ b/guest-libs/verify_stark/guest/Cargo.toml @@ -0,0 +1,25 @@ +[package] +name = "openvm-verify-stark" +description = "OpenVM guest library for verifying STARKs" +version.workspace = true +edition.workspace = true +rust-version.workspace = true +authors.workspace = true +homepage.workspace = true +repository.workspace = true +license.workspace = true + +[target.'cfg(not(target_os = "zkvm"))'.dependencies] +openvm-native-recursion.workspace = true +openvm-rv32im-guest.workspace = true +openvm-sdk = { workspace = true } +openvm-stark-sdk = { workspace = true } + +[dev-dependencies] +openvm-sdk = { workspace = true } +openvm-circuit = { workspace = true, features = ["parallel"] } +openvm-stark-sdk = { workspace = true } +openvm-build.workspace = true +openvm-native-compiler.workspace = true +openvm-verify-stark.workspace = true +eyre.workspace = true \ No newline at end of file diff --git a/crates/sdk/guest/verify_openvm_stark/.gitignore b/guest-libs/verify_stark/guest/examples/verify_openvm_stark/.gitignore similarity index 100% rename from crates/sdk/guest/verify_openvm_stark/.gitignore rename to guest-libs/verify_stark/guest/examples/verify_openvm_stark/.gitignore diff --git a/crates/sdk/guest/verify_openvm_stark/Cargo.toml b/guest-libs/verify_stark/guest/examples/verify_openvm_stark/Cargo.toml similarity index 50% rename from crates/sdk/guest/verify_openvm_stark/Cargo.toml rename to guest-libs/verify_stark/guest/examples/verify_openvm_stark/Cargo.toml index f788fc95b1..772d00014c 100644 --- a/crates/sdk/guest/verify_openvm_stark/Cargo.toml +++ b/guest-libs/verify_stark/guest/examples/verify_openvm_stark/Cargo.toml @@ -5,9 +5,8 @@ version = "0.0.0" edition = "2021" [dependencies] -openvm = { path = "../../../toolchain/openvm", features = ["std"] } -hex-literal = { version = "0.4.1", default-features = false } -bytemuck = { version = "1.20.0", features = ["extern_crate_alloc"] } +openvm = { path = "../../../../../crates/toolchain/openvm", features = ["std"] } +openvm-verify-stark = { path = "../../../guest" } [features] default = [] diff --git a/guest-libs/verify_stark/guest/examples/verify_openvm_stark/openvm.toml b/guest-libs/verify_stark/guest/examples/verify_openvm_stark/openvm.toml new file mode 100644 index 0000000000..cb8c41f306 --- /dev/null +++ b/guest-libs/verify_stark/guest/examples/verify_openvm_stark/openvm.toml @@ -0,0 +1,4 @@ +[app_vm_config.rv32i] +[app_vm_config.rv32m] +[app_vm_config.io] +[app_vm_config.native] diff --git a/guest-libs/verify_stark/guest/examples/verify_openvm_stark/src/main.rs b/guest-libs/verify_stark/guest/examples/verify_openvm_stark/src/main.rs new file mode 100644 index 0000000000..bb8f8c18e4 --- /dev/null +++ b/guest-libs/verify_stark/guest/examples/verify_openvm_stark/src/main.rs @@ -0,0 +1,18 @@ +extern crate alloc; +use alloc::vec::Vec; + +use openvm::io::read; +use openvm_verify_stark::define_verify_openvm_stark; + +define_verify_openvm_stark!( + verify_openvm_stark, + env!("CARGO_MANIFEST_DIR"), + "root_verifier.asm" +); + +pub fn main() { + let app_exe_commit: [u32; 8] = read(); + let app_vm_commit: [u32; 8] = read(); + let pvs: Vec = read(); + verify_openvm_stark(&app_exe_commit, &app_vm_commit, &pvs); +} diff --git a/guest-libs/verify_stark/guest/src/host.rs b/guest-libs/verify_stark/guest/src/host.rs new file mode 100644 index 0000000000..92e3b53969 --- /dev/null +++ b/guest-libs/verify_stark/guest/src/host.rs @@ -0,0 +1,28 @@ +use openvm_native_recursion::hints::Hintable; +use openvm_rv32im_guest::hint_load_by_key_encode; +use openvm_sdk::SC; +use openvm_stark_sdk::{openvm_stark_backend::proof::Proof, p3_baby_bear::BabyBear}; + +/// Compute the hint key for `verify_openvm_stark` function, which reads a stark proof from stream +/// `kv_store`. +pub fn compute_hint_key_for_verify_openvm_stark( + asm_filename: &str, + exe_commit_u32: &[u32; 8], + vm_commit_u32: &[u32; 8], + pvs: &[u8], +) -> Vec { + asm_filename + .as_bytes() + .iter() + .cloned() + .chain(exe_commit_u32.iter().flat_map(|x| x.to_le_bytes())) + .chain(vm_commit_u32.iter().flat_map(|x| x.to_le_bytes())) + .chain(pvs.iter().cloned()) + .collect() +} + +/// Encode a proof into a KV store value so `verify_openvm_stark` can hint it. +pub fn encode_proof_to_kv_store_value(proof: &Proof) -> Vec { + let to_encode: Vec> = proof.write(); + hint_load_by_key_encode(&to_encode) +} diff --git a/crates/toolchain/openvm/src/verify_stark.rs b/guest-libs/verify_stark/guest/src/lib.rs similarity index 78% rename from crates/toolchain/openvm/src/verify_stark.rs rename to guest-libs/verify_stark/guest/src/lib.rs index 1382dcd9d5..02ce76748b 100644 --- a/crates/toolchain/openvm/src/verify_stark.rs +++ b/guest-libs/verify_stark/guest/src/lib.rs @@ -1,3 +1,6 @@ +#[cfg(not(target_os = "zkvm"))] +pub mod host; + /// Define a function that verifies an OpenVM Stark proof. /// To define this function, users need to specify the function name and an ASM file containing the /// assembly code for the verification(this ASM file can be generated by @@ -5,18 +8,17 @@ /// parent folder and filename of the ASM file. /// To call this function: /// 1. users need to provide `app_exe_commit`/`app_vm_commit`/`user_pvs`(user public values) as the -/// arguments. CAREFUL: `app_exe_commit`/`app_vm_commit`/`user_pvs` are in u32 and are interpreted -/// as native fields. `user_pvs` are the exact public values of that proof. Which means, if a guest -/// program calls `reveal_u32` to publish an u32(e.g. `258`) in the proof, that u32 is decomposed -/// into 4 field elements in byte. In `user_pvs`, it should be an u32 array, `[2, 1, 0, 0]`. +/// arguments. CAREFUL: `app_exe_commit`/`app_vm_commit` are in u32 and are interpreted as native +/// fields. `user_pvs` are the exact public values of that proof. Here we assume all the public +/// values are `u8`s. /// 2. Provide the corresponding stark proof in the key-value store in OpenVM streams. The key -/// should -/// be the concatenation of the filename, `app_exe_commit`, `app_vm_commit`, and `user_pvs` in -/// little-endian bytes. +/// should be the concatenation of the filename, `app_exe_commit`, `app_vm_commit`, and +/// `user_pvs` in little-endian bytes. Users can use +/// `openvm::host::compute_hint_key_for_verify_openvm_stark` to compute the hint key. #[macro_export] macro_rules! define_verify_openvm_stark { ($fn_name: ident, $asm_folder: expr, $asm_filename: literal) => { - pub fn $fn_name(app_exe_commit: &[u32; 8], app_vm_commit: &[u32; 8], user_pvs: &[u32]) { + pub fn $fn_name(app_exe_commit: &[u32; 8], app_vm_commit: &[u32; 8], user_pvs: &[u8]) { // The memory location for the start of the heap. const HEAP_START_ADDRESS: u32 = 1 << 24; const FIELDS_PER_U32: u32 = 4; @@ -28,7 +30,7 @@ macro_rules! define_verify_openvm_stark { .cloned() .chain(app_exe_commit.iter().flat_map(|x| x.to_le_bytes())) .chain(app_vm_commit.iter().flat_map(|x| x.to_le_bytes())) - .chain(user_pvs.iter().flat_map(|x| x.to_le_bytes())) + .chain(user_pvs.iter().cloned()) .collect(); openvm::io::hint_load_by_key(&hint_key); // Store the expected public values into the beginning of the native heap. @@ -42,7 +44,7 @@ macro_rules! define_verify_openvm_stark { native_addr += FIELDS_PER_U32; } for &x in user_pvs { - openvm::io::store_u32_to_native(native_addr, x); + openvm::io::store_u32_to_native(native_addr, x as u32); native_addr += FIELDS_PER_U32; } // Assumption: the asm file should be generated by SDK. The code block should: diff --git a/guest-libs/verify_stark/guest/tests/integration_test.rs b/guest-libs/verify_stark/guest/tests/integration_test.rs new file mode 100644 index 0000000000..4f5ba6d6f7 --- /dev/null +++ b/guest-libs/verify_stark/guest/tests/integration_test.rs @@ -0,0 +1,126 @@ +#[cfg(test)] +mod tests { + use std::{path::PathBuf, sync::Arc}; + + use eyre::Result; + use openvm_circuit::arch::{SystemConfig, DEFAULT_MAX_NUM_PUBLIC_VALUES}; + use openvm_native_compiler::conversion::CompilerOptions; + use openvm_sdk::{ + commit::AppExecutionCommit, + config::{AggStarkConfig, AppConfig, SdkSystemConfig, SdkVmConfig}, + keygen::AggStarkProvingKey, + Sdk, StdIn, + }; + use openvm_stark_sdk::config::FriParameters; + use openvm_verify_stark::host::{ + compute_hint_key_for_verify_openvm_stark, encode_proof_to_kv_store_value, + }; + + const LEAF_LOG_BLOWUP: usize = 2; + const INTERNAL_LOG_BLOWUP: usize = 3; + const ROOT_LOG_BLOWUP: usize = 4; + + #[test] + fn test_verify_openvm_stark_e2e() -> Result<()> { + const ASM_FILENAME: &str = "root_verifier.asm"; + let sdk = Sdk::new(); + let mut pkg_dir = PathBuf::from(env!("CARGO_MANIFEST_DIR")).to_path_buf(); + pkg_dir.pop(); + pkg_dir.pop(); + pkg_dir.pop(); + pkg_dir.push("crates/sdk/guest/fib"); + + let vm_config = SdkVmConfig::builder() + .system(SdkSystemConfig { + config: SystemConfig::default().with_continuations(), + }) + .rv32i(Default::default()) + .rv32m(Default::default()) + .io(Default::default()) + .native(Default::default()) + .build(); + assert!(vm_config.system.config.continuation_enabled); + let elf = sdk.build( + Default::default(), + &vm_config, + pkg_dir, + &Default::default(), + None, + )?; + + let app_exe = sdk.transpile(elf, vm_config.transpiler())?; + let fri_params = FriParameters::new_for_testing(LEAF_LOG_BLOWUP); + let app_config = + AppConfig::new_with_leaf_fri_params(fri_params, vm_config.clone(), fri_params); + + let app_pk = sdk.app_keygen(app_config.clone())?; + let committed_app_exe = sdk.commit_app_exe(fri_params, app_exe.clone())?; + + let commits = + AppExecutionCommit::compute(&vm_config, &committed_app_exe, &app_pk.leaf_committed_exe); + + let agg_pk = AggStarkProvingKey::keygen(AggStarkConfig { + max_num_user_public_values: DEFAULT_MAX_NUM_PUBLIC_VALUES, + leaf_fri_params: FriParameters::new_for_testing(LEAF_LOG_BLOWUP), + internal_fri_params: FriParameters::new_for_testing(INTERNAL_LOG_BLOWUP), + root_fri_params: FriParameters::new_for_testing(ROOT_LOG_BLOWUP), + profiling: false, + compiler_options: CompilerOptions { + enable_cycle_tracker: true, + ..Default::default() + }, + root_max_constraint_degree: (1 << ROOT_LOG_BLOWUP) + 1, + }); + let asm = sdk.generate_root_verifier_asm(&agg_pk); + let asm_path = format!( + "{}/examples/verify_openvm_stark/{}", + env!("CARGO_MANIFEST_DIR"), + ASM_FILENAME + ); + std::fs::write(asm_path, asm)?; + + let e2e_stark_proof = sdk.generate_e2e_stark_proof( + Arc::new(app_pk), + committed_app_exe, + agg_pk, + StdIn::default(), + )?; + + let verify_exe = { + let mut pkg_dir = PathBuf::from(env!("CARGO_MANIFEST_DIR")).to_path_buf(); + pkg_dir.push("examples/verify_openvm_stark"); + let elf = sdk.build( + Default::default(), + &vm_config, + pkg_dir, + &Default::default(), + None, + )?; + sdk.transpile(elf, vm_config.transpiler())? + }; + + // app_exe publishes 7th and 8th fibonacci numbers. + let pvs: Vec = [13u32, 21, 0, 0, 0, 0, 0, 0] + .iter() + .flat_map(|x| x.to_le_bytes()) + .collect(); + + let mut stdin = StdIn::default(); + let key = compute_hint_key_for_verify_openvm_stark( + ASM_FILENAME, + &commits.exe_commit, + &commits.vm_commit, + &pvs, + ); + let value = encode_proof_to_kv_store_value(&e2e_stark_proof.proof); + stdin.add_key_value(key, value); + + stdin.write(&commits.exe_commit); + stdin.write(&commits.vm_commit); + stdin.write(&pvs); + + sdk.execute(verify_exe, vm_config, stdin)?; + + Ok(()) + } +} From c0c3090326e435f0b5a66d8c7499aead0b3bb0e5 Mon Sep 17 00:00:00 2001 From: Xinding Wei Date: Thu, 15 May 2025 16:09:27 -0700 Subject: [PATCH 11/15] refactor: CLI Build & Setup (#1649) - `cargo openvm setup` supports skipping halo2 proving keys. - `cargo openvm setup` outputs halo2 PK and stark PK as separated files. - `cargo openvm build` outputs `AppExecutionCommit` in json. The old output(`exe_commit.bytes`) was incorrect. close INT-3950 --- Cargo.lock | 1 - crates/cli/src/commands/build.rs | 26 +---- crates/cli/src/commands/keygen.rs | 19 +++- crates/cli/src/commands/prove.rs | 17 ++- crates/cli/src/commands/setup.rs | 134 +++++++++++++++-------- crates/cli/src/default.rs | 9 +- crates/cli/src/util.rs | 27 +++-- crates/cli/tests/build.rs | 3 +- crates/sdk/src/commit.rs | 4 - crates/sdk/src/fs.rs | 15 ++- guest-libs/verify_stark/guest/Cargo.toml | 1 - 11 files changed, 152 insertions(+), 104 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index a726c28bde..7031868f8f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5024,7 +5024,6 @@ name = "openvm-verify-stark" version = "1.1.2" dependencies = [ "eyre", - "openvm-build", "openvm-circuit", "openvm-native-compiler", "openvm-native-recursion", diff --git a/crates/cli/src/commands/build.rs b/crates/cli/src/commands/build.rs index 576a9e044d..1d495b2bcf 100644 --- a/crates/cli/src/commands/build.rs +++ b/crates/cli/src/commands/build.rs @@ -12,18 +12,11 @@ use openvm_build::{ GuestOptions, }; use openvm_circuit::arch::{InitFileGenerator, OPENVM_DEFAULT_INIT_FILE_NAME}; -use openvm_sdk::{ - commit::{commit_app_exe, committed_exe_as_bn254}, - fs::write_exe_to_file, - Sdk, -}; +use openvm_sdk::{commit::commit_app_exe, fs::write_exe_to_file, Sdk}; use openvm_transpiler::{elf::Elf, openvm_platform::memory::MEM_SIZE}; use crate::{ - default::{ - DEFAULT_APP_CONFIG_PATH, DEFAULT_APP_EXE_PATH, DEFAULT_COMMITTED_APP_EXE_PATH, - DEFAULT_EXE_COMMIT_PATH, - }, + default::{DEFAULT_APP_CONFIG_PATH, DEFAULT_APP_EXE_PATH, DEFAULT_COMMITTED_APP_EXE_PATH}, util::{find_manifest_dir, read_config_toml_or_default}, }; @@ -78,14 +71,6 @@ pub struct BuildArgs { )] pub committed_exe_output: PathBuf, - #[arg( - long, - default_value = DEFAULT_EXE_COMMIT_PATH, - help = "Output path for the exe commit (bn254 commit of committed program)", - help_heading = "OpenVM Options" - )] - pub exe_commit_output: PathBuf, - #[arg( long, default_value = OPENVM_DEFAULT_INIT_FILE_NAME, @@ -428,13 +413,6 @@ pub fn build(build_args: &BuildArgs, cargo_args: &BuildCargoArgs) -> Result Result<()> { - let app_config = read_config_toml_or_default(&self.config)?; - let app_pk = Sdk::new().app_keygen(app_config)?; - write_app_vk_to_file(app_pk.get_app_vk(), &self.vk_output)?; - write_app_pk_to_file(app_pk, &self.output)?; + keygen(&self.config, &self.output, &self.vk_output)?; Ok(()) } } + +pub(crate) fn keygen( + config: impl AsRef, + output: impl AsRef, + vk_output: impl AsRef, +) -> Result<()> { + let app_config = read_config_toml_or_default(config)?; + let app_pk = Sdk::new().app_keygen(app_config)?; + write_app_vk_to_file(app_pk.get_app_vk(), vk_output.as_ref())?; + write_app_pk_to_file(app_pk, output.as_ref())?; + Ok(()) +} diff --git a/crates/cli/src/commands/prove.rs b/crates/cli/src/commands/prove.rs index db33cf766a..9c53e111f8 100644 --- a/crates/cli/src/commands/prove.rs +++ b/crates/cli/src/commands/prove.rs @@ -8,13 +8,15 @@ use openvm_sdk::{ commit::AppExecutionCommit, config::{AggregationTreeConfig, SdkVmConfig}, fs::{ - encode_to_file, read_agg_pk_from_file, read_app_pk_from_file, read_exe_from_file, + encode_to_file, read_agg_stark_pk_from_file, read_app_pk_from_file, read_exe_from_file, write_app_proof_to_file, }, keygen::AppProvingKey, NonRootCommittedExe, Sdk, StdIn, }; +#[cfg(feature = "evm-prove")] +use crate::util::read_default_agg_pk; use crate::{ default::*, input::{read_to_stdin, Input}, @@ -110,16 +112,11 @@ impl ProveCmd { ); println!("exe commit: {:?}", commits.exe_commit); println!("vm commit: {:?}", commits.vm_commit); - - let agg_pk = read_agg_pk_from_file(default_agg_pk_path()).map_err(|e| { + let agg_stark_pk = read_agg_stark_pk_from_file(default_agg_stark_pk_path()).map_err(|e| { eyre::eyre!("Failed to read aggregation proving key: {}\nPlease run 'cargo openvm setup' first", e) })?; - let stark_proof = sdk.generate_e2e_stark_proof( - app_pk, - committed_exe, - agg_pk.agg_stark_pk, - input, - )?; + let stark_proof = + sdk.generate_e2e_stark_proof(app_pk, committed_exe, agg_stark_pk, input)?; encode_to_file(output, stark_proof)?; } #[cfg(feature = "evm-prove")] @@ -146,7 +143,7 @@ impl ProveCmd { println!("vm commit: {:?}", commits.vm_commit_to_bn254()); println!("Generating EVM proof, this may take a lot of compute and memory..."); - let agg_pk = read_agg_pk_from_file(default_agg_pk_path()).map_err(|e| { + let agg_pk = read_default_agg_pk().map_err(|e| { eyre::eyre!("Failed to read aggregation proving key: {}\nPlease run 'cargo openvm setup' first", e) })?; let evm_proof = diff --git a/crates/cli/src/commands/setup.rs b/crates/cli/src/commands/setup.rs index c617112707..d84255c7f4 100644 --- a/crates/cli/src/commands/setup.rs +++ b/crates/cli/src/commands/setup.rs @@ -9,16 +9,21 @@ use clap::Parser; use eyre::{eyre, Result}; use openvm_native_recursion::halo2::utils::CacheHalo2ParamsReader; use openvm_sdk::{ - config::AggConfig, + config::{AggConfig, AggStarkConfig}, fs::{ - write_agg_pk_to_file, write_evm_halo2_verifier_to_folder, EVM_HALO2_VERIFIER_BASE_NAME, - EVM_HALO2_VERIFIER_INTERFACE_NAME, EVM_HALO2_VERIFIER_PARENT_NAME, + write_agg_halo2_pk_to_file, write_agg_stark_pk_to_file, write_evm_halo2_verifier_to_folder, + EVM_HALO2_VERIFIER_BASE_NAME, EVM_HALO2_VERIFIER_INTERFACE_NAME, + EVM_HALO2_VERIFIER_PARENT_NAME, }, DefaultStaticVerifierPvHandler, Sdk, }; -use crate::default::{ - default_agg_pk_path, default_asm_path, default_evm_halo2_verifier_path, default_params_dir, +use crate::{ + default::{ + default_agg_halo2_pk_path, default_agg_stark_pk_path, default_asm_path, + default_evm_halo2_verifier_path, default_params_dir, + }, + util::read_default_agg_pk, }; #[derive(Parser)] @@ -26,57 +31,100 @@ use crate::default::{ name = "evm-proving-setup", about = "Set up for generating EVM proofs. ATTENTION: this requires large amounts of computation and memory. " )] -pub struct EvmProvingSetupCmd {} +pub struct EvmProvingSetupCmd { + #[arg( + long, + default_value = "false", + help = "use --evm to also generate proving keys for EVM verifier" + )] + pub evm: bool, + #[arg( + long, + default_value = "false", + help = "force keygen even if the proving keys already exist" + )] + pub force_agg_keygen: bool, +} impl EvmProvingSetupCmd { pub async fn run(&self) -> Result<()> { - let default_agg_pk_path = default_agg_pk_path(); + let default_agg_stark_pk_path = default_agg_stark_pk_path(); let default_params_dir = default_params_dir(); let default_evm_halo2_verifier_path = default_evm_halo2_verifier_path(); + let default_asm_path = default_asm_path(); + if !self.evm { + if PathBuf::from(&default_agg_stark_pk_path).exists() { + println!("Aggregation stark proving key already exists"); + return Ok(()); + } + let agg_stark_config = AggStarkConfig::default(); + let sdk = Sdk::new(); + let agg_stark_pk = sdk.agg_stark_keygen(agg_stark_config)?; + + println!("Writing stark proving key to file..."); + write_agg_stark_pk_to_file(&agg_stark_pk, default_agg_stark_pk_path)?; + + println!("Generating root verifier ASM..."); + let root_verifier_asm = sdk.generate_root_verifier_asm(&agg_stark_pk); + + println!("Writing root verifier ASM to file..."); + write(&default_asm_path, root_verifier_asm)?; + } else { + let default_agg_halo2_pk_path = default_agg_halo2_pk_path(); + if PathBuf::from(&default_agg_stark_pk_path).exists() + && PathBuf::from(&default_agg_halo2_pk_path).exists() + && PathBuf::from(&default_evm_halo2_verifier_path) + .join(EVM_HALO2_VERIFIER_PARENT_NAME) + .exists() + && PathBuf::from(&default_evm_halo2_verifier_path) + .join(EVM_HALO2_VERIFIER_BASE_NAME) + .exists() + && PathBuf::from(&default_evm_halo2_verifier_path) + .join("interfaces") + .join(EVM_HALO2_VERIFIER_INTERFACE_NAME) + .exists() + { + println!("Aggregation proving key and verifier contract already exist"); + return Ok(()); + } else if !Self::check_solc_installed() { + return Err(eyre!( + "solc is not installed, please install solc to continue" + )); + } - if PathBuf::from(&default_agg_pk_path).exists() - && PathBuf::from(&default_evm_halo2_verifier_path) - .join(EVM_HALO2_VERIFIER_PARENT_NAME) - .exists() - && PathBuf::from(&default_evm_halo2_verifier_path) - .join(EVM_HALO2_VERIFIER_BASE_NAME) - .exists() - && PathBuf::from(&default_evm_halo2_verifier_path) - .join("interfaces") - .join(EVM_HALO2_VERIFIER_INTERFACE_NAME) - .exists() - { - println!("Aggregation proving key and verifier contract already exist"); - return Ok(()); - } else if !Self::check_solc_installed() { - return Err(eyre!( - "solc is not installed, please install solc to continue" - )); - } - - Self::download_params(10, 24).await?; - let params_reader = CacheHalo2ParamsReader::new(default_params_dir); - let agg_config = AggConfig::default(); - let sdk = Sdk::new(); + Self::download_params(10, 24).await?; + let params_reader = CacheHalo2ParamsReader::new(&default_params_dir); + let agg_config = AggConfig::default(); + let sdk = Sdk::new(); - println!("Generating proving key..."); - let agg_pk = sdk.agg_keygen(agg_config, ¶ms_reader, &DefaultStaticVerifierPvHandler)?; + let agg_pk = if !self.force_agg_keygen + && PathBuf::from(&default_agg_stark_pk_path).exists() + && PathBuf::from(&default_agg_halo2_pk_path).exists() + { + read_default_agg_pk()? + } else { + println!("Generating proving key..."); + sdk.agg_keygen(agg_config, ¶ms_reader, &DefaultStaticVerifierPvHandler)? + }; - println!("Generating root verifier ASM..."); - let root_verifier_asm = sdk.generate_root_verifier_asm(&agg_pk.agg_stark_pk); + println!("Generating root verifier ASM..."); + let root_verifier_asm = sdk.generate_root_verifier_asm(&agg_pk.agg_stark_pk); - println!("Generating verifier contract..."); - let verifier = sdk.generate_halo2_verifier_solidity(¶ms_reader, &agg_pk)?; + println!("Generating verifier contract..."); + let verifier = sdk.generate_halo2_verifier_solidity(¶ms_reader, &agg_pk)?; - println!("Writing proving key to file..."); - write_agg_pk_to_file(agg_pk, &default_agg_pk_path)?; + println!("Writing stark proving key to file..."); + write_agg_stark_pk_to_file(&agg_pk.agg_stark_pk, &default_agg_stark_pk_path)?; - println!("Writing root verifier ASM to file..."); - write(default_asm_path(), root_verifier_asm)?; + println!("Writing halo2 proving key to file..."); + write_agg_halo2_pk_to_file(&agg_pk.halo2_pk, &default_agg_halo2_pk_path)?; - println!("Writing verifier contract to file..."); - write_evm_halo2_verifier_to_folder(verifier, &default_evm_halo2_verifier_path)?; + println!("Writing root verifier ASM to file..."); + write(&default_asm_path, root_verifier_asm)?; + println!("Writing verifier contract to file..."); + write_evm_halo2_verifier_to_folder(verifier, &default_evm_halo2_verifier_path)?; + } Ok(()) } diff --git a/crates/cli/src/default.rs b/crates/cli/src/default.rs index 224d3dc219..641c74b8ea 100644 --- a/crates/cli/src/default.rs +++ b/crates/cli/src/default.rs @@ -7,15 +7,18 @@ pub const DEFAULT_MANIFEST_DIR: &str = "."; pub const DEFAULT_APP_CONFIG_PATH: &str = "./openvm.toml"; pub const DEFAULT_APP_EXE_PATH: &str = "./openvm/app.vmexe"; -pub const DEFAULT_EXE_COMMIT_PATH: &str = "./openvm/exe_commit.bytes"; pub const DEFAULT_COMMITTED_APP_EXE_PATH: &str = "./openvm/committed_app_exe.bc"; pub const DEFAULT_APP_PK_PATH: &str = "./openvm/app.pk"; pub const DEFAULT_APP_VK_PATH: &str = "./openvm/app.vk"; pub const DEFAULT_APP_PROOF_PATH: &str = "./openvm/app.proof"; pub const DEFAULT_EVM_PROOF_PATH: &str = "./openvm/evm.proof"; -pub fn default_agg_pk_path() -> String { - env::var("HOME").unwrap() + "/.openvm/agg.pk" +pub fn default_agg_stark_pk_path() -> String { + env::var("HOME").unwrap() + "/.openvm/agg_stark.pk" +} + +pub fn default_agg_halo2_pk_path() -> String { + env::var("HOME").unwrap() + "/.openvm/agg_halo2.pk" } pub fn default_asm_path() -> String { diff --git a/crates/cli/src/util.rs b/crates/cli/src/util.rs index 8db24aa89e..aa4eaceba4 100644 --- a/crates/cli/src/util.rs +++ b/crates/cli/src/util.rs @@ -4,29 +4,42 @@ use std::{ }; use eyre::Result; -use openvm_sdk::config::{AppConfig, SdkVmConfig}; +use openvm_sdk::{ + config::{AppConfig, SdkVmConfig}, + fs::{read_agg_halo2_pk_from_file, read_agg_stark_pk_from_file}, + keygen::AggProvingKey, +}; use serde::de::DeserializeOwned; -use crate::default::default_app_config; +use crate::default::{default_agg_halo2_pk_path, default_agg_stark_pk_path, default_app_config}; -pub(crate) fn read_to_struct_toml(path: &PathBuf) -> Result { - let toml = read_to_string(path.as_ref() as &Path)?; +pub(crate) fn read_to_struct_toml(path: impl AsRef) -> Result { + let toml = read_to_string(path)?; let ret = toml::from_str(&toml)?; Ok(ret) } -pub fn read_config_toml_or_default(config: &PathBuf) -> Result> { - if config.exists() { +pub fn read_config_toml_or_default(config: impl AsRef) -> Result> { + if config.as_ref().exists() { read_to_struct_toml(config) } else { println!( "{:?} not found, using default application configuration", - config + config.as_ref() ); Ok(default_app_config()) } } +pub fn read_default_agg_pk() -> Result { + let agg_stark_pk = read_agg_stark_pk_from_file(default_agg_stark_pk_path())?; + let halo2_pk = read_agg_halo2_pk_from_file(default_agg_halo2_pk_path())?; + Ok(AggProvingKey { + agg_stark_pk, + halo2_pk, + }) +} + pub fn find_manifest_dir(mut current_dir: PathBuf) -> Result { current_dir = current_dir.canonicalize()?; while !current_dir.join("Cargo.toml").exists() { diff --git a/crates/cli/tests/build.rs b/crates/cli/tests/build.rs index 1037d07e2d..42d3f09ee6 100644 --- a/crates/cli/tests/build.rs +++ b/crates/cli/tests/build.rs @@ -2,7 +2,7 @@ use std::path::PathBuf; use cargo_openvm::{ commands::{build, BuildArgs, BuildCargoArgs}, - default::{DEFAULT_APP_EXE_PATH, DEFAULT_COMMITTED_APP_EXE_PATH, DEFAULT_EXE_COMMIT_PATH}, + default::{DEFAULT_APP_EXE_PATH, DEFAULT_COMMITTED_APP_EXE_PATH}, }; use eyre::Result; use openvm_build::RUSTC_TARGET; @@ -18,7 +18,6 @@ fn default_build_args(example: &str) -> BuildArgs { .join("openvm.toml"), exe_output: PathBuf::from(DEFAULT_APP_EXE_PATH), committed_exe_output: PathBuf::from(DEFAULT_COMMITTED_APP_EXE_PATH), - exe_commit_output: PathBuf::from(DEFAULT_EXE_COMMIT_PATH), init_file_name: OPENVM_DEFAULT_INIT_FILE_NAME.to_string(), } } diff --git a/crates/sdk/src/commit.rs b/crates/sdk/src/commit.rs index 931d44e4b3..4a9a43a9b9 100644 --- a/crates/sdk/src/commit.rs +++ b/crates/sdk/src/commit.rs @@ -85,7 +85,3 @@ pub fn commit_app_exe( let app_engine = BabyBearPoseidon2Engine::new(app_fri_params); Arc::new(VmCommittedExe::::commit(exe, app_engine.config.pcs())) } - -pub fn committed_exe_as_bn254(committed_exe: &NonRootCommittedExe) -> Bn254Fr { - babybear_digest_to_bn254(&committed_exe.get_program_commit().into()) -} diff --git a/crates/sdk/src/fs.rs b/crates/sdk/src/fs.rs index e2b11075bd..d212a899f1 100644 --- a/crates/sdk/src/fs.rs +++ b/crates/sdk/src/fs.rs @@ -11,7 +11,7 @@ use serde::{de::DeserializeOwned, Serialize}; use crate::{ codec::{Decode, Encode}, - keygen::{AggProvingKey, AppProvingKey, AppVerifyingKey}, + keygen::{AggStarkProvingKey, AppProvingKey, AppVerifyingKey, Halo2ProvingKey}, types::{EvmHalo2Verifier, EvmProof}, F, OPENVM_VERSION, SC, }; @@ -74,12 +74,19 @@ pub fn write_root_verifier_input_to_file>( encode_to_file(path, input) } -pub fn read_agg_pk_from_file>(path: P) -> Result { +pub fn read_agg_stark_pk_from_file>(path: P) -> Result { read_from_file_bitcode(path) } +pub fn read_agg_halo2_pk_from_file>(path: P) -> Result { + read_from_file_bitcode(path) +} + +pub fn write_agg_halo2_pk_to_file>(pk: &Halo2ProvingKey, path: P) -> Result<()> { + write_to_file_bitcode(path, pk) +} -pub fn write_agg_pk_to_file>(agg_pk: AggProvingKey, path: P) -> Result<()> { - write_to_file_bitcode(path, agg_pk) +pub fn write_agg_stark_pk_to_file>(pk: &AggStarkProvingKey, path: P) -> Result<()> { + write_to_file_bitcode(path, pk) } pub fn read_evm_proof_from_file>(path: P) -> Result { diff --git a/guest-libs/verify_stark/guest/Cargo.toml b/guest-libs/verify_stark/guest/Cargo.toml index 794bcdc1da..070083edad 100644 --- a/guest-libs/verify_stark/guest/Cargo.toml +++ b/guest-libs/verify_stark/guest/Cargo.toml @@ -19,7 +19,6 @@ openvm-stark-sdk = { workspace = true } openvm-sdk = { workspace = true } openvm-circuit = { workspace = true, features = ["parallel"] } openvm-stark-sdk = { workspace = true } -openvm-build.workspace = true openvm-native-compiler.workspace = true openvm-verify-stark.workspace = true eyre.workspace = true \ No newline at end of file From bc5745663291d6d462cc36d865a0696182d5e453 Mon Sep 17 00:00:00 2001 From: HrikB <23041116+HrikB@users.noreply.github.com> Date: Fri, 16 May 2025 13:03:04 -0400 Subject: [PATCH 12/15] fix: verifier read dir (#1658) Fixes the path from which verifier contract is read. I tested the full CI flow on an aws instance to double check everything is working. --- crates/sdk/src/fs.rs | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/crates/sdk/src/fs.rs b/crates/sdk/src/fs.rs index d212a899f1..f6a6d46edf 100644 --- a/crates/sdk/src/fs.rs +++ b/crates/sdk/src/fs.rs @@ -100,17 +100,20 @@ pub fn write_evm_proof_to_file>(proof: EvmProof, path: P) -> Resu } pub fn read_evm_halo2_verifier_from_folder>(folder: P) -> Result { - let halo2_verifier_code_path = folder.as_ref().join(EVM_HALO2_VERIFIER_PARENT_NAME); - let openvm_verifier_code_path = folder.as_ref().join(EVM_HALO2_VERIFIER_BASE_NAME); - let interface_path = folder + let folder = folder .as_ref() + .join("src") + .join(format!("v{}", OPENVM_VERSION)); + let halo2_verifier_code_path = folder.join(EVM_HALO2_VERIFIER_PARENT_NAME); + let openvm_verifier_code_path = folder.join(EVM_HALO2_VERIFIER_BASE_NAME); + let interface_path = folder .join("interfaces") .join(EVM_HALO2_VERIFIER_INTERFACE_NAME); let halo2_verifier_code = read_to_string(halo2_verifier_code_path)?; let openvm_verifier_code = read_to_string(openvm_verifier_code_path)?; let interface = read_to_string(interface_path)?; - let artifact_path = folder.as_ref().join(EVM_VERIFIER_ARTIFACT_FILENAME); + let artifact_path = folder.join(EVM_VERIFIER_ARTIFACT_FILENAME); let artifact: EvmVerifierByteCode = serde_json::from_reader(File::open(artifact_path)?)?; Ok(EvmHalo2Verifier { From 2ab0573ca57c4feb8acae5b8c92e55b6caa169b8 Mon Sep 17 00:00:00 2001 From: HrikB <23041116+HrikB@users.noreply.github.com> Date: Fri, 16 May 2025 14:31:32 -0400 Subject: [PATCH 13/15] docs: add CHANGELOG.md (#1660) Adds CHANGELOG.md to the repo. - [x] CHANGELOG notes for `v1.1.0` to be added --------- Co-authored-by: Jonathan Wang <31040440+jonathanpwang@users.noreply.github.com> --- CHANGELOG.md | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 CHANGELOG.md diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000000..1e014d4030 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,19 @@ +# Changelog + +## v1.1.2 (2025-05-08) + +- The solidity verifier contract no longer has any awareness of the OpenVM patch version. `{MAJOR_VERSION}.{MINOR_VERSION}` is the minimum information necessary to identify the verifier contract since any verifier contract changes will be accompanied by a minor version bump. + +## v1.1.1 (2025-05-03) + +- Adds `OpenVmHalo2Verifier` generation to the SDK which is a thin wrapper around the original `Halo2Verifier` contract exposing a more user-friendly interface. +- Updates the CLI to generate the new `OpenVmHalo2Verifier` contract during `cargo openvm setup`. +- Removes the ability to generate the old `Halo2Verifier` contract from the SDK and CLI. +- Changes the `EvmProof` struct to align with the interface of the `OpenVmHalo2Verifier` contract. +- Formats the verifier contract during generation for better readability on block explorers. +- For verifier contract compilation, explicitly sets the `solc` config via standard-json input for metadata consistency. + +## v1.1.0 (2025-05-02) + +### Security Fixes +- Fixes security vulnerability [OpenVM allows the byte decomposition of pc in AUIPC chip to overflow](https://github.com/advisories/GHSA-jf2r-x3j4-23m7) From 274c4229b429dfa740a201b5335526ba423713e6 Mon Sep 17 00:00:00 2001 From: stephenh-axiom-xyz Date: Mon, 19 May 2025 18:15:35 -0400 Subject: [PATCH 14/15] chore: add entry point to book examples without openvm::init!() (#1663) Resolves INT-4065. --- .github/workflows/cli.yml | 4 +++- examples/i256/src/main.rs | 2 ++ examples/keccak/src/main.rs | 2 ++ examples/u256/src/main.rs | 2 ++ 4 files changed, 9 insertions(+), 1 deletion(-) diff --git a/.github/workflows/cli.yml b/.github/workflows/cli.yml index 4602e9c05f..bc92fb3607 100644 --- a/.github/workflows/cli.yml +++ b/.github/workflows/cli.yml @@ -56,11 +56,13 @@ jobs: - name: Build and run book examples working-directory: examples run: | + set -e for dir in */; do if [ -f "${dir}Cargo.toml" ]; then echo "Building ${dir%/}" cd "$dir" - cargo openvm build && cargo openvm run + cargo openvm build + cargo openvm run cd .. fi done diff --git a/examples/i256/src/main.rs b/examples/i256/src/main.rs index 0cd58e82e8..63f685bf2c 100644 --- a/examples/i256/src/main.rs +++ b/examples/i256/src/main.rs @@ -3,6 +3,8 @@ use core::array; use openvm_bigint_guest::I256; +openvm::entry!(main); + const N: usize = 16; type Matrix = [[I256; N]; N]; diff --git a/examples/keccak/src/main.rs b/examples/keccak/src/main.rs index b17be3fd8f..3debdc75a0 100644 --- a/examples/keccak/src/main.rs +++ b/examples/keccak/src/main.rs @@ -6,6 +6,8 @@ use openvm_keccak256_guest::keccak256; // ANCHOR_END: imports // ANCHOR: main +openvm::entry!(main); + pub fn main() { let test_vectors = [ ( diff --git a/examples/u256/src/main.rs b/examples/u256/src/main.rs index 37e0e17ca9..663b3f12e0 100644 --- a/examples/u256/src/main.rs +++ b/examples/u256/src/main.rs @@ -3,6 +3,8 @@ use core::array; use openvm_bigint_guest::U256; +openvm::entry!(main); + const N: usize = 16; type Matrix = [[U256; N]; N]; From 1ec4308c444805333dd9f564f71ff9fad330ea6a Mon Sep 17 00:00:00 2001 From: stephenh-axiom-xyz Date: Tue, 20 May 2025 15:27:39 -0400 Subject: [PATCH 15/15] feat: cargo CLI update target directory (#1662) Resolves INT-3882, and records changes in the book as per INT-4038. Most importantly, reworks command outputs to primarily use the `target/` directory that Cargo uses and assumes full control over. - User can specify `target_dir`; if unspecified, `cargo openvm` will use the Cargo default - Keys are stored in `${target_dir}/openvm/`, while all other artifacts are stored in `${target_dir}/openvm/${profile}` - User can copy outputs to `${output_dir}` - **NOTE** - generated proofs are stored at `.`, as this file should generally be the final output Other changes include: - `build` no longer generates app commits - `run` will call `build` before attempting to execute - `prove` will call `build` before attempting to generate a proof - Replaced all references to `evm-proving-setup` with references to `setup` - `openvm.toml`'s default location is in the working directory instead of `.` The basic workflow should now be `cargo openvm prove ...` into `cargo openvm verify ...`. --- .github/workflows/cli.yml | 6 +- Cargo.lock | 2 +- benchmarks/execute/src/main.rs | 4 +- book/src/SUMMARY.md | 1 + book/src/writing-apps/build.md | 165 ++++++------ book/src/writing-apps/prove.md | 29 +- book/src/writing-apps/run.md | 75 ++++++ book/src/writing-apps/verify.md | 4 +- crates/cli/Cargo.toml | 2 +- crates/cli/src/bin/cargo-openvm.rs | 2 +- crates/cli/src/commands/build.rs | 246 +++++++++-------- crates/cli/src/commands/keygen.rs | 90 +++++-- crates/cli/src/commands/prove.rs | 217 ++++++++++----- crates/cli/src/commands/run.rs | 248 +++++++++++++++++- crates/cli/src/commands/setup.rs | 6 +- crates/cli/src/commands/verify.rs | 61 ++++- crates/cli/src/default.rs | 13 +- crates/cli/src/util.rs | 93 ++++++- crates/cli/tests/app_e2e.rs | 99 +++++-- crates/cli/tests/build.rs | 203 ++++++++------ .../programs/multi/calculator/Cargo.toml | 2 +- .../programs/multi/string-utils/Cargo.toml | 2 +- crates/sdk/src/config/mod.rs | 21 +- crates/sdk/src/fs.rs | 3 + crates/sdk/src/lib.rs | 9 +- 25 files changed, 1166 insertions(+), 437 deletions(-) create mode 100644 book/src/writing-apps/run.md diff --git a/.github/workflows/cli.yml b/.github/workflows/cli.yml index bc92fb3607..6ac63a8489 100644 --- a/.github/workflows/cli.yml +++ b/.github/workflows/cli.yml @@ -49,9 +49,9 @@ jobs: working-directory: examples/ecc run: | cargo openvm build - mv openvm/app.vmexe app1.vmexe + mv target/openvm/release/ecc-example.vmexe app1.vmexe cargo openvm build - cmp app1.vmexe openvm/app.vmexe || (echo "Build is not deterministic!" && exit 1) + cmp app1.vmexe target/openvm/release/ecc-example.vmexe || (echo "Build is not deterministic!" && exit 1) - name: Build and run book examples working-directory: examples @@ -72,7 +72,7 @@ jobs: run: | export RUST_BACKTRACE=1 cargo build - cargo run --bin cargo-openvm -- openvm keygen --config ./example/app_config.toml --output app.pk --vk-output app.vk + cargo run --bin cargo-openvm -- openvm keygen --config ./example/app_config.toml --output-dir . - name: Run CLI tests working-directory: crates/cli diff --git a/Cargo.lock b/Cargo.lock index 7031868f8f..3959c0d0cb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1287,10 +1287,10 @@ version = "1.1.2" dependencies = [ "aws-config", "aws-sdk-s3", - "bitcode", "clap", "eyre", "hex", + "itertools 0.14.0", "openvm-build", "openvm-circuit", "openvm-native-recursion", diff --git a/benchmarks/execute/src/main.rs b/benchmarks/execute/src/main.rs index 80db3ec5a4..a05baeea44 100644 --- a/benchmarks/execute/src/main.rs +++ b/benchmarks/execute/src/main.rs @@ -1,4 +1,4 @@ -use cargo_openvm::{default::DEFAULT_APP_CONFIG_PATH, util::read_config_toml_or_default}; +use cargo_openvm::util::read_config_toml_or_default; use clap::{Parser, ValueEnum}; use eyre::Result; use openvm_benchmarks_utils::{get_elf_path, get_programs_dir, read_elf_file}; @@ -106,7 +106,7 @@ fn main() -> Result<()> { let elf_path = get_elf_path(&program_dir); let elf = read_elf_file(&elf_path)?; - let config_path = program_dir.join(DEFAULT_APP_CONFIG_PATH); + let config_path = program_dir.join("openvm.toml"); let vm_config = read_config_toml_or_default(&config_path)?.app_vm_config; let exe = VmExe::from_elf(elf, vm_config.transpiler())?; diff --git a/book/src/SUMMARY.md b/book/src/SUMMARY.md index 104ee4bd74..0bbe1daf92 100644 --- a/book/src/SUMMARY.md +++ b/book/src/SUMMARY.md @@ -12,6 +12,7 @@ - [Overview](./writing-apps/overview.md) - [Writing a Program](./writing-apps/write-program.md) - [Compiling](./writing-apps/build.md) +- [Running a Program](./writing-apps/run.md) - [Generating Proofs](./writing-apps/prove.md) - [Verifying Proofs](./writing-apps/verify.md) - [Solidity SDK](./writing-apps/solidity.md) diff --git a/book/src/writing-apps/build.md b/book/src/writing-apps/build.md index cb65db10c9..4f0d207598 100644 --- a/book/src/writing-apps/build.md +++ b/book/src/writing-apps/build.md @@ -9,135 +9,142 @@ The command `cargo openvm build` compiles the program on host to an executable f It first compiles the program normally on your _host_ platform with RISC-V and then transpiles it to a different target. See here for some explanation of [cross-compilation](https://rust-lang.github.io/rustup/cross-compilation.html). Right now we use `riscv32im-risc0-zkvm-elf` target which is available in the [Rust toolchain](https://doc.rust-lang.org/rustc/platform-support/riscv32im-risc0-zkvm-elf.html), but we will contribute an OpenVM target to Rust in the future. -## Build flags +## Build Flags -The following flags are available for the `cargo openvm build` command: +The following flags are available for the `cargo openvm build` command. You can run `cargo openvm build --help` for this list within the command line. -- `--manifest-dir ` +Generally, outputs will always be built to the **target directory**, which will either be determined by the manifest path or explicitly set using the `--target-dir` option. By default Cargo sets this to be `/target/`. - **Description**: Specifies the directory containing the `Cargo.toml` file for the guest code. +OpenVM-specific artifacts will be placed in `${target_dir}/openvm/`, but if `--output-dir` is specified they will be copied to `${output-dir}/` as well. - **Default**: The current directory (`.`). +### OpenVM Options - **Usage Example**: If your `Cargo.toml` is located in `my_project/`, you can run: +- `--no-transpile` - ```bash - cargo openvm build --manifest-dir my_project - ``` + **Description**: Skips transpilation into an OpenVM-compatible `.vmexe` executable when set. - This ensures the build command is executed in that directory. +- `--config ` -- `--target-dir ` + **Description**: Path to the OpenVM config `.toml` file that specifies the VM extensions. By default will search the manifest directory for `openvm.toml`. If no file is found, OpenVM will use a default configuration. Currently the CLI only supports known extensions listed in the [Using Existing Extensions](../custom-extensions/overview.md) section. To use other extensions, use the [SDK](../advanced-usage/sdk.md). - **Description**: Specifies the directory where the guest binary will be built. If not specified, the default target directory is used. +- `--output_dir ` - **Default**: The `target` directory in the package root directory. + **Description**: Output directory for OpenVM artifacts to be copied to. Keys will be placed in `${output-dir}/`, while all other artifacts will be in `${output-dir}/${profile}`. - **Usage Example**: To build the guest binary in the `my_target` directory: +- `--init-file-name ` - ```bash - cargo openvm build --target-dir my_target - ``` + **Description**: Name of the generated initialization file, which will be written into the manifest directory. -- `--features ` + **Default**: `openvm_init.rs` - **Description**: Passes a list of feature flags to the Cargo build process. These flags enable or disable conditional compilation features defined in your `Cargo.toml`. +### Package Selection - **Usage Example**: To enable the `my_feature` feature: +As with `cargo build`, default package selection depends on the working directory. If the working directory is a subdirectory of a specific package, then only that package will be built. Else, all packages in the workspace will be built by default. - ```bash - cargo openvm build --features my_feature - ``` +- `--package ` -- `--bin ` + **Description**: Builds only the specified packages. This flag may be specified multiple times or as a comma-separated list. - **Description**: Restricts the build to the binary target with the given name, similar to `cargo build --bin `. If your project has multiple target types (binaries, libraries, examples, etc.), using `--bin ` narrows down the build to the binary target with the given name. +- `--workspace` - **Usage Example**: + **Description**: Builds all members of the workspace (alias `--all`). - ```bash - cargo openvm build --bin my_bin - ``` +- `--exclude ` -- `--example ` + **Description**: Excludes the specified packages. Must be used in conjunction with `--workspace`. This flag may be specified multiple times or as a comma-separated list. - **Description**: Restricts the build to the example target with the given name, similar to `cargo build --example `. Projects often include code samples or demos under the examples directory, and this flag focuses on compiling a specific example. +### Target Selection - **Usage Example**: +By default all package libraries and binaries will be built. To build samples or demos under the `examples` directory, use either the `--example` or `--examples` option. - ```bash - cargo openvm build --example my_example - ``` +- `--lib` -- `--no-transpile` + **Description**: Builds the package's library. - **Description**: After building the guest code, doesn't transpile the target ELF into an OpenVM-compatible executable (by default it does). +- `--bin ` - **Usage Example**: + **Description**: Builds the specified binary. This flag may be specified multiple times or as a comma-separated list. - ```bash - cargo openvm build --no-transpile - ``` +- `--bins` -- `--config ` + **Description**: Builds all binary targets. + +- `--example ` + + **Description**: Builds the specified example. This flag may be specified multiple times or as a comma-separated list. + +- `--examples` + + **Description**: Builds all example targets. + +- `--all-targets` + + **Description**: Builds all package targets. Equivalent to specifying `--lib` `--bins` `--examples`. + +### Feature Selection + +The following options enable or disable conditional compilation features defined in your `Cargo.toml`. + +- `-F`, `--features ` + + **Description**: Space or comma separated list of features to activate. Features of workspace members may be enabled with `package-name/feature-name` syntax. This flag may also be specified multiple times. + +- `--all-features` + + **Description**: Activates all available features of all selected packages. + +- `--no-default-features` + + **Description**: Do not activate the `default` feature of the selected packages. + +### Compilation Options + +- `--profile ` - **Description**: Specifies the path to a .toml configuration file that defines which VM extensions to use. + **Description**: Builds with the given profile. Common profiles are `dev` (faster builds, less optimization) and `release` (slower builds, more optimization). For more information on profiles, see [Cargo's reference page](https://doc.rust-lang.org/cargo/reference/profiles.html). - **Default**: `./openvm.toml` if `--config` flag is not provided. + **Default**: `release` - **Usage Example**: +### Output Options - ```bash - cargo openvm build --config path/to/openvm.toml - ``` +- `--target_dir ` - This allows you to customize the extensions. Currently the CLI only supports known extensions listed in the [Using Existing Extensions](../custom-extensions/overview.md) section. To use other extensions, use the [SDK](../advanced-usage/sdk.md). + **Description**: Directory for all generated artifacts and intermediate files. Defaults to directory `target/` at the root of the workspace. -- `--exe-output ` +### Display Options - **Description**: Sets the output path for the transpiled program. +- `-v`, `--verbose` - **Default**: `./openvm/app.vmexe` if `--exe-output` flag is not provided. + **Description**: Use verbose output. - **Usage Example**: To specify a custom output filename: +- `-q`, `--quiet` - ```bash - cargo openvm build --exe-output ./output/custom_name.vmexe - ``` + **Description**: Do not print Cargo log messages. -- `--profile ` +- `--color ` - **Description**: Determines the build profile used by Cargo. Common profiles are dev (faster builds, less optimization) and release (slower builds, more optimization). + **Description**: Controls when colored output is used. - **Default**: release + **Default**: `always` - **Usage Example**: +### Manifest Options - ```bash - cargo openvm build --profile dev - ``` +- `--manifest-path ` -- `--help` + **Description**: Path to the guest code Cargo.toml file. By default, `build` searches for the file in the current or any parent directory. The `build` command will be executed in that directory. - **Description**: Prints a help message describing the available options and their usage. +- `--ignore-rust-version` - **Usage Example**: + **Description**: Ignores rust-version specification in packages. - ```bash - cargo openvm build --help - ``` +- `--locked` -## Running a Program + **Description**: Asserts the same dependencies and versions are used as when the existing Cargo.lock file was originally generated. -After building and transpiling a program, you can execute it using the `run` command. The `run` command has the following arguments: +- `--offline` -```bash -cargo openvm run - --exe - --config - --input -``` + **Description**: Prevents Cargo from accessing the network for any reason. -If `--exe` and/or `--config` are not provided, the command will search for these files in `./openvm/app.vmexe` and `./openvm.toml` respectively. If `./openvm.toml` is not present, a default configuration will be used. +- `--frozen` -If your program doesn't require inputs, you can (and should) omit the `--input` flag. + **Description**: Equivalent to specifying both `--locked` and `--offline`. diff --git a/book/src/writing-apps/prove.md b/book/src/writing-apps/prove.md index 88d7a40548..1cdfbc9cfa 100644 --- a/book/src/writing-apps/prove.md +++ b/book/src/writing-apps/prove.md @@ -4,41 +4,46 @@ Generating a proof using the CLI is simple - first generate a key, then generate ```bash cargo openvm keygen -cargo openvm prove [app | evm] +cargo openvm prove [app | stark | evm] ``` ## Key Generation -The `keygen` CLI command has the following optional arguments: +The `keygen` command generates both an application proving and verification key. ```bash cargo openvm keygen --config - --output - --vk_output ``` -If `--config` is not provided, the command will search for `./openvm.toml` and use that as the application configuration if present. If it is not present, a default configuration will be used. +Similarly to `build`, `run`, and `prove`, options `--manifest-path`, `--target-dir`, and `--output-dir` are provided. -If `--output` and/or `--vk_output` are not provided, the keys will be written to default locations `./openvm/app.pk` and/or `./openvm/app.vk` respectively. +If `--config` is not specified, the command will search for `openvm.toml` in the manifest directory. If the file isn't found, a default configuration will be used. + +The proving and verification key will be written to `${target_dir}/openvm/` (and `--output-dir` if specified). ## Proof Generation -The `prove` CLI command has the following optional arguments: +The `prove` CLI command, at its core, uses the options below. `prove` gets access to all of the options that `run` has (see [Running a Program](../writing-apps/run.md) for more information). ```bash -cargo openvm prove [app | evm] - --app_pk +cargo openvm prove [app | stark | evm] + --app-pk --exe --input - --output + --proof ``` +If `--app-pk` is not provided, the command will search for a proving key at `${target_dir}/openvm/app.pk`. + +If `--exe` is not provided, the command will call `build` before generating a proof. + If your program doesn't require inputs, you can (and should) omit the `--input` flag. -If `--app_pk` and/or `--exe` are not provided, the command will search for these files in `./openvm/app.pk` and `./openvm/app.vmexe` respectively. Similarly, if `--output` is not provided then the command will write the proof to `./openvm/[app | evm].proof` by default. +If `--proof` is not provided then the command will write the proof to `./[app | stark | evm].proof` by default. + -The `app` subcommand is used to generate an application-level proof, while the `evm` command generates an end-to-end EVM proof. +The `app` subcommand generates an application-level proof, the `stark` command generates an aggregated root-level proof, while the `evm` command generates an end-to-end EVM proof. For more information on aggregation, see [this specification](https://github.com/openvm-org/openvm/blob/bf8df90b13f4e80bb76dbb71f255a12154c84838/docs/specs/continuations.md). > ⚠️ **WARNING** > In order to run the `evm` subcommand, you must have previously called the costly `cargo openvm setup`, which requires very large amounts of computation and memory (~200 GB). diff --git a/book/src/writing-apps/run.md b/book/src/writing-apps/run.md new file mode 100644 index 0000000000..66a8dbf439 --- /dev/null +++ b/book/src/writing-apps/run.md @@ -0,0 +1,75 @@ +# Running a Program + +After building and transpiling a program, you can execute it using the `run` command. For example, you can call: + +```bash +cargo openvm run + --exe + --config + --input +``` + +If `--exe` is not provided, OpenVM will call `build` prior to attempting to run the executable. Note that only one executable may be run, so if your project contains multiple targets you will have to specify which one to run using the `--bin` or `--example` flag. + +If your program doesn't require inputs, you can (and should) omit the `--input` flag. + +## Run Flags + +Many of the options for `cargo openvm run` will be passed to `cargo openvm build` if `--exe` is not specified. For more information on `build` (or `run`'s **Feature Selection**, **Compilation**, **Output**, **Display**, and/or **Manifest** options) see [Compiling](./writing-apps/build.md). + +### OpenVM Options + +- `--exe ` + + **Description**: Path to the OpenVM executable, if specified `build` will be skipped. + +- `--config ` + + **Description**: Path to the OpenVM config `.toml` file that specifies the VM extensions. By default will search the manifest directory for `openvm.toml`. If no file is found, OpenVM will use a default configuration. Currently the CLI only supports known extensions listed in the [Using Existing Extensions](../custom-extensions/overview.md) section. To use other extensions, use the [SDK](../advanced-usage/sdk.md). + +- `--output_dir ` + + **Description**: Output directory for OpenVM artifacts to be copied to. Keys will be placed in `${output-dir}/`, while all other artifacts will be in `${output-dir}/${profile}`. + +- `--input ` + + **Description**: Input to the OpenVM program, or a hex string. + +- `--init-file-name ` + + **Description**: Name of the generated initialization file, which will be written into the manifest directory. + + **Default**: `openvm_init.rs` + +### Package Selection + +- `--package ` + + **Description**: The package to run, by default the package in the current workspace. + +### Target Selection + +Only one target may be built and run. + +- `--bin ` + + **Description**: Runs the specified binary. + +- `--example ` + + **Description**: Runs the specified example. + +## Examples + +### Running a Specific Binary + +```bash +cargo openvm run --bin bin_name +``` + +### Skipping Build Using `--exe` + +```bash +cargo openvm build --output-dir ./my_output_dir +cargo openvm run --exe ./my_output_dir/bin_name.vmexe +``` diff --git a/book/src/writing-apps/verify.md b/book/src/writing-apps/verify.md index 06e44b7389..bf813a350b 100644 --- a/book/src/writing-apps/verify.md +++ b/book/src/writing-apps/verify.md @@ -10,9 +10,7 @@ cargo openvm verify app --proof ``` -If you omit `--app_vk` and/or `--proof`, the command will search for those files at `./openvm/app.vk` and `./openvm/app.proof` respectively. - -Once again, if you omitted `--output` and `--vk_output` in the `keygen` and `prove` commands, you can omit `--app_vk` and `--proof` in the `verify` command. +Options `--manifest-path`, `--target-dir` are also available to `verify`. If you omit `--app_vk` and/or `--proof`, the command will search for those files at `${target_dir}/openvm/app.vk` and `./app.proof` respectively. ## EVM Level diff --git a/crates/cli/Cargo.toml b/crates/cli/Cargo.toml index 9e916c9641..4f91df352e 100644 --- a/crates/cli/Cargo.toml +++ b/crates/cli/Cargo.toml @@ -37,7 +37,7 @@ hex = "0.4.3" target-lexicon = "0.12.15" tempfile = "3.10.1" toml = { workspace = true } -bitcode.workspace = true +itertools.workspace = true [features] default = ["parallel", "jemalloc", "evm-verify", "bench-metrics"] diff --git a/crates/cli/src/bin/cargo-openvm.rs b/crates/cli/src/bin/cargo-openvm.rs index 1a5e9f5c58..6e2fca5cc8 100644 --- a/crates/cli/src/bin/cargo-openvm.rs +++ b/crates/cli/src/bin/cargo-openvm.rs @@ -26,7 +26,7 @@ pub enum VmCliCommands { Prove(ProveCmd), Run(RunCmd), #[cfg(feature = "evm-verify")] - Setup(EvmProvingSetupCmd), + Setup(SetupCmd), Verify(VerifyCmd), } diff --git a/crates/cli/src/commands/build.rs b/crates/cli/src/commands/build.rs index 1d495b2bcf..496ca7e010 100644 --- a/crates/cli/src/commands/build.rs +++ b/crates/cli/src/commands/build.rs @@ -1,23 +1,21 @@ use std::{ env::var, - fs::{create_dir_all, read, write}, + fs::{copy, create_dir_all, read}, path::PathBuf, - sync::Arc, }; use clap::Parser; use eyre::Result; +use itertools::izip; use openvm_build::{ - build_generic, get_package, get_target_dir, get_workspace_packages, get_workspace_root, - GuestOptions, + build_generic, get_package, get_workspace_packages, get_workspace_root, GuestOptions, }; use openvm_circuit::arch::{InitFileGenerator, OPENVM_DEFAULT_INIT_FILE_NAME}; -use openvm_sdk::{commit::commit_app_exe, fs::write_exe_to_file, Sdk}; +use openvm_sdk::{fs::write_exe_to_file, Sdk}; use openvm_transpiler::{elf::Elf, openvm_platform::memory::MEM_SIZE}; -use crate::{ - default::{DEFAULT_APP_CONFIG_PATH, DEFAULT_APP_EXE_PATH, DEFAULT_COMMITTED_APP_EXE_PATH}, - util::{find_manifest_dir, read_config_toml_or_default}, +use crate::util::{ + get_manifest_path_and_dir, get_target_dir, get_target_output_dir, read_config_toml_or_default, }; #[derive(Parser)] @@ -41,7 +39,6 @@ impl BuildCmd { pub struct BuildArgs { #[arg( long, - default_value = "false", help = "Skips transpilation into exe when set", help_heading = "OpenVM Options" )] @@ -49,27 +46,17 @@ pub struct BuildArgs { #[arg( long, - default_value = DEFAULT_APP_CONFIG_PATH, - help = "Path to the OpenVM config .toml file that specifies the VM extensions", + help = "Path to the OpenVM config .toml file that specifies the VM extensions, by default will search for the file at ${manifest_dir}/openvm.toml", help_heading = "OpenVM Options" )] - pub config: PathBuf, + pub config: Option, #[arg( long, - default_value = DEFAULT_APP_EXE_PATH, - help = "Output path for the transpiled program", + help = "Output directory that OpenVM proving artifacts will be copied to", help_heading = "OpenVM Options" )] - pub exe_output: PathBuf, - - #[arg( - long, - default_value = DEFAULT_COMMITTED_APP_EXE_PATH, - help = "Output path for the committed program", - help_heading = "OpenVM Options" - )] - pub committed_exe_output: PathBuf, + pub output_dir: Option, #[arg( long, @@ -80,6 +67,17 @@ pub struct BuildArgs { pub init_file_name: String, } +impl Default for BuildArgs { + fn default() -> Self { + Self { + no_transpile: false, + config: None, + output_dir: None, + init_file_name: OPENVM_DEFAULT_INIT_FILE_NAME.to_string(), + } + } +} + #[derive(Clone, Parser)] pub struct BuildCargoArgs { #[arg( @@ -254,30 +252,43 @@ pub struct BuildCargoArgs { pub frozen: bool, } -// Returns the paths to the ELF file for each built executable target. -pub fn build(build_args: &BuildArgs, cargo_args: &BuildCargoArgs) -> Result> { - println!("[openvm] Building the package..."); - - // Find manifest directory using either manifest_path or find_manifest_dir - let manifest_dir = if let Some(manifest_path) = &cargo_args.manifest_path { - if !manifest_path.ends_with("Cargo.toml") { - return Err(eyre::eyre!( - "manifest_path must be a path to a Cargo.toml file" - )); +impl Default for BuildCargoArgs { + fn default() -> Self { + Self { + package: vec![], + workspace: false, + exclude: vec![], + lib: false, + bin: vec![], + bins: false, + example: vec![], + examples: false, + all_targets: false, + features: vec![], + all_features: false, + no_default_features: false, + profile: "release".to_string(), + target_dir: None, + verbose: false, + quiet: false, + color: "always".to_string(), + manifest_path: None, + ignore_rust_version: false, + locked: false, + offline: false, + frozen: false, } - manifest_path.parent().unwrap().to_path_buf() - } else { - find_manifest_dir(PathBuf::from("."))? - }; - let manifest_path = manifest_dir.join("Cargo.toml"); - println!("[openvm] Manifest directory: {}", manifest_dir.display()); + } +} - // Get target path - let target_path = if let Some(target_path) = &cargo_args.target_dir { - target_path.to_path_buf() - } else { - get_target_dir(&manifest_path) - }; +// Returns either a) the default transpilation output directory or b) the ELF output +// directory if no_transpile is set to true. +pub fn build(build_args: &BuildArgs, cargo_args: &BuildCargoArgs) -> Result { + println!("[openvm] Building the package..."); + + // Find manifest_path, manifest_dir, and target_dir + let (manifest_path, manifest_dir) = get_manifest_path_and_dir(&cargo_args.manifest_path)?; + let target_dir = get_target_dir(&cargo_args.target_dir, &manifest_path); // Set guest options using build arguments; use found manifest directory for consistency let mut guest_options = GuestOptions::default() @@ -285,7 +296,7 @@ pub fn build(build_args: &BuildArgs, cargo_args: &BuildCargoArgs) -> Result Result target_dir, + let elf_target_dir = match build_generic(&guest_options) { + Ok(raw_target_dir) => raw_target_dir, Err(None) => { return Err(eyre::eyre!("Failed to build guest")); } @@ -344,9 +355,23 @@ pub fn build(build_args: &BuildArgs, cargo_args: &BuildCargoArgs) -> Result Result = packages + let elf_targets = packages + .iter() + .flat_map(|pkg| pkg.targets.iter()) + .filter(|target| { + // We only build bin and example targets (note they are mutually exclusive + // types). If no target selection flags are set, then all bin targets are + // built by default. + if target.is_example() { + all_examples || cargo_args.example.contains(&target.name) + } else if target.is_bin() { + all_bins + || cargo_args.bin.contains(&target.name) + || (!cargo_args.examples + && !cargo_args.lib + && cargo_args.bin.is_empty() + && cargo_args.example.is_empty()) + } else { + false + } + }) + .collect::>(); + let elf_paths = elf_targets .iter() - .flat_map(|pkg| { - pkg.targets - .iter() - .filter(move |target| { - // We only build bin and example targets (note they are mutually exclusive - // types). If no target selection flags are set, then all bin targets are - // built by default. - if target.is_example() { - return all_examples || cargo_args.example.contains(&target.name); - } else if target.is_bin() { - return all_bins - || cargo_args.bin.contains(&target.name) - || (!cargo_args.examples - && !cargo_args.lib - && cargo_args.bin.is_empty() - && cargo_args.example.is_empty()); - } - false - }) - .map(|target| { - if target.is_example() { - target_dir.join("examples") - } else { - target_dir.clone() - } - .join(&target.name) - }) - .collect::>() + .map(|target| { + if target.is_example() { + elf_target_dir.join("examples") + } else { + elf_target_dir.clone() + } + .join(&target.name) }) .collect::>(); - // Transpile and commit, storing in target_dir/openvm/${profile} by default - // TODO[stephen]: actually implement target_dir change - if !build_args.no_transpile { - for elf_path in &elf_paths { - println!("[openvm] Transpiling the package..."); - let output_path = &build_args.exe_output; - let transpiler = app_config.app_vm_config.transpiler(); - - let data = read(elf_path.clone())?; - let elf = Elf::decode(&data, MEM_SIZE as u32)?; - let exe = Sdk::new().transpile(elf, transpiler)?; - let committed_exe = commit_app_exe(app_config.app_fri_params.fri_params, exe.clone()); - write_exe_to_file(exe, output_path)?; - - if let Some(parent) = build_args.committed_exe_output.parent() { - create_dir_all(parent)?; - } - let committed_exe = match Arc::try_unwrap(committed_exe) { - Ok(exe) => exe, - Err(_) => return Err(eyre::eyre!("Failed to unwrap committed_exe Arc")), - }; - write( - &build_args.committed_exe_output, - bitcode::serialize(&committed_exe)?, - )?; - - println!( - "[openvm] Successfully transpiled to {}", - output_path.display() - ); + // Transpile, storing in ${target_dir}/openvm/${profile} by default + let target_output_dir = get_target_output_dir(&target_dir, &cargo_args.profile); + + println!("[openvm] Transpiling the package..."); + for (elf_path, target) in izip!(&elf_paths, &elf_targets) { + let transpiler = app_config.app_vm_config.transpiler(); + let data = read(elf_path.clone())?; + let elf = Elf::decode(&data, MEM_SIZE as u32)?; + let exe = Sdk::new().transpile(elf, transpiler)?; + + let target_name = if target.is_example() { + &format!("examples/{}", target.name) + } else { + &target.name + }; + let file_name = format!("{}.vmexe", target_name); + let file_path = target_output_dir.join(&file_name); + + write_exe_to_file(exe, &file_path)?; + if let Some(output_dir) = &build_args.output_dir { + create_dir_all(output_dir)?; + copy(file_path, output_dir.join(file_name))?; } } - // Return elf paths of all targets for all built packages - println!("[openvm] Successfully built the packages"); - Ok(elf_paths) + let final_output_dir = if let Some(output_dir) = &build_args.output_dir { + output_dir + } else { + &target_output_dir + }; + println!( + "[openvm] Successfully transpiled to {}", + final_output_dir.display() + ); + Ok(final_output_dir.clone()) } diff --git a/crates/cli/src/commands/keygen.rs b/crates/cli/src/commands/keygen.rs index b3070a10de..a021a183e1 100644 --- a/crates/cli/src/commands/keygen.rs +++ b/crates/cli/src/commands/keygen.rs @@ -1,4 +1,7 @@ -use std::path::{Path, PathBuf}; +use std::{ + fs::{copy, create_dir_all}, + path::{Path, PathBuf}, +}; use clap::Parser; use eyre::Result; @@ -8,48 +11,99 @@ use openvm_sdk::{ }; use crate::{ - default::{DEFAULT_APP_CONFIG_PATH, DEFAULT_APP_PK_PATH, DEFAULT_APP_VK_PATH}, - util::read_config_toml_or_default, + default::{DEFAULT_APP_PK_NAME, DEFAULT_APP_VK_NAME}, + util::{ + get_app_pk_path, get_app_vk_path, get_manifest_path_and_dir, get_target_dir, + read_config_toml_or_default, + }, }; #[derive(Parser)] #[command(name = "keygen", about = "Generate an application proving key")] pub struct KeygenCmd { - #[arg(long, action, help = "Path to app config TOML file", default_value = DEFAULT_APP_CONFIG_PATH)] - config: PathBuf, + #[arg( + long, + help = "Path to the OpenVM config .toml file that specifies the VM extensions, by default will search for the file at ${manifest_dir}/openvm.toml", + help_heading = "OpenVM Options" + )] + config: Option, #[arg( long, - action, - help = "Path to output app proving key file", - default_value = DEFAULT_APP_PK_PATH + help = "Output directory that OpenVM proving artifacts will be copied to", + help_heading = "OpenVM Options" )] - output: PathBuf, + output_dir: Option, + #[command(flatten)] + cargo_args: KeygenCargoArgs, +} + +#[derive(Parser)] +pub struct KeygenCargoArgs { #[arg( long, - action, - help = "Path to output app verifying key file", - default_value = DEFAULT_APP_VK_PATH + value_name = "DIR", + help = "Directory for all Cargo-generated artifacts and intermediate files", + help_heading = "Cargo Options" )] - vk_output: PathBuf, + pub(crate) target_dir: Option, + + #[arg( + long, + value_name = "PATH", + help = "Path to the Cargo.toml file, by default searches for the file in the current or any parent directory", + help_heading = "Cargo Options" + )] + pub(crate) manifest_path: Option, } impl KeygenCmd { pub fn run(&self) -> Result<()> { - keygen(&self.config, &self.output, &self.vk_output)?; + let (manifest_path, manifest_dir) = + get_manifest_path_and_dir(&self.cargo_args.manifest_path)?; + let target_dir = get_target_dir(&self.cargo_args.target_dir, &manifest_path); + let app_pk_path = get_app_pk_path(&target_dir); + let app_vk_path = get_app_vk_path(&target_dir); + + keygen( + self.config + .to_owned() + .unwrap_or_else(|| manifest_dir.join("openvm.toml")), + &app_pk_path, + &app_vk_path, + self.output_dir.as_ref(), + )?; + println!( + "Successfully generated app pk and vk in {}", + if let Some(output_dir) = self.output_dir.as_ref() { + output_dir.display() + } else { + app_pk_path.parent().unwrap().display() + } + ); Ok(()) } } pub(crate) fn keygen( config: impl AsRef, - output: impl AsRef, - vk_output: impl AsRef, + app_pk_path: impl AsRef, + app_vk_path: impl AsRef, + output_dir: Option>, ) -> Result<()> { let app_config = read_config_toml_or_default(config)?; let app_pk = Sdk::new().app_keygen(app_config)?; - write_app_vk_to_file(app_pk.get_app_vk(), vk_output.as_ref())?; - write_app_pk_to_file(app_pk, output.as_ref())?; + let app_vk = app_pk.get_app_vk(); + write_app_vk_to_file(app_vk, &app_vk_path)?; + write_app_pk_to_file(app_pk, &app_pk_path)?; + + if let Some(output_dir) = output_dir { + let output_dir = output_dir.as_ref(); + create_dir_all(output_dir)?; + copy(&app_pk_path, output_dir.join(DEFAULT_APP_PK_NAME))?; + copy(&app_vk_path, output_dir.join(DEFAULT_APP_VK_NAME))?; + } + Ok(()) } diff --git a/crates/cli/src/commands/prove.rs b/crates/cli/src/commands/prove.rs index 9c53e111f8..cc71011021 100644 --- a/crates/cli/src/commands/prove.rs +++ b/crates/cli/src/commands/prove.rs @@ -12,14 +12,17 @@ use openvm_sdk::{ write_app_proof_to_file, }, keygen::AppProvingKey, - NonRootCommittedExe, Sdk, StdIn, + NonRootCommittedExe, Sdk, }; +use super::{RunArgs, RunCargoArgs}; #[cfg(feature = "evm-prove")] use crate::util::read_default_agg_pk; use crate::{ + commands::build, default::*, - input::{read_to_stdin, Input}, + input::read_to_stdin, + util::{get_app_pk_path, get_manifest_path_and_dir, get_single_target_name, get_target_dir}, }; #[derive(Parser)] @@ -32,47 +35,80 @@ pub struct ProveCmd { #[derive(Parser)] enum ProveSubCommand { App { - #[arg(long, action, help = "Path to app proving key", default_value = DEFAULT_APP_PK_PATH)] - app_pk: PathBuf, + #[arg( + long, + action, + default_value = DEFAULT_APP_PROOF_PATH, + help = "Path to app proof output", + help_heading = "Output" + )] + proof: PathBuf, + + #[arg( + long, + action, + help = "Path to app proving key, by default will be ${target_dir}/openvm/app.pk", + help_heading = "OpenVM Options" + )] + app_pk: Option, - #[arg(long, action, help = "Path to OpenVM executable", default_value = DEFAULT_APP_EXE_PATH)] - exe: PathBuf, - - #[arg(long, value_parser, help = "Input to OpenVM program")] - input: Option, + #[command(flatten)] + run_args: RunArgs, - #[arg(long, action, help = "Path to output proof", default_value = DEFAULT_APP_PROOF_PATH)] - output: PathBuf, + #[command(flatten)] + cargo_args: RunCargoArgs, }, Stark { - #[arg(long, action, help = "Path to app proving key", default_value = DEFAULT_APP_PK_PATH)] - app_pk: PathBuf, + #[arg( + long, + action, + default_value = DEFAULT_STARK_PROOF_PATH, + help = "Path to STARK proof output", + help_heading = "Output" + )] + proof: PathBuf, + + #[arg( + long, + action, + help = "Path to app proving key, by default will be ${target_dir}/openvm/app.pk", + help_heading = "OpenVM Options" + )] + app_pk: Option, - #[arg(long, action, help = "Path to OpenVM executable", default_value = DEFAULT_APP_EXE_PATH)] - exe: PathBuf, - - #[arg(long, value_parser, help = "Input to OpenVM program")] - input: Option, + #[command(flatten)] + run_args: RunArgs, - #[arg(long, action, help = "Path to output proof", default_value = DEFAULT_APP_PROOF_PATH)] - output: PathBuf, + #[command(flatten)] + cargo_args: RunCargoArgs, #[command(flatten)] agg_tree_config: AggregationTreeConfig, }, #[cfg(feature = "evm-prove")] Evm { - #[arg(long, action, help = "Path to app proving key", default_value = DEFAULT_APP_PK_PATH)] - app_pk: PathBuf, - - #[arg(long, action, help = "Path to OpenVM executable", default_value = DEFAULT_APP_EXE_PATH)] - exe: PathBuf, + #[arg( + long, + action, + default_value = DEFAULT_EVM_PROOF_PATH, + help = "Path to EVM proof output", + help_heading = "Output" + )] + proof: PathBuf, + + #[arg( + long, + action, + help = "Path to app proving key, by default will be ${target_dir}/openvm/app.pk", + help_heading = "OpenVM Options" + )] + app_pk: Option, - #[arg(long, value_parser, help = "Input to OpenVM program")] - input: Option, + #[command(flatten)] + run_args: RunArgs, - #[arg(long, action, help = "Path to output proof", default_value = DEFAULT_EVM_PROOF_PATH)] - output: PathBuf, + #[command(flatten)] + cargo_args: RunCargoArgs, #[command(flatten)] agg_tree_config: AggregationTreeConfig, @@ -81,30 +117,34 @@ enum ProveSubCommand { impl ProveCmd { pub fn run(&self) -> Result<()> { - let sdk = Sdk::new(); match &self.command { ProveSubCommand::App { app_pk, - exe, - input, - output, + proof, + run_args, + cargo_args, } => { - let (app_pk, committed_exe, input) = - Self::prepare_execution(&sdk, app_pk, exe, input)?; - let app_proof = sdk.generate_app_proof(app_pk, committed_exe, input)?; - write_app_proof_to_file(app_proof, output)?; + let sdk = Sdk::new(); + let app_pk = Self::load_app_pk(app_pk, cargo_args)?; + let committed_exe = + Self::load_or_build_and_commit_exe(&sdk, run_args, cargo_args, &app_pk)?; + + let app_proof = + sdk.generate_app_proof(app_pk, committed_exe, read_to_stdin(&run_args.input)?)?; + write_app_proof_to_file(app_proof, proof)?; } ProveSubCommand::Stark { app_pk, - exe, - input, - output, + proof, + run_args, + cargo_args, agg_tree_config, } => { - let mut sdk = sdk; - sdk.set_agg_tree_config(*agg_tree_config); - let (app_pk, committed_exe, input) = - Self::prepare_execution(&sdk, app_pk, exe, input)?; + let sdk = Sdk::new().with_agg_tree_config(*agg_tree_config); + let app_pk = Self::load_app_pk(app_pk, cargo_args)?; + let committed_exe = + Self::load_or_build_and_commit_exe(&sdk, run_args, cargo_args, &app_pk)?; + let commits = AppExecutionCommit::compute( &app_pk.app_vm_pk.vm_config, &committed_exe, @@ -112,28 +152,34 @@ impl ProveCmd { ); println!("exe commit: {:?}", commits.exe_commit); println!("vm commit: {:?}", commits.vm_commit); + let agg_stark_pk = read_agg_stark_pk_from_file(default_agg_stark_pk_path()).map_err(|e| { eyre::eyre!("Failed to read aggregation proving key: {}\nPlease run 'cargo openvm setup' first", e) })?; - let stark_proof = - sdk.generate_e2e_stark_proof(app_pk, committed_exe, agg_stark_pk, input)?; - encode_to_file(output, stark_proof)?; + let stark_proof = sdk.generate_e2e_stark_proof( + app_pk, + committed_exe, + agg_stark_pk, + read_to_stdin(&run_args.input)?, + )?; + + encode_to_file(proof, stark_proof)?; } #[cfg(feature = "evm-prove")] ProveSubCommand::Evm { app_pk, - exe, - input, - output, + proof, + run_args, + cargo_args, agg_tree_config, } => { use openvm_native_recursion::halo2::utils::CacheHalo2ParamsReader; - let mut sdk = sdk; - sdk.set_agg_tree_config(*agg_tree_config); - let params_reader = CacheHalo2ParamsReader::new(default_params_dir()); - let (app_pk, committed_exe, input) = - Self::prepare_execution(&sdk, app_pk, exe, input)?; + let sdk = Sdk::new().with_agg_tree_config(*agg_tree_config); + let app_pk = Self::load_app_pk(app_pk, cargo_args)?; + let committed_exe = + Self::load_or_build_and_commit_exe(&sdk, run_args, cargo_args, &app_pk)?; + let commits = AppExecutionCommit::compute( &app_pk.app_vm_pk.vm_config, &committed_exe, @@ -146,29 +192,56 @@ impl ProveCmd { let agg_pk = read_default_agg_pk().map_err(|e| { eyre::eyre!("Failed to read aggregation proving key: {}\nPlease run 'cargo openvm setup' first", e) })?; - let evm_proof = - sdk.generate_evm_proof(¶ms_reader, app_pk, committed_exe, agg_pk, input)?; - write_evm_proof_to_file(evm_proof, output)?; + let params_reader = CacheHalo2ParamsReader::new(default_params_dir()); + let evm_proof = sdk.generate_evm_proof( + ¶ms_reader, + app_pk, + committed_exe, + agg_pk, + read_to_stdin(&run_args.input)?, + )?; + + write_evm_proof_to_file(evm_proof, proof)?; } } Ok(()) } - fn prepare_execution( + fn load_app_pk( + app_pk: &Option, + cargo_args: &RunCargoArgs, + ) -> Result>> { + let (manifest_path, _) = get_manifest_path_and_dir(&cargo_args.manifest_path)?; + let target_dir = get_target_dir(&cargo_args.target_dir, &manifest_path); + + let app_pk_path = if let Some(app_pk) = app_pk { + app_pk.to_path_buf() + } else { + get_app_pk_path(&target_dir) + }; + + Ok(Arc::new(read_app_pk_from_file(app_pk_path)?)) + } + + fn load_or_build_and_commit_exe( sdk: &Sdk, - app_pk: &PathBuf, - exe: &PathBuf, - input: &Option, - ) -> Result<( - Arc>, - Arc, - StdIn, - )> { - let app_pk: Arc> = Arc::new(read_app_pk_from_file(app_pk)?); - let app_exe = read_exe_from_file(exe)?; + run_args: &RunArgs, + cargo_args: &RunCargoArgs, + app_pk: &Arc>, + ) -> Result> { + let exe_path = if let Some(exe) = &run_args.exe { + exe + } else { + // Build and get the executable name + let target_name = get_single_target_name(cargo_args)?; + let build_args = run_args.clone().into(); + let cargo_args = cargo_args.clone().into(); + let output_dir = build(&build_args, &cargo_args)?; + &output_dir.join(format!("{}.vmexe", target_name)) + }; + + let app_exe = read_exe_from_file(exe_path)?; let committed_exe = sdk.commit_app_exe(app_pk.app_fri_params(), app_exe)?; - - let input = read_to_stdin(input)?; - Ok((app_pk, committed_exe, input)) + Ok(committed_exe) } } diff --git a/crates/cli/src/commands/run.rs b/crates/cli/src/commands/run.rs index 0bdcd438de..133e7eebf5 100644 --- a/crates/cli/src/commands/run.rs +++ b/crates/cli/src/commands/run.rs @@ -2,33 +2,259 @@ use std::path::PathBuf; use clap::Parser; use eyre::Result; +use openvm_circuit::arch::OPENVM_DEFAULT_INIT_FILE_NAME; use openvm_sdk::{fs::read_exe_from_file, Sdk}; +use super::{build, BuildArgs, BuildCargoArgs}; use crate::{ - default::{DEFAULT_APP_CONFIG_PATH, DEFAULT_APP_EXE_PATH}, input::{read_to_stdin, Input}, - util::read_config_toml_or_default, + util::{get_manifest_path_and_dir, get_single_target_name, read_config_toml_or_default}, }; #[derive(Parser)] #[command(name = "run", about = "Run an OpenVM program")] pub struct RunCmd { - #[arg(long, action, help = "Path to OpenVM executable", default_value = DEFAULT_APP_EXE_PATH)] - exe: PathBuf, + #[clap(flatten)] + run_args: RunArgs, - #[arg(long, action, help = "Path to app config TOML file", default_value = DEFAULT_APP_CONFIG_PATH)] - config: PathBuf, + #[clap(flatten)] + cargo_args: RunCargoArgs, +} + +#[derive(Clone, Parser)] +pub struct RunArgs { + #[arg( + long, + action, + help = "Path to OpenVM executable, if specified build will be skipped", + help_heading = "OpenVM Options" + )] + pub exe: Option, + + #[arg( + long, + help = "Path to the OpenVM config .toml file that specifies the VM extensions, by default will search for the file at ${manifest_dir}/openvm.toml", + help_heading = "OpenVM Options" + )] + pub config: Option, + + #[arg( + long, + help = "Output directory that OpenVM proving artifacts will be copied to", + help_heading = "OpenVM Options" + )] + pub output_dir: Option, + + #[arg( + long, + value_parser, + help = "Input to OpenVM program", + help_heading = "OpenVM Options" + )] + pub input: Option, + + #[arg( + long, + default_value = OPENVM_DEFAULT_INIT_FILE_NAME, + help = "Name of the init file", + help_heading = "OpenVM Options" + )] + pub init_file_name: String, +} + +impl From for BuildArgs { + fn from(args: RunArgs) -> Self { + BuildArgs { + config: args.config, + output_dir: args.output_dir, + init_file_name: args.init_file_name, + ..Default::default() + } + } +} + +#[derive(Clone, Parser)] +pub struct RunCargoArgs { + #[arg( + long, + short = 'p', + value_name = "PACKAGES", + help = "The package to run; by default is the package in the current workspace", + help_heading = "Package Selection" + )] + pub package: Option, + + #[arg( + long, + value_name = "BIN", + help = "Run the specified binary", + help_heading = "Target Selection" + )] + pub bin: Vec, + + #[arg( + long, + value_name = "EXAMPLE", + help = "Run the specified example", + help_heading = "Target Selection" + )] + pub example: Vec, + + #[arg( + long, + short = 'F', + value_name = "FEATURES", + value_delimiter = ',', + help = "Space/comma separated list of features to activate", + help_heading = "Feature Selection" + )] + pub features: Vec, + + #[arg( + long, + help = "Activate all available features of all selected packages", + help_heading = "Feature Selection" + )] + pub all_features: bool, - #[arg(long, value_parser, help = "Input to OpenVM program")] - input: Option, + #[arg( + long, + help = "Do not activate the `default` feature of the selected packages", + help_heading = "Feature Selection" + )] + pub no_default_features: bool, + + #[arg( + long, + value_name = "NAME", + default_value = "release", + help = "Run with the given profile", + help_heading = "Compilation Options" + )] + pub profile: String, + + #[arg( + long, + value_name = "DIR", + help = "Directory for all generated artifacts and intermediate files", + help_heading = "Output Options" + )] + pub target_dir: Option, + + #[arg( + long, + short = 'v', + help = "Use verbose output", + help_heading = "Display Options" + )] + pub verbose: bool, + + #[arg( + long, + short = 'q', + help = "Do not print cargo log messages", + help_heading = "Display Options" + )] + pub quiet: bool, + + #[arg( + long, + value_name = "WHEN", + default_value = "always", + help = "Control when colored output is used", + help_heading = "Display Options" + )] + pub color: String, + + #[arg( + long, + value_name = "PATH", + help = "Path to the Cargo.toml file, by default searches for the file in the current or any parent directory", + help_heading = "Manifest Options" + )] + pub manifest_path: Option, + + #[arg( + long, + help = "Ignore rust-version specification in packages", + help_heading = "Manifest Options" + )] + pub ignore_rust_version: bool, + + #[arg( + long, + help = "Asserts same dependencies and versions are used as when the existing Cargo.lock file was originally generated", + help_heading = "Manifest Options" + )] + pub locked: bool, + + #[arg( + long, + help = "Prevents Cargo from accessing the network for any reason", + help_heading = "Manifest Options" + )] + pub offline: bool, + + #[arg( + long, + help = "Equivalent to specifying both --locked and --offline", + help_heading = "Manifest Options" + )] + pub frozen: bool, +} + +impl From for BuildCargoArgs { + fn from(args: RunCargoArgs) -> Self { + BuildCargoArgs { + package: args.package.into_iter().collect(), + bin: args.bin, + example: args.example, + features: args.features, + all_features: args.all_features, + no_default_features: args.no_default_features, + profile: args.profile, + target_dir: args.target_dir, + verbose: args.verbose, + quiet: args.quiet, + color: args.color, + manifest_path: args.manifest_path, + ignore_rust_version: args.ignore_rust_version, + locked: args.locked, + offline: args.offline, + frozen: args.frozen, + ..Default::default() + } + } } impl RunCmd { pub fn run(&self) -> Result<()> { - let exe = read_exe_from_file(&self.exe)?; - let app_config = read_config_toml_or_default(&self.config)?; + let exe_path = if let Some(exe) = &self.run_args.exe { + exe + } else { + // Build and get the executable name + let target_name = get_single_target_name(&self.cargo_args)?; + let build_args = self.run_args.clone().into(); + let cargo_args = self.cargo_args.clone().into(); + let output_dir = build(&build_args, &cargo_args)?; + &output_dir.join(format!("{}.vmexe", target_name)) + }; + + let (_, manifest_dir) = get_manifest_path_and_dir(&self.cargo_args.manifest_path)?; + let app_config = read_config_toml_or_default( + self.run_args + .config + .to_owned() + .unwrap_or_else(|| manifest_dir.join("openvm.toml")), + )?; + let exe = read_exe_from_file(exe_path)?; + let sdk = Sdk::new(); - let output = sdk.execute(exe, app_config.app_vm_config, read_to_stdin(&self.input)?)?; + let output = sdk.execute( + exe, + app_config.app_vm_config, + read_to_stdin(&self.run_args.input)?, + )?; println!("Execution output: {:?}", output); Ok(()) } diff --git a/crates/cli/src/commands/setup.rs b/crates/cli/src/commands/setup.rs index d84255c7f4..770be24d4c 100644 --- a/crates/cli/src/commands/setup.rs +++ b/crates/cli/src/commands/setup.rs @@ -28,10 +28,10 @@ use crate::{ #[derive(Parser)] #[command( - name = "evm-proving-setup", + name = "setup", about = "Set up for generating EVM proofs. ATTENTION: this requires large amounts of computation and memory. " )] -pub struct EvmProvingSetupCmd { +pub struct SetupCmd { #[arg( long, default_value = "false", @@ -46,7 +46,7 @@ pub struct EvmProvingSetupCmd { pub force_agg_keygen: bool, } -impl EvmProvingSetupCmd { +impl SetupCmd { pub async fn run(&self) -> Result<()> { let default_agg_stark_pk_path = default_agg_stark_pk_path(); let default_params_dir = default_params_dir(); diff --git a/crates/cli/src/commands/verify.rs b/crates/cli/src/commands/verify.rs index 94c7f9e355..09662fb96c 100644 --- a/crates/cli/src/commands/verify.rs +++ b/crates/cli/src/commands/verify.rs @@ -7,7 +7,11 @@ use openvm_sdk::{ Sdk, }; -use crate::default::*; +use super::KeygenCargoArgs; +use crate::{ + default::*, + util::{get_app_vk_path, get_manifest_path_and_dir, get_target_dir}, +}; #[derive(Parser)] #[command(name = "verify", about = "Verify a proof")] @@ -19,15 +23,35 @@ pub struct VerifyCmd { #[derive(Parser)] enum VerifySubCommand { App { - #[arg(long, action, help = "Path to app verifying key", default_value = DEFAULT_APP_VK_PATH)] - app_vk: PathBuf, + #[arg( + long, + action, + help = "Path to app verifying key, by default will search for it in ${target_dir}/openvm/app.vk", + help_heading = "OpenVM Options" + )] + app_vk: Option, - #[arg(long, action, help = "Path to app proof", default_value = DEFAULT_APP_PROOF_PATH)] + #[arg( + long, + action, + default_value = DEFAULT_APP_PROOF_PATH, + help = "Path to app proof", + help_heading = "OpenVM Options" + )] proof: PathBuf, + + #[command(flatten)] + cargo_args: KeygenCargoArgs, }, #[cfg(feature = "evm-verify")] Evm { - #[arg(long, action, help = "Path to EVM proof", default_value = DEFAULT_EVM_PROOF_PATH)] + #[arg( + long, + action, + default_value = DEFAULT_EVM_PROOF_PATH, + help = "Path to EVM proof", + help_heading = "OpenVM Options" + )] proof: PathBuf, }, } @@ -36,8 +60,20 @@ impl VerifyCmd { pub fn run(&self) -> Result<()> { let sdk = Sdk::new(); match &self.command { - VerifySubCommand::App { app_vk, proof } => { - let app_vk = read_app_vk_from_file(app_vk)?; + VerifySubCommand::App { + app_vk, + proof, + cargo_args, + } => { + let app_vk_path = if let Some(app_vk) = app_vk { + app_vk.to_path_buf() + } else { + let (manifest_path, _) = get_manifest_path_and_dir(&cargo_args.manifest_path)?; + let target_dir = get_target_dir(&cargo_args.target_dir, &manifest_path); + get_app_vk_path(&target_dir) + }; + + let app_vk = read_app_vk_from_file(app_vk_path)?; let app_proof = read_app_proof_from_file(proof)?; sdk.verify_app_proof(&app_vk, &app_proof)?; } @@ -47,9 +83,14 @@ impl VerifyCmd { read_evm_halo2_verifier_from_folder, read_evm_proof_from_file, }; - let evm_verifier = read_evm_halo2_verifier_from_folder(default_evm_halo2_verifier_path()).map_err(|e| { - eyre::eyre!("Failed to read EVM verifier: {}\nPlease run 'cargo openvm evm-proving-setup' first", e) - })?; + let evm_verifier = + read_evm_halo2_verifier_from_folder(default_evm_halo2_verifier_path()) + .map_err(|e| { + eyre::eyre!( + "Failed to read EVM verifier: {}\nPlease run 'cargo openvm setup' first", + e + ) + })?; let evm_proof = read_evm_proof_from_file(proof)?; sdk.verify_evm_halo2_proof(&evm_verifier, evm_proof)?; } diff --git a/crates/cli/src/default.rs b/crates/cli/src/default.rs index 641c74b8ea..269499a332 100644 --- a/crates/cli/src/default.rs +++ b/crates/cli/src/default.rs @@ -5,13 +5,12 @@ use openvm_stark_sdk::config::FriParameters; pub const DEFAULT_MANIFEST_DIR: &str = "."; -pub const DEFAULT_APP_CONFIG_PATH: &str = "./openvm.toml"; -pub const DEFAULT_APP_EXE_PATH: &str = "./openvm/app.vmexe"; -pub const DEFAULT_COMMITTED_APP_EXE_PATH: &str = "./openvm/committed_app_exe.bc"; -pub const DEFAULT_APP_PK_PATH: &str = "./openvm/app.pk"; -pub const DEFAULT_APP_VK_PATH: &str = "./openvm/app.vk"; -pub const DEFAULT_APP_PROOF_PATH: &str = "./openvm/app.proof"; -pub const DEFAULT_EVM_PROOF_PATH: &str = "./openvm/evm.proof"; +pub const DEFAULT_APP_PK_NAME: &str = "app.pk"; +pub const DEFAULT_APP_VK_NAME: &str = "app.vk"; + +pub const DEFAULT_APP_PROOF_PATH: &str = "./app.proof"; +pub const DEFAULT_STARK_PROOF_PATH: &str = "./stark.proof"; +pub const DEFAULT_EVM_PROOF_PATH: &str = "./evm.proof"; pub fn default_agg_stark_pk_path() -> String { env::var("HOME").unwrap() + "/.openvm/agg_stark.pk" diff --git a/crates/cli/src/util.rs b/crates/cli/src/util.rs index aa4eaceba4..a78467b3b4 100644 --- a/crates/cli/src/util.rs +++ b/crates/cli/src/util.rs @@ -4,6 +4,7 @@ use std::{ }; use eyre::Result; +use openvm_build::get_workspace_packages; use openvm_sdk::{ config::{AppConfig, SdkVmConfig}, fs::{read_agg_halo2_pk_from_file, read_agg_stark_pk_from_file}, @@ -11,7 +12,13 @@ use openvm_sdk::{ }; use serde::de::DeserializeOwned; -use crate::default::{default_agg_halo2_pk_path, default_agg_stark_pk_path, default_app_config}; +use crate::{ + commands::RunCargoArgs, + default::{ + default_agg_halo2_pk_path, default_agg_stark_pk_path, default_app_config, + DEFAULT_APP_PK_NAME, DEFAULT_APP_VK_NAME, + }, +}; pub(crate) fn read_to_struct_toml(path: impl AsRef) -> Result { let toml = read_to_string(path)?; @@ -50,3 +57,87 @@ pub fn find_manifest_dir(mut current_dir: PathBuf) -> Result { } Ok(current_dir) } + +pub fn get_manifest_path_and_dir(manifest_path: &Option) -> Result<(PathBuf, PathBuf)> { + let manifest_dir = if let Some(manifest_path) = &manifest_path { + if !manifest_path.ends_with("Cargo.toml") { + return Err(eyre::eyre!( + "manifest_path must be a path to a Cargo.toml file" + )); + } + manifest_path.parent().unwrap().canonicalize()? + } else { + find_manifest_dir(PathBuf::from("."))? + }; + let manifest_path = manifest_dir.join("Cargo.toml"); + Ok((manifest_path.clone(), manifest_dir)) +} + +pub fn get_target_dir(target_dir: &Option, manifest_path: &PathBuf) -> PathBuf { + target_dir + .clone() + .unwrap_or_else(|| openvm_build::get_target_dir(manifest_path)) +} + +pub fn get_target_output_dir(target_dir: &Path, profile: &str) -> PathBuf { + target_dir.join("openvm").join(profile).to_path_buf() +} + +pub fn get_app_pk_path(target_dir: &Path) -> PathBuf { + target_dir.join("openvm").join(DEFAULT_APP_PK_NAME) +} + +pub fn get_app_vk_path(target_dir: &Path) -> PathBuf { + target_dir.join("openvm").join(DEFAULT_APP_VK_NAME) +} + +// Given the arguments to a run command, this function isolates the executable to +// run. If a specific binary or example is specified it will return that, else it +// will search the workspace/package for binary targets. If there is a single +// binary that will be returned, else an error will be raised. +pub fn get_single_target_name(cargo_args: &RunCargoArgs) -> Result { + let num_targets = cargo_args.bin.len() + cargo_args.example.len(); + let single_target_name = if num_targets > 1 { + return Err(eyre::eyre!( + "`cargo openvm run` can run at most one executable, but multiple were specified" + )); + } else if num_targets == 0 { + let (_, manifest_dir) = get_manifest_path_and_dir(&cargo_args.manifest_path)?; + + let packages = get_workspace_packages(&manifest_dir) + .into_iter() + .filter(|pkg| { + if let Some(package) = &cargo_args.package { + pkg.name == *package + } else { + true + } + }) + .collect::>(); + + let binaries = packages + .iter() + .flat_map(|pkg| pkg.targets.iter()) + .filter(|t| t.is_bin()) + .collect::>(); + + if binaries.len() > 1 { + return Err(eyre::eyre!( + "Could not determine which binary to run. Use the --bin flag to specify.\n\ + Available targets: {:?}", + binaries.iter().map(|t| t.name.clone()).collect::>() + )); + } else if binaries.is_empty() { + return Err(eyre::eyre!( + "No binaries found. If you would like to run an example, use the --example flag.", + )); + } else { + binaries[0].name.clone() + } + } else if cargo_args.bin.is_empty() { + format!("examples/{}", cargo_args.example[0]) + } else { + cargo_args.bin[0].clone() + }; + Ok(single_target_name) +} diff --git a/crates/cli/tests/app_e2e.rs b/crates/cli/tests/app_e2e.rs index 60bd5b2d68..3d146c52d2 100644 --- a/crates/cli/tests/app_e2e.rs +++ b/crates/cli/tests/app_e2e.rs @@ -7,9 +7,9 @@ use tempfile::tempdir; fn test_cli_app_e2e() -> Result<()> { let temp_dir = tempdir()?; run_cmd("cargo", &["install", "--path", ".", "--force"])?; - let temp_exe = temp_dir.path().join("fibonacci.vmexe"); - let temp_pk = temp_dir.path().join("fibonacci.pk"); - let temp_vk = temp_dir.path().join("fibonacci.vk"); + let exe_path = "tests/programs/fibonacci/target/openvm/release/openvm-cli-example-test.vmexe"; + let temp_pk = temp_dir.path().join("app.pk"); + let temp_vk = temp_dir.path().join("app.vk"); let temp_proof = temp_dir.path().join("fibonacci.apppf"); run_cmd( @@ -21,8 +21,6 @@ fn test_cli_app_e2e() -> Result<()> { "tests/programs/fibonacci/Cargo.toml", "--config", "tests/programs/fibonacci/openvm.toml", - "--exe-output", - temp_exe.to_str().unwrap(), ], )?; @@ -33,10 +31,8 @@ fn test_cli_app_e2e() -> Result<()> { "keygen", "--config", "tests/programs/fibonacci/openvm.toml", - "--output", - temp_pk.to_str().unwrap(), - "--vk-output", - temp_vk.to_str().unwrap(), + "--output-dir", + temp_dir.path().to_str().unwrap(), ], )?; @@ -46,7 +42,7 @@ fn test_cli_app_e2e() -> Result<()> { "openvm", "run", "--exe", - temp_exe.to_str().unwrap(), + exe_path, "--config", "tests/programs/fibonacci/openvm.toml", ], @@ -61,8 +57,8 @@ fn test_cli_app_e2e() -> Result<()> { "--app-pk", temp_pk.to_str().unwrap(), "--exe", - temp_exe.to_str().unwrap(), - "--output", + exe_path, + "--proof", temp_proof.to_str().unwrap(), ], )?; @@ -95,10 +91,81 @@ fn test_cli_app_e2e_default_paths() -> Result<()> { "tests/programs/fibonacci/Cargo.toml", ], )?; - run_cmd("cargo", &["openvm", "keygen"])?; - run_cmd("cargo", &["openvm", "run"])?; - run_cmd("cargo", &["openvm", "prove", "app"])?; - run_cmd("cargo", &["openvm", "verify", "app"])?; + run_cmd( + "cargo", + &[ + "openvm", + "keygen", + "--manifest-path", + "tests/programs/fibonacci/Cargo.toml", + ], + )?; + run_cmd( + "cargo", + &[ + "openvm", + "run", + "--manifest-path", + "tests/programs/fibonacci/Cargo.toml", + ], + )?; + run_cmd( + "cargo", + &[ + "openvm", + "prove", + "app", + "--manifest-path", + "tests/programs/fibonacci/Cargo.toml", + ], + )?; + run_cmd( + "cargo", + &[ + "openvm", + "verify", + "app", + "--manifest-path", + "tests/programs/fibonacci/Cargo.toml", + ], + )?; + Ok(()) +} + +#[test] +fn test_cli_app_e2e_simplified() -> Result<()> { + run_cmd("cargo", &["install", "--path", ".", "--force"])?; + run_cmd( + "cargo", + &[ + "openvm", + "keygen", + "--manifest-path", + "tests/programs/multi/Cargo.toml", + ], + )?; + run_cmd( + "cargo", + &[ + "openvm", + "prove", + "app", + "--manifest-path", + "tests/programs/multi/Cargo.toml", + "--example", + "fibonacci", + ], + )?; + run_cmd( + "cargo", + &[ + "openvm", + "verify", + "app", + "--manifest-path", + "tests/programs/multi/Cargo.toml", + ], + )?; Ok(()) } diff --git a/crates/cli/tests/build.rs b/crates/cli/tests/build.rs index 42d3f09ee6..084db50367 100644 --- a/crates/cli/tests/build.rs +++ b/crates/cli/tests/build.rs @@ -1,46 +1,25 @@ use std::path::PathBuf; -use cargo_openvm::{ - commands::{build, BuildArgs, BuildCargoArgs}, - default::{DEFAULT_APP_EXE_PATH, DEFAULT_COMMITTED_APP_EXE_PATH}, -}; +use cargo_openvm::commands::{build, BuildArgs, BuildCargoArgs}; use eyre::Result; use openvm_build::RUSTC_TARGET; -use openvm_circuit::arch::OPENVM_DEFAULT_INIT_FILE_NAME; -fn default_build_args(example: &str) -> BuildArgs { +fn default_build_test_args(example: &str) -> BuildArgs { BuildArgs { no_transpile: true, - config: PathBuf::from(env!("CARGO_MANIFEST_DIR")) - .join("tests") - .join("programs") - .join(example) - .join("openvm.toml"), - exe_output: PathBuf::from(DEFAULT_APP_EXE_PATH), - committed_exe_output: PathBuf::from(DEFAULT_COMMITTED_APP_EXE_PATH), - init_file_name: OPENVM_DEFAULT_INIT_FILE_NAME.to_string(), + config: Some( + PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("tests") + .join("programs") + .join(example) + .join("openvm.toml"), + ), + ..Default::default() } } -fn default_cargo_args(example: &str) -> BuildCargoArgs { +fn default_cargo_test_args(example: &str) -> BuildCargoArgs { BuildCargoArgs { - package: vec![], - workspace: false, - exclude: vec![], - lib: false, - bin: vec![], - bins: false, - example: vec![], - examples: false, - all_targets: false, - all_features: false, - no_default_features: false, - features: vec![], - profile: "release".to_string(), - target_dir: None, - verbose: false, - quiet: false, - color: "always".to_string(), manifest_path: Some( PathBuf::from(env!("CARGO_MANIFEST_DIR")) .join("tests") @@ -48,10 +27,7 @@ fn default_cargo_args(example: &str) -> BuildCargoArgs { .join(example) .join("Cargo.toml"), ), - ignore_rust_version: false, - locked: false, - offline: false, - frozen: false, + ..Default::default() } } @@ -60,8 +36,8 @@ fn test_build_with_profile() -> Result<()> { let temp_dir = tempfile::tempdir()?; let target_dir = temp_dir.path(); - let build_args = default_build_args("fibonacci"); - let mut cargo_args = default_cargo_args("fibonacci"); + let build_args = default_build_test_args("fibonacci"); + let mut cargo_args = default_cargo_test_args("fibonacci"); cargo_args.target_dir = Some(target_dir.to_path_buf()); cargo_args.profile = "dev".to_string(); @@ -79,63 +55,136 @@ fn test_multi_target_build() -> Result<()> { let temp_dir = tempfile::tempdir()?; let target_dir = temp_dir.path(); - let build_args = default_build_args("multi"); - let mut cargo_args = default_cargo_args("multi"); + let build_args = default_build_test_args("multi"); + let mut cargo_args = default_cargo_test_args("multi"); cargo_args.target_dir = Some(target_dir.to_path_buf()); // Build lib cargo_args.lib = true; - let build_result = build(&build_args, &cargo_args)?; - assert!(build_result.is_empty()); + let elf_dir = build(&build_args, &cargo_args)?; + assert!(!elf_dir.join("calculator").exists()); + assert!(!elf_dir.join("string-utils").exists()); + assert!(!elf_dir.join("examples/fibonacci").exists()); + assert!(!elf_dir.join("examples/palindrome").exists()); + std::fs::remove_dir_all(&elf_dir)?; + std::fs::create_dir_all(&elf_dir)?; // Build bins cargo_args.lib = false; - let build_result = build(&build_args, &cargo_args)?; - let binary_names: Vec = build_result - .iter() - .map(|path| path.file_stem().unwrap().to_string_lossy().to_string()) - .collect(); - assert!(binary_names.len() == 2); - assert!(binary_names.contains(&"calculator".to_string())); - assert!(binary_names.contains(&"string-utils".to_string())); + let elf_dir = build(&build_args, &cargo_args)?; + assert!(elf_dir.join("calculator").exists()); + assert!(elf_dir.join("string-utils").exists()); + assert!(!elf_dir.join("examples/fibonacci").exists()); + assert!(!elf_dir.join("examples/palindrome").exists()); + std::fs::remove_dir_all(&elf_dir)?; + std::fs::create_dir_all(&elf_dir)?; // Build examples cargo_args.examples = true; - let build_result = build(&build_args, &cargo_args)?; - let example_names: Vec = build_result - .iter() - .map(|path| path.file_stem().unwrap().to_string_lossy().to_string()) - .collect(); - assert!(example_names.len() == 2); - assert!(example_names.contains(&"fibonacci".to_string())); - assert!(example_names.contains(&"palindrome".to_string())); + let elf_dir = build(&build_args, &cargo_args)?; + assert!(!elf_dir.join("calculator").exists()); + assert!(!elf_dir.join("string-utils").exists()); + assert!(elf_dir.join("examples/fibonacci").exists()); + assert!(elf_dir.join("examples/palindrome").exists()); + std::fs::remove_dir_all(&elf_dir)?; + std::fs::create_dir_all(&elf_dir)?; // Build examples and a single bin cargo_args.bin = vec!["calculator".to_string()]; - let build_result = build(&build_args, &cargo_args)?; - let exe_names: Vec = build_result - .iter() - .map(|path| path.file_stem().unwrap().to_string_lossy().to_string()) - .collect(); - assert!(exe_names.len() == 3); - assert!(exe_names.contains(&"calculator".to_string())); - assert!(exe_names.contains(&"fibonacci".to_string())); - assert!(exe_names.contains(&"palindrome".to_string())); + let elf_dir = build(&build_args, &cargo_args)?; + assert!(elf_dir.join("calculator").exists()); + assert!(!elf_dir.join("string-utils").exists()); + assert!(elf_dir.join("examples/fibonacci").exists()); + assert!(elf_dir.join("examples/palindrome").exists()); + std::fs::remove_dir_all(&elf_dir)?; + std::fs::create_dir_all(&elf_dir)?; // Build all targets cargo_args.bin = vec![]; cargo_args.examples = false; cargo_args.all_targets = true; - let build_result = build(&build_args, &cargo_args)?; - let all_names: Vec = build_result - .iter() - .map(|path| path.file_stem().unwrap().to_string_lossy().to_string()) - .collect(); - assert!(all_names.len() == 4); - assert!(all_names.contains(&"calculator".to_string())); - assert!(all_names.contains(&"string-utils".to_string())); - assert!(all_names.contains(&"fibonacci".to_string())); - assert!(all_names.contains(&"palindrome".to_string())); + let elf_dir = build(&build_args, &cargo_args)?; + assert!(elf_dir.join("calculator").exists()); + assert!(elf_dir.join("string-utils").exists()); + assert!(elf_dir.join("examples/fibonacci").exists()); + assert!(elf_dir.join("examples/palindrome").exists()); + + temp_dir.close()?; + Ok(()) +} + +#[test] +fn test_multi_target_transpile_default() -> Result<()> { + let temp_dir = tempfile::tempdir()?; + let target_dir = temp_dir.path(); + + let mut build_args = default_build_test_args("multi"); + let mut cargo_args = default_cargo_test_args("multi"); + build_args.no_transpile = false; + cargo_args.target_dir = Some(target_dir.to_path_buf()); + cargo_args.all_targets = true; + + build(&build_args, &cargo_args)?; + + // Check for openvm directory + let openvm_dir = target_dir.join("openvm"); + assert!(openvm_dir.exists(),); + + // Check for release directory + let release_dir = openvm_dir.join("release"); + assert!(release_dir.exists()); + + // Check for expected vmexe files + let calculator_exe = release_dir.join("calculator.vmexe"); + let string_utils_exe = release_dir.join("string-utils.vmexe"); + assert!(calculator_exe.exists()); + assert!(string_utils_exe.exists()); + + // Check for example directory + let examples_dir = release_dir.join("examples"); + assert!(examples_dir.exists()); + + // Check for expected example files + let fibonacci_exe = examples_dir.join("fibonacci.vmexe"); + let palindrome_exe = examples_dir.join("palindrome.vmexe"); + assert!(fibonacci_exe.exists()); + assert!(palindrome_exe.exists()); + + Ok(()) +} + +#[test] +fn test_output_dir_copy() -> Result<()> { + let temp_dir = tempfile::tempdir()?; + let target_dir = temp_dir.path(); + + let temp_dir_2 = tempfile::tempdir()?; + let output_dir = temp_dir_2.path(); + + let mut build_args = default_build_test_args("fibonacci"); + let mut cargo_args = default_cargo_test_args("fibonacci"); + build_args.output_dir = Some(output_dir.to_path_buf()); + build_args.no_transpile = false; + cargo_args.target_dir = Some(target_dir.to_path_buf()); + + build(&build_args, &cargo_args)?; + + // Check for executable in target_dir + let default_target = target_dir + .join("openvm") + .join("release") + .join("openvm-cli-example-test.vmexe"); + assert!(default_target.exists()); + + // Check for executable in output_dir + let copied_target = output_dir.join("openvm-cli-example-test.vmexe"); + assert!(copied_target.exists()); + + // Check that the executable is the same + assert_eq!( + std::fs::read(default_target)?, + std::fs::read(copied_target)? + ); Ok(()) } diff --git a/crates/cli/tests/programs/multi/calculator/Cargo.toml b/crates/cli/tests/programs/multi/calculator/Cargo.toml index 1f8543e2ab..9362608c61 100644 --- a/crates/cli/tests/programs/multi/calculator/Cargo.toml +++ b/crates/cli/tests/programs/multi/calculator/Cargo.toml @@ -1,5 +1,5 @@ [package] -name = "openvm-calculator" +name = "openvm-cli-example-calculator" version = "0.0.0" edition = "2021" diff --git a/crates/cli/tests/programs/multi/string-utils/Cargo.toml b/crates/cli/tests/programs/multi/string-utils/Cargo.toml index 245b777912..f2f0561d70 100644 --- a/crates/cli/tests/programs/multi/string-utils/Cargo.toml +++ b/crates/cli/tests/programs/multi/string-utils/Cargo.toml @@ -1,5 +1,5 @@ [package] -name = "openvm-string-utils" +name = "openvm-cli-example-string-utils" version = "0.0.0" edition = "2021" diff --git a/crates/sdk/src/config/mod.rs b/crates/sdk/src/config/mod.rs index 95581c6084..3a231f180d 100644 --- a/crates/sdk/src/config/mod.rs +++ b/crates/sdk/src/config/mod.rs @@ -68,16 +68,31 @@ pub struct Halo2Config { #[derive(Clone, Copy, Debug, Serialize, Deserialize, Args)] pub struct AggregationTreeConfig { /// Each leaf verifier circuit will aggregate this many App VM proofs. - #[arg(long, default_value_t = DEFAULT_NUM_CHILDREN_LEAF)] + #[arg( + long, + default_value_t = DEFAULT_NUM_CHILDREN_LEAF, + help = "Number of children per leaf verifier circuit", + help_heading = "Aggregation Tree Options" + )] pub num_children_leaf: usize, /// Each internal verifier circuit will aggregate this many proofs, /// where each proof may be of either leaf or internal verifier (self) circuit. - #[arg(long, default_value_t = DEFAULT_NUM_CHILDREN_INTERNAL)] + #[arg( + long, + default_value_t = DEFAULT_NUM_CHILDREN_INTERNAL, + help = "Number of children per internal verifier circuit", + help_heading = "Aggregation Tree Options" + )] pub num_children_internal: usize, /// Safety threshold: how many times to do 1-to-1 aggregation of the "last" internal /// verifier proof before it is small enough for the root verifier circuit. /// Note: almost always no wrapping is needed. - #[arg(long, default_value_t = DEFAULT_MAX_INTERNAL_WRAPPER_LAYERS)] + #[arg( + long, + default_value_t = DEFAULT_MAX_INTERNAL_WRAPPER_LAYERS, + help = "Maximum number of internal wrapper layers", + help_heading = "Aggregation Tree Options" + )] pub max_internal_wrapper_layers: usize, // root currently always has 1 child for now } diff --git a/crates/sdk/src/fs.rs b/crates/sdk/src/fs.rs index f6a6d46edf..e849f1b49c 100644 --- a/crates/sdk/src/fs.rs +++ b/crates/sdk/src/fs.rs @@ -95,6 +95,9 @@ pub fn read_evm_proof_from_file>(path: P) -> Result { } pub fn write_evm_proof_to_file>(proof: EvmProof, path: P) -> Result<()> { + if let Some(parent) = path.as_ref().parent() { + create_dir_all(parent)?; + } serde_json::to_writer(File::create(path)?, &proof)?; Ok(()) } diff --git a/crates/sdk/src/lib.rs b/crates/sdk/src/lib.rs index 91b2db0cc0..7148c8e44e 100644 --- a/crates/sdk/src/lib.rs +++ b/crates/sdk/src/lib.rs @@ -119,12 +119,13 @@ impl> GenericSdk { Self::default() } - pub fn agg_tree_config(&self) -> &AggregationTreeConfig { - &self.agg_tree_config + pub fn with_agg_tree_config(mut self, agg_tree_config: AggregationTreeConfig) -> Self { + self.agg_tree_config = agg_tree_config; + self } - pub fn set_agg_tree_config(&mut self, agg_tree_config: AggregationTreeConfig) { - self.agg_tree_config = agg_tree_config; + pub fn agg_tree_config(&self) -> &AggregationTreeConfig { + &self.agg_tree_config } pub fn build>(