fix: 1GiB sectors porep & post key (#791)

Author: th7nder

Diff for: .gitattributes

@@ -1 +1,3 @@
 Cargo.lock -diff
+*.vk binary
+*.vk.scale binary

Diff for: .gitignore

@@ -31,13 +31,13 @@ coverage/
 
 # PoRep Params
 *.porep.params
-*.porep.vk
-*.porep.vk.scale
+/*.porep.vk
+/*.porep.vk.scale
 
 # PoSt Params
 *.post.params
-*.post.vk
-*.post.vk.scale
+/*.post.vk
+/*.post.vk.scale
 
 # File logs from the SP
 logs/

Diff for: examples/1GiB.porep.vk

@@ -40,7 +40,7 @@ workspace = true
 
 [features]
 cuda = ["filecoin-proofs?/cuda", "filecoin-proofs?/multicore-sdr", "std"]
-default = ["opencl"]
+default = ["opencl", "substrate"]
 opencl = ["filecoin-proofs?/multicore-sdr", "filecoin-proofs?/opencl", "std"]
 std = [
   "codec?/std",

Diff for: examples/1GiB.porep.vk.scale

@@ -50,6 +50,15 @@ const G1AFFINE_UNCOMPRESSED_BYTES: usize = 96;
 /// The number of bytes when serialising a `G1Affine` by using `G1Affine::to_uncompressed()`.
 const G2AFFINE_UNCOMPRESSED_BYTES: usize = 192;
 
+/// Number of G1Affine constants in a [`VerifyingKey`].
+const G1AFFINE_PARAMS: usize = 3;
+/// Number of G2Affine constants in a [`VerifyingKey`].
+const G2AFFINE_PARAMS: usize = 3;
+/// Maximum number of ic params in a [`VerifyingKey`] in a secure PoRep of Sector Size of 1GiB.
+const MAX_PRODUCTION_POREP_VK_IC_LEN: usize = 328;
+/// Maximum number of ic params in a [`VerifyingKey`] in a secure PoSt of Sector Size of 1GiB.
+const MAX_PRODUCTION_POST_VK_IC_LEN: usize = 25840;
+
 /// This constant specifies the minimum number of bytes of a serialised `VerifyingKey`.
 ///
 /// It gets calculated by the defined number of serialised bytes of `G1Affine` and `G2Affine` in
@@ -58,7 +67,7 @@ const G2AFFINE_UNCOMPRESSED_BYTES: usize = 192;
 /// 3 x `G2Affine`. One serialised `u32` variable will be added.
 /// That computes to: 3 x 48 + 3 * 96 + 4 = 436.
 pub const VERIFYINGKEY_MIN_BYTES: usize =
-    3 * G1AFFINE_COMPRESSED_BYTES + 3 * G2AFFINE_COMPRESSED_BYTES + 4;
+    G1AFFINE_PARAMS * G1AFFINE_COMPRESSED_BYTES + G2AFFINE_PARAMS * G2AFFINE_COMPRESSED_BYTES + 4;
 
 /// This constant specifies the minimum number of bytes of a serialised `VerifyingKey` of usual
 /// public implementations. Usual public implementations means similar implementations in crates
@@ -69,15 +78,21 @@ pub const VERIFYINGKEY_MIN_BYTES: usize =
 /// serialised `G2Affine` are 192 bytes. In `VerifyingKey` we have for sure 3 x `G1Affine` and
 /// 3 x `G2Affine`. One serialised `u32` variable will be added.
 /// That computes to: 3 x 96 + 3 * 192 + 4 = 868.
-pub const VERIFYINGKEY_MIN_BYTES_STD: usize =
-    3 * G1AFFINE_UNCOMPRESSED_BYTES + 3 * G2AFFINE_UNCOMPRESSED_BYTES + 4;
-
-/// This constant specifies the maximum number of bytes of a serialised `VerifyingKey`.
-///
-/// The maximum number of parameters in field `ic` is 40 because its depedency can be resolved to
-/// possible sector sizes. This computes to: 3 * 96 + 3 * 192 + 4 + 40 * 96 = 4704.
-pub const VERIFYINGKEY_MAX_BYTES: usize =
-    43 * G1AFFINE_UNCOMPRESSED_BYTES + 3 * G2AFFINE_UNCOMPRESSED_BYTES + 4;
+pub const VERIFYINGKEY_MIN_BYTES_STD: usize = G1AFFINE_PARAMS * G1AFFINE_UNCOMPRESSED_BYTES
+    + G2AFFINE_PARAMS * G2AFFINE_UNCOMPRESSED_BYTES
+    + 4;
+
+/// This constant specifies the maximum number of bytes of a serialised PoRep 1GiB `VerifyingKey`.
+pub const POREP_VERIFYINGKEY_MAX_BYTES: usize = (G1AFFINE_PARAMS + MAX_PRODUCTION_POREP_VK_IC_LEN)
+    * G1AFFINE_COMPRESSED_BYTES
+    + G2AFFINE_PARAMS * G2AFFINE_COMPRESSED_BYTES
+    + 4;
+
+/// This constant specifies the maximum number of bytes of a serialised PoSt 1GiB `VerifyingKey`.
+pub const POST_VERIFYINGKEY_MAX_BYTES: usize = (G1AFFINE_PARAMS + MAX_PRODUCTION_POST_VK_IC_LEN)
+    * G1AFFINE_COMPRESSED_BYTES
+    + G2AFFINE_PARAMS * G2AFFINE_COMPRESSED_BYTES
+    + 4;
 
 /// This constant specifies the number of bytes of a serialised `Proof`.
 ///

Diff for: examples/1GiB.post.vk

@@ -190,4 +190,36 @@ mod tests {
         // Compare initial struct with this one.
         assert_eq!(bp_proof, bp_proof_result);
     }
+
+    #[test]
+    fn decodes_production_1gib_porep_verifying_key() {
+        let vk_bytes = include_bytes!("../../../../examples/1GiB.porep.vk");
+        // decode expects &mut mutability
+        let vk_bytes = vk_bytes.to_vec();
+        let key = bp_g16::VerifyingKey::<blstrs::Bls12>::read(&mut vk_bytes.as_slice());
+
+        assert!(key.is_ok(), "failed to parse 1GiB PoRep verifying key");
+        let key = key.unwrap();
+        let substrate_key = VerifyingKey::<Bls12>::try_from(key);
+        assert!(
+            substrate_key.is_ok(),
+            "failed to convert 1GiB PoRep bellperson key to substrate key"
+        );
+    }
+
+    #[test]
+    fn decodes_production_1gib_post_verifying_key() {
+        let vk_bytes = include_bytes!("../../../../examples/1GiB.post.vk");
+        // decode expects &mut mutability
+        let vk_bytes = vk_bytes.to_vec();
+        let key = bp_g16::VerifyingKey::<blstrs::Bls12>::read(&mut vk_bytes.as_slice());
+
+        assert!(key.is_ok(), "failed to parse 1GiB PoSt verifying key");
+        let key = key.unwrap();
+        let substrate_key = VerifyingKey::<Bls12>::try_from(key);
+        assert!(
+            substrate_key.is_ok(),
+            "failed to convert 1GiB PoSt bellperson key to substrate key"
+        );
+    }
 }

Diff for: examples/1GiB.post.vk.scale

@@ -25,10 +25,17 @@ where
     E: Engine<G1Affine = G1Affine, G2Affine = G2Affine>,
 {
     fn decode<I: ::codec::Input>(input: &mut I) -> Result<Self, ::codec::Error> {
-        let mut buffer = [0u8; VERIFYINGKEY_MAX_BYTES];
+        // We can't allocate 1.4MiB required for PoSt on stack.
+        let mut buffer = alloc::vec::Vec::with_capacity(POST_VERIFYINGKEY_MAX_BYTES);
         let Some(n_bytes) = input.remaining_len()? else {
             return Err(::codec::Error::from("unable to get remaining_len"));
         };
+        if n_bytes > POST_VERIFYINGKEY_MAX_BYTES {
+            return Err(::codec::Error::from(
+                "provided verifying key is too big for the current limit of bytes",
+            ));
+        }
+        buffer.resize(n_bytes, 0);
         input.read(&mut buffer[..n_bytes])?;
         VerifyingKey::<E>::from_bytes(&buffer[..n_bytes])
             .map_err(|e| codec::Error::from(e.as_static_str()))
@@ -73,7 +80,7 @@ where
     E: Engine<G1Affine = G1Affine, G2Affine = G2Affine>,
 {
     fn max_encoded_len() -> usize {
-        VERIFYINGKEY_MAX_BYTES
+        MAX_PRODUCTION_POST_VK_IC_LEN
     }
 }
 
@@ -240,4 +247,24 @@ mod tests {
         proof.into_bytes(&mut bytes_regular.as_mut_slice()).unwrap();
         assert_eq!(bytes_regular.as_slice(), bytes_scale.as_slice());
     }
+
+    #[test]
+    fn decodes_production_1gib_porep_verifying_key() {
+        let vk_bytes = include_bytes!("../../../../examples/1GiB.porep.vk.scale");
+        // decode expects &mut mutability
+        let vk_bytes = vk_bytes.to_vec();
+        let key = VerifyingKey::<Bls12>::decode(&mut vk_bytes.as_slice());
+
+        assert!(key.is_ok(), "failed to parse 1GiB PoRep verifying key");
+    }
+
+    #[test]
+    fn decodes_production_1gib_post_verifying_key() {
+        let vk_bytes = include_bytes!("../../../../examples/1GiB.post.vk.scale");
+        // decode expects &mut mutability
+        let vk_bytes = vk_bytes.to_vec();
+        let key = VerifyingKey::<Bls12>::decode(&mut vk_bytes.as_slice());
+
+        assert!(key.is_ok(), "failed to parse 1GiB PoSt verifying key");
+    }
 }

Diff for: lib/polka-storage-proofs/Cargo.toml

@@ -24,12 +24,13 @@ mod tests;
 #[cfg(feature = "runtime-benchmarks")]
 mod benchmarking;
 
-#[frame_support::pallet]
+#[frame_support::pallet(dev_mode)]
 pub mod pallet {
     pub const LOG_TARGET: &'static str = "runtime::proofs";
 
     use frame_support::{pallet_prelude::*, sp_runtime::BoundedBTreeMap};
     use frame_system::pallet_prelude::*;
+    use polka_storage_proofs::POREP_VERIFYINGKEY_MAX_BYTES;
     use primitives::{
         commitment::RawCommitment,
         pallets::ProofVerification,
@@ -89,14 +90,19 @@ pub mod pallet {
 
     #[pallet::call]
     impl<T: Config> Pallet<T> {
-        #[pallet::call_index(0)]
-        #[pallet::weight((T::WeightInfo::set_porep_verifying_key(), DispatchClass::Operational))]
+        // TODO(@th7nder,#790,03/03/2025): benchmarking was previously done for small params, not for production 1GiB sectors
+        // #[pallet::call_index(0)]
+        // #[pallet::weight((T::WeightInfo::set_porep_verifying_key(), DispatchClass::Operational))]
         pub fn set_porep_verifying_key(
             origin: OriginFor<T>,
             registered_seal_proof: RegisteredSealProof,
             verifying_key: crate::Vec<u8>,
         ) -> DispatchResult {
             let caller = ensure_signed(origin)?;
+            if verifying_key.len() > POREP_VERIFYINGKEY_MAX_BYTES {
+                log::error!(target: LOG_TARGET, "verifying key is longer ({}) than the maximum expected ({})", verifying_key.len(), POREP_VERIFYINGKEY_MAX_BYTES);
+                return Err(Error::<T>::InvalidVerifyingKey.into());
+            }
             let vkey =
                 VerifyingKey::<Bls12>::decode(&mut verifying_key.as_slice()).map_err(|e| {
                     log::error!(target: LOG_TARGET, "failed to parse PoRep verifying key {:?}", e);
@@ -111,8 +117,9 @@ pub mod pallet {
             Ok(())
         }
 
-        #[pallet::call_index(1)]
-        #[pallet::weight((T::WeightInfo::set_post_verifying_key(), DispatchClass::Operational))]
+        // TODO(@th7nder,#790,03/03/2025): benchmarking was previously done for small params, not for production 1GiB sectors
+        // #[pallet::call_index(1)]
+        // #[pallet::weight((T::WeightInfo::set_post_verifying_key(), DispatchClass::Operational))]
         pub fn set_post_verifying_key(
             origin: OriginFor<T>,
             registered_post_proof: RegisteredPoStProof,