Merge rust-bitcoin/rust-bitcoin#1088: Add BIP152 (Compact Blocks) structures

ee29910 BIP152: Test net msg ser/der and diff encoding (0xb10c)
cd1aaaf BIP152: Add Compact Block unit test based on Elements (Steven Roose)
d4d92a8 BIP152: Add Compact Blocks network messages (Steven Roose)
f2fcdc8 BIP152: Add basic Compact Block structures (Steven Roose)
a9a39c4 blockdata: Derive PartialOrd, Ord and Hash for BlockHeader (Steven Roose)

Pull request description:

  > Adds the basic structures for BIP152 and a method to create a compact block from a full block.

  This is a rebase of #249 by stevenroose (see rust-bitcoin/rust-bitcoin#249 (comment)) with a milestone for 29.0. I've added deserialization and serialization tests for the network messages and fixed an off-by-one bug in the deserialization of the differentially encoded varints in the `getblocktxn` message (see rust-bitcoin/rust-bitcoin#249 (comment)).

  Closes #249.

ACKs for top commit:
  tcharding:
    ACK ee29910
  apoelstra:
    ACK ee29910

Tree-SHA512: 462a91576281f5a2ffdc2610769ea93970b60dac75a150c827966c48daec7cf93f526f9f202e7ba1dbb1410b49148579880260a3c3df298b98330c0d891a4cca

Author: apoelstra

src/blockdata/block.rs

@@ -37,7 +37,7 @@ use crate::internal_macros::impl_consensus_encoding;
 /// ### Bitcoin Core References
 ///
 /// * [CBlockHeader definition](https://github.com/bitcoin/bitcoin/blob/345457b542b6a980ccfbc868af0970a6f91d1b82/src/primitives/block.h#L20)
-#[derive(Copy, PartialEq, Eq, Clone, Debug)]
+#[derive(Copy, PartialEq, Eq, Clone, Debug, PartialOrd, Ord, Hash)]
 #[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
 #[cfg_attr(feature = "serde", serde(crate = "actual_serde"))]
 pub struct BlockHeader {

src/consensus/encode.rs

@@ -27,6 +27,7 @@ use crate::io::{self, Cursor, Read};
 
 use crate::util::endian;
 use crate::util::psbt;
+use crate::util::bip152::{ShortId, PrefilledTransaction};
 use crate::util::taproot::TapLeafHash;
 use crate::hashes::hex::ToHex;
 
@@ -558,6 +559,7 @@ macro_rules! impl_array {
 
 impl_array!(2);
 impl_array!(4);
+impl_array!(6);
 impl_array!(8);
 impl_array!(10);
 impl_array!(12);
@@ -629,6 +631,9 @@ impl_vec!(TxIn);
 impl_vec!(Vec<u8>);
 impl_vec!(u64);
 impl_vec!(TapLeafHash);
+impl_vec!(VarInt);
+impl_vec!(ShortId);
+impl_vec!(PrefilledTransaction);
 
 #[cfg(feature = "std")] impl_vec!(Inventory);
 #[cfg(feature = "std")] impl_vec!((u32, Address));

src/network/message.rs

@@ -19,6 +19,7 @@ use crate::network::address::{Address, AddrV2Message};
 use crate::network::{message_network, message_bloom};
 use crate::network::message_blockdata;
 use crate::network::message_filter;
+use crate::network::message_compact_blocks;
 use crate::consensus::encode::{CheckedData, Decodable, Encodable, VarInt};
 use crate::consensus::{encode, serialize};
 use crate::util::merkleblock::MerkleBlock;
@@ -181,6 +182,14 @@ pub enum NetworkMessage {
     GetCFCheckpt(message_filter::GetCFCheckpt),
     /// BIP157 cfcheckpt
     CFCheckpt(message_filter::CFCheckpt),
+    /// BIP152 sendcmpct
+    SendCmpct(message_compact_blocks::SendCmpct),
+    /// BIP152 cmpctblock
+    CmpctBlock(message_compact_blocks::CmpctBlock),
+    /// BIP152 getblocktxn
+    GetBlockTxn(message_compact_blocks::GetBlockTxn),
+    /// BIP152 blocktxn
+    BlockTxn(message_compact_blocks::BlockTxn),
     /// `alert`
     Alert(Vec<u8>),
     /// `reject`
@@ -237,6 +246,10 @@ impl NetworkMessage {
             NetworkMessage::CFHeaders(_) => "cfheaders",
             NetworkMessage::GetCFCheckpt(_) => "getcfcheckpt",
             NetworkMessage::CFCheckpt(_) => "cfcheckpt",
+            NetworkMessage::SendCmpct(_) => "sendcmpct",
+            NetworkMessage::CmpctBlock(_) => "cmpctblock",
+            NetworkMessage::GetBlockTxn(_) => "getblocktxn",
+            NetworkMessage::BlockTxn(_) => "blocktxn",
             NetworkMessage::Alert(_)    => "alert",
             NetworkMessage::Reject(_)    => "reject",
             NetworkMessage::FeeFilter(_) => "feefilter",
@@ -314,6 +327,10 @@ impl Encodable for RawNetworkMessage {
             NetworkMessage::CFHeaders(ref dat) => serialize(dat),
             NetworkMessage::GetCFCheckpt(ref dat) => serialize(dat),
             NetworkMessage::CFCheckpt(ref dat) => serialize(dat),
+            NetworkMessage::SendCmpct(ref dat) => serialize(dat),
+            NetworkMessage::CmpctBlock(ref dat) => serialize(dat),
+            NetworkMessage::GetBlockTxn(ref dat) => serialize(dat),
+            NetworkMessage::BlockTxn(ref dat) => serialize(dat),
             NetworkMessage::Alert(ref dat)    => serialize(dat),
             NetworkMessage::Reject(ref dat) => serialize(dat),
             NetworkMessage::FeeFilter(ref data) => serialize(data),
@@ -394,6 +411,10 @@ impl Decodable for RawNetworkMessage {
             "reject" => NetworkMessage::Reject(Decodable::consensus_decode_from_finite_reader(&mut mem_d)?),
             "alert"   => NetworkMessage::Alert(Decodable::consensus_decode_from_finite_reader(&mut mem_d)?),
             "feefilter" => NetworkMessage::FeeFilter(Decodable::consensus_decode_from_finite_reader(&mut mem_d)?),
+            "sendcmpct" => NetworkMessage::SendCmpct(Decodable::consensus_decode_from_finite_reader(&mut mem_d)?),
+            "cmpctblock" => NetworkMessage::CmpctBlock(Decodable::consensus_decode_from_finite_reader(&mut mem_d)?),
+            "getblocktxn" => NetworkMessage::GetBlockTxn(Decodable::consensus_decode_from_finite_reader(&mut mem_d)?),
+            "blocktxn" => NetworkMessage::BlockTxn(Decodable::consensus_decode_from_finite_reader(&mut mem_d)?),
             "wtxidrelay" => NetworkMessage::WtxidRelay,
             "addrv2" => NetworkMessage::AddrV2(Decodable::consensus_decode_from_finite_reader(&mut mem_d)?),
             "sendaddrv2" => NetworkMessage::SendAddrV2,
@@ -432,6 +453,8 @@ mod test {
     use crate::blockdata::script::Script;
     use crate::network::message_bloom::{FilterAdd, FilterLoad, BloomFlags};
     use crate::MerkleBlock;
+    use crate::network::message_compact_blocks::{GetBlockTxn, SendCmpct};
+    use crate::util::bip152::BlockTransactionsRequest;
 
     fn hash(slice: [u8;32]) -> Hash {
         Hash::from_slice(&slice).unwrap()
@@ -446,6 +469,9 @@ mod test {
         let header: BlockHeader = deserialize(&Vec::from_hex("010000004ddccd549d28f385ab457e98d1b11ce80bfea2c5ab93015ade4973e400000000bf4473e53794beae34e64fccc471dace6ae544180816f89591894e0f417a914cd74d6e49ffff001d323b3a7b").unwrap()).unwrap();
         let script: Script = deserialize(&Vec::from_hex("1976a91431a420903c05a0a7de2de40c9f02ebedbacdc17288ac").unwrap()).unwrap();
         let merkle_block: MerkleBlock = deserialize(&Vec::from_hex("0100000079cda856b143d9db2c1caff01d1aecc8630d30625d10e8b4b8b0000000000000b50cc069d6a3e33e3ff84a5c41d9d3febe7c770fdcc96b2c3ff60abe184f196367291b4d4c86041b8fa45d630100000001b50cc069d6a3e33e3ff84a5c41d9d3febe7c770fdcc96b2c3ff60abe184f19630101").unwrap()).unwrap();
+        let cmptblock = deserialize(&Vec::from_hex("00000030d923ad36ff2d955abab07f8a0a6e813bc6e066b973e780c5e36674cad5d1cd1f6e265f2a17a0d35cbe701fe9d06e2c6324cfe135f6233e8b767bfa3fb4479b71115dc562ffff7f2006000000000000000000000000010002000000010000000000000000000000000000000000000000000000000000000000000000ffffffff0302ee00ffffffff0100f9029500000000015100000000").unwrap()).unwrap();
+        let blocktxn = deserialize(&Vec::from_hex("2e93c0cff39ff605020072d96bc3a8d20b8447e294d08092351c8583e08d9b5a01020000000001010000000000000000000000000000000000000000000000000000000000000000ffffffff0402dc0000ffffffff0200f90295000000001976a9142b4569203694fc997e13f2c0a1383b9e16c77a0d88ac0000000000000000266a24aa21a9ede2f61c3f71d1defd3fa999dfa36953755c690689799962b48bebd836974e8cf90120000000000000000000000000000000000000000000000000000000000000000000000000").unwrap()).unwrap();
+
 
         let msgs = vec![
             NetworkMessage::Version(version_msg),
@@ -481,6 +507,10 @@ mod test {
             NetworkMessage::WtxidRelay,
             NetworkMessage::AddrV2(vec![AddrV2Message{ addr: AddrV2::Ipv4(Ipv4Addr::new(127, 0, 0, 1)), port: 0, services: ServiceFlags::NONE, time: 0 }]),
             NetworkMessage::SendAddrV2,
+            NetworkMessage::CmpctBlock(cmptblock),
+            NetworkMessage::GetBlockTxn(GetBlockTxn { txs_request: BlockTransactionsRequest { block_hash: hash([11u8; 32]).into(), indexes: vec![0, 1, 2, 3, 10, 3002] } }),
+            NetworkMessage::BlockTxn(blocktxn),
+            NetworkMessage::SendCmpct(SendCmpct{send_compact: true, version: 8333}),
         ];
 
         for msg in msgs {

src/network/message_blockdata.rs

@@ -27,6 +27,8 @@ pub enum Inventory {
     Transaction(Txid),
     /// Block
     Block(BlockHash),
+    /// Compact Block
+    CompactBlock(BlockHash),
     /// Witness Transaction by Wtxid
     WTx(Wtxid),
     /// Witness Transaction
@@ -54,6 +56,7 @@ impl Encodable for Inventory {
             Inventory::Error => encode_inv!(0, sha256d::Hash::all_zeros()),
             Inventory::Transaction(ref t) => encode_inv!(1, t),
             Inventory::Block(ref b) => encode_inv!(2, b),
+            Inventory::CompactBlock(ref b) => encode_inv!(4, b),
             Inventory::WTx(w) => encode_inv!(5, w),
             Inventory::WitnessTransaction(ref t) => encode_inv!(0x40000001, t),
             Inventory::WitnessBlock(ref b) => encode_inv!(0x40000002, b),
@@ -70,6 +73,7 @@ impl Decodable for Inventory {
             0 => Inventory::Error,
             1 => Inventory::Transaction(Decodable::consensus_decode(r)?),
             2 => Inventory::Block(Decodable::consensus_decode(r)?),
+            4 => Inventory::CompactBlock(Decodable::consensus_decode(r)?),
             5 => Inventory::WTx(Decodable::consensus_decode(r)?),
             0x40000001 => Inventory::WitnessTransaction(Decodable::consensus_decode(r)?),
             0x40000002 => Inventory::WitnessBlock(Decodable::consensus_decode(r)?),

src/network/message_compact_blocks.rs

@@ -0,0 +1,45 @@
+//!
+//! BIP152  Compact Blocks network messages
+//!
+
+use crate::internal_macros::impl_consensus_encoding;
+use crate::util::bip152;
+
+/// sendcmpct message
+#[derive(PartialEq, Eq, Clone, Debug, Copy, PartialOrd, Ord, Hash)]
+pub struct SendCmpct {
+    /// Request to be send compact blocks.
+    pub send_compact: bool,
+    /// Compact Blocks protocol version number.
+    pub version: u64,
+}
+impl_consensus_encoding!(SendCmpct, send_compact, version);
+
+/// cmpctblock message
+///
+/// Note that the rules for validation before relaying compact blocks is
+/// different from headers and regular block messages. Thus, you shouldn't use
+/// compact blocks when relying on an upstream full node to have validated data
+/// being forwarded to you.
+#[derive(PartialEq, Eq, Clone, Debug, PartialOrd, Ord, Hash)]
+pub struct CmpctBlock {
+    /// The Compact Block.
+    pub compact_block: bip152::HeaderAndShortIds,
+}
+impl_consensus_encoding!(CmpctBlock, compact_block);
+
+/// getblocktxn message
+#[derive(PartialEq, Eq, Clone, Debug, PartialOrd, Ord, Hash)]
+pub struct GetBlockTxn {
+    /// The block transactions request.
+    pub txs_request: bip152::BlockTransactionsRequest,
+}
+impl_consensus_encoding!(GetBlockTxn, txs_request);
+
+/// blocktxn message
+#[derive(PartialEq, Eq, Clone, Debug, PartialOrd, Ord, Hash)]
+pub struct BlockTxn {
+    /// The requested block transactions.
+    pub transactions: bip152::BlockTransactions,
+}
+impl_consensus_encoding!(BlockTxn, transactions);

src/network/mod.rs

@@ -26,6 +26,9 @@ pub mod message_blockdata;
 pub mod message_bloom;
 #[cfg(feature = "std")]
 #[cfg_attr(docsrs, doc(cfg(feature = "std")))]
+pub mod message_compact_blocks;
+#[cfg(feature = "std")]
+#[cfg_attr(docsrs, doc(cfg(feature = "std")))]
 pub mod message_network;
 #[cfg(feature = "std")]
 #[cfg_attr(docsrs, doc(cfg(feature = "std")))]