Skip to content

Segwit arbitrary commitment trees & faster hashing #75

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 4 commits into
base: segwit-master
Choose a base branch
from
Open

Segwit arbitrary commitment trees & faster hashing #75

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
516c41f
Allow arbitrary commitment tree structure.
maaku Jan 22, 2016
7dd2bd7
Add mistate output to CSHA256.
maaku Jan 22, 2016
6dadca6
Add fast Merkle branch functions.
maaku Jan 22, 2016
f2ad398
Switch to fast Merkle trees for witness.
maaku Jan 22, 2016
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
76 changes: 68 additions & 8 deletions src/consensus/merkle.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -42,14 +42,32 @@
root.
*/

typedef enum {
MERKLE_COMPUTATION_MUTABLE = 0x1,
MERKLE_COMPUTATION_FAST = 0x2
} merklecomputationopts;

static void MerkleHash_Hash256(uint256& parent, const uint256& left, const uint256& right) {
CHash256().Write(left.begin(), 32).Write(right.begin(), 32).Finalize(parent.begin());
}

static void MerkleHash_Sha256Midstate(uint256& parent, const uint256& left, const uint256& right) {
CSHA256().Write(left.begin(), 32).Write(right.begin(), 32).Midstate(parent.begin(), NULL, NULL);
}

/* This implements a constant-space merkle root/path calculator, limited to 2^32 leaves. */
static void MerkleComputation(const std::vector<uint256>& leaves, uint256* proot, bool* pmutated, uint32_t branchpos, std::vector<uint256>* pbranch) {
static void MerkleComputation(const std::vector<uint256>& leaves, uint256* proot, bool* pmutated, uint32_t branchpos, std::vector<uint256>* pbranch, merklecomputationopts flags) {
if (pbranch) pbranch->clear();
if (leaves.size() == 0) {
if (pmutated) *pmutated = false;
if (proot) *proot = uint256();
return;
}
bool fMutable = flags & MERKLE_COMPUTATION_MUTABLE;
void (*MerkleHash)(uint256&, const uint256&, const uint256&) = MerkleHash_Hash256;
if (flags & MERKLE_COMPUTATION_FAST) {
MerkleHash = MerkleHash_Sha256Midstate;
}
bool mutated = false;
// count is the number of leaves processed so far.
uint32_t count = 0;
Expand Down Expand Up @@ -80,7 +98,7 @@ static void MerkleComputation(const std::vector<uint256>& leaves, uint256* proot
}
}
mutated |= (inner[level] == h);
CHash256().Write(inner[level].begin(), 32).Write(h.begin(), 32).Finalize(h.begin());
MerkleHash(h, inner[level], h);
}
// Store the resulting hash at inner position level.
inner[level] = h;
Expand All @@ -106,7 +124,9 @@ static void MerkleComputation(const std::vector<uint256>& leaves, uint256* proot
if (pbranch && matchh) {
pbranch->push_back(h);
}
CHash256().Write(h.begin(), 32).Write(h.begin(), 32).Finalize(h.begin());
if (fMutable) {
MerkleHash(h, h, h);
}
// Increment count to the value it would have if two entries at this
// level had existed.
count += (((uint32_t)1) << level);
Expand All @@ -121,7 +141,7 @@ static void MerkleComputation(const std::vector<uint256>& leaves, uint256* proot
matchh = true;
}
}
CHash256().Write(inner[level].begin(), 32).Write(h.begin(), 32).Finalize(h.begin());
MerkleHash(h, inner[level], h);
level++;
}
}
Expand All @@ -132,13 +152,13 @@ static void MerkleComputation(const std::vector<uint256>& leaves, uint256* proot

uint256 ComputeMerkleRoot(const std::vector<uint256>& leaves, bool* mutated) {
uint256 hash;
MerkleComputation(leaves, &hash, mutated, -1, NULL);
MerkleComputation(leaves, &hash, mutated, -1, NULL, MERKLE_COMPUTATION_MUTABLE);
return hash;
}

std::vector<uint256> ComputeMerkleBranch(const std::vector<uint256>& leaves, uint32_t position) {
std::vector<uint256> ret;
MerkleComputation(leaves, NULL, NULL, position, &ret);
MerkleComputation(leaves, NULL, NULL, position, &ret, MERKLE_COMPUTATION_MUTABLE);
return ret;
}

Expand All @@ -155,6 +175,46 @@ uint256 ComputeMerkleRootFromBranch(const uint256& leaf, const std::vector<uint2
return hash;
}

uint256 ComputeFastMerkleRoot(const std::vector<uint256>& leaves) {
uint256 hash;
MerkleComputation(leaves, &hash, NULL, -1, NULL, MERKLE_COMPUTATION_FAST);
return hash;
}

std::vector<uint256> ComputeFastMerkleBranch(const std::vector<uint256>& leaves, uint32_t position) {
std::vector<uint256> ret;
MerkleComputation(leaves, NULL, NULL, position, &ret, MERKLE_COMPUTATION_FAST);
return ret;
}

uint256 ComputeFastMerkleRootFromBranch(const uint256& leaf, const std::vector<uint256>& vMerkleBranch, uint32_t nIndex) {
size_t max = 0;
for (int i = 0; i < 32; ++i)
if (nIndex & ((uint32_t)1)<<i)
max = i + 1;
while (max > vMerkleBranch.size()) {
int i;
for (i = max-1; i >= 0; --i)
if (!(nIndex & ((uint32_t)1)<<i))
break;
if (i < 0)
return uint256();
nIndex = (((((uint32_t)1)<<(i+1))-1)>>1) |
((((uint32_t)1)<< i )-1);
--max;
}
uint256 hash = leaf;
for (std::vector<uint256>::const_iterator it = vMerkleBranch.begin(); it != vMerkleBranch.end(); ++it) {
if (nIndex & 1) {
MerkleHash_Sha256Midstate(hash, *it, hash);
} else {
MerkleHash_Sha256Midstate(hash, hash, *it);
}
nIndex >>= 1;
}
return hash;
}

uint256 BlockMerkleRoot(const CBlock& block, bool* mutated)
{
std::vector<uint256> leaves;
Expand All @@ -165,15 +225,15 @@ uint256 BlockMerkleRoot(const CBlock& block, bool* mutated)
return ComputeMerkleRoot(leaves, mutated);
}

uint256 BlockWitnessMerkleRoot(const CBlock& block, bool* mutated)
uint256 BlockWitnessMerkleRoot(const CBlock& block)
{
std::vector<uint256> leaves;
leaves.resize(block.vtx.size());
leaves[0].SetNull(); // The witness hash of the coinbase is 0.
for (size_t s = 1; s < block.vtx.size(); s++) {
leaves[s] = block.vtx[s].GetWitnessHash();
}
return ComputeMerkleRoot(leaves, mutated);
return ComputeFastMerkleRoot(leaves);
}

std::vector<uint256> BlockMerkleBranch(const CBlock& block, uint32_t position)
Expand Down
12 changes: 11 additions & 1 deletion src/consensus/merkle.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,16 @@ uint256 ComputeMerkleRoot(const std::vector<uint256>& leaves, bool* mutated = NU
std::vector<uint256> ComputeMerkleBranch(const std::vector<uint256>& leaves, uint32_t position);
uint256 ComputeMerkleRootFromBranch(const uint256& leaf, const std::vector<uint256>& branch, uint32_t position);

/*
* Has similar API semantics, but produces Merkle roots and validates
* branches 3x as fast, and without the mutation vulnerability. Cannot
* be substituted for the non-fast variants because the hash values are
* different.
*/
uint256 ComputeFastMerkleRoot(const std::vector<uint256>& leaves);
std::vector<uint256> ComputeFastMerkleBranch(const std::vector<uint256>& leaves, uint32_t position);
uint256 ComputeFastMerkleRootFromBranch(const uint256& leaf, const std::vector<uint256>& branch, uint32_t position);

/*
* Compute the Merkle root of the transactions in a block.
* *mutated is set to true if a duplicated subtree was found.
Expand All @@ -26,7 +36,7 @@ uint256 BlockMerkleRoot(const CBlock& block, bool* mutated = NULL);
* Compute the Merkle root of the witness transactions in a block.
* *mutated is set to true if a duplicated subtree was found.
*/
uint256 BlockWitnessMerkleRoot(const CBlock& block, bool* mutated = NULL);
uint256 BlockWitnessMerkleRoot(const CBlock& block);

/*
* Compute the Merkle branch for the tree of transactions in a block, for a
Expand Down
11 changes: 11 additions & 0 deletions src/crypto/sha256.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -171,6 +171,11 @@ void CSHA256::Finalize(unsigned char hash[OUTPUT_SIZE])
WriteBE64(sizedesc, bytes << 3);
Write(pad, 1 + ((119 - (bytes % 64)) % 64));
Write(sizedesc, 8);
Midstate(hash, NULL, NULL);
}

void CSHA256::Midstate(unsigned char hash[OUTPUT_SIZE], uint64_t* len, unsigned char *buffer)
{
WriteBE32(hash, s[0]);
WriteBE32(hash + 4, s[1]);
WriteBE32(hash + 8, s[2]);
Expand All @@ -179,6 +184,12 @@ void CSHA256::Finalize(unsigned char hash[OUTPUT_SIZE])
WriteBE32(hash + 20, s[5]);
WriteBE32(hash + 24, s[6]);
WriteBE32(hash + 28, s[7]);
if (len) {
*len = bytes << 3;
}
if (buffer) {
memcpy(buffer, buf, bytes % 64);
}
}

CSHA256& CSHA256::Reset()
Expand Down
1 change: 1 addition & 0 deletions src/crypto/sha256.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ class CSHA256
CSHA256();
CSHA256& Write(const unsigned char* data, size_t len);
void Finalize(unsigned char hash[OUTPUT_SIZE]);
void Midstate(unsigned char hash[OUTPUT_SIZE], uint64_t* bytes, unsigned char *buffer);
CSHA256& Reset();
};

Expand Down
77 changes: 58 additions & 19 deletions src/main.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3353,19 +3353,35 @@ bool IsWitnessEnabled(const CBlockIndex* pindexPrev, const Consensus::Params& pa
return (VersionBitsState(pindexPrev, params, Consensus::DEPLOYMENT_WITNESS, versionbitscache) == THRESHOLD_ACTIVE);
}

inline bool IsCoinbaseCommitment(const CTxOut& txout)
{
if ( txout.scriptPubKey.size() >= 39
&& txout.scriptPubKey[0] == OP_RETURN
&& txout.scriptPubKey[1] >= 0x24
&& txout.scriptPubKey[1] <= 0x4b
&& txout.scriptPubKey[2] == 0xaa
&& txout.scriptPubKey[3] == 0x21
&& txout.scriptPubKey[4] == 0xa9
&& txout.scriptPubKey[5] == 0xed)
{
return true;
}

return false;
}

void UpdateUncommitedBlockStructures(CBlock& block, const CBlockIndex* pindexPrev, const Consensus::Params& consensusParams)
{
int commitpos = -1;
for (size_t o = 0; o < block.vtx[0].vout.size(); o++) {
if (block.vtx[0].vout[o].scriptPubKey.size() >= 38 && block.vtx[0].vout[o].scriptPubKey[0] == OP_RETURN && block.vtx[0].vout[o].scriptPubKey[1] == 0x24 && block.vtx[0].vout[o].scriptPubKey[2] == 0xaa && block.vtx[0].vout[o].scriptPubKey[3] == 0x21 && block.vtx[0].vout[o].scriptPubKey[4] == 0xa9 && block.vtx[0].vout[o].scriptPubKey[5] == 0xed) {
if (IsCoinbaseCommitment(block.vtx[0].vout[o])) {
commitpos = o;
}
}
static const std::vector<unsigned char> nonce(32, 0x00);
if (commitpos != -1 && IsWitnessEnabled(pindexPrev, consensusParams) && block.vtx[0].wit.IsEmpty()) {
block.vtx[0].wit.vtxinwit.resize(1);
block.vtx[0].wit.vtxinwit[0].scriptWitness.stack.resize(1);
block.vtx[0].wit.vtxinwit[0].scriptWitness.stack[0] = nonce;
block.vtx[0].wit.vtxinwit[0].scriptWitness.stack[0] = std::vector<unsigned char>(1, 0x00);
}
}

Expand All @@ -3374,7 +3390,7 @@ std::vector<unsigned char> GenerateCoinbaseCommitment(CBlock& block, const CBloc
std::vector<unsigned char> commitment;
int commitpos = -1;
for (size_t o = 0; o < block.vtx[0].vout.size(); o++) {
if (block.vtx[0].vout[o].scriptPubKey.size() >= 38 && block.vtx[0].vout[o].scriptPubKey[0] == OP_RETURN && block.vtx[0].vout[o].scriptPubKey[1] == 0x24 && block.vtx[0].vout[o].scriptPubKey[2] == 0xaa && block.vtx[0].vout[o].scriptPubKey[3] == 0x21 && block.vtx[0].vout[o].scriptPubKey[4] == 0xa9 && block.vtx[0].vout[o].scriptPubKey[5] == 0xed) {
if (IsCoinbaseCommitment(block.vtx[0].vout[o])) {
commitpos = o;
}
}
Expand All @@ -3385,21 +3401,20 @@ std::vector<unsigned char> GenerateCoinbaseCommitment(CBlock& block, const CBloc
break;
}
}
std::vector<unsigned char> ret(32, 0x00);
if (fHaveWitness && IsWitnessEnabled(pindexPrev, consensusParams)) {
if (commitpos == -1) {
uint256 witnessroot = BlockWitnessMerkleRoot(block, NULL);
CHash256().Write(witnessroot.begin(), 32).Write(&ret[0], 32).Finalize(witnessroot.begin());
uint256 witnessroot = BlockWitnessMerkleRoot(block);
CTxOut out;
out.nValue = 0;
out.scriptPubKey.resize(38);
out.scriptPubKey.resize(39);
out.scriptPubKey[0] = OP_RETURN;
out.scriptPubKey[1] = 0x24;
out.scriptPubKey[2] = 0xaa;
out.scriptPubKey[3] = 0x21;
out.scriptPubKey[4] = 0xa9;
out.scriptPubKey[5] = 0xed;
memcpy(&out.scriptPubKey[6], witnessroot.begin(), 32);
out.scriptPubKey[6] = 0x01;
memcpy(&out.scriptPubKey[7], witnessroot.begin(), 32);
commitment = std::vector<unsigned char>(out.scriptPubKey.begin(), out.scriptPubKey.end());
const_cast<std::vector<CTxOut>*>(&block.vtx[0].vout)->push_back(out);
block.vtx[0].UpdateHash();
Expand Down Expand Up @@ -3465,30 +3480,54 @@ bool ContextualCheckBlock(const CBlock& block, CValidationState& state, CBlockIn
// Validation for witness commitments.
// * We compute the witness hash (which is the hash including witnesses) of all the block's transactions, except the
// coinbase (where 0x0000....0000 is used instead).
// * The coinbase scriptWitness is a stack of a single 32-byte vector, containing a witness nonce (unconstrained).
// * The coinbase scriptWitness is a stack of a serialize N*32-byte vector, where N is the number of hashes in the
// (unconstrained) Merkle path from the commitment to the witness tree (0 <= N <= 7).
// * We build a merkle tree with all those witness hashes as leaves (similar to the hashMerkleRoot in the block header).
// * The must be at least one output whose scriptPubKey is a single 36-byte push, the first 4 bytes of which are
// {0xaa, 0x21, 0xa9, 0xed}, and the following 32 bytes are SHA256(witness root, witness nonce). In case there are
// multiple, the last one is used.
// * There must be at least one output whose scriptPubKey is a single >=37-byte push, the first 4 bytes of which are
// {0xaa, 0x21, 0xa9, 0xed}, followed by between 0 and 38 unconstrained bytes, then a single byte encoding of the
// path from the commitment root to the witness root, and the final 32 bytes are the commitment root. In case there
// are multiple, the last one is used.
bool fHaveWitness = false;
if (IsWitnessEnabled(pindexPrev, consensusParams)) {
int commitpos = -1;
for (size_t o = 0; o < block.vtx[0].vout.size(); o++) {
if (block.vtx[0].vout[o].scriptPubKey.size() >= 38 && block.vtx[0].vout[o].scriptPubKey[0] == OP_RETURN && block.vtx[0].vout[o].scriptPubKey[1] == 0x24 && block.vtx[0].vout[o].scriptPubKey[2] == 0xaa && block.vtx[0].vout[o].scriptPubKey[3] == 0x21 && block.vtx[0].vout[o].scriptPubKey[4] == 0xa9 && block.vtx[0].vout[o].scriptPubKey[5] == 0xed) {
if ( block.vtx[0].vout[o].scriptPubKey.size() >= 39
&& block.vtx[0].vout[o].scriptPubKey[0] == OP_RETURN
&& block.vtx[0].vout[o].scriptPubKey[1] >= 0x24
&& block.vtx[0].vout[o].scriptPubKey[1] <= 0x4b
&& block.vtx[0].vout[o].scriptPubKey[2] == 0xaa
&& block.vtx[0].vout[o].scriptPubKey[3] == 0x21
&& block.vtx[0].vout[o].scriptPubKey[4] == 0xa9
&& block.vtx[0].vout[o].scriptPubKey[5] == 0xed
&& block.vtx[0].vout[o].scriptPubKey[
block.vtx[0].vout[o].scriptPubKey.size()-33] != 0x00)
{
commitpos = o;
}
}
if (commitpos != -1) {
bool malleated = false;
uint256 hashWitness = BlockWitnessMerkleRoot(block, &malleated);
uint256 hashWitness = BlockWitnessMerkleRoot(block);
// The malleation check is ignored; as the transaction tree itself
// already does not permit it, it is impossible to trigger in the
// witness tree.
if (block.vtx[0].wit.vtxinwit.size() != 1 || block.vtx[0].wit.vtxinwit[0].scriptWitness.stack.size() != 1 || block.vtx[0].wit.vtxinwit[0].scriptWitness.stack[0].size() != 32) {
const CScript& commitscript = block.vtx[0].vout[commitpos].scriptPubKey;
const unsigned char& witnesspath = commitscript[commitscript.size()-33];
size_t witnessdepth = 0;
for (int pos = 0; pos < 8; ++pos)
if (witnesspath & (1<<pos))
witnessdepth = pos;
if ( block.vtx[0].wit.vtxinwit.size() != 1
|| block.vtx[0].wit.vtxinwit[0].scriptWitness.stack.size() != 1
|| block.vtx[0].wit.vtxinwit[0].scriptWitness.stack[0].size() != 1+witnessdepth*32
|| block.vtx[0].wit.vtxinwit[0].scriptWitness.stack[0][0] != witnessdepth)
{
return state.DoS(100, error("%s : invalid witness commitment size", __func__), REJECT_INVALID, "bad-witness-merkle-size", true);
}
CHash256().Write(hashWitness.begin(), 32).Write(&block.vtx[0].wit.vtxinwit[0].scriptWitness.stack[0][0], 32).Finalize(hashWitness.begin());
if (memcmp(hashWitness.begin(), &block.vtx[0].vout[commitpos].scriptPubKey[6], 32)) {
CDataStream ds(block.vtx[0].wit.vtxinwit[0].scriptWitness.stack[0], SER_NETWORK, PROTOCOL_VERSION);
std::vector<uint256> branch;
ds >> branch;
hashWitness = ComputeMerkleRootFromBranch(hashWitness, branch, witnesspath ^ (1<<witnessdepth));
if (memcmp(hashWitness.begin(), &commitscript[commitscript.size()-32], 32)) {
return state.DoS(100, error("%s : witness merkle commitment mismatch", __func__), REJECT_INVALID, "bad-witness-merkle-match", true);
}
fHaveWitness = true;
Expand Down