Skip to content

Commit 66db2d6

Browse files
committed
Merge bitcoin#7600: Mining: Select transactions using feerate-with-ancestors
29fac19 Add unit tests for ancestor feerate mining (Suhas Daftuar) c82a4e9 Use ancestor-feerate based transaction selection for mining (Suhas Daftuar)
2 parents 9c3d0fa + 29fac19 commit 66db2d6

File tree

3 files changed

+432
-1
lines changed

3 files changed

+432
-1
lines changed

src/miner.cpp

Lines changed: 205 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@
2525
#include "utilmoneystr.h"
2626
#include "validationinterface.h"
2727

28+
#include <algorithm>
2829
#include <boost/thread.hpp>
2930
#include <boost/tuple/tuple.hpp>
3031
#include <queue>
@@ -134,7 +135,7 @@ CBlockTemplate* BlockAssembler::CreateNewBlock(const CScript& scriptPubKeyIn)
134135
: pblock->GetBlockTime();
135136

136137
addPriorityTxs();
137-
addScoreTxs();
138+
addPackageTxs();
138139

139140
nLastBlockTx = nBlockTx;
140141
nLastBlockSize = nBlockSize;
@@ -177,7 +178,38 @@ bool BlockAssembler::isStillDependent(CTxMemPool::txiter iter)
177178
return false;
178179
}
179180

181+
void BlockAssembler::onlyUnconfirmed(CTxMemPool::setEntries& testSet)
182+
{
183+
for (CTxMemPool::setEntries::iterator iit = testSet.begin(); iit != testSet.end(); ) {
184+
// Only test txs not already in the block
185+
if (inBlock.count(*iit)) {
186+
testSet.erase(iit++);
187+
}
188+
else {
189+
iit++;
190+
}
191+
}
192+
}
193+
194+
bool BlockAssembler::TestPackage(uint64_t packageSize, unsigned int packageSigOps)
195+
{
196+
if (nBlockSize + packageSize >= nBlockMaxSize)
197+
return false;
198+
if (nBlockSigOps + packageSigOps >= MAX_BLOCK_SIGOPS)
199+
return false;
200+
return true;
201+
}
180202

203+
// Block size and sigops have already been tested. Check that all transactions
204+
// are final.
205+
bool BlockAssembler::TestPackageFinality(const CTxMemPool::setEntries& package)
206+
{
207+
BOOST_FOREACH (const CTxMemPool::txiter it, package) {
208+
if (!IsFinalTx(it->GetTx(), nHeight, nLockTimeCutoff))
209+
return false;
210+
}
211+
return true;
212+
}
181213

182214
bool BlockAssembler::TestForBlock(CTxMemPool::txiter iter)
183215
{
@@ -297,6 +329,178 @@ void BlockAssembler::addScoreTxs()
297329
}
298330
}
299331

332+
void BlockAssembler::UpdatePackagesForAdded(const CTxMemPool::setEntries& alreadyAdded,
333+
indexed_modified_transaction_set &mapModifiedTx)
334+
{
335+
BOOST_FOREACH(const CTxMemPool::txiter it, alreadyAdded) {
336+
CTxMemPool::setEntries descendants;
337+
mempool.CalculateDescendants(it, descendants);
338+
// Insert all descendants (not yet in block) into the modified set
339+
BOOST_FOREACH(CTxMemPool::txiter desc, descendants) {
340+
if (alreadyAdded.count(desc))
341+
continue;
342+
modtxiter mit = mapModifiedTx.find(desc);
343+
if (mit == mapModifiedTx.end()) {
344+
CTxMemPoolModifiedEntry modEntry(desc);
345+
modEntry.nSizeWithAncestors -= it->GetTxSize();
346+
modEntry.nModFeesWithAncestors -= it->GetModifiedFee();
347+
modEntry.nSigOpCountWithAncestors -= it->GetSigOpCount();
348+
mapModifiedTx.insert(modEntry);
349+
} else {
350+
mapModifiedTx.modify(mit, update_for_parent_inclusion(it));
351+
}
352+
}
353+
}
354+
}
355+
356+
// Skip entries in mapTx that are already in a block or are present
357+
// in mapModifiedTx (which implies that the mapTx ancestor state is
358+
// stale due to ancestor inclusion in the block)
359+
// Also skip transactions that we've already failed to add. This can happen if
360+
// we consider a transaction in mapModifiedTx and it fails: we can then
361+
// potentially consider it again while walking mapTx. It's currently
362+
// guaranteed to fail again, but as a belt-and-suspenders check we put it in
363+
// failedTx and avoid re-evaluation, since the re-evaluation would be using
364+
// cached size/sigops/fee values that are not actually correct.
365+
bool BlockAssembler::SkipMapTxEntry(CTxMemPool::txiter it, indexed_modified_transaction_set &mapModifiedTx, CTxMemPool::setEntries &failedTx)
366+
{
367+
assert (it != mempool.mapTx.end());
368+
if (mapModifiedTx.count(it) || inBlock.count(it) || failedTx.count(it))
369+
return true;
370+
return false;
371+
}
372+
373+
void BlockAssembler::SortForBlock(const CTxMemPool::setEntries& package, CTxMemPool::txiter entry, std::vector<CTxMemPool::txiter>& sortedEntries)
374+
{
375+
// Sort package by ancestor count
376+
// If a transaction A depends on transaction B, then A's ancestor count
377+
// must be greater than B's. So this is sufficient to validly order the
378+
// transactions for block inclusion.
379+
sortedEntries.clear();
380+
sortedEntries.insert(sortedEntries.begin(), package.begin(), package.end());
381+
std::sort(sortedEntries.begin(), sortedEntries.end(), CompareTxIterByAncestorCount());
382+
}
383+
384+
// This transaction selection algorithm orders the mempool based
385+
// on feerate of a transaction including all unconfirmed ancestors.
386+
// Since we don't remove transactions from the mempool as we select them
387+
// for block inclusion, we need an alternate method of updating the feerate
388+
// of a transaction with its not-yet-selected ancestors as we go.
389+
// This is accomplished by walking the in-mempool descendants of selected
390+
// transactions and storing a temporary modified state in mapModifiedTxs.
391+
// Each time through the loop, we compare the best transaction in
392+
// mapModifiedTxs with the next transaction in the mempool to decide what
393+
// transaction package to work on next.
394+
void BlockAssembler::addPackageTxs()
395+
{
396+
// mapModifiedTx will store sorted packages after they are modified
397+
// because some of their txs are already in the block
398+
indexed_modified_transaction_set mapModifiedTx;
399+
// Keep track of entries that failed inclusion, to avoid duplicate work
400+
CTxMemPool::setEntries failedTx;
401+
402+
// Start by adding all descendants of previously added txs to mapModifiedTx
403+
// and modifying them for their already included ancestors
404+
UpdatePackagesForAdded(inBlock, mapModifiedTx);
405+
406+
CTxMemPool::indexed_transaction_set::index<ancestor_score>::type::iterator mi = mempool.mapTx.get<ancestor_score>().begin();
407+
CTxMemPool::txiter iter;
408+
while (mi != mempool.mapTx.get<ancestor_score>().end() || !mapModifiedTx.empty())
409+
{
410+
// First try to find a new transaction in mapTx to evaluate.
411+
if (mi != mempool.mapTx.get<ancestor_score>().end() &&
412+
SkipMapTxEntry(mempool.mapTx.project<0>(mi), mapModifiedTx, failedTx)) {
413+
++mi;
414+
continue;
415+
}
416+
417+
// Now that mi is not stale, determine which transaction to evaluate:
418+
// the next entry from mapTx, or the best from mapModifiedTx?
419+
bool fUsingModified = false;
420+
421+
modtxscoreiter modit = mapModifiedTx.get<ancestor_score>().begin();
422+
if (mi == mempool.mapTx.get<ancestor_score>().end()) {
423+
// We're out of entries in mapTx; use the entry from mapModifiedTx
424+
iter = modit->iter;
425+
fUsingModified = true;
426+
} else {
427+
// Try to compare the mapTx entry to the mapModifiedTx entry
428+
iter = mempool.mapTx.project<0>(mi);
429+
if (modit != mapModifiedTx.get<ancestor_score>().end() &&
430+
CompareModifiedEntry()(*modit, CTxMemPoolModifiedEntry(iter))) {
431+
// The best entry in mapModifiedTx has higher score
432+
// than the one from mapTx.
433+
// Switch which transaction (package) to consider
434+
iter = modit->iter;
435+
fUsingModified = true;
436+
} else {
437+
// Either no entry in mapModifiedTx, or it's worse than mapTx.
438+
// Increment mi for the next loop iteration.
439+
++mi;
440+
}
441+
}
442+
443+
// We skip mapTx entries that are inBlock, and mapModifiedTx shouldn't
444+
// contain anything that is inBlock.
445+
assert(!inBlock.count(iter));
446+
447+
uint64_t packageSize = iter->GetSizeWithAncestors();
448+
CAmount packageFees = iter->GetModFeesWithAncestors();
449+
unsigned int packageSigOps = iter->GetSigOpCountWithAncestors();
450+
if (fUsingModified) {
451+
packageSize = modit->nSizeWithAncestors;
452+
packageFees = modit->nModFeesWithAncestors;
453+
packageSigOps = modit->nSigOpCountWithAncestors;
454+
}
455+
456+
if (packageFees < ::minRelayTxFee.GetFee(packageSize) && nBlockSize >= nBlockMinSize) {
457+
// Everything else we might consider has a lower fee rate
458+
return;
459+
}
460+
461+
if (!TestPackage(packageSize, packageSigOps)) {
462+
if (fUsingModified) {
463+
// Since we always look at the best entry in mapModifiedTx,
464+
// we must erase failed entries so that we can consider the
465+
// next best entry on the next loop iteration
466+
mapModifiedTx.get<ancestor_score>().erase(modit);
467+
failedTx.insert(iter);
468+
}
469+
continue;
470+
}
471+
472+
CTxMemPool::setEntries ancestors;
473+
uint64_t nNoLimit = std::numeric_limits<uint64_t>::max();
474+
std::string dummy;
475+
mempool.CalculateMemPoolAncestors(*iter, ancestors, nNoLimit, nNoLimit, nNoLimit, nNoLimit, dummy, false);
476+
477+
onlyUnconfirmed(ancestors);
478+
ancestors.insert(iter);
479+
480+
// Test if all tx's are Final
481+
if (!TestPackageFinality(ancestors)) {
482+
if (fUsingModified) {
483+
mapModifiedTx.get<ancestor_score>().erase(modit);
484+
failedTx.insert(iter);
485+
}
486+
continue;
487+
}
488+
489+
// Package can be added. Sort the entries in a valid order.
490+
vector<CTxMemPool::txiter> sortedEntries;
491+
SortForBlock(ancestors, iter, sortedEntries);
492+
493+
for (size_t i=0; i<sortedEntries.size(); ++i) {
494+
AddToBlock(sortedEntries[i]);
495+
// Erase from the modified set, if present
496+
mapModifiedTx.erase(sortedEntries[i]);
497+
}
498+
499+
// Update transactions that depend on each of these
500+
UpdatePackagesForAdded(ancestors, mapModifiedTx);
501+
}
502+
}
503+
300504
void BlockAssembler::addPriorityTxs()
301505
{
302506
// How much of the block should be dedicated to high-priority transactions,

src/miner.h

Lines changed: 118 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,8 @@
1111

1212
#include <stdint.h>
1313
#include <memory>
14+
#include "boost/multi_index_container.hpp"
15+
#include "boost/multi_index/ordered_index.hpp"
1416

1517
class CBlockIndex;
1618
class CChainParams;
@@ -29,6 +31,104 @@ struct CBlockTemplate
2931
std::vector<int64_t> vTxSigOps;
3032
};
3133

34+
// Container for tracking updates to ancestor feerate as we include (parent)
35+
// transactions in a block
36+
struct CTxMemPoolModifiedEntry {
37+
CTxMemPoolModifiedEntry(CTxMemPool::txiter entry)
38+
{
39+
iter = entry;
40+
nSizeWithAncestors = entry->GetSizeWithAncestors();
41+
nModFeesWithAncestors = entry->GetModFeesWithAncestors();
42+
nSigOpCountWithAncestors = entry->GetSigOpCountWithAncestors();
43+
}
44+
45+
CTxMemPool::txiter iter;
46+
uint64_t nSizeWithAncestors;
47+
CAmount nModFeesWithAncestors;
48+
unsigned int nSigOpCountWithAncestors;
49+
};
50+
51+
/** Comparator for CTxMemPool::txiter objects.
52+
* It simply compares the internal memory address of the CTxMemPoolEntry object
53+
* pointed to. This means it has no meaning, and is only useful for using them
54+
* as key in other indexes.
55+
*/
56+
struct CompareCTxMemPoolIter {
57+
bool operator()(const CTxMemPool::txiter& a, const CTxMemPool::txiter& b) const
58+
{
59+
return &(*a) < &(*b);
60+
}
61+
};
62+
63+
struct modifiedentry_iter {
64+
typedef CTxMemPool::txiter result_type;
65+
result_type operator() (const CTxMemPoolModifiedEntry &entry) const
66+
{
67+
return entry.iter;
68+
}
69+
};
70+
71+
// This matches the calculation in CompareTxMemPoolEntryByAncestorFee,
72+
// except operating on CTxMemPoolModifiedEntry.
73+
// TODO: refactor to avoid duplication of this logic.
74+
struct CompareModifiedEntry {
75+
bool operator()(const CTxMemPoolModifiedEntry &a, const CTxMemPoolModifiedEntry &b)
76+
{
77+
double f1 = (double)a.nModFeesWithAncestors * b.nSizeWithAncestors;
78+
double f2 = (double)b.nModFeesWithAncestors * a.nSizeWithAncestors;
79+
if (f1 == f2) {
80+
return CTxMemPool::CompareIteratorByHash()(a.iter, b.iter);
81+
}
82+
return f1 > f2;
83+
}
84+
};
85+
86+
// A comparator that sorts transactions based on number of ancestors.
87+
// This is sufficient to sort an ancestor package in an order that is valid
88+
// to appear in a block.
89+
struct CompareTxIterByAncestorCount {
90+
bool operator()(const CTxMemPool::txiter &a, const CTxMemPool::txiter &b)
91+
{
92+
if (a->GetCountWithAncestors() != b->GetCountWithAncestors())
93+
return a->GetCountWithAncestors() < b->GetCountWithAncestors();
94+
return CTxMemPool::CompareIteratorByHash()(a, b);
95+
}
96+
};
97+
98+
typedef boost::multi_index_container<
99+
CTxMemPoolModifiedEntry,
100+
boost::multi_index::indexed_by<
101+
boost::multi_index::ordered_unique<
102+
modifiedentry_iter,
103+
CompareCTxMemPoolIter
104+
>,
105+
// sorted by modified ancestor fee rate
106+
boost::multi_index::ordered_non_unique<
107+
// Reuse same tag from CTxMemPool's similar index
108+
boost::multi_index::tag<ancestor_score>,
109+
boost::multi_index::identity<CTxMemPoolModifiedEntry>,
110+
CompareModifiedEntry
111+
>
112+
>
113+
> indexed_modified_transaction_set;
114+
115+
typedef indexed_modified_transaction_set::nth_index<0>::type::iterator modtxiter;
116+
typedef indexed_modified_transaction_set::index<ancestor_score>::type::iterator modtxscoreiter;
117+
118+
struct update_for_parent_inclusion
119+
{
120+
update_for_parent_inclusion(CTxMemPool::txiter it) : iter(it) {}
121+
122+
void operator() (CTxMemPoolModifiedEntry &e)
123+
{
124+
e.nModFeesWithAncestors -= iter->GetFee();
125+
e.nSizeWithAncestors -= iter->GetTxSize();
126+
e.nSigOpCountWithAncestors -= iter->GetSigOpCount();
127+
}
128+
129+
CTxMemPool::txiter iter;
130+
};
131+
32132
/** Generate a new block, without valid proof-of-work */
33133
class BlockAssembler
34134
{
@@ -74,12 +174,30 @@ class BlockAssembler
74174
void addScoreTxs();
75175
/** Add transactions based on tx "priority" */
76176
void addPriorityTxs();
177+
/** Add transactions based on feerate including unconfirmed ancestors */
178+
void addPackageTxs();
77179

78180
// helper function for addScoreTxs and addPriorityTxs
79181
/** Test if tx will still "fit" in the block */
80182
bool TestForBlock(CTxMemPool::txiter iter);
81183
/** Test if tx still has unconfirmed parents not yet in block */
82184
bool isStillDependent(CTxMemPool::txiter iter);
185+
186+
// helper functions for addPackageTxs()
187+
/** Remove confirmed (inBlock) entries from given set */
188+
void onlyUnconfirmed(CTxMemPool::setEntries& testSet);
189+
/** Test if a new package would "fit" in the block */
190+
bool TestPackage(uint64_t packageSize, unsigned int packageSigOps);
191+
/** Test if a set of transactions are all final */
192+
bool TestPackageFinality(const CTxMemPool::setEntries& package);
193+
/** Return true if given transaction from mapTx has already been evaluated,
194+
* or if the transaction's cached data in mapTx is incorrect. */
195+
bool SkipMapTxEntry(CTxMemPool::txiter it, indexed_modified_transaction_set &mapModifiedTx, CTxMemPool::setEntries &failedTx);
196+
/** Sort the package in an order that is valid to appear in a block */
197+
void SortForBlock(const CTxMemPool::setEntries& package, CTxMemPool::txiter entry, std::vector<CTxMemPool::txiter>& sortedEntries);
198+
/** Add descendants of given transactions to mapModifiedTx with ancestor
199+
* state updated assuming given transactions are inBlock. */
200+
void UpdatePackagesForAdded(const CTxMemPool::setEntries& alreadyAdded, indexed_modified_transaction_set &mapModifiedTx);
83201
};
84202

85203
/** Modify the extranonce in a block */

0 commit comments

Comments
 (0)