Skip to content

Commit 26d123a

Browse files
committed
Implement on-the-fly mempool size limitation.
1 parent 550ae8a commit 26d123a

File tree

6 files changed

+171
-22
lines changed

6 files changed

+171
-22
lines changed

src/init.cpp

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -283,6 +283,7 @@ std::string HelpMessage(HelpMessageMode mode)
283283
strUsage += HelpMessageOpt("-dbcache=<n>", strprintf(_("Set database cache size in megabytes (%d to %d, default: %d)"), nMinDbCache, nMaxDbCache, nDefaultDbCache));
284284
strUsage += HelpMessageOpt("-loadblock=<file>", _("Imports blocks from external blk000??.dat file") + " " + _("on startup"));
285285
strUsage += HelpMessageOpt("-maxorphantx=<n>", strprintf(_("Keep at most <n> unconnectable transactions in memory (default: %u)"), DEFAULT_MAX_ORPHAN_TRANSACTIONS));
286+
strUsage += HelpMessageOpt("-maxmempool=<n>", strprintf(_("Keep the transaction memory pool below <n> megabytes (default: %u)"), DEFAULT_MAX_MEMPOOL_SIZE));
286287
strUsage += HelpMessageOpt("-par=<n>", strprintf(_("Set the number of script verification threads (%u to %d, 0 = auto, <0 = leave that many cores free, default: %d)"),
287288
-GetNumCores(), MAX_SCRIPTCHECK_THREADS, DEFAULT_SCRIPTCHECK_THREADS));
288289
#ifndef WIN32

src/main.cpp

Lines changed: 17 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -882,22 +882,29 @@ bool AcceptToMemoryPool(CTxMemPool& pool, CValidationState &state, const CTransa
882882
CTxMemPoolEntry entry(tx, nFees, GetTime(), dPriority, chainActive.Height(), mempool.HasNoInputsOf(tx));
883883
unsigned int nSize = entry.GetTxSize();
884884

885+
// Try to make space in mempool
886+
std::set<uint256> stagedelete;
887+
CAmount nFeesDeleted = 0;
888+
if (!mempool.StageTrimToSize(GetArg("-maxmempool", DEFAULT_MAX_MEMPOOL_SIZE) * 1000000, entry, stagedelete, nFeesDeleted)) {
889+
return state.DoS(0, false, REJECT_INSUFFICIENTFEE, "mempool full");
890+
}
891+
885892
// Don't accept it if it can't get into a block
886893
CAmount txMinFee = GetMinRelayFee(tx, nSize, true);
887-
if (fLimitFree && nFees < txMinFee)
894+
if (fLimitFree && nFees < txMinFee + nFeesDeleted)
888895
return state.DoS(0, error("AcceptToMemoryPool: not enough fees %s, %d < %d",
889-
hash.ToString(), nFees, txMinFee),
896+
hash.ToString(), nFees, txMinFee + nFeesDeleted),
890897
REJECT_INSUFFICIENTFEE, "insufficient fee");
891898

892899
// Require that free transactions have sufficient priority to be mined in the next block.
893-
if (GetBoolArg("-relaypriority", true) && nFees < ::minRelayTxFee.GetFee(nSize) && !AllowFree(view.GetPriority(tx, chainActive.Height() + 1))) {
900+
if (GetBoolArg("-relaypriority", true) && nFees - nFeesDeleted < ::minRelayTxFee.GetFee(nSize) && !AllowFree(view.GetPriority(tx, chainActive.Height() + 1))) {
894901
return state.DoS(0, false, REJECT_INSUFFICIENTFEE, "insufficient priority");
895902
}
896903

897904
// Continuously rate-limit free (really, very-low-fee) transactions
898905
// This mitigates 'penny-flooding' -- sending thousands of free transactions just to
899906
// be annoying or make others' transactions take longer to confirm.
900-
if (fLimitFree && nFees < ::minRelayTxFee.GetFee(nSize))
907+
if (fLimitFree && nFees - nFeesDeleted < ::minRelayTxFee.GetFee(nSize))
901908
{
902909
static CCriticalSection csFreeLimiter;
903910
static double dFreeCount;
@@ -944,6 +951,12 @@ bool AcceptToMemoryPool(CTxMemPool& pool, CValidationState &state, const CTransa
944951
return error("AcceptToMemoryPool: BUG! PLEASE REPORT THIS! ConnectInputs failed against MANDATORY but not STANDARD flags %s", hash.ToString());
945952
}
946953

954+
// Make actually space
955+
if (!stagedelete.empty()) {
956+
LogPrint("mempool", "Removing %u transactions (%d fees) from the mempool to make space for %s\n", stagedelete.size(), nFeesDeleted, tx.GetHash().ToString());
957+
pool.RemoveStaged(stagedelete);
958+
}
959+
947960
// Store transaction in memory
948961
pool.addUnchecked(hash, entry, !IsInitialBlockDownload());
949962
}

src/main.h

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -50,6 +50,8 @@ struct CNodeStateStats;
5050
static const bool DEFAULT_ALERTS = true;
5151
/** Default for -maxorphantx, maximum number of orphan transactions kept in memory */
5252
static const unsigned int DEFAULT_MAX_ORPHAN_TRANSACTIONS = 100;
53+
/** Default for -maxmempool, maximum megabytes of mempool memory usage */
54+
static const unsigned int DEFAULT_MAX_MEMPOOL_SIZE = 300;
5355
/** The maximum size of a blk?????.dat file (since 0.8) */
5456
static const unsigned int MAX_BLOCKFILE_SIZE = 0x8000000; // 128 MiB
5557
/** The pre-allocation chunk size for blk?????.dat files (since 0.8) */

src/memusage.h

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -104,6 +104,12 @@ static inline size_t DynamicUsage(const std::set<X>& s)
104104
return MallocUsage(sizeof(stl_tree_node<X>)) * s.size();
105105
}
106106

107+
template<typename X>
108+
static inline size_t IncrementalDynamicUsage(const std::set<X>& s)
109+
{
110+
return MallocUsage(sizeof(stl_tree_node<X>));
111+
}
112+
107113
template<typename X>
108114
static inline size_t RecursiveDynamicUsage(const std::set<X>& v)
109115
{
@@ -120,6 +126,12 @@ static inline size_t DynamicUsage(const std::map<X, Y>& m)
120126
return MallocUsage(sizeof(stl_tree_node<std::pair<const X, Y> >)) * m.size();
121127
}
122128

129+
template<typename X, typename Y>
130+
static inline size_t IncrementalDynamicUsage(const std::map<X, Y>& m)
131+
{
132+
return MallocUsage(sizeof(stl_tree_node<std::pair<const X, Y> >));
133+
}
134+
123135
template<typename X, typename Y>
124136
static inline size_t RecursiveDynamicUsage(const std::map<X, Y>& v)
125137
{

src/txmempool.cpp

Lines changed: 118 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,6 @@ CTxMemPoolEntry::CTxMemPoolEntry(const CTransaction& _tx, const CAmount& _nFee,
3232
nTxSize = ::GetSerializeSize(tx, SER_NETWORK, PROTOCOL_VERSION);
3333
nModSize = tx.CalculateModifiedSize(nTxSize);
3434
nUsageSize = tx.DynamicMemoryUsage();
35-
feeRate = CFeeRate(nFee, nTxSize);
3635
}
3736

3837
CTxMemPoolEntry::CTxMemPoolEntry(const CTxMemPoolEntry& other)
@@ -49,9 +48,10 @@ CTxMemPoolEntry::GetPriority(unsigned int currentHeight) const
4948
return dResult;
5049
}
5150

52-
CTxMemPool::CTxMemPool(const CFeeRate& _minRelayFee) :
53-
nTransactionsUpdated(0)
51+
CTxMemPool::CTxMemPool(const CFeeRate& _minRelayFee)
5452
{
53+
clear();
54+
5555
// Sanity checks off by default for performance, because otherwise
5656
// accepting transactions becomes O(N^2) where N is the number
5757
// of transactions in the pool
@@ -109,6 +109,19 @@ bool CTxMemPool::addUnchecked(const uint256& hash, const CTxMemPoolEntry &entry,
109109
return true;
110110
}
111111

112+
void CTxMemPool::removeUnchecked(const uint256& hash)
113+
{
114+
indexed_transaction_set::iterator it = mapTx.find(hash);
115+
116+
BOOST_FOREACH(const CTxIn& txin, it->GetTx().vin)
117+
mapNextTx.erase(txin.prevout);
118+
119+
totalTxSize -= it->GetTxSize();
120+
cachedInnerUsage -= it->DynamicMemoryUsage();
121+
mapTx.erase(it);
122+
nTransactionsUpdated++;
123+
minerPolicyEstimator->removeTx(hash);
124+
}
112125

113126
void CTxMemPool::remove(const CTransaction &origTx, std::list<CTransaction>& removed, bool fRecursive)
114127
{
@@ -144,15 +157,8 @@ void CTxMemPool::remove(const CTransaction &origTx, std::list<CTransaction>& rem
144157
txToRemove.push_back(it->second.ptx->GetHash());
145158
}
146159
}
147-
BOOST_FOREACH(const CTxIn& txin, tx.vin)
148-
mapNextTx.erase(txin.prevout);
149-
150160
removed.push_back(tx);
151-
totalTxSize -= mapTx.find(hash)->GetTxSize();
152-
cachedInnerUsage -= mapTx.find(hash)->DynamicMemoryUsage();
153-
mapTx.erase(hash);
154-
nTransactionsUpdated++;
155-
minerPolicyEstimator->removeTx(hash);
161+
removeUnchecked(hash);
156162
}
157163
}
158164
}
@@ -435,3 +441,104 @@ size_t CTxMemPool::DynamicMemoryUsage() const {
435441
// Estimate the overhead of mapTx to be 6 pointers + an allocation, as no exact formula for boost::multi_index_contained is implemented.
436442
return memusage::MallocUsage(sizeof(CTxMemPoolEntry) + 6 * sizeof(void*)) * mapTx.size() + memusage::DynamicUsage(mapNextTx) + memusage::DynamicUsage(mapDeltas) + cachedInnerUsage;
437443
}
444+
445+
size_t CTxMemPool::GuessDynamicMemoryUsage(const CTxMemPoolEntry& entry) const {
446+
return memusage::MallocUsage(sizeof(CTxMemPoolEntry) + 5 * sizeof(void*)) + entry.DynamicMemoryUsage() + memusage::IncrementalDynamicUsage(mapNextTx) * entry.GetTx().vin.size();
447+
}
448+
449+
bool CTxMemPool::StageTrimToSize(size_t sizelimit, const CTxMemPoolEntry& toadd, std::set<uint256>& stage, CAmount& nFeesRemoved) {
450+
size_t nSizeRemoved = 0;
451+
std::set<uint256> protect;
452+
BOOST_FOREACH(const CTxIn& in, toadd.GetTx().vin) {
453+
protect.insert(in.prevout.hash);
454+
}
455+
456+
size_t expsize = DynamicMemoryUsage() + GuessDynamicMemoryUsage(toadd); // Track the expected resulting memory usage of the mempool.
457+
indexed_transaction_set::nth_index<1>::type::reverse_iterator it = mapTx.get<1>().rbegin();
458+
int fails = 0; // Number of mempool transactions iterated over that were not included in the stage.
459+
// Iterate from lowest feerate to highest feerate in the mempool:
460+
while (expsize > sizelimit && it != mapTx.get<1>().rend()) {
461+
const uint256& hash = it->GetTx().GetHash();
462+
if (stage.count(hash)) {
463+
// If the transaction is already staged for deletion, we know its descendants are already processed, so skip it.
464+
it++;
465+
continue;
466+
}
467+
if (GetRand(10)) {
468+
// Only try 1/10 of the transactions, in order to have some chance to avoid very big chains.
469+
it++;
470+
continue;
471+
}
472+
if (CompareTxMemPoolEntryByFeeRate()(*it, toadd)) {
473+
// If the transaction's feerate is worse than what we're looking for, we have processed everything in the mempool
474+
// that could improve the staged set. If we don't have an acceptable solution by now, bail out.
475+
return false;
476+
}
477+
std::deque<uint256> todo; // List of hashes that we still need to process (descendants of 'hash').
478+
std::set<uint256> now; // Set of hashes that will need to be added to stage if 'hash' is included.
479+
CAmount nowfee = 0; // Sum of the fees in 'now'.
480+
size_t nowsize = 0; // Sum of the tx sizes in 'now'.
481+
size_t nowusage = 0; // Sum of the memory usages of transactions in 'now'.
482+
int iternow = 0; // Transactions we've inspected so far while determining whether 'hash' is acceptable.
483+
todo.push_back(it->GetTx().GetHash()); // Add 'hash' to the todo list, to initiate processing its children.
484+
bool good = true; // Whether including 'hash' (and all its descendants) is a good idea.
485+
// Iterate breadth-first over all descendants of transaction with hash 'hash'.
486+
while (!todo.empty()) {
487+
uint256 hashnow = todo.front();
488+
if (protect.count(hashnow)) {
489+
// If this transaction is in the protected set, we're done with 'hash'.
490+
good = false;
491+
break;
492+
}
493+
iternow++; // We only count transactions we actually had to go find in the mempool.
494+
if (iternow + fails > 20) {
495+
return false;
496+
}
497+
const CTxMemPoolEntry* origTx = &*mapTx.find(hashnow);
498+
nowfee += origTx->GetFee();
499+
if (nFeesRemoved + nowfee > toadd.GetFee()) {
500+
// If this pushes up to the total fees deleted too high, we're done with 'hash'.
501+
good = false;
502+
break;
503+
}
504+
todo.pop_front();
505+
// Add 'hashnow' to the 'now' set, and update its statistics.
506+
now.insert(hashnow);
507+
nowusage += GuessDynamicMemoryUsage(*origTx);
508+
nowsize += origTx->GetTxSize();
509+
// Find dependencies of 'hashnow' and them to todo.
510+
std::map<COutPoint, CInPoint>::iterator iter = mapNextTx.lower_bound(COutPoint(hashnow, 0));
511+
while (iter != mapNextTx.end() && iter->first.hash == hashnow) {
512+
const uint256& nexthash = iter->second.ptx->GetHash();
513+
if (!(stage.count(nexthash) || now.count(nexthash))) {
514+
todo.push_back(nexthash);
515+
}
516+
iter++;
517+
}
518+
}
519+
if (good && (double)nowfee * toadd.GetTxSize() > (double)toadd.GetFee() * nowsize) {
520+
// The new transaction's feerate is below that of the set we're removing.
521+
good = false;
522+
}
523+
if (good) {
524+
stage.insert(now.begin(), now.end());
525+
nFeesRemoved += nowfee;
526+
nSizeRemoved += nowsize;
527+
expsize -= nowusage;
528+
} else {
529+
fails += iternow;
530+
if (fails > 10) {
531+
// Bail out after traversing 32 transactions that are not acceptable.
532+
return false;
533+
}
534+
}
535+
it++;
536+
}
537+
return true;
538+
}
539+
540+
void CTxMemPool::RemoveStaged(std::set<uint256>& stage) {
541+
BOOST_FOREACH(const uint256& hash, stage) {
542+
removeUnchecked(hash);
543+
}
544+
}

src/txmempool.h

Lines changed: 21 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -45,7 +45,6 @@ class CTxMemPoolEntry
4545
size_t nTxSize; //! ... and avoid recomputing tx size
4646
size_t nModSize; //! ... and modified size for priority
4747
size_t nUsageSize; //! ... and total memory usage
48-
CFeeRate feeRate; //! ... and fee per kB
4948
int64_t nTime; //! Local time when entering the mempool
5049
double dPriority; //! Priority when entering the mempool
5150
unsigned int nHeight; //! Chain height when entering the mempool
@@ -59,8 +58,7 @@ class CTxMemPoolEntry
5958

6059
const CTransaction& GetTx() const { return this->tx; }
6160
double GetPriority(unsigned int currentHeight) const;
62-
CAmount GetFee() const { return nFee; }
63-
CFeeRate GetFeeRate() const { return feeRate; }
61+
const CAmount& GetFee() const { return nFee; }
6462
size_t GetTxSize() const { return nTxSize; }
6563
int64_t GetTime() const { return nTime; }
6664
unsigned int GetHeight() const { return nHeight; }
@@ -78,14 +76,18 @@ struct mempoolentry_txid
7876
}
7977
};
8078

81-
class CompareTxMemPoolEntryByFee
79+
class CompareTxMemPoolEntryByFeeRate
8280
{
8381
public:
8482
bool operator()(const CTxMemPoolEntry& a, const CTxMemPoolEntry& b)
8583
{
86-
if (a.GetFeeRate() == b.GetFeeRate())
84+
// Avoid a division by rewriting (a/b > c/d) as (a*d > c*b).
85+
double f1 = (double)a.GetFee() * b.GetTxSize();
86+
double f2 = (double)b.GetFee() * a.GetTxSize();
87+
if (f1 == f2) {
8788
return a.GetTime() < b.GetTime();
88-
return a.GetFeeRate() > b.GetFeeRate();
89+
}
90+
return f1 > f2;
8991
}
9092
};
9193

@@ -134,7 +136,7 @@ class CTxMemPool
134136
// sorted by fee rate
135137
boost::multi_index::ordered_non_unique<
136138
boost::multi_index::identity<CTxMemPoolEntry>,
137-
CompareTxMemPoolEntryByFee
139+
CompareTxMemPoolEntryByFeeRate
138140
>
139141
>
140142
> indexed_transaction_set;
@@ -157,6 +159,7 @@ class CTxMemPool
157159
void setSanityCheck(bool _fSanityCheck) { fSanityCheck = _fSanityCheck; }
158160

159161
bool addUnchecked(const uint256& hash, const CTxMemPoolEntry &entry, bool fCurrentEstimate = true);
162+
void removeUnchecked(const uint256& hash);
160163
void remove(const CTransaction &tx, std::list<CTransaction>& removed, bool fRecursive = false);
161164
void removeCoinbaseSpends(const CCoinsViewCache *pcoins, unsigned int nMemPoolHeight);
162165
void removeConflicts(const CTransaction &tx, std::list<CTransaction>& removed);
@@ -178,6 +181,16 @@ class CTxMemPool
178181
void ApplyDeltas(const uint256 hash, double &dPriorityDelta, CAmount &nFeeDelta);
179182
void ClearPrioritisation(const uint256 hash);
180183

184+
/** Build a list of transaction (hashes) to remove such that:
185+
* - The list is consistent (if a parent is included, all its dependencies are included as well).
186+
* - No dependencies of toadd are removed.
187+
* - The total fees removed are not more than the fees added by toadd.
188+
* - The feerate of what is removed is not better than the feerate of toadd.
189+
* - Removing said list will reduce the DynamicMemoryUsage after adding toadd, below sizelimit.
190+
*/
191+
bool StageTrimToSize(size_t sizelimit, const CTxMemPoolEntry& toadd, std::set<uint256>& stage, CAmount& nFeeRemoved);
192+
void RemoveStaged(std::set<uint256>& stage);
193+
181194
unsigned long size()
182195
{
183196
LOCK(cs);
@@ -209,6 +222,7 @@ class CTxMemPool
209222
bool ReadFeeEstimates(CAutoFile& filein);
210223

211224
size_t DynamicMemoryUsage() const;
225+
size_t GuessDynamicMemoryUsage(const CTxMemPoolEntry& entry) const;
212226
};
213227

214228
/**

0 commit comments

Comments
 (0)