diff --git a/doc/release-notes-7882.md b/doc/release-notes-7882.md new file mode 100644 index 000000000000..f8f05449eb8c --- /dev/null +++ b/doc/release-notes-7882.md @@ -0,0 +1,14 @@ +Pruning +------- + +- Pruned nodes now keep the blocks that block validation reads back from disk + for the credit pool and for quorum members. From V20 on that is at least the + last 2074 blocks on mainnet, testnet and devnet, instead of 288. Before, a + node pruned below that could reject a valid block and stop following the + chain. At current mainnet block sizes this is far below the minimum `-prune` + target, so it mostly affects manual pruning: `pruneblockchain` now returns a + lower height when the requested height is within that range. A pruned node + now refuses to start, and asks to restart with `-reindex`, if blocks that it + needs to validate the next blocks were already pruned, for example by an + earlier release. A node that has already rejected a valid block this way also + needs `-reindex`. (#7882) diff --git a/src/Makefile.test.include b/src/Makefile.test.include index 90a0ed75ed64..c2a651177a46 100644 --- a/src/Makefile.test.include +++ b/src/Makefile.test.include @@ -118,6 +118,7 @@ BITCOIN_TESTS =\ test/evo_mnauth_tests.cpp \ test/evo_mnhf_tests.cpp \ test/evo_netinfo_tests.cpp \ + test/evo_prune_tests.cpp \ test/evo_sharedmn_tests.cpp \ test/evo_simplifiedmns_tests.cpp \ test/evo_snapshot_tests.cpp \ diff --git a/src/evo/creditpool.h b/src/evo/creditpool.h index e7676beace54..863a4f0f1033 100644 --- a/src/evo/creditpool.h +++ b/src/evo/creditpool.h @@ -123,9 +123,9 @@ class CCreditPoolManager CEvoDB& evoDb; const ChainstateManager& m_chainman; +public: static constexpr int DISK_SNAPSHOT_PERIOD = 576; // once per day -public: static constexpr CAmount LimitAmountLow = 100 * COIN; static constexpr CAmount LimitAmountHigh = 1000 * COIN; static constexpr CAmount LimitAmountV22 = 2000 * COIN; diff --git a/src/evo/specialtxman.cpp b/src/evo/specialtxman.cpp index aef331c99d8d..32d9ab553158 100644 --- a/src/evo/specialtxman.cpp +++ b/src/evo/specialtxman.cpp @@ -18,6 +18,7 @@ #include #include #include +#include #include #include @@ -50,6 +51,26 @@ CSpecialTxProcessor::CSpecialTxProcessor(CEvoDB& evodb, CDeterministicMNManager& CSpecialTxProcessor::~CSpecialTxProcessor() = default; +int GetEvoPruneLockHeight(const CChainParams& chainparams, int tip_height) +{ + // On a cold cache the credit pool is rebuilt from the last disk snapshot, up to DISK_SNAPSHOT_PERIOD blocks back + // (the snapshot of a block is only written once the next block connects), reading every block on the way and the + // block CreditPoolPeriodBlocks before each of them + int depth{CCreditPoolManager::DISK_SNAPSHOT_PERIOD + chainparams.CreditPoolPeriodBlocks()}; + for (const auto& llmq_params : chainparams.GetConsensus().llmqs) { + // Quorum members depend on the coinbase ChainLock in the work block of the quorum. Rotated quorum members + // also depend on the work blocks of the three previous cycles, and are stored as a snapshot that the next + // three cycles build on. Members and their snapshot are built and stored again whenever they are not cached, + // so keep the work blocks of a cycle and of the three cycles before it until the last commitment of the third + // cycle after it can be mined. + const int cycles{llmq_params.useRotation ? 6 : 0}; + const int llmq_depth{cycles * llmq_params.dkgInterval + llmq_params.dkgMiningWindowEnd + llmq::WORK_DIFF_DEPTH}; + depth = std::max(depth, llmq_depth); + } + // Nothing before the credit pool window of the first V20 block is read + return std::max(tip_height - depth, chainparams.GetConsensus().V20Height - chainparams.CreditPoolPeriodBlocks()); +} + static bool AddNetInfoEntries(const std::shared_ptr& net_info, NetInfoPurpose purpose, const NetInfoList& entries, BlockValidationState& state) { diff --git a/src/evo/specialtxman.h b/src/evo/specialtxman.h index 1481ad01683a..0e0e1c8279e3 100644 --- a/src/evo/specialtxman.h +++ b/src/evo/specialtxman.h @@ -19,6 +19,7 @@ class CBlock; class CBlockIndex; class CCbTx; class CChain; +class CChainParams; class CCoinsViewCache; class CCreditPoolManager; class CDeterministicMNList; @@ -115,6 +116,12 @@ class CSpecialTxProcessor CAmount blockSubsidy, BlockValidationState& state) EXCLUSIVE_LOCKS_REQUIRED(::cs_main); }; +/** + * Lowest height whose block data the credit pool and quorum member code may read back from disk while + * validating the blocks after tip_height. Pruning has to keep these blocks, see Chainstate::FlushStateToDisk(). + */ +[[nodiscard]] int GetEvoPruneLockHeight(const CChainParams& chainparams, int tip_height); + /** * This helper does some trivial validations that doesn't depends on collateral and * masternode list. Use CheckPro*Tx for the full validation of transaction, diff --git a/src/init.cpp b/src/init.cpp index 8deac741233e..11cedf91de1a 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -597,6 +597,7 @@ void SetupServerArgs(ArgsManager& argsman) #endif argsman.AddArg("-assumevalid=", strprintf("If this block is in the chain assume that it and its ancestors are valid and potentially skip their script verification (0 to verify all, default: %s, testnet: %s, devnet: %s)", defaultChainParams->GetConsensus().defaultAssumeValid.GetHex(), testnetChainParams->GetConsensus().defaultAssumeValid.GetHex(), devnetChainParams->GetConsensus().defaultAssumeValid.GetHex()), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); argsman.AddArg("-blocksdir=", "Specify directory to hold blocks subdirectory for *.dat files (default: )", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); + argsman.AddArg("-evoprunelock", strprintf("Keep the blocks that evo validation reads back from disk when pruning, only disable it for testing (default: %u)", DEFAULT_EVO_PRUNE_LOCK), ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::DEBUG_TEST); argsman.AddArg("-fastprune", "Use smaller block files and lower minimum prune height for testing purposes", ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::DEBUG_TEST); argsman.AddArg("-tinyblk", "Use smaller block files for testing purposes", ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::DEBUG_TEST); #if HAVE_SYSTEM @@ -2134,7 +2135,7 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info) if (!options.reindex) { bool fRet = uiInterface.ThreadSafeQuestion( error + Untranslated(".\n\n") + _("Do you want to rebuild the block database now?"), - error.original + ".\nPlease restart with -reindex or -reindex-chainstate to recover.", + error.original + (args.GetIntArg("-prune", 0) ? ".\nPlease restart with -reindex to recover." : ".\nPlease restart with -reindex or -reindex-chainstate to recover."), "", CClientUIInterface::MSG_ERROR | CClientUIInterface::BTN_ABORT); if (fRet) { fReindex = true; diff --git a/src/kernel/chainstatemanager_opts.h b/src/kernel/chainstatemanager_opts.h index 1ae3637b3700..6d822020e706 100644 --- a/src/kernel/chainstatemanager_opts.h +++ b/src/kernel/chainstatemanager_opts.h @@ -20,6 +20,7 @@ struct SuperblockStatus; static constexpr bool DEFAULT_CHECKPOINTS_ENABLED{true}; static constexpr auto DEFAULT_MAX_TIP_AGE{6h}; // ~144 blocks behind -> 2 x fork detection time, was 24h in bitcoin +static constexpr bool DEFAULT_EVO_PRUNE_LOCK{true}; namespace kernel { @@ -38,6 +39,9 @@ struct ChainstateManagerOpts { std::optional assumed_valid_block{}; //! If the tip is older than this, the node is considered to be in initial block download. std::chrono::seconds max_tip_age{DEFAULT_MAX_TIP_AGE}; + //! Dash: keep the blocks evo validation reads back from disk when pruning, see GetEvoPruneLockHeight(). Only + //! disabled in tests, to get a datadir pruned without them. + bool evo_prune_lock{DEFAULT_EVO_PRUNE_LOCK}; //! Dash: governance's superblock status for a block height, consulted by //! block validation and the miner. Unset means superblock validation is disabled. std::function superblock_status{}; diff --git a/src/node/blockstorage.h b/src/node/blockstorage.h index 02d259d3a868..b862943bd6a1 100644 --- a/src/node/blockstorage.h +++ b/src/node/blockstorage.h @@ -149,7 +149,8 @@ class BlockManager std::set m_dirty_fileinfo; /** - * Map from external index name to oldest block that must not be pruned. + * Map from external index name (or "evo", see Chainstate::FlushStateToDisk()) to oldest block that must not be + * pruned. * * @note Internally, only blocks at height (height_first - PRUNE_LOCK_BUFFER - 1) and * below will be pruned, but callers should avoid assuming any particular buffer size. diff --git a/src/node/chainstate.cpp b/src/node/chainstate.cpp index d270725d05dd..fc590f68ff3f 100644 --- a/src/node/chainstate.cpp +++ b/src/node/chainstate.cpp @@ -262,6 +262,19 @@ static ChainstateLoadResult CompleteChainstateInitialization(ChainstateManager& } } + // Blocks pruned without the "evo" prune lock, e.g. by an earlier release, may be missing blocks that validating the + // next blocks reads. A snapshot chainstate has no blocks below its base, -reindex can't help it. + if (chainman.m_blockman.m_have_pruned && !chainman.IsSnapshotActive() && + !is_coinsview_empty(&chainman.ActiveChainstate())) { + const CBlockIndex* missing{FindMissingEvoBlockData(chainman.GetParams(), chainman.ActiveTip())}; + if (missing) { + return {ChainstateLoadStatus::FAILURE, + strprintf(_("Block %d, which is needed to validate new blocks, has been pruned. You need to " + "rebuild the database using -reindex. This will redownload the entire blockchain"), + missing->nHeight)}; + } + } + if (!special_tx.m_mnhfman->ForceSignalDBUpdate(chainman.ActiveTip())) { return {ChainstateLoadStatus::FAILURE, _("Error upgrading evo database for EHF")}; } diff --git a/src/node/chainstatemanager_args.cpp b/src/node/chainstatemanager_args.cpp index b0d929626bde..eab6cab2ab15 100644 --- a/src/node/chainstatemanager_args.cpp +++ b/src/node/chainstatemanager_args.cpp @@ -34,6 +34,8 @@ std::optional ApplyArgsManOptions(const ArgsManager& args, Chains if (auto value{args.GetIntArg("-maxtipage")}) opts.max_tip_age = std::chrono::seconds{*value}; + if (auto value{args.GetBoolArg("-evoprunelock")}) opts.evo_prune_lock = *value; + return std::nullopt; } } // namespace node diff --git a/src/test/evo_prune_tests.cpp b/src/test/evo_prune_tests.cpp new file mode 100644 index 000000000000..6ef177727ac3 --- /dev/null +++ b/src/test/evo_prune_tests.cpp @@ -0,0 +1,162 @@ +// Copyright (c) 2026 The Dash Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include + +BOOST_FIXTURE_TEST_SUITE(evo_prune_tests, BasicTestingSetup) + +BOOST_AUTO_TEST_CASE(evo_prune_lock_depth) +{ + // Mainnet, testnet and devnet: rebuilding LLMQ_60_75 members (dkgInterval 288, dkgMiningWindowEnd 50) dominates. + // Regtest: rebuilding the credit pool (snapshot period 576, CreditPoolPeriodBlocks 100) dominates. + const std::pair expected_depths[]{ + {CBaseChainParams::MAIN, 6 * 288 + 50 + 8}, + {CBaseChainParams::TESTNET, 6 * 288 + 50 + 8}, + {CBaseChainParams::DEVNET, 6 * 288 + 50 + 8}, + {CBaseChainParams::REGTEST, 576 + 100}, + }; + for (const auto& [chain, depth] : expected_depths) { + const auto params{CreateChainParams(ArgsManager{}, chain)}; + const int tip{params->GetConsensus().V20Height + 10'000}; + BOOST_CHECK_EQUAL(GetEvoPruneLockHeight(*params, tip), tip - depth); + } +} + +BOOST_AUTO_TEST_CASE(evo_prune_lock_keeps_rotated_quorum_work_blocks) +{ + for (const auto& chain : + {CBaseChainParams::MAIN, CBaseChainParams::TESTNET, CBaseChainParams::DEVNET, CBaseChainParams::REGTEST}) { + const auto params{CreateChainParams(ArgsManager{}, chain)}; + const auto& consensus{params->GetConsensus()}; + for (const auto& llmq_params : consensus.llmqs) { + if (!llmq_params.useRotation) continue; + const int interval{llmq_params.dkgInterval}; + const int cycle_base{(consensus.V20Height / interval + 10) * interval}; + // The last block that can carry a commitment of the cycle connects on top of this tip + const int tip{cycle_base + llmq_params.dkgMiningWindowEnd - 1}; + // Its members are built from the work blocks of the cycle and of the three previous cycles, and the + // members of the oldest of these cycles were built from the work blocks of the three cycles before it + const int oldest_work_block{cycle_base - 6 * interval - llmq::WORK_DIFF_DEPTH}; + BOOST_CHECK_LE(GetEvoPruneLockHeight(*params, tip), oldest_work_block); + } + } +} + +BOOST_AUTO_TEST_CASE(evo_prune_lock_keeps_credit_pool_blocks) +{ + for (const auto& chain : + {CBaseChainParams::MAIN, CBaseChainParams::TESTNET, CBaseChainParams::DEVNET, CBaseChainParams::REGTEST}) { + const auto params{CreateChainParams(ArgsManager{}, chain)}; + const int period{CCreditPoolManager::DISK_SNAPSHOT_PERIOD}; + // A rebuild on a cold cache starts furthest back when the tip is at a disk snapshot height, as the snapshot of + // the tip is only written once the next block connects + const int tip{(params->GetConsensus().V20Height / period + 10) * period}; + // The first block after the previous snapshot reads the block CreditPoolPeriodBlocks before it + const int oldest_read{tip - period + 1 - params->CreditPoolPeriodBlocks()}; + BOOST_CHECK_LE(GetEvoPruneLockHeight(*params, tip), oldest_read); + } +} + +struct RegTestV20At2000Setup : public BasicTestingSetup { + RegTestV20At2000Setup() : + BasicTestingSetup{CBaseChainParams::REGTEST, {"-testactivationheight=v20@2000", "-testactivationheight=mn_rr@2000"}} + { + } +}; + +BOOST_FIXTURE_TEST_CASE(evo_prune_lock_before_v20, RegTestV20At2000Setup) +{ + const CChainParams& params{Params()}; + BOOST_REQUIRE_EQUAL(params.GetConsensus().V20Height, 2000); + // Nothing before the credit pool window of the first V20 block is read + BOOST_CHECK_EQUAL(GetEvoPruneLockHeight(params, 1000), 2000 - 100); + BOOST_CHECK_EQUAL(GetEvoPruneLockHeight(params, 2000 + 576), 2000 - 100); + BOOST_CHECK_EQUAL(GetEvoPruneLockHeight(params, 2000 + 576 + 1), 2000 - 100 + 1); + BOOST_CHECK_EQUAL(GetEvoPruneLockHeight(params, 5000), 5000 - (576 + 100)); +} + +// Link blocks into a chain of block indexes from first_height on, all with data +static void MakeChain(std::vector& blocks, int first_height) EXCLUSIVE_LOCKS_REQUIRED(::cs_main) +{ + for (size_t i{0}; i < blocks.size(); ++i) { + blocks[i].nHeight = first_height + static_cast(i); + blocks[i].pprev = i > 0 ? &blocks[i - 1] : nullptr; + blocks[i].nStatus = BLOCK_HAVE_DATA; + } +} + +// The height of the block FindMissingEvoBlockData() reports for the last block when only the block at missing_height +// has no data, or -1 for none +static int FindMissingWith(const CChainParams& params, std::vector& blocks, int missing_height) + EXCLUSIVE_LOCKS_REQUIRED(::cs_main) +{ + CBlockIndex& missing{blocks.at(missing_height - blocks.front().nHeight)}; + missing.nStatus &= ~BLOCK_HAVE_DATA; + const CBlockIndex* found{FindMissingEvoBlockData(params, &blocks.back())}; + missing.nStatus |= BLOCK_HAVE_DATA; + return found ? found->nHeight : -1; +} + +BOOST_AUTO_TEST_CASE(evo_prune_find_missing_block_data) +{ + LOCK(cs_main); + const auto params{CreateChainParams(ArgsManager{}, CBaseChainParams::MAIN)}; + const auto llmq_params{params->GetLLMQ(Consensus::LLMQType::LLMQ_60_75)}; + BOOST_REQUIRE(llmq_params && llmq_params->useRotation); + const int interval{llmq_params->dkgInterval}; + const int period{CCreditPoolManager::DISK_SNAPSHOT_PERIOD}; + // A cycle of rotated quorums that starts at a credit pool snapshot height, with a commitment mined on top of tip + const int cycle_base{(params->GetConsensus().V20Height / period + 10) * period}; + BOOST_REQUIRE_EQUAL(cycle_base % interval, 0); + const int tip{cycle_base + 40}; + // Its members are built from the work block of the third cycle before it, which is older than any block that the + // credit pool reads for the next block (from the snapshot of cycle_base on) + const int work_block{cycle_base - 3 * interval - llmq::WORK_DIFF_DEPTH}; + BOOST_REQUIRE_LT(work_block, cycle_base + 1 - params->CreditPoolPeriodBlocks()); + + const int height_first{GetEvoPruneLockHeight(*params, tip)}; + BOOST_CHECK_EQUAL(height_first, tip - (6 * 288 + 50 + 8)); + // The lock keeps more blocks below them for a reorg, which a rewound tip may have used up, so these aren't checked + const int lock_first{GetEvoPruneLockHeightForTip(*params, tip)}; + BOOST_CHECK_EQUAL(lock_first, height_first - static_cast(MIN_BLOCKS_TO_KEEP)); + std::vector blocks(tip - (lock_first - 10) + 1); + MakeChain(blocks, lock_first - 10); + BOOST_CHECK(FindMissingEvoBlockData(*params, &blocks.back()) == nullptr); + BOOST_CHECK_EQUAL(FindMissingWith(*params, blocks, work_block), work_block); + BOOST_CHECK_EQUAL(FindMissingWith(*params, blocks, tip), tip); + BOOST_CHECK_EQUAL(FindMissingWith(*params, blocks, height_first), height_first); + BOOST_CHECK_EQUAL(FindMissingWith(*params, blocks, height_first - 1), -1); + BOOST_CHECK_EQUAL(FindMissingWith(*params, blocks, lock_first), -1); +} + +BOOST_AUTO_TEST_CASE(evo_prune_find_missing_block_data_before_v20) +{ + LOCK(cs_main); + const auto params{CreateChainParams(ArgsManager{}, CBaseChainParams::REGTEST)}; + // Nothing before the credit pool window of the V20 activation block is kept or checked + const int floor{params->GetConsensus().V20Height - params->CreditPoolPeriodBlocks()}; + std::vector blocks(floor + 11); + MakeChain(blocks, 0); + BOOST_CHECK_EQUAL(FindMissingWith(*params, blocks, floor), floor); + BOOST_CHECK_EQUAL(FindMissingWith(*params, blocks, floor - 1), -1); + // A chain that ends below it isn't checked at all + std::vector short_blocks(floor); + MakeChain(short_blocks, 0); + BOOST_CHECK_EQUAL(FindMissingWith(*params, short_blocks, floor - 1), -1); +} + +BOOST_AUTO_TEST_SUITE_END() diff --git a/src/validation.cpp b/src/validation.cpp index f596fe5fe0b1..5bb63054a295 100644 --- a/src/validation.cpp +++ b/src/validation.cpp @@ -3191,6 +3191,11 @@ bool Chainstate::FlushStateToDisk( CoinsCacheSizeState cache_state = GetCoinsCacheSizeState(); LOCK(m_blockman.cs_LastBlockFile); if (m_blockman.IsPruneMode() && (m_blockman.m_check_for_pruning || nManualPruneHeight > 0) && !fReindex) { + if (m_chainman.m_options.evo_prune_lock) { + m_blockman.UpdatePruneLock("evo", {.height_first = GetEvoPruneLockHeightForTip(m_chainman.GetParams(), + m_chain.Height())}); + } + // make sure we don't prune above any of the prune locks bestblocks // pruning is height-based int last_prune{m_chain.Height()}; // last height we can prune @@ -5186,6 +5191,26 @@ void PruneBlockFilesManual(Chainstate& active_chainstate, int nManualPruneHeight } } +int GetEvoPruneLockHeightForTip(const CChainParams& chainparams, int tip_height) +{ + // evo and LLMQ validation reads blocks from disk further back than MIN_BLOCKS_TO_KEEP, also for the blocks + // connected after a reorg of up to MIN_BLOCKS_TO_KEEP blocks + return GetEvoPruneLockHeight(chainparams, tip_height - static_cast(MIN_BLOCKS_TO_KEEP)); +} + +const CBlockIndex* FindMissingEvoBlockData(const CChainParams& chainparams, const CBlockIndex* tip) +{ + AssertLockHeld(::cs_main); + if (tip == nullptr) return nullptr; + // The blocks that validating the next blocks reads. The lock keeps more below them for a reorg, but a node that + // rewound its tip, e.g. with invalidateblock, may have used those up and can still connect the next blocks. + const int height_first{GetEvoPruneLockHeight(chainparams, tip->nHeight)}; + for (const CBlockIndex* pindex{tip}; pindex != nullptr && pindex->nHeight >= height_first; pindex = pindex->pprev) { + if (!(pindex->nStatus & BLOCK_HAVE_DATA)) return pindex; + } + return nullptr; +} + void Chainstate::LoadMempool(const fs::path& load_path, FopenFn mockable_fopen_function) { if (!m_mempool) return; diff --git a/src/validation.h b/src/validation.h index 5a6e73279474..84b2a6bb2b91 100644 --- a/src/validation.h +++ b/src/validation.h @@ -143,6 +143,16 @@ double GuessVerificationProgress(const ChainTxData& data, const CBlockIndex* pin /** Prune block files up to a given height */ void PruneBlockFilesManual(Chainstate& active_chainstate, int nManualPruneHeight); +/** Height of the oldest block that the "evo" prune lock keeps for a chain tip at tip_height */ +[[nodiscard]] int GetEvoPruneLockHeightForTip(const CChainParams& chainparams, int tip_height); + +/** + * @return the newest block from tip down to GetEvoPruneLockHeight() that has no data on disk (e.g. because it was + * pruned before the "evo" prune lock existed), or nullptr if there is none + */ +const CBlockIndex* FindMissingEvoBlockData(const CChainParams& chainparams, const CBlockIndex* tip) + EXCLUSIVE_LOCKS_REQUIRED(::cs_main); + /** * Validation result for a single transaction mempool acceptance. */ diff --git a/test/functional/feature_pruning_evo.py b/test/functional/feature_pruning_evo.py new file mode 100755 index 000000000000..a1da8e4ef424 --- /dev/null +++ b/test/functional/feature_pruning_evo.py @@ -0,0 +1,133 @@ +#!/usr/bin/env python3 +# Copyright (c) 2026 The Dash Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +"""Test that pruning keeps the blocks evo validation reads back from disk. + +After V20, connecting a block on a cold credit pool cache (e.g. after a restart) +rebuilds the credit pool from the last disk snapshot, taken every 576 blocks. It +reads every block on the way and the block CreditPoolPeriodBlocks before each of +them, which is further back than MIN_BLOCKS_TO_KEEP. A pruned node has to keep +these blocks, also when pruning manually and for the blocks it connects after a +reorg, or it rejects valid blocks. It refuses to start on a datadir pruned +without them. +""" + +import os + +from test_framework.governance import EXPECTED_STDERR_NO_GOV_PRUNE +from test_framework.test_framework import BitcoinTestFramework +from test_framework.util import assert_equal, assert_greater_than, assert_greater_than_or_equal + +V20_HEIGHT = 150 +DEPLOYMENT_ARGS = [ + "-dip3params=2:2", + f"-testactivationheight=v20@{V20_HEIGHT}", + f"-testactivationheight=mn_rr@{V20_HEIGHT}", +] +MIN_BLOCKS_TO_KEEP = 288 +CREDIT_POOL_SNAPSHOT_PERIOD = 576 # CCreditPoolManager::DISK_SNAPSHOT_PERIOD +CREDIT_POOL_PERIOD_BLOCKS = 100 # CChainParams::CreditPoolPeriodBlocks() on regtest +# Validating the next blocks reads this many blocks below the tip, the evo prune lock keeps MIN_BLOCKS_TO_KEEP more +# for a reorg +EVO_BLOCKS_READ = CREDIT_POOL_SNAPSHOT_PERIOD + CREDIT_POOL_PERIOD_BLOCKS +EVO_BLOCKS_TO_KEEP = MIN_BLOCKS_TO_KEEP + EVO_BLOCKS_READ +PRUNE_LOCK_BUFFER = 10 + + +class FeaturePruningEvoTest(BitcoinTestFramework): + def set_test_params(self): + self.num_nodes = 2 + self.setup_clean_chain = True + self.extra_args = [ + DEPLOYMENT_ARGS, + ["-prune=1", "-fastprune", "-debug=prune"] + DEPLOYMENT_ARGS, + ] + + def mine_to(self, height): + while self.nodes[0].getblockcount() < height: + self.generate(self.nodes[0], min(250, height - self.nodes[0].getblockcount())) + + def prune_and_restart(self, tip): + """Mine to tip, prune as far as allowed and restart the pruned node, so that its credit pool cache is cold""" + self.mine_to(tip) + pruned_height = self.nodes[1].pruneblockchain(tip) + assert_greater_than(pruned_height, self.prev_pruned_height) + self.prev_pruned_height = pruned_height + self.restart_node(1, extra_args=self.extra_args[1], expected_stderr=EXPECTED_STDERR_NO_GOV_PRUNE) + return pruned_height + + def check_evo_prune_lock(self, tip): + lock_height = tip - EVO_BLOCKS_TO_KEEP - PRUNE_LOCK_BUFFER - 1 + with self.nodes[1].assert_debug_log([f"evo limited pruning to height {lock_height}"]): + self.nodes[1].pruneblockchain(tip) + + def run_test(self): + miner, pruned = self.nodes + self.prev_pruned_height = 0 + + self.log.info("Reorg a pruned node by MIN_BLOCKS_TO_KEEP blocks after a restart at the fork point") + # The first block connected after the reorg rebuilds the credit pool from the snapshot before the fork point, + # which reaches furthest back when the fork point is one block before a snapshot height + fork_height = 2 * CREDIT_POOL_SNAPSHOT_PERIOD - 1 + tip = fork_height + MIN_BLOCKS_TO_KEEP + pruned_height = self.prune_and_restart(tip) + fork_child = pruned.getblockhash(fork_height + 1) + pruned.invalidateblock(fork_child) + assert_equal(pruned.getblockcount(), fork_height) + # Restart at the fork point, which reconsiders the invalid blocks ahead of the tip + fork_hash = pruned.getblockhash(fork_height) + with pruned.assert_debug_log([f"Loaded best chain: hashBestChain={fork_hash} height={fork_height}", + "is ahead of our tip, reconsidering"]): + self.restart_node(1, extra_args=self.extra_args[1], expected_stderr=EXPECTED_STDERR_NO_GOV_PRUNE) + pruned.reconsiderblock(fork_child) + assert_equal(pruned.getbestblockhash(), miner.getbestblockhash()) + last_snapshot = fork_height - fork_height % CREDIT_POOL_SNAPSHOT_PERIOD + assert_greater_than(last_snapshot + 1 - CREDIT_POOL_PERIOD_BLOCKS, pruned_height) + # It started at the fork point although the blocks that the lock would keep there for a further reorg, below + # the ones that validating the next blocks reads, were pruned + assert_greater_than(pruned_height, fork_height - EVO_BLOCKS_TO_KEEP) + self.check_evo_prune_lock(tip) + self.connect_nodes(0, 1) + + self.log.info("Restart a pruned node at a credit pool snapshot height and connect the next block") + # The snapshot of the tip is only written once the next block connects, so the rebuild starts from the + # snapshot before it + tip = 3 * CREDIT_POOL_SNAPSHOT_PERIOD + pruned_height = self.prune_and_restart(tip) + self.connect_nodes(0, 1) + self.generate(miner, 1) + last_snapshot = tip - CREDIT_POOL_SNAPSHOT_PERIOD + assert_greater_than(last_snapshot + 1 - CREDIT_POOL_PERIOD_BLOCKS, pruned_height) + self.check_evo_prune_lock(tip + 1) + + self.log.info("Refuse to start on a datadir missing blocks that validating the next blocks reads") + # Without the lock, pruneblockchain prunes up to MIN_BLOCKS_TO_KEEP blocks below the tip. Prune blocks that + # validating the next blocks reads, further back than the ones building the credit pool reads. + self.restart_node(1, extra_args=self.extra_args[1] + ["-evoprunelock=0"], expected_stderr=EXPECTED_STDERR_NO_GOV_PRUNE) + last_snapshot = tip + tip = pruned.getblockcount() + pruned_height = pruned.pruneblockchain(tip) + assert_greater_than_or_equal(pruned_height, tip - EVO_BLOCKS_READ) + assert_greater_than(last_snapshot + 1 - CREDIT_POOL_PERIOD_BLOCKS, pruned_height) + self.stop_node(1, expected_stderr=EXPECTED_STDERR_NO_GOV_PRUNE) + # The error names the newest missing block, the last pruned one + pruned.assert_start_raises_init_error( + extra_args=self.extra_args[1], + expected_msg=f"{EXPECTED_STDERR_NO_GOV_PRUNE}{os.linesep}: Block {pruned_height}, which is needed to " + "validate new blocks, has been pruned. You need to rebuild the database using -reindex. This " + f"will redownload the entire blockchain.{os.linesep}" + "Please restart with -reindex to recover.", + ) + + self.log.info("Recover with -reindex") + self.start_node(1, extra_args=self.extra_args[1] + ["-reindex"]) + self.connect_nodes(0, 1) + self.sync_blocks() + assert_equal(pruned.getbestblockhash(), miner.getbestblockhash()) + + self.stop_node(1, expected_stderr=EXPECTED_STDERR_NO_GOV_PRUNE) + + +if __name__ == '__main__': + FeaturePruningEvoTest().main() diff --git a/test/functional/feature_remove_pruned_files_on_startup.py b/test/functional/feature_remove_pruned_files_on_startup.py index 8ce4cd86c0e3..7bd7ed8c1a7a 100755 --- a/test/functional/feature_remove_pruned_files_on_startup.py +++ b/test/functional/feature_remove_pruned_files_on_startup.py @@ -26,7 +26,8 @@ def run_test(self): rev0 = os.path.join(self.nodes[0].datadir, self.nodes[0].chain, 'blocks', 'rev00000.dat') blk1 = os.path.join(self.nodes[0].datadir, self.nodes[0].chain, 'blocks', 'blk00001.dat') rev1 = os.path.join(self.nodes[0].datadir, self.nodes[0].chain, 'blocks', 'rev00001.dat') - self.mine_batches(800) + # Mine past the blocks that pruning keeps for evo validation, see feature_pruning_evo.py + self.mine_batches(1400) fo1 = os.open(blk0, os.O_RDONLY) fo2 = os.open(rev1, os.O_RDONLY) fd1 = os.fdopen(fo1) diff --git a/test/functional/interface_usdt_utxocache.py b/test/functional/interface_usdt_utxocache.py index 338c9e589b43..b0921028699d 100755 --- a/test/functional/interface_usdt_utxocache.py +++ b/test/functional/interface_usdt_utxocache.py @@ -392,9 +392,11 @@ def handle_utxocache_flush(_, data, __): self.log.info("restart the node with -prune") self.start_node(0, ["-fastprune=1", "-prune=1"]) - BLOCKS_TO_MINE = 450 + # Pruning keeps the blocks evo validation reads, see feature_pruning_evo.py + BLOCKS_TO_MINE = 1200 self.log.info(f"mine {BLOCKS_TO_MINE} blocks to be able to prune") - self.generate(self.wallet, BLOCKS_TO_MINE) + for _ in range(4): + self.generate(self.wallet, BLOCKS_TO_MINE // 4) # we added BLOCKS_TO_MINE coinbase UTXOs to the cache possible_cache_sizes = {BLOCKS_TO_MINE} expected_flushes.append( diff --git a/test/functional/rpc_getblockfrompeer.py b/test/functional/rpc_getblockfrompeer.py index e4c42459d055..8e3d2a324bc0 100755 --- a/test/functional/rpc_getblockfrompeer.py +++ b/test/functional/rpc_getblockfrompeer.py @@ -118,7 +118,8 @@ def run_test(self): # (-fastprune) block file. More blocks are needed before the first block # file wraps and can be pruned, and the resulting prune heights are not the # same as upstream, hence they are not compared against fixed values here. - self.generate(self.nodes[0], 900, sync_fun=self.no_op) + # Pruning also keeps the blocks evo validation reads, see feature_pruning_evo.py. + self.generate(self.nodes[0], 1200, sync_fun=self.no_op) self.sync_blocks([self.nodes[0], pruned_node]) pruneheight = pruned_node.pruneblockchain(800) assert_greater_than(pruneheight, 2) diff --git a/test/functional/rpc_rawtransaction.py b/test/functional/rpc_rawtransaction.py index 04aa11023052..6dbab23cb3f1 100755 --- a/test/functional/rpc_rawtransaction.py +++ b/test/functional/rpc_rawtransaction.py @@ -247,8 +247,10 @@ def getrawtransaction_verbosity_tests(self): # check that verbosity 2 for a mempool tx will fallback to verbosity 1 # Do this with a pruned chain, as a regression test for https://github.com/bitcoin/bitcoin/pull/29003 # Pre-DIP3 regtest blocks are small enough that the first 64 KiB block file only ends around height 365, - # so prune above that while staying MIN_BLOCKS_TO_KEEP below the tip - self.generate(self.nodes[2], 600) + # so prune above that while staying below the blocks that pruning keeps for evo validation, see + # feature_pruning_evo.py + for _ in range(4): + self.generate(self.nodes[2], 300) assert_greater_than(self.nodes[2].pruneblockchain(400), 0) mempool_tx = self.wallet.send_self_transfer(from_node=self.nodes[2])['txid'] gottx = self.nodes[2].getrawtransaction(txid=mempool_tx, verbosity=2) diff --git a/test/functional/test_runner.py b/test/functional/test_runner.py index 6ac6b127f41d..cccd8f342eee 100755 --- a/test/functional/test_runner.py +++ b/test/functional/test_runner.py @@ -165,6 +165,7 @@ 'wallet_groups.py --descriptors', 'p2p_blockfilters.py', 'feature_assumevalid.py', + 'feature_pruning_evo.py', 'feature_bip68_sequence.py', 'rpc_packages.py', 'rpc_bind.py --ipv4',