VextaProject/ Vexta
Vexta Vexta-CPU-Miner Vexta-Android-Wallet
Watch
1
0
Fork
You've already forked Vexta
 0
Code Issues Pull requests Projects Packages Wiki Activity Actions

Initial Vexta Core v0.1.0 release

Browse source
This commit is contained in:
Capri 2026-07-15 14:07:28 +00:00
commit 25c454329d
2481 changed files with 689289 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

372
src/index/base.cpp Normal file
View file

@ -0,0 +1,372 @@
// Copyright (c) 2017-2020 The Bitcoin Core developers
// Copyright (c) 2017-2020 The DigiByte Core developers
// Distributed under the MIT software license, see the accompanying
// file COPYING or http://www.opensource.org/licenses/mit-license.php.
#include <chainparams.h>
#include <index/base.h>
#include <node/blockstorage.h>
#include <node/ui_interface.h>
#include <shutdown.h>
#include <tinyformat.h>
#include <util/thread.h>
#include <util/translation.h>
#include <validation.h> // For g_chainman
#include <warnings.h>
constexpr uint8_t DB_BEST_BLOCK{'B'};
constexpr int64_t SYNC_LOG_INTERVAL = 30; // seconds
constexpr int64_t SYNC_LOCATOR_WRITE_INTERVAL = 30; // seconds
template <typename... Args>
static void FatalError(const char* fmt, const Args&... args)
{
std::string strMessage = tfm::format(fmt, args...);
SetMiscWarning(Untranslated(strMessage));
LogPrintf("*** %s\n", strMessage);
AbortError(_("A fatal internal error occurred, see debug.log for details"));
StartShutdown();
}
BaseIndex::DB::DB(const fs::path& path, size_t n_cache_size, bool f_memory, bool f_wipe, bool f_obfuscate) :
CDBWrapper(path, n_cache_size, f_memory, f_wipe, f_obfuscate)
{}
bool BaseIndex::DB::ReadBestBlock(CBlockLocator& locator) const
{
bool success = Read(DB_BEST_BLOCK, locator);
if (!success) {
locator.SetNull();
}
return success;
}
void BaseIndex::DB::WriteBestBlock(CDBBatch& batch, const CBlockLocator& locator)
{
batch.Write(DB_BEST_BLOCK, locator);
}
BaseIndex::~BaseIndex()
{
Interrupt();
Stop();
}
bool BaseIndex::Init()
{
CBlockLocator locator;
if (!GetDB().ReadBestBlock(locator)) {
locator.SetNull();
}
LOCK(cs_main);
CChain& active_chain = m_chainstate->m_chain;
if (locator.IsNull()) {
m_best_block_index = nullptr;
} else {
m_best_block_index = m_chainstate->m_blockman.FindForkInGlobalIndex(active_chain, locator);
}
m_synced = m_best_block_index.load() == active_chain.Tip();
if (!m_synced) {
bool prune_violation = false;
if (!m_best_block_index) {
// index is not built yet
// make sure we have all block data back to the genesis
const CBlockIndex* block = active_chain.Tip();
while (block->pprev && (block->pprev->nStatus & BLOCK_HAVE_DATA)) {
block = block->pprev;
}
prune_violation = block != active_chain.Genesis();
}
// in case the index has a best block set and is not fully synced
// check if we have the required blocks to continue building the index
else {
const CBlockIndex* block_to_test = m_best_block_index.load();
if (!active_chain.Contains(block_to_test)) {
// if the bestblock is not part of the mainchain, find the fork
// and make sure we have all data down to the fork
block_to_test = active_chain.FindFork(block_to_test);
}
const CBlockIndex* block = active_chain.Tip();
prune_violation = true;
// check backwards from the tip if we have all block data until we reach the indexes bestblock
while (block_to_test && block->pprev && (block->pprev->nStatus & BLOCK_HAVE_DATA)) {
if (block_to_test == block) {
prune_violation = false;
break;
}
block = block->pprev;
}
}
if (prune_violation) {
return InitError(strprintf(Untranslated("%s best block of the index goes beyond pruned data. Please disable the index or reindex (which will download the whole blockchain again)"), GetName()));
}
}
return true;
}
static const CBlockIndex* NextSyncBlock(const CBlockIndex* pindex_prev, CChain& chain) EXCLUSIVE_LOCKS_REQUIRED(cs_main)
{
AssertLockHeld(cs_main);
if (!pindex_prev) {
return chain.Genesis();
}
const CBlockIndex* pindex = chain.Next(pindex_prev);
if (pindex) {
return pindex;
}
return chain.Next(chain.FindFork(pindex_prev));
}
void BaseIndex::ThreadSync()
{
const CBlockIndex* pindex = m_best_block_index.load();
if (!m_synced) {
auto& consensus_params = Params().GetConsensus();
int64_t last_log_time = 0;
int64_t last_locator_write_time = 0;
while (true) {
if (m_interrupt) {
m_best_block_index = pindex;
// No need to handle errors in Commit. If it fails, the error will be already be
// logged. The best way to recover is to continue, as index cannot be corrupted by
// a missed commit to disk for an advanced index state.
Commit();
return;
}
{
LOCK(cs_main);
const CBlockIndex* pindex_next = NextSyncBlock(pindex, m_chainstate->m_chain);
if (!pindex_next) {
m_best_block_index = pindex;
m_synced = true;
// No need to handle errors in Commit. See rationale above.
Commit();
break;
}
if (pindex_next->pprev != pindex && !Rewind(pindex, pindex_next->pprev)) {
FatalError("%s: Failed to rewind index %s to a previous chain tip",
__func__, GetName());
return;
}
pindex = pindex_next;
}
int64_t current_time = GetTime();
if (last_log_time + SYNC_LOG_INTERVAL < current_time) {
LogPrintf("Syncing %s with block chain from height %d\n",
GetName(), pindex->nHeight);
last_log_time = current_time;
}
if (last_locator_write_time + SYNC_LOCATOR_WRITE_INTERVAL < current_time) {
m_best_block_index = pindex;
last_locator_write_time = current_time;
// No need to handle errors in Commit. See rationale above.
Commit();
}
CBlock block;
if (!ReadBlockFromDisk(block, pindex, consensus_params)) {
FatalError("%s: Failed to read block %s from disk",
__func__, pindex->GetBlockHash().ToString());
return;
}
if (!WriteBlock(block, pindex)) {
FatalError("%s: Failed to write block %s to index database",
__func__, pindex->GetBlockHash().ToString());
return;
}
}
}
if (pindex) {
LogPrintf("%s is enabled at height %d\n", GetName(), pindex->nHeight);
} else {
LogPrintf("%s is enabled\n", GetName());
}
}
bool BaseIndex::Commit()
{
CDBBatch batch(GetDB());
if (!CommitInternal(batch) || !GetDB().WriteBatch(batch)) {
return error("%s: Failed to commit latest %s state", __func__, GetName());
}
return true;
}
bool BaseIndex::CommitInternal(CDBBatch& batch)
{
LOCK(cs_main);
GetDB().WriteBestBlock(batch, m_chainstate->m_chain.GetLocator(m_best_block_index));
return true;
}
bool BaseIndex::Rewind(const CBlockIndex* current_tip, const CBlockIndex* new_tip)
{
assert(current_tip == m_best_block_index);
assert(current_tip->GetAncestor(new_tip->nHeight) == new_tip);
// In the case of a reorg, ensure persisted block locator is not stale.
// Pruning has a minimum of 288 blocks-to-keep and getting the index
// out of sync may be possible but a users fault.
// In case we reorg beyond the pruned depth, ReadBlockFromDisk would
// throw and lead to a graceful shutdown
m_best_block_index = new_tip;
if (!Commit()) {
// If commit fails, revert the best block index to avoid corruption.
m_best_block_index = current_tip;
return false;
}
return true;
}
void BaseIndex::BlockConnected(const std::shared_ptr<const CBlock>& block, const CBlockIndex* pindex)
{
if (!m_synced) {
return;
}
const CBlockIndex* best_block_index = m_best_block_index.load();
if (!best_block_index) {
if (pindex->nHeight != 0) {
FatalError("%s: First block connected is not the genesis block (height=%d)",
__func__, pindex->nHeight);
return;
}
} else {
// Ensure block connects to an ancestor of the current best block. This should be the case
// most of the time, but may not be immediately after the sync thread catches up and sets
// m_synced. Consider the case where there is a reorg and the blocks on the stale branch are
// in the ValidationInterface queue backlog even after the sync thread has caught up to the
// new chain tip. In this unlikely event, log a warning and let the queue clear.
if (best_block_index->GetAncestor(pindex->nHeight - 1) != pindex->pprev) {
LogPrintf("%s: WARNING: Block %s does not connect to an ancestor of " /* Continued */
"known best chain (tip=%s); not updating index\n",
__func__, pindex->GetBlockHash().ToString(),
best_block_index->GetBlockHash().ToString());
return;
}
if (best_block_index != pindex->pprev && !Rewind(best_block_index, pindex->pprev)) {
FatalError("%s: Failed to rewind index %s to a previous chain tip",
__func__, GetName());
return;
}
}
if (WriteBlock(*block, pindex)) {
m_best_block_index = pindex;
} else {
FatalError("%s: Failed to write block %s to index",
__func__, pindex->GetBlockHash().ToString());
return;
}
}
void BaseIndex::ChainStateFlushed(const CBlockLocator& locator)
{
if (!m_synced) {
return;
}
const uint256& locator_tip_hash = locator.vHave.front();
const CBlockIndex* locator_tip_index;
{
LOCK(cs_main);
locator_tip_index = m_chainstate->m_blockman.LookupBlockIndex(locator_tip_hash);
}
if (!locator_tip_index) {
FatalError("%s: First block (hash=%s) in locator was not found",
__func__, locator_tip_hash.ToString());
return;
}
// This checks that ChainStateFlushed callbacks are received after BlockConnected. The check may fail
// immediately after the sync thread catches up and sets m_synced. Consider the case where
// there is a reorg and the blocks on the stale branch are in the ValidationInterface queue
// backlog even after the sync thread has caught up to the new chain tip. In this unlikely
// event, log a warning and let the queue clear.
const CBlockIndex* best_block_index = m_best_block_index.load();
if (best_block_index->GetAncestor(locator_tip_index->nHeight) != locator_tip_index) {
LogPrintf("%s: WARNING: Locator contains block (hash=%s) not on known best " /* Continued */
"chain (tip=%s); not writing index locator\n",
__func__, locator_tip_hash.ToString(),
best_block_index->GetBlockHash().ToString());
return;
}
// No need to handle errors in Commit. If it fails, the error will be already be logged. The
// best way to recover is to continue, as index cannot be corrupted by a missed commit to disk
// for an advanced index state.
Commit();
}
bool BaseIndex::BlockUntilSyncedToCurrentChain() const
{
AssertLockNotHeld(cs_main);
if (!m_synced) {
return false;
}
{
// Skip the queue-draining stuff if we know we're caught up with
// ::ChainActive().Tip().
LOCK(cs_main);
const CBlockIndex* chain_tip = m_chainstate->m_chain.Tip();
const CBlockIndex* best_block_index = m_best_block_index.load();
if (best_block_index->GetAncestor(chain_tip->nHeight) == chain_tip) {
return true;
}
}
LogPrintf("%s: %s is catching up on block notifications\n", __func__, GetName());
SyncWithValidationInterfaceQueue();
return true;
}
void BaseIndex::Interrupt()
{
m_interrupt();
}
bool BaseIndex::Start(CChainState& active_chainstate)
{
m_chainstate = &active_chainstate;
// Need to register this ValidationInterface before running Init(), so that
// callbacks are not missed if Init sets m_synced to true.
RegisterValidationInterface(this);
if (!Init()) {
return false;
}
m_thread_sync = std::thread(&util::TraceThread, GetName(), [this] { ThreadSync(); });
return true;
}
void BaseIndex::Stop()
{
UnregisterValidationInterface(this);
if (m_thread_sync.joinable()) {
m_thread_sync.join();
}
}
IndexSummary BaseIndex::GetSummary() const
{
IndexSummary summary{};
summary.name = GetName();
summary.synced = m_synced;
summary.best_block_height = m_best_block_index ? m_best_block_index.load()->nHeight : 0;
return summary;
}

132
src/index/base.h Normal file
View file

@ -0,0 +1,132 @@
// Copyright (c) 2017-2020 The Bitcoin Core developers
// Copyright (c) 2017-2020 The DigiByte Core developers
// Distributed under the MIT software license, see the accompanying
// file COPYING or http://www.opensource.org/licenses/mit-license.php.
#ifndef DIGIBYTE_INDEX_BASE_H
#define DIGIBYTE_INDEX_BASE_H
#include <dbwrapper.h>
#include <primitives/block.h>
#include <primitives/transaction.h>
#include <threadinterrupt.h>
#include <validationinterface.h>
class CBlockIndex;
class CChainState;
struct IndexSummary {
std::string name;
bool synced{false};
int best_block_height{0};
};
/**
* Base class for indices of blockchain data. This implements
* CValidationInterface and ensures blocks are indexed sequentially according
* to their position in the active chain.
*/
class BaseIndex : public CValidationInterface
{
protected:
/**
* The database stores a block locator of the chain the database is synced to
* so that the index can efficiently determine the point it last stopped at.
* A locator is used instead of a simple hash of the chain tip because blocks
* and block index entries may not be flushed to disk until after this database
* is updated.
*/
class DB : public CDBWrapper
{
public:
DB(const fs::path& path, size_t n_cache_size,
bool f_memory = false, bool f_wipe = false, bool f_obfuscate = false);
/// Read block locator of the chain that the txindex is in sync with.
bool ReadBestBlock(CBlockLocator& locator) const;
/// Write block locator of the chain that the txindex is in sync with.
void WriteBestBlock(CDBBatch& batch, const CBlockLocator& locator);
};
private:
/// Whether the index is in sync with the main chain. The flag is flipped
/// from false to true once, after which point this starts processing
/// ValidationInterface notifications to stay in sync.
std::atomic<bool> m_synced{false};
/// The last block in the chain that the index is in sync with.
std::atomic<const CBlockIndex*> m_best_block_index{nullptr};
std::thread m_thread_sync;
CThreadInterrupt m_interrupt;
/// Sync the index with the block index starting from the current best block.
/// Intended to be run in its own thread, m_thread_sync, and can be
/// interrupted with m_interrupt. Once the index gets in sync, the m_synced
/// flag is set and the BlockConnected ValidationInterface callback takes
/// over and the sync thread exits.
void ThreadSync();
/// Write the current index state (eg. chain block locator and subclass-specific items) to disk.
///
/// Recommendations for error handling:
/// If called on a successor of the previous committed best block in the index, the index can
/// continue processing without risk of corruption, though the index state will need to catch up
/// from further behind on reboot. If the new state is not a successor of the previous state (due
/// to a chain reorganization), the index must halt until Commit succeeds or else it could end up
/// getting corrupted.
bool Commit();
protected:
CChainState* m_chainstate{nullptr};
void BlockConnected(const std::shared_ptr<const CBlock>& block, const CBlockIndex* pindex) override;
void ChainStateFlushed(const CBlockLocator& locator) override;
const CBlockIndex* CurrentIndex() { return m_best_block_index.load(); };
/// Initialize internal state from the database and block index.
[[nodiscard]] virtual bool Init();
/// Write update index entries for a newly connected block.
virtual bool WriteBlock(const CBlock& block, const CBlockIndex* pindex) { return true; }
/// Virtual method called internally by Commit that can be overridden to atomically
/// commit more index state.
virtual bool CommitInternal(CDBBatch& batch);
/// Rewind index to an earlier chain tip during a chain reorg. The tip must
/// be an ancestor of the current best block.
virtual bool Rewind(const CBlockIndex* current_tip, const CBlockIndex* new_tip);
virtual DB& GetDB() const = 0;
/// Get the name of the index for display in logs.
virtual const char* GetName() const = 0;
public:
/// Destructor interrupts sync thread if running and blocks until it exits.
virtual ~BaseIndex();
/// Blocks the current thread until the index is caught up to the current
/// state of the block chain. This only blocks if the index has gotten in
/// sync once and only needs to process blocks in the ValidationInterface
/// queue. If the index is catching up from far behind, this method does
/// not block and immediately returns false.
bool BlockUntilSyncedToCurrentChain() const LOCKS_EXCLUDED(::cs_main);
void Interrupt();
/// Start initializes the sync state and registers the instance as a
/// ValidationInterface so that it stays in sync with blockchain updates.
[[nodiscard]] bool Start(CChainState& active_chainstate);
/// Stops the instance from staying in sync with blockchain updates.
void Stop();
/// Get a summary of the index and its state.
IndexSummary GetSummary() const;
};
#endif // DIGIBYTE_INDEX_BASE_H

481
src/index/blockfilterindex.cpp Normal file
View file

@ -0,0 +1,481 @@
// Copyright (c) 2018-2020 The DigiByte Core developers
// Distributed under the MIT software license, see the accompanying
// file COPYING or http://www.opensource.org/licenses/mit-license.php.
#include <map>
#include <dbwrapper.h>
#include <index/blockfilterindex.h>
#include <node/blockstorage.h>
#include <util/system.h>
/* The index database stores three items for each block: the disk location of the encoded filter,
* its dSHA256 hash, and the header. Those belonging to blocks on the active chain are indexed by
* height, and those belonging to blocks that have been reorganized out of the active chain are
* indexed by block hash. This ensures that filter data for any block that becomes part of the
* active chain can always be retrieved, alleviating timing concerns.
*
* The filters themselves are stored in flat files and referenced by the LevelDB entries. This
* minimizes the amount of data written to LevelDB and keeps the database values constant size. The
* disk location of the next block filter to be written (represented as a FlatFilePos) is stored
* under the DB_FILTER_POS key.
*
* Keys for the height index have the type [DB_BLOCK_HEIGHT, uint32 (BE)]. The height is represented
* as big-endian so that sequential reads of filters by height are fast.
* Keys for the hash index have the type [DB_BLOCK_HASH, uint256].
*/
constexpr uint8_t DB_BLOCK_HASH{'s'};
constexpr uint8_t DB_BLOCK_HEIGHT{'t'};
constexpr uint8_t DB_FILTER_POS{'P'};
constexpr unsigned int MAX_FLTR_FILE_SIZE = 0x1000000; // 16 MiB
/** The pre-allocation chunk size for fltr?????.dat files */
constexpr unsigned int FLTR_FILE_CHUNK_SIZE = 0x100000; // 1 MiB
/** Maximum size of the cfheaders cache
* We have a limit to prevent a bug in filling this cache
* potentially turning into an OOM. At 2000 entries, this cache
* is big enough for a 2,000,000 length block chain, which
* we should be enough until ~2047. */
constexpr size_t CF_HEADERS_CACHE_MAX_SZ{2000};
namespace {
struct DBVal {
uint256 hash;
uint256 header;
FlatFilePos pos;
SERIALIZE_METHODS(DBVal, obj) { READWRITE(obj.hash, obj.header, obj.pos); }
};
struct DBHeightKey {
int height;
explicit DBHeightKey(int height_in) : height(height_in) {}
template<typename Stream>
void Serialize(Stream& s) const
{
ser_writedata8(s, DB_BLOCK_HEIGHT);
ser_writedata32be(s, height);
}
template<typename Stream>
void Unserialize(Stream& s)
{
const uint8_t prefix{ser_readdata8(s)};
if (prefix != DB_BLOCK_HEIGHT) {
throw std::ios_base::failure("Invalid format for block filter index DB height key");
}
height = ser_readdata32be(s);
}
};
struct DBHashKey {
uint256 hash;
explicit DBHashKey(const uint256& hash_in) : hash(hash_in) {}
SERIALIZE_METHODS(DBHashKey, obj) {
uint8_t prefix{DB_BLOCK_HASH};
READWRITE(prefix);
if (prefix != DB_BLOCK_HASH) {
throw std::ios_base::failure("Invalid format for block filter index DB hash key");
}
READWRITE(obj.hash);
}
};
}; // namespace
static std::map<BlockFilterType, BlockFilterIndex> g_filter_indexes;
BlockFilterIndex::BlockFilterIndex(BlockFilterType filter_type,
size_t n_cache_size, bool f_memory, bool f_wipe)
: m_filter_type(filter_type)
{
const std::string& filter_name = BlockFilterTypeName(filter_type);
if (filter_name.empty()) throw std::invalid_argument("unknown filter_type");
fs::path path = gArgs.GetDataDirNet() / "indexes" / "blockfilter" / filter_name;
fs::create_directories(path);
m_name = filter_name + " block filter index";
m_db = std::make_unique<BaseIndex::DB>(path / "db", n_cache_size, f_memory, f_wipe);
m_filter_fileseq = std::make_unique<FlatFileSeq>(std::move(path), "fltr", FLTR_FILE_CHUNK_SIZE);
}
bool BlockFilterIndex::Init()
{
if (!m_db->Read(DB_FILTER_POS, m_next_filter_pos)) {
// Check that the cause of the read failure is that the key does not exist. Any other errors
// indicate database corruption or a disk failure, and starting the index would cause
// further corruption.
if (m_db->Exists(DB_FILTER_POS)) {
return error("%s: Cannot read current %s state; index may be corrupted",
__func__, GetName());
}
// If the DB_FILTER_POS is not set, then initialize to the first location.
m_next_filter_pos.nFile = 0;
m_next_filter_pos.nPos = 0;
}
return BaseIndex::Init();
}
bool BlockFilterIndex::CommitInternal(CDBBatch& batch)
{
const FlatFilePos& pos = m_next_filter_pos;
// Flush current filter file to disk.
CAutoFile file(m_filter_fileseq->Open(pos), SER_DISK, CLIENT_VERSION);
if (file.IsNull()) {
return error("%s: Failed to open filter file %d", __func__, pos.nFile);
}
if (!FileCommit(file.Get())) {
return error("%s: Failed to commit filter file %d", __func__, pos.nFile);
}
batch.Write(DB_FILTER_POS, pos);
return BaseIndex::CommitInternal(batch);
}
bool BlockFilterIndex::ReadFilterFromDisk(const FlatFilePos& pos, BlockFilter& filter) const
{
CAutoFile filein(m_filter_fileseq->Open(pos, true), SER_DISK, CLIENT_VERSION);
if (filein.IsNull()) {
return false;
}
uint256 block_hash;
std::vector<uint8_t> encoded_filter;
try {
filein >> block_hash >> encoded_filter;
filter = BlockFilter(GetFilterType(), block_hash, std::move(encoded_filter));
}
catch (const std::exception& e) {
return error("%s: Failed to deserialize block filter from disk: %s", __func__, e.what());
}
return true;
}
size_t BlockFilterIndex::WriteFilterToDisk(FlatFilePos& pos, const BlockFilter& filter)
{
assert(filter.GetFilterType() == GetFilterType());
size_t data_size =
GetSerializeSize(filter.GetBlockHash(), CLIENT_VERSION) +
GetSerializeSize(filter.GetEncodedFilter(), CLIENT_VERSION);
// If writing the filter would overflow the file, flush and move to the next one.
if (pos.nPos + data_size > MAX_FLTR_FILE_SIZE) {
CAutoFile last_file(m_filter_fileseq->Open(pos), SER_DISK, CLIENT_VERSION);
if (last_file.IsNull()) {
LogPrintf("%s: Failed to open filter file %d\n", __func__, pos.nFile);
return 0;
}
if (!TruncateFile(last_file.Get(), pos.nPos)) {
LogPrintf("%s: Failed to truncate filter file %d\n", __func__, pos.nFile);
return 0;
}
if (!FileCommit(last_file.Get())) {
LogPrintf("%s: Failed to commit filter file %d\n", __func__, pos.nFile);
return 0;
}
pos.nFile++;
pos.nPos = 0;
}
// Pre-allocate sufficient space for filter data.
bool out_of_space;
m_filter_fileseq->Allocate(pos, data_size, out_of_space);
if (out_of_space) {
LogPrintf("%s: out of disk space\n", __func__);
return 0;
}
CAutoFile fileout(m_filter_fileseq->Open(pos), SER_DISK, CLIENT_VERSION);
if (fileout.IsNull()) {
LogPrintf("%s: Failed to open filter file %d\n", __func__, pos.nFile);
return 0;
}
fileout << filter.GetBlockHash() << filter.GetEncodedFilter();
return data_size;
}
bool BlockFilterIndex::WriteBlock(const CBlock& block, const CBlockIndex* pindex)
{
CBlockUndo block_undo;
uint256 prev_header;
if (pindex->nHeight > 0) {
if (!UndoReadFromDisk(block_undo, pindex)) {
return false;
}
std::pair<uint256, DBVal> read_out;
if (!m_db->Read(DBHeightKey(pindex->nHeight - 1), read_out)) {
return false;
}
uint256 expected_block_hash = pindex->pprev->GetBlockHash();
if (read_out.first != expected_block_hash) {
return error("%s: previous block header belongs to unexpected block %s; expected %s",
__func__, read_out.first.ToString(), expected_block_hash.ToString());
}
prev_header = read_out.second.header;
}
BlockFilter filter(m_filter_type, block, block_undo);
size_t bytes_written = WriteFilterToDisk(m_next_filter_pos, filter);
if (bytes_written == 0) return false;
std::pair<uint256, DBVal> value;
value.first = pindex->GetBlockHash();
value.second.hash = filter.GetHash();
value.second.header = filter.ComputeHeader(prev_header);
value.second.pos = m_next_filter_pos;
if (!m_db->Write(DBHeightKey(pindex->nHeight), value)) {
return false;
}
m_next_filter_pos.nPos += bytes_written;
return true;
}
static bool CopyHeightIndexToHashIndex(CDBIterator& db_it, CDBBatch& batch,
const std::string& index_name,
int start_height, int stop_height)
{
DBHeightKey key(start_height);
db_it.Seek(key);
for (int height = start_height; height <= stop_height; ++height) {
if (!db_it.GetKey(key) || key.height != height) {
return error("%s: unexpected key in %s: expected (%c, %d)",
__func__, index_name, DB_BLOCK_HEIGHT, height);
}
std::pair<uint256, DBVal> value;
if (!db_it.GetValue(value)) {
return error("%s: unable to read value in %s at key (%c, %d)",
__func__, index_name, DB_BLOCK_HEIGHT, height);
}
batch.Write(DBHashKey(value.first), std::move(value.second));
db_it.Next();
}
return true;
}
bool BlockFilterIndex::Rewind(const CBlockIndex* current_tip, const CBlockIndex* new_tip)
{
assert(current_tip->GetAncestor(new_tip->nHeight) == new_tip);
CDBBatch batch(*m_db);
std::unique_ptr<CDBIterator> db_it(m_db->NewIterator());
// During a reorg, we need to copy all filters for blocks that are getting disconnected from the
// height index to the hash index so we can still find them when the height index entries are
// overwritten.
if (!CopyHeightIndexToHashIndex(*db_it, batch, m_name, new_tip->nHeight, current_tip->nHeight)) {
return false;
}
// The latest filter position gets written in Commit by the call to the BaseIndex::Rewind.
// But since this creates new references to the filter, the position should get updated here
// atomically as well in case Commit fails.
batch.Write(DB_FILTER_POS, m_next_filter_pos);
if (!m_db->WriteBatch(batch)) return false;
return BaseIndex::Rewind(current_tip, new_tip);
}
static bool LookupOne(const CDBWrapper& db, const CBlockIndex* block_index, DBVal& result)
{
// First check if the result is stored under the height index and the value there matches the
// block hash. This should be the case if the block is on the active chain.
std::pair<uint256, DBVal> read_out;
if (!db.Read(DBHeightKey(block_index->nHeight), read_out)) {
return false;
}
if (read_out.first == block_index->GetBlockHash()) {
result = std::move(read_out.second);
return true;
}
// If value at the height index corresponds to an different block, the result will be stored in
// the hash index.
return db.Read(DBHashKey(block_index->GetBlockHash()), result);
}
static bool LookupRange(CDBWrapper& db, const std::string& index_name, int start_height,
const CBlockIndex* stop_index, std::vector<DBVal>& results)
{
if (start_height < 0) {
return error("%s: start height (%d) is negative", __func__, start_height);
}
if (start_height > stop_index->nHeight) {
return error("%s: start height (%d) is greater than stop height (%d)",
__func__, start_height, stop_index->nHeight);
}
size_t results_size = static_cast<size_t>(stop_index->nHeight - start_height + 1);
std::vector<std::pair<uint256, DBVal>> values(results_size);
DBHeightKey key(start_height);
std::unique_ptr<CDBIterator> db_it(db.NewIterator());
db_it->Seek(DBHeightKey(start_height));
for (int height = start_height; height <= stop_index->nHeight; ++height) {
if (!db_it->Valid() || !db_it->GetKey(key) || key.height != height) {
return false;
}
size_t i = static_cast<size_t>(height - start_height);
if (!db_it->GetValue(values[i])) {
return error("%s: unable to read value in %s at key (%c, %d)",
__func__, index_name, DB_BLOCK_HEIGHT, height);
}
db_it->Next();
}
results.resize(results_size);
// Iterate backwards through block indexes collecting results in order to access the block hash
// of each entry in case we need to look it up in the hash index.
for (const CBlockIndex* block_index = stop_index;
block_index && block_index->nHeight >= start_height;
block_index = block_index->pprev) {
uint256 block_hash = block_index->GetBlockHash();
size_t i = static_cast<size_t>(block_index->nHeight - start_height);
if (block_hash == values[i].first) {
results[i] = std::move(values[i].second);
continue;
}
if (!db.Read(DBHashKey(block_hash), results[i])) {
return error("%s: unable to read value in %s at key (%c, %s)",
__func__, index_name, DB_BLOCK_HASH, block_hash.ToString());
}
}
return true;
}
bool BlockFilterIndex::LookupFilter(const CBlockIndex* block_index, BlockFilter& filter_out) const
{
DBVal entry;
if (!LookupOne(*m_db, block_index, entry)) {
return false;
}
return ReadFilterFromDisk(entry.pos, filter_out);
}
bool BlockFilterIndex::LookupFilterHeader(const CBlockIndex* block_index, uint256& header_out)
{
LOCK(m_cs_headers_cache);
bool is_checkpoint{block_index->nHeight % CFCHECKPT_INTERVAL == 0};
if (is_checkpoint) {
// Try to find the block in the headers cache if this is a checkpoint height.
auto header = m_headers_cache.find(block_index->GetBlockHash());
if (header != m_headers_cache.end()) {
header_out = header->second;
return true;
}
}
DBVal entry;
if (!LookupOne(*m_db, block_index, entry)) {
return false;
}
if (is_checkpoint &&
m_headers_cache.size() < CF_HEADERS_CACHE_MAX_SZ) {
// Add to the headers cache if this is a checkpoint height.
m_headers_cache.emplace(block_index->GetBlockHash(), entry.header);
}
header_out = entry.header;
return true;
}
bool BlockFilterIndex::LookupFilterRange(int start_height, const CBlockIndex* stop_index,
std::vector<BlockFilter>& filters_out) const
{
std::vector<DBVal> entries;
if (!LookupRange(*m_db, m_name, start_height, stop_index, entries)) {
return false;
}
filters_out.resize(entries.size());
auto filter_pos_it = filters_out.begin();
for (const auto& entry : entries) {
if (!ReadFilterFromDisk(entry.pos, *filter_pos_it)) {
return false;
}
++filter_pos_it;
}
return true;
}
bool BlockFilterIndex::LookupFilterHashRange(int start_height, const CBlockIndex* stop_index,
std::vector<uint256>& hashes_out) const
{
std::vector<DBVal> entries;
if (!LookupRange(*m_db, m_name, start_height, stop_index, entries)) {
return false;
}
hashes_out.clear();
hashes_out.reserve(entries.size());
for (const auto& entry : entries) {
hashes_out.push_back(entry.hash);
}
return true;
}
BlockFilterIndex* GetBlockFilterIndex(BlockFilterType filter_type)
{
auto it = g_filter_indexes.find(filter_type);
return it != g_filter_indexes.end() ? &it->second : nullptr;
}
void ForEachBlockFilterIndex(std::function<void (BlockFilterIndex&)> fn)
{
for (auto& entry : g_filter_indexes) fn(entry.second);
}
bool InitBlockFilterIndex(BlockFilterType filter_type,
size_t n_cache_size, bool f_memory, bool f_wipe)
{
auto result = g_filter_indexes.emplace(std::piecewise_construct,
std::forward_as_tuple(filter_type),
std::forward_as_tuple(filter_type,
n_cache_size, f_memory, f_wipe));
return result.second;
}
bool DestroyBlockFilterIndex(BlockFilterType filter_type)
{
return g_filter_indexes.erase(filter_type);
}
void DestroyAllBlockFilterIndexes()
{
g_filter_indexes.clear();
}

102
src/index/blockfilterindex.h Normal file
View file

@ -0,0 +1,102 @@
// Copyright (c) 2018-2020 The DigiByte Core developers
// Distributed under the MIT software license, see the accompanying
// file COPYING or http://www.opensource.org/licenses/mit-license.php.
#ifndef DIGIBYTE_INDEX_BLOCKFILTERINDEX_H
#define DIGIBYTE_INDEX_BLOCKFILTERINDEX_H
#include <blockfilter.h>
#include <chain.h>
#include <flatfile.h>
#include <index/base.h>
#include <util/hasher.h>
/** Interval between compact filter checkpoints. See BIP 157. */
static constexpr int CFCHECKPT_INTERVAL = 1000;
/**
* BlockFilterIndex is used to store and retrieve block filters, hashes, and headers for a range of
* blocks by height. An index is constructed for each supported filter type with its own database
* (ie. filter data for different types are stored in separate databases).
*
* This index is used to serve BIP 157 net requests.
*/
class BlockFilterIndex final : public BaseIndex
{
private:
BlockFilterType m_filter_type;
std::string m_name;
std::unique_ptr<BaseIndex::DB> m_db;
FlatFilePos m_next_filter_pos;
std::unique_ptr<FlatFileSeq> m_filter_fileseq;
bool ReadFilterFromDisk(const FlatFilePos& pos, BlockFilter& filter) const;
size_t WriteFilterToDisk(FlatFilePos& pos, const BlockFilter& filter);
Mutex m_cs_headers_cache;
/** cache of block hash to filter header, to avoid disk access when responding to getcfcheckpt. */
std::unordered_map<uint256, uint256, FilterHeaderHasher> m_headers_cache GUARDED_BY(m_cs_headers_cache);
protected:
bool Init() override;
bool CommitInternal(CDBBatch& batch) override;
bool WriteBlock(const CBlock& block, const CBlockIndex* pindex) override;
bool Rewind(const CBlockIndex* current_tip, const CBlockIndex* new_tip) override;
BaseIndex::DB& GetDB() const override { return *m_db; }
const char* GetName() const override { return m_name.c_str(); }
public:
/** Constructs the index, which becomes available to be queried. */
explicit BlockFilterIndex(BlockFilterType filter_type,
size_t n_cache_size, bool f_memory = false, bool f_wipe = false);
BlockFilterType GetFilterType() const { return m_filter_type; }
/** Get a single filter by block. */
bool LookupFilter(const CBlockIndex* block_index, BlockFilter& filter_out) const;
/** Get a single filter header by block. */
bool LookupFilterHeader(const CBlockIndex* block_index, uint256& header_out);
/** Get a range of filters between two heights on a chain. */
bool LookupFilterRange(int start_height, const CBlockIndex* stop_index,
std::vector<BlockFilter>& filters_out) const;
/** Get a range of filter hashes between two heights on a chain. */
bool LookupFilterHashRange(int start_height, const CBlockIndex* stop_index,
std::vector<uint256>& hashes_out) const;
};
/**
* Get a block filter index by type. Returns nullptr if index has not been initialized or was
* already destroyed.
*/
BlockFilterIndex* GetBlockFilterIndex(BlockFilterType filter_type);
/** Iterate over all running block filter indexes, invoking fn on each. */
void ForEachBlockFilterIndex(std::function<void (BlockFilterIndex&)> fn);
/**
* Initialize a block filter index for the given type if one does not already exist. Returns true if
* a new index is created and false if one has already been initialized.
*/
bool InitBlockFilterIndex(BlockFilterType filter_type,
size_t n_cache_size, bool f_memory = false, bool f_wipe = false);
/**
* Destroy the block filter index with the given type. Returns false if no such index exists. This
* just releases the allocated memory and closes the database connection, it does not delete the
* index data.
*/
bool DestroyBlockFilterIndex(BlockFilterType filter_type);
/** Destroy all open block filter indexes. */
void DestroyAllBlockFilterIndexes();
#endif // DIGIBYTE_INDEX_BLOCKFILTERINDEX_H

472
src/index/coinstatsindex.cpp Normal file
View file

@ -0,0 +1,472 @@
// Copyright (c) 2020-2021 The DigiByte Core developers
// Distributed under the MIT software license, see the accompanying
// file COPYING or http://www.opensource.org/licenses/mit-license.php.
#include <chainparams.h>
#include <coins.h>
#include <crypto/muhash.h>
#include <index/coinstatsindex.h>
#include <node/blockstorage.h>
#include <serialize.h>
#include <txdb.h>
#include <undo.h>
#include <validation.h>
static constexpr uint8_t DB_BLOCK_HASH{'s'};
static constexpr uint8_t DB_BLOCK_HEIGHT{'t'};
static constexpr uint8_t DB_MUHASH{'M'};
namespace {
struct DBVal {
uint256 muhash;
uint64_t transaction_output_count;
uint64_t bogo_size;
CAmount total_amount;
CAmount total_subsidy;
CAmount block_unspendable_amount;
CAmount block_prevout_spent_amount;
CAmount block_new_outputs_ex_coinbase_amount;
CAmount block_coinbase_amount;
CAmount unspendables_genesis_block;
CAmount unspendables_bip30;
CAmount unspendables_scripts;
CAmount unspendables_unclaimed_rewards;
SERIALIZE_METHODS(DBVal, obj)
{
READWRITE(obj.muhash);
READWRITE(obj.transaction_output_count);
READWRITE(obj.bogo_size);
READWRITE(obj.total_amount);
READWRITE(obj.total_subsidy);
READWRITE(obj.block_unspendable_amount);
READWRITE(obj.block_prevout_spent_amount);
READWRITE(obj.block_new_outputs_ex_coinbase_amount);
READWRITE(obj.block_coinbase_amount);
READWRITE(obj.unspendables_genesis_block);
READWRITE(obj.unspendables_bip30);
READWRITE(obj.unspendables_scripts);
READWRITE(obj.unspendables_unclaimed_rewards);
}
};
struct DBHeightKey {
int height;
explicit DBHeightKey(int height_in) : height(height_in) {}
template <typename Stream>
void Serialize(Stream& s) const
{
ser_writedata8(s, DB_BLOCK_HEIGHT);
ser_writedata32be(s, height);
}
template <typename Stream>
void Unserialize(Stream& s)
{
const uint8_t prefix{ser_readdata8(s)};
if (prefix != DB_BLOCK_HEIGHT) {
throw std::ios_base::failure("Invalid format for coinstatsindex DB height key");
}
height = ser_readdata32be(s);
}
};
struct DBHashKey {
uint256 block_hash;
explicit DBHashKey(const uint256& hash_in) : block_hash(hash_in) {}
SERIALIZE_METHODS(DBHashKey, obj)
{
uint8_t prefix{DB_BLOCK_HASH};
READWRITE(prefix);
if (prefix != DB_BLOCK_HASH) {
throw std::ios_base::failure("Invalid format for coinstatsindex DB hash key");
}
READWRITE(obj.block_hash);
}
};
}; // namespace
std::unique_ptr<CoinStatsIndex> g_coin_stats_index;
CoinStatsIndex::CoinStatsIndex(size_t n_cache_size, bool f_memory, bool f_wipe)
{
fs::path path{gArgs.GetDataDirNet() / "indexes" / "coinstats"};
fs::create_directories(path);
m_db = std::make_unique<CoinStatsIndex::DB>(path / "db", n_cache_size, f_memory, f_wipe);
}
bool CoinStatsIndex::WriteBlock(const CBlock& block, const CBlockIndex* pindex)
{
CBlockUndo block_undo;
const CAmount block_subsidy{GetBlockSubsidy(pindex->nHeight, Params().GetConsensus())};
m_total_subsidy += block_subsidy;
// Ignore genesis block
if (pindex->nHeight > 0) {
if (!UndoReadFromDisk(block_undo, pindex)) {
return false;
}
std::pair<uint256, DBVal> read_out;
if (!m_db->Read(DBHeightKey(pindex->nHeight - 1), read_out)) {
return false;
}
uint256 expected_block_hash{pindex->pprev->GetBlockHash()};
if (read_out.first != expected_block_hash) {
if (!m_db->Read(DBHashKey(expected_block_hash), read_out)) {
return error("%s: previous block header belongs to unexpected block %s; expected %s",
__func__, read_out.first.ToString(), expected_block_hash.ToString());
}
}
// TODO: Deduplicate BIP30 related code
bool is_bip30_block{(pindex->nHeight == 91722 && pindex->GetBlockHash() == uint256S("0x00000000000271a2dc26e7667f8419f2e15416dc6955e5a6c6cdf3f2574dd08e")) ||
(pindex->nHeight == 91812 && pindex->GetBlockHash() == uint256S("0x00000000000af0aed4792b1acee3d966af36cf5def14935db8de83d6f9306f2f"))};
// Add the new utxos created from the block
for (size_t i = 0; i < block.vtx.size(); ++i) {
const auto& tx{block.vtx.at(i)};
// Skip duplicate txid coinbase transactions (BIP30).
if (is_bip30_block && tx->IsCoinBase()) {
m_block_unspendable_amount += block_subsidy;
m_unspendables_bip30 += block_subsidy;
continue;
}
for (size_t j = 0; j < tx->vout.size(); ++j) {
const CTxOut& out{tx->vout[j]};
Coin coin{out, pindex->nHeight, tx->IsCoinBase()};
COutPoint outpoint{tx->GetHash(), static_cast<uint32_t>(j)};
// Skip unspendable coins
if (coin.out.scriptPubKey.IsUnspendable()) {
m_block_unspendable_amount += coin.out.nValue;
m_unspendables_scripts += coin.out.nValue;
continue;
}
m_muhash.Insert(MakeUCharSpan(TxOutSer(outpoint, coin)));
if (tx->IsCoinBase()) {
m_block_coinbase_amount += coin.out.nValue;
} else {
m_block_new_outputs_ex_coinbase_amount += coin.out.nValue;
}
++m_transaction_output_count;
m_total_amount += coin.out.nValue;
m_bogo_size += GetBogoSize(coin.out.scriptPubKey);
}
// The coinbase tx has no undo data since no former output is spent
if (!tx->IsCoinBase()) {
const auto& tx_undo{block_undo.vtxundo.at(i - 1)};
for (size_t j = 0; j < tx_undo.vprevout.size(); ++j) {
Coin coin{tx_undo.vprevout[j]};
COutPoint outpoint{tx->vin[j].prevout.hash, tx->vin[j].prevout.n};
m_muhash.Remove(MakeUCharSpan(TxOutSer(outpoint, coin)));
m_block_prevout_spent_amount += coin.out.nValue;
--m_transaction_output_count;
m_total_amount -= coin.out.nValue;
m_bogo_size -= GetBogoSize(coin.out.scriptPubKey);
}
}
}
} else {
// genesis block
m_block_unspendable_amount += block_subsidy;
m_unspendables_genesis_block += block_subsidy;
}
// If spent prevouts + block subsidy are still a higher amount than
// new outputs + coinbase + current unspendable amount this means
// the miner did not claim the full block reward. Unclaimed block
// rewards are also unspendable.
const CAmount unclaimed_rewards{(m_block_prevout_spent_amount + m_total_subsidy) - (m_block_new_outputs_ex_coinbase_amount + m_block_coinbase_amount + m_block_unspendable_amount)};
m_block_unspendable_amount += unclaimed_rewards;
m_unspendables_unclaimed_rewards += unclaimed_rewards;
std::pair<uint256, DBVal> value;
value.first = pindex->GetBlockHash();
value.second.transaction_output_count = m_transaction_output_count;
value.second.bogo_size = m_bogo_size;
value.second.total_amount = m_total_amount;
value.second.total_subsidy = m_total_subsidy;
value.second.block_unspendable_amount = m_block_unspendable_amount;
value.second.block_prevout_spent_amount = m_block_prevout_spent_amount;
value.second.block_new_outputs_ex_coinbase_amount = m_block_new_outputs_ex_coinbase_amount;
value.second.block_coinbase_amount = m_block_coinbase_amount;
value.second.unspendables_genesis_block = m_unspendables_genesis_block;
value.second.unspendables_bip30 = m_unspendables_bip30;
value.second.unspendables_scripts = m_unspendables_scripts;
value.second.unspendables_unclaimed_rewards = m_unspendables_unclaimed_rewards;
uint256 out;
m_muhash.Finalize(out);
value.second.muhash = out;
return m_db->Write(DBHeightKey(pindex->nHeight), value) && m_db->Write(DB_MUHASH, m_muhash);
}
static bool CopyHeightIndexToHashIndex(CDBIterator& db_it, CDBBatch& batch,
const std::string& index_name,
int start_height, int stop_height)
{
DBHeightKey key{start_height};
db_it.Seek(key);
for (int height = start_height; height <= stop_height; ++height) {
if (!db_it.GetKey(key) || key.height != height) {
return error("%s: unexpected key in %s: expected (%c, %d)",
__func__, index_name, DB_BLOCK_HEIGHT, height);
}
std::pair<uint256, DBVal> value;
if (!db_it.GetValue(value)) {
return error("%s: unable to read value in %s at key (%c, %d)",
__func__, index_name, DB_BLOCK_HEIGHT, height);
}
batch.Write(DBHashKey(value.first), std::move(value.second));
db_it.Next();
}
return true;
}
bool CoinStatsIndex::Rewind(const CBlockIndex* current_tip, const CBlockIndex* new_tip)
{
assert(current_tip->GetAncestor(new_tip->nHeight) == new_tip);
CDBBatch batch(*m_db);
std::unique_ptr<CDBIterator> db_it(m_db->NewIterator());
// During a reorg, we need to copy all hash digests for blocks that are
// getting disconnected from the height index to the hash index so we can
// still find them when the height index entries are overwritten.
if (!CopyHeightIndexToHashIndex(*db_it, batch, m_name, new_tip->nHeight, current_tip->nHeight)) {
return false;
}
if (!m_db->WriteBatch(batch)) return false;
{
LOCK(cs_main);
CBlockIndex* iter_tip{m_chainstate->m_blockman.LookupBlockIndex(current_tip->GetBlockHash())};
const auto& consensus_params{Params().GetConsensus()};
do {
CBlock block;
if (!ReadBlockFromDisk(block, iter_tip, consensus_params)) {
return error("%s: Failed to read block %s from disk",
__func__, iter_tip->GetBlockHash().ToString());
}
ReverseBlock(block, iter_tip);
iter_tip = iter_tip->GetAncestor(iter_tip->nHeight - 1);
} while (new_tip != iter_tip);
}
return BaseIndex::Rewind(current_tip, new_tip);
}
static bool LookUpOne(const CDBWrapper& db, const CBlockIndex* block_index, DBVal& result)
{
// First check if the result is stored under the height index and the value
// there matches the block hash. This should be the case if the block is on
// the active chain.
std::pair<uint256, DBVal> read_out;
if (!db.Read(DBHeightKey(block_index->nHeight), read_out)) {
return false;
}
if (read_out.first == block_index->GetBlockHash()) {
result = std::move(read_out.second);
return true;
}
// If value at the height index corresponds to an different block, the
// result will be stored in the hash index.
return db.Read(DBHashKey(block_index->GetBlockHash()), result);
}
bool CoinStatsIndex::LookUpStats(const CBlockIndex* block_index, CCoinsStats& coins_stats) const
{
DBVal entry;
if (!LookUpOne(*m_db, block_index, entry)) {
return false;
}
coins_stats.hashSerialized = entry.muhash;
coins_stats.nTransactionOutputs = entry.transaction_output_count;
coins_stats.nBogoSize = entry.bogo_size;
coins_stats.nTotalAmount = entry.total_amount;
coins_stats.total_subsidy = entry.total_subsidy;
coins_stats.block_unspendable_amount = entry.block_unspendable_amount;
coins_stats.block_prevout_spent_amount = entry.block_prevout_spent_amount;
coins_stats.block_new_outputs_ex_coinbase_amount = entry.block_new_outputs_ex_coinbase_amount;
coins_stats.block_coinbase_amount = entry.block_coinbase_amount;
coins_stats.unspendables_genesis_block = entry.unspendables_genesis_block;
coins_stats.unspendables_bip30 = entry.unspendables_bip30;
coins_stats.unspendables_scripts = entry.unspendables_scripts;
coins_stats.unspendables_unclaimed_rewards = entry.unspendables_unclaimed_rewards;
return true;
}
bool CoinStatsIndex::Init()
{
if (!m_db->Read(DB_MUHASH, m_muhash)) {
// Check that the cause of the read failure is that the key does not
// exist. Any other errors indicate database corruption or a disk
// failure, and starting the index would cause further corruption.
if (m_db->Exists(DB_MUHASH)) {
return error("%s: Cannot read current %s state; index may be corrupted",
__func__, GetName());
}
}
if (BaseIndex::Init()) {
const CBlockIndex* pindex{CurrentIndex()};
if (pindex) {
DBVal entry;
if (!LookUpOne(*m_db, pindex, entry)) {
return false;
}
m_transaction_output_count = entry.transaction_output_count;
m_bogo_size = entry.bogo_size;
m_total_amount = entry.total_amount;
m_total_subsidy = entry.total_subsidy;
m_block_unspendable_amount = entry.block_unspendable_amount;
m_block_prevout_spent_amount = entry.block_prevout_spent_amount;
m_block_new_outputs_ex_coinbase_amount = entry.block_new_outputs_ex_coinbase_amount;
m_block_coinbase_amount = entry.block_coinbase_amount;
m_unspendables_genesis_block = entry.unspendables_genesis_block;
m_unspendables_bip30 = entry.unspendables_bip30;
m_unspendables_scripts = entry.unspendables_scripts;
m_unspendables_unclaimed_rewards = entry.unspendables_unclaimed_rewards;
}
return true;
}
return false;
}
// Reverse a single block as part of a reorg
bool CoinStatsIndex::ReverseBlock(const CBlock& block, const CBlockIndex* pindex)
{
CBlockUndo block_undo;
std::pair<uint256, DBVal> read_out;
const CAmount block_subsidy{GetBlockSubsidy(pindex->nHeight, Params().GetConsensus())};
m_total_subsidy -= block_subsidy;
// Ignore genesis block
if (pindex->nHeight > 0) {
if (!UndoReadFromDisk(block_undo, pindex)) {
return false;
}
if (!m_db->Read(DBHeightKey(pindex->nHeight - 1), read_out)) {
return false;
}
uint256 expected_block_hash{pindex->pprev->GetBlockHash()};
if (read_out.first != expected_block_hash) {
if (!m_db->Read(DBHashKey(expected_block_hash), read_out)) {
return error("%s: previous block header belongs to unexpected block %s; expected %s",
__func__, read_out.first.ToString(), expected_block_hash.ToString());
}
}
}
// Remove the new UTXOs that were created from the block
for (size_t i = 0; i < block.vtx.size(); ++i) {
const auto& tx{block.vtx.at(i)};
for (size_t j = 0; j < tx->vout.size(); ++j) {
const CTxOut& out{tx->vout[j]};
COutPoint outpoint{tx->GetHash(), static_cast<uint32_t>(j)};
Coin coin{out, pindex->nHeight, tx->IsCoinBase()};
// Skip unspendable coins
if (coin.out.scriptPubKey.IsUnspendable()) {
m_block_unspendable_amount -= coin.out.nValue;
m_unspendables_scripts -= coin.out.nValue;
continue;
}
m_muhash.Remove(MakeUCharSpan(TxOutSer(outpoint, coin)));
if (tx->IsCoinBase()) {
m_block_coinbase_amount -= coin.out.nValue;
} else {
m_block_new_outputs_ex_coinbase_amount -= coin.out.nValue;
}
--m_transaction_output_count;
m_total_amount -= coin.out.nValue;
m_bogo_size -= GetBogoSize(coin.out.scriptPubKey);
}
// The coinbase tx has no undo data since no former output is spent
if (!tx->IsCoinBase()) {
const auto& tx_undo{block_undo.vtxundo.at(i - 1)};
for (size_t j = 0; j < tx_undo.vprevout.size(); ++j) {
Coin coin{tx_undo.vprevout[j]};
COutPoint outpoint{tx->vin[j].prevout.hash, tx->vin[j].prevout.n};
m_muhash.Insert(MakeUCharSpan(TxOutSer(outpoint, coin)));
m_block_prevout_spent_amount -= coin.out.nValue;
m_transaction_output_count++;
m_total_amount += coin.out.nValue;
m_bogo_size += GetBogoSize(coin.out.scriptPubKey);
}
}
}
const CAmount unclaimed_rewards{(m_block_new_outputs_ex_coinbase_amount + m_block_coinbase_amount + m_block_unspendable_amount) - (m_block_prevout_spent_amount + m_total_subsidy)};
m_block_unspendable_amount -= unclaimed_rewards;
m_unspendables_unclaimed_rewards -= unclaimed_rewards;
// Check that the rolled back internal values are consistent with the DB read out
uint256 out;
m_muhash.Finalize(out);
Assert(read_out.second.muhash == out);
Assert(m_transaction_output_count == read_out.second.transaction_output_count);
Assert(m_total_amount == read_out.second.total_amount);
Assert(m_bogo_size == read_out.second.bogo_size);
Assert(m_total_subsidy == read_out.second.total_subsidy);
Assert(m_block_unspendable_amount == read_out.second.block_unspendable_amount);
Assert(m_block_prevout_spent_amount == read_out.second.block_prevout_spent_amount);
Assert(m_block_new_outputs_ex_coinbase_amount == read_out.second.block_new_outputs_ex_coinbase_amount);
Assert(m_block_coinbase_amount == read_out.second.block_coinbase_amount);
Assert(m_unspendables_genesis_block == read_out.second.unspendables_genesis_block);
Assert(m_unspendables_bip30 == read_out.second.unspendables_bip30);
Assert(m_unspendables_scripts == read_out.second.unspendables_scripts);
Assert(m_unspendables_unclaimed_rewards == read_out.second.unspendables_unclaimed_rewards);
return m_db->Write(DB_MUHASH, m_muhash);
}

61
src/index/coinstatsindex.h Normal file
View file

@ -0,0 +1,61 @@
// Copyright (c) 2020-2021 The DigiByte Core developers
// Distributed under the MIT software license, see the accompanying
// file COPYING or http://www.opensource.org/licenses/mit-license.php.
#ifndef DIGIBYTE_INDEX_COINSTATSINDEX_H
#define DIGIBYTE_INDEX_COINSTATSINDEX_H
#include <chain.h>
#include <crypto/muhash.h>
#include <flatfile.h>
#include <index/base.h>
#include <node/coinstats.h>
/**
* CoinStatsIndex maintains statistics on the UTXO set.
*/
class CoinStatsIndex final : public BaseIndex
{
private:
std::string m_name;
std::unique_ptr<BaseIndex::DB> m_db;
MuHash3072 m_muhash;
uint64_t m_transaction_output_count{0};
uint64_t m_bogo_size{0};
CAmount m_total_amount{0};
CAmount m_total_subsidy{0};
CAmount m_block_unspendable_amount{0};
CAmount m_block_prevout_spent_amount{0};
CAmount m_block_new_outputs_ex_coinbase_amount{0};
CAmount m_block_coinbase_amount{0};
CAmount m_unspendables_genesis_block{0};
CAmount m_unspendables_bip30{0};
CAmount m_unspendables_scripts{0};
CAmount m_unspendables_unclaimed_rewards{0};
bool ReverseBlock(const CBlock& block, const CBlockIndex* pindex);
protected:
bool Init() override;
bool WriteBlock(const CBlock& block, const CBlockIndex* pindex) override;
bool Rewind(const CBlockIndex* current_tip, const CBlockIndex* new_tip) override;
BaseIndex::DB& GetDB() const override { return *m_db; }
const char* GetName() const override { return "coinstatsindex"; }
public:
// Constructs the index, which becomes available to be queried.
explicit CoinStatsIndex(size_t n_cache_size, bool f_memory = false, bool f_wipe = false);
// Look up stats for a specific block using CBlockIndex
bool LookUpStats(const CBlockIndex* block_index, CCoinsStats& coins_stats) const;
};
/// The global UTXO set hash object.
extern std::unique_ptr<CoinStatsIndex> g_coin_stats_index;
#endif // DIGIBYTE_INDEX_COINSTATSINDEX_H

35
src/index/disktxpos.h Normal file
View file

@ -0,0 +1,35 @@
// Copyright (c) 2019-2020 The DigiByte Core developers
// Distributed under the MIT software license, see the accompanying
// file COPYING or http://www.opensource.org/licenses/mit-license.php.
#ifndef DIGIBYTE_INDEX_DISKTXPOS_H
#define DIGIBYTE_INDEX_DISKTXPOS_H
#include <flatfile.h>
#include <serialize.h>
struct CDiskTxPos : public FlatFilePos
{
unsigned int nTxOffset; // after header
SERIALIZE_METHODS(CDiskTxPos, obj)
{
READWRITEAS(FlatFilePos, obj);
READWRITE(VARINT(obj.nTxOffset));
}
CDiskTxPos(const FlatFilePos &blockIn, unsigned int nTxOffsetIn) : FlatFilePos(blockIn.nFile, blockIn.nPos), nTxOffset(nTxOffsetIn) {
}
CDiskTxPos() {
SetNull();
}
void SetNull() {
FlatFilePos::SetNull();
nTxOffset = 0;
}
};
#endif // DIGIBYTE_INDEX_DISKTXPOS_H

258
src/index/txindex.cpp Normal file
View file

@ -0,0 +1,258 @@
// Copyright (c) 2017-2020 The Bitcoin Core developers
// Copyright (c) 2017-2020 The DigiByte Core developers
// Distributed under the MIT software license, see the accompanying
// file COPYING or http://www.opensource.org/licenses/mit-license.php.
#include <index/disktxpos.h>
#include <index/txindex.h>
#include <node/blockstorage.h>
#include <node/ui_interface.h>
#include <shutdown.h>
#include <util/system.h>
#include <util/translation.h>
#include <validation.h>
constexpr uint8_t DB_BEST_BLOCK{'B'};
constexpr uint8_t DB_TXINDEX{'t'};
constexpr uint8_t DB_TXINDEX_BLOCK{'T'};
std::unique_ptr<TxIndex> g_txindex;
/** Access to the txindex database (indexes/txindex/) */
class TxIndex::DB : public BaseIndex::DB
{
public:
explicit DB(size_t n_cache_size, bool f_memory = false, bool f_wipe = false);
/// Read the disk location of the transaction data with the given hash. Returns false if the
/// transaction hash is not indexed.
bool ReadTxPos(const uint256& txid, CDiskTxPos& pos) const;
/// Write a batch of transaction positions to the DB.
bool WriteTxs(const std::vector<std::pair<uint256, CDiskTxPos>>& v_pos);
/// Migrate txindex data from the block tree DB, where it may be for older nodes that have not
/// been upgraded yet to the new database.
bool MigrateData(CBlockTreeDB& block_tree_db, const CBlockLocator& best_locator);
};
TxIndex::DB::DB(size_t n_cache_size, bool f_memory, bool f_wipe) :
BaseIndex::DB(gArgs.GetDataDirNet() / "indexes" / "txindex", n_cache_size, f_memory, f_wipe)
{}
bool TxIndex::DB::ReadTxPos(const uint256 &txid, CDiskTxPos& pos) const
{
return Read(std::make_pair(DB_TXINDEX, txid), pos);
}
bool TxIndex::DB::WriteTxs(const std::vector<std::pair<uint256, CDiskTxPos>>& v_pos)
{
CDBBatch batch(*this);
for (const auto& tuple : v_pos) {
batch.Write(std::make_pair(DB_TXINDEX, tuple.first), tuple.second);
}
return WriteBatch(batch);
}
/*
* Safely persist a transfer of data from the old txindex database to the new one, and compact the
* range of keys updated. This is used internally by MigrateData.
*/
static void WriteTxIndexMigrationBatches(CDBWrapper& newdb, CDBWrapper& olddb,
CDBBatch& batch_newdb, CDBBatch& batch_olddb,
const std::pair<uint8_t, uint256>& begin_key,
const std::pair<uint8_t, uint256>& end_key)
{
// Sync new DB changes to disk before deleting from old DB.
newdb.WriteBatch(batch_newdb, /*fSync=*/ true);
olddb.WriteBatch(batch_olddb);
olddb.CompactRange(begin_key, end_key);
batch_newdb.Clear();
batch_olddb.Clear();
}
bool TxIndex::DB::MigrateData(CBlockTreeDB& block_tree_db, const CBlockLocator& best_locator)
{
// The prior implementation of txindex was always in sync with block index
// and presence was indicated with a boolean DB flag. If the flag is set,
// this means the txindex from a previous version is valid and in sync with
// the chain tip. The first step of the migration is to unset the flag and
// write the chain hash to a separate key, DB_TXINDEX_BLOCK. After that, the
// index entries are copied over in batches to the new database. Finally,
// DB_TXINDEX_BLOCK is erased from the old database and the block hash is
// written to the new database.
//
// Unsetting the boolean flag ensures that if the node is downgraded to a
// previous version, it will not see a corrupted, partially migrated index
// -- it will see that the txindex is disabled. When the node is upgraded
// again, the migration will pick up where it left off and sync to the block
// with hash DB_TXINDEX_BLOCK.
bool f_legacy_flag = false;
block_tree_db.ReadFlag("txindex", f_legacy_flag);
if (f_legacy_flag) {
if (!block_tree_db.Write(DB_TXINDEX_BLOCK, best_locator)) {
return error("%s: cannot write block indicator", __func__);
}
if (!block_tree_db.WriteFlag("txindex", false)) {
return error("%s: cannot write block index db flag", __func__);
}
}
CBlockLocator locator;
if (!block_tree_db.Read(DB_TXINDEX_BLOCK, locator)) {
return true;
}
int64_t count = 0;
LogPrintf("Upgrading txindex database... [0%%]\n");
uiInterface.ShowProgress(_("Upgrading txindex database").translated, 0, true);
int report_done = 0;
const size_t batch_size = 1 << 24; // 16 MiB
CDBBatch batch_newdb(*this);
CDBBatch batch_olddb(block_tree_db);
std::pair<uint8_t, uint256> key;
std::pair<uint8_t, uint256> begin_key{DB_TXINDEX, uint256()};
std::pair<uint8_t, uint256> prev_key = begin_key;
bool interrupted = false;
std::unique_ptr<CDBIterator> cursor(block_tree_db.NewIterator());
for (cursor->Seek(begin_key); cursor->Valid(); cursor->Next()) {
if (ShutdownRequested()) {
interrupted = true;
break;
}
if (!cursor->GetKey(key)) {
return error("%s: cannot get key from valid cursor", __func__);
}
if (key.first != DB_TXINDEX) {
break;
}
// Log progress every 10%.
if (++count % 256 == 0) {
// Since txids are uniformly random and traversed in increasing order, the high 16 bits
// of the hash can be used to estimate the current progress.
const uint256& txid = key.second;
uint32_t high_nibble =
(static_cast<uint32_t>(*(txid.begin() + 0)) << 8) +
(static_cast<uint32_t>(*(txid.begin() + 1)) << 0);
int percentage_done = (int)(high_nibble * 100.0 / 65536.0 + 0.5);
uiInterface.ShowProgress(_("Upgrading txindex database").translated, percentage_done, true);
if (report_done < percentage_done/10) {
LogPrintf("Upgrading txindex database... [%d%%]\n", percentage_done);
report_done = percentage_done/10;
}
}
CDiskTxPos value;
if (!cursor->GetValue(value)) {
return error("%s: cannot parse txindex record", __func__);
}
batch_newdb.Write(key, value);
batch_olddb.Erase(key);
if (batch_newdb.SizeEstimate() > batch_size || batch_olddb.SizeEstimate() > batch_size) {
// NOTE: it's OK to delete the key pointed at by the current DB cursor while iterating
// because LevelDB iterators are guaranteed to provide a consistent view of the
// underlying data, like a lightweight snapshot.
WriteTxIndexMigrationBatches(*this, block_tree_db,
batch_newdb, batch_olddb,
prev_key, key);
prev_key = key;
}
}
// If these final DB batches complete the migration, write the best block
// hash marker to the new database and delete from the old one. This signals
// that the former is fully caught up to that point in the blockchain and
// that all txindex entries have been removed from the latter.
if (!interrupted) {
batch_olddb.Erase(DB_TXINDEX_BLOCK);
batch_newdb.Write(DB_BEST_BLOCK, locator);
}
WriteTxIndexMigrationBatches(*this, block_tree_db,
batch_newdb, batch_olddb,
begin_key, key);
if (interrupted) {
LogPrintf("[CANCELLED].\n");
return false;
}
uiInterface.ShowProgress("", 100, false);
LogPrintf("[DONE].\n");
return true;
}
TxIndex::TxIndex(size_t n_cache_size, bool f_memory, bool f_wipe)
: m_db(std::make_unique<TxIndex::DB>(n_cache_size, f_memory, f_wipe))
{}
TxIndex::~TxIndex() {}
bool TxIndex::Init()
{
LOCK(cs_main);
// Attempt to migrate txindex from the old database to the new one. Even if
// chain_tip is null, the node could be reindexing and we still want to
// delete txindex records in the old database.
if (!m_db->MigrateData(*pblocktree, m_chainstate->m_chain.GetLocator())) {
return false;
}
return BaseIndex::Init();
}
bool TxIndex::WriteBlock(const CBlock& block, const CBlockIndex* pindex)
{
// Exclude genesis block transaction because outputs are not spendable.
if (pindex->nHeight == 0) return true;
CDiskTxPos pos(pindex->GetBlockPos(), GetSizeOfCompactSize(block.vtx.size()));
std::vector<std::pair<uint256, CDiskTxPos>> vPos;
vPos.reserve(block.vtx.size());
for (const auto& tx : block.vtx) {
vPos.emplace_back(tx->GetHash(), pos);
pos.nTxOffset += ::GetSerializeSize(*tx, CLIENT_VERSION);
}
return m_db->WriteTxs(vPos);
}
BaseIndex::DB& TxIndex::GetDB() const { return *m_db; }
bool TxIndex::FindTx(const uint256& tx_hash, uint256& block_hash, CTransactionRef& tx) const
{
CDiskTxPos postx;
if (!m_db->ReadTxPos(tx_hash, postx)) {
return false;
}
CAutoFile file(OpenBlockFile(postx, true), SER_DISK, CLIENT_VERSION);
if (file.IsNull()) {
return error("%s: OpenBlockFile failed", __func__);
}
CBlockHeader header;
try {
file >> header;
if (fseek(file.Get(), postx.nTxOffset, SEEK_CUR)) {
return error("%s: fseek(...) failed", __func__);
}
file >> tx;
} catch (const std::exception& e) {
return error("%s: Deserialize or I/O error - %s", __func__, e.what());
}
if (tx->GetHash() != tx_hash) {
return error("%s: txid mismatch", __func__);
}
block_hash = header.GetHash();
return true;
}

54
src/index/txindex.h Normal file
View file

@ -0,0 +1,54 @@
// Copyright (c) 2017-2018 The DigiByte Core developers
// Distributed under the MIT software license, see the accompanying
// file COPYING or http://www.opensource.org/licenses/mit-license.php.
#ifndef DIGIBYTE_INDEX_TXINDEX_H
#define DIGIBYTE_INDEX_TXINDEX_H
#include <chain.h>
#include <index/base.h>
#include <txdb.h>
/**
* TxIndex is used to look up transactions included in the blockchain by hash.
* The index is written to a LevelDB database and records the filesystem
* location of each transaction by transaction hash.
*/
class TxIndex final : public BaseIndex
{
protected:
class DB;
private:
const std::unique_ptr<DB> m_db;
protected:
/// Override base class init to migrate from old database.
bool Init() override;
bool WriteBlock(const CBlock& block, const CBlockIndex* pindex) override;
BaseIndex::DB& GetDB() const override;
const char* GetName() const override { return "txindex"; }
public:
/// Constructs the index, which becomes available to be queried.
explicit TxIndex(size_t n_cache_size, bool f_memory = false, bool f_wipe = false);
// Destructor is declared because this class contains a unique_ptr to an incomplete type.
virtual ~TxIndex() override;
/// Look up a transaction by hash.
///
/// @param[in] tx_hash The hash of the transaction to be returned.
/// @param[out] block_hash The hash of the block the transaction is found in.
/// @param[out] tx The transaction itself.
/// @return true if transaction is found, false otherwise
bool FindTx(const uint256& tx_hash, uint256& block_hash, CTransactionRef& tx) const;
};
/// The global transaction index, used in GetTransaction. May be null.
extern std::unique_ptr<TxIndex> g_txindex;
#endif // DIGIBYTE_INDEX_TXINDEX_H