Initial Vexta Core v0.1.0 release

This commit is contained in:
Capri 2026-07-15 14:07:28 +00:00
commit 25c454329d
2481 changed files with 689289 additions and 0 deletions

43
test/lint/README.md Normal file
View file

@ -0,0 +1,43 @@
This folder contains lint scripts.
check-doc.py
============
Check for missing documentation of command line options.
commit-script-check.sh
======================
Verification of [scripted diffs](/doc/developer-notes.md#scripted-diffs).
Scripted diffs are only assumed to run on the latest LTS release of Ubuntu. Running them on other operating systems
might require installing GNU tools, such as GNU sed.
git-subtree-check.sh
====================
Run this script from the root of the repository to verify that a subtree matches the contents of
the commit it claims to have been updated to.
```
Usage: test/lint/git-subtree-check.sh [-r] DIR [COMMIT]
test/lint/git-subtree-check.sh -?
```
- `DIR` is the prefix within the repository to check.
- `COMMIT` is the commit to check, if it is not provided, HEAD will be used.
- `-r` checks that subtree commit is present in repository.
To do a full check with `-r`, make sure that you have fetched the upstream repository branch in which the subtree is
maintained:
* for `src/secp256k1`: https://github.com/digibyte-core/secp256k1.git (branch master)
* for `src/leveldb`: https://github.com/digibyte-core/leveldb.git (branch digibyte-fork)
* for `src/univalue`: https://github.com/digibyte-core/univalue.git (branch master)
* for `src/crypto/ctaes`: https://github.com/digibyte-core/ctaes.git (branch master)
* for `src/crc32c`: https://github.com/google/crc32c.git (branch master)
To do so, add the upstream repository as remote:
```
git remote add --fetch secp256k1 https://github.com/digibyte-core/secp256k1.git
```
lint-all.sh
===========
Calls other scripts with the `lint-` prefix.

67
test/lint/check-doc.py Executable file
View file

@ -0,0 +1,67 @@
#!/usr/bin/env python3
# Copyright (c) 2009-2020 The Bitcoin Core developers
# Copyright (c) 2014-2020 The DigiByte Core developers
# Distributed under the MIT software license, see the accompanying
# file COPYING or http://www.opensource.org/licenses/mit-license.php.
'''
This checks if all command line args are documented.
Return value is 0 to indicate no error.
Author: @MarcoFalke
'''
from subprocess import check_output
import re
FOLDER_GREP = 'src'
FOLDER_TEST = 'src/test/'
REGEX_ARG = r'(?:ForceSet|SoftSet|Get|Is)(?:Bool)?Args?(?:Set)?\("(-[^"]+)"'
REGEX_DOC = r'AddArg\("(-[^"=]+?)(?:=|")'
CMD_ROOT_DIR = '$(git rev-parse --show-toplevel)/{}'.format(FOLDER_GREP)
CMD_GREP_ARGS = r"git grep --perl-regexp '{}' -- {} ':(exclude){}'".format(REGEX_ARG, CMD_ROOT_DIR, FOLDER_TEST)
CMD_GREP_WALLET_ARGS = r"git grep --function-context 'void WalletInit::AddWalletOptions' -- {} | grep AddArg".format(CMD_ROOT_DIR)
CMD_GREP_WALLET_HIDDEN_ARGS = r"git grep --function-context 'void DummyWalletInit::AddWalletOptions' -- {}".format(CMD_ROOT_DIR)
CMD_GREP_DOCS = r"git grep --perl-regexp '{}' {}".format(REGEX_DOC, CMD_ROOT_DIR)
# list unsupported, deprecated and duplicate args as they need no documentation
SET_DOC_OPTIONAL = set(['-h', '-help', '-dbcrashratio', '-forcecompactdb', '-zapwallettxes'])
def lint_missing_argument_documentation():
used = check_output(CMD_GREP_ARGS, shell=True).decode('utf8').strip()
docd = check_output(CMD_GREP_DOCS, shell=True).decode('utf8').strip()
args_used = set(re.findall(re.compile(REGEX_ARG), used))
args_docd = set(re.findall(re.compile(REGEX_DOC), docd)).union(SET_DOC_OPTIONAL)
args_need_doc = args_used.difference(args_docd)
args_unknown = args_docd.difference(args_used)
print("Args used : {}".format(len(args_used)))
print("Args documented : {}".format(len(args_docd)))
print("Args undocumented: {}".format(len(args_need_doc)))
print(args_need_doc)
print("Args unknown : {}".format(len(args_unknown)))
print(args_unknown)
assert 0 == len(args_need_doc), "Please document the following arguments: {}".format(args_need_doc)
def lint_missing_hidden_wallet_args():
wallet_args = check_output(CMD_GREP_WALLET_ARGS, shell=True).decode('utf8').strip()
wallet_hidden_args = check_output(CMD_GREP_WALLET_HIDDEN_ARGS, shell=True).decode('utf8').strip()
wallet_args = set(re.findall(re.compile(REGEX_DOC), wallet_args))
wallet_hidden_args = set(re.findall(re.compile(r' "([^"=]+)'), wallet_hidden_args))
hidden_missing = wallet_args.difference(wallet_hidden_args)
if hidden_missing:
assert 0, "Please add {} to the hidden args in DummyWalletInit::AddWalletOptions".format(hidden_missing)
def main():
lint_missing_argument_documentation()
lint_missing_hidden_wallet_args()
if __name__ == "__main__":
main()

159
test/lint/check-rpc-mappings.py Executable file
View file

@ -0,0 +1,159 @@
#!/usr/bin/env python3
# Copyright (c) 2009-2019 The Bitcoin Core developers
# Copyright (c) 2014-2019 The DigiByte Core developers
# Distributed under the MIT software license, see the accompanying
# file COPYING or http://www.opensource.org/licenses/mit-license.php.
"""Check RPC argument consistency."""
from collections import defaultdict
import os
import re
import sys
# Source files (relative to root) to scan for dispatch tables
SOURCES = [
"src/rpc/server.cpp",
"src/rpc/blockchain.cpp",
"src/rpc/mining.cpp",
"src/rpc/misc.cpp",
"src/rpc/net.cpp",
"src/rpc/rawtransaction.cpp",
"src/wallet/rpcwallet.cpp",
]
# Source file (relative to root) containing conversion mapping
SOURCE_CLIENT = 'src/rpc/client.cpp'
# Argument names that should be ignored in consistency checks
IGNORE_DUMMY_ARGS = {'dummy', 'arg0', 'arg1', 'arg2', 'arg3', 'arg4', 'arg5', 'arg6', 'arg7', 'arg8', 'arg9'}
class RPCCommand:
def __init__(self, name, args):
self.name = name
self.args = args
class RPCArgument:
def __init__(self, names, idx):
self.names = names
self.idx = idx
self.convert = False
def parse_string(s):
assert s[0] == '"'
assert s[-1] == '"'
return s[1:-1]
def process_commands(fname):
"""Find and parse dispatch table in implementation file `fname`."""
cmds = []
in_rpcs = False
with open(fname, "r", encoding="utf8") as f:
for line in f:
line = line.rstrip()
if not in_rpcs:
if re.match("static const CRPCCommand .*\[\] =", line):
in_rpcs = True
else:
if line.startswith('};'):
in_rpcs = False
elif '{' in line and '"' in line:
m = re.search('{ *("[^"]*"), *("[^"]*"), *&([^,]*), *{([^}]*)} *},', line)
assert m, 'No match to table expression: %s' % line
name = parse_string(m.group(2))
args_str = m.group(4).strip()
if args_str:
args = [RPCArgument(parse_string(x.strip()).split('|'), idx) for idx, x in enumerate(args_str.split(','))]
else:
args = []
cmds.append(RPCCommand(name, args))
assert not in_rpcs and cmds, "Something went wrong with parsing the C++ file: update the regexps"
return cmds
def process_mapping(fname):
"""Find and parse conversion table in implementation file `fname`."""
cmds = []
in_rpcs = False
with open(fname, "r", encoding="utf8") as f:
for line in f:
line = line.rstrip()
if not in_rpcs:
if line == 'static const CRPCConvertParam vRPCConvertParams[] =':
in_rpcs = True
else:
if line.startswith('};'):
in_rpcs = False
elif '{' in line and '"' in line:
m = re.search('{ *("[^"]*"), *([0-9]+) *, *("[^"]*") *},', line)
assert m, 'No match to table expression: %s' % line
name = parse_string(m.group(1))
idx = int(m.group(2))
argname = parse_string(m.group(3))
cmds.append((name, idx, argname))
assert not in_rpcs and cmds
return cmds
def main():
root = sys.argv[1]
# Get all commands from dispatch tables
cmds = []
for fname in SOURCES:
cmds += process_commands(os.path.join(root, fname))
cmds_by_name = {}
for cmd in cmds:
cmds_by_name[cmd.name] = cmd
# Get current convert mapping for client
client = SOURCE_CLIENT
mapping = set(process_mapping(os.path.join(root, client)))
print('* Checking consistency between dispatch tables and vRPCConvertParams')
# Check mapping consistency
errors = 0
for (cmdname, argidx, argname) in mapping:
try:
rargnames = cmds_by_name[cmdname].args[argidx].names
except IndexError:
print('ERROR: %s argument %i (named %s in vRPCConvertParams) is not defined in dispatch table' % (cmdname, argidx, argname))
errors += 1
continue
if argname not in rargnames:
print('ERROR: %s argument %i is named %s in vRPCConvertParams but %s in dispatch table' % (cmdname, argidx, argname, rargnames), file=sys.stderr)
errors += 1
# Check for conflicts in vRPCConvertParams conversion
# All aliases for an argument must either be present in the
# conversion table, or not. Anything in between means an oversight
# and some aliases won't work.
for cmd in cmds:
for arg in cmd.args:
convert = [((cmd.name, arg.idx, argname) in mapping) for argname in arg.names]
if any(convert) != all(convert):
print('ERROR: %s argument %s has conflicts in vRPCConvertParams conversion specifier %s' % (cmd.name, arg.names, convert))
errors += 1
arg.convert = all(convert)
# Check for conversion difference by argument name.
# It is preferable for API consistency that arguments with the same name
# have the same conversion, so bin by argument name.
all_methods_by_argname = defaultdict(list)
converts_by_argname = defaultdict(list)
for cmd in cmds:
for arg in cmd.args:
for argname in arg.names:
all_methods_by_argname[argname].append(cmd.name)
converts_by_argname[argname].append(arg.convert)
for argname, convert in converts_by_argname.items():
if all(convert) != any(convert):
if argname in IGNORE_DUMMY_ARGS:
# these are testing or dummy, don't warn for them
continue
print('WARNING: conversion mismatch for argument named %s (%s)' %
(argname, list(zip(all_methods_by_argname[argname], converts_by_argname[argname]))))
sys.exit(errors > 0)
if __name__ == '__main__':
main()

View file

@ -0,0 +1,48 @@
#!/bin/sh
# Copyright (c) 2009-2020 The Bitcoin Core developers
# Copyright (c) 2014-2020 The DigiByte Core developers
# Distributed under the MIT software license, see the accompanying
# file COPYING or http://www.opensource.org/licenses/mit-license.php.
# This simple script checks for commits beginning with: scripted-diff:
# If found, looks for a script between the lines -BEGIN VERIFY SCRIPT- and
# -END VERIFY SCRIPT-. If no ending is found, it reads until the end of the
# commit message.
# The resulting script should exactly transform the previous commit into the current
# one. Any remaining diff signals an error.
export LC_ALL=C
if test -z $1; then
echo "Usage: $0 <commit>..."
exit 1
fi
RET=0
PREV_BRANCH=$(git name-rev --name-only HEAD)
PREV_HEAD=$(git rev-parse HEAD)
for commit in $(git rev-list --reverse $1); do
if git rev-list -n 1 --pretty="%s" $commit | grep -q "^scripted-diff:"; then
git checkout --quiet $commit^ || exit
SCRIPT="$(git rev-list --format=%b -n1 $commit | sed '/^-BEGIN VERIFY SCRIPT-$/,/^-END VERIFY SCRIPT-$/{//!b};d')"
if test -z "$SCRIPT"; then
echo "Error: missing script for: $commit"
echo "Failed"
RET=1
else
echo "Running script for: $commit"
echo "$SCRIPT"
(eval "$SCRIPT")
git --no-pager diff --exit-code $commit && echo "OK" || (echo "Failed"; false) || RET=1
fi
git reset --quiet --hard HEAD
else
if git rev-list "--format=%b" -n1 $commit | grep -q '^-\(BEGIN\|END\)[ a-zA-Z]*-$'; then
echo "Error: script block marker but no scripted-diff in title of commit $commit"
echo "Failed"
RET=1
fi
fi
done
git checkout --quiet $PREV_BRANCH 2>/dev/null || git checkout --quiet $PREV_HEAD
exit $RET

26
test/lint/extended-lint-all.sh Executable file
View file

@ -0,0 +1,26 @@
#!/usr/bin/env bash
#
# Copyright (c) 2019-2020 The DigiByte Core developers
# Distributed under the MIT software license, see the accompanying
# file COPYING or http://www.opensource.org/licenses/mit-license.php.
#
# This script runs all contrib/devtools/extended-lint-*.sh files, and fails if
# any exit with a non-zero status code.
# This script is intentionally locale dependent by not setting "export LC_ALL=C"
# in order to allow for the executed lint scripts to opt in or opt out of locale
# dependence themselves.
set -u
SCRIPTDIR=$(dirname "${BASH_SOURCE[0]}")
LINTALL=$(basename "${BASH_SOURCE[0]}")
for f in "${SCRIPTDIR}"/extended-lint-*.sh; do
if [ "$(basename "$f")" != "$LINTALL" ]; then
if ! "$f"; then
echo "^---- failure generated from $f"
exit 1
fi
fi
done

View file

@ -0,0 +1,88 @@
#!/usr/bin/env bash
#
# Copyright (c) 2019-2020 The DigiByte Core developers
# Distributed under the MIT software license, see the accompanying
# file COPYING or http://www.opensource.org/licenses/mit-license.php.
#
export LC_ALL=C
ENABLED_CHECKS=(
"Class '.*' has a constructor with 1 argument that is not explicit."
"Struct '.*' has a constructor with 1 argument that is not explicit."
)
IGNORED_WARNINGS=(
"src/arith_uint256.h:.* Class 'arith_uint256' has a constructor with 1 argument that is not explicit."
"src/arith_uint256.h:.* Class 'base_uint < 256 >' has a constructor with 1 argument that is not explicit."
"src/arith_uint256.h:.* Class 'base_uint' has a constructor with 1 argument that is not explicit."
"src/coins.h:.* Class 'CCoinsViewBacked' has a constructor with 1 argument that is not explicit."
"src/coins.h:.* Class 'CCoinsViewCache' has a constructor with 1 argument that is not explicit."
"src/coins.h:.* Class 'CCoinsViewCursor' has a constructor with 1 argument that is not explicit."
"src/net.h:.* Class 'CNetMessage' has a constructor with 1 argument that is not explicit."
"src/policy/feerate.h:.* Class 'CFeeRate' has a constructor with 1 argument that is not explicit."
"src/prevector.h:.* Class 'const_iterator' has a constructor with 1 argument that is not explicit."
"src/prevector.h:.* Class 'const_reverse_iterator' has a constructor with 1 argument that is not explicit."
"src/prevector.h:.* Class 'iterator' has a constructor with 1 argument that is not explicit."
"src/prevector.h:.* Class 'reverse_iterator' has a constructor with 1 argument that is not explicit."
"src/primitives/block.h:.* Class 'CBlock' has a constructor with 1 argument that is not explicit."
"src/primitives/transaction.h:.* Class 'CTransaction' has a constructor with 1 argument that is not explicit."
"src/protocol.h:.* Class 'CMessageHeader' has a constructor with 1 argument that is not explicit."
"src/qt/guiutil.h:.* Class 'ItemDelegate' has a constructor with 1 argument that is not explicit."
"src/rpc/util.h:.* Struct 'RPCResults' has a constructor with 1 argument that is not explicit."
"src/rpc/util.h:.* Struct 'UniValueType' has a constructor with 1 argument that is not explicit."
"src/rpc/util.h:.* style: Struct 'UniValueType' has a constructor with 1 argument that is not explicit."
"src/script/descriptor.cpp:.* Class 'AddressDescriptor' has a constructor with 1 argument that is not explicit."
"src/script/descriptor.cpp:.* Class 'ComboDescriptor' has a constructor with 1 argument that is not explicit."
"src/script/descriptor.cpp:.* Class 'ConstPubkeyProvider' has a constructor with 1 argument that is not explicit."
"src/script/descriptor.cpp:.* Class 'PKDescriptor' has a constructor with 1 argument that is not explicit."
"src/script/descriptor.cpp:.* Class 'PKHDescriptor' has a constructor with 1 argument that is not explicit."
"src/script/descriptor.cpp:.* Class 'RawDescriptor' has a constructor with 1 argument that is not explicit."
"src/script/descriptor.cpp:.* Class 'SHDescriptor' has a constructor with 1 argument that is not explicit."
"src/script/descriptor.cpp:.* Class 'WPKHDescriptor' has a constructor with 1 argument that is not explicit."
"src/script/descriptor.cpp:.* Class 'WSHDescriptor' has a constructor with 1 argument that is not explicit."
"src/script/script.h:.* Class 'CScript' has a constructor with 1 argument that is not explicit."
"src/script/standard.h:.* Class 'CScriptID' has a constructor with 1 argument that is not explicit."
"src/span.h:.* Class 'Span < const CRPCCommand >' has a constructor with 1 argument that is not explicit."
"src/span.h:.* Class 'Span < const char >' has a constructor with 1 argument that is not explicit."
"src/span.h:.* Class 'Span < const std :: vector <unsigned char > >' has a constructor with 1 argument that is not explicit."
"src/span.h:.* Class 'Span < const uint8_t >' has a constructor with 1 argument that is not explicit."
"src/span.h:.* Class 'Span' has a constructor with 1 argument that is not explicit."
"src/support/allocators/secure.h:.* Struct 'secure_allocator < char >' has a constructor with 1 argument that is not explicit."
"src/support/allocators/secure.h:.* Struct 'secure_allocator < RNGState >' has a constructor with 1 argument that is not explicit."
"src/support/allocators/secure.h:.* Struct 'secure_allocator < unsigned char >' has a constructor with 1 argument that is not explicit."
"src/support/allocators/zeroafterfree.h:.* Struct 'zero_after_free_allocator < char >' has a constructor with 1 argument that is not explicit."
"src/test/checkqueue_tests.cpp:.* Struct 'FailingCheck' has a constructor with 1 argument that is not explicit."
"src/test/checkqueue_tests.cpp:.* Struct 'MemoryCheck' has a constructor with 1 argument that is not explicit."
"src/test/checkqueue_tests.cpp:.* Struct 'UniqueCheck' has a constructor with 1 argument that is not explicit."
"src/test/fuzz/util.h:.* Class 'FuzzedFileProvider' has a constructor with 1 argument that is not explicit."
"src/test/fuzz/util.h:.* Class 'FuzzedAutoFileProvider' has a constructor with 1 argument that is not explicit."
"src/wallet/db.h:.* Class 'BerkeleyEnvironment' has a constructor with 1 argument that is not explicit."
)
if ! command -v cppcheck > /dev/null; then
echo "Skipping cppcheck linting since cppcheck is not installed. Install by running \"apt install cppcheck\""
exit 0
fi
function join_array {
local IFS="$1"
shift
echo "$*"
}
ENABLED_CHECKS_REGEXP=$(join_array "|" "${ENABLED_CHECKS[@]}")
IGNORED_WARNINGS_REGEXP=$(join_array "|" "${IGNORED_WARNINGS[@]}")
WARNINGS=$(git ls-files -- "*.cpp" "*.h" ":(exclude)src/leveldb/" ":(exclude)src/crc32c/" ":(exclude)src/secp256k1/" ":(exclude)src/univalue/" | \
xargs cppcheck --enable=all -j "$(getconf _NPROCESSORS_ONLN)" --language=c++ --std=c++17 --template=gcc -D__cplusplus -DCLIENT_VERSION_BUILD -DCLIENT_VERSION_IS_RELEASE -DCLIENT_VERSION_MAJOR -DCLIENT_VERSION_MINOR -DCOPYRIGHT_YEAR -DDEBUG -I src/ -q 2>&1 | sort -u | \
grep -E "${ENABLED_CHECKS_REGEXP}" | \
grep -vE "${IGNORED_WARNINGS_REGEXP}")
if [[ ${WARNINGS} != "" ]]; then
echo "${WARNINGS}"
echo
echo "Advice not applicable in this specific case? Add an exception by updating"
echo "IGNORED_WARNINGS in $0"
# Uncomment to enforce the developer note policy "By default, declare single-argument constructors `explicit`"
# exit 1
fi
exit 0

132
test/lint/git-subtree-check.sh Executable file
View file

@ -0,0 +1,132 @@
#!/bin/sh
# Copyright (c) 2009-2020 The Bitcoin Core developers
# Copyright (c) 2014-2020 The DigiByte Core developers
# Distributed under the MIT software license, see the accompanying
# file COPYING or http://www.opensource.org/licenses/mit-license.php.
export LC_ALL=C
check_remote=0
while getopts "?hr" opt; do
case $opt in
'?' | h)
echo "Usage: $0 [-r] DIR [COMMIT]"
echo " $0 -?"
echo ""
echo "Checks that a certain prefix is pure subtree, and optionally whether the"
echo "referenced commit is present in any fetched remote."
echo ""
echo "DIR is the prefix within the repository to check."
echo "COMMIT is the commit to check, if it is not provided, HEAD will be used."
echo ""
echo "-r Check that subtree commit is present in repository."
echo " To do this check, fetch the subtreed remote first. Example:"
echo ""
echo " git fetch https://github.com/digibyte-core/secp256k1.git"
echo " test/lint/git-subtree-check.sh -r src/secp256k1"
exit 1
;;
r)
check_remote=1
;;
esac
done
shift $((OPTIND-1))
if [ -z "$1" ]; then
echo "Need to provide a DIR, see $0 -?"
exit 1
fi
# Strip trailing / from directory path (in case it was added by autocomplete)
DIR="${1%/}"
COMMIT="$2"
if [ -z "$COMMIT" ]; then
COMMIT=HEAD
fi
# Taken from git-subtree (Copyright (C) 2009 Avery Pennarun <apenwarr@gmail.com>)
find_latest_squash()
{
dir="$1"
sq=
main=
sub=
git log --grep="^git-subtree-dir: $dir/*\$" \
--pretty=format:'START %H%n%s%n%n%b%nEND%n' "$COMMIT" |
while read a b _; do
case "$a" in
START) sq="$b" ;;
git-subtree-mainline:) main="$b" ;;
git-subtree-split:) sub="$b" ;;
END)
if [ -n "$sub" ]; then
if [ -n "$main" ]; then
# a rejoin commit?
# Pretend its sub was a squash.
sq="$sub"
fi
echo "$sq" "$sub"
break
fi
sq=
main=
sub=
;;
esac
done
}
# find latest subtree update
latest_squash="$(find_latest_squash "$DIR")"
if [ -z "$latest_squash" ]; then
echo "ERROR: $DIR is not a subtree" >&2
exit 2
fi
set $latest_squash
old=$1
rev=$2
# get the tree in the current commit
tree_actual=$(git ls-tree -d "$COMMIT" "$DIR" | head -n 1)
if [ -z "$tree_actual" ]; then
echo "FAIL: subtree directory $DIR not found in $COMMIT" >&2
exit 1
fi
set $tree_actual
tree_actual_type=$2
tree_actual_tree=$3
echo "$DIR in $COMMIT currently refers to $tree_actual_type $tree_actual_tree"
if [ "d$tree_actual_type" != "dtree" ]; then
echo "FAIL: subtree directory $DIR is not a tree in $COMMIT" >&2
exit 1
fi
# get the tree at the time of the last subtree update
tree_commit=$(git show -s --format="%T" $old)
echo "$DIR in $COMMIT was last updated in commit $old (tree $tree_commit)"
# ... and compare the actual tree with it
if [ "$tree_actual_tree" != "$tree_commit" ]; then
git diff $tree_commit $tree_actual_tree >&2
echo "FAIL: subtree directory was touched without subtree merge" >&2
exit 1
fi
if [ "$check_remote" != "0" ]; then
# get the tree in the subtree commit referred to
if [ "d$(git cat-file -t $rev 2>/dev/null)" != dcommit ]; then
echo "subtree commit $rev unavailable: cannot compare. Did you add and fetch the remote?" >&2
exit 1
fi
tree_subtree=$(git show -s --format="%T" $rev)
echo "$DIR in $COMMIT was last updated to upstream commit $rev (tree $tree_subtree)"
# ... and compare the actual tree with it
if [ "$tree_actual_tree" != "$tree_subtree" ]; then
echo "FAIL: subtree update commit differs from upstream tree!" >&2
exit 1
fi
fi
echo "GOOD"

31
test/lint/lint-all.sh Executable file
View file

@ -0,0 +1,31 @@
#!/usr/bin/env bash
#
# Copyright (c) 2009-2020 The Bitcoin Core developers
# Copyright (c) 2014-2020 The DigiByte Core developers
# Distributed under the MIT software license, see the accompanying
# file COPYING or http://www.opensource.org/licenses/mit-license.php.
#
# This script runs all contrib/devtools/lint-*.sh files, and fails if any exit
# with a non-zero status code.
# This script is intentionally locale dependent by not setting "export LC_ALL=C"
# in order to allow for the executed lint scripts to opt in or opt out of locale
# dependence themselves.
set -u
SCRIPTDIR=$(dirname "${BASH_SOURCE[0]}")
LINTALL=$(basename "${BASH_SOURCE[0]}")
EXIT_CODE=0
for f in "${SCRIPTDIR}"/lint-*.sh; do
if [ "$(basename "$f")" != "$LINTALL" ]; then
if ! "$f"; then
echo "^---- failure generated from $f"
EXIT_CODE=1
fi
fi
done
exit ${EXIT_CODE}

34
test/lint/lint-assertions.sh Executable file
View file

@ -0,0 +1,34 @@
#!/usr/bin/env bash
#
# Copyright (c) 2018-2020 The DigiByte Core developers
# Distributed under the MIT software license, see the accompanying
# file COPYING or http://www.opensource.org/licenses/mit-license.php.
#
# Check for assertions with obvious side effects.
export LC_ALL=C
EXIT_CODE=0
# PRE31-C (SEI CERT C Coding Standard):
# "Assertions should not contain assignments, increment, or decrement operators."
OUTPUT=$(git grep -E '[^_]assert\(.*(\+\+|\-\-|[^=!<>]=[^=!<>]).*\);' -- "*.cpp" "*.h")
if [[ ${OUTPUT} != "" ]]; then
echo "Assertions should not have side effects:"
echo
echo "${OUTPUT}"
EXIT_CODE=1
fi
# Macro CHECK_NONFATAL(condition) should be used instead of assert for RPC code, where it
# is undesirable to crash the whole program. See: src/util/check.h
# src/rpc/server.cpp is excluded from this check since it's mostly meta-code.
OUTPUT=$(git grep -nE '\<(A|a)ssert *\(.*\);' -- "src/rpc/" "src/wallet/rpc*" ":(exclude)src/rpc/server.cpp")
if [[ ${OUTPUT} != "" ]]; then
echo "CHECK_NONFATAL(condition) should be used instead of assert for RPC code."
echo
echo "${OUTPUT}"
EXIT_CODE=1
fi
exit ${EXIT_CODE}

View file

@ -0,0 +1,72 @@
#!/usr/bin/env bash
#
# Copyright (c) 2009-2020 The Bitcoin Core developers
# Copyright (c) 2014-2020 The DigiByte Core developers
# Distributed under the MIT software license, see the accompanying
# file COPYING or http://www.opensource.org/licenses/mit-license.php.
#
# Check for circular dependencies
export LC_ALL=C
EXPECTED_CIRCULAR_DEPENDENCIES=(
"chainparamsbase -> util/system -> chainparamsbase"
"index/txindex -> validation -> index/txindex"
"node/blockstorage -> validation -> node/blockstorage"
"index/blockfilterindex -> node/blockstorage -> validation -> index/blockfilterindex"
"index/base -> validation -> index/blockfilterindex -> index/base"
"index/coinstatsindex -> node/coinstats -> index/coinstatsindex"
"policy/fees -> txmempool -> policy/fees"
"qt/addresstablemodel -> qt/walletmodel -> qt/addresstablemodel"
"qt/recentrequeststablemodel -> qt/walletmodel -> qt/recentrequeststablemodel"
"qt/sendcoinsdialog -> qt/walletmodel -> qt/sendcoinsdialog"
"qt/transactiontablemodel -> qt/walletmodel -> qt/transactiontablemodel"
"txmempool -> validation -> txmempool"
"wallet/fees -> wallet/wallet -> wallet/fees"
"wallet/wallet -> wallet/walletdb -> wallet/wallet"
"node/coinstats -> validation -> node/coinstats"
# Temporary circular dependencies that allow wallet.h/wallet.cpp to be
# split up in a MOVEONLY commit. These are removed in #21206.
"wallet/receive -> wallet/wallet -> wallet/receive"
"wallet/spend -> wallet/wallet -> wallet/spend"
)
EXIT_CODE=0
CIRCULAR_DEPENDENCIES=()
IFS=$'\n'
for CIRC in $(cd src && ../contrib/devtools/circular-dependencies.py {*,*/*,*/*/*}.{h,cpp} | sed -e 's/^Circular dependency: //'); do
CIRCULAR_DEPENDENCIES+=( "$CIRC" )
IS_EXPECTED_CIRC=0
for EXPECTED_CIRC in "${EXPECTED_CIRCULAR_DEPENDENCIES[@]}"; do
if [[ "${CIRC}" == "${EXPECTED_CIRC}" ]]; then
IS_EXPECTED_CIRC=1
break
fi
done
if [[ ${IS_EXPECTED_CIRC} == 0 ]]; then
echo "A new circular dependency in the form of \"${CIRC}\" appears to have been introduced."
echo
EXIT_CODE=1
fi
done
for EXPECTED_CIRC in "${EXPECTED_CIRCULAR_DEPENDENCIES[@]}"; do
IS_PRESENT_EXPECTED_CIRC=0
for CIRC in "${CIRCULAR_DEPENDENCIES[@]}"; do
if [[ "${CIRC}" == "${EXPECTED_CIRC}" ]]; then
IS_PRESENT_EXPECTED_CIRC=1
break
fi
done
if [[ ${IS_PRESENT_EXPECTED_CIRC} == 0 ]]; then
echo "Good job! The circular dependency \"${EXPECTED_CIRC}\" is no longer present."
echo "Please remove it from EXPECTED_CIRCULAR_DEPENDENCIES in $0"
echo "to make sure this circular dependency is not accidentally reintroduced."
echo
EXIT_CODE=1
fi
done
exit ${EXIT_CODE}

21
test/lint/lint-cpp.sh Executable file
View file

@ -0,0 +1,21 @@
#!/usr/bin/env bash
#
# Copyright (c) 2020 The DigiByte Core developers
# Distributed under the MIT software license, see the accompanying
# file COPYING or http://www.opensource.org/licenses/mit-license.php.
#
# Check for various C++ code patterns we want to avoid.
export LC_ALL=C
EXIT_CODE=0
OUTPUT=$(git grep -E "boost::bind\(" -- "*.cpp" "*.h")
if [[ ${OUTPUT} != "" ]]; then
echo "Use of boost::bind detected. Use std::bind instead."
echo
echo "${OUTPUT}"
EXIT_CODE=1
fi
exit ${EXIT_CODE}

24
test/lint/lint-filenames.sh Executable file
View file

@ -0,0 +1,24 @@
#!/usr/bin/env bash
#
# Copyright (c) 2018 The DigiByte Core developers
# Distributed under the MIT software license, see the accompanying
# file COPYING or http://www.opensource.org/licenses/mit-license.php.
#
# Make sure only lowercase alphanumerics (a-z0-9), underscores (_),
# hyphens (-) and dots (.) are used in source code filenames.
export LC_ALL=C
EXIT_CODE=0
OUTPUT=$(git ls-files --full-name -- "*.[cC][pP][pP]" "*.[hH]" "*.[pP][yY]" "*.[sS][hH]" | \
grep -vE '^[a-z0-9_./-]+$' | \
grep -vE '^src/(secp256k1|univalue)/')
if [[ ${OUTPUT} != "" ]]; then
echo "Use only lowercase alphanumerics (a-z0-9), underscores (_), hyphens (-) and dots (.)"
echo "in source code filenames:"
echo
echo "${OUTPUT}"
EXIT_CODE=1
fi
exit ${EXIT_CODE}

203
test/lint/lint-files.py Executable file
View file

@ -0,0 +1,203 @@
#!/usr/bin/env python3
# Copyright (c) 2021 The DigiByte Core developers
# Distributed under the MIT software license, see the accompanying
# file COPYING or http://www.opensource.org/licenses/mit-license.php.
"""
This checks that all files in the repository have correct filenames and permissions
"""
import os
import re
import sys
from subprocess import check_output
from typing import Optional, NoReturn
CMD_ALL_FILES = "git ls-files -z --full-name"
CMD_SOURCE_FILES = 'git ls-files -z --full-name -- "*.[cC][pP][pP]" "*.[hH]" "*.[pP][yY]" "*.[sS][hH]"'
CMD_SHEBANG_FILES = "git grep --full-name --line-number -I '^#!'"
ALLOWED_FILENAME_REGEXP = "^[a-zA-Z0-9/_.@][a-zA-Z0-9/_.@-]*$"
ALLOWED_SOURCE_FILENAME_REGEXP = "^[a-z0-9_./-]+$"
ALLOWED_SOURCE_FILENAME_EXCEPTION_REGEXP = (
"^src/(secp256k1/|univalue/|test/fuzz/FuzzedDataProvider.h)"
)
ALLOWED_PERMISSION_NON_EXECUTABLES = 644
ALLOWED_PERMISSION_EXECUTABLES = 755
ALLOWED_EXECUTABLE_SHEBANG = {
"py": [b"#!/usr/bin/env python3"],
"sh": [b"#!/usr/bin/env bash", b"#!/bin/sh"],
}
class FileMeta(object):
def __init__(self, file_path: str):
self.file_path = file_path
@property
def extension(self) -> Optional[str]:
"""
Returns the file extension for a given filename string.
eg:
'ci/lint_run_all.sh' -> 'sh'
'ci/retry/retry' -> None
'contrib/devtools/split-debug.sh.in' -> 'in'
"""
return str(os.path.splitext(self.file_path)[1].strip(".") or None)
@property
def full_extension(self) -> Optional[str]:
"""
Returns the full file extension for a given filename string.
eg:
'ci/lint_run_all.sh' -> 'sh'
'ci/retry/retry' -> None
'contrib/devtools/split-debug.sh.in' -> 'sh.in'
"""
filename_parts = self.file_path.split(os.extsep, 1)
try:
return filename_parts[1]
except IndexError:
return None
@property
def permissions(self) -> int:
"""
Returns the octal file permission of the file
"""
return int(oct(os.stat(self.file_path).st_mode)[-3:])
def check_all_filenames() -> int:
"""
Checks every file in the repository against an allowed regexp to make sure only lowercase or uppercase
alphanumerics (a-zA-Z0-9), underscores (_), hyphens (-), at (@) and dots (.) are used in repository filenames.
"""
filenames = check_output(CMD_ALL_FILES, shell=True).decode("utf8").rstrip("\0").split("\0")
filename_regex = re.compile(ALLOWED_FILENAME_REGEXP)
failed_tests = 0
for filename in filenames:
if not filename_regex.match(filename):
print(
f"""File {repr(filename)} does not not match the allowed filename regexp ('{ALLOWED_FILENAME_REGEXP}')."""
)
failed_tests += 1
return failed_tests
def check_source_filenames() -> int:
"""
Checks only source files (*.cpp, *.h, *.py, *.sh) against a stricter allowed regexp to make sure only lowercase
alphanumerics (a-z0-9), underscores (_), hyphens (-) and dots (.) are used in source code filenames.
Additionally there is an exception regexp for directories or files which are excepted from matching this regexp.
"""
filenames = check_output(CMD_SOURCE_FILES, shell=True).decode("utf8").rstrip("\0").split("\0")
filename_regex = re.compile(ALLOWED_SOURCE_FILENAME_REGEXP)
filename_exception_regex = re.compile(ALLOWED_SOURCE_FILENAME_EXCEPTION_REGEXP)
failed_tests = 0
for filename in filenames:
if not filename_regex.match(filename) and not filename_exception_regex.match(filename):
print(
f"""File {repr(filename)} does not not match the allowed source filename regexp ('{ALLOWED_SOURCE_FILENAME_REGEXP}'), or the exception regexp ({ALLOWED_SOURCE_FILENAME_EXCEPTION_REGEXP})."""
)
failed_tests += 1
return failed_tests
def check_all_file_permissions() -> int:
"""
Checks all files in the repository match an allowed executable or non-executable file permission octal.
Additionally checks that for executable files, the file contains a shebang line
"""
filenames = check_output(CMD_ALL_FILES, shell=True).decode("utf8").rstrip("\0").split("\0")
failed_tests = 0
for filename in filenames:
file_meta = FileMeta(filename)
if file_meta.permissions == ALLOWED_PERMISSION_EXECUTABLES:
with open(filename, "rb") as f:
shebang = f.readline().rstrip(b"\n")
# For any file with executable permissions the first line must contain a shebang
if not shebang.startswith(b"#!"):
print(
f"""File "{filename}" has permission {ALLOWED_PERMISSION_EXECUTABLES} (executable) and is thus expected to contain a shebang '#!'. Add shebang or do "chmod {ALLOWED_PERMISSION_NON_EXECUTABLES} {filename}" to make it non-executable."""
)
failed_tests += 1
# For certain file extensions that have been defined, we also check that the shebang conforms to a specific
# allowable set of shebangs
if file_meta.extension in ALLOWED_EXECUTABLE_SHEBANG.keys():
if shebang not in ALLOWED_EXECUTABLE_SHEBANG[file_meta.extension]:
print(
f"""File "{filename}" is missing expected shebang """
+ " or ".join(
[
x.decode("utf-8")
for x in ALLOWED_EXECUTABLE_SHEBANG[file_meta.extension]
]
)
)
failed_tests += 1
elif file_meta.permissions == ALLOWED_PERMISSION_NON_EXECUTABLES:
continue
else:
print(
f"""File "{filename}" has unexpected permission {file_meta.permissions}. Do "chmod {ALLOWED_PERMISSION_NON_EXECUTABLES} {filename}" (if non-executable) or "chmod {ALLOWED_PERMISSION_EXECUTABLES} {filename}" (if executable)."""
)
failed_tests += 1
return failed_tests
def check_shebang_file_permissions() -> int:
"""
Checks every file that contains a shebang line to ensure it has an executable permission
"""
filenames = check_output(CMD_SHEBANG_FILES, shell=True).decode("utf8").strip().split("\n")
# The git grep command we use returns files which contain a shebang on any line within the file
# so we need to filter the list to only files with the shebang on the first line
filenames = [filename.split(":1:")[0] for filename in filenames if ":1:" in filename]
failed_tests = 0
for filename in filenames:
file_meta = FileMeta(filename)
if file_meta.permissions != ALLOWED_PERMISSION_EXECUTABLES:
# These file types are typically expected to be sourced and not executed directly
if file_meta.full_extension in ["bash", "init", "openrc", "sh.in"]:
continue
# *.py files which don't contain an `if __name__ == '__main__'` are not expected to be executed directly
if file_meta.extension == "py":
with open(filename, "r", encoding="utf8") as f:
file_data = f.read()
if not re.search("""if __name__ == ['"]__main__['"]:""", file_data):
continue
print(
f"""File "{filename}" contains a shebang line, but has the file permission {file_meta.permissions} instead of the expected executable permission {ALLOWED_PERMISSION_EXECUTABLES}. Do "chmod {ALLOWED_PERMISSION_EXECUTABLES} {filename}" (or remove the shebang line)."""
)
failed_tests += 1
return failed_tests
def main() -> NoReturn:
failed_tests = 0
failed_tests += check_all_filenames()
failed_tests += check_source_filenames()
failed_tests += check_all_file_permissions()
failed_tests += check_shebang_file_permissions()
if failed_tests:
print(
f"ERROR: There were {failed_tests} failed tests in the lint-files.py lint test. Please resolve the above errors."
)
sys.exit(1)
else:
sys.exit(0)
if __name__ == "__main__":
main()

7
test/lint/lint-files.sh Executable file
View file

@ -0,0 +1,7 @@
#!/usr/bin/env bash
export LC_ALL=C
set -e
cd "$(dirname $0)/../.."
test/lint/lint-files.py

294
test/lint/lint-format-strings.py Executable file
View file

@ -0,0 +1,294 @@
#!/usr/bin/env python3
#
# Copyright (c) 2009-2020 The Bitcoin Core developers
# Copyright (c) 2014-2020 The DigiByte Core developers
# Distributed under the MIT software license, see the accompanying
# file COPYING or http://www.opensource.org/licenses/mit-license.php.
#
# Lint format strings: This program checks that the number of arguments passed
# to a variadic format string function matches the number of format specifiers
# in the format string.
import argparse
import re
import sys
FALSE_POSITIVES = [
("src/dbwrapper.cpp", "vsnprintf(p, limit - p, format, backup_ap)"),
("src/index/base.cpp", "FatalError(const char* fmt, const Args&... args)"),
("src/netbase.cpp", "LogConnectFailure(bool manual_connection, const char* fmt, const Args&... args)"),
("src/util/system.cpp", "strprintf(_(COPYRIGHT_HOLDERS).translated, COPYRIGHT_HOLDERS_SUBSTITUTION)"),
("src/validationinterface.cpp", "LogPrint(BCLog::VALIDATION, fmt \"\\n\", __VA_ARGS__)"),
("src/wallet/wallet.h", "WalletLogPrintf(std::string fmt, Params... parameters)"),
("src/wallet/wallet.h", "LogPrintf((\"%s \" + fmt).c_str(), GetDisplayName(), parameters...)"),
("src/wallet/scriptpubkeyman.h", "WalletLogPrintf(std::string fmt, Params... parameters)"),
("src/wallet/scriptpubkeyman.h", "LogPrintf((\"%s \" + fmt).c_str(), m_storage.GetDisplayName(), parameters...)"),
("src/logging.h", "LogPrintf(const char* fmt, const Args&... args)"),
("src/wallet/scriptpubkeyman.h", "WalletLogPrintf(const std::string& fmt, const Params&... parameters)"),
]
def parse_function_calls(function_name, source_code):
"""Return an array with all calls to function function_name in string source_code.
Preprocessor directives and C++ style comments ("//") in source_code are removed.
>>> len(parse_function_calls("foo", "foo();bar();foo();bar();"))
2
>>> parse_function_calls("foo", "foo(1);bar(1);foo(2);bar(2);")[0].startswith("foo(1);")
True
>>> parse_function_calls("foo", "foo(1);bar(1);foo(2);bar(2);")[1].startswith("foo(2);")
True
>>> len(parse_function_calls("foo", "foo();bar();// foo();bar();"))
1
>>> len(parse_function_calls("foo", "#define FOO foo();"))
0
"""
assert type(function_name) is str and type(source_code) is str and function_name
lines = [re.sub("// .*", " ", line).strip()
for line in source_code.split("\n")
if not line.strip().startswith("#")]
return re.findall(r"[^a-zA-Z_](?=({}\(.*).*)".format(function_name), " " + " ".join(lines))
def normalize(s):
"""Return a normalized version of string s with newlines, tabs and C style comments ("/* ... */")
replaced with spaces. Multiple spaces are replaced with a single space.
>>> normalize(" /* nothing */ foo\tfoo /* bar */ foo ")
'foo foo foo'
"""
assert type(s) is str
s = s.replace("\n", " ")
s = s.replace("\t", " ")
s = re.sub(r"/\*.*?\*/", " ", s)
s = re.sub(" {2,}", " ", s)
return s.strip()
ESCAPE_MAP = {
r"\n": "[escaped-newline]",
r"\t": "[escaped-tab]",
r'\"': "[escaped-quote]",
}
def escape(s):
"""Return the escaped version of string s with "\\\"", "\\n" and "\\t" escaped as
"[escaped-backslash]", "[escaped-newline]" and "[escaped-tab]".
>>> unescape(escape("foo")) == "foo"
True
>>> escape(r'foo \\t foo \\n foo \\\\ foo \\ foo \\"bar\\"')
'foo [escaped-tab] foo [escaped-newline] foo \\\\\\\\ foo \\\\ foo [escaped-quote]bar[escaped-quote]'
"""
assert type(s) is str
for raw_value, escaped_value in ESCAPE_MAP.items():
s = s.replace(raw_value, escaped_value)
return s
def unescape(s):
"""Return the unescaped version of escaped string s.
Reverses the replacements made in function escape(s).
>>> unescape(escape("bar"))
'bar'
>>> unescape("foo [escaped-tab] foo [escaped-newline] foo \\\\\\\\ foo \\\\ foo [escaped-quote]bar[escaped-quote]")
'foo \\\\t foo \\\\n foo \\\\\\\\ foo \\\\ foo \\\\"bar\\\\"'
"""
assert type(s) is str
for raw_value, escaped_value in ESCAPE_MAP.items():
s = s.replace(escaped_value, raw_value)
return s
def parse_function_call_and_arguments(function_name, function_call):
"""Split string function_call into an array of strings consisting of:
* the string function_call followed by "("
* the function call argument #1
* ...
* the function call argument #n
* a trailing ");"
The strings returned are in escaped form. See escape(...).
>>> parse_function_call_and_arguments("foo", 'foo("%s", "foo");')
['foo(', '"%s",', ' "foo"', ')']
>>> parse_function_call_and_arguments("foo", 'foo("%s", "foo");')
['foo(', '"%s",', ' "foo"', ')']
>>> parse_function_call_and_arguments("foo", 'foo("%s %s", "foo", "bar");')
['foo(', '"%s %s",', ' "foo",', ' "bar"', ')']
>>> parse_function_call_and_arguments("fooprintf", 'fooprintf("%050d", i);')
['fooprintf(', '"%050d",', ' i', ')']
>>> parse_function_call_and_arguments("foo", 'foo(bar(foobar(barfoo("foo"))), foobar); barfoo')
['foo(', 'bar(foobar(barfoo("foo"))),', ' foobar', ')']
>>> parse_function_call_and_arguments("foo", "foo()")
['foo(', '', ')']
>>> parse_function_call_and_arguments("foo", "foo(123)")
['foo(', '123', ')']
>>> parse_function_call_and_arguments("foo", 'foo("foo")')
['foo(', '"foo"', ')']
>>> parse_function_call_and_arguments("strprintf", 'strprintf("%s (%d)", std::wstring_convert<std::codecvt_utf8_utf16<wchar_t>,wchar_t>().to_bytes(buf), err);')
['strprintf(', '"%s (%d)",', ' std::wstring_convert<std::codecvt_utf8_utf16<wchar_t>,wchar_t>().to_bytes(buf),', ' err', ')']
>>> parse_function_call_and_arguments("strprintf", 'strprintf("%s (%d)", foo<wchar_t>().to_bytes(buf), err);')
['strprintf(', '"%s (%d)",', ' foo<wchar_t>().to_bytes(buf),', ' err', ')']
>>> parse_function_call_and_arguments("strprintf", 'strprintf("%s (%d)", foo().to_bytes(buf), err);')
['strprintf(', '"%s (%d)",', ' foo().to_bytes(buf),', ' err', ')']
>>> parse_function_call_and_arguments("strprintf", 'strprintf("%s (%d)", foo << 1, err);')
['strprintf(', '"%s (%d)",', ' foo << 1,', ' err', ')']
>>> parse_function_call_and_arguments("strprintf", 'strprintf("%s (%d)", foo<bar>() >> 1, err);')
['strprintf(', '"%s (%d)",', ' foo<bar>() >> 1,', ' err', ')']
>>> parse_function_call_and_arguments("strprintf", 'strprintf("%s (%d)", foo < 1 ? bar : foobar, err);')
['strprintf(', '"%s (%d)",', ' foo < 1 ? bar : foobar,', ' err', ')']
>>> parse_function_call_and_arguments("strprintf", 'strprintf("%s (%d)", foo < 1, err);')
['strprintf(', '"%s (%d)",', ' foo < 1,', ' err', ')']
>>> parse_function_call_and_arguments("strprintf", 'strprintf("%s (%d)", foo > 1 ? bar : foobar, err);')
['strprintf(', '"%s (%d)",', ' foo > 1 ? bar : foobar,', ' err', ')']
>>> parse_function_call_and_arguments("strprintf", 'strprintf("%s (%d)", foo > 1, err);')
['strprintf(', '"%s (%d)",', ' foo > 1,', ' err', ')']
>>> parse_function_call_and_arguments("strprintf", 'strprintf("%s (%d)", foo <= 1, err);')
['strprintf(', '"%s (%d)",', ' foo <= 1,', ' err', ')']
>>> parse_function_call_and_arguments("strprintf", 'strprintf("%s (%d)", foo <= bar<1, 2>(1, 2), err);')
['strprintf(', '"%s (%d)",', ' foo <= bar<1, 2>(1, 2),', ' err', ')']
>>> parse_function_call_and_arguments("strprintf", 'strprintf("%s (%d)", foo>foo<1,2>(1,2)?bar:foobar,err)');
['strprintf(', '"%s (%d)",', ' foo>foo<1,2>(1,2)?bar:foobar,', 'err', ')']
>>> parse_function_call_and_arguments("strprintf", 'strprintf("%s (%d)", foo>foo<1,2>(1,2),err)');
['strprintf(', '"%s (%d)",', ' foo>foo<1,2>(1,2),', 'err', ')']
"""
assert type(function_name) is str and type(function_call) is str and function_name
remaining = normalize(escape(function_call))
expected_function_call = "{}(".format(function_name)
assert remaining.startswith(expected_function_call)
parts = [expected_function_call]
remaining = remaining[len(expected_function_call):]
open_parentheses = 1
open_template_arguments = 0
in_string = False
parts.append("")
for i, char in enumerate(remaining):
parts.append(parts.pop() + char)
if char == "\"":
in_string = not in_string
continue
if in_string:
continue
if char == "(":
open_parentheses += 1
continue
if char == ")":
open_parentheses -= 1
if open_parentheses > 1:
continue
if open_parentheses == 0:
parts.append(parts.pop()[:-1])
parts.append(char)
break
prev_char = remaining[i - 1] if i - 1 >= 0 else None
next_char = remaining[i + 1] if i + 1 <= len(remaining) - 1 else None
if char == "<" and next_char not in [" ", "<", "="] and prev_char not in [" ", "<"]:
open_template_arguments += 1
continue
if char == ">" and next_char not in [" ", ">", "="] and prev_char not in [" ", ">"] and open_template_arguments > 0:
open_template_arguments -= 1
if open_template_arguments > 0:
continue
if char == ",":
parts.append("")
return parts
def parse_string_content(argument):
"""Return the text within quotes in string argument.
>>> parse_string_content('1 "foo %d bar" 2')
'foo %d bar'
>>> parse_string_content('1 foobar 2')
''
>>> parse_string_content('1 "bar" 2')
'bar'
>>> parse_string_content('1 "foo" 2 "bar" 3')
'foobar'
>>> parse_string_content('1 "foo" 2 " " "bar" 3')
'foo bar'
>>> parse_string_content('""')
''
>>> parse_string_content('')
''
>>> parse_string_content('1 2 3')
''
"""
assert type(argument) is str
string_content = ""
in_string = False
for char in normalize(escape(argument)):
if char == "\"":
in_string = not in_string
elif in_string:
string_content += char
return string_content
def count_format_specifiers(format_string):
"""Return the number of format specifiers in string format_string.
>>> count_format_specifiers("foo bar foo")
0
>>> count_format_specifiers("foo %d bar foo")
1
>>> count_format_specifiers("foo %d bar %i foo")
2
>>> count_format_specifiers("foo %d bar %i foo %% foo")
2
>>> count_format_specifiers("foo %d bar %i foo %% foo %d foo")
3
>>> count_format_specifiers("foo %d bar %i foo %% foo %*d foo")
4
"""
assert type(format_string) is str
format_string = format_string.replace('%%', 'X')
n = 0
in_specifier = False
for i, char in enumerate(format_string):
if char == "%":
in_specifier = True
n += 1
elif char in "aAcdeEfFgGinopsuxX":
in_specifier = False
elif in_specifier and char == "*":
n += 1
return n
def main():
parser = argparse.ArgumentParser(description="This program checks that the number of arguments passed "
"to a variadic format string function matches the number of format "
"specifiers in the format string.")
parser.add_argument("--skip-arguments", type=int, help="number of arguments before the format string "
"argument (e.g. 1 in the case of fprintf)", default=0)
parser.add_argument("function_name", help="function name (e.g. fprintf)", default=None)
parser.add_argument("file", nargs="*", help="C++ source code file (e.g. foo.cpp)")
args = parser.parse_args()
exit_code = 0
for filename in args.file:
with open(filename, "r", encoding="utf-8") as f:
for function_call_str in parse_function_calls(args.function_name, f.read()):
parts = parse_function_call_and_arguments(args.function_name, function_call_str)
relevant_function_call_str = unescape("".join(parts))[:512]
if (f.name, relevant_function_call_str) in FALSE_POSITIVES:
continue
if len(parts) < 3 + args.skip_arguments:
exit_code = 1
print("{}: Could not parse function call string \"{}(...)\": {}".format(f.name, args.function_name, relevant_function_call_str))
continue
argument_count = len(parts) - 3 - args.skip_arguments
format_str = parse_string_content(parts[1 + args.skip_arguments])
format_specifier_count = count_format_specifiers(format_str)
if format_specifier_count != argument_count:
exit_code = 1
print("{}: Expected {} argument(s) after format string but found {} argument(s): {}".format(f.name, format_specifier_count, argument_count, relevant_function_call_str))
continue
sys.exit(exit_code)
if __name__ == "__main__":
main()

View file

@ -0,0 +1,45 @@
#!/usr/bin/env bash
#
# Copyright (c) 2009-2020 The Bitcoin Core developers
# Copyright (c) 2014-2020 The DigiByte Core developers
# Distributed under the MIT software license, see the accompanying
# file COPYING or http://www.opensource.org/licenses/mit-license.php.
#
# Lint format strings: This program checks that the number of arguments passed
# to a variadic format string function matches the number of format specifiers
# in the format string.
export LC_ALL=C
FUNCTION_NAMES_AND_NUMBER_OF_LEADING_ARGUMENTS=(
"FatalError,0"
"fprintf,1"
"tfm::format,1" # Assuming tfm::::format(std::ostream&, ...
"LogConnectFailure,1"
"LogPrint,1"
"LogPrintf,0"
"printf,0"
"snprintf,2"
"sprintf,1"
"strprintf,0"
"vfprintf,1"
"vprintf,1"
"vsnprintf,1"
"vsprintf,1"
"WalletLogPrintf,0"
)
EXIT_CODE=0
if ! python3 -m doctest test/lint/lint-format-strings.py; then
EXIT_CODE=1
fi
for S in "${FUNCTION_NAMES_AND_NUMBER_OF_LEADING_ARGUMENTS[@]}"; do
IFS="," read -r FUNCTION_NAME SKIP_ARGUMENTS <<< "${S}"
for MATCHING_FILE in $(git grep --full-name -l "${FUNCTION_NAME}" -- "*.c" "*.cpp" "*.h" | sort | grep -vE "^src/(leveldb|secp256k1|tinyformat|univalue|test/fuzz/strprintf.cpp)"); do
MATCHING_FILES+=("${MATCHING_FILE}")
done
if ! test/lint/lint-format-strings.py --skip-arguments "${SKIP_ARGUMENTS}" "${FUNCTION_NAME}" "${MATCHING_FILES[@]}"; then
EXIT_CODE=1
fi
done
exit ${EXIT_CODE}

View file

@ -0,0 +1,48 @@
#!/usr/bin/env bash
# Copyright (c) 2020 The DigiByte Core developers
# Distributed under the MIT software license, see the accompanying
# file COPYING or http://www.opensource.org/licenses/mit-license.php.
#
# Linter to check that commit messages have a new line before the body
# or no body at all
export LC_ALL=C
EXIT_CODE=0
while getopts "?" opt; do
case $opt in
?)
echo "Usage: $0 [N]"
echo " COMMIT_RANGE='<commit range>' $0"
echo " $0 -?"
echo "Checks unmerged commits, the previous N commits, or a commit range."
echo "COMMIT_RANGE='47ba2c3...ee50c9e' $0"
exit ${EXIT_CODE}
;;
esac
done
if [ -z "${COMMIT_RANGE}" ]; then
if [ -n "$1" ]; then
COMMIT_RANGE="HEAD~$1...HEAD"
else
# This assumes that the target branch of the pull request will be master.
MERGE_BASE=$(git merge-base HEAD master)
COMMIT_RANGE="$MERGE_BASE..HEAD"
fi
fi
while IFS= read -r commit_hash || [[ -n "$commit_hash" ]]; do
n_line=0
while IFS= read -r line || [[ -n "$line" ]]; do
n_line=$((n_line+1))
length=${#line}
if [ $n_line -eq 2 ] && [ $length -ne 0 ]; then
echo "The subject line of commit hash ${commit_hash} is followed by a non-empty line. Subject lines should always be followed by a blank line."
EXIT_CODE=1
fi
done < <(git log --format=%B -n 1 "$commit_hash")
done < <(git log "${COMMIT_RANGE}" --format=%H)
exit ${EXIT_CODE}

View file

@ -0,0 +1,31 @@
#!/usr/bin/env bash
#
# Copyright (c) 2009-2020 The Bitcoin Core developers
# Copyright (c) 2014-2020 The DigiByte Core developers
# Distributed under the MIT software license, see the accompanying
# file COPYING or http://www.opensource.org/licenses/mit-license.php.
#
# Check include guards.
export LC_ALL=C
HEADER_ID_PREFIX="DIGIBYTE_"
HEADER_ID_SUFFIX="_H"
REGEXP_EXCLUDE_FILES_WITH_PREFIX="src/(crypto/ctaes/|leveldb/|crc32c/|secp256k1/|test/fuzz/FuzzedDataProvider.h|tinyformat.h|bench/nanobench.h|univalue/)"
EXIT_CODE=0
for HEADER_FILE in $(git ls-files -- "*.h" | grep -vE "^${REGEXP_EXCLUDE_FILES_WITH_PREFIX}")
do
HEADER_ID_BASE=$(cut -f2- -d/ <<< "${HEADER_FILE}" | sed "s/\.h$//g" | tr / _ | tr - _ | tr "[:lower:]" "[:upper:]")
HEADER_ID="${HEADER_ID_PREFIX}${HEADER_ID_BASE}${HEADER_ID_SUFFIX}"
if [[ $(grep -cE "^#(ifndef|define) ${HEADER_ID}" "${HEADER_FILE}") != 2 ]]; then
echo "${HEADER_FILE} seems to be missing the expected include guard:"
echo " #ifndef ${HEADER_ID}"
echo " #define ${HEADER_ID}"
echo " ..."
echo " #endif // ${HEADER_ID}"
echo
EXIT_CODE=1
fi
done
exit ${EXIT_CODE}

105
test/lint/lint-includes.sh Executable file
View file

@ -0,0 +1,105 @@
#!/usr/bin/env bash
#
# Copyright (c) 2009-2020 The Bitcoin Core developers
# Copyright (c) 2014-2020 The DigiByte Core developers
# Distributed under the MIT software license, see the accompanying
# file COPYING or http://www.opensource.org/licenses/mit-license.php.
#
# Check for duplicate includes.
# Guard against accidental introduction of new Boost dependencies.
# Check includes: Check for duplicate includes. Enforce bracket syntax includes.
export LC_ALL=C
IGNORE_REGEXP="/(leveldb|secp256k1|univalue|crc32c)/"
# cd to root folder of git repo for git ls-files to work properly
cd "$(dirname $0)/../.." || exit 1
filter_suffix() {
git ls-files | grep -E "^src/.*\.${1}"'$' | grep -Ev "${IGNORE_REGEXP}"
}
EXIT_CODE=0
for HEADER_FILE in $(filter_suffix h); do
DUPLICATE_INCLUDES_IN_HEADER_FILE=$(grep -E "^#include " < "${HEADER_FILE}" | sort | uniq -d)
if [[ ${DUPLICATE_INCLUDES_IN_HEADER_FILE} != "" ]]; then
echo "Duplicate include(s) in ${HEADER_FILE}:"
echo "${DUPLICATE_INCLUDES_IN_HEADER_FILE}"
echo
EXIT_CODE=1
fi
done
for CPP_FILE in $(filter_suffix cpp); do
DUPLICATE_INCLUDES_IN_CPP_FILE=$(grep -E "^#include " < "${CPP_FILE}" | sort | uniq -d)
if [[ ${DUPLICATE_INCLUDES_IN_CPP_FILE} != "" ]]; then
echo "Duplicate include(s) in ${CPP_FILE}:"
echo "${DUPLICATE_INCLUDES_IN_CPP_FILE}"
echo
EXIT_CODE=1
fi
done
INCLUDED_CPP_FILES=$(git grep -E "^#include [<\"][^>\"]+\.cpp[>\"]" -- "*.cpp" "*.h")
if [[ ${INCLUDED_CPP_FILES} != "" ]]; then
echo "The following files #include .cpp files:"
echo "${INCLUDED_CPP_FILES}"
echo
EXIT_CODE=1
fi
EXPECTED_BOOST_INCLUDES=(
boost/algorithm/string.hpp
boost/algorithm/string/classification.hpp
boost/algorithm/string/replace.hpp
boost/algorithm/string/split.hpp
boost/date_time/posix_time/posix_time.hpp
boost/filesystem.hpp
boost/filesystem/fstream.hpp
boost/multi_index/hashed_index.hpp
boost/multi_index/ordered_index.hpp
boost/multi_index/sequenced_index.hpp
boost/multi_index_container.hpp
boost/process.hpp
boost/signals2/connection.hpp
boost/signals2/optional_last_value.hpp
boost/signals2/signal.hpp
boost/test/unit_test.hpp
)
for BOOST_INCLUDE in $(git grep '^#include <boost/' -- "*.cpp" "*.h" | cut -f2 -d: | cut -f2 -d'<' | cut -f1 -d'>' | sort -u); do
IS_EXPECTED_INCLUDE=0
for EXPECTED_BOOST_INCLUDE in "${EXPECTED_BOOST_INCLUDES[@]}"; do
if [[ "${BOOST_INCLUDE}" == "${EXPECTED_BOOST_INCLUDE}" ]]; then
IS_EXPECTED_INCLUDE=1
break
fi
done
if [[ ${IS_EXPECTED_INCLUDE} == 0 ]]; then
EXIT_CODE=1
echo "A new Boost dependency in the form of \"${BOOST_INCLUDE}\" appears to have been introduced:"
git grep "${BOOST_INCLUDE}" -- "*.cpp" "*.h"
echo
fi
done
for EXPECTED_BOOST_INCLUDE in "${EXPECTED_BOOST_INCLUDES[@]}"; do
if ! git grep -q "^#include <${EXPECTED_BOOST_INCLUDE}>" -- "*.cpp" "*.h"; then
echo "Good job! The Boost dependency \"${EXPECTED_BOOST_INCLUDE}\" is no longer used."
echo "Please remove it from EXPECTED_BOOST_INCLUDES in $0"
echo "to make sure this dependency is not accidentally reintroduced."
echo
EXIT_CODE=1
fi
done
QUOTE_SYNTAX_INCLUDES=$(git grep '^#include "' -- "*.cpp" "*.h" | grep -Ev "${IGNORE_REGEXP}")
if [[ ${QUOTE_SYNTAX_INCLUDES} != "" ]]; then
echo "Please use bracket syntax includes (\"#include <foo.h>\") instead of quote syntax includes:"
echo "${QUOTE_SYNTAX_INCLUDES}"
echo
EXIT_CODE=1
fi
exit ${EXIT_CODE}

View file

@ -0,0 +1,253 @@
#!/usr/bin/env bash
# Copyright (c) 2018-2020 The DigiByte Core developers
# Distributed under the MIT software license, see the accompanying
# file COPYING or http://www.opensource.org/licenses/mit-license.php.
export LC_ALL=C
# Be aware that digibyted and digibyte-qt differ in terms of localization: Qt
# opts in to POSIX localization by running setlocale(LC_ALL, "") on startup,
# whereas no such call is made in digibyted.
#
# Qt runs setlocale(LC_ALL, "") on initialization. This installs the locale
# specified by the user's LC_ALL (or LC_*) environment variable as the new
# C locale.
#
# In contrast, digibyted does not opt in to localization -- no call to
# setlocale(LC_ALL, "") is made and the environment variables LC_* are
# thus ignored.
#
# This results in situations where digibyted is guaranteed to be running
# with the classic locale ("C") whereas the locale of digibyte-qt will vary
# depending on the user's environment variables.
#
# An example: Assuming the environment variable LC_ALL=de_DE then the
# call std::to_string(1.23) will return "1.230000" in digibyted but
# "1,230000" in digibyte-qt.
#
# From the Qt documentation:
# "On Unix/Linux Qt is configured to use the system locale settings by default.
# This can cause a conflict when using POSIX functions, for instance, when
# converting between data types such as floats and strings, since the notation
# may differ between locales. To get around this problem, call the POSIX function
# setlocale(LC_NUMERIC,"C") right after initializing QApplication, QGuiApplication
# or QCoreApplication to reset the locale that is used for number formatting to
# "C"-locale."
#
# See https://doc.qt.io/qt-5/qcoreapplication.html#locale-settings and
# https://stackoverflow.com/a/34878283 for more details.
KNOWN_VIOLATIONS=(
"src/digibyte-tx.cpp.*stoul"
"src/digibyte-tx.cpp.*trim_right"
"src/dbwrapper.cpp.*stoul"
"src/dbwrapper.cpp:.*vsnprintf"
"src/httprpc.cpp.*trim"
"src/node/blockstorage.cpp:.*atoi"
"src/qt/rpcconsole.cpp:.*atoi"
"src/rest.cpp:.*strtol"
"src/test/dbwrapper_tests.cpp:.*snprintf"
"src/test/fuzz/locale.cpp"
"src/test/fuzz/parse_numbers.cpp:.*atoi"
"src/torcontrol.cpp:.*atoi"
"src/torcontrol.cpp:.*strtol"
"src/util/strencodings.cpp:.*atoi"
"src/util/strencodings.cpp:.*strtol"
"src/util/strencodings.cpp:.*strtoul"
"src/util/strencodings.h:.*atoi"
"src/util/system.cpp:.*atoi"
)
REGEXP_IGNORE_EXTERNAL_DEPENDENCIES="^src/(crypto/ctaes/|leveldb/|secp256k1/|tinyformat.h|univalue/)"
LOCALE_DEPENDENT_FUNCTIONS=(
alphasort # LC_COLLATE (via strcoll)
asctime # LC_TIME (directly)
asprintf # (via vasprintf)
atof # LC_NUMERIC (via strtod)
atoi # LC_NUMERIC (via strtol)
atol # LC_NUMERIC (via strtol)
atoll # (via strtoll)
atoq
btowc # LC_CTYPE (directly)
ctime # (via asctime or localtime)
dprintf # (via vdprintf)
fgetwc
fgetws
fold_case # boost::locale::fold_case
fprintf # (via vfprintf)
fputwc
fputws
fscanf # (via __vfscanf)
fwprintf # (via __vfwprintf)
getdate # via __getdate_r => isspace // __localtime_r
getwc
getwchar
is_digit # boost::algorithm::is_digit
is_space # boost::algorithm::is_space
isalnum # LC_CTYPE
isalpha # LC_CTYPE
isblank # LC_CTYPE
iscntrl # LC_CTYPE
isctype # LC_CTYPE
isdigit # LC_CTYPE
isgraph # LC_CTYPE
islower # LC_CTYPE
isprint # LC_CTYPE
ispunct # LC_CTYPE
isspace # LC_CTYPE
isupper # LC_CTYPE
iswalnum # LC_CTYPE
iswalpha # LC_CTYPE
iswblank # LC_CTYPE
iswcntrl # LC_CTYPE
iswctype # LC_CTYPE
iswdigit # LC_CTYPE
iswgraph # LC_CTYPE
iswlower # LC_CTYPE
iswprint # LC_CTYPE
iswpunct # LC_CTYPE
iswspace # LC_CTYPE
iswupper # LC_CTYPE
iswxdigit # LC_CTYPE
isxdigit # LC_CTYPE
localeconv # LC_NUMERIC + LC_MONETARY
mblen # LC_CTYPE
mbrlen
mbrtowc
mbsinit
mbsnrtowcs
mbsrtowcs
mbstowcs # LC_CTYPE
mbtowc # LC_CTYPE
mktime
normalize # boost::locale::normalize
printf # LC_NUMERIC
putwc
putwchar
scanf # LC_NUMERIC
setlocale
snprintf
sprintf
sscanf
std::locale::global
std::to_string
stod
stof
stoi
stol
stold
stoll
stoul
stoull
strcasecmp
strcasestr
strcoll # LC_COLLATE
# strerror
strfmon
strftime # LC_TIME
strncasecmp
strptime
strtod # LC_NUMERIC
strtof
strtoimax
strtol # LC_NUMERIC
strtold
strtoll
strtoq
strtoul # LC_NUMERIC
strtoull
strtoumax
strtouq
strxfrm # LC_COLLATE
swprintf
to_lower # boost::locale::to_lower
to_title # boost::locale::to_title
to_upper # boost::locale::to_upper
tolower # LC_CTYPE
toupper # LC_CTYPE
towctrans
towlower # LC_CTYPE
towupper # LC_CTYPE
trim # boost::algorithm::trim
trim_left # boost::algorithm::trim_left
trim_right # boost::algorithm::trim_right
ungetwc
vasprintf
vdprintf
versionsort
vfprintf
vfscanf
vfwprintf
vprintf
vscanf
vsnprintf
vsprintf
vsscanf
vswprintf
vwprintf
wcrtomb
wcscasecmp
wcscoll # LC_COLLATE
wcsftime # LC_TIME
wcsncasecmp
wcsnrtombs
wcsrtombs
wcstod # LC_NUMERIC
wcstof
wcstoimax
wcstol # LC_NUMERIC
wcstold
wcstoll
wcstombs # LC_CTYPE
wcstoul # LC_NUMERIC
wcstoull
wcstoumax
wcswidth
wcsxfrm # LC_COLLATE
wctob
wctomb # LC_CTYPE
wctrans
wctype
wcwidth
wprintf
)
function join_array {
local IFS="$1"
shift
echo "$*"
}
REGEXP_IGNORE_KNOWN_VIOLATIONS=$(join_array "|" "${KNOWN_VIOLATIONS[@]}")
# Invoke "git grep" only once in order to minimize run-time
REGEXP_LOCALE_DEPENDENT_FUNCTIONS=$(join_array "|" "${LOCALE_DEPENDENT_FUNCTIONS[@]}")
GIT_GREP_OUTPUT=$(git grep -E "[^a-zA-Z0-9_\`'\"<>](${REGEXP_LOCALE_DEPENDENT_FUNCTIONS}(_r|_s)?)[^a-zA-Z0-9_\`'\"<>]" -- "*.cpp" "*.h")
EXIT_CODE=0
for LOCALE_DEPENDENT_FUNCTION in "${LOCALE_DEPENDENT_FUNCTIONS[@]}"; do
MATCHES=$(grep -E "[^a-zA-Z0-9_\`'\"<>]${LOCALE_DEPENDENT_FUNCTION}(_r|_s)?[^a-zA-Z0-9_\`'\"<>]" <<< "${GIT_GREP_OUTPUT}" | \
grep -vE "\.(c|cpp|h):\s*(//|\*|/\*|\").*${LOCALE_DEPENDENT_FUNCTION}")
if [[ ${REGEXP_IGNORE_EXTERNAL_DEPENDENCIES} != "" ]]; then
MATCHES=$(grep -vE "${REGEXP_IGNORE_EXTERNAL_DEPENDENCIES}" <<< "${MATCHES}")
fi
if [[ ${REGEXP_IGNORE_KNOWN_VIOLATIONS} != "" ]]; then
MATCHES=$(grep -vE "${REGEXP_IGNORE_KNOWN_VIOLATIONS}" <<< "${MATCHES}")
fi
if [[ ${MATCHES} != "" ]]; then
echo "The locale dependent function ${LOCALE_DEPENDENT_FUNCTION}(...) appears to be used:"
echo "${MATCHES}"
echo
EXIT_CODE=1
fi
done
if [[ ${EXIT_CODE} != 0 ]]; then
echo "Unnecessary locale dependence can cause bugs that are very"
echo "tricky to isolate and fix. Please avoid using locale dependent"
echo "functions if possible."
echo
echo "Advice not applicable in this specific case? Add an exception"
echo "by updating the ignore list in $0"
fi
exit ${EXIT_CODE}

29
test/lint/lint-logs.sh Executable file
View file

@ -0,0 +1,29 @@
#!/usr/bin/env bash
#
# Copyright (c) 2009-2020 The Bitcoin Core developers
# Copyright (c) 2014-2020 The DigiByte Core developers
# Distributed under the MIT software license, see the accompanying
# file COPYING or http://www.opensource.org/licenses/mit-license.php.
#
# Check that all logs are terminated with '\n'
#
# Some logs are continued over multiple lines. They should be explicitly
# commented with \* Continued *\
#
# There are some instances of LogPrintf() in comments. Those can be
# ignored
export LC_ALL=C
UNTERMINATED_LOGS=$(git grep --extended-regexp "LogPrintf?\(" -- "*.cpp" | \
grep -v '\\n"' | \
grep -v '\.\.\.' | \
grep -v "/\* Continued \*/" | \
grep -v "LogPrint()" | \
grep -v "LogPrintf()")
if [[ ${UNTERMINATED_LOGS} != "" ]]; then
# shellcheck disable=SC2028
echo "All calls to LogPrintf() and LogPrint() should be terminated with \\n"
echo
echo "${UNTERMINATED_LOGS}"
exit 1
fi

View file

@ -0,0 +1,23 @@
#!/usr/bin/env bash
#
# Copyright (c) 2021 The DigiByte Core developers
# Distributed under the MIT software license, see the accompanying
# file COPYING or http://www.opensource.org/licenses/mit-license.php.
#
# Find dead Python code.
export LC_ALL=C
if ! command -v vulture > /dev/null; then
echo "Skipping Python dead code linting since vulture is not installed. Install by running \"pip3 install vulture\""
exit 0
fi
# --min-confidence 100 will only report code that is guaranteed to be unused within the analyzed files.
# Any value below 100 introduces the risk of false positives, which would create an unacceptable maintenance burden.
if ! vulture \
--min-confidence 100 \
$(git ls-files -- "*.py"); then
echo "Python dead code detection found some issues"
exit 1
fi

View file

@ -0,0 +1,52 @@
#!/usr/bin/env bash
#
# Copyright (c) 2019 The DigiByte Core developers
# Distributed under the MIT software license, see the accompanying
# file COPYING or http://www.opensource.org/licenses/mit-license.php.
#
# Detect when a mutable list or dict is used as a default parameter value in a Python function.
export LC_ALL=C
EXIT_CODE=0
OUTPUT=$(git grep -E '^\s*def [a-zA-Z0-9_]+\(.*=\s*(\[|\{)' -- "*.py")
if [[ ${OUTPUT} != "" ]]; then
echo "A mutable list or dict seems to be used as default parameter value:"
echo
echo "${OUTPUT}"
echo
cat << EXAMPLE
This is how mutable list and dict default parameter values behave:
>>> def f(i, j=[], k={}):
... j.append(i)
... k[i] = True
... return j, k
...
>>> f(1)
([1], {1: True})
>>> f(1)
([1, 1], {1: True})
>>> f(2)
([1, 1, 2], {1: True, 2: True})
The intended behaviour was likely:
>>> def f(i, j=None, k=None):
... if j is None:
... j = []
... if k is None:
... k = {}
... j.append(i)
... k[i] = True
... return j, k
...
>>> f(1)
([1], {1: True})
>>> f(1)
([1], {1: True})
>>> f(2)
([2], {2: True})
EXAMPLE
EXIT_CODE=1
fi
exit ${EXIT_CODE}

View file

@ -0,0 +1,29 @@
#!/usr/bin/env bash
#
# Copyright (c) 2009-2020 The Bitcoin Core developers
# Copyright (c) 2014-2020 The DigiByte Core developers
# Distributed under the MIT software license, see the accompanying
# file COPYING or http://www.opensource.org/licenses/mit-license.php.
#
# Make sure we explicitly open all text files using UTF-8 (or ASCII) encoding to
# avoid potential issues on the BSDs where the locale is not always set.
export LC_ALL=C
EXIT_CODE=0
OUTPUT=$(git grep " open(" -- "*.py" ":(exclude)src/crc32c/" | grep -vE "encoding=.(ascii|utf8|utf-8)." | grep -vE "open\([^,]*, ['\"][^'\"]*b[^'\"]*['\"]")
if [[ ${OUTPUT} != "" ]]; then
echo "Python's open(...) seems to be used to open text files without explicitly"
echo "specifying encoding=\"utf8\":"
echo
echo "${OUTPUT}"
EXIT_CODE=1
fi
OUTPUT=$(git grep "check_output(" -- "*.py" ":(exclude)src/crc32c/"| grep "universal_newlines=True" | grep -vE "encoding=.(ascii|utf8|utf-8).")
if [[ ${OUTPUT} != "" ]]; then
echo "Python's check_output(...) seems to be used to get program outputs without explicitly"
echo "specifying encoding=\"utf8\":"
echo
echo "${OUTPUT}"
EXIT_CODE=1
fi
exit ${EXIT_CODE}

110
test/lint/lint-python.sh Executable file
View file

@ -0,0 +1,110 @@
#!/usr/bin/env bash
#
# Copyright (c) 2009-2020 The Bitcoin Core developers
# Copyright (c) 2014-2020 The DigiByte Core developers
# Distributed under the MIT software license, see the accompanying
# file COPYING or http://www.opensource.org/licenses/mit-license.php.
#
# Check for specified flake8 warnings in python files.
export LC_ALL=C
export MYPY_CACHE_DIR="${BASE_ROOT_DIR}/test/.mypy_cache"
enabled=(
E101 # indentation contains mixed spaces and tabs
E112 # expected an indented block
E113 # unexpected indentation
E115 # expected an indented block (comment)
E116 # unexpected indentation (comment)
E125 # continuation line with same indent as next logical line
E129 # visually indented line with same indent as next logical line
E131 # continuation line unaligned for hanging indent
E133 # closing bracket is missing indentation
E223 # tab before operator
E224 # tab after operator
E242 # tab after ','
E266 # too many leading '#' for block comment
E271 # multiple spaces after keyword
E272 # multiple spaces before keyword
E273 # tab after keyword
E274 # tab before keyword
E275 # missing whitespace after keyword
E304 # blank lines found after function decorator
E306 # expected 1 blank line before a nested definition
E401 # multiple imports on one line
E402 # module level import not at top of file
E502 # the backslash is redundant between brackets
E701 # multiple statements on one line (colon)
E702 # multiple statements on one line (semicolon)
E703 # statement ends with a semicolon
E711 # comparison to None should be 'if cond is None:'
E714 # test for object identity should be "is not"
E721 # do not compare types, use "isinstance()"
E742 # do not define classes named "l", "O", or "I"
E743 # do not define functions named "l", "O", or "I"
E901 # SyntaxError: invalid syntax
E902 # TokenError: EOF in multi-line string
F401 # module imported but unused
F402 # import module from line N shadowed by loop variable
F403 # 'from foo_module import *' used; unable to detect undefined names
F404 # future import(s) name after other statements
F405 # foo_function may be undefined, or defined from star imports: bar_module
F406 # "from module import *" only allowed at module level
F407 # an undefined __future__ feature name was imported
F601 # dictionary key name repeated with different values
F602 # dictionary key variable name repeated with different values
F621 # too many expressions in an assignment with star-unpacking
F622 # two or more starred expressions in an assignment (a, *b, *c = d)
F631 # assertion test is a tuple, which are always True
F632 # use ==/!= to compare str, bytes, and int literals
F701 # a break statement outside of a while or for loop
F702 # a continue statement outside of a while or for loop
F703 # a continue statement in a finally block in a loop
F704 # a yield or yield from statement outside of a function
F705 # a return statement with arguments inside a generator
F706 # a return statement outside of a function/method
F707 # an except: block as not the last exception handler
F811 # redefinition of unused name from line N
F812 # list comprehension redefines 'foo' from line N
F821 # undefined name 'Foo'
F822 # undefined name name in __all__
F823 # local variable name … referenced before assignment
F831 # duplicate argument name in function definition
F841 # local variable 'foo' is assigned to but never used
W191 # indentation contains tabs
W291 # trailing whitespace
W292 # no newline at end of file
W293 # blank line contains whitespace
W601 # .has_key() is deprecated, use "in"
W602 # deprecated form of raising exception
W603 # "<>" is deprecated, use "!="
W604 # backticks are deprecated, use "repr()"
W605 # invalid escape sequence "x"
W606 # 'async' and 'await' are reserved keywords starting with Python 3.7
)
if ! command -v flake8 > /dev/null; then
echo "Skipping Python linting since flake8 is not installed."
exit 0
elif PYTHONWARNINGS="ignore" flake8 --version | grep -q "Python 2"; then
echo "Skipping Python linting since flake8 is running under Python 2. Install the Python 3 version of flake8."
exit 0
fi
EXIT_CODE=0
if ! PYTHONWARNINGS="ignore" flake8 --ignore=B,C,E,F,I,N,W --select=$(IFS=","; echo "${enabled[*]}") $(
if [[ $# == 0 ]]; then
git ls-files "*.py"
else
echo "$@"
fi
); then
EXIT_CODE=1
fi
if ! mypy --ignore-missing-imports --show-error-codes $(git ls-files "test/functional/*.py" "contrib/devtools/*.py"); then
EXIT_CODE=1
fi
exit $EXIT_CODE

20
test/lint/lint-qt.sh Executable file
View file

@ -0,0 +1,20 @@
#!/usr/bin/env bash
#
# Copyright (c) 2018 The DigiByte Core developers
# Distributed under the MIT software license, see the accompanying
# file COPYING or http://www.opensource.org/licenses/mit-license.php.
#
# Check for SIGNAL/SLOT connect style, removed since Qt4 support drop.
export LC_ALL=C
EXIT_CODE=0
OUTPUT=$(git grep -E '(SIGNAL|, ?SLOT)\(' -- src/qt)
if [[ ${OUTPUT} != "" ]]; then
echo "Use Qt5 connect style in:"
echo "$OUTPUT"
EXIT_CODE=1
fi
exit ${EXIT_CODE}

25
test/lint/lint-shell-locale.sh Executable file
View file

@ -0,0 +1,25 @@
#!/usr/bin/env bash
#
# Copyright (c) 2018 The DigiByte Core developers
# Distributed under the MIT software license, see the accompanying
# file COPYING or http://www.opensource.org/licenses/mit-license.php.
#
# Make sure all shell scripts:
# a.) explicitly opt out of locale dependence using
# "export LC_ALL=C" or "export LC_ALL=C.UTF-8", or
# b.) explicitly opt in to locale dependence using the annotation below.
export LC_ALL=C
EXIT_CODE=0
for SHELL_SCRIPT in $(git ls-files -- "*.sh" | grep -vE "src/(secp256k1|univalue)/"); do
if grep -q "# This script is intentionally locale dependent by not setting \"export LC_ALL=C\"" "${SHELL_SCRIPT}"; then
continue
fi
FIRST_NON_COMMENT_LINE=$(grep -vE '^(#.*)?$' "${SHELL_SCRIPT}" | head -1)
if [[ ${FIRST_NON_COMMENT_LINE} != "export LC_ALL=C" && ${FIRST_NON_COMMENT_LINE} != "export LC_ALL=C.UTF-8" ]]; then
echo "Missing \"export LC_ALL=C\" (to avoid locale dependence) as first non-comment non-empty line in ${SHELL_SCRIPT}"
EXIT_CODE=1
fi
done
exit ${EXIT_CODE}

55
test/lint/lint-shell.sh Executable file
View file

@ -0,0 +1,55 @@
#!/usr/bin/env bash
#
# Copyright (c) 2009-2020 The Bitcoin Core developers
# Copyright (c) 2014-2020 The DigiByte Core developers
# Distributed under the MIT software license, see the accompanying
# file COPYING or http://www.opensource.org/licenses/mit-license.php.
#
# Check for shellcheck warnings in shell scripts.
export LC_ALL=C
# Disabled warnings:
disabled=(
SC2046 # Quote this to prevent word splitting.
SC2086 # Double quote to prevent globbing and word splitting.
SC2162 # read without -r will mangle backslashes.
)
disabled_gitian=(
SC2094 # Make sure not to read and write the same file in the same pipeline.
SC2129 # Consider using { cmd1; cmd2; } >> file instead of individual redirects.
)
EXIT_CODE=0
if ! command -v shellcheck > /dev/null; then
echo "Skipping shell linting since shellcheck is not installed."
exit $EXIT_CODE
fi
SHELLCHECK_CMD=(shellcheck --external-sources --check-sourced)
EXCLUDE="--exclude=$(IFS=','; echo "${disabled[*]}")"
SOURCED_FILES=$(git ls-files | xargs gawk '/^# shellcheck shell=/ {print FILENAME} {nextfile}') # Check shellcheck directive used for sourced files
if ! "${SHELLCHECK_CMD[@]}" "$EXCLUDE" $SOURCED_FILES $(git ls-files -- '*.sh' | grep -vE 'src/(leveldb|secp256k1|univalue)/'); then
EXIT_CODE=1
fi
if ! command -v yq > /dev/null; then
echo "Skipping Gitian descriptor scripts checking since yq is not installed."
exit $EXIT_CODE
fi
EXCLUDE_GITIAN=${EXCLUDE}",$(IFS=','; echo "${disabled_gitian[*]}")"
for descriptor in $(git ls-files -- 'contrib/gitian-descriptors/*.yml')
do
script=$(basename "$descriptor")
# Use #!/bin/bash as gitian-builder/bin/gbuild does to complete a script.
echo "#!/bin/bash" > $script
yq -r .script "$descriptor" >> $script
if ! "${SHELLCHECK_CMD[@]}" "$EXCLUDE_GITIAN" $script; then
EXIT_CODE=1
fi
rm $script
done
exit $EXIT_CODE

View file

@ -0,0 +1,17 @@
asend
blockin
cachable
fo
fpr
hights
hist
inout
invokable
keypair
mor
nin
ser
unparseable
unser
useable
wit

20
test/lint/lint-spelling.sh Executable file
View file

@ -0,0 +1,20 @@
#!/usr/bin/env bash
#
# Copyright (c) 2018-2019 The DigiByte Core developers
# Distributed under the MIT software license, see the accompanying
# file COPYING or http://www.opensource.org/licenses/mit-license.php.
#
# Warn in case of spelling errors.
# Note: Will exit successfully regardless of spelling errors.
export LC_ALL=C
if ! command -v codespell > /dev/null; then
echo "Skipping spell check linting since codespell is not installed."
exit 0
fi
IGNORE_WORDS_FILE=test/lint/lint-spelling.ignore-words.txt
if ! codespell --check-filenames --disable-colors --quiet-level=7 --ignore-words=${IGNORE_WORDS_FILE} $(git ls-files -- ":(exclude)build-aux/m4/" ":(exclude)contrib/seeds/*.txt" ":(exclude)depends/" ":(exclude)doc/release-notes/" ":(exclude)src/leveldb/" ":(exclude)src/crc32c/" ":(exclude)src/qt/locale/" ":(exclude)src/qt/*.qrc" ":(exclude)src/secp256k1/" ":(exclude)src/univalue/" ":(exclude)contrib/builder-keys/keys.txt" ":(exclude)contrib/guix/patches"); then
echo "^ Warning: codespell identified likely spelling errors. Any false positives? Add them to the list of ignored words in ${IGNORE_WORDS_FILE}"
fi

20
test/lint/lint-submodule.sh Executable file
View file

@ -0,0 +1,20 @@
#!/usr/bin/env bash
#
# Copyright (c) 2020 The DigiByte Core developers
# Distributed under the MIT software license, see the accompanying
# file COPYING or http://www.opensource.org/licenses/mit-license.php.
#
# This script checks for git modules
export LC_ALL=C
EXIT_CODE=0
CMD=$(git submodule status --recursive)
if test -n "$CMD";
then
echo These submodules were found, delete them:
echo "$CMD"
EXIT_CODE=1
fi
exit $EXIT_CODE

35
test/lint/lint-tests.sh Executable file
View file

@ -0,0 +1,35 @@
#!/usr/bin/env bash
#
# Copyright (c) 2018 The DigiByte Core developers
# Distributed under the MIT software license, see the accompanying
# file COPYING or http://www.opensource.org/licenses/mit-license.php.
#
# Check the test suite naming conventions
export LC_ALL=C
EXIT_CODE=0
NAMING_INCONSISTENCIES=$(git grep -E '^BOOST_FIXTURE_TEST_SUITE\(' -- \
"src/test/**.cpp" "src/wallet/test/**.cpp" | \
grep -vE '/(.*?)\.cpp:BOOST_FIXTURE_TEST_SUITE\(\1, .*\)$')
if [[ ${NAMING_INCONSISTENCIES} != "" ]]; then
echo "The test suite in file src/test/foo_tests.cpp should be named"
echo "\"foo_tests\". Please make sure the following test suites follow"
echo "that convention:"
echo
echo "${NAMING_INCONSISTENCIES}"
EXIT_CODE=1
fi
TEST_SUITE_NAME_COLLISSIONS=$(git grep -E '^BOOST_FIXTURE_TEST_SUITE\(' -- \
"src/test/**.cpp" "src/wallet/test/**.cpp" | cut -f2 -d'(' | cut -f1 -d, | \
sort | uniq -d)
if [[ ${TEST_SUITE_NAME_COLLISSIONS} != "" ]]; then
echo "Test suite names must be unique. The following test suite names"
echo "appear to be used more than once:"
echo
echo "${TEST_SUITE_NAME_COLLISSIONS}"
EXIT_CODE=1
fi
exit ${EXIT_CODE}

116
test/lint/lint-whitespace.sh Executable file
View file

@ -0,0 +1,116 @@
#!/usr/bin/env bash
#
# Copyright (c) 2009-2020 The Bitcoin Core developers
# Copyright (c) 2014-2020 The DigiByte Core developers
# Distributed under the MIT software license, see the accompanying
# file COPYING or http://www.opensource.org/licenses/mit-license.php.
#
# Check for new lines in diff that introduce trailing whitespace.
# We can't run this check unless we know the commit range for the PR.
export LC_ALL=C
while getopts "?" opt; do
case $opt in
?)
echo "Usage: $0 [N]"
echo " COMMIT_RANGE='<commit range>' $0"
echo " $0 -?"
echo "Checks unstaged changes, the previous N commits, or a commit range."
echo "COMMIT_RANGE='47ba2c3...ee50c9e' $0"
exit 0
;;
esac
done
if [ -z "${COMMIT_RANGE}" ]; then
if [ -n "$1" ]; then
COMMIT_RANGE="HEAD~$1...HEAD"
else
# This assumes that the target branch of the pull request will be master.
MERGE_BASE=$(git merge-base HEAD master)
COMMIT_RANGE="$MERGE_BASE..HEAD"
fi
fi
showdiff() {
if ! git diff -U0 "${COMMIT_RANGE}" -- "." ":(exclude)depends/patches/" ":(exclude)contrib/guix/patches/" ":(exclude)src/leveldb/" ":(exclude)src/crc32c/" ":(exclude)src/secp256k1/" ":(exclude)src/univalue/" ":(exclude)doc/release-notes/" ":(exclude)src/qt/locale/"; then
echo "Failed to get a diff"
exit 1
fi
}
showcodediff() {
if ! git diff -U0 "${COMMIT_RANGE}" -- *.cpp *.h *.md *.py *.sh ":(exclude)src/leveldb/" ":(exclude)src/crc32c/" ":(exclude)src/secp256k1/" ":(exclude)src/univalue/" ":(exclude)doc/release-notes/" ":(exclude)src/qt/locale/"; then
echo "Failed to get a diff"
exit 1
fi
}
RET=0
# Check if trailing whitespace was found in the diff.
if showdiff | grep -E -q '^\+.*\s+$'; then
echo "This diff appears to have added new lines with trailing whitespace."
echo "The following changes were suspected:"
FILENAME=""
SEEN=0
SEENLN=0
while read -r line; do
if [[ "$line" =~ ^diff ]]; then
FILENAME="$line"
SEEN=0
elif [[ "$line" =~ ^@@ ]]; then
LINENUMBER="$line"
SEENLN=0
else
if [ "$SEEN" -eq 0 ]; then
# The first time a file is seen with trailing whitespace, we print the
# filename (preceded by a newline).
echo
echo "$FILENAME"
SEEN=1
fi
if [ "$SEENLN" -eq 0 ]; then
echo "$LINENUMBER"
SEENLN=1
fi
echo "$line"
fi
done < <(showdiff | grep -E '^(diff --git |@@|\+.*\s+$)')
RET=1
fi
# Check if tab characters were found in the diff.
if showcodediff | perl -nle '$MATCH++ if m{^\+.*\t}; END{exit 1 unless $MATCH>0}' > /dev/null; then
echo "This diff appears to have added new lines with tab characters instead of spaces."
echo "The following changes were suspected:"
FILENAME=""
SEEN=0
SEENLN=0
while read -r line; do
if [[ "$line" =~ ^diff ]]; then
FILENAME="$line"
SEEN=0
elif [[ "$line" =~ ^@@ ]]; then
LINENUMBER="$line"
SEENLN=0
else
if [ "$SEEN" -eq 0 ]; then
# The first time a file is seen with a tab character, we print the
# filename (preceded by a newline).
echo
echo "$FILENAME"
SEEN=1
fi
if [ "$SEENLN" -eq 0 ]; then
echo "$LINENUMBER"
SEENLN=1
fi
echo "$line"
fi
done < <(showcodediff | perl -nle 'print if m{^(diff --git |@@|\+.*\t)}')
RET=1
fi
exit $RET