diff --git a/.travis.yml b/.travis.yml index 95ef36bf028ee..64227ac2a883b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,18 +1,17 @@ -# errata: -# - A travis bug causes caches to trample eachother when using the same -# compiler key (which we don't use anyway). This is worked around for now by -# replacing the "compilers" with a build name prefixed by the no-op ":" -# command. See: https://github.com/travis-ci/travis-ci/issues/4393 -# - sudo/dist/group are set so as to get Blue Box VMs, necessary for [loopback] -# IPv6 support - sudo: required -dist: precise -group: legacy +dist: trusty + +#workaround for https://github.com/travis-ci/travis-ci/issues/5227 +addons: + hostname: bitcoin-tester os: linux -language: cpp -compiler: gcc +language: generic +cache: + directories: + - depends/built + - depends/sdk-sources + - $HOME/.ccache env: global: - MAKEJOBS=-j3 @@ -26,34 +25,27 @@ env: - SDK_URL=https://bitcoincore.org/depends-sources/sdks - PYTHON_DEBUG=1 - WINEDEBUG=fixme-all -cache: - apt: true - directories: - - depends/built - - depends/sdk-sources - - $HOME/.ccache -matrix: - fast_finish: true - include: - - compiler: ": ARM" - env: HOST=arm-linux-gnueabihf PACKAGES="g++-arm-linux-gnueabihf" DEP_OPTS="NO_QT=1" CHECK_DOC=1 GOAL="install" BITCOIN_CONFIG="--enable-glibc-back-compat --enable-reduce-exports" - - compiler: ": Win32" - env: HOST=i686-w64-mingw32 PPA="ppa:ubuntu-wine/ppa" PACKAGES="nsis gcc-mingw-w64-i686 g++-mingw-w64-i686 binutils-mingw-w64-i686 mingw-w64-dev wine1.7 bc" RUN_TESTS=true GOAL="deploy" BITCOIN_CONFIG="--enable-gui --enable-reduce-exports" MAKEJOBS="-j2" - - compiler: ": 32-bit + dash" - env: HOST=i686-pc-linux-gnu PACKAGES="g++-multilib bc python-zmq" PPA="ppa:chris-lea/zeromq" RUN_TESTS=true GOAL="install" BITCOIN_CONFIG="--enable-zmq --enable-glibc-back-compat --enable-reduce-exports LDFLAGS=-static-libstdc++" USE_SHELL="/bin/dash" - - compiler: ": Win64" - env: HOST=x86_64-w64-mingw32 PPA="ppa:ubuntu-wine/ppa" PACKAGES="nsis gcc-mingw-w64-x86-64 g++-mingw-w64-x86-64 binutils-mingw-w64-x86-64 mingw-w64-dev wine1.7 bc" RUN_TESTS=true GOAL="deploy" BITCOIN_CONFIG="--enable-gui --enable-reduce-exports" MAKEJOBS="-j2" - - compiler: ": bitcoind" - env: HOST=x86_64-unknown-linux-gnu PACKAGES="bc python-zmq" PPA="ppa:chris-lea/zeromq" DEP_OPTS="NO_QT=1 NO_UPNP=1 DEBUG=1" RUN_TESTS=true GOAL="install" BITCOIN_CONFIG="--enable-zmq --enable-glibc-back-compat --enable-reduce-exports CPPFLAGS=-DDEBUG_LOCKORDER" - - compiler: ": No wallet" - env: HOST=x86_64-unknown-linux-gnu DEP_OPTS="NO_WALLET=1" RUN_TESTS=true GOAL="install" BITCOIN_CONFIG="--enable-glibc-back-compat --enable-reduce-exports" - - compiler: ": Cross-Mac" - env: HOST=x86_64-apple-darwin11 PACKAGES="cmake imagemagick libcap-dev librsvg2-bin libz-dev libbz2-dev libtiff-tools python-dev" BITCOIN_CONFIG="--enable-reduce-exports" OSX_SDK=10.9 GOAL="deploy" - exclude: - - compiler: gcc + matrix: +# ARM + - HOST=arm-linux-gnueabihf PACKAGES="g++-arm-linux-gnueabihf" DEP_OPTS="NO_QT=1" CHECK_DOC=1 GOAL="install" BITCOIN_CONFIG="--enable-glibc-back-compat --enable-reduce-exports" +# Win32 + - HOST=i686-w64-mingw32 DPKG_ADD_ARCH="i386" DEP_OPTS="NO_QT=1" PACKAGES="python3 nsis g++-mingw-w64-i686 wine1.6 bc openjdk-7-jre-headless" RUN_TESTS=true GOAL="install" BITCOIN_CONFIG="--enable-reduce-exports" +# 32-bit + dash + - HOST=i686-pc-linux-gnu PACKAGES="g++-multilib bc python3-zmq openjdk-7-jre-headless" DEP_OPTS="NO_QT=1" RUN_TESTS=true GOAL="install" BITCOIN_CONFIG="--enable-zmq --enable-glibc-back-compat --enable-reduce-exports LDFLAGS=-static-libstdc++" USE_SHELL="/bin/dash" +# Win64 + - HOST=x86_64-w64-mingw32 DPKG_ADD_ARCH="i386" DEP_OPTS="NO_QT=1" PACKAGES="python3 nsis g++-mingw-w64-x86-64 wine1.6 bc openjdk-7-jre-headless" RUN_TESTS=true GOAL="install" BITCOIN_CONFIG="--enable-reduce-exports" +# bitcoind + - HOST=x86_64-unknown-linux-gnu PACKAGES="bc python3-zmq openjdk-7-jre-headless" DEP_OPTS="NO_QT=1 NO_UPNP=1 DEBUG=1" RUN_TESTS=true GOAL="install" BITCOIN_CONFIG="--enable-zmq --enable-glibc-back-compat --enable-reduce-exports CPPFLAGS=-DDEBUG_LOCKORDER" +# No wallet + - HOST=x86_64-unknown-linux-gnu PACKAGES=" openjdk-7-jre-headless python3" DEP_OPTS="NO_WALLET=1" RUN_TESTS=true GOAL="install" BITCOIN_CONFIG="--enable-glibc-back-compat --enable-reduce-exports" +# Cross-Mac + - HOST=x86_64-apple-darwin11 PACKAGES="cmake imagemagick libcap-dev librsvg2-bin libz-dev libbz2-dev libtiff-tools python-dev" BITCOIN_CONFIG="--enable-reduce-exports" OSX_SDK=10.9 GOAL="deploy" + +before_install: + - export PATH=$(echo $PATH | tr ':' "\n" | sed '/\/opt\/python/d' | tr "\n" ":" | sed "s|::|:|g") install: - - if [ -n "$PACKAGES" ]; then sudo rm -f /etc/apt/sources.list.d/travis_ci_zeromq3-source.list; fi - if [ -n "$PPA" ]; then travis_retry sudo add-apt-repository "$PPA" -y; fi + - if [ -n "$DPKG_ADD_ARCH" ]; then sudo dpkg --add-architecture "$DPKG_ADD_ARCH" ; fi - if [ -n "$PACKAGES" ]; then travis_retry sudo apt-get update; fi - if [ -n "$PACKAGES" ]; then travis_retry sudo apt-get install --no-install-recommends --no-upgrade -qq $PACKAGES; fi before_script: @@ -69,17 +61,13 @@ script: - OUTDIR=$BASE_OUTDIR/$TRAVIS_PULL_REQUEST/$TRAVIS_JOB_NUMBER-$HOST - BITCOIN_CONFIG_ALL="--disable-dependency-tracking --prefix=$TRAVIS_BUILD_DIR/depends/$HOST --bindir=$OUTDIR/bin --libdir=$OUTDIR/lib" - depends/$HOST/native/bin/ccache --max-size=$CCACHE_SIZE - - if [ "$TRAVIS_PULL_REQUEST" != "false" ]; then export CCACHE_READONLY=1; fi - test -n "$USE_SHELL" && eval '"$USE_SHELL" -c "./autogen.sh"' || ./autogen.sh - - ./configure --cache-file=config.cache $BITCOIN_CONFIG_ALL $BITCOIN_CONFIG || ( cat config.log && false) - - make distdir PACKAGE=bitcoin VERSION=$HOST - - cd bitcoin-$HOST - - ./configure --cache-file=../config.cache $BITCOIN_CONFIG_ALL $BITCOIN_CONFIG || ( cat config.log && false) + - mkdir build && cd build + - ../configure $BITCOIN_CONFIG_ALL $BITCOIN_CONFIG || ( cat config.log && false) - make $MAKEJOBS $GOAL || ( echo "Build failure. Verbose build follows." && make $GOAL V=1 ; false ) - export LD_LIBRARY_PATH=$TRAVIS_BUILD_DIR/depends/$HOST/lib - - if [ "$RUN_TESTS" = "true" ]; then make check; fi + - if [ "$RUN_TESTS" = "true" ]; then make $MAKEJOBS check VERBOSE=1; fi - if [ "$RUN_TESTS" = "true" ]; then qa/pull-tester/rpc-tests.py --coverage; fi after_script: - echo $TRAVIS_COMMIT_RANGE - echo $TRAVIS_COMMIT_LOG - - if [ "$TRAVIS_PULL_REQUEST" != "false" ]; then (echo "Upload goes here. Something like: scp -r $BASE_OUTDIR server" || echo "upload failed"); fi diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 53d6527d40718..5c1138b812d47 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -17,13 +17,13 @@ To contribute a patch, the workflow is as follows: - Create topic branch - Commit patches -The project coding conventions in [doc/developer-notes.md](doc/developer-notes.md) must be adhered to. +The project coding conventions in the [developer notes](doc/developer-notes.md) must be adhered to. In general [commits should be atomic](https://en.wikipedia.org/wiki/Atomic_commit#Atomic_commit_convention) and diffs should be easy to read. For this reason do not mix any formatting fixes or code moves with actual code changes. Commit messages should be verbose by default consisting of a short subject line (50 chars max), a blank line and detailed explanatory text as separate paragraph(s); unless the title alone is self-explanatory (like "Corrected typo in main.cpp") then a single title line is sufficient. Commit messages should be helpful to people reading your code in the future, so explain the reasoning for your decisions. Further explanation [here](http://chris.beams.io/posts/git-commit/). -If a particular commit references another issue, please add the reference, for example "refs #1234", or "fixes #4321". Using "fixes or closes" keywords will cause the corresponding issue to be closed when the pull request is merged. +If a particular commit references another issue, please add the reference, for example `refs #1234`, or `fixes #4321`. Using the `fixes` or `closes` keywords will cause the corresponding issue to be closed when the pull request is merged. Please refer to the [Git manual](https://git-scm.com/doc) for more information about Git. @@ -35,13 +35,27 @@ The title of the pull request should be prefixed by the component or area that t Consensus: Add new opcode for BIP-XXXX OP_CHECKAWESOMESIG Net: Automatically create hidden service, listen on Tor Qt: Add feed bump button - Trivial: fix typo + Trivial: Fix typo in main.cpp -If a pull request is specifically not to be considered for merging (yet) please prefix the title with [WIP] or use [Tasks Lists](https://github.com/blog/1375-task-lists-in-gfm-issues-pulls-comments) in the body of the pull request to indicate tasks are pending. +If a pull request is specifically not to be considered for merging (yet) please prefix the title with [WIP] or use [Tasks Lists](https://help.github.com/articles/basic-writing-and-formatting-syntax/#task-lists) in the body of the pull request to indicate tasks are pending. The body of the pull request should contain enough description about what the patch does together with any justification/reasoning. You should include references to any discussions (for example other tickets or mailing list discussions). -At this stage one should expect comments and review from other contributors. You can add more commits to your pull request by committing them locally and pushing to your fork until you have satisfied all feedback. If your pull request is accepted for merging, you may be asked by a maintainer to squash and or rebase your commits before it will be merged. The length of time required for peer review is unpredictable and will vary from patch to patch. +At this stage one should expect comments and review from other contributors. You can add more commits to your pull request by committing them locally and pushing to your fork until you have satisfied all feedback. + +Squashing Commits +--------------------------- +If your pull request is accepted for merging, you may be asked by a maintainer to squash and or [rebase](https://git-scm.com/docs/git-rebase) your commits before it will be merged. The basic squashing workflow is shown below. + + git checkout your_branch_name + git rebase -i HEAD~n + # n is normally the number of commits in the pull + # set commits from 'pick' to 'squash', save and quit + # on the next screen, edit/refine commit messages + # save and quit + git push -f # (force push to GitHub) + +The length of time required for peer review is unpredictable and will vary from pull request to pull request. Pull Request Philosophy @@ -61,7 +75,7 @@ Refactoring is a necessary part of any software project's evolution. The followi There are three categories of refactoring, code only moves, code style fixes, code refactoring. In general refactoring pull requests should not mix these three kinds of activity in order to make refactoring pull requests easy to review and uncontroversial. In all cases, refactoring PRs must not change the behaviour of code within the pull request (bugs must be preserved as is). -Project maintainers aim for a quick turnaround on refactoring pull requests, so where possible keep them short, uncomplex and easy to verify. +Project maintainers aim for a quick turnaround on refactoring pull requests, so where possible keep them short, uncomplex and easy to verify. "Decision Making" Process @@ -69,7 +83,7 @@ Project maintainers aim for a quick turnaround on refactoring pull requests, so The following applies to code changes to the Bitcoin Core project (and related projects such as libsecp256k1), and is not to be confused with overall Bitcoin Network Protocol consensus changes. -Whether a pull request is merged into Bitcoin Core rests with the project merge maintainers and ultimately the project lead. +Whether a pull request is merged into Bitcoin Core rests with the project merge maintainers and ultimately the project lead. Maintainers will take into consideration if a patch is in line with the general principles of the project; meets the minimum standards for inclusion; and will judge the general consensus of contributors. diff --git a/Makefile.am b/Makefile.am index 5783c1fdd8f55..b10d0850664c8 100644 --- a/Makefile.am +++ b/Makefile.am @@ -28,7 +28,7 @@ OSX_DSSTORE_GEN=$(top_srcdir)/contrib/macdeploy/custom_dsstore.py OSX_DEPLOY_SCRIPT=$(top_srcdir)/contrib/macdeploy/macdeployqtplus OSX_FANCY_PLIST=$(top_srcdir)/contrib/macdeploy/fancy.plist OSX_INSTALLER_ICONS=$(top_srcdir)/src/qt/res/icons/bitcoin.icns -OSX_PLIST=$(top_srcdir)/share/qt/Info.plist #not installed +OSX_PLIST=$(top_builddir)/share/qt/Info.plist #not installed OSX_QT_TRANSLATIONS = da,de,es,hu,ru,uk,zh_CN,zh_TW DIST_DOCS = $(wildcard doc/*.md) $(wildcard doc/release-notes/*.md) @@ -53,18 +53,8 @@ COVERAGE_INFO = baseline_filtered_combined.info baseline.info block_test.info \ leveldb_baseline_filtered.info test_bitcoin_coverage.info test_bitcoin.info dist-hook: - -$(MAKE) -C $(top_distdir)/src/leveldb clean - -$(MAKE) -C $(top_distdir)/src/secp256k1 distclean -$(GIT) archive --format=tar HEAD -- src/clientversion.cpp | $(AMTAR) -C $(top_distdir) -xf - -distcheck-hook: - $(MKDIR_P) $(top_distdir)/_build/src/leveldb - cp -rf $(top_srcdir)/src/leveldb/* $(top_distdir)/_build/src/leveldb/ - -$(MAKE) -C $(top_distdir)/_build/src/leveldb clean - -distcleancheck: - @: - $(BITCOIN_WIN_INSTALLER): all-recursive $(MKDIR_P) $(top_builddir)/release STRIPPROG="$(STRIP)" $(INSTALL_STRIP_PROGRAM) $(BITCOIND_BIN) $(top_builddir)/release @@ -234,7 +224,11 @@ EXTRA_DIST = $(top_srcdir)/share/genbuild.sh qa/pull-tester/rpc-tests.py qa/rpc- CLEANFILES = $(OSX_DMG) $(BITCOIN_WIN_INSTALLER) +# This file is problematic for out-of-tree builds if it exists. +DISTCLEANFILES = qa/pull-tester/tests_config.pyc + .INTERMEDIATE: $(COVERAGE_INFO) clean-local: rm -rf coverage_percent.txt test_bitcoin.coverage/ total.coverage/ qa/tmp/ cache/ $(OSX_APP) + rm -rf qa/pull-tester/__pycache__ diff --git a/README.md b/README.md index 85b1985560c8f..8e816e7a43fa4 100644 --- a/README.md +++ b/README.md @@ -57,8 +57,7 @@ There are also [regression and integration tests](/qa) of the RPC interface, wri in Python, that are run automatically on the build server. These tests can be run (if the [test dependencies](/qa) are installed) with: `qa/pull-tester/rpc-tests.py` -The Travis CI system makes sure that every pull request is built for Windows -and Linux, OS X, and that unit and sanity tests are automatically run. +The Travis CI system makes sure that every pull request is built for Windows, Linux, and OS X, and that unit/sanity tests are run automatically. ### Manual Quality Assurance (QA) Testing diff --git a/build-aux/m4/ax_cxx_compile_stdcxx.m4 b/build-aux/m4/ax_cxx_compile_stdcxx.m4 new file mode 100644 index 0000000000000..2c18e49c56c0b --- /dev/null +++ b/build-aux/m4/ax_cxx_compile_stdcxx.m4 @@ -0,0 +1,562 @@ +# =========================================================================== +# http://www.gnu.org/software/autoconf-archive/ax_cxx_compile_stdcxx.html +# =========================================================================== +# +# SYNOPSIS +# +# AX_CXX_COMPILE_STDCXX(VERSION, [ext|noext], [mandatory|optional]) +# +# DESCRIPTION +# +# Check for baseline language coverage in the compiler for the specified +# version of the C++ standard. If necessary, add switches to CXX and +# CXXCPP to enable support. VERSION may be '11' (for the C++11 standard) +# or '14' (for the C++14 standard). +# +# The second argument, if specified, indicates whether you insist on an +# extended mode (e.g. -std=gnu++11) or a strict conformance mode (e.g. +# -std=c++11). If neither is specified, you get whatever works, with +# preference for an extended mode. +# +# The third argument, if specified 'mandatory' or if left unspecified, +# indicates that baseline support for the specified C++ standard is +# required and that the macro should error out if no mode with that +# support is found. If specified 'optional', then configuration proceeds +# regardless, after defining HAVE_CXX${VERSION} if and only if a +# supporting mode is found. +# +# LICENSE +# +# Copyright (c) 2008 Benjamin Kosnik +# Copyright (c) 2012 Zack Weinberg +# Copyright (c) 2013 Roy Stogner +# Copyright (c) 2014, 2015 Google Inc.; contributed by Alexey Sokolov +# Copyright (c) 2015 Paul Norman +# Copyright (c) 2015 Moritz Klammler +# +# Copying and distribution of this file, with or without modification, are +# permitted in any medium without royalty provided the copyright notice +# and this notice are preserved. This file is offered as-is, without any +# warranty. + +#serial 4 + +dnl This macro is based on the code from the AX_CXX_COMPILE_STDCXX_11 macro +dnl (serial version number 13). + +AC_DEFUN([AX_CXX_COMPILE_STDCXX], [dnl + m4_if([$1], [11], [], + [$1], [14], [], + [$1], [17], [m4_fatal([support for C++17 not yet implemented in AX_CXX_COMPILE_STDCXX])], + [m4_fatal([invalid first argument `$1' to AX_CXX_COMPILE_STDCXX])])dnl + m4_if([$2], [], [], + [$2], [ext], [], + [$2], [noext], [], + [m4_fatal([invalid second argument `$2' to AX_CXX_COMPILE_STDCXX])])dnl + m4_if([$3], [], [ax_cxx_compile_cxx$1_required=true], + [$3], [mandatory], [ax_cxx_compile_cxx$1_required=true], + [$3], [optional], [ax_cxx_compile_cxx$1_required=false], + [m4_fatal([invalid third argument `$3' to AX_CXX_COMPILE_STDCXX])]) + AC_LANG_PUSH([C++])dnl + ac_success=no + AC_CACHE_CHECK(whether $CXX supports C++$1 features by default, + ax_cv_cxx_compile_cxx$1, + [AC_COMPILE_IFELSE([AC_LANG_SOURCE([_AX_CXX_COMPILE_STDCXX_testbody_$1])], + [ax_cv_cxx_compile_cxx$1=yes], + [ax_cv_cxx_compile_cxx$1=no])]) + if test x$ax_cv_cxx_compile_cxx$1 = xyes; then + ac_success=yes + fi + + m4_if([$2], [noext], [], [dnl + if test x$ac_success = xno; then + for switch in -std=gnu++$1 -std=gnu++0x; do + cachevar=AS_TR_SH([ax_cv_cxx_compile_cxx$1_$switch]) + AC_CACHE_CHECK(whether $CXX supports C++$1 features with $switch, + $cachevar, + [ac_save_CXX="$CXX" + CXX="$CXX $switch" + AC_COMPILE_IFELSE([AC_LANG_SOURCE([_AX_CXX_COMPILE_STDCXX_testbody_$1])], + [eval $cachevar=yes], + [eval $cachevar=no]) + CXX="$ac_save_CXX"]) + if eval test x\$$cachevar = xyes; then + CXX="$CXX $switch" + if test -n "$CXXCPP" ; then + CXXCPP="$CXXCPP $switch" + fi + ac_success=yes + break + fi + done + fi]) + + m4_if([$2], [ext], [], [dnl + if test x$ac_success = xno; then + dnl HP's aCC needs +std=c++11 according to: + dnl http://h21007.www2.hp.com/portal/download/files/unprot/aCxx/PDF_Release_Notes/769149-001.pdf + dnl Cray's crayCC needs "-h std=c++11" + for switch in -std=c++$1 -std=c++0x +std=c++$1 "-h std=c++$1"; do + cachevar=AS_TR_SH([ax_cv_cxx_compile_cxx$1_$switch]) + AC_CACHE_CHECK(whether $CXX supports C++$1 features with $switch, + $cachevar, + [ac_save_CXX="$CXX" + CXX="$CXX $switch" + AC_COMPILE_IFELSE([AC_LANG_SOURCE([_AX_CXX_COMPILE_STDCXX_testbody_$1])], + [eval $cachevar=yes], + [eval $cachevar=no]) + CXX="$ac_save_CXX"]) + if eval test x\$$cachevar = xyes; then + CXX="$CXX $switch" + if test -n "$CXXCPP" ; then + CXXCPP="$CXXCPP $switch" + fi + ac_success=yes + break + fi + done + fi]) + AC_LANG_POP([C++]) + if test x$ax_cxx_compile_cxx$1_required = xtrue; then + if test x$ac_success = xno; then + AC_MSG_ERROR([*** A compiler with support for C++$1 language features is required.]) + fi + fi + if test x$ac_success = xno; then + HAVE_CXX$1=0 + AC_MSG_NOTICE([No compiler with C++$1 support was found]) + else + HAVE_CXX$1=1 + AC_DEFINE(HAVE_CXX$1,1, + [define if the compiler supports basic C++$1 syntax]) + fi + AC_SUBST(HAVE_CXX$1) +]) + + +dnl Test body for checking C++11 support + +m4_define([_AX_CXX_COMPILE_STDCXX_testbody_11], + _AX_CXX_COMPILE_STDCXX_testbody_new_in_11 +) + + +dnl Test body for checking C++14 support + +m4_define([_AX_CXX_COMPILE_STDCXX_testbody_14], + _AX_CXX_COMPILE_STDCXX_testbody_new_in_11 + _AX_CXX_COMPILE_STDCXX_testbody_new_in_14 +) + + +dnl Tests for new features in C++11 + +m4_define([_AX_CXX_COMPILE_STDCXX_testbody_new_in_11], [[ + +// If the compiler admits that it is not ready for C++11, why torture it? +// Hopefully, this will speed up the test. + +#ifndef __cplusplus + +#error "This is not a C++ compiler" + +#elif __cplusplus < 201103L + +#error "This is not a C++11 compiler" + +#else + +namespace cxx11 +{ + + namespace test_static_assert + { + + template + struct check + { + static_assert(sizeof(int) <= sizeof(T), "not big enough"); + }; + + } + + namespace test_final_override + { + + struct Base + { + virtual void f() {} + }; + + struct Derived : public Base + { + virtual void f() override {} + }; + + } + + namespace test_double_right_angle_brackets + { + + template < typename T > + struct check {}; + + typedef check single_type; + typedef check> double_type; + typedef check>> triple_type; + typedef check>>> quadruple_type; + + } + + namespace test_decltype + { + + int + f() + { + int a = 1; + decltype(a) b = 2; + return a + b; + } + + } + + namespace test_type_deduction + { + + template < typename T1, typename T2 > + struct is_same + { + static const bool value = false; + }; + + template < typename T > + struct is_same + { + static const bool value = true; + }; + + template < typename T1, typename T2 > + auto + add(T1 a1, T2 a2) -> decltype(a1 + a2) + { + return a1 + a2; + } + + int + test(const int c, volatile int v) + { + static_assert(is_same::value == true, ""); + static_assert(is_same::value == false, ""); + static_assert(is_same::value == false, ""); + auto ac = c; + auto av = v; + auto sumi = ac + av + 'x'; + auto sumf = ac + av + 1.0; + static_assert(is_same::value == true, ""); + static_assert(is_same::value == true, ""); + static_assert(is_same::value == true, ""); + static_assert(is_same::value == false, ""); + static_assert(is_same::value == true, ""); + return (sumf > 0.0) ? sumi : add(c, v); + } + + } + + namespace test_noexcept + { + + int f() { return 0; } + int g() noexcept { return 0; } + + static_assert(noexcept(f()) == false, ""); + static_assert(noexcept(g()) == true, ""); + + } + + namespace test_constexpr + { + + template < typename CharT > + unsigned long constexpr + strlen_c_r(const CharT *const s, const unsigned long acc) noexcept + { + return *s ? strlen_c_r(s + 1, acc + 1) : acc; + } + + template < typename CharT > + unsigned long constexpr + strlen_c(const CharT *const s) noexcept + { + return strlen_c_r(s, 0UL); + } + + static_assert(strlen_c("") == 0UL, ""); + static_assert(strlen_c("1") == 1UL, ""); + static_assert(strlen_c("example") == 7UL, ""); + static_assert(strlen_c("another\0example") == 7UL, ""); + + } + + namespace test_rvalue_references + { + + template < int N > + struct answer + { + static constexpr int value = N; + }; + + answer<1> f(int&) { return answer<1>(); } + answer<2> f(const int&) { return answer<2>(); } + answer<3> f(int&&) { return answer<3>(); } + + void + test() + { + int i = 0; + const int c = 0; + static_assert(decltype(f(i))::value == 1, ""); + static_assert(decltype(f(c))::value == 2, ""); + static_assert(decltype(f(0))::value == 3, ""); + } + + } + + namespace test_uniform_initialization + { + + struct test + { + static const int zero {}; + static const int one {1}; + }; + + static_assert(test::zero == 0, ""); + static_assert(test::one == 1, ""); + + } + + namespace test_lambdas + { + + void + test1() + { + auto lambda1 = [](){}; + auto lambda2 = lambda1; + lambda1(); + lambda2(); + } + + int + test2() + { + auto a = [](int i, int j){ return i + j; }(1, 2); + auto b = []() -> int { return '0'; }(); + auto c = [=](){ return a + b; }(); + auto d = [&](){ return c; }(); + auto e = [a, &b](int x) mutable { + const auto identity = [](int y){ return y; }; + for (auto i = 0; i < a; ++i) + a += b--; + return x + identity(a + b); + }(0); + return a + b + c + d + e; + } + + int + test3() + { + const auto nullary = [](){ return 0; }; + const auto unary = [](int x){ return x; }; + using nullary_t = decltype(nullary); + using unary_t = decltype(unary); + const auto higher1st = [](nullary_t f){ return f(); }; + const auto higher2nd = [unary](nullary_t f1){ + return [unary, f1](unary_t f2){ return f2(unary(f1())); }; + }; + return higher1st(nullary) + higher2nd(nullary)(unary); + } + + } + + namespace test_variadic_templates + { + + template + struct sum; + + template + struct sum + { + static constexpr auto value = N0 + sum::value; + }; + + template <> + struct sum<> + { + static constexpr auto value = 0; + }; + + static_assert(sum<>::value == 0, ""); + static_assert(sum<1>::value == 1, ""); + static_assert(sum<23>::value == 23, ""); + static_assert(sum<1, 2>::value == 3, ""); + static_assert(sum<5, 5, 11>::value == 21, ""); + static_assert(sum<2, 3, 5, 7, 11, 13>::value == 41, ""); + + } + + // http://stackoverflow.com/questions/13728184/template-aliases-and-sfinae + // Clang 3.1 fails with headers of libstd++ 4.8.3 when using std::function + // because of this. + namespace test_template_alias_sfinae + { + + struct foo {}; + + template + using member = typename T::member_type; + + template + void func(...) {} + + template + void func(member*) {} + + void test(); + + void test() { func(0); } + + } + +} // namespace cxx11 + +#endif // __cplusplus >= 201103L + +]]) + + +dnl Tests for new features in C++14 + +m4_define([_AX_CXX_COMPILE_STDCXX_testbody_new_in_14], [[ + +// If the compiler admits that it is not ready for C++14, why torture it? +// Hopefully, this will speed up the test. + +#ifndef __cplusplus + +#error "This is not a C++ compiler" + +#elif __cplusplus < 201402L + +#error "This is not a C++14 compiler" + +#else + +namespace cxx14 +{ + + namespace test_polymorphic_lambdas + { + + int + test() + { + const auto lambda = [](auto&&... args){ + const auto istiny = [](auto x){ + return (sizeof(x) == 1UL) ? 1 : 0; + }; + const int aretiny[] = { istiny(args)... }; + return aretiny[0]; + }; + return lambda(1, 1L, 1.0f, '1'); + } + + } + + namespace test_binary_literals + { + + constexpr auto ivii = 0b0000000000101010; + static_assert(ivii == 42, "wrong value"); + + } + + namespace test_generalized_constexpr + { + + template < typename CharT > + constexpr unsigned long + strlen_c(const CharT *const s) noexcept + { + auto length = 0UL; + for (auto p = s; *p; ++p) + ++length; + return length; + } + + static_assert(strlen_c("") == 0UL, ""); + static_assert(strlen_c("x") == 1UL, ""); + static_assert(strlen_c("test") == 4UL, ""); + static_assert(strlen_c("another\0test") == 7UL, ""); + + } + + namespace test_lambda_init_capture + { + + int + test() + { + auto x = 0; + const auto lambda1 = [a = x](int b){ return a + b; }; + const auto lambda2 = [a = lambda1(x)](){ return a; }; + return lambda2(); + } + + } + + namespace test_digit_seperators + { + + constexpr auto ten_million = 100'000'000; + static_assert(ten_million == 100000000, ""); + + } + + namespace test_return_type_deduction + { + + auto f(int& x) { return x; } + decltype(auto) g(int& x) { return x; } + + template < typename T1, typename T2 > + struct is_same + { + static constexpr auto value = false; + }; + + template < typename T > + struct is_same + { + static constexpr auto value = true; + }; + + int + test() + { + auto x = 0; + static_assert(is_same::value, ""); + static_assert(is_same::value, ""); + return x; + } + + } + +} // namespace cxx14 + +#endif // __cplusplus >= 201402L + +]]) diff --git a/configure.ac b/configure.ac index 6e463dfc5540d..97af58bd7cd85 100644 --- a/configure.ac +++ b/configure.ac @@ -55,6 +55,8 @@ case $host in lt_cv_deplibs_check_method="pass_all" ;; esac +dnl Require C++11 compiler (no GNU extensions) +AX_CXX_COMPILE_STDCXX([11], [noext], [mandatory]) dnl Libtool init checks. LT_INIT([pic-only]) @@ -73,6 +75,7 @@ AC_PATH_PROG(XGETTEXT,xgettext) AC_PATH_PROG(HEXDUMP,hexdump) AC_PATH_TOOL(READELF, readelf) AC_PATH_TOOL(CPPFILT, c++filt) +AC_PATH_TOOL(OBJCOPY, objcopy) AC_ARG_VAR(PYTHONPATH, Augments the default search path for python module files) @@ -185,6 +188,9 @@ AC_ARG_ENABLE([debug], [enable_debug=$enableval], [enable_debug=no]) +AC_LANG_PUSH([C++]) +AX_CHECK_COMPILE_FLAG([-Werror],[CXXFLAG_WERROR="-Werror"],[CXXFLAG_WERROR=""]) + if test "x$enable_debug" = xyes; then CPPFLAGS="$CPPFLAGS -DDEBUG -DDEBUG_LOCKORDER" if test "x$GCC" = xyes; then @@ -196,11 +202,19 @@ if test "x$enable_debug" = xyes; then fi fi -## TODO: Remove these hard-coded paths and flags. They are here for the sake of -## compatibility with the legacy buildsystem. -## if test "x$CXXFLAGS_overridden" = "xno"; then - CXXFLAGS="$CXXFLAGS -Wall -Wextra -Wformat -Wformat-security -Wno-unused-parameter -Wno-self-assign" + AX_CHECK_COMPILE_FLAG([-Wall],[CXXFLAGS="$CXXFLAGS -Wall"],,[[$CXXFLAG_WERROR]]) + AX_CHECK_COMPILE_FLAG([-Wextra],[CXXFLAGS="$CXXFLAGS -Wextra"],,[[$CXXFLAG_WERROR]]) + AX_CHECK_COMPILE_FLAG([-Wformat],[CXXFLAGS="$CXXFLAGS -Wformat"],,[[$CXXFLAG_WERROR]]) + AX_CHECK_COMPILE_FLAG([-Wformat-security],[CXXFLAGS="$CXXFLAGS -Wformat-security"],,[[$CXXFLAG_WERROR]]) + + ## Some compilers (gcc) ignore unknown -Wno-* options, but warn about all + ## unknown options if any other warning is produced. Test the -Wfoo case, and + ## set the -Wno-foo case if it works. + AX_CHECK_COMPILE_FLAG([-Wunused-parameter],[CXXFLAGS="$CXXFLAGS -Wno-unused-parameter"],,[[$CXXFLAG_WERROR]]) + AX_CHECK_COMPILE_FLAG([-Wself-assign],[CXXFLAGS="$CXXFLAGS -Wno-self-assign"],,[[$CXXFLAG_WERROR]]) + AX_CHECK_COMPILE_FLAG([-Wunused-local-typedef],[CXXFLAGS="$CXXFLAGS -Wno-unused-local-typedef"],,[[$CXXFLAG_WERROR]]) + AX_CHECK_COMPILE_FLAG([-Wdeprecated-register],[CXXFLAGS="$CXXFLAGS -Wno-deprecated-register"],,[[$CXXFLAG_WERROR]]) fi CPPFLAGS="$CPPFLAGS -DHAVE_BUILD_INFO -D__STDC_FORMAT_MACROS" @@ -222,8 +236,6 @@ AC_ARG_WITH([daemon], [build_bitcoind=$withval], [build_bitcoind=yes]) -AC_LANG_PUSH([C++]) - use_pkgconfig=yes case $host in *mingw*) @@ -267,7 +279,7 @@ case $host in fi CPPFLAGS="$CPPFLAGS -D_MT -DWIN32 -D_WINDOWS -DBOOST_THREAD_USE_LIB" - LEVELDB_TARGET_FLAGS="TARGET_OS=OS_WINDOWS_CROSSCOMPILE" + LEVELDB_TARGET_FLAGS="-DOS_WINDOWS" if test "x$CXXFLAGS_overridden" = "xno"; then CXXFLAGS="$CXXFLAGS -w" fi @@ -289,7 +301,7 @@ case $host in ;; *darwin*) TARGET_OS=darwin - LEVELDB_TARGET_FLAGS="TARGET_OS=Darwin" + LEVELDB_TARGET_FLAGS="-DOS_MACOSX" if test x$cross_compiling != xyes; then BUILD_OS=darwin AC_CHECK_PROG([PORT],port, port) @@ -355,8 +367,11 @@ case $host in ;; *linux*) TARGET_OS=linux + LEVELDB_TARGET_FLAGS="-DOS_LINUX" ;; *) + OTHER_OS=`echo ${host_os} | awk '{print toupper($0)}'` + LEVELDB_TARGET_FLAGS="-DOS_${OTHER_OS}" ;; esac @@ -1046,6 +1061,8 @@ AC_SUBST(MINIUPNPC_LIBS) AC_CONFIG_FILES([Makefile src/Makefile share/setup.nsi share/qt/Info.plist src/test/buildenv.py]) AC_CONFIG_FILES([qa/pull-tester/run-bitcoind-for-test.sh],[chmod +x qa/pull-tester/run-bitcoind-for-test.sh]) AC_CONFIG_FILES([qa/pull-tester/tests_config.py],[chmod +x qa/pull-tester/tests_config.py]) +AC_CONFIG_FILES([contrib/devtools/split-debug.sh],[chmod +x contrib/devtools/split-debug.sh]) +AC_CONFIG_LINKS([qa/pull-tester/rpc-tests.py:qa/pull-tester/rpc-tests.py]) dnl boost's m4 checks do something really nasty: they export these vars. As a dnl result, they leak into secp256k1's configure and crazy things happen. diff --git a/contrib/README.md b/contrib/README.md index 32b3a170ac7ec..a23b197cc6122 100644 --- a/contrib/README.md +++ b/contrib/README.md @@ -51,5 +51,5 @@ Test and Verify Tools ### [TestGen](/contrib/testgen) ### Utilities to generate test vectors for the data-driven Bitcoin tests. -### [Verify SF Binaries](/contrib/verifysfbinaries) ### -This script attempts to download and verify the signature file SHA256SUMS.asc from SourceForge. +### [Verify Binaries](/contrib/verifybinaries) ### +This script attempts to download and verify the signature file SHA256SUMS.asc from bitcoin.org. diff --git a/contrib/devtools/README.md b/contrib/devtools/README.md index 1103ca86c5e82..bb8b9246b8781 100644 --- a/contrib/devtools/README.md +++ b/contrib/devtools/README.md @@ -48,9 +48,9 @@ the commit it claims to have been updated to. To use, make sure that you have fetched the upstream repository branch in which the subtree is maintained: -* for `src/secp256k1`: https://github.com/bitcoin/secp256k1.git (branch master) -* for `src/leveldb`: https://github.com/bitcoin/leveldb.git (branch bitcoin-fork) -* for `src/univalue`: https://github.com/bitcoin/univalue.git (branch master) +* for `src/secp256k1`: https://github.com/bitcoin-core/secp256k1.git (branch master) +* for `src/leveldb`: https://github.com/bitcoin-core/leveldb.git (branch bitcoin-fork) +* for `src/univalue`: https://github.com/bitcoin-core/univalue.git (branch master) Usage: `git-subtree-check.sh DIR COMMIT` diff --git a/contrib/devtools/split-debug.sh.in b/contrib/devtools/split-debug.sh.in new file mode 100644 index 0000000000000..deda49cc54104 --- /dev/null +++ b/contrib/devtools/split-debug.sh.in @@ -0,0 +1,10 @@ +#!/bin/sh + +if [ $# -ne 3 ]; + then echo "usage: $0 " +fi + +@OBJCOPY@ --enable-deterministic-archives -p --only-keep-debug $1 $3 +@OBJCOPY@ --enable-deterministic-archives -p --strip-debug $1 $2 +@STRIP@ --enable-deterministic-archives -p -s $2 +@OBJCOPY@ --enable-deterministic-archives -p --add-gnu-debuglink=$3 $2 diff --git a/contrib/gitian-descriptors/gitian-linux.yml b/contrib/gitian-descriptors/gitian-linux.yml index 13941f5d55ea3..fb629578d65f0 100644 --- a/contrib/gitian-descriptors/gitian-linux.yml +++ b/contrib/gitian-descriptors/gitian-linux.yml @@ -5,9 +5,19 @@ suites: - "trusty" architectures: - "amd64" -packages: +packages: - "curl" -- "g++-multilib" +- "g++-aarch64-linux-gnu" +- "g++-4.8-aarch64-linux-gnu" +- "gcc-4.8-aarch64-linux-gnu" +- "binutils-aarch64-linux-gnu" +- "g++-arm-linux-gnueabihf" +- "g++-4.8-arm-linux-gnueabihf" +- "gcc-4.8-arm-linux-gnueabihf" +- "binutils-arm-linux-gnueabihf" +- "g++-4.8-multilib" +- "gcc-4.8-multilib" +- "binutils-gold" - "git-core" - "pkg-config" - "autoconf" @@ -15,20 +25,28 @@ packages: - "automake" - "faketime" - "bsdmainutils" -- "binutils-gold" - "ca-certificates" - "python" -reference_datetime: "2016-01-01 00:00:00" remotes: - "url": "https://github.com/bitcoin/bitcoin.git" "dir": "bitcoin" files: [] script: | + + #unlock sudo + echo "ubuntu" | sudo -S true + + sudo mkdir -p /usr/include/i386-linux-gnu/ + sudo ln -s /usr/include/x86_64-linux-gnu/asm /usr/include/i386-linux-gnu/asm + WRAP_DIR=$HOME/wrapped - HOSTS="i686-pc-linux-gnu x86_64-unknown-linux-gnu" - CONFIGFLAGS="--enable-glibc-back-compat --enable-reduce-exports --disable-bench --disable-gui-tests LDFLAGS=-static-libstdc++" + HOSTS="i686-pc-linux-gnu x86_64-linux-gnu arm-linux-gnueabihf aarch64-linux-gnu" + CONFIGFLAGS="--enable-glibc-back-compat --enable-reduce-exports --disable-bench --disable-gui-tests" FAKETIME_HOST_PROGS="" - FAKETIME_PROGS="date ar ranlib nm strip" + FAKETIME_PROGS="date ar ranlib nm" + HOST_CFLAGS="-O2 -g" + HOST_CXXFLAGS="-O2 -g" + HOST_LDFLAGS=-static-libstdc++ export QT_RCC_TEST=1 export GZIP="-9n" @@ -42,29 +60,36 @@ script: | mkdir -p ${BASE_CACHE} ${SOURCES_PATH} fi - # Create global faketime wrappers + function create_global_faketime_wrappers { for prog in ${FAKETIME_PROGS}; do echo '#!/bin/bash' > ${WRAP_DIR}/${prog} echo "REAL=\`which -a ${prog} | grep -v ${WRAP_DIR}/${prog} | head -1\`" >> ${WRAP_DIR}/${prog} echo 'export LD_PRELOAD=/usr/lib/x86_64-linux-gnu/faketime/libfaketime.so.1' >> ${WRAP_DIR}/${prog} - echo "export FAKETIME=\"${REFERENCE_DATETIME}\"" >> ${WRAP_DIR}/${prog} + echo "export FAKETIME=\"$1\"" >> ${WRAP_DIR}/${prog} echo "\$REAL \$@" >> $WRAP_DIR/${prog} chmod +x ${WRAP_DIR}/${prog} done + } - # Create per-host faketime wrappers + function create_per-host_faketime_wrappers { for i in $HOSTS; do for prog in ${FAKETIME_HOST_PROGS}; do echo '#!/bin/bash' > ${WRAP_DIR}/${i}-${prog} echo "REAL=\`which -a ${i}-${prog} | grep -v ${WRAP_DIR}/${i}-${prog} | head -1\`" >> ${WRAP_DIR}/${i}-${prog} echo 'export LD_PRELOAD=/usr/lib/x86_64-linux-gnu/faketime/libfaketime.so.1' >> ${WRAP_DIR}/${i}-${prog} - echo "export FAKETIME=\"${REFERENCE_DATETIME}\"" >> ${WRAP_DIR}/${i}-${prog} + echo "export FAKETIME=\"$1\"" >> ${WRAP_DIR}/${i}-${prog} echo "\$REAL \$@" >> $WRAP_DIR/${i}-${prog} chmod +x ${WRAP_DIR}/${i}-${prog} done done + } + export PATH=${WRAP_DIR}:${PATH} + # Faketime for depends so intermediate results are comparable + create_global_faketime_wrappers "2000-01-01 12:00:00" + create_per-host_faketime_wrappers "2000-01-01 12:00:00" + cd bitcoin BASEPREFIX=`pwd`/depends # Build dependencies for each host @@ -72,9 +97,13 @@ script: | make ${MAKEOPTS} -C ${BASEPREFIX} HOST="${i}" done + # Faketime for binaries + create_global_faketime_wrappers "${REFERENCE_DATETIME}" + create_per-host_faketime_wrappers "${REFERENCE_DATETIME}" + # Create the release tarball using (arbitrarily) the first host ./autogen.sh - ./configure --prefix=${BASEPREFIX}/`echo "${HOSTS}" | awk '{print $1;}'` + CONFIG_SITE=${BASEPREFIX}/`echo "${HOSTS}" | awk '{print $1;}'`/share/config.site ./configure --prefix=/ make dist SOURCEDIST=`echo bitcoin-*.tar.gz` DISTNAME=`echo ${SOURCEDIST} | sed 's/.tar.*//'` @@ -95,20 +124,31 @@ script: | mkdir -p ${INSTALLPATH} tar --strip-components=1 -xf ../$SOURCEDIST - ./configure --prefix=${BASEPREFIX}/${i} --bindir=${INSTALLPATH}/bin --includedir=${INSTALLPATH}/include --libdir=${INSTALLPATH}/lib --disable-ccache --disable-maintainer-mode --disable-dependency-tracking ${CONFIGFLAGS} + CONFIG_SITE=${BASEPREFIX}/${i}/share/config.site ./configure --prefix=/ --disable-ccache --disable-maintainer-mode --disable-dependency-tracking ${CONFIGFLAGS} CFLAGS="${HOST_CFLAGS}" CXXFLAGS="${HOST_CXXFLAGS}" LDFLAGS="${HOST_LDFLAGS}" make ${MAKEOPTS} make ${MAKEOPTS} -C src check-security - make ${MAKEOPTS} -C src check-symbols - make install-strip + + #TODO: This is a quick hack that disables symbol checking for arm. + # Instead, we should investigate why these are popping up. + # For aarch64, we'll need to bump up the min GLIBC version, as the abi + # support wasn't introduced until 2.17. + case $i in + aarch64-*) : ;; + arm-*) : ;; + *) make ${MAKEOPTS} -C src check-symbols ;; + esac + + make install DESTDIR=${INSTALLPATH} cd installed find . -name "lib*.la" -delete find . -name "lib*.a" -delete rm -rf ${DISTNAME}/lib/pkgconfig - find ${DISTNAME} | sort | tar --no-recursion --mode='u+rw,go+r-w,a+X' --owner=0 --group=0 -c -T - | gzip -9n > ${OUTDIR}/${DISTNAME}-${i}.tar.gz + find ${DISTNAME}/bin -type f -executable -exec ../contrib/devtools/split-debug.sh {} {} {}.dbg \; + find ${DISTNAME}/lib -type f -exec ../contrib/devtools/split-debug.sh {} {} {}.dbg \; + find ${DISTNAME} -not -name "*.dbg" | sort | tar --no-recursion --mode='u+rw,go+r-w,a+X' --owner=0 --group=0 -c -T - | gzip -9n > ${OUTDIR}/${DISTNAME}-${i}.tar.gz + find ${DISTNAME} -name "*.dbg" | sort | tar --no-recursion --mode='u+rw,go+r-w,a+X' --owner=0 --group=0 -c -T - | gzip -9n > ${OUTDIR}/${DISTNAME}-${i}-debug.tar.gz cd ../../ + rm -rf distsrc-${i} done mkdir -p $OUTDIR/src mv $SOURCEDIST $OUTDIR/src - mv ${OUTDIR}/${DISTNAME}-x86_64-*.tar.gz ${OUTDIR}/${DISTNAME}-linux64.tar.gz - mv ${OUTDIR}/${DISTNAME}-i686-*.tar.gz ${OUTDIR}/${DISTNAME}-linux32.tar.gz - diff --git a/contrib/gitian-descriptors/gitian-osx-signer.yml b/contrib/gitian-descriptors/gitian-osx-signer.yml index c4165470af132..f6e9414ab1bf2 100644 --- a/contrib/gitian-descriptors/gitian-osx-signer.yml +++ b/contrib/gitian-descriptors/gitian-osx-signer.yml @@ -6,9 +6,8 @@ architectures: - "amd64" packages: - "faketime" -reference_datetime: "2016-01-01 00:00:00" remotes: -- "url": "https://github.com/bitcoin/bitcoin-detached-sigs.git" +- "url": "https://github.com/bitcoin-core/bitcoin-detached-sigs.git" "dir": "signature" files: - "bitcoin-osx-unsigned.tar.gz" @@ -34,5 +33,5 @@ script: | tar -xf ${UNSIGNED} OSX_VOLNAME="$(cat osx_volname)" ./detached-sig-apply.sh ${UNSIGNED} signature/osx - ${WRAP_DIR}/genisoimage -no-cache-inodes -D -l -probe -V "${OSX_VOLNAME}" -no-pad -r -apple -o uncompressed.dmg signed-app + ${WRAP_DIR}/genisoimage -no-cache-inodes -D -l -probe -V "${OSX_VOLNAME}" -no-pad -r -dir-mode 0755 -apple -o uncompressed.dmg signed-app ${WRAP_DIR}/dmg dmg uncompressed.dmg ${OUTDIR}/${SIGNED} diff --git a/contrib/gitian-descriptors/gitian-osx.yml b/contrib/gitian-descriptors/gitian-osx.yml index c430932f9da0e..8436cd612a57d 100644 --- a/contrib/gitian-descriptors/gitian-osx.yml +++ b/contrib/gitian-descriptors/gitian-osx.yml @@ -5,7 +5,7 @@ suites: - "trusty" architectures: - "amd64" -packages: +packages: - "ca-certificates" - "curl" - "g++" @@ -27,7 +27,6 @@ packages: - "python-dev" - "python-setuptools" - "fonts-tuffy" -reference_datetime: "2016-01-01 00:00:00" remotes: - "url": "https://github.com/bitcoin/bitcoin.git" "dir": "bitcoin" @@ -54,29 +53,36 @@ script: | export ZERO_AR_DATE=1 - # Create global faketime wrappers + function create_global_faketime_wrappers { for prog in ${FAKETIME_PROGS}; do echo '#!/bin/bash' > ${WRAP_DIR}/${prog} echo "REAL=\`which -a ${prog} | grep -v ${WRAP_DIR}/${prog} | head -1\`" >> ${WRAP_DIR}/${prog} echo 'export LD_PRELOAD=/usr/lib/x86_64-linux-gnu/faketime/libfaketime.so.1' >> ${WRAP_DIR}/${prog} - echo "export FAKETIME=\"${REFERENCE_DATETIME}\"" >> ${WRAP_DIR}/${prog} + echo "export FAKETIME=\"$1\"" >> ${WRAP_DIR}/${prog} echo "\$REAL \$@" >> $WRAP_DIR/${prog} chmod +x ${WRAP_DIR}/${prog} done + } - # Create per-host faketime wrappers + function create_per-host_faketime_wrappers { for i in $HOSTS; do for prog in ${FAKETIME_HOST_PROGS}; do echo '#!/bin/bash' > ${WRAP_DIR}/${i}-${prog} echo "REAL=\`which -a ${i}-${prog} | grep -v ${WRAP_DIR}/${i}-${prog} | head -1\`" >> ${WRAP_DIR}/${i}-${prog} echo 'export LD_PRELOAD=/usr/lib/x86_64-linux-gnu/faketime/libfaketime.so.1' >> ${WRAP_DIR}/${i}-${prog} - echo "export FAKETIME=\"${REFERENCE_DATETIME}\"" >> ${WRAP_DIR}/${i}-${prog} + echo "export FAKETIME=\"$1\"" >> ${WRAP_DIR}/${i}-${prog} echo "\$REAL \$@" >> $WRAP_DIR/${i}-${prog} chmod +x ${WRAP_DIR}/${i}-${prog} done done + } + export PATH=${WRAP_DIR}:${PATH} + # Faketime for depends so intermediate results are comparable + create_global_faketime_wrappers "2000-01-01 12:00:00" + create_per-host_faketime_wrappers "2000-01-01 12:00:00" + cd bitcoin BASEPREFIX=`pwd`/depends @@ -88,9 +94,13 @@ script: | make ${MAKEOPTS} -C ${BASEPREFIX} HOST="${i}" done + # Faketime for binaries + create_global_faketime_wrappers "${REFERENCE_DATETIME}" + create_per-host_faketime_wrappers "${REFERENCE_DATETIME}" + # Create the release tarball using (arbitrarily) the first host ./autogen.sh - ./configure --prefix=${BASEPREFIX}/`echo "${HOSTS}" | awk '{print $1;}'` + CONFIG_SITE=${BASEPREFIX}/`echo "${HOSTS}" | awk '{print $1;}'`/share/config.site ./configure --prefix=/ make dist SOURCEDIST=`echo bitcoin-*.tar.gz` DISTNAME=`echo ${SOURCEDIST} | sed 's/.tar.*//'` @@ -112,9 +122,9 @@ script: | mkdir -p ${INSTALLPATH} tar --strip-components=1 -xf ../$SOURCEDIST - ./configure --prefix=${BASEPREFIX}/${i} --bindir=${INSTALLPATH}/bin --includedir=${INSTALLPATH}/include --libdir=${INSTALLPATH}/lib --disable-ccache --disable-maintainer-mode --disable-dependency-tracking ${CONFIGFLAGS} + CONFIG_SITE=${BASEPREFIX}/${i}/share/config.site ./configure --prefix=/ --disable-ccache --disable-maintainer-mode --disable-dependency-tracking ${CONFIGFLAGS} make ${MAKEOPTS} - make install-strip + make install-strip DESTDIR=${INSTALLPATH} make osx_volname make deploydir diff --git a/contrib/gitian-descriptors/gitian-win-signer.yml b/contrib/gitian-descriptors/gitian-win-signer.yml index 27c4f01eb4ed9..3c1e0214a0aa6 100644 --- a/contrib/gitian-descriptors/gitian-win-signer.yml +++ b/contrib/gitian-descriptors/gitian-win-signer.yml @@ -7,9 +7,8 @@ architectures: packages: - "libssl-dev" - "autoconf" -reference_datetime: "2016-01-01 00:00:00" remotes: -- "url": "https://github.com/bitcoin/bitcoin-detached-sigs.git" +- "url": "https://github.com/bitcoin-core/bitcoin-detached-sigs.git" "dir": "signature" files: - "osslsigncode-1.7.1.tar.gz" diff --git a/contrib/gitian-descriptors/gitian-win.yml b/contrib/gitian-descriptors/gitian-win.yml index 9f7322f0b5a21..1d3a876dfb559 100644 --- a/contrib/gitian-descriptors/gitian-win.yml +++ b/contrib/gitian-descriptors/gitian-win.yml @@ -5,7 +5,7 @@ suites: - "trusty" architectures: - "amd64" -packages: +packages: - "curl" - "g++" - "git-core" @@ -21,7 +21,6 @@ packages: - "zip" - "ca-certificates" - "python" -reference_datetime: "2016-01-01 00:00:00" remotes: - "url": "https://github.com/bitcoin/bitcoin.git" "dir": "bitcoin" @@ -29,9 +28,11 @@ files: [] script: | WRAP_DIR=$HOME/wrapped HOSTS="x86_64-w64-mingw32 i686-w64-mingw32" - CONFIGFLAGS="--enable-reduce-exports --disable-gui-tests" - FAKETIME_HOST_PROGS="g++ ar ranlib nm windres strip" + CONFIGFLAGS="--enable-reduce-exports --disable-bench --disable-gui-tests" + FAKETIME_HOST_PROGS="g++ ar ranlib nm windres strip objcopy" FAKETIME_PROGS="date makensis zip" + HOST_CFLAGS="-O2 -g" + HOST_CXXFLAGS="-O2 -g" export QT_RCC_TEST=1 export GZIP="-9n" @@ -45,29 +46,31 @@ script: | mkdir -p ${BASE_CACHE} ${SOURCES_PATH} fi - # Create global faketime wrappers + function create_global_faketime_wrappers { for prog in ${FAKETIME_PROGS}; do echo '#!/bin/bash' > ${WRAP_DIR}/${prog} echo "REAL=\`which -a ${prog} | grep -v ${WRAP_DIR}/${prog} | head -1\`" >> ${WRAP_DIR}/${prog} echo 'export LD_PRELOAD=/usr/lib/x86_64-linux-gnu/faketime/libfaketime.so.1' >> ${WRAP_DIR}/${prog} - echo "export FAKETIME=\"${REFERENCE_DATETIME}\"" >> ${WRAP_DIR}/${prog} + echo "export FAKETIME=\"$1\"" >> ${WRAP_DIR}/${prog} echo "\$REAL \$@" >> $WRAP_DIR/${prog} chmod +x ${WRAP_DIR}/${prog} done + } - # Create per-host faketime wrappers + function create_per-host_faketime_wrappers { for i in $HOSTS; do for prog in ${FAKETIME_HOST_PROGS}; do echo '#!/bin/bash' > ${WRAP_DIR}/${i}-${prog} echo "REAL=\`which -a ${i}-${prog} | grep -v ${WRAP_DIR}/${i}-${prog} | head -1\`" >> ${WRAP_DIR}/${i}-${prog} echo 'export LD_PRELOAD=/usr/lib/x86_64-linux-gnu/faketime/libfaketime.so.1' >> ${WRAP_DIR}/${i}-${prog} - echo "export FAKETIME=\"${REFERENCE_DATETIME}\"" >> ${WRAP_DIR}/${i}-${prog} + echo "export FAKETIME=\"$1\"" >> ${WRAP_DIR}/${i}-${prog} echo "\$REAL \$@" >> $WRAP_DIR/${i}-${prog} chmod +x ${WRAP_DIR}/${i}-${prog} done done + } - # Create per-host linker wrapper + function create_per-host_linker_wrapper { # This is only needed for trusty, as the mingw linker leaks a few bytes of # heap, causing non-determinism. See discussion in https://github.com/bitcoin/bitcoin/pull/6900 for i in $HOSTS; do @@ -83,15 +86,21 @@ script: | echo '#!/bin/bash' > ${WRAP_DIR}/${i}-${prog} echo "REAL=\`which -a ${i}-${prog} | grep -v ${WRAP_DIR}/${i}-${prog} | head -1\`" >> ${WRAP_DIR}/${i}-${prog} echo 'export LD_PRELOAD=/usr/lib/x86_64-linux-gnu/faketime/libfaketime.so.1' >> ${WRAP_DIR}/${i}-${prog} - echo "export FAKETIME=\"${REFERENCE_DATETIME}\"" >> ${WRAP_DIR}/${i}-${prog} + echo "export FAKETIME=\"$1\"" >> ${WRAP_DIR}/${i}-${prog} echo "export COMPILER_PATH=${WRAP_DIR}/${i}" >> ${WRAP_DIR}/${i}-${prog} echo "\$REAL \$@" >> $WRAP_DIR/${i}-${prog} chmod +x ${WRAP_DIR}/${i}-${prog} done done + } export PATH=${WRAP_DIR}:${PATH} + # Faketime for depends so intermediate results are comparable + create_global_faketime_wrappers "2000-01-01 12:00:00" + create_per-host_faketime_wrappers "2000-01-01 12:00:00" + create_per-host_linker_wrapper "2000-01-01 12:00:00" + cd bitcoin BASEPREFIX=`pwd`/depends # Build dependencies for each host @@ -99,9 +108,14 @@ script: | make ${MAKEOPTS} -C ${BASEPREFIX} HOST="${i}" done + # Faketime for binaries + create_global_faketime_wrappers "${REFERENCE_DATETIME}" + create_per-host_faketime_wrappers "${REFERENCE_DATETIME}" + create_per-host_linker_wrapper "${REFERENCE_DATETIME}" + # Create the release tarball using (arbitrarily) the first host ./autogen.sh - ./configure --prefix=${BASEPREFIX}/`echo "${HOSTS}" | awk '{print $1;}'` + CONFIG_SITE=${BASEPREFIX}/`echo "${HOSTS}" | awk '{print $1;}'`/share/config.site ./configure --prefix=/ make dist SOURCEDIST=`echo bitcoin-*.tar.gz` DISTNAME=`echo ${SOURCEDIST} | sed 's/.tar.*//'` @@ -125,22 +139,28 @@ script: | mkdir -p ${INSTALLPATH} tar --strip-components=1 -xf ../$SOURCEDIST - ./configure --prefix=${BASEPREFIX}/${i} --bindir=${INSTALLPATH}/bin --includedir=${INSTALLPATH}/include --libdir=${INSTALLPATH}/lib --disable-ccache --disable-maintainer-mode --disable-dependency-tracking ${CONFIGFLAGS} + CONFIG_SITE=${BASEPREFIX}/${i}/share/config.site ./configure --prefix=/ --disable-ccache --disable-maintainer-mode --disable-dependency-tracking ${CONFIGFLAGS} CFLAGS="${HOST_CFLAGS}" CXXFLAGS="${HOST_CXXFLAGS}" make ${MAKEOPTS} make ${MAKEOPTS} -C src check-security make deploy - make install-strip + make install DESTDIR=${INSTALLPATH} cp -f bitcoin-*setup*.exe $OUTDIR/ cd installed mv ${DISTNAME}/bin/*.dll ${DISTNAME}/lib/ find . -name "lib*.la" -delete find . -name "lib*.a" -delete rm -rf ${DISTNAME}/lib/pkgconfig - find ${DISTNAME} -type f | sort | zip -X@ ${OUTDIR}/${DISTNAME}-${i}.zip - cd ../.. + find ${DISTNAME}/bin -type f -executable -exec ${i}-objcopy --only-keep-debug {} {}.dbg \; -exec ${i}-strip -s {} \; -exec ${i}-objcopy --add-gnu-debuglink={}.dbg {} \; + find ${DISTNAME}/lib -type f -exec ${i}-objcopy --only-keep-debug {} {}.dbg \; -exec ${i}-strip -s {} \; -exec ${i}-objcopy --add-gnu-debuglink={}.dbg {} \; + find ${DISTNAME} -not -name "*.dbg" -type f | sort | zip -X@ ${OUTDIR}/${DISTNAME}-${i}.zip + find ${DISTNAME} -name "*.dbg" -type f | sort | zip -X@ ${OUTDIR}/${DISTNAME}-${i}-debug.zip + cd ../../ + rm -rf distsrc-${i} done cd $OUTDIR rename 's/-setup\.exe$/-setup-unsigned.exe/' *-setup.exe find . -name "*-setup-unsigned.exe" | sort | tar --no-recursion --mode='u+rw,go+r-w,a+X' --owner=0 --group=0 -c -T - | gzip -9n > ${OUTDIR}/${DISTNAME}-win-unsigned.tar.gz + mv ${OUTDIR}/${DISTNAME}-x86_64-*-debug.zip ${OUTDIR}/${DISTNAME}-win64-debug.zip + mv ${OUTDIR}/${DISTNAME}-i686-*-debug.zip ${OUTDIR}/${DISTNAME}-win32-debug.zip mv ${OUTDIR}/${DISTNAME}-x86_64-*.zip ${OUTDIR}/${DISTNAME}-win64.zip mv ${OUTDIR}/${DISTNAME}-i686-*.zip ${OUTDIR}/${DISTNAME}-win32.zip diff --git a/contrib/gitian-keys/luke-jr-key.pgp b/contrib/gitian-keys/luke-jr-key.pgp index 4406e6d5be02c..a2d34e75e1c16 100644 Binary files a/contrib/gitian-keys/luke-jr-key.pgp and b/contrib/gitian-keys/luke-jr-key.pgp differ diff --git a/contrib/macdeploy/macdeployqtplus b/contrib/macdeploy/macdeployqtplus index 685ed8e5b89c0..f8201e72c649d 100755 --- a/contrib/macdeploy/macdeployqtplus +++ b/contrib/macdeploy/macdeployqtplus @@ -210,8 +210,8 @@ def getFrameworks(binaryPath, verbose): sys.stderr.write(o_stderr) sys.stderr.flush() raise RuntimeError("otool failed with return code %d" % otool.returncode) - - otoolLines = o_stdout.split("\n") + + otoolLines = o_stdout.decode().split("\n") otoolLines.pop(0) # First line is the inspected binary if ".framework" in binaryPath or binaryPath.endswith(".dylib"): otoolLines.pop(0) # Frameworks and dylibs list themselves as a dependency. @@ -676,7 +676,7 @@ if verbose >= 2: print("+ Installing qt.conf +") f = open(os.path.join(applicationBundle.resourcesPath, "qt.conf"), "wb") -f.write(qt_conf) +f.write(qt_conf.encode()) f.close() # ------------------------------------------------ diff --git a/contrib/verify-commits/trusted-keys b/contrib/verify-commits/trusted-keys index a0d0f82db3f43..ad1b28be0cc47 100644 --- a/contrib/verify-commits/trusted-keys +++ b/contrib/verify-commits/trusted-keys @@ -5,3 +5,4 @@ AF8BE07C7049F3A26B239D5325B3083201782B2F 81291FA67D2C379A006A053FEAB5AF94D9E9ABE7 3F1888C6DCA92A6499C4911FDBA1A67379A1A931 32EE5C4C3FA15CCADB46ABE529D4BCB6416F53EC +FE09B823E6D83A3BC7983EAA2D7F2372E50FE137 diff --git a/contrib/verifysfbinaries/README.md b/contrib/verifybinaries/README.md similarity index 81% rename from contrib/verifysfbinaries/README.md rename to contrib/verifybinaries/README.md index 1db3fe52fc2c3..8970f3daa406a 100644 --- a/contrib/verifysfbinaries/README.md +++ b/contrib/verifybinaries/README.md @@ -1,6 +1,13 @@ -### Verify Binaries ### +### Verify Binaries This script attempts to download the signature file `SHA256SUMS.asc` from https://bitcoin.org. It first checks if the signature passes, and then downloads the files specified in the file, and checks if the hashes of these files match those that are specified in the signature file. The script returns 0 if everything passes the checks. It returns 1 if either the signature check or the hash check doesn't pass. If an error occurs the return value is 2. + +Usage: + +```sh +./verify.sh bitcoin-core-0.11.2 +./verify.sh bitcoin-core-0.12.0 +``` diff --git a/contrib/verifysfbinaries/verify.sh b/contrib/verifybinaries/verify.sh similarity index 94% rename from contrib/verifysfbinaries/verify.sh rename to contrib/verifybinaries/verify.sh index 847c507553057..657c3bd33c2e2 100755 --- a/contrib/verifysfbinaries/verify.sh +++ b/contrib/verifybinaries/verify.sh @@ -23,9 +23,6 @@ BASEDIR="https://bitcoin.org/bin/" VERSIONPREFIX="bitcoin-core-" RCVERSIONSTRING="rc" -#this URL is used if a version number is not specified as an argument to the script -SIGNATUREFILE="$BASEDIR""$VERSIONPREFIX""0.10.4/""$RCSUBDIR""$SIGNATUREFILENAME" - if [ ! -d "$WORKINGDIR" ]; then mkdir "$WORKINGDIR" fi @@ -53,7 +50,8 @@ if [ -n "$1" ]; then SIGNATUREFILE="$BASEDIR$SIGNATUREFILENAME" else - BASEDIR="${SIGNATUREFILE%/*}/" + echo "Error: need to specify a version on the command line" + exit 2 fi #first we fetch the file containing the signature diff --git a/contrib/zmq/zmq_sub.py b/contrib/zmq/zmq_sub.py index decf29d42a4c1..6268123dd8e53 100755 --- a/contrib/zmq/zmq_sub.py +++ b/contrib/zmq/zmq_sub.py @@ -3,6 +3,7 @@ import array import binascii import zmq +import struct port = 28332 @@ -19,18 +20,21 @@ msg = zmqSubSocket.recv_multipart() topic = str(msg[0]) body = msg[1] - + sequence = "Unknown"; + if len(msg[-1]) == 4: + msgSequence = struct.unpack('/dev/null) $(host_arch)_$(host_os)_id_string+=$(shell $(host_RANLIB) --version 2>/dev/null) $(host_arch)_$(host_os)_id_string+=$(shell $(host_STRIP) --version 2>/dev/null) -qt_packages_$(NO_QT) = $(qt_packages) $(qt_$(host_os)_packages) -qt_native_packages_$(NO_QT) = $(qt_native_packages) +qt_packages_$(NO_QT) = $(qt_packages) $(qt_$(host_os)_packages) $(qt_$(host_arch)_$(host_os)_packages) wallet_packages_$(NO_WALLET) = $(wallet_packages) upnp_packages_$(NO_UPNP) = $(upnp_packages) packages += $($(host_arch)_$(host_os)_packages) $($(host_os)_packages) $(qt_packages_) $(wallet_packages_) $(upnp_packages_) -native_packages += $($(host_arch)_$(host_os)_native_packages) $($(host_os)_native_packages) $(qt_native_packages_) +native_packages += $($(host_arch)_$(host_os)_native_packages) $($(host_os)_native_packages) + +ifneq ($(qt_packages_),) +native_packages += $(qt_native_packages) +endif + all_packages = $(packages) $(native_packages) meta_depends = Makefile funcs.mk builders/default.mk hosts/default.mk hosts/$(host_os).mk builders/$(build_os).mk diff --git a/depends/builders/darwin.mk b/depends/builders/darwin.mk index 200d6ed22a230..27f550ab036ae 100644 --- a/depends/builders/darwin.mk +++ b/depends/builders/darwin.mk @@ -11,7 +11,7 @@ build_darwin_DOWNLOAD = curl --location --fail --connect-timeout $(DOWNLOAD_CONN #darwin host on darwin builder. overrides darwin host preferences. darwin_CC=$(shell xcrun -f clang) -mmacosx-version-min=$(OSX_MIN_VERSION) -darwin_CXX:=$(shell xcrun -f clang++) -mmacosx-version-min=$(OSX_MIN_VERSION) +darwin_CXX:=$(shell xcrun -f clang++) -mmacosx-version-min=$(OSX_MIN_VERSION) -stdlib=libc++ darwin_AR:=$(shell xcrun -f ar) darwin_RANLIB:=$(shell xcrun -f ranlib) darwin_STRIP:=$(shell xcrun -f strip) diff --git a/depends/config.guess b/depends/config.guess index 373a659a06760..c4bd827a7bedc 100755 --- a/depends/config.guess +++ b/depends/config.guess @@ -2,7 +2,7 @@ # Attempt to guess a canonical system name. # Copyright 1992-2016 Free Software Foundation, Inc. -timestamp='2016-02-11' +timestamp='2016-05-15' # This file is free software; you can redistribute it and/or modify it # under the terms of the GNU General Public License as published by @@ -186,9 +186,12 @@ case "${UNAME_MACHINE}:${UNAME_SYSTEM}:${UNAME_RELEASE}:${UNAME_VERSION}" in *) machine=${UNAME_MACHINE_ARCH}-unknown ;; esac # The Operating System including object format, if it has switched - # to ELF recently, or will in the future. + # to ELF recently (or will in the future) and ABI. case "${UNAME_MACHINE_ARCH}" in - arm*|earm*|i386|m68k|ns32k|sh3*|sparc|vax) + earm*) + os=netbsdelf + ;; + arm*|i386|m68k|ns32k|sh3*|sparc|vax) eval $set_cc_for_build if echo __ELF__ | $CC_FOR_BUILD -E - 2>/dev/null \ | grep -q __ELF__ @@ -386,7 +389,7 @@ case "${UNAME_MACHINE}:${UNAME_SYSTEM}:${UNAME_RELEASE}:${UNAME_VERSION}" in # This test works for both compilers. if [ "$CC_FOR_BUILD" != no_compiler_found ]; then if (echo '#ifdef __amd64'; echo IS_64BIT_ARCH; echo '#endif') | \ - (CCOPTS= $CC_FOR_BUILD -E - 2>/dev/null) | \ + (CCOPTS="" $CC_FOR_BUILD -E - 2>/dev/null) | \ grep IS_64BIT_ARCH >/dev/null then SUN_ARCH=x86_64 @@ -684,7 +687,7 @@ EOF exit (0); } EOF - (CCOPTS= $CC_FOR_BUILD -o $dummy $dummy.c 2>/dev/null) && HP_ARCH=`$dummy` + (CCOPTS="" $CC_FOR_BUILD -o $dummy $dummy.c 2>/dev/null) && HP_ARCH=`$dummy` test -z "$HP_ARCH" && HP_ARCH=hppa fi ;; esac @@ -701,7 +704,7 @@ EOF # $ CC_FOR_BUILD="cc +DA2.0w" ./config.guess # => hppa64-hp-hpux11.23 - if echo __LP64__ | (CCOPTS= $CC_FOR_BUILD -E - 2>/dev/null) | + if echo __LP64__ | (CCOPTS="" $CC_FOR_BUILD -E - 2>/dev/null) | grep -q __LP64__ then HP_ARCH=hppa2.0w @@ -900,7 +903,7 @@ EOF exit ;; *:GNU/*:*:*) # other systems with GNU libc and userland - echo ${UNAME_MACHINE}-unknown-`echo ${UNAME_SYSTEM} | sed 's,^[^/]*/,,' | tr '[A-Z]' '[a-z]'``echo ${UNAME_RELEASE}|sed -e 's/[-(].*//'`-${LIBC} + echo ${UNAME_MACHINE}-unknown-`echo ${UNAME_SYSTEM} | sed 's,^[^/]*/,,' | tr "[:upper:]" "[:lower:]"``echo ${UNAME_RELEASE}|sed -e 's/[-(].*//'`-${LIBC} exit ;; i*86:Minix:*:*) echo ${UNAME_MACHINE}-pc-minix @@ -1276,6 +1279,9 @@ EOF SX-8R:SUPER-UX:*:*) echo sx8r-nec-superux${UNAME_RELEASE} exit ;; + SX-ACE:SUPER-UX:*:*) + echo sxace-nec-superux${UNAME_RELEASE} + exit ;; Power*:Rhapsody:*:*) echo powerpc-apple-rhapsody${UNAME_RELEASE} exit ;; @@ -1291,7 +1297,7 @@ EOF if test `echo "$UNAME_RELEASE" | sed -e 's/\..*//'` -le 10 ; then if [ "$CC_FOR_BUILD" != no_compiler_found ]; then if (echo '#ifdef __LP64__'; echo IS_64BIT_ARCH; echo '#endif') | \ - (CCOPTS= $CC_FOR_BUILD -E - 2>/dev/null) | \ + (CCOPTS="" $CC_FOR_BUILD -E - 2>/dev/null) | \ grep IS_64BIT_ARCH >/dev/null then case $UNAME_PROCESSOR in @@ -1386,7 +1392,7 @@ EOF echo i386-pc-xenix exit ;; i*86:skyos:*:*) - echo ${UNAME_MACHINE}-pc-skyos`echo ${UNAME_RELEASE}` | sed -e 's/ .*$//' + echo ${UNAME_MACHINE}-pc-skyos`echo ${UNAME_RELEASE} | sed -e 's/ .*$//'` exit ;; i*86:rdos:*:*) echo ${UNAME_MACHINE}-pc-rdos @@ -1405,18 +1411,17 @@ esac cat >&2 < in order to provide the needed -information to handle your system. +If $0 has already been updated, send the following data and any +information you think might be pertinent to config-patches@gnu.org to +provide the necessary information to handle your system. config.guess timestamp = $timestamp diff --git a/depends/config.site.in b/depends/config.site.in index 984ddb1e62b01..e731537bf76bd 100644 --- a/depends/config.site.in +++ b/depends/config.site.in @@ -1,24 +1,26 @@ +depends_prefix="`dirname ${ac_site_file}`/.." + cross_compiling=maybe host_alias=@HOST@ ac_tool_prefix=${host_alias}- if test -z $with_boost; then - with_boost=$prefix + with_boost=$depends_prefix fi if test -z $with_qt_plugindir; then - with_qt_plugindir=$prefix/plugins + with_qt_plugindir=$depends_prefix/plugins fi if test -z $with_qt_translationdir; then - with_qt_translationdir=$prefix/translations + with_qt_translationdir=$depends_prefix/translations fi if test -z $with_qt_bindir; then - with_qt_bindir=$prefix/native/bin + with_qt_bindir=$depends_prefix/native/bin fi if test -z $with_protoc_bindir; then - with_protoc_bindir=$prefix/native/bin + with_protoc_bindir=$depends_prefix/native/bin fi if test -z $with_comparison_tool; then - with_comparison_tool=$prefix/native/share/BitcoindComparisonTool_jar/BitcoindComparisonTool.jar + with_comparison_tool=$depends_prefix/native/share/BitcoindComparisonTool_jar/BitcoindComparisonTool.jar fi @@ -41,32 +43,32 @@ fi if test x@host_os@ = xmingw32; then if test -z $with_qt_incdir; then - with_qt_incdir=$prefix/include + with_qt_incdir=$depends_prefix/include fi if test -z $with_qt_libdir; then - with_qt_libdir=$prefix/lib + with_qt_libdir=$depends_prefix/lib fi fi -PATH=$prefix/native/bin:$PATH +PATH=$depends_prefix/native/bin:$PATH PKG_CONFIG="`which pkg-config` --static" # These two need to remain exported because pkg-config does not see them # otherwise. That means they must be unexported at the end of configure.ac to # avoid ruining the cache. Sigh. -export PKG_CONFIG_LIBDIR=$prefix/lib/pkgconfig -export PKG_CONFIG_PATH=$prefix/share/pkgconfig +export PKG_CONFIG_LIBDIR=$depends_prefix/lib/pkgconfig +export PKG_CONFIG_PATH=$depends_prefix/share/pkgconfig -CPPFLAGS="-I$prefix/include/ $CPPFLAGS" -LDFLAGS="-L$prefix/lib $LDFLAGS" +CPPFLAGS="-I$depends_prefix/include/ $CPPFLAGS" +LDFLAGS="-L$depends_prefix/lib $LDFLAGS" CC="@CC@" CXX="@CXX@" OBJC="${CC}" OBJCXX="${CXX}" -CCACHE=$prefix/native/bin/ccache -PYTHONPATH=$prefix/native/lib/python/dist-packages:$PYTHONPATH +CCACHE=$depends_prefix/native/bin/ccache +PYTHONPATH=$depends_prefix/native/lib/python/dist-packages:$PYTHONPATH if test -n "@AR@"; then AR=@AR@ diff --git a/depends/config.sub b/depends/config.sub index 6223dde931918..6d86a1e2f77b3 100755 --- a/depends/config.sub +++ b/depends/config.sub @@ -2,7 +2,7 @@ # Configuration validation subroutine script. # Copyright 1992-2016 Free Software Foundation, Inc. -timestamp='2016-01-01' +timestamp='2016-05-10' # This file is free software; you can redistribute it and/or modify it # under the terms of the GNU General Public License as published by @@ -1399,7 +1399,7 @@ case $os in | -morphos* | -superux* | -rtmk* | -rtmk-nova* | -windiss* \ | -powermax* | -dnix* | -nx6 | -nx7 | -sei* | -dragonfly* \ | -skyos* | -haiku* | -rdos* | -toppers* | -drops* | -es* \ - | -onefs* | -tirtos*) + | -onefs* | -tirtos* | -phoenix*) # Remember, each alternative MUST END IN *, to match a version number. ;; -qnx*) @@ -1531,6 +1531,8 @@ case $os in ;; -nacl*) ;; + -ios) + ;; -none) ;; *) diff --git a/depends/hosts/darwin.mk b/depends/hosts/darwin.mk index 2958dc50cc8dc..dbe6d00795e00 100644 --- a/depends/hosts/darwin.mk +++ b/depends/hosts/darwin.mk @@ -3,7 +3,7 @@ OSX_SDK_VERSION=10.9 OSX_SDK=$(SDK_PATH)/MacOSX$(OSX_SDK_VERSION).sdk LD64_VERSION=241.9 darwin_CC=clang -target $(host) -mmacosx-version-min=$(OSX_MIN_VERSION) --sysroot $(OSX_SDK) -mlinker-version=$(LD64_VERSION) -darwin_CXX=clang++ -target $(host) -mmacosx-version-min=$(OSX_MIN_VERSION) --sysroot $(OSX_SDK) -mlinker-version=$(LD64_VERSION) +darwin_CXX=clang++ -target $(host) -mmacosx-version-min=$(OSX_MIN_VERSION) --sysroot $(OSX_SDK) -mlinker-version=$(LD64_VERSION) -stdlib=libc++ darwin_CFLAGS=-pipe darwin_CXXFLAGS=$(darwin_CFLAGS) diff --git a/depends/packages/bdb.mk b/depends/packages/bdb.mk index e2f85ad4fcad8..6c9876c2c7c7c 100644 --- a/depends/packages/bdb.mk +++ b/depends/packages/bdb.mk @@ -9,11 +9,13 @@ define $(package)_set_vars $(package)_config_opts=--disable-shared --enable-cxx --disable-replication $(package)_config_opts_mingw32=--enable-mingw $(package)_config_opts_linux=--with-pic +$(package)_cxxflags=-std=c++11 endef define $(package)_preprocess_cmds sed -i.old 's/__atomic_compare_exchange/__atomic_compare_exchange_db/' dbinc/atomic.h && \ - sed -i.old 's/atomic_init/atomic_init_db/' dbinc/atomic.h mp/mp_region.c mp/mp_mvcc.c mp/mp_fget.c mutex/mut_method.c mutex/mut_tas.c + sed -i.old 's/atomic_init/atomic_init_db/' dbinc/atomic.h mp/mp_region.c mp/mp_mvcc.c mp/mp_fget.c mutex/mut_method.c mutex/mut_tas.c && \ + cp -f $(BASEDIR)/config.guess $(BASEDIR)/config.sub dist endef define $(package)_config_cmds diff --git a/depends/packages/boost.mk b/depends/packages/boost.mk index 215c694b6b82b..ef1307c241b66 100644 --- a/depends/packages/boost.mk +++ b/depends/packages/boost.mk @@ -20,7 +20,7 @@ $(package)_archiver_$(host_os)=$($(package)_ar) $(package)_toolset_darwin=darwin $(package)_archiver_darwin=$($(package)_libtool) $(package)_config_libraries=chrono,filesystem,program_options,system,thread,test -$(package)_cxxflags=-fvisibility=hidden +$(package)_cxxflags=-std=c++11 -fvisibility=hidden $(package)_cxxflags_linux=-fPIC endef diff --git a/depends/packages/expat.mk b/depends/packages/expat.mk index 1ac443537420b..bd2927563850b 100644 --- a/depends/packages/expat.mk +++ b/depends/packages/expat.mk @@ -1,8 +1,8 @@ package=expat -$(package)_version=2.1.0 -$(package)_download_path=http://sourceforge.net/projects/expat/files/expat/$($(package)_version) -$(package)_file_name=$(package)-$($(package)_version).tar.gz -$(package)_sha256_hash=823705472f816df21c8f6aa026dd162b280806838bb55b3432b0fb1fcca7eb86 +$(package)_version=2.1.1 +$(package)_download_path=https://downloads.sourceforge.net/project/expat/expat/$($(package)_version) +$(package)_file_name=$(package)-$($(package)_version).tar.bz2 +$(package)_sha256_hash=aff584e5a2f759dcfc6d48671e9529f6afe1e30b0cd6a4cec200cbe3f793de67 define $(package)_set_vars $(package)_config_opts=--disable-static diff --git a/depends/packages/freetype.mk b/depends/packages/freetype.mk index f7d6e0f9fc58d..7cea28ff0bbf9 100644 --- a/depends/packages/freetype.mk +++ b/depends/packages/freetype.mk @@ -1,8 +1,8 @@ package=freetype -$(package)_version=2.5.3 -$(package)_download_path=http://downloads.sourceforge.net/$(package) +$(package)_version=2.6.3 +$(package)_download_path=http://download.savannah.gnu.org/releases/$(package) $(package)_file_name=$(package)-$($(package)_version).tar.bz2 -$(package)_sha256_hash=c0848b29d52ef3ca27ad92e08351f023c5e24ce8cea7d8fe69fc96358e65f75e +$(package)_sha256_hash=371e707aa522acf5b15ce93f11183c725b8ed1ee8546d7b3af549863045863a2 define $(package)_set_vars $(package)_config_opts=--without-zlib --without-png --disable-static diff --git a/depends/packages/miniupnpc.mk b/depends/packages/miniupnpc.mk index 45fa03631f89d..e34cf7be2f1bf 100644 --- a/depends/packages/miniupnpc.mk +++ b/depends/packages/miniupnpc.mk @@ -1,8 +1,8 @@ package=miniupnpc -$(package)_version=1.9.20160209 +$(package)_version=2.0 $(package)_download_path=http://miniupnp.free.fr/files $(package)_file_name=$(package)-$($(package)_version).tar.gz -$(package)_sha256_hash=572171eacc1d72537ce47b6f4571260757ab7bcfdaf54c3a55c7f88594d94b6f +$(package)_sha256_hash=d434ceb8986efbe199c5ca53f90ed53eab290b1e6d0530b717eb6fa49d61f93b define $(package)_set_vars $(package)_build_opts=CC="$($(package)_cc)" diff --git a/depends/packages/native_ccache.mk b/depends/packages/native_ccache.mk index cc76f9a794655..9216e175983a9 100644 --- a/depends/packages/native_ccache.mk +++ b/depends/packages/native_ccache.mk @@ -1,8 +1,8 @@ package=native_ccache -$(package)_version=3.2.4 +$(package)_version=3.2.5 $(package)_download_path=http://samba.org/ftp/ccache $(package)_file_name=ccache-$($(package)_version).tar.bz2 -$(package)_sha256_hash=ffeb967edb549e67da0bd5f44f729a2022de9fdde65dfd80d2a7204d7f75332e +$(package)_sha256_hash=7a553809e90faf9de3a23ee9c5b5f786cfd4836bf502744bedb824a24bee1097 define $(package)_set_vars $(package)_config_opts= diff --git a/depends/packages/native_mac_alias.mk b/depends/packages/native_mac_alias.mk index d117c1c9a3f68..85a8a402bf92c 100644 --- a/depends/packages/native_mac_alias.mk +++ b/depends/packages/native_mac_alias.mk @@ -5,6 +5,11 @@ $(package)_download_file=v$($(package)_version).tar.bz2 $(package)_file_name=$(package)-$($(package)_version).tar.bz2 $(package)_sha256_hash=87ad827e66790028361e43fc754f68ed041a9bdb214cca03c853f079b04fb120 $(package)_install_libdir=$(build_prefix)/lib/python/dist-packages +$(package)_patches=python3.patch + +define $(package)_preprocess_cmds + patch -p1 < $($(package)_patch_dir)/python3.patch +endef define $(package)_build_cmds python setup.py build diff --git a/depends/packages/packages.mk b/depends/packages/packages.mk index 59b009b66a27b..ac43ef4a2e304 100644 --- a/depends/packages/packages.mk +++ b/depends/packages/packages.mk @@ -6,7 +6,9 @@ native_packages := native_ccache native_comparisontool qt_native_packages = native_protobuf qt_packages = qrencode protobuf -qt_linux_packages= qt expat dbus libxcb xcb_proto libXau xproto freetype fontconfig libX11 xextproto libXext xtrans +qt_x86_64_linux_packages:=qt expat dbus libxcb xcb_proto libXau xproto freetype fontconfig libX11 xextproto libXext xtrans +qt_i686_linux_packages:=$(qt_x86_64_linux_packages) + qt_darwin_packages=qt qt_mingw32_packages=qt diff --git a/depends/packages/protobuf.mk b/depends/packages/protobuf.mk index 5affad28375a9..54d3fd924519b 100644 --- a/depends/packages/protobuf.mk +++ b/depends/packages/protobuf.mk @@ -4,6 +4,7 @@ $(package)_download_path=$(native_$(package)_download_path) $(package)_file_name=$(native_$(package)_file_name) $(package)_sha256_hash=$(native_$(package)_sha256_hash) $(package)_dependencies=native_$(package) +$(package)_cxxflags=-std=c++11 define $(package)_set_vars $(package)_config_opts=--disable-shared --with-protoc=$(build_prefix)/bin/protoc diff --git a/depends/packages/qt.mk b/depends/packages/qt.mk index 77df77b73f7f0..c1fc8e3058e9e 100644 --- a/depends/packages/qt.mk +++ b/depends/packages/qt.mk @@ -31,7 +31,7 @@ $(package)_config_opts += -no-iconv $(package)_config_opts += -no-gif $(package)_config_opts += -no-freetype $(package)_config_opts += -no-nis -$(package)_config_opts += -no-pch +$(package)_config_opts += -pch $(package)_config_opts += -no-qml-debug $(package)_config_opts += -nomake examples $(package)_config_opts += -nomake tests @@ -49,7 +49,7 @@ $(package)_config_opts += -no-sql-sqlite2 $(package)_config_opts += -prefix $(host_prefix) $(package)_config_opts += -hostprefix $(build_prefix) $(package)_config_opts += -bindir $(build_prefix)/bin -$(package)_config_opts += -no-c++11 +$(package)_config_opts += -c++11 $(package)_config_opts += -openssl-linked $(package)_config_opts += -v $(package)_config_opts += -static diff --git a/depends/packages/zeromq.mk b/depends/packages/zeromq.mk index 7b866e9c0f662..f8901f72c26c6 100644 --- a/depends/packages/zeromq.mk +++ b/depends/packages/zeromq.mk @@ -1,12 +1,13 @@ package=zeromq -$(package)_version=4.0.7 +$(package)_version=4.1.4 $(package)_download_path=http://download.zeromq.org $(package)_file_name=$(package)-$($(package)_version).tar.gz -$(package)_sha256_hash=e00b2967e074990d0538361cc79084a0a92892df2c6e7585da34e4c61ee47b03 +$(package)_sha256_hash=e99f44fde25c2e4cb84ce440f87ca7d3fe3271c2b8cfbc67d55e4de25e6fe378 define $(package)_set_vars - $(package)_config_opts=--without-documentation --disable-shared + $(package)_config_opts=--without-documentation --disable-shared --without-libsodium $(package)_config_opts_linux=--with-pic + $(package)_cxxflags=-std=c++11 endef define $(package)_config_cmds @@ -14,11 +15,11 @@ define $(package)_config_cmds endef define $(package)_build_cmds - $(MAKE) -C src + $(MAKE) libzmq.la endef define $(package)_stage_cmds - $(MAKE) -C src DESTDIR=$($(package)_staging_dir) install + $(MAKE) DESTDIR=$($(package)_staging_dir) install-libLTLIBRARIES install-includeHEADERS install-pkgconfigDATA endef define $(package)_postprocess_cmds diff --git a/depends/patches/native_mac_alias/python3.patch b/depends/patches/native_mac_alias/python3.patch new file mode 100644 index 0000000000000..1a32340be5b13 --- /dev/null +++ b/depends/patches/native_mac_alias/python3.patch @@ -0,0 +1,72 @@ +diff -dur a/mac_alias/alias.py b/mac_alias/alias.py +--- a/mac_alias/alias.py 2015-10-19 12:12:48.000000000 +0200 ++++ b/mac_alias/alias.py 2016-04-03 12:13:12.037159417 +0200 +@@ -243,10 +243,10 @@ + alias = Alias() + alias.appinfo = appinfo + +- alias.volume = VolumeInfo (volname.replace('/',':'), ++ alias.volume = VolumeInfo (volname.decode().replace('/',':'), + voldate, fstype, disktype, + volattrs, volfsid) +- alias.target = TargetInfo (kind, filename.replace('/',':'), ++ alias.target = TargetInfo (kind, filename.decode().replace('/',':'), + folder_cnid, cnid, + crdate, creator_code, type_code) + alias.target.levels_from = levels_from +@@ -261,9 +261,9 @@ + b.read(1) + + if tag == TAG_CARBON_FOLDER_NAME: +- alias.target.folder_name = value.replace('/',':') ++ alias.target.folder_name = value.decode().replace('/',':') + elif tag == TAG_CNID_PATH: +- alias.target.cnid_path = struct.unpack(b'>%uI' % (length // 4), ++ alias.target.cnid_path = struct.unpack('>%uI' % (length // 4), + value) + elif tag == TAG_CARBON_PATH: + alias.target.carbon_path = value +@@ -298,9 +298,9 @@ + alias.target.creation_date \ + = mac_epoch + datetime.timedelta(seconds=seconds) + elif tag == TAG_POSIX_PATH: +- alias.target.posix_path = value ++ alias.target.posix_path = value.decode() + elif tag == TAG_POSIX_PATH_TO_MOUNTPOINT: +- alias.volume.posix_path = value ++ alias.volume.posix_path = value.decode() + elif tag == TAG_RECURSIVE_ALIAS_OF_DISK_IMAGE: + alias.volume.disk_image_alias = Alias.from_bytes(value) + elif tag == TAG_USER_HOME_LENGTH_PREFIX: +@@ -422,13 +422,13 @@ + # (so doing so is ridiculous, and nothing could rely on it). + b.write(struct.pack(b'>h28pI2shI64pII4s4shhI2s10s', + self.target.kind, +- carbon_volname, voldate, ++ carbon_volname, int(voldate), + self.volume.fs_type, + self.volume.disk_type, + self.target.folder_cnid, + carbon_filename, + self.target.cnid, +- crdate, ++ int(crdate), + self.target.creator_code, + self.target.type_code, + self.target.levels_from, +@@ -449,12 +449,12 @@ + + b.write(struct.pack(b'>hhQhhQ', + TAG_HIGH_RES_VOLUME_CREATION_DATE, +- 8, long(voldate * 65536), ++ 8, int(voldate * 65536), + TAG_HIGH_RES_CREATION_DATE, +- 8, long(crdate * 65536))) ++ 8, int(crdate * 65536))) + + if self.target.cnid_path: +- cnid_path = struct.pack(b'>%uI' % len(self.target.cnid_path), ++ cnid_path = struct.pack('>%uI' % len(self.target.cnid_path), + *self.target.cnid_path) + b.write(struct.pack(b'>hh', TAG_CNID_PATH, + len(cnid_path))) diff --git a/doc/README.md b/doc/README.md index cf475ef18e5fc..c30f29452b786 100644 --- a/doc/README.md +++ b/doc/README.md @@ -11,16 +11,10 @@ The following are some helpful notes on how to run Bitcoin on your native platfo ### Unix -You need the Qt4 run-time libraries to run Bitcoin-Qt. On Debian or Ubuntu: - - sudo apt-get install libqtgui4 - Unpack the files into a directory and run: -- bin/32/bitcoin-qt (GUI, 32-bit) or bin/32/bitcoind (headless, 32-bit) -- bin/64/bitcoin-qt (GUI, 64-bit) or bin/64/bitcoind (headless, 64-bit) - - +- `bin/bitcoin-qt` (GUI) or +- `bin/bitcoind` (headless) ### Windows @@ -63,6 +57,7 @@ The Bitcoin repo's [root README](/README.md) contains relevant information on th - [Shared Libraries](shared-libraries.md) - [BIPS](bips.md) - [Dnsseed Policy](dnsseed-policy.md) +- [Benchmarking](benchmarking.md) ### Resources * Discuss on the [BitcoinTalk](https://bitcointalk.org/) forums, in the [Development & Technical Discussion board](https://bitcointalk.org/index.php?board=6.0). diff --git a/doc/benchmarking.md b/doc/benchmarking.md new file mode 100644 index 0000000000000..0ba75afcda3cb --- /dev/null +++ b/doc/benchmarking.md @@ -0,0 +1,30 @@ +Benchmarking +============ + +Bitcoin Core has an internal benchmarking framework, with benchmarks +for cryptographic algorithms such as SHA1, SHA256, SHA512 and RIPEMD160. As well as the rolling bloom filter. + +After compiling bitcoin-core, the benchmarks can be run with: +`src/bench/bench_bitcoin` + +The output will look similar to: +``` +#Benchmark,count,min,max,average +RIPEMD160,448,0.001245033173334,0.002638196945190,0.002461894814457 +RollingBloom-refresh,1,0.000635000000000,0.000635000000000,0.000635000000000 +RollingBloom-refresh,1,0.000108000000000,0.000108000000000,0.000108000000000 +RollingBloom-refresh,1,0.000107000000000,0.000107000000000,0.000107000000000 +RollingBloom-refresh,1,0.000204000000000,0.000204000000000,0.000204000000000 +SHA1,640,0.000909024336207,0.001938136418660,0.001843086257577 +SHA256,256,0.002209486499909,0.008500099182129,0.004300644621253 +SHA512,384,0.001319904176016,0.002813005447388,0.002615700786312 +Sleep100ms,10,0.205592155456543,0.210056066513062,0.104166316986084 +Trig,67108864,0.000000014997003,0.000000015448112,0.000000015188842 +``` + +More benchmarks are needed for, in no particular order: +- Script Validation +- CCoinDBView caching +- Coins database +- Memory pool +- Wallet coin selection diff --git a/doc/bips.md b/doc/bips.md index b8efabbcf251c..b4b62e781ec61 100644 --- a/doc/bips.md +++ b/doc/bips.md @@ -1,5 +1,6 @@ -BIPs that are implemented by Bitcoin Core (up-to-date up to **v0.12.0**): +BIPs that are implemented by Bitcoin Core (up-to-date up to **v0.13.0**): +* [`BIP 9`](https://github.com/bitcoin/bips/blob/master/bip-0009.mediawiki): The changes allowing multiple soft-forks to be deployed in parallel have been implemented since **v0.12.1** ([PR #7575](https://github.com/bitcoin/bitcoin/pull/7575)) * [`BIP 11`](https://github.com/bitcoin/bips/blob/master/bip-0011.mediawiki): Multisig outputs are standard since **v0.6.0** ([PR #669](https://github.com/bitcoin/bitcoin/pull/669)). * [`BIP 13`](https://github.com/bitcoin/bips/blob/master/bip-0013.mediawiki): The address format for P2SH addresses has been implemented since **v0.6.0** ([PR #669](https://github.com/bitcoin/bitcoin/pull/669)). * [`BIP 14`](https://github.com/bitcoin/bips/blob/master/bip-0014.mediawiki): The subversion string is being used as User Agent since **v0.6.0** ([PR #669](https://github.com/bitcoin/bitcoin/pull/669)). @@ -16,8 +17,11 @@ BIPs that are implemented by Bitcoin Core (up-to-date up to **v0.12.0**): * [`BIP 61`](https://github.com/bitcoin/bips/blob/master/bip-0061.mediawiki): The 'reject' protocol message (and the protocol version bump to 70002) was added in **v0.9.0** ([PR #3185](https://github.com/bitcoin/bitcoin/pull/3185)). * [`BIP 65`](https://github.com/bitcoin/bips/blob/master/bip-0065.mediawiki): The CHECKLOCKTIMEVERIFY softfork was merged in **v0.12.0** ([PR #6351](https://github.com/bitcoin/bitcoin/pull/6351)), and backported to **v0.11.2** and **v0.10.4**. Mempool-only CLTV was added in [PR #6124](https://github.com/bitcoin/bitcoin/pull/6124). * [`BIP 66`](https://github.com/bitcoin/bips/blob/master/bip-0066.mediawiki): The strict DER rules and associated version 3 blocks have been implemented since **v0.10.0** ([PR #5713](https://github.com/bitcoin/bitcoin/pull/5713)). +* [`BIP 68`](https://github.com/bitcoin/bips/blob/master/bip-0068.mediawiki): Sequence locks have been implemented as of **v0.12.1** ([PR #7184](https://github.com/bitcoin/bitcoin/pull/7184)). * [`BIP 70`](https://github.com/bitcoin/bips/blob/master/bip-0070.mediawiki) [`71`](https://github.com/bitcoin/bips/blob/master/bip-0071.mediawiki) [`72`](https://github.com/bitcoin/bips/blob/master/bip-0072.mediawiki): Payment Protocol support has been available in Bitcoin Core GUI since **v0.9.0** ([PR #5216](https://github.com/bitcoin/bitcoin/pull/5216)). * [`BIP 111`](https://github.com/bitcoin/bips/blob/master/bip-0111.mediawiki): `NODE_BLOOM` service bit added, and enforced for all peer versions as of **v0.13.0** ([PR #6579](https://github.com/bitcoin/bitcoin/pull/6579) and [PR #6641](https://github.com/bitcoin/bitcoin/pull/6641)). +* [`BIP 112`](https://github.com/bitcoin/bips/blob/master/bip-0112.mediawiki): The CHECKSEQUENCEVERIFY opcode has been implemented since **v0.12.1** ([PR #7524](https://github.com/bitcoin/bitcoin/pull/7524)). +* [`BIP 113`](https://github.com/bitcoin/bips/blob/master/bip-0113.mediawiki): Median time past lock-time calculations have been implemented since **v0.12.1** ([PR #6566](https://github.com/bitcoin/bitcoin/pull/6566)). * [`BIP 125`](https://github.com/bitcoin/bips/blob/master/bip-0125.mediawiki): Opt-in full replace-by-fee signaling honoured in mempool and mining as of **v0.12.0** ([PR 6871](https://github.com/bitcoin/bitcoin/pull/6871)). * [`BIP 130`](https://github.com/bitcoin/bips/blob/master/bip-0130.mediawiki): direct headers announcement is negotiated with peer versions `>=70012` as of **v0.12.0** ([PR 6494](https://github.com/bitcoin/bitcoin/pull/6494)). * [`BIP 133`](https://github.com/bitcoin/bips/blob/master/bip-0133.mediawiki): feefilter messages are respected and sent for peer versions `>=70013` as of **v0.13.0** ([PR 7542](https://github.com/bitcoin/bitcoin/pull/7542)). diff --git a/doc/build-osx.md b/doc/build-osx.md index 296e0aa1f0726..89d7816c9a42b 100644 --- a/doc/build-osx.md +++ b/doc/build-osx.md @@ -1,129 +1,93 @@ Mac OS X Build Instructions and Notes ==================================== -This guide will show you how to build Bitcoin Core for OS X. - -Notes ------ - -* Tested on OS X 10.7 through 10.11 on 64-bit Intel processors only. - -* All of the commands should be executed in a Terminal application. The -built-in one is located in `/Applications/Utilities`. +The commands in this guide should be executed in a Terminal application. +The built-in one is located in `/Applications/Utilities/Terminal.app`. Preparation ----------- +Download and install [Xcode](https://developer.apple.com/xcode/download). -You need to install Xcode with all the options checked so that the compiler -and everything is available in /usr not just /Developer. Xcode should be -available on your OS X installation media, but if not, you can get the -current version from https://developer.apple.com/xcode/. If you install -Xcode 4.3 or later, you'll need to install its command line tools. This can -be done in `Xcode > Preferences > Downloads > Components` and generally must -be re-done or updated every time Xcode is updated. - -You will also need to install [Homebrew](http://brew.sh) in order to install library -dependencies. +Once installed, run `xcode-select --install` to install the OS X command line tools. -The installation of the actual dependencies is covered in the instructions -sections below. +Install [Homebrew](http://brew.sh). -Instructions: Homebrew +Dependencies ---------------------- -#### Install dependencies using Homebrew + brew install automake berkeley-db4 libtool boost --c++11 miniupnpc openssl pkg-config protobuf --c++11 qt5 libevent - brew install autoconf automake berkeley-db4 libtool boost miniupnpc openssl pkg-config protobuf qt5 libevent +NOTE: Building with Qt4 is still supported, however, could result in a broken UI. Building with Qt5 is recommended. -NOTE: Building with Qt4 is still supported, however, could result in a broken UI. As such, building with Qt5 is recommended. - -### Building `bitcoin` +Build Bitcoin Core +------------------------ -1. Clone the GitHub tree to get the source code and go into the directory. +1. Clone the bitcoin source code and cd into `bitcoin` - git clone https://github.com/bitcoin/bitcoin.git + git clone https://github.com/bitcoin/bitcoin cd bitcoin 2. Build bitcoin-core: - This will configure and build the headless bitcoin binaries as well as the gui (if Qt is found). - You can disable the gui build by passing `--without-gui` to configure. + + Configure and build the headless bitcoin binaries as well as the GUI (if Qt is found). + + You can disable the GUI build by passing `--without-gui` to configure. ./autogen.sh ./configure make -3. It is also a good idea to build and run the unit tests: +3. It is recommended to build and run the unit tests: make check -4. (Optional) You can also install bitcoind to your path: - - make install - -Use Qt Creator as IDE ------------------------- -You can use Qt Creator as IDE, for debugging and for manipulating forms, etc. -Download Qt Creator from https://www.qt.io/download/. Download the "community edition" and only install Qt Creator (uncheck the rest during the installation process). - -1. Make sure you installed everything through Homebrew mentioned above -2. Do a proper ./configure --enable-debug -3. In Qt Creator do "New Project" -> Import Project -> Import Existing Project -4. Enter "bitcoin-qt" as project name, enter src/qt as location -5. Leave the file selection as it is -6. Confirm the "summary page" -7. In the "Projects" tab select "Manage Kits..." -8. Select the default "Desktop" kit and select "Clang (x86 64bit in /usr/bin)" as compiler -9. Select LLDB as debugger (you might need to set the path to your installation) -10. Start debugging with Qt Creator - -Creating a release build ------------------------- -You can ignore this section if you are building `bitcoind` for your own use. - -bitcoind/bitcoin-cli binaries are not included in the Bitcoin-Qt.app bundle. - -If you are building `bitcoind` or `Bitcoin Core` for others, your build machine should be set up -as follows for maximum compatibility: +4. You can also create a .dmg that contains the .app bundle (optional): -All dependencies should be compiled with these flags: - - -mmacosx-version-min=10.7 - -arch x86_64 - -isysroot $(xcode-select --print-path)/Platforms/MacOSX.platform/Developer/SDKs/MacOSX10.7.sdk - -Once dependencies are compiled, see [doc/release-process.md](release-process.md) for how the Bitcoin Core -bundle is packaged and signed to create the .dmg disk image that is distributed. + make deploy Running ------- -It's now available at `./bitcoind`, provided that you are still in the `src` -directory. We have to first create the RPC configuration file, though. +Bitcoin Core is now available at `./src/bitcoind` -Run `./bitcoind` to get the filename where it should be put, or just try these -commands: +Before running, it's recommended you create an RPC configuration file. echo -e "rpcuser=bitcoinrpc\nrpcpassword=$(xxd -l 16 -p /dev/urandom)" > "/Users/${USER}/Library/Application Support/Bitcoin/bitcoin.conf" + chmod 600 "/Users/${USER}/Library/Application Support/Bitcoin/bitcoin.conf" -The next time you run it, it will start downloading the blockchain, but it won't -output anything while it's doing this. This process may take several hours; -you can monitor its process by looking at the debug.log file, like this: +The first time you run bitcoind, it will start downloading the blockchain. This process could take several hours. + +You can monitor the download process by looking at the debug.log file: tail -f $HOME/Library/Application\ Support/Bitcoin/debug.log Other commands: ------- - ./bitcoind -daemon # to start the bitcoin daemon. - ./bitcoin-cli --help # for a list of command-line options. - ./bitcoin-cli help # When the daemon is running, to get a list of RPC commands + ./src/bitcoind -daemon # Starts the bitcoin daemon. + ./src/bitcoin-cli --help # Outputs a list of command-line options. + ./src/bitcoin-cli help # Outputs a list of RPC commands when the daemon is running. + +Using Qt Creator as IDE +------------------------ +You can use Qt Creator as an IDE, for bitcoin development. +Download and install the community edition of [Qt Creator](https://www.qt.io/download/). +Uncheck everything except Qt Creator during the installation process. -Using Qt official installer while building ------------------------------------------- +1. Make sure you installed everything through Homebrew mentioned above +2. Do a proper ./configure --enable-debug +3. In Qt Creator do "New Project" -> Import Project -> Import Existing Project +4. Enter "bitcoin-qt" as project name, enter src/qt as location +5. Leave the file selection as it is +6. Confirm the "summary page" +7. In the "Projects" tab select "Manage Kits..." +8. Select the default "Desktop" kit and select "Clang (x86 64bit in /usr/bin)" as compiler +9. Select LLDB as debugger (you might need to set the path to your installation) +10. Start debugging with Qt Creator + +Notes +----- + +* Tested on OS X 10.7 through 10.11 on 64-bit Intel processors only. -If you prefer to use the latest Qt installed from the official binary -installer over the brew version, you have to make several changes to -the installed tree and its binaries (all these changes are contained -in the brew version already). The changes needed are described in -[#7714](https://github.com/bitcoin/bitcoin/issues/7714). We do not -support building Bitcoin Core this way though. +* Building with downloaded Qt binaries is not officially supported. See the notes in [#7714](https://github.com/bitcoin/bitcoin/issues/7714) diff --git a/doc/build-unix.md b/doc/build-unix.md index dc754fc7339c9..bd89978cc2a74 100644 --- a/doc/build-unix.md +++ b/doc/build-unix.md @@ -124,6 +124,24 @@ libqrencode (optional) can be installed with: Once these are installed, they will be found by configure and a bitcoin-qt executable will be built by default. +Dependency Build Instructions: Fedora +------------------------------------- +Build requirements: + + sudo dnf install gcc-c++ libtool make autoconf automake openssl-devel libevent-devel boost-devel libdb4-devel libdb4-cxx-devel + +Optional: + + sudo dnf install miniupnpc-devel + +To build with Qt 5 (recommended) you need the following: + + sudo dnf install qt5-qttools-devel qt5-qtbase-devel protobuf-devel + +libqrencode (optional) can be installed with: + + sudo dnf install qrencode-devel + Notes ----- The release is built with GCC and then "strip bitcoind" to strip the debug diff --git a/doc/developer-notes.md b/doc/developer-notes.md index 8affb2158a913..95c46b05fe203 100644 --- a/doc/developer-notes.md +++ b/doc/developer-notes.md @@ -5,7 +5,9 @@ Various coding styles have been used during the history of the codebase, and the result is not very consistent. However, we're now trying to converge to a single style, so please use it in new code. Old code will be converted gradually. -- Basic rules specified in src/.clang-format. Use a recent clang-format-3.5 to format automatically. +- Basic rules specified in [src/.clang-format](/src/.clang-format). + Use a recent clang-format to format automatically using one of the [dev scripts] + (/contrib/devtools/README.md#clang-formatpy). - Braces on new lines for namespaces, classes, functions, methods. - Braces on the same line for everything else. - 4 space indentation (no tabs) for every block except namespaces. @@ -263,7 +265,7 @@ General C++ the `.h` to the `.cpp` should not result in build errors - Use the RAII (Resource Acquisition Is Initialization) paradigm where possible. For example by using - `scoped_pointer` for allocations in a function. + `unique_ptr` for allocations in a function. - *Rationale*: This avoids memory and resource leaks, and ensures exception safety @@ -282,10 +284,9 @@ C++ data structures - *Rationale*: Behavior is undefined. In C++ parlor this means "may reformat the universe", in practice this has resulted in at least one hard-to-debug crash bug -- Watch out for vector out-of-bounds exceptions. `&vch[0]` is illegal for an - empty vector, `&vch[vch.size()]` is always illegal. Use `begin_ptr(vch)` and - `end_ptr(vch)` to get the begin and end pointer instead (defined in - `serialize.h`) +- Watch out for out-of-bounds vector access. `&vch[vch.size()]` is illegal, + including `&vch[0]` for an empty vector. Use `vch.data()` and `vch.data() + + vch.size()` instead. - Vector bounds checking is only enabled in debug mode. Do not rely on it @@ -321,7 +322,7 @@ Strings and formatting buffer overflows and surprises with `\0` characters. Also some C string manipulations tend to act differently depending on platform, or even the user locale -- Use `ParseInt32`, `ParseInt64`, `ParseDouble` from `utilstrencodings.h` for number parsing +- Use `ParseInt32`, `ParseInt64`, `ParseUInt32`, `ParseUInt64`, `ParseDouble` from `utilstrencodings.h` for number parsing - *Rationale*: These functions do overflow checking, and avoid pesky locale issues @@ -379,3 +380,51 @@ GUI - *Rationale*: Model classes pass through events and data from the core, they should not interact with the user. That's where View classes come in. The converse also holds: try to not directly access core data structures from Views. + +Git and github tips +--------------------- + +- For resolving merge/rebase conflicts, it can be useful to enable diff3 style using + `git config merge.conflictstyle diff3`. Instead of + + <<< + yours + === + theirs + >>> + + you will see + + <<< + yours + ||| + original + === + theirs + >>> + + This may make it much clearer what caused the conflict. In this style, you can often just look + at what changed between *original* and *theirs*, and mechanically apply that to *yours* (or the other way around). + +- When reviewing patches which change indentation in C++ files, use `git diff -w` and `git show -w`. This makes + the diff algorithm ignore whitespace changes. This feature is also available on github.com, by adding `?w=1` + at the end of any URL which shows a diff. + +- When reviewing patches that change symbol names in many places, use `git diff --word-diff`. This will instead + of showing the patch as deleted/added *lines*, show deleted/added *words*. + +- When reviewing patches that move code around, try using + `git diff --patience commit~:old/file.cpp commit:new/file/name.cpp`, and ignoring everything except the + moved body of code which should show up as neither `+` or `-` lines. In case it was not a pure move, this may + even work when combined with the `-w` or `--word-diff` options described above. + +- When looking at other's pull requests, it may make sense to add the following section to your `.git/config` + file: + + [remote "upstream-pull"] + fetch = +refs/pull/*:refs/remotes/upstream-pull/* + url = git@github.com:bitcoin/bitcoin.git + + This will add an `upstream-pull` remote to your git repository, which can be fetched using `git fetch --all` + or `git fetch upstream-pull`. Afterwards, you can use `upstream-pull/NUMBER/head` in arguments to `git show`, + `git checkout` and anywhere a commit id would be acceptable to see the changes from pull request NUMBER. diff --git a/doc/gitian-building.md b/doc/gitian-building.md index 3126218518690..7796a5fc9c151 100644 --- a/doc/gitian-building.md +++ b/doc/gitian-building.md @@ -1,7 +1,7 @@ Gitian building ================ -*Setup instructions for a Gitian build of Bitcoin using a Debian VM or physical system.* +*Setup instructions for a Gitian build of Bitcoin Core using a Debian VM or physical system.* Gitian is the deterministic build process that is used to build the Bitcoin Core executables. It provides a way to be reasonably sure that the @@ -26,7 +26,7 @@ Table of Contents - [Installing Gitian](#installing-gitian) - [Setting up the Gitian image](#setting-up-the-gitian-image) - [Getting and building the inputs](#getting-and-building-the-inputs) -- [Building Bitcoin](#building-bitcoin) +- [Building Bitcoin Core](#building-bitcoin-core) - [Building an alternative repository](#building-an-alternative-repository) - [Signing externally](#signing-externally) - [Uploading signatures](#uploading-signatures) @@ -95,11 +95,11 @@ After creating the VM, we need to configure it. - Click `Ok` twice to save. -Get the [Debian 8.x net installer](http://cdimage.debian.org/debian-cd/8.4.0/amd64/iso-cd/debian-8.4.0-amd64-netinst.iso) (a more recent minor version should also work, see also [Debian Network installation](https://www.debian.org/CD/netinst/)). +Get the [Debian 8.x net installer](http://cdimage.debian.org/debian-cd/8.5.0/amd64/iso-cd/debian-8.5.0-amd64-netinst.iso) (a more recent minor version should also work, see also [Debian Network installation](https://www.debian.org/CD/netinst/)). This DVD image can be validated using a SHA256 hashing tool, for example on Unixy OSes by entering the following in a terminal: - echo "7a6b418e6a4ee3ca75dda04d79ed96c9e2c33bb0c703ca7e40c6374ab4590748 debian-8.4.0-amd64-netinst.iso" | sha256sum -c + echo "ad4e8c27c561ad8248d5ebc1d36eb172f884057bfeb2c22ead823f59fa8c3dff debian-8.5.0-amd64-netinst.iso" | sha256sum -c # (must return OK) Then start the VM. On the first launch you will be asked for a CD or DVD image. Choose the downloaded iso. @@ -310,7 +310,7 @@ Clone the git repositories for bitcoin and Gitian. ```bash git clone https://github.com/devrandom/gitian-builder.git git clone https://github.com/bitcoin/bitcoin -git clone https://github.com/bitcoin/gitian.sigs.git +git clone https://github.com/bitcoin-core/gitian.sigs.git ``` Setting up the Gitian image @@ -342,10 +342,10 @@ manual intervention. Also optionally follow the next step: 'Seed the Gitian sour and offline git repositories' which will fetch the remaining files required for building offline. -Building Bitcoin +Building Bitcoin Core ---------------- -To build Bitcoin (for Linux, OS X and Windows) just follow the steps under 'perform +To build Bitcoin Core (for Linux, OS X and Windows) just follow the steps under 'perform Gitian builds' in [doc/release-process.md](release-process.md#perform-gitian-builds) in the bitcoin repository. This may take some time as it will build all the dependencies needed for each descriptor. @@ -441,10 +441,10 @@ Then when building, override the remote URLs that gbuild would otherwise pull fr ```bash cd /some/root/path/ -git clone https://github.com/bitcoin/bitcoin-detached-sigs.git +git clone https://github.com/bitcoin-core/bitcoin-detached-sigs.git -BTCPATH=/some/root/path/bitcoin.git -SIGPATH=/some/root/path/bitcoin-detached-sigs.git +BTCPATH=/some/root/path/bitcoin +SIGPATH=/some/root/path/bitcoin-detached-sigs ./bin/gbuild --url bitcoin=${BTCPATH},signature=${SIGPATH} ../bitcoin/contrib/gitian-descriptors/gitian-win-signer.yml ``` @@ -473,5 +473,5 @@ Uploading signatures --------------------- After building and signing you can push your signatures (both the `.assert` and `.assert.sig` files) to the -[bitcoin/gitian.sigs](https://github.com/bitcoin/gitian.sigs/) repository, or if that's not possible create a pull +[bitcoin-core/gitian.sigs](https://github.com/bitcoin-core/gitian.sigs/) repository, or if that's not possible create a pull request. You can also mail the files to Wladimir (laanwj@gmail.com) and he will commit them. diff --git a/doc/release-notes.md b/doc/release-notes.md index 8360cc4816c36..be619e41c6ae0 100644 --- a/doc/release-notes.md +++ b/doc/release-notes.md @@ -1,6 +1,21 @@ (note: this is a temporary file, to be added-to by anybody, and moved to release-notes at release time) +Bitcoin Core version *version* is now available from: + + + +This is a new major version release, including new features, various bugfixes +and performance improvements, as well as updated translations. + +Please report bugs using the issue tracker at github: + + + +To receive security and update notifications, please subscribe to: + + + Notable changes =============== @@ -28,6 +43,19 @@ RPC low-level changes 32-bit and 64-bit platforms, and the txids were missing in the hashed data. This has been fixed, but this means that the output will be different than from previous versions. +C++11 and Python 3 +------------------- + +Various code modernizations have been done. The Bitcoin Core code base has +started using C++11. This means that a C++11-capable compiler is now needed for +building. Effectively this means GCC 4.7 or higher, or Clang 3.3 or higher. + +When cross-compiling for a target that doesn't have C++11 libraries, configure with +`./configure --enable-glibc-back-compat ... LDFLAGS=-static-libstdc++`. + +For running the functional tests in `qa/rpc-tests`, Python3.4 or higher is now +required. + 0.13.0 Change log ================= @@ -52,6 +80,22 @@ The following outputs are affected by this change: - REST `/rest/block/` (JSON format when including extended tx details) - `bitcoin-tx -json` +New mempool information RPC calls +--------------------------------- + +RPC calls have been added to output detailed statistics for individual mempool +entries, as well as to calculate the in-mempool ancestors or descendants of a +transaction: see `getmempoolentry`, `getmempoolancestors`, `getmempooldescendants`. + +### ZMQ + +Each ZMQ notification now contains an up-counting sequence number that allows +listeners to detect lost notifications. +The sequence number is always the last element in a multi-part ZMQ notification and +therefore backward compatible. +Each message type has its own counter. +(https://github.com/bitcoin/bitcoin/pull/7762) + ### Configuration and command-line options ### Block and transaction handling diff --git a/doc/release-notes/release-notes-0.12.0.md b/doc/release-notes/release-notes-0.12.0.md index 135cd68a7c78f..1b7bd06ece6a2 100644 --- a/doc/release-notes/release-notes-0.12.0.md +++ b/doc/release-notes/release-notes-0.12.0.md @@ -61,7 +61,7 @@ Signature validation using libsecp256k1 --------------------------------------- ECDSA signatures inside Bitcoin transactions now use validation using -[libsecp256k1](https://github.com/bitcoin/secp256k1) instead of OpenSSL. +[libsecp256k1](https://github.com/bitcoin-core/secp256k1) instead of OpenSSL. Depending on the platform, this means a significant speedup for raw signature validation speed. The advantage is largest on x86_64, where validation is over diff --git a/doc/release-process.md b/doc/release-process.md index 5a6ac8482b674..3bfcc38177bb2 100644 --- a/doc/release-process.md +++ b/doc/release-process.md @@ -1,181 +1,228 @@ Release Process ==================== -* Update translations (ping wumpus, Diapolo or tcatm on IRC) see [translation_process.md](https://github.com/bitcoin/bitcoin/blob/master/doc/translation_process.md#syncing-with-transifex) +Before every release candidate: + +* Update translations (ping wumpus on IRC) see [translation_process.md](https://github.com/bitcoin/bitcoin/blob/master/doc/translation_process.md#syncing-with-transifex) + +Before every minor and major release: + * Update [bips.md](bips.md) to account for changes since the last release. -* Update hardcoded [seeds](/contrib/seeds) +* Update version in sources (see below) +* Write release notes (see below) + +Before every major release: -* * * +* Update hardcoded [seeds](/contrib/seeds/README.md), see [this pull request](https://github.com/bitcoin/bitcoin/pull/7415) for an example. + +### First time / New builders -###First time / New builders Check out the source code in the following directory hierarchy. - cd /path/to/your/toplevel/build - git clone https://github.com/bitcoin/gitian.sigs.git - git clone https://github.com/bitcoin/bitcoin-detached-sigs.git - git clone https://github.com/devrandom/gitian-builder.git - git clone https://github.com/bitcoin/bitcoin.git + cd /path/to/your/toplevel/build + git clone https://github.com/bitcoin-core/gitian.sigs.git + git clone https://github.com/bitcoin-core/bitcoin-detached-sigs.git + git clone https://github.com/devrandom/gitian-builder.git + git clone https://github.com/bitcoin/bitcoin.git + +### Bitcoin maintainers/release engineers, update version in sources + +Update the following: + +- `configure.ac`: + - `_CLIENT_VERSION_MAJOR` + - `_CLIENT_VERSION_MINOR` + - `_CLIENT_VERSION_REVISION` + - Don't forget to set `_CLIENT_VERSION_IS_RELEASE` to `true` +- `src/clientversion.h`: (this mirrors `configure.ac` - see issue #3539) + - `CLIENT_VERSION_MAJOR` + - `CLIENT_VERSION_MINOR` + - `CLIENT_VERSION_REVISION` + - Don't forget to set `CLIENT_VERSION_IS_RELEASE` to `true` +- `doc/README.md` and `doc/README_windows.txt` +- `doc/Doxyfile`: `PROJECT_NUMBER` contains the full version +- `contrib/gitian-descriptors/*.yml`: usually one'd want to do this on master after branching off the release - but be sure to at least do it before a new major release -###Bitcoin maintainers/release engineers, update (commit) version in sources +Write release notes. git shortlog helps a lot, for example: - pushd ./bitcoin - contrib/verifysfbinaries/verify.sh - configure.ac - doc/README* - doc/Doxyfile - contrib/gitian-descriptors/*.yml - src/clientversion.h (change CLIENT_VERSION_IS_RELEASE to true) + git shortlog --no-merges v(current version, e.g. 0.7.2)..v(new version, e.g. 0.8.0) - # tag version in git +(or ping @wumpus on IRC, he has specific tooling to generate the list of merged pulls +and sort them into categories based on labels) - git tag -s v(new version, e.g. 0.8.0) +Generate list of authors: - # write release notes. git shortlog helps a lot, for example: + git log --format='%aN' "$*" | sort -ui | sed -e 's/^/- /' - git shortlog --no-merges v(current version, e.g. 0.7.2)..v(new version, e.g. 0.8.0) - popd +Tag version (or release candidate) in git -* * * + git tag -s v(new version, e.g. 0.8.0) -###Setup and perform Gitian builds +### Setup and perform Gitian builds - Setup Gitian descriptors: +Setup Gitian descriptors: - pushd ./bitcoin - export SIGNER=(your Gitian key, ie bluematt, sipa, etc) - export VERSION=(new version, e.g. 0.8.0) - git fetch - git checkout v${VERSION} - popd + pushd ./bitcoin + export SIGNER=(your Gitian key, ie bluematt, sipa, etc) + export VERSION=(new version, e.g. 0.8.0) + git fetch + git checkout v${VERSION} + popd - Ensure your gitian.sigs are up-to-date if you wish to gverify your builds against other Gitian signatures. +Ensure your gitian.sigs are up-to-date if you wish to gverify your builds against other Gitian signatures. - pushd ./gitian.sigs - git pull - popd + pushd ./gitian.sigs + git pull + popd - Ensure gitian-builder is up-to-date to take advantage of new caching features (`e9741525c` or later is recommended). +Ensure gitian-builder is up-to-date to take advantage of new caching features (`e9741525c` or later is recommended). - pushd ./gitian-builder - git pull + pushd ./gitian-builder + git pull + popd -###Fetch and create inputs: (first time, or when dependency versions change) +### Fetch and create inputs: (first time, or when dependency versions change) - mkdir -p inputs - wget -P inputs https://bitcoincore.org/cfields/osslsigncode-Backports-to-1.7.1.patch - wget -P inputs http://downloads.sourceforge.net/project/osslsigncode/osslsigncode/osslsigncode-1.7.1.tar.gz + pushd ./gitian-builder + mkdir -p inputs + wget -P inputs https://bitcoincore.org/cfields/osslsigncode-Backports-to-1.7.1.patch + wget -P inputs http://downloads.sourceforge.net/project/osslsigncode/osslsigncode/osslsigncode-1.7.1.tar.gz + popd - Register and download the Apple SDK: see [OS X readme](README_osx.txt) for details. +Register and download the Apple SDK: see [OS X readme](README_osx.txt) for details. - https://developer.apple.com/devcenter/download.action?path=/Developer_Tools/xcode_6.1.1/xcode_6.1.1.dmg +https://developer.apple.com/devcenter/download.action?path=/Developer_Tools/xcode_6.1.1/xcode_6.1.1.dmg - Using a Mac, create a tarball for the 10.9 SDK and copy it to the inputs directory: +Using a Mac, create a tarball for the 10.9 SDK and copy it to the inputs directory: - tar -C /Volumes/Xcode/Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/ -czf MacOSX10.9.sdk.tar.gz MacOSX10.9.sdk + tar -C /Volumes/Xcode/Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/ -czf MacOSX10.9.sdk.tar.gz MacOSX10.9.sdk -###Optional: Seed the Gitian sources cache and offline git repositories +### Optional: Seed the Gitian sources cache and offline git repositories By default, Gitian will fetch source files as needed. To cache them ahead of time: - make -C ../bitcoin/depends download SOURCES_PATH=`pwd`/cache/common + pushd ./gitian-builder + make -C ../bitcoin/depends download SOURCES_PATH=`pwd`/cache/common + popd Only missing files will be fetched, so this is safe to re-run for each build. NOTE: Offline builds must use the --url flag to ensure Gitian fetches only from local URLs. For example: -``` -./bin/gbuild --url bitcoin=/path/to/bitcoin,signature=/path/to/sigs {rest of arguments} -``` + + pushd ./gitian-builder + ./bin/gbuild --url bitcoin=/path/to/bitcoin,signature=/path/to/sigs {rest of arguments} + popd + The gbuild invocations below DO NOT DO THIS by default. -###Build and sign Bitcoin Core for Linux, Windows, and OS X: +### Build and sign Bitcoin Core for Linux, Windows, and OS X: - ./bin/gbuild --commit bitcoin=v${VERSION} ../bitcoin/contrib/gitian-descriptors/gitian-linux.yml - ./bin/gsign --signer $SIGNER --release ${VERSION}-linux --destination ../gitian.sigs/ ../bitcoin/contrib/gitian-descriptors/gitian-linux.yml + pushd ./gitian-builder + ./bin/gbuild --commit bitcoin=v${VERSION} ../bitcoin/contrib/gitian-descriptors/gitian-linux.yml + ./bin/gsign --signer $SIGNER --release ${VERSION}-linux --destination ../gitian.sigs/ ../bitcoin/contrib/gitian-descriptors/gitian-linux.yml mv build/out/bitcoin-*.tar.gz build/out/src/bitcoin-*.tar.gz ../ - ./bin/gbuild --commit bitcoin=v${VERSION} ../bitcoin/contrib/gitian-descriptors/gitian-win.yml - ./bin/gsign --signer $SIGNER --release ${VERSION}-win-unsigned --destination ../gitian.sigs/ ../bitcoin/contrib/gitian-descriptors/gitian-win.yml + ./bin/gbuild --commit bitcoin=v${VERSION} ../bitcoin/contrib/gitian-descriptors/gitian-win.yml + ./bin/gsign --signer $SIGNER --release ${VERSION}-win-unsigned --destination ../gitian.sigs/ ../bitcoin/contrib/gitian-descriptors/gitian-win.yml mv build/out/bitcoin-*-win-unsigned.tar.gz inputs/bitcoin-win-unsigned.tar.gz mv build/out/bitcoin-*.zip build/out/bitcoin-*.exe ../ - ./bin/gbuild --commit bitcoin=v${VERSION} ../bitcoin/contrib/gitian-descriptors/gitian-osx.yml - ./bin/gsign --signer $SIGNER --release ${VERSION}-osx-unsigned --destination ../gitian.sigs/ ../bitcoin/contrib/gitian-descriptors/gitian-osx.yml + ./bin/gbuild --commit bitcoin=v${VERSION} ../bitcoin/contrib/gitian-descriptors/gitian-osx.yml + ./bin/gsign --signer $SIGNER --release ${VERSION}-osx-unsigned --destination ../gitian.sigs/ ../bitcoin/contrib/gitian-descriptors/gitian-osx.yml mv build/out/bitcoin-*-osx-unsigned.tar.gz inputs/bitcoin-osx-unsigned.tar.gz mv build/out/bitcoin-*.tar.gz build/out/bitcoin-*.dmg ../ + popd - Build output expected: +Build output expected: - 1. source tarball (bitcoin-${VERSION}.tar.gz) - 2. linux 32-bit and 64-bit dist tarballs (bitcoin-${VERSION}-linux[32|64].tar.gz) - 3. windows 32-bit and 64-bit unsigned installers and dist zips (bitcoin-${VERSION}-win[32|64]-setup-unsigned.exe, bitcoin-${VERSION}-win[32|64].zip) - 4. OS X unsigned installer and dist tarball (bitcoin-${VERSION}-osx-unsigned.dmg, bitcoin-${VERSION}-osx64.tar.gz) - 5. Gitian signatures (in gitian.sigs/${VERSION}-/(your Gitian key)/ + 1. source tarball (`bitcoin-${VERSION}.tar.gz`) + 2. linux 32-bit and 64-bit dist tarballs (`bitcoin-${VERSION}-linux[32|64].tar.gz`) + 3. windows 32-bit and 64-bit unsigned installers and dist zips (`bitcoin-${VERSION}-win[32|64]-setup-unsigned.exe`, `bitcoin-${VERSION}-win[32|64].zip`) + 4. OS X unsigned installer and dist tarball (`bitcoin-${VERSION}-osx-unsigned.dmg`, `bitcoin-${VERSION}-osx64.tar.gz`) + 5. Gitian signatures (in `gitian.sigs/${VERSION}-/(your Gitian key)/`) -###Verify other gitian builders signatures to your own. (Optional) +### Verify other gitian builders signatures to your own. (Optional) - Add other gitian builders keys to your gpg keyring +Add other gitian builders keys to your gpg keyring - gpg --import ../bitcoin/contrib/gitian-keys/*.pgp + gpg --import bitcoin/contrib/gitian-keys/*.pgp - Verify the signatures +Verify the signatures - ./bin/gverify -v -d ../gitian.sigs/ -r ${VERSION}-linux ../bitcoin/contrib/gitian-descriptors/gitian-linux.yml - ./bin/gverify -v -d ../gitian.sigs/ -r ${VERSION}-win-unsigned ../bitcoin/contrib/gitian-descriptors/gitian-win.yml - ./bin/gverify -v -d ../gitian.sigs/ -r ${VERSION}-osx-unsigned ../bitcoin/contrib/gitian-descriptors/gitian-osx.yml + pushd ./gitian-builder + ./bin/gverify -v -d ../gitian.sigs/ -r ${VERSION}-linux ../bitcoin/contrib/gitian-descriptors/gitian-linux.yml + ./bin/gverify -v -d ../gitian.sigs/ -r ${VERSION}-win-unsigned ../bitcoin/contrib/gitian-descriptors/gitian-win.yml + ./bin/gverify -v -d ../gitian.sigs/ -r ${VERSION}-osx-unsigned ../bitcoin/contrib/gitian-descriptors/gitian-osx.yml + popd - popd - -###Next steps: +### Next steps: Commit your signature to gitian.sigs: - pushd gitian.sigs - git add ${VERSION}-linux/${SIGNER} - git add ${VERSION}-win-unsigned/${SIGNER} - git add ${VERSION}-osx-unsigned/${SIGNER} - git commit -a - git push # Assuming you can push to the gitian.sigs tree - popd - - Wait for Windows/OS X detached signatures: - Once the Windows/OS X builds each have 3 matching signatures, they will be signed with their respective release keys. - Detached signatures will then be committed to the [bitcoin-detached-sigs](https://github.com/bitcoin/bitcoin-detached-sigs) repository, which can be combined with the unsigned apps to create signed binaries. - - Create (and optionally verify) the signed OS X binary: - - pushd ./gitian-builder - ./bin/gbuild -i --commit signature=v${VERSION} ../bitcoin/contrib/gitian-descriptors/gitian-osx-signer.yml - ./bin/gsign --signer $SIGNER --release ${VERSION}-osx-signed --destination ../gitian.sigs/ ../bitcoin/contrib/gitian-descriptors/gitian-osx-signer.yml - ./bin/gverify -v -d ../gitian.sigs/ -r ${VERSION}-osx-signed ../bitcoin/contrib/gitian-descriptors/gitian-osx-signer.yml - mv build/out/bitcoin-osx-signed.dmg ../bitcoin-${VERSION}-osx.dmg - popd - - Create (and optionally verify) the signed Windows binaries: - - pushd ./gitian-builder - ./bin/gbuild -i --commit signature=v${VERSION} ../bitcoin/contrib/gitian-descriptors/gitian-win-signer.yml - ./bin/gsign --signer $SIGNER --release ${VERSION}-win-signed --destination ../gitian.sigs/ ../bitcoin/contrib/gitian-descriptors/gitian-win-signer.yml - ./bin/gverify -v -d ../gitian.sigs/ -r ${VERSION}-win-signed ../bitcoin/contrib/gitian-descriptors/gitian-win-signer.yml - mv build/out/bitcoin-*win64-setup.exe ../bitcoin-${VERSION}-win64-setup.exe - mv build/out/bitcoin-*win32-setup.exe ../bitcoin-${VERSION}-win32-setup.exe - popd + pushd gitian.sigs + git add ${VERSION}-linux/${SIGNER} + git add ${VERSION}-win-unsigned/${SIGNER} + git add ${VERSION}-osx-unsigned/${SIGNER} + git commit -a + git push # Assuming you can push to the gitian.sigs tree + popd -Commit your signature for the signed OS X/Windows binaries: +Wait for Windows/OS X detached signatures: + +- Once the Windows/OS X builds each have 3 matching signatures, they will be signed with their respective release keys. +- Detached signatures will then be committed to the [bitcoin-detached-sigs](https://github.com/bitcoin-core/bitcoin-detached-sigs) repository, which can be combined with the unsigned apps to create signed binaries. + +Create (and optionally verify) the signed OS X binary: - pushd gitian.sigs - git add ${VERSION}-osx-signed/${SIGNER} - git add ${VERSION}-win-signed/${SIGNER} - git commit -a - git push # Assuming you can push to the gitian.sigs tree - popd + pushd ./gitian-builder + ./bin/gbuild -i --commit signature=v${VERSION} ../bitcoin/contrib/gitian-descriptors/gitian-osx-signer.yml + ./bin/gsign --signer $SIGNER --release ${VERSION}-osx-signed --destination ../gitian.sigs/ ../bitcoin/contrib/gitian-descriptors/gitian-osx-signer.yml + ./bin/gverify -v -d ../gitian.sigs/ -r ${VERSION}-osx-signed ../bitcoin/contrib/gitian-descriptors/gitian-osx-signer.yml + mv build/out/bitcoin-osx-signed.dmg ../bitcoin-${VERSION}-osx.dmg + popd -------------------------------------------------------------------------- +Create (and optionally verify) the signed Windows binaries: + + pushd ./gitian-builder + ./bin/gbuild -i --commit signature=v${VERSION} ../bitcoin/contrib/gitian-descriptors/gitian-win-signer.yml + ./bin/gsign --signer $SIGNER --release ${VERSION}-win-signed --destination ../gitian.sigs/ ../bitcoin/contrib/gitian-descriptors/gitian-win-signer.yml + ./bin/gverify -v -d ../gitian.sigs/ -r ${VERSION}-win-signed ../bitcoin/contrib/gitian-descriptors/gitian-win-signer.yml + mv build/out/bitcoin-*win64-setup.exe ../bitcoin-${VERSION}-win64-setup.exe + mv build/out/bitcoin-*win32-setup.exe ../bitcoin-${VERSION}-win32-setup.exe + popd + +Commit your signature for the signed OS X/Windows binaries: + + pushd gitian.sigs + git add ${VERSION}-osx-signed/${SIGNER} + git add ${VERSION}-win-signed/${SIGNER} + git commit -a + git push # Assuming you can push to the gitian.sigs tree + popd ### After 3 or more people have gitian-built and their results match: - Create `SHA256SUMS.asc` for the builds, and GPG-sign it: + ```bash sha256sum * > SHA256SUMS +``` + +The list of files should be: +``` +bitcoin-${VERSION}-linux32.tar.gz +bitcoin-${VERSION}-linux64.tar.gz +bitcoin-${VERSION}-osx64.tar.gz +bitcoin-${VERSION}-osx.dmg +bitcoin-${VERSION}.tar.gz +bitcoin-${VERSION}-win32-setup.exe +bitcoin-${VERSION}-win32.zip +bitcoin-${VERSION}-win64-setup.exe +bitcoin-${VERSION}-win64.zip +``` + +- GPG-sign it, delete the unsigned file: +``` gpg --digest-algo sha256 --clearsign SHA256SUMS # outputs SHA256SUMS.asc rm SHA256SUMS ``` @@ -185,6 +232,15 @@ Note: check that SHA256SUMS itself doesn't end up in SHA256SUMS, which is a spur - Upload zips and installers, as well as `SHA256SUMS.asc` from last step, to the bitcoin.org server into `/var/www/bin/bitcoin-core-${VERSION}` +- A `.torrent` will appear in the directory after a few minutes. Optionally help seed this torrent. To get the `magnet:` URI use: +```bash +transmission-show -m +``` +Insert the magnet URI into the announcement sent to mailing lists. This permits +people without access to `bitcoin.org` to download the binary distribution. +Also put it into the `optional_magnetlink:` slot in the YAML file for +bitcoin.org (see below for bitcoin.org update instructions). + - Update bitcoin.org version - First, check to see if the Bitcoin.org maintainers have prepared a @@ -202,16 +258,18 @@ Note: check that SHA256SUMS itself doesn't end up in SHA256SUMS, which is a spur - Announce the release: - - Release sticky on bitcointalk: https://bitcointalk.org/index.php?board=1.0 + - bitcoin-dev and bitcoin-core-dev mailing list + + - Bitcoin Core announcements list https://bitcoincore.org/en/list/announcements/join/ - - Bitcoin-development mailing list + - bitcoincore.org blog post - Update title of #bitcoin on Freenode IRC - - Optionally reddit /r/Bitcoin, ... but this will usually sort out itself + - Optionally twitter, reddit /r/Bitcoin, ... but this will usually sort out itself -- Notify BlueMatt so that he can start building [the PPAs](https://launchpad.net/~bitcoin/+archive/ubuntu/bitcoin) + - Notify BlueMatt so that he can start building [the PPAs](https://launchpad.net/~bitcoin/+archive/ubuntu/bitcoin) -- Add release notes for the new version to the directory `doc/release-notes` in git master + - Add release notes for the new version to the directory `doc/release-notes` in git master -- Celebrate + - Celebrate diff --git a/doc/zmq.md b/doc/zmq.md index 902d1124c766d..6079e3254f783 100644 --- a/doc/zmq.md +++ b/doc/zmq.md @@ -38,7 +38,7 @@ newer. Typically, it is packaged by distributions as something like *libzmq3-dev*. The C++ wrapper for ZeroMQ is *not* needed. In order to run the example Python client scripts in contrib/ one must -also install *python-zmq*, though this is not necessary for daemon +also install *python3-zmq*, though this is not necessary for daemon operation. ## Enabling @@ -99,3 +99,8 @@ using other means such as firewalling. Note that when the block chain tip changes, a reorganisation may occur and just the tip will be notified. It is up to the subscriber to retrieve the chain from the last known block to the new tip. + +There are several possibilities that ZMQ notification can get lost +during transmission depending on the communication type your are +using. Bitcoind appends an up-counting sequence number to each +notification which allows listeners to detect lost notifications. diff --git a/qa/README.md b/qa/README.md index 2b476c4d8d93f..7489eb5130ac0 100644 --- a/qa/README.md +++ b/qa/README.md @@ -11,23 +11,36 @@ Before running the tests, the following must be installed. Unix ---- -The python-zmq library is required. On Ubuntu or Debian it can be installed via: +The python3-zmq library is required. On Ubuntu or Debian it can be installed via: ``` -sudo apt-get install python-zmq +sudo apt-get install python3-zmq ``` Running tests ============= -You can run any single test by calling `qa/pull-tester/rpc-tests.py `. +You can run any single test by calling -Or you can run any combination of tests by calling `qa/pull-tester/rpc-tests.py ...` + qa/pull-tester/rpc-tests.py -Run the regression test suite with `qa/pull-tester/rpc-tests.py` +Or you can run any combination of tests by calling -Run all possible tests with `qa/pull-tester/rpc-tests.py -extended` + qa/pull-tester/rpc-tests.py ... -Possible options: +Run the regression test suite with + + qa/pull-tester/rpc-tests.py + +Run all possible tests with + + qa/pull-tester/rpc-tests.py -extended + +By default, tests will be run in parallel if you want to specify how many +tests should be run in parallel, append `-paralell=n` (default n=4). + +If you want to create a basic coverage report for the rpc test suite, append `--coverage`. + +Possible options, which apply to each individual test run: ``` -h, --help show this help message and exit diff --git a/qa/pull-tester/rpc-tests.py b/qa/pull-tester/rpc-tests.py index 9200811b14558..fba1ad473fd67 100755 --- a/qa/pull-tester/rpc-tests.py +++ b/qa/pull-tester/rpc-tests.py @@ -1,5 +1,5 @@ -#!/usr/bin/env python2 -# Copyright (c) 2014-2015 The Bitcoin Core developers +#!/usr/bin/env python3 +# Copyright (c) 2014-2016 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -29,8 +29,17 @@ import tempfile import re +sys.path.append("qa/pull-tester/") from tests_config import * +BOLD = ("","") +if os.name == 'posix': + # primitive formatting on supported + # terminal via ANSI escape sequences: + BOLD = ('\033[0m', '\033[1m') + +RPC_TESTS_DIR = SRCDIR + '/qa/rpc-tests/' + #If imported values are not defined then set to zero (or disabled) if 'ENABLE_WALLET' not in vars(): ENABLE_WALLET=0 @@ -43,51 +52,56 @@ ENABLE_COVERAGE=0 -#Create a set to store arguments and create the passOn string +#Create a set to store arguments and create the passon string opts = set() -passOn = "" -p = re.compile("^--") +passon_args = [] +PASSON_REGEX = re.compile("^--") +PARALLEL_REGEX = re.compile('^-parallel=') -bold = ("","") -if (os.name == 'posix'): - bold = ('\033[0m', '\033[1m') +print_help = False +run_parallel = 4 for arg in sys.argv[1:]: + if arg == "--help" or arg == "-h" or arg == "-?": + print_help = True + break if arg == '--coverage': ENABLE_COVERAGE = 1 - elif (p.match(arg) or arg == "-h"): - passOn += " " + arg + elif PASSON_REGEX.match(arg): + passon_args.append(arg) + elif PARALLEL_REGEX.match(arg): + run_parallel = int(arg.split(sep='=', maxsplit=1)[1]) else: opts.add(arg) #Set env vars -buildDir = BUILDDIR if "BITCOIND" not in os.environ: - os.environ["BITCOIND"] = buildDir + '/src/bitcoind' + EXEEXT + os.environ["BITCOIND"] = BUILDDIR + '/src/bitcoind' + EXEEXT if "BITCOINCLI" not in os.environ: - os.environ["BITCOINCLI"] = buildDir + '/src/bitcoin-cli' + EXEEXT + os.environ["BITCOINCLI"] = BUILDDIR + '/src/bitcoin-cli' + EXEEXT if EXEEXT == ".exe" and "-win" not in opts: # https://github.com/bitcoin/bitcoin/commit/d52802551752140cf41f0d9a225a43e84404d3e9 # https://github.com/bitcoin/bitcoin/pull/5677#issuecomment-136646964 - print "Win tests currently disabled by default. Use -win option to enable" + print("Win tests currently disabled by default. Use -win option to enable") sys.exit(0) if not (ENABLE_WALLET == 1 and ENABLE_UTILS == 1 and ENABLE_BITCOIND == 1): - print "No rpc tests to run. Wallet, utils, and bitcoind must all be enabled" + print("No rpc tests to run. Wallet, utils, and bitcoind must all be enabled") sys.exit(0) -# python-zmq may not be installed. Handle this gracefully and with some helpful info +# python3-zmq may not be installed. Handle this gracefully and with some helpful info if ENABLE_ZMQ: try: import zmq except ImportError as e: - print("ERROR: \"import zmq\" failed. Set ENABLE_ZMQ=0 or " \ + print("WARNING: \"import zmq\" failed. Set ENABLE_ZMQ=0 or " \ "to run zmq tests, see dependency info in /qa/README.md.") - raise e + ENABLE_ZMQ=0 #Tests testScripts = [ + 'walletbackup.py', 'bip68-112-113-p2p.py', 'wallet.py', 'listtransactions.py', @@ -108,7 +122,6 @@ 'merkle_blocks.py', 'fundrawtransaction.py', 'signrawtransactions.py', - 'walletbackup.py', 'nodehandling.py', 'reindex.py', 'decodescript.py', @@ -125,6 +138,7 @@ 'p2p-segwit.py', 'segwit.py', 'importprunedfunds.py', + 'signmessages.py', ] if ENABLE_ZMQ: testScripts.append('zmq_test.py') @@ -153,50 +167,56 @@ 'pruning.py', # leave pruning last as it takes a REALLY long time ] + def runtests(): + test_list = [] + if '-extended' in opts: + test_list = testScripts + testScriptsExt + elif len(opts) == 0 or (len(opts) == 1 and "-win" in opts): + test_list = testScripts + else: + for t in testScripts + testScriptsExt: + if t in opts or re.sub(".py$", "", t) in opts: + test_list.append(t) + + if print_help: + # Only print help of the first script and exit + subprocess.check_call((RPC_TESTS_DIR + test_list[0]).split() + ['-h']) + sys.exit(0) + coverage = None if ENABLE_COVERAGE: coverage = RPCCoverage() print("Initializing coverage directory at %s\n" % coverage.dir) + flags = ["--srcdir=%s/src" % BUILDDIR] + passon_args + if coverage: + flags.append(coverage.flag) - rpcTestDir = buildDir + '/qa/rpc-tests/' - run_extended = '-extended' in opts - cov_flag = coverage.flag if coverage else '' - flags = " --srcdir %s/src %s %s" % (buildDir, cov_flag, passOn) + if len(test_list) > 1: + # Populate cache + subprocess.check_output([RPC_TESTS_DIR + 'create_cache.py'] + flags) #Run Tests - for i in range(len(testScripts)): - if (len(opts) == 0 - or (len(opts) == 1 and "-win" in opts ) - or run_extended - or testScripts[i] in opts - or re.sub(".py$", "", testScripts[i]) in opts ): - - print("Running testscript %s%s%s ..." % (bold[1], testScripts[i], bold[0])) - time0 = time.time() - subprocess.check_call( - rpcTestDir + testScripts[i] + flags, shell=True) - print("Duration: %s s\n" % (int(time.time() - time0))) - - # exit if help is called so we print just one set of - # instructions - p = re.compile(" -h| --help") - if p.match(passOn): - sys.exit(0) - - # Run Extended Tests - for i in range(len(testScriptsExt)): - if (run_extended or testScriptsExt[i] in opts - or re.sub(".py$", "", testScriptsExt[i]) in opts): - - print( - "Running 2nd level testscript " - + "%s%s%s ..." % (bold[1], testScriptsExt[i], bold[0])) - time0 = time.time() - subprocess.check_call( - rpcTestDir + testScriptsExt[i] + flags, shell=True) - print("Duration: %s s\n" % (int(time.time() - time0))) + max_len_name = len(max(test_list, key=len)) + time_sum = 0 + time0 = time.time() + job_queue = RPCTestHandler(run_parallel, test_list, flags) + results = BOLD[1] + "%s | %s | %s\n\n" % ("TEST".ljust(max_len_name), "PASSED", "DURATION") + BOLD[0] + all_passed = True + for _ in range(len(test_list)): + (name, stdout, stderr, passed, duration) = job_queue.get_next() + all_passed = all_passed and passed + time_sum += duration + + print('\n' + BOLD[1] + name + BOLD[0] + ":") + print(stdout) + print('stderr:\n' if not stderr == '' else '', stderr) + results += "%s | %s | %s s\n" % (name.ljust(max_len_name), str(passed).ljust(6), duration) + print("Pass: %s%s%s, Duration: %s s\n" % (BOLD[1], passed, BOLD[0], duration)) + results += BOLD[1] + "\n%s | %s | %s s (accumulated)" % ("ALL".ljust(max_len_name), str(all_passed).ljust(6), time_sum) + BOLD[0] + print(results) + print("\nRuntime: %s s" % (int(time.time() - time0))) if coverage: coverage.report_rpc_coverage() @@ -204,6 +224,49 @@ def runtests(): print("Cleaning up coverage data") coverage.cleanup() + sys.exit(not all_passed) + + +class RPCTestHandler: + """ + Trigger the testscrips passed in via the list. + """ + + def __init__(self, num_tests_parallel, test_list=None, flags=None): + assert(num_tests_parallel >= 1) + self.num_jobs = num_tests_parallel + self.test_list = test_list + self.flags = flags + self.num_running = 0 + self.jobs = [] + + def get_next(self): + while self.num_running < self.num_jobs and self.test_list: + # Add tests + self.num_running += 1 + t = self.test_list.pop(0) + port_seed = ["--portseed=%s" % len(self.test_list)] + self.jobs.append((t, + time.time(), + subprocess.Popen((RPC_TESTS_DIR + t).split() + self.flags + port_seed, + universal_newlines=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE))) + if not self.jobs: + raise IndexError('%s from empty list' % __name__) + while True: + # Return first proc that finishes + time.sleep(.5) + for j in self.jobs: + (name, time0, proc) = j + if proc.poll() is not None: + (stdout, stderr) = proc.communicate(timeout=3) + passed = stderr == "" and proc.returncode == 0 + self.num_running -= 1 + self.jobs.remove(j) + return name, stdout, stderr, passed, int(time.time() - time0) + print('.', end='', flush=True) + class RPCCoverage(object): """ @@ -222,7 +285,7 @@ class RPCCoverage(object): """ def __init__(self): self.dir = tempfile.mkdtemp(prefix="coverage") - self.flag = '--coveragedir %s' % self.dir + self.flag = '--coveragedir=%s' % self.dir def report_rpc_coverage(self): """ diff --git a/qa/pull-tester/tests_config.py.in b/qa/pull-tester/tests_config.py.in index 937b4231f12f8..a0d0a3d98a86b 100644 --- a/qa/pull-tester/tests_config.py.in +++ b/qa/pull-tester/tests_config.py.in @@ -1,8 +1,9 @@ -#!/usr/bin/env python2 -# Copyright (c) 2013-2014 The Bitcoin Core developers +#!/usr/bin/env python3 +# Copyright (c) 2013-2016 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. +SRCDIR="@abs_top_srcdir@" BUILDDIR="@abs_top_builddir@" EXEEXT="@EXEEXT@" @@ -11,5 +12,3 @@ EXEEXT="@EXEEXT@" @BUILD_BITCOIN_UTILS_TRUE@ENABLE_UTILS=1 @BUILD_BITCOIND_TRUE@ENABLE_BITCOIND=1 @ENABLE_ZMQ_TRUE@ENABLE_ZMQ=1 - - diff --git a/qa/rpc-tests/abandonconflict.py b/qa/rpc-tests/abandonconflict.py index a83aa97fcd806..c50c3cc562e5f 100755 --- a/qa/rpc-tests/abandonconflict.py +++ b/qa/rpc-tests/abandonconflict.py @@ -1,17 +1,18 @@ -#!/usr/bin/env python2 -# Copyright (c) 2014-2015 The Bitcoin Core developers +#!/usr/bin/env python3 +# Copyright (c) 2014-2016 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. from test_framework.test_framework import BitcoinTestFramework from test_framework.util import * -try: - import urllib.parse as urlparse -except ImportError: - import urlparse +import urllib.parse class AbandonConflictTest(BitcoinTestFramework): + def __init__(self): + super().__init__() + self.num_nodes = 2 + self.setup_clean_chain = False def setup_network(self): self.nodes = [] @@ -34,7 +35,7 @@ def run_test(self): assert(balance - newbalance < Decimal("0.001")) #no more than fees lost balance = newbalance - url = urlparse.urlparse(self.nodes[1].url) + url = urllib.parse.urlparse(self.nodes[1].url) self.nodes[0].disconnectnode(url.hostname+":"+str(p2p_port(1))) # Identify the 10btc outputs @@ -151,9 +152,9 @@ def run_test(self): self.nodes[0].invalidateblock(self.nodes[0].getbestblockhash()) newbalance = self.nodes[0].getbalance() #assert(newbalance == balance - Decimal("10")) - print "If balance has not declined after invalidateblock then out of mempool wallet tx which is no longer" - print "conflicted has not resumed causing its inputs to be seen as spent. See Issue #7315" - print balance , " -> " , newbalance , " ?" + print("If balance has not declined after invalidateblock then out of mempool wallet tx which is no longer") + print("conflicted has not resumed causing its inputs to be seen as spent. See Issue #7315") + print(str(balance) + " -> " + str(newbalance) + " ?") if __name__ == '__main__': AbandonConflictTest().main() diff --git a/qa/rpc-tests/bip65-cltv-p2p.py b/qa/rpc-tests/bip65-cltv-p2p.py index 99d74344aef95..754b6873b71ff 100755 --- a/qa/rpc-tests/bip65-cltv-p2p.py +++ b/qa/rpc-tests/bip65-cltv-p2p.py @@ -1,8 +1,7 @@ -#!/usr/bin/env python2 -# Copyright (c) 2015 The Bitcoin Core developers -# Distributed under the MIT/X11 software license, see the accompanying +#!/usr/bin/env python3 +# Copyright (c) 2015-2016 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. -# from test_framework.test_framework import ComparisonTestFramework from test_framework.util import * @@ -38,11 +37,12 @@ def cltv_invalidate(tx): class BIP65Test(ComparisonTestFramework): def __init__(self): + super().__init__() self.num_nodes = 1 def setup_network(self): # Must set the blockversion for this test - self.nodes = start_nodes(1, self.options.tmpdir, + self.nodes = start_nodes(self.num_nodes, self.options.tmpdir, extra_args=[['-debug', '-whitelist=127.0.0.1', '-blockversion=3']], binary=[self.options.testbinary]) @@ -67,13 +67,13 @@ def get_tests(self): self.coinbase_blocks = self.nodes[0].generate(2) height = 3 # height of the next block to build - self.tip = int ("0x" + self.nodes[0].getbestblockhash() + "L", 0) + self.tip = int("0x" + self.nodes[0].getbestblockhash(), 0) self.nodeaddress = self.nodes[0].getnewaddress() self.last_block_time = int(time.time()) ''' 98 more version 3 blocks ''' test_blocks = [] - for i in xrange(98): + for i in range(98): block = create_block(self.tip, create_coinbase(height), self.last_block_time + 1) block.nVersion = 3 block.rehash() @@ -86,7 +86,7 @@ def get_tests(self): ''' Mine 749 version 4 blocks ''' test_blocks = [] - for i in xrange(749): + for i in range(749): block = create_block(self.tip, create_coinbase(height), self.last_block_time + 1) block.nVersion = 4 block.rehash() @@ -138,7 +138,7 @@ def get_tests(self): ''' Mine 199 new version blocks on last valid tip ''' test_blocks = [] - for i in xrange(199): + for i in range(199): block = create_block(self.tip, create_coinbase(height), self.last_block_time + 1) block.nVersion = 4 block.rehash() diff --git a/qa/rpc-tests/bip65-cltv.py b/qa/rpc-tests/bip65-cltv.py index f666a07c9b769..abba7fc20e7d9 100755 --- a/qa/rpc-tests/bip65-cltv.py +++ b/qa/rpc-tests/bip65-cltv.py @@ -1,5 +1,5 @@ -#!/usr/bin/env python2 -# Copyright (c) 2015 The Bitcoin Core developers +#!/usr/bin/env python3 +# Copyright (c) 2015-2016 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -11,6 +11,10 @@ from test_framework.util import * class BIP65Test(BitcoinTestFramework): + def __init__(self): + super().__init__() + self.num_nodes = 3 + self.setup_clean_chain = False def setup_network(self): self.nodes = [] @@ -32,7 +36,7 @@ def run_test(self): raise AssertionError("Failed to mine 100 version=3 blocks") # Mine 750 new-version blocks - for i in xrange(15): + for i in range(15): self.nodes[2].generate(50) self.sync_all() if (self.nodes[0].getblockcount() != cnt + 850): @@ -49,7 +53,7 @@ def run_test(self): # TODO: check that new CHECKLOCKTIMEVERIFY rules are enforced # Mine 198 new-version blocks - for i in xrange(2): + for i in range(2): self.nodes[2].generate(99) self.sync_all() if (self.nodes[0].getblockcount() != cnt + 1049): diff --git a/qa/rpc-tests/bip68-112-113-p2p.py b/qa/rpc-tests/bip68-112-113-p2p.py index 3bcfdabe269a1..8ba07043849a6 100755 --- a/qa/rpc-tests/bip68-112-113-p2p.py +++ b/qa/rpc-tests/bip68-112-113-p2p.py @@ -1,8 +1,7 @@ -#!/usr/bin/env python2 +#!/usr/bin/env python3 # Copyright (c) 2015 The Bitcoin Core developers -# Distributed under the MIT/X11 software license, see the accompanying +# Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. -# from test_framework.test_framework import ComparisonTestFramework from test_framework.util import * @@ -63,13 +62,13 @@ # b31,b25,b22,b18 represent the 31st, 25th, 22nd and 18th bits respectively in the nSequence field # relative_locktimes[b31][b25][b22][b18] is a base_relative_locktime with the indicated bits set if their indices are 1 relative_locktimes = [] -for b31 in xrange(2): +for b31 in range(2): b25times = [] - for b25 in xrange(2): + for b25 in range(2): b22times = [] - for b22 in xrange(2): + for b22 in range(2): b18times = [] - for b18 in xrange(2): + for b18 in range(2): rlt = base_relative_locktime if (b31): rlt = rlt | seq_disable_flag @@ -86,20 +85,21 @@ def all_rlt_txs(txarray): txs = [] - for b31 in xrange(2): - for b25 in xrange(2): - for b22 in xrange(2): - for b18 in xrange(2): + for b31 in range(2): + for b25 in range(2): + for b22 in range(2): + for b18 in range(2): txs.append(txarray[b31][b25][b22][b18]) return txs class BIP68_112_113Test(ComparisonTestFramework): def __init__(self): + super().__init__() self.num_nodes = 1 def setup_network(self): # Must set the blockversion for this test - self.nodes = start_nodes(1, self.options.tmpdir, + self.nodes = start_nodes(self.num_nodes, self.options.tmpdir, extra_args=[['-debug', '-whitelist=127.0.0.1', '-blockversion=4']], binary=[self.options.testbinary]) @@ -131,7 +131,7 @@ def sign_transaction(self, node, unsignedtx): return tx def generate_blocks(self, number, version, test_blocks = []): - for i in xrange(number): + for i in range(number): block = self.create_test_block([], version) test_blocks.append([block, True]) self.last_block_time += 600 @@ -152,13 +152,13 @@ def create_bip68txs(self, bip68inputs, txversion, locktime_delta = 0): txs = [] assert(len(bip68inputs) >= 16) i = 0 - for b31 in xrange(2): + for b31 in range(2): b25txs = [] - for b25 in xrange(2): + for b25 in range(2): b22txs = [] - for b22 in xrange(2): + for b22 in range(2): b18txs = [] - for b18 in xrange(2): + for b18 in range(2): tx = self.create_transaction(self.nodes[0], bip68inputs[i], self.nodeaddress, Decimal("49.98")) i += 1 tx.nVersion = txversion @@ -180,13 +180,13 @@ def create_bip112txs(self, bip112inputs, varyOP_CSV, txversion, locktime_delta = txs = [] assert(len(bip112inputs) >= 16) i = 0 - for b31 in xrange(2): + for b31 in range(2): b25txs = [] - for b25 in xrange(2): + for b25 in range(2): b22txs = [] - for b22 in xrange(2): + for b22 in range(2): b18txs = [] - for b18 in xrange(2): + for b18 in range(2): tx = self.create_transaction(self.nodes[0], bip112inputs[i], self.nodeaddress, Decimal("49.98")) i += 1 if (varyOP_CSV): # if varying OP_CSV, nSequence is fixed @@ -212,7 +212,7 @@ def get_tests(self): self.nodes[0].setmocktime(0) # set time back to present so yielded blocks aren't in the future as we advance last_block_time self.tipheight = 82 # height of the next block to build self.last_block_time = long_past_time - self.tip = int ("0x" + self.nodes[0].getbestblockhash() + "L", 0) + self.tip = int("0x" + self.nodes[0].getbestblockhash(), 0) self.nodeaddress = self.nodes[0].getnewaddress() assert_equal(get_bip9_status(self.nodes[0], 'csv')['status'], 'defined') @@ -250,20 +250,20 @@ def get_tests(self): # Note we reuse inputs for v1 and v2 txs so must test these separately # 16 normal inputs bip68inputs = [] - for i in xrange(16): + for i in range(16): bip68inputs.append(self.send_generic_input_tx(self.nodes[0], self.coinbase_blocks)) # 2 sets of 16 inputs with 10 OP_CSV OP_DROP (actually will be prepended to spending scriptSig) bip112basicinputs = [] - for j in xrange(2): + for j in range(2): inputs = [] - for i in xrange(16): + for i in range(16): inputs.append(self.send_generic_input_tx(self.nodes[0], self.coinbase_blocks)) bip112basicinputs.append(inputs) # 2 sets of 16 varied inputs with (relative_lock_time) OP_CSV OP_DROP (actually will be prepended to spending scriptSig) bip112diverseinputs = [] - for j in xrange(2): + for j in range(2): inputs = [] - for i in xrange(16): + for i in range(16): inputs.append(self.send_generic_input_tx(self.nodes[0], self.coinbase_blocks)) bip112diverseinputs.append(inputs) # 1 special input with -1 OP_CSV OP_DROP (actually will be prepended to spending scriptSig) @@ -274,7 +274,7 @@ def get_tests(self): self.nodes[0].setmocktime(self.last_block_time + 600) inputblockhash = self.nodes[0].generate(1)[0] # 1 block generated for inputs to be in chain at height 572 self.nodes[0].setmocktime(0) - self.tip = int("0x" + inputblockhash + "L", 0) + self.tip = int("0x" + inputblockhash, 0) self.tipheight += 1 self.last_block_time += 600 assert_equal(len(self.nodes[0].getblock(inputblockhash,True)["tx"]), 82+1) @@ -398,22 +398,22 @@ def get_tests(self): ### Version 2 txs ### bip68success_txs = [] # All txs with SEQUENCE_LOCKTIME_DISABLE_FLAG set pass - for b25 in xrange(2): - for b22 in xrange(2): - for b18 in xrange(2): + for b25 in range(2): + for b22 in range(2): + for b18 in range(2): bip68success_txs.append(bip68txs_v2[1][b25][b22][b18]) yield TestInstance([[self.create_test_block(bip68success_txs), True]]) # 15 self.nodes[0].invalidateblock(self.nodes[0].getbestblockhash()) # All txs without flag fail as we are at delta height = 8 < 10 and delta time = 8 * 600 < 10 * 512 bip68timetxs = [] - for b25 in xrange(2): - for b18 in xrange(2): + for b25 in range(2): + for b18 in range(2): bip68timetxs.append(bip68txs_v2[0][b25][1][b18]) for tx in bip68timetxs: yield TestInstance([[self.create_test_block([tx]), False]]) # 16 - 19 bip68heighttxs = [] - for b25 in xrange(2): - for b18 in xrange(2): + for b25 in range(2): + for b18 in range(2): bip68heighttxs.append(bip68txs_v2[0][b25][0][b18]) for tx in bip68heighttxs: yield TestInstance([[self.create_test_block([tx]), False]]) # 20 - 23 @@ -445,9 +445,9 @@ def get_tests(self): yield TestInstance([[self.create_test_block([bip112tx_special_v1]), False]]) #32 # If SEQUENCE_LOCKTIME_DISABLE_FLAG is set in argument to OP_CSV, version 1 txs should still pass success_txs = [] - for b25 in xrange(2): - for b22 in xrange(2): - for b18 in xrange(2): + for b25 in range(2): + for b22 in range(2): + for b18 in range(2): success_txs.append(bip112txs_vary_OP_CSV_v1[1][b25][b22][b18]) success_txs.append(bip112txs_vary_OP_CSV_9_v1[1][b25][b22][b18]) yield TestInstance([[self.create_test_block(success_txs), True]]) # 33 @@ -457,9 +457,9 @@ def get_tests(self): fail_txs = [] fail_txs.extend(all_rlt_txs(bip112txs_vary_nSequence_v1)) fail_txs.extend(all_rlt_txs(bip112txs_vary_nSequence_9_v1)) - for b25 in xrange(2): - for b22 in xrange(2): - for b18 in xrange(2): + for b25 in range(2): + for b22 in range(2): + for b18 in range(2): fail_txs.append(bip112txs_vary_OP_CSV_v1[0][b25][b22][b18]) fail_txs.append(bip112txs_vary_OP_CSV_9_v1[0][b25][b22][b18]) @@ -472,9 +472,9 @@ def get_tests(self): # If SEQUENCE_LOCKTIME_DISABLE_FLAG is set in argument to OP_CSV, version 2 txs should pass (all sequence locks are met) success_txs = [] - for b25 in xrange(2): - for b22 in xrange(2): - for b18 in xrange(2): + for b25 in range(2): + for b22 in range(2): + for b18 in range(2): success_txs.append(bip112txs_vary_OP_CSV_v2[1][b25][b22][b18]) # 8/16 of vary_OP_CSV success_txs.append(bip112txs_vary_OP_CSV_9_v2[1][b25][b22][b18]) # 8/16 of vary_OP_CSV_9 @@ -485,9 +485,9 @@ def get_tests(self): # All txs with nSequence 9 should fail either due to earlier mismatch or failing the CSV check fail_txs = [] fail_txs.extend(all_rlt_txs(bip112txs_vary_nSequence_9_v2)) # 16/16 of vary_nSequence_9 - for b25 in xrange(2): - for b22 in xrange(2): - for b18 in xrange(2): + for b25 in range(2): + for b22 in range(2): + for b18 in range(2): fail_txs.append(bip112txs_vary_OP_CSV_9_v2[0][b25][b22][b18]) # 16/16 of vary_OP_CSV_9 for tx in fail_txs: @@ -495,17 +495,17 @@ def get_tests(self): # If SEQUENCE_LOCKTIME_DISABLE_FLAG is set in nSequence, tx should fail fail_txs = [] - for b25 in xrange(2): - for b22 in xrange(2): - for b18 in xrange(2): + for b25 in range(2): + for b22 in range(2): + for b18 in range(2): fail_txs.append(bip112txs_vary_nSequence_v2[1][b25][b22][b18]) # 8/16 of vary_nSequence for tx in fail_txs: yield TestInstance([[self.create_test_block([tx]), False]]) # 108-115 # If sequencelock types mismatch, tx should fail fail_txs = [] - for b25 in xrange(2): - for b18 in xrange(2): + for b25 in range(2): + for b18 in range(2): fail_txs.append(bip112txs_vary_nSequence_v2[0][b25][1][b18]) # 12/16 of vary_nSequence fail_txs.append(bip112txs_vary_OP_CSV_v2[0][b25][1][b18]) # 12/16 of vary_OP_CSV for tx in fail_txs: @@ -513,8 +513,8 @@ def get_tests(self): # Remaining txs should pass, just test masking works properly success_txs = [] - for b25 in xrange(2): - for b18 in xrange(2): + for b25 in range(2): + for b18 in range(2): success_txs.append(bip112txs_vary_nSequence_v2[0][b25][0][b18]) # 16/16 of vary_nSequence success_txs.append(bip112txs_vary_OP_CSV_v2[0][b25][0][b18]) # 16/16 of vary_OP_CSV yield TestInstance([[self.create_test_block(success_txs), True]]) # 124 @@ -522,8 +522,8 @@ def get_tests(self): # Additional test, of checking that comparison of two time types works properly time_txs = [] - for b25 in xrange(2): - for b18 in xrange(2): + for b25 in range(2): + for b18 in range(2): tx = bip112txs_vary_OP_CSV_v2[0][b25][1][b18] tx.vin[0].nSequence = base_relative_locktime | seq_type_flag signtx = self.sign_transaction(self.nodes[0], tx) diff --git a/qa/rpc-tests/bip68-sequence.py b/qa/rpc-tests/bip68-sequence.py index 33e05dfc51a09..a12bf10ebd758 100755 --- a/qa/rpc-tests/bip68-sequence.py +++ b/qa/rpc-tests/bip68-sequence.py @@ -1,5 +1,5 @@ -#!/usr/bin/env python2 -# Copyright (c) 2014-2015 The Bitcoin Core developers +#!/usr/bin/env python3 +# Copyright (c) 2014-2016 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -22,6 +22,10 @@ NOT_FINAL_ERROR = "64: non-BIP68-final" class BIP68Test(BitcoinTestFramework): + def __init__(self): + super().__init__() + self.num_nodes = 2 + self.setup_clean_chain = False def setup_network(self): self.nodes = [] @@ -35,28 +39,28 @@ def run_test(self): # Generate some coins self.nodes[0].generate(110) - print "Running test disable flag" + print("Running test disable flag") self.test_disable_flag() - print "Running test sequence-lock-confirmed-inputs" + print("Running test sequence-lock-confirmed-inputs") self.test_sequence_lock_confirmed_inputs() - print "Running test sequence-lock-unconfirmed-inputs" + print("Running test sequence-lock-unconfirmed-inputs") self.test_sequence_lock_unconfirmed_inputs() - print "Running test BIP68 not consensus before versionbits activation" + print("Running test BIP68 not consensus before versionbits activation") self.test_bip68_not_consensus() - print "Verifying nVersion=2 transactions aren't standard" + print("Verifying nVersion=2 transactions aren't standard") self.test_version2_relay(before_activation=True) - print "Activating BIP68 (and 112/113)" + print("Activating BIP68 (and 112/113)") self.activateCSV() - print "Verifying nVersion=2 transactions are now standard" + print("Verifying nVersion=2 transactions are now standard") self.test_version2_relay(before_activation=False) - print "Passed\n" + print("Passed\n") # Test that BIP68 is not in effect if tx version is 1, or if # the first sequence bit is set. @@ -125,7 +129,7 @@ def test_sequence_lock_confirmed_inputs(self): random.shuffle(addresses) num_outputs = random.randint(1, max_outputs) outputs = {} - for i in xrange(num_outputs): + for i in range(num_outputs): outputs[addresses[i]] = random.randint(1, 20)*0.01 self.nodes[0].sendmany("", outputs) self.nodes[0].generate(1) @@ -137,7 +141,7 @@ def test_sequence_lock_confirmed_inputs(self): # some of those inputs to be sequence locked (and randomly choose # between height/time locking). Small random chance of making the locks # all pass. - for i in xrange(400): + for i in range(400): # Randomly choose up to 10 inputs num_inputs = random.randint(1, 10) random.shuffle(utxos) @@ -151,7 +155,7 @@ def test_sequence_lock_confirmed_inputs(self): tx = CTransaction() tx.nVersion = 2 value = 0 - for j in xrange(num_inputs): + for j in range(num_inputs): sequence_value = 0xfffffffe # this disables sequence locks # 50% chance we enable sequence locks @@ -259,7 +263,7 @@ def test_nonzero_locks(orig_tx, node, relayfee, use_height_lock): # Use prioritisetransaction to lower the effective feerate to 0 self.nodes[0].prioritisetransaction(tx2.hash, -1e15, int(-self.relayfee*COIN)) cur_time = int(time.time()) - for i in xrange(10): + for i in range(10): self.nodes[0].setmocktime(cur_time + 600) self.nodes[0].generate(1) cur_time += 600 @@ -324,7 +328,7 @@ def test_nonzero_locks(orig_tx, node, relayfee, use_height_lock): # tx3 to be removed. tip = int(self.nodes[0].getblockhash(self.nodes[0].getblockcount()-1), 16) height = self.nodes[0].getblockcount() - for i in xrange(2): + for i in range(2): block = create_block(tip, create_coinbase(height), cur_time) block.nVersion = 3 block.rehash() diff --git a/qa/rpc-tests/bip9-softforks.py b/qa/rpc-tests/bip9-softforks.py index ddca3c2e33d34..d7e8e5e5a5588 100755 --- a/qa/rpc-tests/bip9-softforks.py +++ b/qa/rpc-tests/bip9-softforks.py @@ -1,9 +1,9 @@ -#!/usr/bin/env python2 +#!/usr/bin/env python3 # Copyright (c) 2015 The Bitcoin Core developers -# Distributed under the MIT/X11 software license, see the accompanying +# Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. -# +from test_framework.blockstore import BlockStore from test_framework.test_framework import ComparisonTestFramework from test_framework.util import * from test_framework.mininode import CTransaction, NetworkThread @@ -32,10 +32,11 @@ class BIP9SoftForksTest(ComparisonTestFramework): def __init__(self): + super().__init__() self.num_nodes = 1 def setup_network(self): - self.nodes = start_nodes(1, self.options.tmpdir, + self.nodes = start_nodes(self.num_nodes, self.options.tmpdir, extra_args=[['-debug', '-whitelist=127.0.0.1']], binary=[self.options.testbinary]) @@ -64,7 +65,7 @@ def sign_transaction(self, node, tx): return tx def generate_blocks(self, number, version, test_blocks = []): - for i in xrange(number): + for i in range(number): block = create_block(self.tip, create_coinbase(self.height), self.last_block_time + 1) block.nVersion = version block.rehash() @@ -79,15 +80,20 @@ def get_bip9_status(self, key): info = self.nodes[0].getblockchaininfo() return info['bip9_softforks'][key] - def test_BIP(self, bipName, activated_version, invalidate, invalidatePostSignature): + def test_BIP(self, bipName, activated_version, invalidate, invalidatePostSignature, bitno): # generate some coins for later self.coinbase_blocks = self.nodes[0].generate(2) self.height = 3 # height of the next block to build - self.tip = int ("0x" + self.nodes[0].getbestblockhash() + "L", 0) + self.tip = int("0x" + self.nodes[0].getbestblockhash(), 0) self.nodeaddress = self.nodes[0].getnewaddress() self.last_block_time = int(time.time()) assert_equal(self.get_bip9_status(bipName)['status'], 'defined') + tmpl = self.nodes[0].getblocktemplate({}) + assert(bipName not in tmpl['rules']) + assert(bipName not in tmpl['vbavailable']) + assert_equal(tmpl['vbrequired'], 0) + assert_equal(tmpl['version'], 0x20000000) # Test 1 # Advance from DEFINED to STARTED @@ -95,6 +101,11 @@ def test_BIP(self, bipName, activated_version, invalidate, invalidatePostSignatu yield TestInstance(test_blocks, sync_every_block=False) assert_equal(self.get_bip9_status(bipName)['status'], 'started') + tmpl = self.nodes[0].getblocktemplate({}) + assert(bipName not in tmpl['rules']) + assert_equal(tmpl['vbavailable'][bipName], bitno) + assert_equal(tmpl['vbrequired'], 0) + assert(tmpl['version'] & activated_version) # Test 2 # Fail to achieve LOCKED_IN 100 out of 144 signal bit 1 @@ -106,6 +117,11 @@ def test_BIP(self, bipName, activated_version, invalidate, invalidatePostSignatu yield TestInstance(test_blocks, sync_every_block=False) assert_equal(self.get_bip9_status(bipName)['status'], 'started') + tmpl = self.nodes[0].getblocktemplate({}) + assert(bipName not in tmpl['rules']) + assert_equal(tmpl['vbavailable'][bipName], bitno) + assert_equal(tmpl['vbrequired'], 0) + assert(tmpl['version'] & activated_version) # Test 3 # 108 out of 144 signal bit 1 to achieve LOCKED_IN @@ -117,6 +133,8 @@ def test_BIP(self, bipName, activated_version, invalidate, invalidatePostSignatu yield TestInstance(test_blocks, sync_every_block=False) assert_equal(self.get_bip9_status(bipName)['status'], 'locked_in') + tmpl = self.nodes[0].getblocktemplate({}) + assert(bipName not in tmpl['rules']) # Test 4 # 143 more version 536870913 blocks (waiting period-1) @@ -124,6 +142,8 @@ def test_BIP(self, bipName, activated_version, invalidate, invalidatePostSignatu yield TestInstance(test_blocks, sync_every_block=False) assert_equal(self.get_bip9_status(bipName)['status'], 'locked_in') + tmpl = self.nodes[0].getblocktemplate({}) + assert(bipName not in tmpl['rules']) # Test 5 # Check that the new rule is enforced @@ -147,6 +167,11 @@ def test_BIP(self, bipName, activated_version, invalidate, invalidatePostSignatu yield TestInstance([[block, True]]) assert_equal(self.get_bip9_status(bipName)['status'], 'active') + tmpl = self.nodes[0].getblocktemplate({}) + assert(bipName in tmpl['rules']) + assert(bipName not in tmpl['vbavailable']) + assert_equal(tmpl['vbrequired'], 0) + assert(not (tmpl['version'] & (1 << bitno))) # Test 6 # Check that the new sequence lock rules are enforced @@ -168,11 +193,13 @@ def test_BIP(self, bipName, activated_version, invalidate, invalidatePostSignatu yield TestInstance([[block, False]]) # Restart all + self.test.block_store.close() stop_nodes(self.nodes) wait_bitcoinds() shutil.rmtree(self.options.tmpdir) self.setup_chain() self.setup_network() + self.test.block_store = BlockStore(self.options.tmpdir) self.test.clear_all_connections() self.test.add_all_connections(self.nodes) NetworkThread().start() # Start up network handling in another thread @@ -180,9 +207,9 @@ def test_BIP(self, bipName, activated_version, invalidate, invalidatePostSignatu def get_tests(self): for test in itertools.chain( - self.test_BIP('csv', 536870913, self.sequence_lock_invalidate, self.donothing), - self.test_BIP('csv', 536870913, self.mtp_invalidate, self.donothing), - self.test_BIP('csv', 536870913, self.donothing, self.csv_invalidate) + self.test_BIP('csv', 0x20000001, self.sequence_lock_invalidate, self.donothing, 0), + self.test_BIP('csv', 0x20000001, self.mtp_invalidate, self.donothing, 0), + self.test_BIP('csv', 0x20000001, self.donothing, self.csv_invalidate, 0) ): yield test diff --git a/qa/rpc-tests/bipdersig-p2p.py b/qa/rpc-tests/bipdersig-p2p.py index bba86a50c4c56..4e4936a4ae6de 100755 --- a/qa/rpc-tests/bipdersig-p2p.py +++ b/qa/rpc-tests/bipdersig-p2p.py @@ -1,8 +1,7 @@ -#!/usr/bin/env python2 -# Copyright (c) 2015 The Bitcoin Core developers -# Distributed under the MIT/X11 software license, see the accompanying +#!/usr/bin/env python3 +# Copyright (c) 2015-2016 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. -# from test_framework.test_framework import ComparisonTestFramework from test_framework.util import * @@ -46,11 +45,12 @@ def unDERify(tx): class BIP66Test(ComparisonTestFramework): def __init__(self): + super().__init__() self.num_nodes = 1 def setup_network(self): # Must set the blockversion for this test - self.nodes = start_nodes(1, self.options.tmpdir, + self.nodes = start_nodes(self.num_nodes, self.options.tmpdir, extra_args=[['-debug', '-whitelist=127.0.0.1', '-blockversion=2']], binary=[self.options.testbinary]) @@ -75,13 +75,13 @@ def get_tests(self): self.coinbase_blocks = self.nodes[0].generate(2) height = 3 # height of the next block to build - self.tip = int ("0x" + self.nodes[0].getbestblockhash() + "L", 0) + self.tip = int("0x" + self.nodes[0].getbestblockhash(), 0) self.nodeaddress = self.nodes[0].getnewaddress() self.last_block_time = int(time.time()) ''' 98 more version 2 blocks ''' test_blocks = [] - for i in xrange(98): + for i in range(98): block = create_block(self.tip, create_coinbase(height), self.last_block_time + 1) block.nVersion = 2 block.rehash() @@ -94,7 +94,7 @@ def get_tests(self): ''' Mine 749 version 3 blocks ''' test_blocks = [] - for i in xrange(749): + for i in range(749): block = create_block(self.tip, create_coinbase(height), self.last_block_time + 1) block.nVersion = 3 block.rehash() @@ -146,7 +146,7 @@ def get_tests(self): ''' Mine 199 new version blocks on last valid tip ''' test_blocks = [] - for i in xrange(199): + for i in range(199): block = create_block(self.tip, create_coinbase(height), self.last_block_time + 1) block.nVersion = 3 block.rehash() diff --git a/qa/rpc-tests/bipdersig.py b/qa/rpc-tests/bipdersig.py index be9121c456831..17c2ced79ac1e 100755 --- a/qa/rpc-tests/bipdersig.py +++ b/qa/rpc-tests/bipdersig.py @@ -1,5 +1,5 @@ -#!/usr/bin/env python2 -# Copyright (c) 2014-2015 The Bitcoin Core developers +#!/usr/bin/env python3 +# Copyright (c) 2014-2016 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -11,6 +11,10 @@ from test_framework.util import * class BIP66Test(BitcoinTestFramework): + def __init__(self): + super().__init__() + self.num_nodes = 3 + self.setup_clean_chain = False def setup_network(self): self.nodes = [] @@ -32,7 +36,7 @@ def run_test(self): raise AssertionError("Failed to mine 100 version=2 blocks") # Mine 750 new-version blocks - for i in xrange(15): + for i in range(15): self.nodes[2].generate(50) self.sync_all() if (self.nodes[0].getblockcount() != cnt + 850): @@ -49,7 +53,7 @@ def run_test(self): # TODO: check that new DERSIG rules are enforced # Mine 198 new-version blocks - for i in xrange(2): + for i in range(2): self.nodes[2].generate(99) self.sync_all() if (self.nodes[0].getblockcount() != cnt + 1049): diff --git a/qa/rpc-tests/blockchain.py b/qa/rpc-tests/blockchain.py index 8f59ee741c366..410b85d15e35f 100755 --- a/qa/rpc-tests/blockchain.py +++ b/qa/rpc-tests/blockchain.py @@ -1,5 +1,5 @@ -#!/usr/bin/env python2 -# Copyright (c) 2014-2015 The Bitcoin Core developers +#!/usr/bin/env python3 +# Copyright (c) 2014-2016 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -13,7 +13,6 @@ from test_framework.test_framework import BitcoinTestFramework from test_framework.authproxy import JSONRPCException from test_framework.util import ( - initialize_chain, assert_equal, assert_raises, assert_is_hex_string, @@ -32,12 +31,13 @@ class BlockchainTest(BitcoinTestFramework): """ - def setup_chain(self): - print("Initializing test directory " + self.options.tmpdir) - initialize_chain(self.options.tmpdir) + def __init__(self): + super().__init__() + self.setup_clean_chain = False + self.num_nodes = 2 def setup_network(self, split=False): - self.nodes = start_nodes(2, self.options.tmpdir) + self.nodes = start_nodes(self.num_nodes, self.options.tmpdir) connect_nodes_bi(self.nodes, 0, 1) self.is_network_split = False self.sync_all() @@ -51,13 +51,13 @@ def _test_gettxoutsetinfo(self): node = self.nodes[0] res = node.gettxoutsetinfo() - assert_equal(res[u'total_amount'], Decimal('8725.00000000')) - assert_equal(res[u'transactions'], 200) - assert_equal(res[u'height'], 200) - assert_equal(res[u'txouts'], 200) - assert_equal(res[u'bytes_serialized'], 13924), - assert_equal(len(res[u'bestblock']), 64) - assert_equal(len(res[u'hash_serialized']), 64) + assert_equal(res['total_amount'], Decimal('8725.00000000')) + assert_equal(res['transactions'], 200) + assert_equal(res['height'], 200) + assert_equal(res['txouts'], 200) + assert_equal(res['bytes_serialized'], 13924), + assert_equal(len(res['bestblock']), 64) + assert_equal(len(res['hash_serialized']), 64) def _test_getblockheader(self): node = self.nodes[0] diff --git a/qa/rpc-tests/create_cache.py b/qa/rpc-tests/create_cache.py new file mode 100755 index 0000000000000..b6161e0917d07 --- /dev/null +++ b/qa/rpc-tests/create_cache.py @@ -0,0 +1,23 @@ +#!/usr/bin/env python3 +# Copyright (c) 2016 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +# +# Helper script to create the cache +# (see BitcoinTestFramework.setup_chain) +# + +from test_framework.test_framework import BitcoinTestFramework + +class CreateCache(BitcoinTestFramework): + + def setup_network(self): + # Don't setup any test nodes + self.options.noshutdown = True + + def run_test(self): + pass + +if __name__ == '__main__': + CreateCache().main() diff --git a/qa/rpc-tests/decodescript.py b/qa/rpc-tests/decodescript.py index 578844f2c5bb0..24768c265577a 100755 --- a/qa/rpc-tests/decodescript.py +++ b/qa/rpc-tests/decodescript.py @@ -1,5 +1,5 @@ -#!/usr/bin/env python2 -# Copyright (c) 2015 The Bitcoin Core developers +#!/usr/bin/env python3 +# Copyright (c) 2015-2016 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -11,12 +11,13 @@ class DecodeScriptTest(BitcoinTestFramework): """Tests decoding scripts via RPC command "decodescript".""" - def setup_chain(self): - print('Initializing test directory ' + self.options.tmpdir) - initialize_chain_clean(self.options.tmpdir, 1) + def __init__(self): + super().__init__() + self.setup_clean_chain = True + self.num_nodes = 1 def setup_network(self, split=False): - self.nodes = start_nodes(1, self.options.tmpdir) + self.nodes = start_nodes(self.num_nodes, self.options.tmpdir) self.is_network_split = False def decodescript_script_sig(self): diff --git a/qa/rpc-tests/disablewallet.py b/qa/rpc-tests/disablewallet.py index cb868029fa327..36c147edad3a6 100755 --- a/qa/rpc-tests/disablewallet.py +++ b/qa/rpc-tests/disablewallet.py @@ -1,5 +1,5 @@ -#!/usr/bin/env python2 -# Copyright (c) 2014-2015 The Bitcoin Core developers +#!/usr/bin/env python3 +# Copyright (c) 2015-2016 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -13,12 +13,13 @@ class DisableWalletTest (BitcoinTestFramework): - def setup_chain(self): - print("Initializing test directory "+self.options.tmpdir) - initialize_chain_clean(self.options.tmpdir, 1) + def __init__(self): + super().__init__() + self.setup_clean_chain = True + self.num_nodes = 1 def setup_network(self, split=False): - self.nodes = start_nodes(1, self.options.tmpdir, [['-disablewallet']]) + self.nodes = start_nodes(self.num_nodes, self.options.tmpdir, [['-disablewallet']]) self.is_network_split = False self.sync_all() diff --git a/qa/rpc-tests/forknotify.py b/qa/rpc-tests/forknotify.py index 20e6ce961954b..5a3f75c808741 100755 --- a/qa/rpc-tests/forknotify.py +++ b/qa/rpc-tests/forknotify.py @@ -1,5 +1,5 @@ -#!/usr/bin/env python2 -# Copyright (c) 2014-2015 The Bitcoin Core developers +#!/usr/bin/env python3 +# Copyright (c) 2014-2016 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -12,6 +12,11 @@ class ForkNotifyTest(BitcoinTestFramework): + def __init__(self): + super().__init__() + self.num_nodes = 2 + self.setup_clean_chain = False + alert_filename = None # Set by setup_network def setup_network(self): diff --git a/qa/rpc-tests/fundrawtransaction.py b/qa/rpc-tests/fundrawtransaction.py index 496c7fe8b0ed0..998f822afe81d 100755 --- a/qa/rpc-tests/fundrawtransaction.py +++ b/qa/rpc-tests/fundrawtransaction.py @@ -1,5 +1,5 @@ -#!/usr/bin/env python2 -# Copyright (c) 2014-2015 The Bitcoin Core developers +#!/usr/bin/env python3 +# Copyright (c) 2014-2016 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -9,12 +9,13 @@ # Create one-input, one-output, no-fee transaction: class RawTransactionsTest(BitcoinTestFramework): - def setup_chain(self): - print("Initializing test directory "+self.options.tmpdir) - initialize_chain_clean(self.options.tmpdir, 4) + def __init__(self): + super().__init__() + self.setup_clean_chain = True + self.num_nodes = 4 def setup_network(self, split=False): - self.nodes = start_nodes(4, self.options.tmpdir) + self.nodes = start_nodes(self.num_nodes, self.options.tmpdir) connect_nodes_bi(self.nodes,0,1) connect_nodes_bi(self.nodes,1,2) @@ -25,7 +26,7 @@ def setup_network(self, split=False): self.sync_all() def run_test(self): - print "Mining blocks..." + print("Mining blocks...") min_relay_tx_fee = self.nodes[0].getnetworkinfo()['relayfee'] # This test is not meant to test fee estimation and we'd like @@ -198,7 +199,7 @@ def run_test(self): try: self.nodes[2].fundrawtransaction(rawtx, {'foo': 'bar'}) raise AssertionError("Accepted invalid option foo") - except JSONRPCException,e: + except JSONRPCException as e: assert("Unexpected key foo" in e.error['message']) @@ -223,7 +224,7 @@ def run_test(self): try: self.nodes[2].fundrawtransaction(rawtx, {'changeAddress': 'foobar'}) raise AssertionError("Accepted invalid bitcoin address") - except JSONRPCException,e: + except JSONRPCException as e: assert("changeAddress must be a valid bitcoin address" in e.error['message']) @@ -521,7 +522,7 @@ def run_test(self): stop_nodes(self.nodes) wait_bitcoinds() - self.nodes = start_nodes(4, self.options.tmpdir) + self.nodes = start_nodes(self.num_nodes, self.options.tmpdir) # This test is not meant to test fee estimation and we'd like # to be sure all txs are sent at a consistent desired feerate for node in self.nodes: @@ -677,6 +678,14 @@ def run_test(self): assert(signedtx["complete"]) self.nodes[0].sendrawtransaction(signedtx["hex"]) + inputs = [] + outputs = {self.nodes[2].getnewaddress() : 1} + rawtx = self.nodes[3].createrawtransaction(inputs, outputs) + result = self.nodes[3].fundrawtransaction(rawtx) # uses min_relay_tx_fee (set by settxfee) + result2 = self.nodes[3].fundrawtransaction(rawtx, {"feeRate": 2*min_relay_tx_fee}) + result3 = self.nodes[3].fundrawtransaction(rawtx, {"feeRate": 10*min_relay_tx_fee}) + assert_equal(result['fee']*2, result2['fee']) + assert_equal(result['fee']*10, result3['fee']) if __name__ == '__main__': RawTransactionsTest().main() diff --git a/qa/rpc-tests/getblocktemplate_longpoll.py b/qa/rpc-tests/getblocktemplate_longpoll.py index 3e85957ae2323..3cddf4046aaf6 100755 --- a/qa/rpc-tests/getblocktemplate_longpoll.py +++ b/qa/rpc-tests/getblocktemplate_longpoll.py @@ -1,33 +1,11 @@ -#!/usr/bin/env python2 -# Copyright (c) 2014-2015 The Bitcoin Core developers +#!/usr/bin/env python3 +# Copyright (c) 2014-2016 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. from test_framework.test_framework import BitcoinTestFramework from test_framework.util import * - -def check_array_result(object_array, to_match, expected): - """ - Pass in array of JSON objects, a dictionary with key/value pairs - to match against, and another dictionary with expected key/value - pairs. - """ - num_matched = 0 - for item in object_array: - all_match = True - for key,value in to_match.items(): - if item[key] != value: - all_match = False - if not all_match: - continue - for key,value in expected.items(): - if item[key] != value: - raise AssertionError("%s : expected %s=%s"%(str(item), str(key), str(value))) - num_matched = num_matched+1 - if num_matched == 0: - raise AssertionError("No objects matched %s"%(str(to_match))) - import threading class LongpollThread(threading.Thread): @@ -48,8 +26,13 @@ class GetBlockTemplateLPTest(BitcoinTestFramework): Test longpolling with getblocktemplate. ''' + def __init__(self): + super().__init__() + self.num_nodes = 4 + self.setup_clean_chain = False + def run_test(self): - print "Warning: this test will take about 70 seconds in the best case. Be patient." + print("Warning: this test will take about 70 seconds in the best case. Be patient.") self.nodes[0].generate(10) templat = self.nodes[0].getblocktemplate() longpollid = templat['longpollid'] diff --git a/qa/rpc-tests/getblocktemplate_proposals.py b/qa/rpc-tests/getblocktemplate_proposals.py index 07bfe69c6d869..7a4f8f8fdc3c2 100755 --- a/qa/rpc-tests/getblocktemplate_proposals.py +++ b/qa/rpc-tests/getblocktemplate_proposals.py @@ -1,5 +1,5 @@ -#!/usr/bin/env python2 -# Copyright (c) 2014-2015 The Bitcoin Core developers +#!/usr/bin/env python3 +# Copyright (c) 2014-2016 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -10,28 +10,6 @@ from hashlib import sha256 from struct import pack - -def check_array_result(object_array, to_match, expected): - """ - Pass in array of JSON objects, a dictionary with key/value pairs - to match against, and another dictionary with expected key/value - pairs. - """ - num_matched = 0 - for item in object_array: - all_match = True - for key,value in to_match.items(): - if item[key] != value: - all_match = False - if not all_match: - continue - for key,value in expected.items(): - if item[key] != value: - raise AssertionError("%s : expected %s=%s"%(str(item), str(key), str(value))) - num_matched = num_matched+1 - if num_matched == 0: - raise AssertionError("No objects matched %s"%(str(to_match))) - def b2x(b): return b2a_hex(b).decode('ascii') @@ -68,7 +46,7 @@ def genmrklroot(leaflist): cur = n return cur[0] -def template_to_bytes(tmpl, txlist): +def template_to_bytearray(tmpl, txlist): blkver = pack(' MAX_REQUESTS: raise AssertionError("Error, too many blocks (%d) requested" % total_requests) - print "Round %d: success (total requests: %d)" % (count, total_requests) + print("Round %d: success (total requests: %d)" % (count, total_requests)) self.disconnectOkay = True self.connection.disconnect_node() @@ -77,12 +76,13 @@ def add_options(self, parser): default=os.getenv("BITCOIND", "bitcoind"), help="Binary to test max block requests behavior") - def setup_chain(self): - print "Initializing test directory "+self.options.tmpdir - initialize_chain_clean(self.options.tmpdir, 1) + def __init__(self): + super().__init__() + self.setup_clean_chain = True + self.num_nodes = 1 def setup_network(self): - self.nodes = start_nodes(1, self.options.tmpdir, + self.nodes = start_nodes(self.num_nodes, self.options.tmpdir, extra_args=[['-debug', '-whitelist=127.0.0.1']], binary=[self.options.testbinary]) diff --git a/qa/rpc-tests/maxuploadtarget.py b/qa/rpc-tests/maxuploadtarget.py index fb290f476b57d..125d4eb275190 100755 --- a/qa/rpc-tests/maxuploadtarget.py +++ b/qa/rpc-tests/maxuploadtarget.py @@ -1,8 +1,7 @@ -#!/usr/bin/env python2 -# -# Distributed under the MIT/X11 software license, see the accompanying +#!/usr/bin/env python3 +# Copyright (c) 2015-2016 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. -# from test_framework.mininode import * from test_framework.test_framework import BitcoinTestFramework @@ -81,17 +80,19 @@ def received_pong(): return success class MaxUploadTest(BitcoinTestFramework): - def __init__(self): - self.utxo = [] - self.txouts = gen_return_txouts() def add_options(self, parser): parser.add_option("--testbinary", dest="testbinary", default=os.getenv("BITCOIND", "bitcoind"), help="bitcoind binary to test") - def setup_chain(self): - initialize_chain_clean(self.options.tmpdir, 2) + def __init__(self): + super().__init__() + self.setup_clean_chain = True + self.num_nodes = 1 + + self.utxo = [] + self.txouts = gen_return_txouts() def setup_network(self): # Start a node with maxuploadtarget of 200 MB (/24h) @@ -101,7 +102,7 @@ def setup_network(self): def mine_full_block(self, node, address): # Want to create a full block # We'll generate a 66k transaction below, and 14 of them is close to the 1MB block limit - for j in xrange(14): + for j in range(14): if len(self.utxo) < 14: self.utxo = node.listunspent() inputs=[] @@ -139,7 +140,7 @@ def run_test(self): test_nodes = [] connections = [] - for i in xrange(3): + for i in range(3): test_nodes.append(TestNode()) connections.append(NodeConn('127.0.0.1', p2p_port(0), self.nodes[0], test_nodes[i])) test_nodes[i].add_connection(connections[i]) @@ -181,7 +182,7 @@ def run_test(self): # 576MB will be reserved for relaying new blocks, so expect this to # succeed for ~235 tries. - for i in xrange(success_count): + for i in range(success_count): test_nodes[0].send_message(getdata_request) test_nodes[0].sync_with_ping() assert_equal(test_nodes[0].block_receive_map[big_old_block], i+1) @@ -189,22 +190,22 @@ def run_test(self): assert_equal(len(self.nodes[0].getpeerinfo()), 3) # At most a couple more tries should succeed (depending on how long # the test has been running so far). - for i in xrange(3): + for i in range(3): test_nodes[0].send_message(getdata_request) test_nodes[0].wait_for_disconnect() assert_equal(len(self.nodes[0].getpeerinfo()), 2) - print "Peer 0 disconnected after downloading old block too many times" + print("Peer 0 disconnected after downloading old block too many times") # Requesting the current block on test_nodes[1] should succeed indefinitely, # even when over the max upload target. # We'll try 800 times getdata_request.inv = [CInv(2, big_new_block)] - for i in xrange(800): + for i in range(800): test_nodes[1].send_message(getdata_request) test_nodes[1].sync_with_ping() assert_equal(test_nodes[1].block_receive_map[big_new_block], i+1) - print "Peer 1 able to repeatedly download new block" + print("Peer 1 able to repeatedly download new block") # But if test_nodes[1] tries for an old block, it gets disconnected too. getdata_request.inv = [CInv(2, big_old_block)] @@ -212,9 +213,9 @@ def run_test(self): test_nodes[1].wait_for_disconnect() assert_equal(len(self.nodes[0].getpeerinfo()), 1) - print "Peer 1 disconnected after trying to download old block" + print("Peer 1 disconnected after trying to download old block") - print "Advancing system time on node to clear counters..." + print("Advancing system time on node to clear counters...") # If we advance the time by 24 hours, then the counters should reset, # and test_nodes[2] should be able to retrieve the old block. @@ -224,12 +225,12 @@ def run_test(self): test_nodes[2].sync_with_ping() assert_equal(test_nodes[2].block_receive_map[big_old_block], 1) - print "Peer 2 able to download old block" + print("Peer 2 able to download old block") [c.disconnect_node() for c in connections] #stop and start node 0 with 1MB maxuploadtarget, whitelist 127.0.0.1 - print "Restarting nodes with -whitelist=127.0.0.1" + print("Restarting nodes with -whitelist=127.0.0.1") stop_node(self.nodes[0], 0) self.nodes[0] = start_node(0, self.options.tmpdir, ["-debug", "-whitelist=127.0.0.1", "-maxuploadtarget=1", "-blockmaxsize=999000"]) @@ -237,7 +238,7 @@ def run_test(self): test_nodes = [] connections = [] - for i in xrange(3): + for i in range(3): test_nodes.append(TestNode()) connections.append(NodeConn('127.0.0.1', p2p_port(0), self.nodes[0], test_nodes[i])) test_nodes[i].add_connection(connections[i]) @@ -247,7 +248,7 @@ def run_test(self): #retrieve 20 blocks which should be enough to break the 1MB limit getdata_request.inv = [CInv(2, big_new_block)] - for i in xrange(20): + for i in range(20): test_nodes[1].send_message(getdata_request) test_nodes[1].sync_with_ping() assert_equal(test_nodes[1].block_receive_map[big_new_block], i+1) @@ -257,7 +258,7 @@ def run_test(self): test_nodes[1].wait_for_disconnect() assert_equal(len(self.nodes[0].getpeerinfo()), 3) #node is still connected because of the whitelist - print "Peer 1 still connected after trying to download old block (whitelisted)" + print("Peer 1 still connected after trying to download old block (whitelisted)") [c.disconnect_node() for c in connections] diff --git a/qa/rpc-tests/mempool_limit.py b/qa/rpc-tests/mempool_limit.py index c19a63c6996d2..4438c152df10a 100755 --- a/qa/rpc-tests/mempool_limit.py +++ b/qa/rpc-tests/mempool_limit.py @@ -1,5 +1,5 @@ -#!/usr/bin/env python2 -# Copyright (c) 2014-2015 The Bitcoin Core developers +#!/usr/bin/env python3 +# Copyright (c) 2014-2016 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -10,9 +10,6 @@ class MempoolLimitTest(BitcoinTestFramework): - def __init__(self): - self.txouts = gen_return_txouts() - def setup_network(self): self.nodes = [] self.nodes.append(start_node(0, self.options.tmpdir, ["-maxmempool=5", "-spendzeroconfchange=0", "-debug"])) @@ -20,9 +17,12 @@ def setup_network(self): self.sync_all() self.relayfee = self.nodes[0].getnetworkinfo()['relayfee'] - def setup_chain(self): - print("Initializing test directory "+self.options.tmpdir) - initialize_chain_clean(self.options.tmpdir, 2) + def __init__(self): + super().__init__() + self.setup_clean_chain = True + self.num_nodes = 1 + + self.txouts = gen_return_txouts() def run_test(self): txids = [] @@ -41,7 +41,7 @@ def run_test(self): relayfee = self.nodes[0].getnetworkinfo()['relayfee'] base_fee = relayfee*100 - for i in xrange (4): + for i in range (4): txids.append([]) txids[i] = create_lots_of_big_transactions(self.nodes[0], self.txouts, utxos[30*i:30*i+30], (i+1)*base_fee) diff --git a/qa/rpc-tests/mempool_packages.py b/qa/rpc-tests/mempool_packages.py index bc3f9e051cef1..45dc0e65c43c2 100755 --- a/qa/rpc-tests/mempool_packages.py +++ b/qa/rpc-tests/mempool_packages.py @@ -1,5 +1,5 @@ -#!/usr/bin/env python2 -# Copyright (c) 2014-2015 The Bitcoin Core developers +#!/usr/bin/env python3 +# Copyright (c) 2014-2016 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -13,6 +13,10 @@ MAX_DESCENDANTS = 25 class MempoolPackagesTest(BitcoinTestFramework): + def __init__(self): + super().__init__() + self.num_nodes = 2 + self.setup_clean_chain = False def setup_network(self): self.nodes = [] @@ -28,7 +32,7 @@ def chain_transaction(self, node, parent_txid, vout, value, fee, num_outputs): send_value = satoshi_round((value - fee)/num_outputs) inputs = [ {'txid' : parent_txid, 'vout' : vout} ] outputs = {} - for i in xrange(num_outputs): + for i in range(num_outputs): outputs[node.getnewaddress()] = send_value rawtx = node.createrawtransaction(inputs, outputs) signedtx = node.signrawtransaction(rawtx) @@ -48,7 +52,7 @@ def run_test(self): fee = Decimal("0.0001") # MAX_ANCESTORS transactions off a confirmed tx should be fine chain = [] - for i in xrange(MAX_ANCESTORS): + for i in range(MAX_ANCESTORS): (txid, sent_value) = self.chain_transaction(self.nodes[0], txid, 0, value, fee, 1) value = sent_value chain.append(txid) @@ -61,7 +65,14 @@ def run_test(self): descendant_fees = 0 descendant_size = 0 + descendants = [] + ancestors = list(chain) for x in reversed(chain): + # Check that getmempoolentry is consistent with getrawmempool + entry = self.nodes[0].getmempoolentry(x) + assert_equal(entry, mempool[x]) + + # Check that the descendant calculations are correct assert_equal(mempool[x]['descendantcount'], descendant_count) descendant_fees += mempool[x]['fee'] assert_equal(mempool[x]['modifiedfee'], mempool[x]['fee']) @@ -70,6 +81,27 @@ def run_test(self): assert_equal(mempool[x]['descendantsize'], descendant_size) descendant_count += 1 + # Check that getmempooldescendants is correct + assert_equal(sorted(descendants), sorted(self.nodes[0].getmempooldescendants(x))) + descendants.append(x) + + # Check that getmempoolancestors is correct + ancestors.remove(x) + assert_equal(sorted(ancestors), sorted(self.nodes[0].getmempoolancestors(x))) + + # Check that getmempoolancestors/getmempooldescendants correctly handle verbose=true + v_ancestors = self.nodes[0].getmempoolancestors(chain[-1], True) + assert_equal(len(v_ancestors), len(chain)-1) + for x in v_ancestors.keys(): + assert_equal(mempool[x], v_ancestors[x]) + assert(chain[-1] not in v_ancestors.keys()) + + v_descendants = self.nodes[0].getmempooldescendants(chain[0], True) + assert_equal(len(v_descendants), len(chain)-1) + for x in v_descendants.keys(): + assert_equal(mempool[x], v_descendants[x]) + assert(chain[0] not in v_descendants.keys()) + # Check that descendant modified fees includes fee deltas from # prioritisetransaction self.nodes[0].prioritisetransaction(chain[-1], 0, 1000) @@ -84,7 +116,7 @@ def run_test(self): try: self.chain_transaction(self.nodes[0], txid, vout, value, fee, 1) except JSONRPCException as e: - print "too-long-ancestor-chain successfully rejected" + print("too-long-ancestor-chain successfully rejected") # Check that prioritising a tx before it's added to the mempool works # First clear the mempool by mining a block. @@ -121,22 +153,22 @@ def run_test(self): # First create one parent tx with 10 children (txid, sent_value) = self.chain_transaction(self.nodes[0], txid, vout, value, fee, 10) parent_transaction = txid - for i in xrange(10): + for i in range(10): transaction_package.append({'txid': txid, 'vout': i, 'amount': sent_value}) - for i in xrange(MAX_DESCENDANTS): + for i in range(MAX_DESCENDANTS): utxo = transaction_package.pop(0) try: (txid, sent_value) = self.chain_transaction(self.nodes[0], utxo['txid'], utxo['vout'], utxo['amount'], fee, 10) - for j in xrange(10): + for j in range(10): transaction_package.append({'txid': txid, 'vout': j, 'amount': sent_value}) if i == MAX_DESCENDANTS - 2: mempool = self.nodes[0].getrawmempool(True) assert_equal(mempool[parent_transaction]['descendantcount'], MAX_DESCENDANTS) except JSONRPCException as e: - print e.error['message'] + print(e.error['message']) assert_equal(i, MAX_DESCENDANTS - 1) - print "tx that would create too large descendant package successfully rejected" + print("tx that would create too large descendant package successfully rejected") # TODO: check that node1's mempool is as expected @@ -171,7 +203,7 @@ def run_test(self): send_value = satoshi_round((value - fee)/2) inputs = [ {'txid' : txid, 'vout' : vout} ] outputs = {} - for i in xrange(2): + for i in range(2): outputs[self.nodes[0].getnewaddress()] = send_value rawtx = self.nodes[0].createrawtransaction(inputs, outputs) signedtx = self.nodes[0].signrawtransaction(rawtx) @@ -185,7 +217,7 @@ def run_test(self): # Create tx2-7 vout = 1 txid = tx0_id - for i in xrange(6): + for i in range(6): (txid, sent_value) = self.chain_transaction(self.nodes[0], txid, vout, value, fee, 1) vout = 0 value = sent_value diff --git a/qa/rpc-tests/mempool_reorg.py b/qa/rpc-tests/mempool_reorg.py index 5e9856e5d4318..301b094eb08ca 100755 --- a/qa/rpc-tests/mempool_reorg.py +++ b/qa/rpc-tests/mempool_reorg.py @@ -1,5 +1,5 @@ -#!/usr/bin/env python2 -# Copyright (c) 2014-2015 The Bitcoin Core developers +#!/usr/bin/env python3 +# Copyright (c) 2014-2016 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -13,6 +13,10 @@ # Create one-input, one-output, no-fee transaction: class MempoolCoinbaseTest(BitcoinTestFramework): + def __init__(self): + super().__init__() + self.num_nodes = 2 + self.setup_clean_chain = False alert_filename = None # Set by setup_network diff --git a/qa/rpc-tests/mempool_resurrect_test.py b/qa/rpc-tests/mempool_resurrect_test.py index 0ba46e6f51e8a..3db12cbf76a68 100755 --- a/qa/rpc-tests/mempool_resurrect_test.py +++ b/qa/rpc-tests/mempool_resurrect_test.py @@ -1,5 +1,5 @@ -#!/usr/bin/env python2 -# Copyright (c) 2014-2015 The Bitcoin Core developers +#!/usr/bin/env python3 +# Copyright (c) 2014-2016 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -14,6 +14,11 @@ # Create one-input, one-output, no-fee transaction: class MempoolCoinbaseTest(BitcoinTestFramework): + def __init__(self): + super().__init__() + self.num_nodes = 1 + self.setup_clean_chain = False + def setup_network(self): # Just need one node for this test args = ["-checkmempool", "-debug=mempool"] diff --git a/qa/rpc-tests/mempool_spendcoinbase.py b/qa/rpc-tests/mempool_spendcoinbase.py index 507b5ff416409..d5e4bf52d26ce 100755 --- a/qa/rpc-tests/mempool_spendcoinbase.py +++ b/qa/rpc-tests/mempool_spendcoinbase.py @@ -1,5 +1,5 @@ -#!/usr/bin/env python2 -# Copyright (c) 2014-2015 The Bitcoin Core developers +#!/usr/bin/env python3 +# Copyright (c) 2014-2016 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -19,6 +19,11 @@ # Create one-input, one-output, no-fee transaction: class MempoolSpendCoinbaseTest(BitcoinTestFramework): + def __init__(self): + super().__init__() + self.num_nodes = 1 + self.setup_clean_chain = False + def setup_network(self): # Just need one node for this test args = ["-checkmempool", "-debug=mempool"] diff --git a/qa/rpc-tests/merkle_blocks.py b/qa/rpc-tests/merkle_blocks.py index eb718f39e41a6..b2155d7fc33c3 100755 --- a/qa/rpc-tests/merkle_blocks.py +++ b/qa/rpc-tests/merkle_blocks.py @@ -1,5 +1,5 @@ -#!/usr/bin/env python2 -# Copyright (c) 2014-2015 The Bitcoin Core developers +#!/usr/bin/env python3 +# Copyright (c) 2014-2016 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -12,9 +12,10 @@ class MerkleBlockTest(BitcoinTestFramework): - def setup_chain(self): - print("Initializing test directory "+self.options.tmpdir) - initialize_chain_clean(self.options.tmpdir, 4) + def __init__(self): + super().__init__() + self.setup_clean_chain = True + self.num_nodes = 4 def setup_network(self): self.nodes = [] @@ -32,7 +33,7 @@ def setup_network(self): self.sync_all() def run_test(self): - print "Mining blocks..." + print("Mining blocks...") self.nodes[0].generate(105) self.sync_all() diff --git a/qa/rpc-tests/multi_rpc.py b/qa/rpc-tests/multi_rpc.py index afb18cf3da20d..24373b257d7e6 100755 --- a/qa/rpc-tests/multi_rpc.py +++ b/qa/rpc-tests/multi_rpc.py @@ -1,5 +1,5 @@ -#!/usr/bin/env python2 -# Copyright (c) 2015 The Bitcoin Core developers +#!/usr/bin/env python3 +# Copyright (c) 2015-2016 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -8,25 +8,21 @@ # from test_framework.test_framework import BitcoinTestFramework -from test_framework.util import * -import base64 - -try: - import http.client as httplib -except ImportError: - import httplib -try: - import urllib.parse as urlparse -except ImportError: - import urlparse +from test_framework.util import str_to_b64str, assert_equal + +import os +import http.client +import urllib.parse class HTTPBasicsTest (BitcoinTestFramework): - def setup_nodes(self): - return start_nodes(4, self.options.tmpdir) + + def __init__(self): + super().__init__() + self.setup_clean_chain = False + self.num_nodes = 1 def setup_chain(self): - print("Initializing test directory "+self.options.tmpdir) - initialize_chain(self.options.tmpdir) + super().setup_chain() #Append rpcauth to bitcoin.conf before initialization rpcauth = "rpcauth=rt:93648e835a54c573682c2eb19f882535$7681e9c5b74bdd85e78166031d2058e1069b3ed7ed967c93fc63abba06f31144" rpcauth2 = "rpcauth=rt2:f8607b1a88861fac29dfccf9b52ff9f$ff36a0c23c8c62b4846112e50fa888416e94c17bfd4c42f88fd8f55ec6a3137e" @@ -34,12 +30,15 @@ def setup_chain(self): f.write(rpcauth+"\n") f.write(rpcauth2+"\n") + def setup_network(self): + self.nodes = self.setup_nodes() + def run_test(self): ################################################## # Check correctness of the rpcauth config option # ################################################## - url = urlparse.urlparse(self.nodes[0].url) + url = urllib.parse.urlparse(self.nodes[0].url) #Old authpair authpair = url.username + ':' + url.password @@ -55,7 +54,7 @@ def run_test(self): headers = {"Authorization": "Basic " + str_to_b64str(authpair)} - conn = httplib.HTTPConnection(url.hostname, url.port) + conn = http.client.HTTPConnection(url.hostname, url.port) conn.connect() conn.request('POST', '/', '{"method": "getbestblockhash"}', headers) resp = conn.getresponse() @@ -65,7 +64,7 @@ def run_test(self): #Use new authpair to confirm both work headers = {"Authorization": "Basic " + str_to_b64str(authpairnew)} - conn = httplib.HTTPConnection(url.hostname, url.port) + conn = http.client.HTTPConnection(url.hostname, url.port) conn.connect() conn.request('POST', '/', '{"method": "getbestblockhash"}', headers) resp = conn.getresponse() @@ -76,7 +75,7 @@ def run_test(self): authpairnew = "rtwrong:"+password headers = {"Authorization": "Basic " + str_to_b64str(authpairnew)} - conn = httplib.HTTPConnection(url.hostname, url.port) + conn = http.client.HTTPConnection(url.hostname, url.port) conn.connect() conn.request('POST', '/', '{"method": "getbestblockhash"}', headers) resp = conn.getresponse() @@ -87,7 +86,7 @@ def run_test(self): authpairnew = "rt:"+password+"wrong" headers = {"Authorization": "Basic " + str_to_b64str(authpairnew)} - conn = httplib.HTTPConnection(url.hostname, url.port) + conn = http.client.HTTPConnection(url.hostname, url.port) conn.connect() conn.request('POST', '/', '{"method": "getbestblockhash"}', headers) resp = conn.getresponse() @@ -98,7 +97,7 @@ def run_test(self): authpairnew = "rt2:"+password2 headers = {"Authorization": "Basic " + str_to_b64str(authpairnew)} - conn = httplib.HTTPConnection(url.hostname, url.port) + conn = http.client.HTTPConnection(url.hostname, url.port) conn.connect() conn.request('POST', '/', '{"method": "getbestblockhash"}', headers) resp = conn.getresponse() @@ -109,7 +108,7 @@ def run_test(self): authpairnew = "rt2:"+password2+"wrong" headers = {"Authorization": "Basic " + str_to_b64str(authpairnew)} - conn = httplib.HTTPConnection(url.hostname, url.port) + conn = http.client.HTTPConnection(url.hostname, url.port) conn.connect() conn.request('POST', '/', '{"method": "getbestblockhash"}', headers) resp = conn.getresponse() diff --git a/qa/rpc-tests/nodehandling.py b/qa/rpc-tests/nodehandling.py index c6c8c436e9701..e9682c4908f65 100755 --- a/qa/rpc-tests/nodehandling.py +++ b/qa/rpc-tests/nodehandling.py @@ -1,5 +1,5 @@ -#!/usr/bin/env python2 -# Copyright (c) 2014-2015 The Bitcoin Core developers +#!/usr/bin/env python3 +# Copyright (c) 2014-2016 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -10,16 +10,16 @@ from test_framework.test_framework import BitcoinTestFramework from test_framework.util import * -try: - import http.client as httplib -except ImportError: - import httplib -try: - import urllib.parse as urlparse -except ImportError: - import urlparse +import http.client +import urllib.parse class NodeHandlingTest (BitcoinTestFramework): + + def __init__(self): + super().__init__() + self.num_nodes = 4 + self.setup_clean_chain = False + def run_test(self): ########################### # setban/listbanned tests # @@ -69,7 +69,7 @@ def run_test(self): ########################### # RPC disconnectnode test # ########################### - url = urlparse.urlparse(self.nodes[1].url) + url = urllib.parse.urlparse(self.nodes[1].url) self.nodes[0].disconnectnode(url.hostname+":"+str(p2p_port(1))) time.sleep(2) #disconnecting a node needs a little bit of time for node in self.nodes[0].getpeerinfo(): diff --git a/qa/rpc-tests/p2p-acceptblock.py b/qa/rpc-tests/p2p-acceptblock.py index e429fcc5fe9d4..015ec34effabe 100755 --- a/qa/rpc-tests/p2p-acceptblock.py +++ b/qa/rpc-tests/p2p-acceptblock.py @@ -1,8 +1,7 @@ -#!/usr/bin/env python2 -# Copyright (c) 2015 The Bitcoin Core developers -# Distributed under the MIT/X11 software license, see the accompanying +#!/usr/bin/env python3 +# Copyright (c) 2015-2016 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. -# from test_framework.mininode import * from test_framework.test_framework import BitcoinTestFramework @@ -112,8 +111,10 @@ def add_options(self, parser): default=os.getenv("BITCOIND", "bitcoind"), help="bitcoind binary to test") - def setup_chain(self): - initialize_chain_clean(self.options.tmpdir, 2) + def __init__(self): + super().__init__() + self.setup_clean_chain = True + self.num_nodes = 2 def setup_network(self): # Node0 will be used to test behavior of processing unrequested blocks @@ -145,13 +146,13 @@ def run_test(self): # 1. Have both nodes mine a block (leave IBD) [ n.generate(1) for n in self.nodes ] - tips = [ int ("0x" + n.getbestblockhash() + "L", 0) for n in self.nodes ] + tips = [ int("0x" + n.getbestblockhash(), 0) for n in self.nodes ] # 2. Send one block that builds on each tip. # This should be accepted. blocks_h2 = [] # the height 2 blocks on each node's chain block_time = int(time.time()) + 1 - for i in xrange(2): + for i in range(2): blocks_h2.append(create_block(tips[i], create_coinbase(2), block_time)) blocks_h2[i].solve() block_time += 1 @@ -161,11 +162,11 @@ def run_test(self): [ x.sync_with_ping() for x in [test_node, white_node] ] assert_equal(self.nodes[0].getblockcount(), 2) assert_equal(self.nodes[1].getblockcount(), 2) - print "First height 2 block accepted by both nodes" + print("First height 2 block accepted by both nodes") # 3. Send another block that builds on the original tip. blocks_h2f = [] # Blocks at height 2 that fork off the main chain - for i in xrange(2): + for i in range(2): blocks_h2f.append(create_block(tips[i], create_coinbase(2), blocks_h2[i].nTime+1)) blocks_h2f[i].solve() test_node.send_message(msg_block(blocks_h2f[0])) @@ -180,11 +181,11 @@ def run_test(self): if x['hash'] == blocks_h2f[1].hash: assert_equal(x['status'], "valid-headers") - print "Second height 2 block accepted only from whitelisted peer" + print("Second height 2 block accepted only from whitelisted peer") # 4. Now send another block that builds on the forking chain. blocks_h3 = [] - for i in xrange(2): + for i in range(2): blocks_h3.append(create_block(blocks_h2f[i].sha256, create_coinbase(3), blocks_h2f[i].nTime+1)) blocks_h3[i].solve() test_node.send_message(msg_block(blocks_h3[0])) @@ -200,13 +201,13 @@ def run_test(self): # But this block should be accepted by node0 since it has more work. try: self.nodes[0].getblock(blocks_h3[0].hash) - print "Unrequested more-work block accepted from non-whitelisted peer" + print("Unrequested more-work block accepted from non-whitelisted peer") except: raise AssertionError("Unrequested more work block was not processed") # Node1 should have accepted and reorged. assert_equal(self.nodes[1].getblockcount(), 3) - print "Successfully reorged to length 3 chain from whitelisted peer" + print("Successfully reorged to length 3 chain from whitelisted peer") # 4b. Now mine 288 more blocks and deliver; all should be processed but # the last (height-too-high) on node0. Node1 should process the tip if @@ -214,8 +215,8 @@ def run_test(self): tips = blocks_h3 headers_message = msg_headers() all_blocks = [] # node0's blocks - for j in xrange(2): - for i in xrange(288): + for j in range(2): + for i in range(288): next_block = create_block(tips[j].sha256, create_coinbase(i + 4), tips[j].nTime+1) next_block.solve() if j==0: @@ -233,7 +234,7 @@ def run_test(self): raise AssertionError("Unrequested block too far-ahead should have been ignored") except: if x == all_blocks[287]: - print "Unrequested block too far-ahead not processed" + print("Unrequested block too far-ahead not processed") else: raise AssertionError("Unrequested block with more work should have been accepted") @@ -243,7 +244,7 @@ def run_test(self): try: white_node.sync_with_ping() self.nodes[1].getblock(tips[1].hash) - print "Unrequested block far ahead of tip accepted from whitelisted peer" + print("Unrequested block far ahead of tip accepted from whitelisted peer") except: raise AssertionError("Unrequested block from whitelisted peer not accepted") @@ -259,7 +260,7 @@ def run_test(self): # a getdata request for this block. test_node.sync_with_ping() assert_equal(self.nodes[0].getblockcount(), 2) - print "Unrequested block that would complete more-work chain was ignored" + print("Unrequested block that would complete more-work chain was ignored") # 6. Try to get node to request the missing block. # Poke the node with an inv for block at height 3 and see if that @@ -275,14 +276,14 @@ def run_test(self): # Check that the getdata includes the right block assert_equal(getdata.inv[0].hash, blocks_h2f[0].sha256) - print "Inv at tip triggered getdata for unprocessed block" + print("Inv at tip triggered getdata for unprocessed block") # 7. Send the missing block for the third time (now it is requested) test_node.send_message(msg_block(blocks_h2f[0])) test_node.sync_with_ping() assert_equal(self.nodes[0].getblockcount(), 290) - print "Successfully reorged to longer chain from non-whitelisted peer" + print("Successfully reorged to longer chain from non-whitelisted peer") [ c.disconnect_node() for c in connections ] diff --git a/qa/rpc-tests/p2p-feefilter.py b/qa/rpc-tests/p2p-feefilter.py index 281b6ca37a0ce..cd0501a3140c3 100755 --- a/qa/rpc-tests/p2p-feefilter.py +++ b/qa/rpc-tests/p2p-feefilter.py @@ -1,6 +1,6 @@ -#!/usr/bin/env python2 +#!/usr/bin/env python3 # Copyright (c) 2016 The Bitcoin Core developers -# Distributed under the MIT/X11 software license, see the accompanying +# Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. # @@ -18,7 +18,7 @@ def hashToHex(hash): # Wait up to 60 secs to see if the testnode has received all the expected invs def allInvsMatch(invsExpected, testnode): - for x in xrange(60): + for x in range(60): with mininode_lock: if (sorted(invsExpected) == sorted(testnode.txinvs)): return True; @@ -46,6 +46,12 @@ def send_filter(self, feerate): self.sync_with_ping() class FeeFilterTest(BitcoinTestFramework): + + def __init__(self): + super().__init__() + self.num_nodes = 2 + self.setup_clean_chain = False + def setup_network(self): # Node1 will be used to generate txs which should be relayed from Node0 # to our test node @@ -69,7 +75,7 @@ def run_test(self): # Test that invs are received for all txs at feerate of 20 sat/byte node1.settxfee(Decimal("0.00020000")) - txids = [node1.sendtoaddress(node1.getnewaddress(), 1) for x in xrange(3)] + txids = [node1.sendtoaddress(node1.getnewaddress(), 1) for x in range(3)] assert(allInvsMatch(txids, test_node)) test_node.clear_invs() @@ -77,13 +83,13 @@ def run_test(self): test_node.send_filter(15000) # Test that txs are still being received (paying 20 sat/byte) - txids = [node1.sendtoaddress(node1.getnewaddress(), 1) for x in xrange(3)] + txids = [node1.sendtoaddress(node1.getnewaddress(), 1) for x in range(3)] assert(allInvsMatch(txids, test_node)) test_node.clear_invs() # Change tx fee rate to 10 sat/byte and test they are no longer received node1.settxfee(Decimal("0.00010000")) - [node1.sendtoaddress(node1.getnewaddress(), 1) for x in xrange(3)] + [node1.sendtoaddress(node1.getnewaddress(), 1) for x in range(3)] sync_mempools(self.nodes) # must be sure node 0 has received all txs time.sleep(10) # wait 10 secs to be sure its doesn't relay any assert(allInvsMatch([], test_node)) @@ -91,7 +97,7 @@ def run_test(self): # Remove fee filter and check that txs are received again test_node.send_filter(0) - txids = [node1.sendtoaddress(node1.getnewaddress(), 1) for x in xrange(3)] + txids = [node1.sendtoaddress(node1.getnewaddress(), 1) for x in range(3)] assert(allInvsMatch(txids, test_node)) test_node.clear_invs() diff --git a/qa/rpc-tests/p2p-fullblocktest.py b/qa/rpc-tests/p2p-fullblocktest.py index ae82d9dca9266..17fd40ef1dffd 100755 --- a/qa/rpc-tests/p2p-fullblocktest.py +++ b/qa/rpc-tests/p2p-fullblocktest.py @@ -1,8 +1,7 @@ -#!/usr/bin/env python2 -# Copyright (c) 2015 The Bitcoin Core developers -# Distributed under the MIT/X11 software license, see the accompanying +#!/usr/bin/env python3 +# Copyright (c) 2015-2016 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. -# from test_framework.test_framework import ComparisonTestFramework from test_framework.util import * @@ -10,7 +9,8 @@ from test_framework.blocktools import * import time from test_framework.key import CECKey -from test_framework.script import CScript, SignatureHash, SIGHASH_ALL, OP_TRUE, OP_FALSE +from test_framework.script import * +import struct class PreviousSpendableOutput(object): def __init__(self, tx = CTransaction(), n = -1): @@ -25,76 +25,107 @@ def __init__(self, tx = CTransaction(), n = -1): each test. ''' +def hash160(s): + return hashlib.new('ripemd160', sha256(s)).digest() + +# Use this class for tests that require behavior other than normal "mininode" behavior. +# For now, it is used to serialize a bloated varint (b64). +class CBrokenBlock(CBlock): + def __init__(self, header=None): + super(CBrokenBlock, self).__init__(header) + + def initialize(self, base_block): + self.vtx = copy.deepcopy(base_block.vtx) + self.hashMerkleRoot = self.calc_merkle_root() + + def serialize(self): + r = b"" + r += super(CBlock, self).serialize() + r += struct.pack(" b1 (0) -> b2 (1) - out0 = get_spendable_output() - block(1, spend=out0) + block(1, spend=out[0]) save_spendable_output() yield accepted() - out1 = get_spendable_output() - b2 = block(2, spend=out1) + block(2, spend=out[1]) yield accepted() - + save_spendable_output() # so fork like this: - # + # # genesis -> b1 (0) -> b2 (1) # \-> b3 (1) - # + # # Nothing should happen at this point. We saw b2 first so it takes priority. tip(1) - b3 = block(3, spend=out1) - txout_b3 = PreviousSpendableOutput(b3.vtx[1], 1) + b3 = block(3, spend=out[1]) + txout_b3 = PreviousSpendableOutput(b3.vtx[1], 0) yield rejected() # Now we add another block to make the alternative chain longer. - # + # # genesis -> b1 (0) -> b2 (1) # \-> b3 (1) -> b4 (2) - out2 = get_spendable_output() - block(4, spend=out2) + block(4, spend=out[2]) yield accepted() @@ -197,46 +236,41 @@ def update_block(block_number, new_transactions): # genesis -> b1 (0) -> b2 (1) -> b5 (2) -> b6 (3) # \-> b3 (1) -> b4 (2) tip(2) - block(5, spend=out2) + block(5, spend=out[2]) save_spendable_output() yield rejected() - out3 = get_spendable_output() - block(6, spend=out3) + block(6, spend=out[3]) yield accepted() - # Try to create a fork that double-spends # genesis -> b1 (0) -> b2 (1) -> b5 (2) -> b6 (3) # \-> b7 (2) -> b8 (4) # \-> b3 (1) -> b4 (2) tip(5) - block(7, spend=out2) + block(7, spend=out[2]) yield rejected() - out4 = get_spendable_output() - block(8, spend=out4) + block(8, spend=out[4]) yield rejected() - # Try to create a block that has too much fee # genesis -> b1 (0) -> b2 (1) -> b5 (2) -> b6 (3) # \-> b9 (4) # \-> b3 (1) -> b4 (2) tip(6) - block(9, spend=out4, additional_coinbase_value=1) + block(9, spend=out[4], additional_coinbase_value=1) yield rejected(RejectResult(16, b'bad-cb-amount')) - # Create a fork that ends in a block with too much fee (the one that causes the reorg) # genesis -> b1 (0) -> b2 (1) -> b5 (2) -> b6 (3) # \-> b10 (3) -> b11 (4) # \-> b3 (1) -> b4 (2) tip(5) - block(10, spend=out3) + block(10, spend=out[3]) yield rejected() - block(11, spend=out4, additional_coinbase_value=1) + block(11, spend=out[4], additional_coinbase_value=1) yield rejected(RejectResult(16, b'bad-cb-amount')) @@ -246,19 +280,17 @@ def update_block(block_number, new_transactions): # (b12 added last) # \-> b3 (1) -> b4 (2) tip(5) - b12 = block(12, spend=out3) + b12 = block(12, spend=out[3]) save_spendable_output() - #yield TestInstance([[b12, False]]) - b13 = block(13, spend=out4) + b13 = block(13, spend=out[4]) # Deliver the block header for b12, and the block b13. # b13 should be accepted but the tip won't advance until b12 is delivered. yield TestInstance([[CBlockHeader(b12), None], [b13, False]]) save_spendable_output() - out5 = get_spendable_output() # b14 is invalid, but the node won't know that until it tries to connect # Tip still can't advance because b12 is missing - block(14, spend=out5, additional_coinbase_value=1) + block(14, spend=out[5], additional_coinbase_value=1) yield rejected() yield TestInstance([[b12, True, b13.sha256]]) # New tip should be b13. @@ -267,18 +299,18 @@ def update_block(block_number, new_transactions): # genesis -> b1 (0) -> b2 (1) -> b5 (2) -> b6 (3) # \-> b12 (3) -> b13 (4) -> b15 (5) -> b16 (6) # \-> b3 (1) -> b4 (2) - + # Test that a block with a lot of checksigs is okay - lots_of_checksigs = CScript([OP_CHECKSIG] * (1000000 // 50 - 1)) + lots_of_checksigs = CScript([OP_CHECKSIG] * (MAX_BLOCK_SIGOPS - 1)) tip(13) - block(15, spend=out5, script=lots_of_checksigs) + block(15, spend=out[5], script=lots_of_checksigs) yield accepted() + save_spendable_output() # Test that a block with too many checksigs is rejected - out6 = get_spendable_output() - too_many_checksigs = CScript([OP_CHECKSIG] * (1000000 // 50)) - block(16, spend=out6, script=too_many_checksigs) + too_many_checksigs = CScript([OP_CHECKSIG] * (MAX_BLOCK_SIGOPS)) + block(16, spend=out[6], script=too_many_checksigs) yield rejected(RejectResult(16, b'bad-blk-sigops')) @@ -299,7 +331,7 @@ def update_block(block_number, new_transactions): block(18, spend=txout_b3) yield rejected() - block(19, spend=out6) + block(19, spend=out[6]) yield rejected() # Attempt to spend a coinbase at depth too low @@ -307,8 +339,7 @@ def update_block(block_number, new_transactions): # \-> b12 (3) -> b13 (4) -> b15 (5) -> b20 (7) # \-> b3 (1) -> b4 (2) tip(15) - out7 = get_spendable_output() - block(20, spend=out7) + block(20, spend=out[7]) yield rejected(RejectResult(16, b'bad-txns-premature-spend-of-coinbase')) # Attempt to spend a coinbase at depth too low (on a fork this time) @@ -317,10 +348,10 @@ def update_block(block_number, new_transactions): # \-> b21 (6) -> b22 (5) # \-> b3 (1) -> b4 (2) tip(13) - block(21, spend=out6) + block(21, spend=out[6]) yield rejected() - block(22, spend=out5) + block(22, spend=out[5]) yield rejected() # Create a block on either side of MAX_BLOCK_SIZE and make sure its accepted/rejected @@ -329,21 +360,21 @@ def update_block(block_number, new_transactions): # \-> b24 (6) -> b25 (7) # \-> b3 (1) -> b4 (2) tip(15) - b23 = block(23, spend=out6) - old_hash = b23.sha256 + b23 = block(23, spend=out[6]) tx = CTransaction() script_length = MAX_BLOCK_SIZE - len(b23.serialize()) - 69 script_output = CScript([b'\x00' * script_length]) tx.vout.append(CTxOut(0, script_output)) - tx.vin.append(CTxIn(COutPoint(b23.vtx[1].sha256, 1))) + tx.vin.append(CTxIn(COutPoint(b23.vtx[1].sha256, 0))) b23 = update_block(23, [tx]) # Make sure the math above worked out to produce a max-sized block assert_equal(len(b23.serialize()), MAX_BLOCK_SIZE) yield accepted() + save_spendable_output() # Make the next block one byte bigger and check that it fails tip(15) - b24 = block(24, spend=out6) + b24 = block(24, spend=out[6]) script_length = MAX_BLOCK_SIZE - len(b24.serialize()) - 69 script_output = CScript([b'\x00' * (script_length+1)]) tx.vout = [CTxOut(0, script_output)] @@ -351,7 +382,7 @@ def update_block(block_number, new_transactions): assert_equal(len(b24.serialize()), MAX_BLOCK_SIZE+1) yield rejected(RejectResult(16, b'bad-blk-length')) - b25 = block(25, spend=out7) + block(25, spend=out[7]) yield rejected() # Create blocks with a coinbase input script size out of range @@ -360,7 +391,7 @@ def update_block(block_number, new_transactions): # \-> ... (6) -> ... (7) # \-> b3 (1) -> b4 (2) tip(15) - b26 = block(26, spend=out6) + b26 = block(26, spend=out[6]) b26.vtx[0].vin[0].scriptSig = b'\x00' b26.vtx[0].rehash() # update_block causes the merkle root to get updated, even with no new @@ -369,23 +400,20 @@ def update_block(block_number, new_transactions): yield rejected(RejectResult(16, b'bad-cb-length')) # Extend the b26 chain to make sure bitcoind isn't accepting b26 - b27 = block(27, spend=out7) - yield rejected() + b27 = block(27, spend=out[7]) + yield rejected(RejectResult(16, b'bad-prevblk')) # Now try a too-large-coinbase script tip(15) - b28 = block(28, spend=out6) + b28 = block(28, spend=out[6]) b28.vtx[0].vin[0].scriptSig = b'\x00' * 101 b28.vtx[0].rehash() b28 = update_block(28, []) yield rejected(RejectResult(16, b'bad-cb-length')) - # Extend the b28 chain to make sure bitcoind isn't accepted b28 - b29 = block(29, spend=out7) - # TODO: Should get a reject message back with "bad-prevblk", except - # there's a bug that prevents this from being detected. Just note - # failure for now, and add the reject result later. - yield rejected() + # Extend the b28 chain to make sure bitcoind isn't accepting b28 + b29 = block(29, spend=out[7]) + yield rejected(RejectResult(16, b'bad-prevblk')) # b30 has a max-sized coinbase scriptSig. tip(23) @@ -394,6 +422,871 @@ def update_block(block_number, new_transactions): b30.vtx[0].rehash() b30 = update_block(30, []) yield accepted() + save_spendable_output() + + # b31 - b35 - check sigops of OP_CHECKMULTISIG / OP_CHECKMULTISIGVERIFY / OP_CHECKSIGVERIFY + # + # genesis -> ... -> b30 (7) -> b31 (8) -> b33 (9) -> b35 (10) + # \-> b36 (11) + # \-> b34 (10) + # \-> b32 (9) + # + + # MULTISIG: each op code counts as 20 sigops. To create the edge case, pack another 19 sigops at the end. + lots_of_multisigs = CScript([OP_CHECKMULTISIG] * ((MAX_BLOCK_SIGOPS-1) // 20) + [OP_CHECKSIG] * 19) + b31 = block(31, spend=out[8], script=lots_of_multisigs) + assert_equal(get_legacy_sigopcount_block(b31), MAX_BLOCK_SIGOPS) + yield accepted() + save_spendable_output() + + # this goes over the limit because the coinbase has one sigop + too_many_multisigs = CScript([OP_CHECKMULTISIG] * (MAX_BLOCK_SIGOPS // 20)) + b32 = block(32, spend=out[9], script=too_many_multisigs) + assert_equal(get_legacy_sigopcount_block(b32), MAX_BLOCK_SIGOPS + 1) + yield rejected(RejectResult(16, b'bad-blk-sigops')) + + + # CHECKMULTISIGVERIFY + tip(31) + lots_of_multisigs = CScript([OP_CHECKMULTISIGVERIFY] * ((MAX_BLOCK_SIGOPS-1) // 20) + [OP_CHECKSIG] * 19) + block(33, spend=out[9], script=lots_of_multisigs) + yield accepted() + save_spendable_output() + + too_many_multisigs = CScript([OP_CHECKMULTISIGVERIFY] * (MAX_BLOCK_SIGOPS // 20)) + block(34, spend=out[10], script=too_many_multisigs) + yield rejected(RejectResult(16, b'bad-blk-sigops')) + + + # CHECKSIGVERIFY + tip(33) + lots_of_checksigs = CScript([OP_CHECKSIGVERIFY] * (MAX_BLOCK_SIGOPS - 1)) + b35 = block(35, spend=out[10], script=lots_of_checksigs) + yield accepted() + save_spendable_output() + + too_many_checksigs = CScript([OP_CHECKSIGVERIFY] * (MAX_BLOCK_SIGOPS)) + block(36, spend=out[11], script=too_many_checksigs) + yield rejected(RejectResult(16, b'bad-blk-sigops')) + + + # Check spending of a transaction in a block which failed to connect + # + # b6 (3) + # b12 (3) -> b13 (4) -> b15 (5) -> b23 (6) -> b30 (7) -> b31 (8) -> b33 (9) -> b35 (10) + # \-> b37 (11) + # \-> b38 (11/37) + # + + # save 37's spendable output, but then double-spend out11 to invalidate the block + tip(35) + b37 = block(37, spend=out[11]) + txout_b37 = PreviousSpendableOutput(b37.vtx[1], 0) + tx = create_and_sign_tx(out[11].tx, out[11].n, 0) + b37 = update_block(37, [tx]) + yield rejected(RejectResult(16, b'bad-txns-inputs-missingorspent')) + + # attempt to spend b37's first non-coinbase tx, at which point b37 was still considered valid + tip(35) + block(38, spend=txout_b37) + yield rejected(RejectResult(16, b'bad-txns-inputs-missingorspent')) + + # Check P2SH SigOp counting + # + # + # 13 (4) -> b15 (5) -> b23 (6) -> b30 (7) -> b31 (8) -> b33 (9) -> b35 (10) -> b39 (11) -> b41 (12) + # \-> b40 (12) + # + # b39 - create some P2SH outputs that will require 6 sigops to spend: + # + # redeem_script = COINBASE_PUBKEY, (OP_2DUP+OP_CHECKSIGVERIFY) * 5, OP_CHECKSIG + # p2sh_script = OP_HASH160, ripemd160(sha256(script)), OP_EQUAL + # + tip(35) + b39 = block(39) + b39_outputs = 0 + b39_sigops_per_output = 6 + + # Build the redeem script, hash it, use hash to create the p2sh script + redeem_script = CScript([self.coinbase_pubkey] + [OP_2DUP, OP_CHECKSIGVERIFY]*5 + [OP_CHECKSIG]) + redeem_script_hash = hash160(redeem_script) + p2sh_script = CScript([OP_HASH160, redeem_script_hash, OP_EQUAL]) + + # Create a transaction that spends one satoshi to the p2sh_script, the rest to OP_TRUE + # This must be signed because it is spending a coinbase + spend = out[11] + tx = create_tx(spend.tx, spend.n, 1, p2sh_script) + tx.vout.append(CTxOut(spend.tx.vout[spend.n].nValue - 1, CScript([OP_TRUE]))) + self.sign_tx(tx, spend.tx, spend.n) + tx.rehash() + b39 = update_block(39, [tx]) + b39_outputs += 1 + + # Until block is full, add tx's with 1 satoshi to p2sh_script, the rest to OP_TRUE + tx_new = None + tx_last = tx + total_size=len(b39.serialize()) + while(total_size < MAX_BLOCK_SIZE): + tx_new = create_tx(tx_last, 1, 1, p2sh_script) + tx_new.vout.append(CTxOut(tx_last.vout[1].nValue - 1, CScript([OP_TRUE]))) + tx_new.rehash() + total_size += len(tx_new.serialize()) + if total_size >= MAX_BLOCK_SIZE: + break + b39.vtx.append(tx_new) # add tx to block + tx_last = tx_new + b39_outputs += 1 + + b39 = update_block(39, []) + yield accepted() + save_spendable_output() + + + # Test sigops in P2SH redeem scripts + # + # b40 creates 3333 tx's spending the 6-sigop P2SH outputs from b39 for a total of 19998 sigops. + # The first tx has one sigop and then at the end we add 2 more to put us just over the max. + # + # b41 does the same, less one, so it has the maximum sigops permitted. + # + tip(39) + b40 = block(40, spend=out[12]) + sigops = get_legacy_sigopcount_block(b40) + numTxes = (MAX_BLOCK_SIGOPS - sigops) // b39_sigops_per_output + assert_equal(numTxes <= b39_outputs, True) + + lastOutpoint = COutPoint(b40.vtx[1].sha256, 0) + new_txs = [] + for i in range(1, numTxes+1): + tx = CTransaction() + tx.vout.append(CTxOut(1, CScript([OP_TRUE]))) + tx.vin.append(CTxIn(lastOutpoint, b'')) + # second input is corresponding P2SH output from b39 + tx.vin.append(CTxIn(COutPoint(b39.vtx[i].sha256, 0), b'')) + # Note: must pass the redeem_script (not p2sh_script) to the signature hash function + (sighash, err) = SignatureHash(redeem_script, tx, 1, SIGHASH_ALL) + sig = self.coinbase_key.sign(sighash) + bytes(bytearray([SIGHASH_ALL])) + scriptSig = CScript([sig, redeem_script]) + + tx.vin[1].scriptSig = scriptSig + tx.rehash() + new_txs.append(tx) + lastOutpoint = COutPoint(tx.sha256, 0) + + b40_sigops_to_fill = MAX_BLOCK_SIGOPS - (numTxes * b39_sigops_per_output + sigops) + 1 + tx = CTransaction() + tx.vin.append(CTxIn(lastOutpoint, b'')) + tx.vout.append(CTxOut(1, CScript([OP_CHECKSIG] * b40_sigops_to_fill))) + tx.rehash() + new_txs.append(tx) + update_block(40, new_txs) + yield rejected(RejectResult(16, b'bad-blk-sigops')) + + # same as b40, but one less sigop + tip(39) + b41 = block(41, spend=None) + update_block(41, b40.vtx[1:-1]) + b41_sigops_to_fill = b40_sigops_to_fill - 1 + tx = CTransaction() + tx.vin.append(CTxIn(lastOutpoint, b'')) + tx.vout.append(CTxOut(1, CScript([OP_CHECKSIG] * b41_sigops_to_fill))) + tx.rehash() + update_block(41, [tx]) + yield accepted() + + # Fork off of b39 to create a constant base again + # + # b23 (6) -> b30 (7) -> b31 (8) -> b33 (9) -> b35 (10) -> b39 (11) -> b42 (12) -> b43 (13) + # \-> b41 (12) + # + tip(39) + block(42, spend=out[12]) + yield rejected() + save_spendable_output() + + block(43, spend=out[13]) + yield accepted() + save_spendable_output() + + + # Test a number of really invalid scenarios + # + # -> b31 (8) -> b33 (9) -> b35 (10) -> b39 (11) -> b42 (12) -> b43 (13) -> b44 (14) + # \-> ??? (15) + + # The next few blocks are going to be created "by hand" since they'll do funky things, such as having + # the first transaction be non-coinbase, etc. The purpose of b44 is to make sure this works. + height = self.block_heights[self.tip.sha256] + 1 + coinbase = create_coinbase(height, self.coinbase_pubkey) + b44 = CBlock() + b44.nTime = self.tip.nTime + 1 + b44.hashPrevBlock = self.tip.sha256 + b44.nBits = 0x207fffff + b44.vtx.append(coinbase) + b44.hashMerkleRoot = b44.calc_merkle_root() + b44.solve() + self.tip = b44 + self.block_heights[b44.sha256] = height + self.blocks[44] = b44 + yield accepted() + + # A block with a non-coinbase as the first tx + non_coinbase = create_tx(out[15].tx, out[15].n, 1) + b45 = CBlock() + b45.nTime = self.tip.nTime + 1 + b45.hashPrevBlock = self.tip.sha256 + b45.nBits = 0x207fffff + b45.vtx.append(non_coinbase) + b45.hashMerkleRoot = b45.calc_merkle_root() + b45.calc_sha256() + b45.solve() + self.block_heights[b45.sha256] = self.block_heights[self.tip.sha256]+1 + self.tip = b45 + self.blocks[45] = b45 + yield rejected(RejectResult(16, b'bad-cb-missing')) + + # A block with no txns + tip(44) + b46 = CBlock() + b46.nTime = b44.nTime+1 + b46.hashPrevBlock = b44.sha256 + b46.nBits = 0x207fffff + b46.vtx = [] + b46.hashMerkleRoot = 0 + b46.solve() + self.block_heights[b46.sha256] = self.block_heights[b44.sha256]+1 + self.tip = b46 + assert 46 not in self.blocks + self.blocks[46] = b46 + s = ser_uint256(b46.hashMerkleRoot) + yield rejected(RejectResult(16, b'bad-blk-length')) + + # A block with invalid work + tip(44) + b47 = block(47, solve=False) + target = uint256_from_compact(b47.nBits) + while b47.sha256 < target: #changed > to < + b47.nNonce += 1 + b47.rehash() + yield rejected(RejectResult(16, b'high-hash')) + + # A block with timestamp > 2 hrs in the future + tip(44) + b48 = block(48, solve=False) + b48.nTime = int(time.time()) + 60 * 60 * 3 + b48.solve() + yield rejected(RejectResult(16, b'time-too-new')) + + # A block with an invalid merkle hash + tip(44) + b49 = block(49) + b49.hashMerkleRoot += 1 + b49.solve() + yield rejected(RejectResult(16, b'bad-txnmrklroot')) + + # A block with an incorrect POW limit + tip(44) + b50 = block(50) + b50.nBits = b50.nBits - 1 + b50.solve() + yield rejected(RejectResult(16, b'bad-diffbits')) + + # A block with two coinbase txns + tip(44) + b51 = block(51) + cb2 = create_coinbase(51, self.coinbase_pubkey) + b51 = update_block(51, [cb2]) + yield rejected(RejectResult(16, b'bad-cb-multiple')) + + # A block w/ duplicate txns + # Note: txns have to be in the right position in the merkle tree to trigger this error + tip(44) + b52 = block(52, spend=out[15]) + tx = create_tx(b52.vtx[1], 0, 1) + b52 = update_block(52, [tx, tx]) + yield rejected(RejectResult(16, b'bad-txns-duplicate')) + + # Test block timestamps + # -> b31 (8) -> b33 (9) -> b35 (10) -> b39 (11) -> b42 (12) -> b43 (13) -> b53 (14) -> b55 (15) + # \-> b54 (15) + # + tip(43) + block(53, spend=out[14]) + yield rejected() # rejected since b44 is at same height + save_spendable_output() + + # invalid timestamp (b35 is 5 blocks back, so its time is MedianTimePast) + b54 = block(54, spend=out[15]) + b54.nTime = b35.nTime - 1 + b54.solve() + yield rejected(RejectResult(16, b'time-too-old')) + + # valid timestamp + tip(53) + b55 = block(55, spend=out[15]) + b55.nTime = b35.nTime + update_block(55, []) + yield accepted() + save_spendable_output() + + + # Test CVE-2012-2459 + # + # -> b42 (12) -> b43 (13) -> b53 (14) -> b55 (15) -> b57p2 (16) + # \-> b57 (16) + # \-> b56p2 (16) + # \-> b56 (16) + # + # Merkle tree malleability (CVE-2012-2459): repeating sequences of transactions in a block without + # affecting the merkle root of a block, while still invalidating it. + # See: src/consensus/merkle.h + # + # b57 has three txns: coinbase, tx, tx1. The merkle root computation will duplicate tx. + # Result: OK + # + # b56 copies b57 but duplicates tx1 and does not recalculate the block hash. So it has a valid merkle + # root but duplicate transactions. + # Result: Fails + # + # b57p2 has six transactions in its merkle tree: + # - coinbase, tx, tx1, tx2, tx3, tx4 + # Merkle root calculation will duplicate as necessary. + # Result: OK. + # + # b56p2 copies b57p2 but adds both tx3 and tx4. The purpose of the test is to make sure the code catches + # duplicate txns that are not next to one another with the "bad-txns-duplicate" error (which indicates + # that the error was caught early, avoiding a DOS vulnerability.) + + # b57 - a good block with 2 txs, don't submit until end + tip(55) + b57 = block(57) + tx = create_and_sign_tx(out[16].tx, out[16].n, 1) + tx1 = create_tx(tx, 0, 1) + b57 = update_block(57, [tx, tx1]) + + # b56 - copy b57, add a duplicate tx + tip(55) + b56 = copy.deepcopy(b57) + self.blocks[56] = b56 + assert_equal(len(b56.vtx),3) + b56 = update_block(56, [tx1]) + assert_equal(b56.hash, b57.hash) + yield rejected(RejectResult(16, b'bad-txns-duplicate')) + + # b57p2 - a good block with 6 tx'es, don't submit until end + tip(55) + b57p2 = block("57p2") + tx = create_and_sign_tx(out[16].tx, out[16].n, 1) + tx1 = create_tx(tx, 0, 1) + tx2 = create_tx(tx1, 0, 1) + tx3 = create_tx(tx2, 0, 1) + tx4 = create_tx(tx3, 0, 1) + b57p2 = update_block("57p2", [tx, tx1, tx2, tx3, tx4]) + + # b56p2 - copy b57p2, duplicate two non-consecutive tx's + tip(55) + b56p2 = copy.deepcopy(b57p2) + self.blocks["b56p2"] = b56p2 + assert_equal(b56p2.hash, b57p2.hash) + assert_equal(len(b56p2.vtx),6) + b56p2 = update_block("b56p2", [tx3, tx4]) + yield rejected(RejectResult(16, b'bad-txns-duplicate')) + + tip("57p2") + yield accepted() + + tip(57) + yield rejected() #rejected because 57p2 seen first + save_spendable_output() + + # Test a few invalid tx types + # + # -> b35 (10) -> b39 (11) -> b42 (12) -> b43 (13) -> b53 (14) -> b55 (15) -> b57 (16) -> b60 (17) + # \-> ??? (17) + # + + # tx with prevout.n out of range + tip(57) + b58 = block(58, spend=out[17]) + tx = CTransaction() + assert(len(out[17].tx.vout) < 42) + tx.vin.append(CTxIn(COutPoint(out[17].tx.sha256, 42), CScript([OP_TRUE]), 0xffffffff)) + tx.vout.append(CTxOut(0, b"")) + tx.calc_sha256() + b58 = update_block(58, [tx]) + yield rejected(RejectResult(16, b'bad-txns-inputs-missingorspent')) + + # tx with output value > input value out of range + tip(57) + b59 = block(59) + tx = create_and_sign_tx(out[17].tx, out[17].n, 51*COIN) + b59 = update_block(59, [tx]) + yield rejected(RejectResult(16, b'bad-txns-in-belowout')) + + # reset to good chain + tip(57) + b60 = block(60, spend=out[17]) + yield accepted() + save_spendable_output() + + # Test BIP30 + # + # -> b39 (11) -> b42 (12) -> b43 (13) -> b53 (14) -> b55 (15) -> b57 (16) -> b60 (17) + # \-> b61 (18) + # + # Blocks are not allowed to contain a transaction whose id matches that of an earlier, + # not-fully-spent transaction in the same chain. To test, make identical coinbases; + # the second one should be rejected. + # + tip(60) + b61 = block(61, spend=out[18]) + b61.vtx[0].vin[0].scriptSig = b60.vtx[0].vin[0].scriptSig #equalize the coinbases + b61.vtx[0].rehash() + b61 = update_block(61, []) + assert_equal(b60.vtx[0].serialize(), b61.vtx[0].serialize()) + yield rejected(RejectResult(16, b'bad-txns-BIP30')) + + + # Test tx.isFinal is properly rejected (not an exhaustive tx.isFinal test, that should be in data-driven transaction tests) + # + # -> b39 (11) -> b42 (12) -> b43 (13) -> b53 (14) -> b55 (15) -> b57 (16) -> b60 (17) + # \-> b62 (18) + # + tip(60) + b62 = block(62) + tx = CTransaction() + tx.nLockTime = 0xffffffff #this locktime is non-final + assert(out[18].n < len(out[18].tx.vout)) + tx.vin.append(CTxIn(COutPoint(out[18].tx.sha256, out[18].n))) # don't set nSequence + tx.vout.append(CTxOut(0, CScript([OP_TRUE]))) + assert(tx.vin[0].nSequence < 0xffffffff) + tx.calc_sha256() + b62 = update_block(62, [tx]) + yield rejected(RejectResult(16, b'bad-txns-nonfinal')) + + + # Test a non-final coinbase is also rejected + # + # -> b39 (11) -> b42 (12) -> b43 (13) -> b53 (14) -> b55 (15) -> b57 (16) -> b60 (17) + # \-> b63 (-) + # + tip(60) + b63 = block(63) + b63.vtx[0].nLockTime = 0xffffffff + b63.vtx[0].vin[0].nSequence = 0xDEADBEEF + b63.vtx[0].rehash() + b63 = update_block(63, []) + yield rejected(RejectResult(16, b'bad-txns-nonfinal')) + + + # This checks that a block with a bloated VARINT between the block_header and the array of tx such that + # the block is > MAX_BLOCK_SIZE with the bloated varint, but <= MAX_BLOCK_SIZE without the bloated varint, + # does not cause a subsequent, identical block with canonical encoding to be rejected. The test does not + # care whether the bloated block is accepted or rejected; it only cares that the second block is accepted. + # + # What matters is that the receiving node should not reject the bloated block, and then reject the canonical + # block on the basis that it's the same as an already-rejected block (which would be a consensus failure.) + # + # -> b39 (11) -> b42 (12) -> b43 (13) -> b53 (14) -> b55 (15) -> b57 (16) -> b60 (17) -> b64 (18) + # \ + # b64a (18) + # b64a is a bloated block (non-canonical varint) + # b64 is a good block (same as b64 but w/ canonical varint) + # + tip(60) + regular_block = block("64a", spend=out[18]) + + # make it a "broken_block," with non-canonical serialization + b64a = CBrokenBlock(regular_block) + b64a.initialize(regular_block) + self.blocks["64a"] = b64a + self.tip = b64a + tx = CTransaction() + + # use canonical serialization to calculate size + script_length = MAX_BLOCK_SIZE - len(b64a.normal_serialize()) - 69 + script_output = CScript([b'\x00' * script_length]) + tx.vout.append(CTxOut(0, script_output)) + tx.vin.append(CTxIn(COutPoint(b64a.vtx[1].sha256, 0))) + b64a = update_block("64a", [tx]) + assert_equal(len(b64a.serialize()), MAX_BLOCK_SIZE + 8) + yield TestInstance([[self.tip, None]]) + + # comptool workaround: to make sure b64 is delivered, manually erase b64a from blockstore + self.test.block_store.erase(b64a.sha256) + + tip(60) + b64 = CBlock(b64a) + b64.vtx = copy.deepcopy(b64a.vtx) + assert_equal(b64.hash, b64a.hash) + assert_equal(len(b64.serialize()), MAX_BLOCK_SIZE) + self.blocks[64] = b64 + update_block(64, []) + yield accepted() + save_spendable_output() + + # Spend an output created in the block itself + # + # -> b42 (12) -> b43 (13) -> b53 (14) -> b55 (15) -> b57 (16) -> b60 (17) -> b64 (18) -> b65 (19) + # + tip(64) + b65 = block(65) + tx1 = create_and_sign_tx(out[19].tx, out[19].n, out[19].tx.vout[0].nValue) + tx2 = create_and_sign_tx(tx1, 0, 0) + update_block(65, [tx1, tx2]) + yield accepted() + save_spendable_output() + + # Attempt to spend an output created later in the same block + # + # -> b43 (13) -> b53 (14) -> b55 (15) -> b57 (16) -> b60 (17) -> b64 (18) -> b65 (19) + # \-> b66 (20) + tip(65) + b66 = block(66) + tx1 = create_and_sign_tx(out[20].tx, out[20].n, out[20].tx.vout[0].nValue) + tx2 = create_and_sign_tx(tx1, 0, 1) + update_block(66, [tx2, tx1]) + yield rejected(RejectResult(16, b'bad-txns-inputs-missingorspent')) + + # Attempt to double-spend a transaction created in a block + # + # -> b43 (13) -> b53 (14) -> b55 (15) -> b57 (16) -> b60 (17) -> b64 (18) -> b65 (19) + # \-> b67 (20) + # + # + tip(65) + b67 = block(67) + tx1 = create_and_sign_tx(out[20].tx, out[20].n, out[20].tx.vout[0].nValue) + tx2 = create_and_sign_tx(tx1, 0, 1) + tx3 = create_and_sign_tx(tx1, 0, 2) + update_block(67, [tx1, tx2, tx3]) + yield rejected(RejectResult(16, b'bad-txns-inputs-missingorspent')) + + # More tests of block subsidy + # + # -> b43 (13) -> b53 (14) -> b55 (15) -> b57 (16) -> b60 (17) -> b64 (18) -> b65 (19) -> b69 (20) + # \-> b68 (20) + # + # b68 - coinbase with an extra 10 satoshis, + # creates a tx that has 9 satoshis from out[20] go to fees + # this fails because the coinbase is trying to claim 1 satoshi too much in fees + # + # b69 - coinbase with extra 10 satoshis, and a tx that gives a 10 satoshi fee + # this succeeds + # + tip(65) + b68 = block(68, additional_coinbase_value=10) + tx = create_and_sign_tx(out[20].tx, out[20].n, out[20].tx.vout[0].nValue-9) + update_block(68, [tx]) + yield rejected(RejectResult(16, b'bad-cb-amount')) + + tip(65) + b69 = block(69, additional_coinbase_value=10) + tx = create_and_sign_tx(out[20].tx, out[20].n, out[20].tx.vout[0].nValue-10) + update_block(69, [tx]) + yield accepted() + save_spendable_output() + + # Test spending the outpoint of a non-existent transaction + # + # -> b53 (14) -> b55 (15) -> b57 (16) -> b60 (17) -> b64 (18) -> b65 (19) -> b69 (20) + # \-> b70 (21) + # + tip(69) + block(70, spend=out[21]) + bogus_tx = CTransaction() + bogus_tx.sha256 = uint256_from_str(b"23c70ed7c0506e9178fc1a987f40a33946d4ad4c962b5ae3a52546da53af0c5c") + tx = CTransaction() + tx.vin.append(CTxIn(COutPoint(bogus_tx.sha256, 0), b"", 0xffffffff)) + tx.vout.append(CTxOut(1, b"")) + update_block(70, [tx]) + yield rejected(RejectResult(16, b'bad-txns-inputs-missingorspent')) + + + # Test accepting an invalid block which has the same hash as a valid one (via merkle tree tricks) + # + # -> b53 (14) -> b55 (15) -> b57 (16) -> b60 (17) -> b64 (18) -> b65 (19) -> b69 (20) -> b72 (21) + # \-> b71 (21) + # + # b72 is a good block. + # b71 is a copy of 72, but re-adds one of its transactions. However, it has the same hash as b71. + # + tip(69) + b72 = block(72) + tx1 = create_and_sign_tx(out[21].tx, out[21].n, 2) + tx2 = create_and_sign_tx(tx1, 0, 1) + b72 = update_block(72, [tx1, tx2]) # now tip is 72 + b71 = copy.deepcopy(b72) + b71.vtx.append(tx2) # add duplicate tx2 + self.block_heights[b71.sha256] = self.block_heights[b69.sha256] + 1 # b71 builds off b69 + self.blocks[71] = b71 + + assert_equal(len(b71.vtx), 4) + assert_equal(len(b72.vtx), 3) + assert_equal(b72.sha256, b71.sha256) + + tip(71) + yield rejected(RejectResult(16, b'bad-txns-duplicate')) + tip(72) + yield accepted() + save_spendable_output() + + + # Test some invalid scripts and MAX_BLOCK_SIGOPS + # + # -> b55 (15) -> b57 (16) -> b60 (17) -> b64 (18) -> b65 (19) -> b69 (20) -> b72 (21) + # \-> b** (22) + # + + # b73 - tx with excessive sigops that are placed after an excessively large script element. + # The purpose of the test is to make sure those sigops are counted. + # + # script is a bytearray of size 20,526 + # + # bytearray[0-19,998] : OP_CHECKSIG + # bytearray[19,999] : OP_PUSHDATA4 + # bytearray[20,000-20,003]: 521 (max_script_element_size+1, in little-endian format) + # bytearray[20,004-20,525]: unread data (script_element) + # bytearray[20,526] : OP_CHECKSIG (this puts us over the limit) + # + tip(72) + b73 = block(73) + size = MAX_BLOCK_SIGOPS - 1 + MAX_SCRIPT_ELEMENT_SIZE + 1 + 5 + 1 + a = bytearray([OP_CHECKSIG] * size) + a[MAX_BLOCK_SIGOPS - 1] = int("4e",16) # OP_PUSHDATA4 + + element_size = MAX_SCRIPT_ELEMENT_SIZE + 1 + a[MAX_BLOCK_SIGOPS] = element_size % 256 + a[MAX_BLOCK_SIGOPS+1] = element_size // 256 + a[MAX_BLOCK_SIGOPS+2] = 0 + a[MAX_BLOCK_SIGOPS+3] = 0 + + tx = create_and_sign_tx(out[22].tx, 0, 1, CScript(a)) + b73 = update_block(73, [tx]) + assert_equal(get_legacy_sigopcount_block(b73), MAX_BLOCK_SIGOPS+1) + yield rejected(RejectResult(16, b'bad-blk-sigops')) + + # b74/75 - if we push an invalid script element, all prevous sigops are counted, + # but sigops after the element are not counted. + # + # The invalid script element is that the push_data indicates that + # there will be a large amount of data (0xffffff bytes), but we only + # provide a much smaller number. These bytes are CHECKSIGS so they would + # cause b75 to fail for excessive sigops, if those bytes were counted. + # + # b74 fails because we put MAX_BLOCK_SIGOPS+1 before the element + # b75 succeeds because we put MAX_BLOCK_SIGOPS before the element + # + # + tip(72) + b74 = block(74) + size = MAX_BLOCK_SIGOPS - 1 + MAX_SCRIPT_ELEMENT_SIZE + 42 # total = 20,561 + a = bytearray([OP_CHECKSIG] * size) + a[MAX_BLOCK_SIGOPS] = 0x4e + a[MAX_BLOCK_SIGOPS+1] = 0xfe + a[MAX_BLOCK_SIGOPS+2] = 0xff + a[MAX_BLOCK_SIGOPS+3] = 0xff + a[MAX_BLOCK_SIGOPS+4] = 0xff + tx = create_and_sign_tx(out[22].tx, 0, 1, CScript(a)) + b74 = update_block(74, [tx]) + yield rejected(RejectResult(16, b'bad-blk-sigops')) + + tip(72) + b75 = block(75) + size = MAX_BLOCK_SIGOPS - 1 + MAX_SCRIPT_ELEMENT_SIZE + 42 + a = bytearray([OP_CHECKSIG] * size) + a[MAX_BLOCK_SIGOPS-1] = 0x4e + a[MAX_BLOCK_SIGOPS] = 0xff + a[MAX_BLOCK_SIGOPS+1] = 0xff + a[MAX_BLOCK_SIGOPS+2] = 0xff + a[MAX_BLOCK_SIGOPS+3] = 0xff + tx = create_and_sign_tx(out[22].tx, 0, 1, CScript(a)) + b75 = update_block(75, [tx]) + yield accepted() + save_spendable_output() + + # Check that if we push an element filled with CHECKSIGs, they are not counted + tip(75) + b76 = block(76) + size = MAX_BLOCK_SIGOPS - 1 + MAX_SCRIPT_ELEMENT_SIZE + 1 + 5 + a = bytearray([OP_CHECKSIG] * size) + a[MAX_BLOCK_SIGOPS-1] = 0x4e # PUSHDATA4, but leave the following bytes as just checksigs + tx = create_and_sign_tx(out[23].tx, 0, 1, CScript(a)) + b76 = update_block(76, [tx]) + yield accepted() + save_spendable_output() + + # Test transaction resurrection + # + # -> b77 (24) -> b78 (25) -> b79 (26) + # \-> b80 (25) -> b81 (26) -> b82 (27) + # + # b78 creates a tx, which is spent in b79. After b82, both should be in mempool + # + # The tx'es must be unsigned and pass the node's mempool policy. It is unsigned for the + # rather obscure reason that the Python signature code does not distinguish between + # Low-S and High-S values (whereas the bitcoin code has custom code which does so); + # as a result of which, the odds are 50% that the python code will use the right + # value and the transaction will be accepted into the mempool. Until we modify the + # test framework to support low-S signing, we are out of luck. + # + # To get around this issue, we construct transactions which are not signed and which + # spend to OP_TRUE. If the standard-ness rules change, this test would need to be + # updated. (Perhaps to spend to a P2SH OP_TRUE script) + # + tip(76) + block(77) + tx77 = create_and_sign_tx(out[24].tx, out[24].n, 10*COIN) + update_block(77, [tx77]) + yield accepted() + save_spendable_output() + + block(78) + tx78 = create_tx(tx77, 0, 9*COIN) + update_block(78, [tx78]) + yield accepted() + + block(79) + tx79 = create_tx(tx78, 0, 8*COIN) + update_block(79, [tx79]) + yield accepted() + + # mempool should be empty + assert_equal(len(self.nodes[0].getrawmempool()), 0) + + tip(77) + block(80, spend=out[25]) + yield rejected() + save_spendable_output() + + block(81, spend=out[26]) + yield rejected() # other chain is same length + save_spendable_output() + + block(82, spend=out[27]) + yield accepted() # now this chain is longer, triggers re-org + save_spendable_output() + + # now check that tx78 and tx79 have been put back into the peer's mempool + mempool = self.nodes[0].getrawmempool() + assert_equal(len(mempool), 2) + assert(tx78.hash in mempool) + assert(tx79.hash in mempool) + + + # Test invalid opcodes in dead execution paths. + # + # -> b81 (26) -> b82 (27) -> b83 (28) + # + b83 = block(83) + op_codes = [OP_IF, OP_INVALIDOPCODE, OP_ELSE, OP_TRUE, OP_ENDIF] + script = CScript(op_codes) + tx1 = create_and_sign_tx(out[28].tx, out[28].n, out[28].tx.vout[0].nValue, script) + + tx2 = create_and_sign_tx(tx1, 0, 0, CScript([OP_TRUE])) + tx2.vin[0].scriptSig = CScript([OP_FALSE]) + tx2.rehash() + + update_block(83, [tx1, tx2]) + yield accepted() + save_spendable_output() + + + # Reorg on/off blocks that have OP_RETURN in them (and try to spend them) + # + # -> b81 (26) -> b82 (27) -> b83 (28) -> b84 (29) -> b87 (30) -> b88 (31) + # \-> b85 (29) -> b86 (30) \-> b89a (32) + # + # + b84 = block(84) + tx1 = create_tx(out[29].tx, out[29].n, 0, CScript([OP_RETURN])) + tx1.vout.append(CTxOut(0, CScript([OP_TRUE]))) + tx1.vout.append(CTxOut(0, CScript([OP_TRUE]))) + tx1.vout.append(CTxOut(0, CScript([OP_TRUE]))) + tx1.vout.append(CTxOut(0, CScript([OP_TRUE]))) + tx1.calc_sha256() + self.sign_tx(tx1, out[29].tx, out[29].n) + tx1.rehash() + tx2 = create_tx(tx1, 1, 0, CScript([OP_RETURN])) + tx2.vout.append(CTxOut(0, CScript([OP_RETURN]))) + tx3 = create_tx(tx1, 2, 0, CScript([OP_RETURN])) + tx3.vout.append(CTxOut(0, CScript([OP_TRUE]))) + tx4 = create_tx(tx1, 3, 0, CScript([OP_TRUE])) + tx4.vout.append(CTxOut(0, CScript([OP_RETURN]))) + tx5 = create_tx(tx1, 4, 0, CScript([OP_RETURN])) + + update_block(84, [tx1,tx2,tx3,tx4,tx5]) + yield accepted() + save_spendable_output() + + tip(83) + block(85, spend=out[29]) + yield rejected() + + block(86, spend=out[30]) + yield accepted() + + tip(84) + block(87, spend=out[30]) + yield rejected() + save_spendable_output() + + block(88, spend=out[31]) + yield accepted() + save_spendable_output() + + # trying to spend the OP_RETURN output is rejected + block("89a", spend=out[32]) + tx = create_tx(tx1, 0, 0, CScript([OP_TRUE])) + update_block("89a", [tx]) + yield rejected() + + + # Test re-org of a week's worth of blocks (1088 blocks) + # This test takes a minute or two and can be accomplished in memory + # + if self.options.runbarelyexpensive: + tip(88) + LARGE_REORG_SIZE = 1088 + test1 = TestInstance(sync_every_block=False) + spend=out[32] + for i in range(89, LARGE_REORG_SIZE + 89): + b = block(i, spend) + tx = CTransaction() + script_length = MAX_BLOCK_SIZE - len(b.serialize()) - 69 + script_output = CScript([b'\x00' * script_length]) + tx.vout.append(CTxOut(0, script_output)) + tx.vin.append(CTxIn(COutPoint(b.vtx[1].sha256, 0))) + b = update_block(i, [tx]) + assert_equal(len(b.serialize()), MAX_BLOCK_SIZE) + test1.blocks_and_transactions.append([self.tip, True]) + save_spendable_output() + spend = get_spendable_output() + + yield test1 + chain1_tip = i + + # now create alt chain of same length + tip(88) + test2 = TestInstance(sync_every_block=False) + for i in range(89, LARGE_REORG_SIZE + 89): + block("alt"+str(i)) + test2.blocks_and_transactions.append([self.tip, False]) + yield test2 + + # extend alt chain to trigger re-org + block("alt" + str(chain1_tip + 1)) + yield accepted() + + # ... and re-org back to the first chain + tip(chain1_tip) + block(chain1_tip + 1) + yield rejected() + block(chain1_tip + 2) + yield accepted() + + chain1_tip += 2 + if __name__ == '__main__': diff --git a/qa/rpc-tests/p2p-mempool.py b/qa/rpc-tests/p2p-mempool.py new file mode 100755 index 0000000000000..5d2daf39f8288 --- /dev/null +++ b/qa/rpc-tests/p2p-mempool.py @@ -0,0 +1,99 @@ +#!/usr/bin/env python3 +# Copyright (c) 2015-2016 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +from test_framework.mininode import * +from test_framework.test_framework import BitcoinTestFramework +from test_framework.util import * +import time + +class TestNode(NodeConnCB): + def __init__(self): + NodeConnCB.__init__(self) + self.connection = None + self.ping_counter = 1 + self.last_pong = msg_pong() + self.block_receive_map = {} + + def add_connection(self, conn): + self.connection = conn + self.peer_disconnected = False + + def on_inv(self, conn, message): + pass + + # Track the last getdata message we receive (used in the test) + def on_getdata(self, conn, message): + self.last_getdata = message + + def on_block(self, conn, message): + message.block.calc_sha256() + try: + self.block_receive_map[message.block.sha256] += 1 + except KeyError as e: + self.block_receive_map[message.block.sha256] = 1 + + # Spin until verack message is received from the node. + # We use this to signal that our test can begin. This + # is called from the testing thread, so it needs to acquire + # the global lock. + def wait_for_verack(self): + def veracked(): + return self.verack_received + return wait_until(veracked, timeout=10) + + def wait_for_disconnect(self): + def disconnected(): + return self.peer_disconnected + return wait_until(disconnected, timeout=10) + + # Wrapper for the NodeConn's send_message function + def send_message(self, message): + self.connection.send_message(message) + + def on_pong(self, conn, message): + self.last_pong = message + + def on_close(self, conn): + self.peer_disconnected = True + + # Sync up with the node after delivery of a block + def sync_with_ping(self, timeout=30): + def received_pong(): + return (self.last_pong.nonce == self.ping_counter) + self.connection.send_message(msg_ping(nonce=self.ping_counter)) + success = wait_until(received_pong, timeout) + self.ping_counter += 1 + return success + + def send_mempool(self): + self.lastInv = [] + self.send_message(msg_mempool()) + +class P2PMempoolTests(BitcoinTestFramework): + def setup_chain(self): + initialize_chain_clean(self.options.tmpdir, 2) + + def setup_network(self): + # Start a node with maxuploadtarget of 200 MB (/24h) + self.nodes = [] + self.nodes.append(start_node(0, self.options.tmpdir, ["-debug", "-peerbloomfilters=0"])) + + def run_test(self): + #connect a mininode + aTestNode = TestNode() + node = NodeConn('127.0.0.1', p2p_port(0), self.nodes[0], aTestNode) + aTestNode.add_connection(node) + NetworkThread().start() + aTestNode.wait_for_verack() + + #request mempool + aTestNode.send_mempool() + aTestNode.wait_for_disconnect() + + #mininode must be disconnected at this point + assert_equal(len(self.nodes[0].getpeerinfo()), 0) + +if __name__ == '__main__': + P2PMempoolTests().main() diff --git a/qa/rpc-tests/p2p-versionbits-warning.py b/qa/rpc-tests/p2p-versionbits-warning.py index 061dcbf0e1e87..962cafef0b17b 100755 --- a/qa/rpc-tests/p2p-versionbits-warning.py +++ b/qa/rpc-tests/p2p-versionbits-warning.py @@ -1,8 +1,7 @@ -#!/usr/bin/env python2 +#!/usr/bin/env python3 # Copyright (c) 2016 The Bitcoin Core developers -# Distributed under the MIT/X11 software license, see the accompanying +# Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. -# from test_framework.mininode import * from test_framework.test_framework import BitcoinTestFramework @@ -60,8 +59,10 @@ def sync_with_ping(self, timeout=30): class VersionBitsWarningTest(BitcoinTestFramework): - def setup_chain(self): - initialize_chain_clean(self.options.tmpdir, 1) + def __init__(self): + super().__init__() + self.setup_clean_chain = True + self.num_nodes = 1 def setup_network(self): self.nodes = [] @@ -82,7 +83,7 @@ def send_blocks_with_version(self, peer, numblocks, nVersionToUse): block_time = self.nodes[0].getblockheader(tip)["time"]+1 tip = int(tip, 16) - for i in xrange(numblocks): + for i in range(numblocks): block = create_block(tip, create_coinbase(height+1), block_time) block.nVersion = nVersionToUse block.solve() diff --git a/qa/rpc-tests/prioritise_transaction.py b/qa/rpc-tests/prioritise_transaction.py index f8d9063b4ebc0..e1771231c066e 100755 --- a/qa/rpc-tests/prioritise_transaction.py +++ b/qa/rpc-tests/prioritise_transaction.py @@ -1,5 +1,5 @@ -#!/usr/bin/env python2 -# Copyright (c) 2015 The Bitcoin Core developers +#!/usr/bin/env python3 +# Copyright (c) 2015-2016 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -14,11 +14,11 @@ class PrioritiseTransactionTest(BitcoinTestFramework): def __init__(self): - self.txouts = gen_return_txouts() + super().__init__() + self.setup_clean_chain = True + self.num_nodes = 1 - def setup_chain(self): - print("Initializing test directory "+self.options.tmpdir) - initialize_chain_clean(self.options.tmpdir, 1) + self.txouts = gen_return_txouts() def setup_network(self): self.nodes = [] @@ -35,7 +35,7 @@ def run_test(self): # Create 3 batches of transactions at 3 different fee rate levels range_size = utxo_count // 3 - for i in xrange(3): + for i in range(3): txids.append([]) start_range = i * range_size end_range = start_range + range_size @@ -46,7 +46,7 @@ def run_test(self): # more transactions. mempool = self.nodes[0].getrawmempool(True) sizes = [0, 0, 0] - for i in xrange(3): + for i in range(3): for j in txids[i]: assert(j in mempool) sizes[i] += mempool[j]['size'] @@ -61,7 +61,7 @@ def run_test(self): self.nodes[0].generate(1) mempool = self.nodes[0].getrawmempool() - print "Assert that prioritised transaction was mined" + print("Assert that prioritised transaction was mined") assert(txids[0][0] not in mempool) assert(txids[0][1] in mempool) @@ -93,7 +93,7 @@ def run_test(self): # High fee transaction should not have been mined, but other high fee rate # transactions should have been. mempool = self.nodes[0].getrawmempool() - print "Assert that de-prioritised transaction is still in mempool" + print("Assert that de-prioritised transaction is still in mempool") assert(high_fee_tx in mempool) for x in txids[2]: if (x != high_fee_tx): @@ -135,7 +135,7 @@ def run_test(self): # accepted. self.nodes[0].prioritisetransaction(tx2_id, 0, int(self.relayfee*COIN)) - print "Assert that prioritised free transaction is accepted to mempool" + print("Assert that prioritised free transaction is accepted to mempool") assert_equal(self.nodes[0].sendrawtransaction(tx2_hex), tx2_id) assert(tx2_id in self.nodes[0].getrawmempool()) diff --git a/qa/rpc-tests/proxy_test.py b/qa/rpc-tests/proxy_test.py index 91c871ddc9003..27160cae07762 100755 --- a/qa/rpc-tests/proxy_test.py +++ b/qa/rpc-tests/proxy_test.py @@ -1,7 +1,8 @@ -#!/usr/bin/env python2 -# Copyright (c) 2015 The Bitcoin Core developers +#!/usr/bin/env python3 +# Copyright (c) 2015-2016 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. + import socket from test_framework.socks5 import Socks5Configuration, Socks5Command, Socks5Server, AddressType @@ -35,6 +36,10 @@ class ProxyTest(BitcoinTestFramework): def __init__(self): + super().__init__() + self.num_nodes = 4 + self.setup_clean_chain = False + self.have_ipv6 = test_ipv6_local() # Create two proxies on different ports # ... one unauthenticated @@ -55,7 +60,7 @@ def __init__(self): self.conf3.unauth = True self.conf3.auth = True else: - print "Warning: testing without local IPv6 support" + print("Warning: testing without local IPv6 support") self.serv1 = Socks5Server(self.conf1) self.serv1.start() @@ -76,7 +81,7 @@ def setup_nodes(self): ] if self.have_ipv6: args[3] = ['-listen', '-debug=net', '-debug=proxy', '-proxy=[%s]:%i' % (self.conf3.addr),'-proxyrandomize=0', '-noonion'] - return start_nodes(4, self.options.tmpdir, extra_args=args) + return start_nodes(self.num_nodes, self.options.tmpdir, extra_args=args) def node_test(self, node, proxies, auth, test_onion=True): rv = [] diff --git a/qa/rpc-tests/pruning.py b/qa/rpc-tests/pruning.py index eccd157e5c085..d225e29b500f2 100755 --- a/qa/rpc-tests/pruning.py +++ b/qa/rpc-tests/pruning.py @@ -1,5 +1,5 @@ -#!/usr/bin/env python2 -# Copyright (c) 2014-2015 The Bitcoin Core developers +#!/usr/bin/env python3 +# Copyright (c) 2014-2016 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -20,14 +20,14 @@ def calc_usage(blockdir): class PruneTest(BitcoinTestFramework): def __init__(self): + super().__init__() + self.setup_clean_chain = True + self.num_nodes = 3 + self.utxo = [] self.address = ["",""] self.txouts = gen_return_txouts() - def setup_chain(self): - print("Initializing test directory "+self.options.tmpdir) - initialize_chain_clean(self.options.tmpdir, 3) - def setup_network(self): self.nodes = [] self.is_network_split = False @@ -57,7 +57,7 @@ def create_big_chain(self): sync_blocks(self.nodes[0:2]) self.nodes[0].generate(150) # Then mine enough full blocks to create more than 550MiB of data - for i in xrange(645): + for i in range(645): self.mine_full_block(self.nodes[0], self.address[0]) sync_blocks(self.nodes[0:3]) @@ -65,30 +65,30 @@ def create_big_chain(self): def test_height_min(self): if not os.path.isfile(self.prunedir+"blk00000.dat"): raise AssertionError("blk00000.dat is missing, pruning too early") - print "Success" - print "Though we're already using more than 550MiB, current usage:", calc_usage(self.prunedir) - print "Mining 25 more blocks should cause the first block file to be pruned" + print("Success") + print("Though we're already using more than 550MiB, current usage:", calc_usage(self.prunedir)) + print("Mining 25 more blocks should cause the first block file to be pruned") # Pruning doesn't run until we're allocating another chunk, 20 full blocks past the height cutoff will ensure this - for i in xrange(25): + for i in range(25): self.mine_full_block(self.nodes[0],self.address[0]) waitstart = time.time() while os.path.isfile(self.prunedir+"blk00000.dat"): time.sleep(0.1) - if time.time() - waitstart > 10: + if time.time() - waitstart > 30: raise AssertionError("blk00000.dat not pruned when it should be") - print "Success" + print("Success") usage = calc_usage(self.prunedir) - print "Usage should be below target:", usage + print("Usage should be below target:", usage) if (usage > 550): raise AssertionError("Pruning target not being met") def create_chain_with_staleblocks(self): # Create stale blocks in manageable sized chunks - print "Mine 24 (stale) blocks on Node 1, followed by 25 (main chain) block reorg from Node 0, for 12 rounds" + print("Mine 24 (stale) blocks on Node 1, followed by 25 (main chain) block reorg from Node 0, for 12 rounds") - for j in xrange(12): + for j in range(12): # Disconnect node 0 so it can mine a longer reorg chain without knowing about node 1's soon-to-be-stale chain # Node 2 stays connected, so it hears about the stale blocks and then reorg's when node0 reconnects # Stopping node 0 also clears its mempool, so it doesn't have node1's transactions to accidentally mine @@ -96,7 +96,7 @@ def create_chain_with_staleblocks(self): self.nodes[0]=start_node(0, self.options.tmpdir, ["-debug","-maxreceivebuffer=20000","-blockmaxsize=999000", "-checkblocks=5"], timewait=900) # Mine 24 blocks in node 1 self.utxo = self.nodes[1].listunspent() - for i in xrange(24): + for i in range(24): if j == 0: self.mine_full_block(self.nodes[1],self.address[1]) else: @@ -104,7 +104,7 @@ def create_chain_with_staleblocks(self): # Reorg back with 25 block chain from node 0 self.utxo = self.nodes[0].listunspent() - for i in xrange(25): + for i in range(25): self.mine_full_block(self.nodes[0],self.address[0]) # Create connections in the order so both nodes can see the reorg at the same time @@ -112,7 +112,7 @@ def create_chain_with_staleblocks(self): connect_nodes(self.nodes[2], 0) sync_blocks(self.nodes[0:3]) - print "Usage can be over target because of high stale rate:", calc_usage(self.prunedir) + print("Usage can be over target because of high stale rate:", calc_usage(self.prunedir)) def reorg_test(self): # Node 1 will mine a 300 block chain starting 287 blocks back from Node 0 and Node 2's tip @@ -123,11 +123,11 @@ def reorg_test(self): self.nodes[1]=start_node(1, self.options.tmpdir, ["-debug","-maxreceivebuffer=20000","-blockmaxsize=5000", "-checkblocks=5", "-disablesafemode"], timewait=900) height = self.nodes[1].getblockcount() - print "Current block height:", height + print("Current block height:", height) invalidheight = height-287 badhash = self.nodes[1].getblockhash(invalidheight) - print "Invalidating block at height:",invalidheight,badhash + print("Invalidating block at height:",invalidheight,badhash) self.nodes[1].invalidateblock(badhash) # We've now switched to our previously mined-24 block fork on node 1, but thats not what we want @@ -139,29 +139,29 @@ def reorg_test(self): curhash = self.nodes[1].getblockhash(invalidheight - 1) assert(self.nodes[1].getblockcount() == invalidheight - 1) - print "New best height", self.nodes[1].getblockcount() + print("New best height", self.nodes[1].getblockcount()) # Reboot node1 to clear those giant tx's from mempool stop_node(self.nodes[1],1) self.nodes[1]=start_node(1, self.options.tmpdir, ["-debug","-maxreceivebuffer=20000","-blockmaxsize=5000", "-checkblocks=5", "-disablesafemode"], timewait=900) - print "Generating new longer chain of 300 more blocks" + print("Generating new longer chain of 300 more blocks") self.nodes[1].generate(300) - print "Reconnect nodes" + print("Reconnect nodes") connect_nodes(self.nodes[0], 1) connect_nodes(self.nodes[2], 1) sync_blocks(self.nodes[0:3]) - print "Verify height on node 2:",self.nodes[2].getblockcount() - print "Usage possibly still high bc of stale blocks in block files:", calc_usage(self.prunedir) + print("Verify height on node 2:",self.nodes[2].getblockcount()) + print("Usage possibly still high bc of stale blocks in block files:", calc_usage(self.prunedir)) - print "Mine 220 more blocks so we have requisite history (some blocks will be big and cause pruning of previous chain)" + print("Mine 220 more blocks so we have requisite history (some blocks will be big and cause pruning of previous chain)") self.nodes[0].generate(220) #node 0 has many large tx's in its mempool from the disconnects sync_blocks(self.nodes[0:3]) usage = calc_usage(self.prunedir) - print "Usage should be below target:", usage + print("Usage should be below target:", usage) if (usage > 550): raise AssertionError("Pruning target not being met") @@ -173,7 +173,7 @@ def reorg_back(self): self.nodes[2].getblock(self.forkhash) raise AssertionError("Old block wasn't pruned so can't test redownload") except JSONRPCException as e: - print "Will need to redownload block",self.forkheight + print("Will need to redownload block",self.forkheight) # Verify that we have enough history to reorg back to the fork point # Although this is more than 288 blocks, because this chain was written more recently @@ -197,14 +197,14 @@ def reorg_back(self): # At this point node 2 is within 288 blocks of the fork point so it will preserve its ability to reorg if self.nodes[2].getblockcount() < self.mainchainheight: blocks_to_mine = first_reorg_height + 1 - self.mainchainheight - print "Rewind node 0 to prev main chain to mine longer chain to trigger redownload. Blocks needed:", blocks_to_mine + print("Rewind node 0 to prev main chain to mine longer chain to trigger redownload. Blocks needed:", blocks_to_mine) self.nodes[0].invalidateblock(curchainhash) assert(self.nodes[0].getblockcount() == self.mainchainheight) assert(self.nodes[0].getbestblockhash() == self.mainchainhash2) goalbesthash = self.nodes[0].generate(blocks_to_mine)[-1] goalbestheight = first_reorg_height + 1 - print "Verify node 2 reorged back to the main chain, some blocks of which it had to redownload" + print("Verify node 2 reorged back to the main chain, some blocks of which it had to redownload") waitstart = time.time() while self.nodes[2].getblockcount() < goalbestheight: time.sleep(0.1) @@ -217,7 +217,7 @@ def reorg_back(self): def mine_full_block(self, node, address): # Want to create a full block # We'll generate a 66k transaction below, and 14 of them is close to the 1MB block limit - for j in xrange(14): + for j in range(14): if len(self.utxo) < 14: self.utxo = node.listunspent() inputs=[] @@ -241,8 +241,8 @@ def mine_full_block(self, node, address): def run_test(self): - print "Warning! This test requires 4GB of disk space and takes over 30 mins (up to 2 hours)" - print "Mining a big blockchain of 995 blocks" + print("Warning! This test requires 4GB of disk space and takes over 30 mins (up to 2 hours)") + print("Mining a big blockchain of 995 blocks") self.create_big_chain() # Chain diagram key: # * blocks on main chain @@ -253,12 +253,12 @@ def run_test(self): # Start by mining a simple chain that all nodes have # N0=N1=N2 **...*(995) - print "Check that we haven't started pruning yet because we're below PruneAfterHeight" + print("Check that we haven't started pruning yet because we're below PruneAfterHeight") self.test_height_min() # Extend this chain past the PruneAfterHeight # N0=N1=N2 **...*(1020) - print "Check that we'll exceed disk space target if we have a very high stale block rate" + print("Check that we'll exceed disk space target if we have a very high stale block rate") self.create_chain_with_staleblocks() # Disconnect N0 # And mine a 24 block chain on N1 and a separate 25 block chain on N0 @@ -282,7 +282,7 @@ def run_test(self): self.mainchainheight = self.nodes[2].getblockcount() #1320 self.mainchainhash2 = self.nodes[2].getblockhash(self.mainchainheight) - print "Check that we can survive a 288 block reorg still" + print("Check that we can survive a 288 block reorg still") (self.forkheight,self.forkhash) = self.reorg_test() #(1033, ) # Now create a 288 block reorg by mining a longer chain on N1 # First disconnect N1 @@ -315,7 +315,7 @@ def run_test(self): # \ # *...**(1320) - print "Test that we can rerequest a block we previously pruned if needed for a reorg" + print("Test that we can rerequest a block we previously pruned if needed for a reorg") self.reorg_back() # Verify that N2 still has block 1033 on current chain (@), but not on main chain (*) # Invalidate 1033 on current chain (@) on N2 and we should be able to reorg to @@ -335,7 +335,7 @@ def run_test(self): # # N1 doesn't change because 1033 on main chain (*) is invalid - print "Done" + print("Done") if __name__ == '__main__': PruneTest().main() diff --git a/qa/rpc-tests/rawtransactions.py b/qa/rpc-tests/rawtransactions.py index e38ef6c8b1071..aa403f058c538 100755 --- a/qa/rpc-tests/rawtransactions.py +++ b/qa/rpc-tests/rawtransactions.py @@ -1,5 +1,5 @@ -#!/usr/bin/env python2 -# Copyright (c) 2014-2015 The Bitcoin Core developers +#!/usr/bin/env python3 +# Copyright (c) 2014-2016 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -14,12 +14,13 @@ # Create one-input, one-output, no-fee transaction: class RawTransactionsTest(BitcoinTestFramework): - def setup_chain(self): - print("Initializing test directory "+self.options.tmpdir) - initialize_chain_clean(self.options.tmpdir, 3) + def __init__(self): + super().__init__() + self.setup_clean_chain = True + self.num_nodes = 3 def setup_network(self, split=False): - self.nodes = start_nodes(3, self.options.tmpdir) + self.nodes = start_nodes(self.num_nodes, self.options.tmpdir) #connect to a local machine for debugging #url = "http://bitcoinrpc:DP6DvqZtqXarpeNWyN3LZTFchCCyCUuHwNF7E8pX99x1@%s:%d" % ('127.0.0.1', 18332) @@ -137,5 +138,11 @@ def run_test(self): self.sync_all() assert_equal(self.nodes[0].getbalance(), bal+Decimal('50.00000000')+Decimal('2.19000000')) #block reward + tx + inputs = [ {'txid' : "1d1d4e24ed99057e84c3f80fd8fbec79ed9e1acee37da269356ecea000000000", 'vout' : 1, 'sequence' : 1000}] + outputs = { self.nodes[0].getnewaddress() : 1 } + rawtx = self.nodes[0].createrawtransaction(inputs, outputs) + decrawtx= self.nodes[0].decoderawtransaction(rawtx) + assert_equal(decrawtx['vin'][0]['sequence'], 1000) + if __name__ == '__main__': RawTransactionsTest().main() diff --git a/qa/rpc-tests/receivedby.py b/qa/rpc-tests/receivedby.py index 606426b394199..4f17b661cb28d 100755 --- a/qa/rpc-tests/receivedby.py +++ b/qa/rpc-tests/receivedby.py @@ -1,5 +1,5 @@ -#!/usr/bin/env python2 -# Copyright (c) 2014-2015 The Bitcoin Core developers +#!/usr/bin/env python3 +# Copyright (c) 2014-2016 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -25,38 +25,17 @@ def get_sub_array_from_array(object_array, to_match): return item return [] -def check_array_result(object_array, to_match, expected, should_not_find = False): - """ - Pass in array of JSON objects, a dictionary with key/value pairs - to match against, and another dictionary with expected key/value - pairs. - If the should_not_find flag is true, to_match should not be found in object_array - """ - if should_not_find == True: - expected = { } - num_matched = 0 - for item in object_array: - all_match = True - for key,value in to_match.items(): - if item[key] != value: - all_match = False - if not all_match: - continue - for key,value in expected.items(): - if item[key] != value: - raise AssertionError("%s : expected %s=%s"%(str(item), str(key), str(value))) - num_matched = num_matched+1 - if num_matched == 0 and should_not_find != True: - raise AssertionError("No objects matched %s"%(str(to_match))) - if num_matched > 0 and should_not_find == True: - raise AssertionError("Objects was matched %s"%(str(to_match))) - class ReceivedByTest(BitcoinTestFramework): + def __init__(self): + super().__init__() + self.num_nodes = 4 + self.setup_clean_chain = False + def setup_nodes(self): #This test requires mocktime enable_mocktime() - return start_nodes(4, self.options.tmpdir) + return start_nodes(self.num_nodes, self.options.tmpdir) def run_test(self): ''' @@ -68,26 +47,26 @@ def run_test(self): self.sync_all() #Check not listed in listreceivedbyaddress because has 0 confirmations - check_array_result(self.nodes[1].listreceivedbyaddress(), + assert_array_result(self.nodes[1].listreceivedbyaddress(), {"address":addr}, { }, True) #Bury Tx under 10 block so it will be returned by listreceivedbyaddress self.nodes[1].generate(10) self.sync_all() - check_array_result(self.nodes[1].listreceivedbyaddress(), + assert_array_result(self.nodes[1].listreceivedbyaddress(), {"address":addr}, {"address":addr, "account":"", "amount":Decimal("0.1"), "confirmations":10, "txids":[txid,]}) #With min confidence < 10 - check_array_result(self.nodes[1].listreceivedbyaddress(5), + assert_array_result(self.nodes[1].listreceivedbyaddress(5), {"address":addr}, {"address":addr, "account":"", "amount":Decimal("0.1"), "confirmations":10, "txids":[txid,]}) #With min confidence > 10, should not find Tx - check_array_result(self.nodes[1].listreceivedbyaddress(11),{"address":addr},{ },True) + assert_array_result(self.nodes[1].listreceivedbyaddress(11),{"address":addr},{ },True) #Empty Tx addr = self.nodes[1].getnewaddress() - check_array_result(self.nodes[1].listreceivedbyaddress(0,True), + assert_array_result(self.nodes[1].listreceivedbyaddress(0,True), {"address":addr}, {"address":addr, "account":"", "amount":0, "confirmations":0, "txids":[]}) @@ -131,7 +110,7 @@ def run_test(self): self.sync_all() # listreceivedbyaccount should return received_by_account_json because of 0 confirmations - check_array_result(self.nodes[1].listreceivedbyaccount(), + assert_array_result(self.nodes[1].listreceivedbyaccount(), {"account":account}, received_by_account_json) @@ -143,7 +122,7 @@ def run_test(self): self.nodes[1].generate(10) self.sync_all() # listreceivedbyaccount should return updated account balance - check_array_result(self.nodes[1].listreceivedbyaccount(), + assert_array_result(self.nodes[1].listreceivedbyaccount(), {"account":account}, {"account":received_by_account_json["account"], "amount":(received_by_account_json["amount"] + Decimal("0.1"))}) diff --git a/qa/rpc-tests/reindex.py b/qa/rpc-tests/reindex.py index 321c2fe422a5f..abbbb1033693b 100755 --- a/qa/rpc-tests/reindex.py +++ b/qa/rpc-tests/reindex.py @@ -1,32 +1,43 @@ -#!/usr/bin/env python2 -# Copyright (c) 2014-2015 The Bitcoin Core developers +#!/usr/bin/env python3 +# Copyright (c) 2014-2016 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. # -# Test -reindex with CheckBlockIndex +# Test -reindex and -reindex-chainstate with CheckBlockIndex # from test_framework.test_framework import BitcoinTestFramework from test_framework.util import * +import time class ReindexTest(BitcoinTestFramework): - def setup_chain(self): - print("Initializing test directory "+self.options.tmpdir) - initialize_chain_clean(self.options.tmpdir, 1) + def __init__(self): + super().__init__() + self.setup_clean_chain = True + self.num_nodes = 1 def setup_network(self): self.nodes = [] self.is_network_split = False self.nodes.append(start_node(0, self.options.tmpdir)) - def run_test(self): + def reindex(self, justchainstate=False): self.nodes[0].generate(3) + blockcount = self.nodes[0].getblockcount() stop_node(self.nodes[0], 0) wait_bitcoinds() - self.nodes[0]=start_node(0, self.options.tmpdir, ["-debug", "-reindex", "-checkblockindex=1"]) - assert_equal(self.nodes[0].getblockcount(), 3) - print "Success" + self.nodes[0]=start_node(0, self.options.tmpdir, ["-debug", "-reindex-chainstate" if justchainstate else "-reindex", "-checkblockindex=1"]) + while self.nodes[0].getblockcount() < blockcount: + time.sleep(0.1) + assert_equal(self.nodes[0].getblockcount(), blockcount) + print("Success") + + def run_test(self): + self.reindex(False) + self.reindex(True) + self.reindex(False) + self.reindex(True) if __name__ == '__main__': ReindexTest().main() diff --git a/qa/rpc-tests/replace-by-fee.py b/qa/rpc-tests/replace-by-fee.py index 4c8ef6de26eb2..34c0f9d7957c7 100755 --- a/qa/rpc-tests/replace-by-fee.py +++ b/qa/rpc-tests/replace-by-fee.py @@ -1,5 +1,5 @@ -#!/usr/bin/env python2 -# Copyright (c) 2014-2015 The Bitcoin Core developers +#!/usr/bin/env python3 +# Copyright (c) 2014-2016 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -68,6 +68,11 @@ def make_utxo(node, amount, confirmed=True, scriptPubKey=CScript([1])): class ReplaceByFeeTest(BitcoinTestFramework): + def __init__(self): + super().__init__() + self.num_nodes = 1 + self.setup_clean_chain = False + def setup_network(self): self.nodes = [] self.nodes.append(start_node(0, self.options.tmpdir, ["-maxorphantx=1000", "-debug", @@ -82,34 +87,34 @@ def setup_network(self): def run_test(self): make_utxo(self.nodes[0], 1*COIN) - print "Running test simple doublespend..." + print("Running test simple doublespend...") self.test_simple_doublespend() - print "Running test doublespend chain..." + print("Running test doublespend chain...") self.test_doublespend_chain() - print "Running test doublespend tree..." + print("Running test doublespend tree...") self.test_doublespend_tree() - print "Running test replacement feeperkb..." + print("Running test replacement feeperkb...") self.test_replacement_feeperkb() - print "Running test spends of conflicting outputs..." + print("Running test spends of conflicting outputs...") self.test_spends_of_conflicting_outputs() - print "Running test new unconfirmed inputs..." + print("Running test new unconfirmed inputs...") self.test_new_unconfirmed_inputs() - print "Running test too many replacements..." + print("Running test too many replacements...") self.test_too_many_replacements() - print "Running test opt-in..." + print("Running test opt-in...") self.test_opt_in() - print "Running test prioritised transactions..." + print("Running test prioritised transactions...") self.test_prioritised_transactions() - print "Passed\n" + print("Passed\n") def test_simple_doublespend(self): """Simple doublespend""" @@ -459,7 +464,7 @@ def test_opt_in(self): except JSONRPCException as exp: assert_equal(exp.error['code'], -26) else: - print tx1b_txid + print(tx1b_txid) assert(False) tx1_outpoint = make_utxo(self.nodes[0], int(1.1*COIN)) diff --git a/qa/rpc-tests/rest.py b/qa/rpc-tests/rest.py index 359f9239fb370..c9c2eaf7f37b9 100755 --- a/qa/rpc-tests/rest.py +++ b/qa/rpc-tests/rest.py @@ -1,5 +1,5 @@ -#!/usr/bin/env python2 -# Copyright (c) 2014-2015 The Bitcoin Core developers +#!/usr/bin/env python3 +# Copyright (c) 2014-2016 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -13,16 +13,9 @@ from struct import * from io import BytesIO from codecs import encode -import binascii -try: - import http.client as httplib -except ImportError: - import httplib -try: - import urllib.parse as urlparse -except ImportError: - import urlparse +import http.client +import urllib.parse def deser_uint256(f): r = 0 @@ -33,7 +26,7 @@ def deser_uint256(f): #allows simple http get calls def http_get_call(host, port, path, response_object = 0): - conn = httplib.HTTPConnection(host, port) + conn = http.client.HTTPConnection(host, port) conn.request('GET', path) if response_object: @@ -43,7 +36,7 @@ def http_get_call(host, port, path, response_object = 0): #allows simple http post calls with a request body def http_post_call(host, port, path, requestdata = '', response_object = 0): - conn = httplib.HTTPConnection(host, port) + conn = http.client.HTTPConnection(host, port) conn.request('POST', path, requestdata) if response_object: @@ -54,12 +47,13 @@ def http_post_call(host, port, path, requestdata = '', response_object = 0): class RESTTest (BitcoinTestFramework): FORMAT_SEPARATOR = "." - def setup_chain(self): - print("Initializing test directory "+self.options.tmpdir) - initialize_chain_clean(self.options.tmpdir, 3) + def __init__(self): + super().__init__() + self.setup_clean_chain = True + self.num_nodes = 3 def setup_network(self, split=False): - self.nodes = start_nodes(3, self.options.tmpdir) + self.nodes = start_nodes(self.num_nodes, self.options.tmpdir) connect_nodes_bi(self.nodes,0,1) connect_nodes_bi(self.nodes,1,2) connect_nodes_bi(self.nodes,0,2) @@ -67,8 +61,8 @@ def setup_network(self, split=False): self.sync_all() def run_test(self): - url = urlparse.urlparse(self.nodes[0].url) - print "Mining blocks..." + url = urllib.parse.urlparse(self.nodes[0].url) + print("Mining blocks...") self.nodes[0].generate(1) self.sync_all() @@ -151,7 +145,7 @@ def run_test(self): output.write(bin_response) output.seek(0) chainHeight = unpack("i", output.read(4))[0] - hashFromBinResponse = hex(deser_uint256(output))[2:].zfill(65).rstrip("L") + hashFromBinResponse = hex(deser_uint256(output))[2:].zfill(64) assert_equal(bb_hash, hashFromBinResponse) #check if getutxo's chaintip during calculation was fine assert_equal(chainHeight, 102) #chain height must be 102 diff --git a/qa/rpc-tests/rpcbind_test.py b/qa/rpc-tests/rpcbind_test.py index 10a48b5556e40..572273566b63a 100755 --- a/qa/rpc-tests/rpcbind_test.py +++ b/qa/rpc-tests/rpcbind_test.py @@ -1,5 +1,5 @@ -#!/usr/bin/env python2 -# Copyright (c) 2014-2015 The Bitcoin Core developers +#!/usr/bin/env python3 +# Copyright (c) 2014-2016 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -24,7 +24,7 @@ def run_bind_test(tmpdir, allow_ips, connect_to, addresses, expected): if allow_ips: base_args += ['-rpcallowip=' + x for x in allow_ips] binds = ['-rpcbind='+addr for addr in addresses] - nodes = start_nodes(1, tmpdir, [base_args + binds], connect_to) + nodes = start_nodes(self.num_nodes, tmpdir, [base_args + binds], connect_to) try: pid = bitcoind_processes[0].pid assert_equal(set(get_bind_addrs(pid)), set(expected)) @@ -38,7 +38,7 @@ def run_allowip_test(tmpdir, allow_ips, rpchost, rpcport): at a non-localhost IP. ''' base_args = ['-disablewallet', '-nolisten'] + ['-rpcallowip='+x for x in allow_ips] - nodes = start_nodes(1, tmpdir, [base_args]) + nodes = start_nodes(self.num_nodes, tmpdir, [base_args]) try: # connect to node through non-loopback interface url = "http://rt:rt@%s:%d" % (rpchost, rpcport,) @@ -51,7 +51,7 @@ def run_allowip_test(tmpdir, allow_ips, rpchost, rpcport): def run_test(tmpdir): - assert(sys.platform == 'linux2') # due to OS-specific network stats queries, this test works only on Linux + assert(sys.platform.startswith('linux')) # due to OS-specific network stats queries, this test works only on Linux # find the first non-loopback interface for testing non_loopback_ip = None for name,ip in all_interfaces(): diff --git a/qa/rpc-tests/sendheaders.py b/qa/rpc-tests/sendheaders.py index 2bc32584b5f8e..6ab17d59b3878 100755 --- a/qa/rpc-tests/sendheaders.py +++ b/qa/rpc-tests/sendheaders.py @@ -1,8 +1,7 @@ -#!/usr/bin/env python2 -# Copyright (c) 2014-2015 The Bitcoin Core developers -# Distributed under the MIT/X11 software license, see the accompanying +#!/usr/bin/env python3 +# Copyright (c) 2014-2016 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. -# from test_framework.mininode import * from test_framework.test_framework import BitcoinTestFramework @@ -209,12 +208,14 @@ def __init__(self): BaseNode.__init__(self) class SendHeadersTest(BitcoinTestFramework): - def setup_chain(self): - initialize_chain_clean(self.options.tmpdir, 2) + def __init__(self): + super().__init__() + self.setup_clean_chain = True + self.num_nodes = 2 def setup_network(self): self.nodes = [] - self.nodes = start_nodes(2, self.options.tmpdir, [["-debug", "-logtimemicros=1"]]*2) + self.nodes = start_nodes(self.num_nodes, self.options.tmpdir, [["-debug", "-logtimemicros=1"]]*2) connect_nodes(self.nodes[0], 1) # mine count blocks and return the new tip @@ -266,8 +267,8 @@ def run_test(self): # PART 1 # 1. Mine a block; expect inv announcements each time - print "Part 1: headers don't start before sendheaders message..." - for i in xrange(4): + print("Part 1: headers don't start before sendheaders message...") + for i in range(4): old_tip = tip tip = self.mine_blocks(1) assert_equal(inv_node.check_last_announcement(inv=[tip]), True) @@ -297,14 +298,14 @@ def run_test(self): inv_node.clear_last_announcement() test_node.clear_last_announcement() - print "Part 1: success!" - print "Part 2: announce blocks with headers after sendheaders message..." + print("Part 1: success!") + print("Part 2: announce blocks with headers after sendheaders message...") # PART 2 # 2. Send a sendheaders message and test that headers announcements # commence and keep working. test_node.send_message(msg_sendheaders()) prev_tip = int(self.nodes[0].getbestblockhash(), 16) - test_node.get_headers(locator=[prev_tip], hashstop=0L) + test_node.get_headers(locator=[prev_tip], hashstop=0) test_node.sync_with_ping() # Now that we've synced headers, headers announcements should work @@ -314,14 +315,14 @@ def run_test(self): height = self.nodes[0].getblockcount()+1 block_time += 10 # Advance far enough ahead - for i in xrange(10): + for i in range(10): # Mine i blocks, and alternate announcing either via # inv (of tip) or via headers. After each, new blocks # mined by the node should successfully be announced # with block header, even though the blocks are never requested - for j in xrange(2): + for j in range(2): blocks = [] - for b in xrange(i+1): + for b in range(i+1): blocks.append(create_block(tip, create_coinbase(height), block_time)) blocks[-1].solve() tip = blocks[-1].sha256 @@ -360,13 +361,13 @@ def run_test(self): height += 1 block_time += 1 - print "Part 2: success!" + print("Part 2: success!") - print "Part 3: headers announcements can stop after large reorg, and resume after headers/inv from peer..." + print("Part 3: headers announcements can stop after large reorg, and resume after headers/inv from peer...") # PART 3. Headers announcements can stop after large reorg, and resume after # getheaders or inv from peer. - for j in xrange(2): + for j in range(2): # First try mining a reorg that can propagate with header announcement new_block_hashes = self.mine_reorg(length=7) tip = new_block_hashes[-1] @@ -392,7 +393,7 @@ def run_test(self): test_node.get_data(new_block_hashes) test_node.wait_for_block(new_block_hashes[-1]) - for i in xrange(3): + for i in range(3): # Mine another block, still should get only an inv tip = self.mine_blocks(1) assert_equal(inv_node.check_last_announcement(inv=[tip]), True) @@ -414,7 +415,7 @@ def run_test(self): # of headers announcements, or mine a new block and inv it, also # triggering resumption of headers announcements. if j == 0: - test_node.get_headers(locator=[tip], hashstop=0L) + test_node.get_headers(locator=[tip], hashstop=0) test_node.sync_with_ping() else: test_node.send_block_inv(tip) @@ -424,9 +425,9 @@ def run_test(self): assert_equal(inv_node.check_last_announcement(inv=[tip]), True) assert_equal(test_node.check_last_announcement(headers=[tip]), True) - print "Part 3: success!" + print("Part 3: success!") - print "Part 4: Testing direct fetch behavior..." + print("Part 4: Testing direct fetch behavior...") tip = self.mine_blocks(1) height = self.nodes[0].getblockcount() + 1 last_time = self.nodes[0].getblock(self.nodes[0].getbestblockhash())['time'] @@ -434,7 +435,7 @@ def run_test(self): # Create 2 blocks. Send the blocks, then send the headers. blocks = [] - for b in xrange(2): + for b in range(2): blocks.append(create_block(tip, create_coinbase(height), block_time)) blocks[-1].solve() tip = blocks[-1].sha256 @@ -452,7 +453,7 @@ def run_test(self): # This time, direct fetch should work blocks = [] - for b in xrange(3): + for b in range(3): blocks.append(create_block(tip, create_coinbase(height), block_time)) blocks[-1].solve() tip = blocks[-1].sha256 @@ -473,7 +474,7 @@ def run_test(self): blocks = [] # Create extra blocks for later - for b in xrange(20): + for b in range(20): blocks.append(create_block(tip, create_coinbase(height), block_time)) blocks[-1].solve() tip = blocks[-1].sha256 @@ -507,7 +508,7 @@ def run_test(self): with mininode_lock: assert_equal(test_node.last_getdata, None) - print "Part 4: success!" + print("Part 4: success!") # Finally, check that the inv node never received a getdata request, # throughout the test diff --git a/qa/rpc-tests/signmessages.py b/qa/rpc-tests/signmessages.py new file mode 100755 index 0000000000000..31b6f14a26562 --- /dev/null +++ b/qa/rpc-tests/signmessages.py @@ -0,0 +1,41 @@ +#!/usr/bin/env python3 +# Copyright (c) 2016 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +from test_framework.test_framework import BitcoinTestFramework +from test_framework.util import * + + +class SignMessagesTest(BitcoinTestFramework): + """Tests RPC commands for signing and verifying messages.""" + + def __init__(self): + super().__init__() + self.setup_clean_chain = True + self.num_nodes = 1 + + def setup_network(self, split=False): + self.nodes = start_nodes(self.num_nodes, self.options.tmpdir) + self.is_network_split = False + + def run_test(self): + message = 'This is just a test message' + + # Test the signing with a privkey + privKey = 'cUeKHd5orzT3mz8P9pxyREHfsWtVfgsfDjiZZBcjUBAaGk1BTj7N' + address = 'mpLQjfK79b7CCV4VMJWEWAj5Mpx8Up5zxB' + signature = self.nodes[0].signmessagewithprivkey(privKey, message) + + # Verify the message + assert(self.nodes[0].verifymessage(address, signature, message)) + + # Test the signing with an address with wallet + address = self.nodes[0].getnewaddress() + signature = self.nodes[0].signmessage(address, message) + + # Verify the message + assert(self.nodes[0].verifymessage(address, signature, message)) + +if __name__ == '__main__': + SignMessagesTest().main() diff --git a/qa/rpc-tests/signrawtransactions.py b/qa/rpc-tests/signrawtransactions.py index d51d6ee6106e0..c61a280616cf7 100755 --- a/qa/rpc-tests/signrawtransactions.py +++ b/qa/rpc-tests/signrawtransactions.py @@ -1,5 +1,5 @@ -#!/usr/bin/env python2 -# Copyright (c) 2015 The Bitcoin Core developers +#!/usr/bin/env python3 +# Copyright (c) 2015-2016 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -10,12 +10,13 @@ class SignRawTransactionsTest(BitcoinTestFramework): """Tests transaction signing via RPC command "signrawtransaction".""" - def setup_chain(self): - print('Initializing test directory ' + self.options.tmpdir) - initialize_chain_clean(self.options.tmpdir, 1) + def __init__(self): + super().__init__() + self.setup_clean_chain = True + self.num_nodes = 1 def setup_network(self, split=False): - self.nodes = start_nodes(1, self.options.tmpdir) + self.nodes = start_nodes(self.num_nodes, self.options.tmpdir) self.is_network_split = False def successful_signing_test(self): diff --git a/qa/rpc-tests/smartfees.py b/qa/rpc-tests/smartfees.py index 2c064ad8a03a9..d76fba4b075ce 100755 --- a/qa/rpc-tests/smartfees.py +++ b/qa/rpc-tests/smartfees.py @@ -1,5 +1,5 @@ -#!/usr/bin/env python2 -# Copyright (c) 2014-2015 The Bitcoin Core developers +#!/usr/bin/env python3 +# Copyright (c) 2014-2016 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -7,6 +7,7 @@ # Test fee estimation code # +from collections import OrderedDict from test_framework.test_framework import BitcoinTestFramework from test_framework.util import * @@ -22,7 +23,7 @@ def small_txpuzzle_randfee(from_node, conflist, unconflist, amount, min_fee, fee_increment): ''' Create and send a transaction with a random fee. - The transaction pays to a trival P2SH script, and assumes that its inputs + The transaction pays to a trivial P2SH script, and assumes that its inputs are of the same form. The function takes a list of confirmed outputs and unconfirmed outputs and attempts to use the confirmed list first for its inputs. @@ -49,10 +50,10 @@ def small_txpuzzle_randfee(from_node, conflist, unconflist, amount, min_fee, fee if total_in <= amount + fee: raise RuntimeError("Insufficient funds: need %d, have %d"%(amount+fee, total_in)) outputs = {} - outputs[P2SH_1] = total_in - amount - fee - outputs[P2SH_2] = amount + outputs = OrderedDict([(P2SH_1, total_in - amount - fee), + (P2SH_2, amount)]) rawtx = from_node.createrawtransaction(inputs, outputs) - # Createrawtransaction constructions a transaction that is ready to be signed + # createrawtransaction constructs a transaction that is ready to be signed. # These transactions don't need to be signed, but we still have to insert the ScriptSig # that will satisfy the ScriptPubKey. completetx = rawtx[0:10] @@ -78,12 +79,10 @@ def split_inputs(from_node, txins, txouts, initial_split = False): ''' prevtxout = txins.pop() inputs = [] - outputs = {} inputs.append({ "txid" : prevtxout["txid"], "vout" : prevtxout["vout"] }) half_change = satoshi_round(prevtxout["amount"]/2) rem_change = prevtxout["amount"] - half_change - Decimal("0.00001000") - outputs[P2SH_1] = half_change - outputs[P2SH_2] = rem_change + outputs = OrderedDict([(P2SH_1, half_change), (P2SH_2, rem_change)]) rawtx = from_node.createrawtransaction(inputs, outputs) # If this is the initial split we actually need to sign the transaction # Otherwise we just need to insert the property ScriptSig @@ -146,6 +145,11 @@ def check_estimates(node, fees_seen, max_invalid, print_estimates = True): class EstimateFeeTest(BitcoinTestFramework): + def __init__(self): + super().__init__() + self.num_nodes = 3 + self.setup_clean_chain = False + def setup_network(self): ''' We'll setup the network to have 3 nodes that all mine with different parameters. @@ -224,7 +228,7 @@ def transact_and_mine(self, numblocks, mining_node): sync_mempools(self.nodes[0:3],.1) mined = mining_node.getblock(mining_node.generate(1)[0],True)["tx"] sync_blocks(self.nodes[0:3],.1) - #update which txouts are confirmed + # update which txouts are confirmed newmem = [] for utx in self.memutxo: if utx["txid"] in mined: @@ -239,7 +243,7 @@ def run_test(self): self.confutxo = self.txouts # Start with the set of confirmed txouts after splitting print("Will output estimates for 1/2/3/6/15/25 blocks") - for i in xrange(2): + for i in range(2): print("Creating transactions and mining them with a block size that can't keep up") # Create transactions and mine 10 small blocks with node 2, but create txs faster than we can mine self.transact_and_mine(10, self.nodes[2]) diff --git a/qa/rpc-tests/test_framework/authproxy.py b/qa/rpc-tests/test_framework/authproxy.py index e5f7ab3656167..95b2be658cffe 100644 --- a/qa/rpc-tests/test_framework/authproxy.py +++ b/qa/rpc-tests/test_framework/authproxy.py @@ -124,6 +124,11 @@ def _request(self, method, path, postdata): return self._get_response() else: raise + except BrokenPipeError: + # Python 3.5+ raises this instead of BadStatusLine when the connection was reset + self.__conn.close() + self.__conn.request(method, path, postdata, headers) + return self._get_response() def __call__(self, *args): AuthServiceProxy.__id_count += 1 diff --git a/qa/rpc-tests/test_framework/bignum.py b/qa/rpc-tests/test_framework/bignum.py index b0c58ccd47447..ef800e4d57369 100644 --- a/qa/rpc-tests/test_framework/bignum.py +++ b/qa/rpc-tests/test_framework/bignum.py @@ -1,16 +1,15 @@ -# +#!/usr/bin/env python3 # # bignum.py # # This file is copied from python-bitcoinlib. # -# Distributed under the MIT/X11 software license, see the accompanying +# Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. # """Bignum routines""" -from __future__ import absolute_import, division, print_function, unicode_literals import struct diff --git a/qa/rpc-tests/test_framework/blockstore.py b/qa/rpc-tests/test_framework/blockstore.py index 73d9ffbb2fe05..6120dd574b5dd 100644 --- a/qa/rpc-tests/test_framework/blockstore.py +++ b/qa/rpc-tests/test_framework/blockstore.py @@ -1,31 +1,43 @@ +#!/usr/bin/env python3 # BlockStore: a helper class that keeps a map of blocks and implements # helper functions for responding to getheaders and getdata, # and for constructing a getheaders message # from .mininode import * -import dbm from io import BytesIO +import dbm.ndbm class BlockStore(object): def __init__(self, datadir): - self.blockDB = dbm.open(datadir + "/blocks", 'c') - self.currentBlock = 0L + self.blockDB = dbm.ndbm.open(datadir + "/blocks", 'c') + self.currentBlock = 0 self.headers_map = dict() - + def close(self): self.blockDB.close() + def erase(self, blockhash): + del self.blockDB[repr(blockhash)] + + # lookup an entry and return the item as raw bytes def get(self, blockhash): - serialized_block = None + value = None try: - serialized_block = self.blockDB[repr(blockhash)] + value = self.blockDB[repr(blockhash)] except KeyError: return None - f = BytesIO(serialized_block) - ret = CBlock() - ret.deserialize(f) - ret.calc_sha256() + return value + + # lookup an entry and return it as a CBlock + def get_block(self, blockhash): + ret = None + serialized_block = self.get(blockhash) + if serialized_block is not None: + f = BytesIO(serialized_block) + ret = CBlock() + ret.deserialize(f) + ret.calc_sha256() return ret def get_header(self, blockhash): @@ -67,20 +79,23 @@ def add_block(self, block): try: self.blockDB[repr(block.sha256)] = bytes(block.serialize()) except TypeError as e: - print "Unexpected error: ", sys.exc_info()[0], e.args + print("Unexpected error: ", sys.exc_info()[0], e.args) self.currentBlock = block.sha256 self.headers_map[block.sha256] = CBlockHeader(block) def add_header(self, header): self.headers_map[header.sha256] = header + # lookup the hashes in "inv", and return p2p messages for delivering + # blocks found. def get_blocks(self, inv): responses = [] for i in inv: if (i.type == 2): # MSG_BLOCK - block = self.get(i.hash) - if block is not None: - responses.append(msg_block(block)) + data = self.get(i.hash) + if data is not None: + # Use msg_generic to avoid re-serialization + responses.append(msg_generic(b"block", data)) return responses def get_locator(self, current_tip=None): @@ -89,11 +104,11 @@ def get_locator(self, current_tip=None): r = [] counter = 0 step = 1 - lastBlock = self.get(current_tip) + lastBlock = self.get_block(current_tip) while lastBlock is not None: r.append(lastBlock.hashPrevBlock) for i in range(step): - lastBlock = self.get(lastBlock.hashPrevBlock) + lastBlock = self.get_block(lastBlock.hashPrevBlock) if lastBlock is None: break counter += 1 @@ -105,21 +120,28 @@ def get_locator(self, current_tip=None): class TxStore(object): def __init__(self, datadir): - self.txDB = dbm.open(datadir + "/transactions", 'c') + self.txDB = dbm.ndbm.open(datadir + "/transactions", 'c') def close(self): self.txDB.close() + # lookup an entry and return the item as raw bytes def get(self, txhash): - serialized_tx = None + value = None try: - serialized_tx = self.txDB[repr(txhash)] + value = self.txDB[repr(txhash)] except KeyError: return None - f = BytesIO(serialized_tx) - ret = CTransaction() - ret.deserialize(f) - ret.calc_sha256() + return value + + def get_transaction(self, txhash): + ret = None + serialized_tx = self.get(txhash) + if serialized_tx is not None: + f = BytesIO(serialized_tx) + ret = CTransaction() + ret.deserialize(f) + ret.calc_sha256() return ret def add_transaction(self, tx): @@ -127,7 +149,7 @@ def add_transaction(self, tx): try: self.txDB[repr(tx.sha256)] = bytes(tx.serialize()) except TypeError as e: - print "Unexpected error: ", sys.exc_info()[0], e.args + print("Unexpected error: ", sys.exc_info()[0], e.args) def get_transactions(self, inv): responses = [] @@ -135,5 +157,5 @@ def get_transactions(self, inv): if (i.type == 1): # MSG_TX tx = self.get(i.hash) if tx is not None: - responses.append(msg_tx(tx)) + responses.append(msg_generic(b"tx", tx)) return responses diff --git a/qa/rpc-tests/test_framework/blocktools.py b/qa/rpc-tests/test_framework/blocktools.py index b06a19d56c202..df4fe13e5c1c6 100644 --- a/qa/rpc-tests/test_framework/blocktools.py +++ b/qa/rpc-tests/test_framework/blocktools.py @@ -1,8 +1,8 @@ +#!/usr/bin/env python3 # blocktools.py - utilities for manipulating blocks and transactions -# Copyright (c) 2015 The Bitcoin Core developers -# Distributed under the MIT/X11 software license, see the accompanying +# Copyright (c) 2015-2016 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. -# from .mininode import * from .script import CScript, OP_TRUE, OP_CHECKSIG, OP_RETURN @@ -79,12 +79,27 @@ def create_coinbase(height, pubkey = None): coinbase.calc_sha256() return coinbase -# Create a transaction with an anyone-can-spend output, that spends the -# nth output of prevtx. -def create_transaction(prevtx, n, sig, value): +# Create a transaction. +# If the scriptPubKey is not specified, make it anyone-can-spend. +def create_transaction(prevtx, n, sig, value, scriptPubKey=CScript()): tx = CTransaction() assert(n < len(prevtx.vout)) tx.vin.append(CTxIn(COutPoint(prevtx.sha256, n), sig, 0xffffffff)) - tx.vout.append(CTxOut(value, b"")) + tx.vout.append(CTxOut(value, scriptPubKey)) tx.calc_sha256() return tx + +def get_legacy_sigopcount_block(block, fAccurate=True): + count = 0 + for tx in block.vtx: + count += get_legacy_sigopcount_tx(tx, fAccurate) + return count + +def get_legacy_sigopcount_tx(tx, fAccurate=True): + count = 0 + for i in tx.vout: + count += i.scriptPubKey.GetSigOpCount(fAccurate) + for j in tx.vin: + # scriptSig might be of type bytes, so convert to CScript for the moment + count += CScript(j.scriptSig).GetSigOpCount(fAccurate) + return count diff --git a/qa/rpc-tests/test_framework/comptool.py b/qa/rpc-tests/test_framework/comptool.py index 17626cf8dada4..7c92d3f828917 100755 --- a/qa/rpc-tests/test_framework/comptool.py +++ b/qa/rpc-tests/test_framework/comptool.py @@ -1,8 +1,7 @@ -#!/usr/bin/env python2 -# Copyright (c) 2015 The Bitcoin Core developers -# Distributed under the MIT/X11 software license, see the accompanying +#!/usr/bin/env python3 +# Copyright (c) 2015-2016 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. -# from .mininode import * from .blockstore import BlockStore, TxStore @@ -259,10 +258,10 @@ def check_results(self, blockhash, outcome): if c.cb.bestblockhash == blockhash: return False if blockhash not in c.cb.block_reject_map: - print 'Block not in reject map: %064x' % (blockhash) + print('Block not in reject map: %064x' % (blockhash)) return False if not outcome.match(c.cb.block_reject_map[blockhash]): - print 'Block rejected with %s instead of expected %s: %064x' % (c.cb.block_reject_map[blockhash], outcome, blockhash) + print('Block rejected with %s instead of expected %s: %064x' % (c.cb.block_reject_map[blockhash], outcome, blockhash)) return False elif ((c.cb.bestblockhash == blockhash) != outcome): # print c.cb.bestblockhash, blockhash, outcome @@ -287,10 +286,10 @@ def check_mempool(self, txhash, outcome): if txhash in c.cb.lastInv: return False if txhash not in c.cb.tx_reject_map: - print 'Tx not in reject map: %064x' % (txhash) + print('Tx not in reject map: %064x' % (txhash)) return False if not outcome.match(c.cb.tx_reject_map[txhash]): - print 'Tx rejected with %s instead of expected %s: %064x' % (c.cb.tx_reject_map[txhash], outcome, txhash) + print('Tx rejected with %s instead of expected %s: %064x' % (c.cb.tx_reject_map[txhash], outcome, txhash)) return False elif ((txhash in c.cb.lastInv) != outcome): # print c.rpc.getrawmempool(), c.cb.lastInv @@ -393,7 +392,7 @@ def run(self): if (not self.check_mempool(tx.sha256, tx_outcome)): raise AssertionError("Mempool test failed at test %d" % test_number) - print "Test %d: PASS" % test_number, [ c.rpc.getblockcount() for c in self.connections ] + print("Test %d: PASS" % test_number, [ c.rpc.getblockcount() for c in self.connections ]) test_number += 1 [ c.disconnect_node() for c in self.connections ] diff --git a/qa/rpc-tests/test_framework/coverage.py b/qa/rpc-tests/test_framework/coverage.py index d21a001b6efe0..23fce61014040 100644 --- a/qa/rpc-tests/test_framework/coverage.py +++ b/qa/rpc-tests/test_framework/coverage.py @@ -1,8 +1,7 @@ -#!/usr/bin/env python2 -# Copyright (c) 2015 The Bitcoin Core developers -# Distributed under the MIT/X11 software license, see the accompanying +#!/usr/bin/env python3 +# Copyright (c) 2015-2016 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. -# """ This module contains utilities for doing coverage analysis on the RPC diff --git a/qa/rpc-tests/test_framework/mininode.py b/qa/rpc-tests/test_framework/mininode.py index 892b645a2a3f4..782a41a82dd9c 100755 --- a/qa/rpc-tests/test_framework/mininode.py +++ b/qa/rpc-tests/test_framework/mininode.py @@ -1,7 +1,12 @@ -# mininode.py - Bitcoin P2P network half-a-node -# -# Distributed under the MIT/X11 software license, see the accompanying +#!/usr/bin/env python3 +# Copyright (c) 2010 ArtForz -- public domain half-a-node +# Copyright (c) 2012 Jeff Garzik +# Copyright (c) 2010-2016 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. + +# +# mininode.py - Bitcoin P2P network half-a-node # # This python code was modified from ArtForz' public domain half-a-node, as # found in the mini-node branch of http://github.com/jgarzik/pynode. @@ -34,12 +39,12 @@ BIP0031_VERSION = 60000 MY_VERSION = 60001 # past bip-31 for ping/pong -MY_SUBVERSION = b"/python-mininode-tester:0.0.2/" +MY_SUBVERSION = b"/python-mininode-tester:0.0.3/" MAX_INV_SZ = 50000 MAX_BLOCK_SIZE = 1000000 -COIN = 100000000L # 1 btc in satoshis +COIN = 100000000 # 1 btc in satoshis NODE_NETWORK = (1 << 0) NODE_GETUTXO = (1 << 1) @@ -80,44 +85,42 @@ def deser_string(f): nit = struct.unpack(">= 32 return rs def uint256_from_str(s): - r = 0L + r = 0 t = struct.unpack("> 24) & 0xFF - v = (c & 0xFFFFFFL) << (8 * (nbytes - 3)) + v = (c & 0xFFFFFF) << (8 * (nbytes - 3)) return v @@ -130,7 +133,7 @@ def deser_vector(f, c): elif nit == 255: nit = struct.unpack(" 1: newhashes = [] - for i in xrange(0, len(hashes), 2): + for i in range(0, len(hashes), 2): i2 = min(i+1, len(hashes)-1) newhashes.append(hash256(hashes[i] + hashes[i2])) hashes = newhashes @@ -768,7 +771,7 @@ def deserialize(self, f): self.vchSig = deser_string(f) def serialize(self): - r = "" + r = b"" r += ser_string(self.vchMsg) r += ser_string(self.vchSig) return r @@ -925,7 +928,7 @@ class msg_getblocks(object): def __init__(self): self.locator = CBlockLocator() - self.hashstop = 0L + self.hashstop = 0 def deserialize(self, f): self.locator = CBlockLocator() @@ -982,13 +985,25 @@ def serialize(self): def __repr__(self): return "msg_block(block=%s)" % (repr(self.block)) +# for cases where a user needs tighter control over what is sent over the wire +# note that the user must supply the name of the command, and the data +class msg_generic(object): + def __init__(self, command, data=None): + self.command = command + self.data = data + + def serialize(self): + return self.data + + def __repr__(self): + return "msg_generic()" + class msg_witness_block(msg_block): def serialize(self): r = self.block.serialize(with_witness=True) return r - class msg_getaddr(object): command = b"getaddr" @@ -1024,7 +1039,7 @@ def __repr__(self): class msg_ping(object): command = b"ping" - def __init__(self, nonce=0L): + def __init__(self, nonce=0): self.nonce = nonce def deserialize(self, f): @@ -1097,7 +1112,7 @@ class msg_getheaders(object): def __init__(self): self.locator = CBlockLocator() - self.hashstop = 0L + self.hashstop = 0 def deserialize(self, f): self.locator = CBlockLocator() @@ -1139,25 +1154,28 @@ def __repr__(self): class msg_reject(object): command = b"reject" + REJECT_MALFORMED = 1 def __init__(self): self.message = b"" self.code = 0 - self.reason = "" - self.data = 0L + self.reason = b"" + self.data = 0 def deserialize(self, f): self.message = deser_string(f) self.code = struct.unpack("= 209: @@ -1332,7 +1350,6 @@ class NodeConn(asyncore.dispatcher): "mainnet": b"\xf9\xbe\xb4\xd9", # mainnet "testnet3": b"\x0b\x11\x09\x07", # testnet3 "regtest": b"\xfa\xbf\xb5\xda", # regtest - "segnet": b"\x2e\x96\xea\xca" # segnet } def __init__(self, dstaddr, dstport, rpc, callback, net="regtest", services=NODE_NETWORK): @@ -1360,8 +1377,8 @@ def __init__(self, dstaddr, dstport, rpc, callback, net="regtest", services=NODE vt.addrFrom.ip = "0.0.0.0" vt.addrFrom.port = 0 self.send_message(vt, True) - print 'MiniNode: Connecting to Bitcoin Node IP # ' + dstaddr + ':' \ - + str(dstport) + print('MiniNode: Connecting to Bitcoin Node IP # ' + dstaddr + ':' \ + + str(dstport)) try: self.connect((dstaddr, dstport)) @@ -1454,7 +1471,9 @@ def got_data(self): self.show_debug_msg("Unknown command: '" + command + "' " + repr(msg)) except Exception as e: - print 'got_data:', repr(e) + print('got_data:', repr(e)) + # import traceback + # traceback.print_tb(sys.exc_info()[2]) def send_message(self, message, pushbuf=False): if self.state != "connected" and not pushbuf: diff --git a/qa/rpc-tests/test_framework/netutil.py b/qa/rpc-tests/test_framework/netutil.py index 52a7ab74890f1..573b06772d7b3 100644 --- a/qa/rpc-tests/test_framework/netutil.py +++ b/qa/rpc-tests/test_framework/netutil.py @@ -1,5 +1,5 @@ -#!/usr/bin/env python2 -# Copyright (c) 2014-2015 The Bitcoin Core developers +#!/usr/bin/env python3 +# Copyright (c) 2014-2016 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/qa/rpc-tests/test_framework/script.py b/qa/rpc-tests/test_framework/script.py index 2907b0ad28c90..7678228c42bb5 100644 --- a/qa/rpc-tests/test_framework/script.py +++ b/qa/rpc-tests/test_framework/script.py @@ -1,18 +1,19 @@ +#!/usr/bin/env python3 +# Copyright (c) 2015-2016 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + # # script.py # # This file is modified from python-bitcoinlib. # -# Distributed under the MIT/X11 software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. -# """Scripts Functionality to build scripts, as well as SignatureHash(). """ -from __future__ import absolute_import, division, print_function, unicode_literals from .mininode import CTransaction, CTxOut, sha256, hash256, uint256_from_str, ser_uint256, ser_string from binascii import hexlify @@ -663,7 +664,7 @@ def __coerce_instance(cls, other): other = bchr(CScriptOp(OP_0)) else: other = CScriptNum.encode(other) - elif isinstance(other, (int, long)): + elif isinstance(other, int): if 0 <= other <= 16: other = bytes(bchr(CScriptOp.encode_op_n(other))) elif other == -1: diff --git a/qa/rpc-tests/test_framework/socks5.py b/qa/rpc-tests/test_framework/socks5.py index f725d97701747..372f5ed605e4a 100644 --- a/qa/rpc-tests/test_framework/socks5.py +++ b/qa/rpc-tests/test_framework/socks5.py @@ -1,11 +1,12 @@ -# Copyright (c) 2015 The Bitcoin Core developers +#!/usr/bin/env python3 +# Copyright (c) 2015-2016 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. ''' Dummy Socks5 server for testing. ''' -from __future__ import print_function, division, unicode_literals -import socket, threading, Queue + +import socket, threading, queue import traceback, sys ### Protocol constants @@ -132,7 +133,7 @@ def __init__(self, conf): self.s.listen(5) self.running = False self.thread = None - self.queue = Queue.Queue() # report connections and exceptions to client + self.queue = queue.Queue() # report connections and exceptions to client def run(self): while self.running: diff --git a/qa/rpc-tests/test_framework/test_framework.py b/qa/rpc-tests/test_framework/test_framework.py index 3b08cd1384d4e..30e8b5755de39 100755 --- a/qa/rpc-tests/test_framework/test_framework.py +++ b/qa/rpc-tests/test_framework/test_framework.py @@ -1,21 +1,20 @@ -#!/usr/bin/env python2 -# Copyright (c) 2014-2015 The Bitcoin Core developers +#!/usr/bin/env python3 +# Copyright (c) 2014-2016 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. # Base class for RPC testing -# Add python-bitcoinrpc to module search path: +import logging +import optparse import os import sys - import shutil import tempfile import traceback from .util import ( initialize_chain, - assert_equal, start_nodes, connect_nodes_bi, sync_blocks, @@ -25,27 +24,33 @@ enable_coverage, check_json_precision, initialize_chain_clean, + PortSeed, ) -from .authproxy import AuthServiceProxy, JSONRPCException +from .authproxy import JSONRPCException class BitcoinTestFramework(object): - # These may be over-ridden by subclasses: + def __init__(self): + self.num_nodes = 4 + self.setup_clean_chain = False + self.nodes = None + def run_test(self): - for node in self.nodes: - assert_equal(node.getblockcount(), 200) - assert_equal(node.getbalance(), 25*50) + raise NotImplementedError def add_options(self, parser): pass def setup_chain(self): print("Initializing test directory "+self.options.tmpdir) - initialize_chain(self.options.tmpdir) + if self.setup_clean_chain: + initialize_chain_clean(self.options.tmpdir, self.num_nodes) + else: + initialize_chain(self.options.tmpdir, self.num_nodes) def setup_nodes(self): - return start_nodes(4, self.options.tmpdir) + return start_nodes(self.num_nodes, self.options.tmpdir) def setup_network(self, split = False): self.nodes = self.setup_nodes() @@ -95,31 +100,35 @@ def join_network(self): self.setup_network(False) def main(self): - import optparse parser = optparse.OptionParser(usage="%prog [options]") parser.add_option("--nocleanup", dest="nocleanup", default=False, action="store_true", help="Leave bitcoinds and test.* datadir on exit or error") parser.add_option("--noshutdown", dest="noshutdown", default=False, action="store_true", help="Don't stop bitcoinds after the test execution") - parser.add_option("--srcdir", dest="srcdir", default="../../src", + parser.add_option("--srcdir", dest="srcdir", default=os.path.normpath(os.path.dirname(os.path.realpath(__file__))+"/../../../src"), help="Source directory containing bitcoind/bitcoin-cli (default: %default)") parser.add_option("--tmpdir", dest="tmpdir", default=tempfile.mkdtemp(prefix="test"), help="Root directory for datadirs") parser.add_option("--tracerpc", dest="trace_rpc", default=False, action="store_true", help="Print out all RPC calls as they are made") + parser.add_option("--portseed", dest="port_seed", default=os.getpid(), type='int', + help="The seed to use for assigning port numbers (default: current process id)") parser.add_option("--coveragedir", dest="coveragedir", help="Write tested RPC commands into this directory") self.add_options(parser) (self.options, self.args) = parser.parse_args() + self.options.tmpdir += '/' + str(self.options.port_seed) + if self.options.trace_rpc: - import logging - logging.basicConfig(level=logging.DEBUG) + logging.basicConfig(level=logging.DEBUG, stream=sys.stdout) if self.options.coveragedir: enable_coverage(self.options.coveragedir) + PortSeed.n = self.options.port_seed + os.environ['PATH'] = self.options.srcdir+":"+self.options.srcdir+"/qt:"+os.environ['PATH'] check_json_precision() @@ -140,14 +149,16 @@ def main(self): print("JSONRPC error: "+e.error['message']) traceback.print_tb(sys.exc_info()[2]) except AssertionError as e: - print("Assertion failed: "+ str(e)) + print("Assertion failed: " + str(e)) traceback.print_tb(sys.exc_info()[2]) except KeyError as e: print("key not found: "+ str(e)) traceback.print_tb(sys.exc_info()[2]) except Exception as e: - print("Unexpected exception caught during testing: "+str(e)) + print("Unexpected exception caught during testing: " + repr(e)) traceback.print_tb(sys.exc_info()[2]) + except KeyboardInterrupt as e: + print("Exiting after " + repr(e)) if not self.options.noshutdown: print("Stopping nodes") @@ -156,9 +167,11 @@ def main(self): else: print("Note: bitcoinds were not stopped and may still be running") - if not self.options.nocleanup and not self.options.noshutdown: + if not self.options.nocleanup and not self.options.noshutdown and success: print("Cleaning up") shutil.rmtree(self.options.tmpdir) + else: + print("Not cleaning up dir %s" % self.options.tmpdir) if success: print("Tests successful") @@ -176,9 +189,10 @@ def main(self): class ComparisonTestFramework(BitcoinTestFramework): - # Can override the num_nodes variable to indicate how many nodes to run. def __init__(self): + super().__init__() self.num_nodes = 2 + self.setup_clean_chain = True def add_options(self, parser): parser.add_option("--testbinary", dest="testbinary", @@ -188,10 +202,6 @@ def add_options(self, parser): default=os.getenv("BITCOIND", "bitcoind"), help="bitcoind binary to use for reference nodes (if any)") - def setup_chain(self): - print "Initializing test directory "+self.options.tmpdir - initialize_chain_clean(self.options.tmpdir, self.num_nodes) - def setup_network(self): self.nodes = start_nodes( self.num_nodes, self.options.tmpdir, diff --git a/qa/rpc-tests/test_framework/util.py b/qa/rpc-tests/test_framework/util.py index 8b720b54a1a3d..fc66ef287d715 100644 --- a/qa/rpc-tests/test_framework/util.py +++ b/qa/rpc-tests/test_framework/util.py @@ -1,4 +1,5 @@ -# Copyright (c) 2014-2015 The Bitcoin Core developers +#!/usr/bin/env python3 +# Copyright (c) 2014-2016 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -7,7 +8,6 @@ # Helpful routines for regression testing # -# Add python-bitcoinrpc to module search path: import os import sys @@ -15,6 +15,7 @@ from base64 import b64encode from decimal import Decimal, ROUND_DOWN import json +import http.client import random import shutil import subprocess @@ -27,6 +28,20 @@ COVERAGE_DIR = None +# The maximum number of nodes a single test can spawn +MAX_NODES = 8 +# Don't assign rpc or p2p ports lower than this +PORT_MIN = 11000 +# The number of ports to "reserve" for p2p and rpc, each +PORT_RANGE = 5000 + +BITCOIND_PROC_WAIT_TIMEOUT = 60 + + +class PortSeed: + # Must be initialized with a unique integer for each process + n = None + #Set Mocktime default to OFF. #MOCKTIME is only needed for scripts that use the #cached version of the blockchain. If the cached @@ -81,9 +96,11 @@ def get_rpc_proxy(url, node_number, timeout=None): def p2p_port(n): - return 11000 + n + os.getpid()%999 + assert(n <= MAX_NODES) + return PORT_MIN + n + (MAX_NODES * PortSeed.n) % (PORT_RANGE - 1 - MAX_NODES) + def rpc_port(n): - return 12000 + n + os.getpid()%999 + return PORT_MIN + PORT_RANGE + n + (MAX_NODES * PortSeed.n) % (PORT_RANGE - 1 - MAX_NODES) def check_json_precision(): """Make sure json library being used does not lose precision converting BTC values""" @@ -104,30 +121,34 @@ def hex_str_to_bytes(hex_str): def str_to_b64str(string): return b64encode(string.encode('utf-8')).decode('ascii') -def sync_blocks(rpc_connections, wait=1): +def sync_blocks(rpc_connections, wait=1, timeout=60): """ - Wait until everybody has the same block count + Wait until everybody has the same tip """ - while True: - counts = [ x.getblockcount() for x in rpc_connections ] - if counts == [ counts[0] ]*len(counts): - break + while timeout > 0: + tips = [ x.getbestblockhash() for x in rpc_connections ] + if tips == [ tips[0] ]*len(tips): + return True time.sleep(wait) + timeout -= wait + raise AssertionError("Block sync failed") -def sync_mempools(rpc_connections, wait=1): +def sync_mempools(rpc_connections, wait=1, timeout=60): """ Wait until everybody has the same transactions in their memory pools """ - while True: + while timeout > 0: pool = set(rpc_connections[0].getrawmempool()) num_match = 1 for i in range(1, len(rpc_connections)): if set(rpc_connections[i].getrawmempool()) == pool: num_match = num_match+1 if num_match == len(rpc_connections): - break + return True time.sleep(wait) + timeout -= wait + raise AssertionError("Mempool sync failed") bitcoind_processes = {} @@ -167,45 +188,52 @@ def wait_for_bitcoind_start(process, url, i): raise # unkown JSON RPC exception time.sleep(0.25) -def initialize_chain(test_dir): +def initialize_chain(test_dir, num_nodes): """ - Create (or copy from cache) a 200-block-long chain and - 4 wallets. + Create a cache of a 200-block-long chain (with wallet) for MAX_NODES + Afterward, create num_nodes copies from the cache """ - if (not os.path.isdir(os.path.join("cache","node0")) - or not os.path.isdir(os.path.join("cache","node1")) - or not os.path.isdir(os.path.join("cache","node2")) - or not os.path.isdir(os.path.join("cache","node3"))): + assert num_nodes <= MAX_NODES + create_cache = False + for i in range(MAX_NODES): + if not os.path.isdir(os.path.join('cache', 'node'+str(i))): + create_cache = True + break + + if create_cache: #find and delete old cache directories if any exist - for i in range(4): + for i in range(MAX_NODES): if os.path.isdir(os.path.join("cache","node"+str(i))): shutil.rmtree(os.path.join("cache","node"+str(i))) # Create cache directories, run bitcoinds: - for i in range(4): + for i in range(MAX_NODES): datadir=initialize_datadir("cache", i) args = [ os.getenv("BITCOIND", "bitcoind"), "-server", "-keypool=1", "-datadir="+datadir, "-discover=0" ] if i > 0: args.append("-connect=127.0.0.1:"+str(p2p_port(0))) bitcoind_processes[i] = subprocess.Popen(args) if os.getenv("PYTHON_DEBUG", ""): - print "initialize_chain: bitcoind started, waiting for RPC to come up" + print("initialize_chain: bitcoind started, waiting for RPC to come up") wait_for_bitcoind_start(bitcoind_processes[i], rpc_url(i), i) if os.getenv("PYTHON_DEBUG", ""): - print "initialize_chain: RPC succesfully started" + print("initialize_chain: RPC succesfully started") rpcs = [] - for i in range(4): + for i in range(MAX_NODES): try: rpcs.append(get_rpc_proxy(rpc_url(i), i)) except: sys.stderr.write("Error connecting to "+url+"\n") sys.exit(1) - # Create a 200-block-long chain; each of the 4 nodes + # Create a 200-block-long chain; each of the 4 first nodes # gets 25 mature blocks and 25 immature. + # Note: To preserve compatibility with older versions of + # initialize_chain, only 4 nodes will generate coins. + # # blocks are created with timestamps 10 minutes apart # starting from 2010 minutes in the past enable_mocktime() @@ -223,13 +251,13 @@ def initialize_chain(test_dir): stop_nodes(rpcs) wait_bitcoinds() disable_mocktime() - for i in range(4): + for i in range(MAX_NODES): os.remove(log_filename("cache", i, "debug.log")) os.remove(log_filename("cache", i, "db.log")) os.remove(log_filename("cache", i, "peers.dat")) os.remove(log_filename("cache", i, "fee_estimates.dat")) - for i in range(4): + for i in range(num_nodes): from_dir = os.path.join("cache", "node"+str(i)) to_dir = os.path.join(test_dir, "node"+str(i)) shutil.copytree(from_dir, to_dir) @@ -275,11 +303,11 @@ def start_node(i, dirname, extra_args=None, rpchost=None, timewait=None, binary= if extra_args is not None: args.extend(extra_args) bitcoind_processes[i] = subprocess.Popen(args) if os.getenv("PYTHON_DEBUG", ""): - print "start_node: bitcoind started, waiting for RPC to come up" + print("start_node: bitcoind started, waiting for RPC to come up") url = rpc_url(i, rpchost) wait_for_bitcoind_start(bitcoind_processes[i], url, i) if os.getenv("PYTHON_DEBUG", ""): - print "start_node: RPC succesfully started" + print("start_node: RPC succesfully started") proxy = get_rpc_proxy(url, i, timeout=timewait) if COVERAGE_DIR: @@ -291,8 +319,8 @@ def start_nodes(num_nodes, dirname, extra_args=None, rpchost=None, binary=None): """ Start multiple bitcoinds, return RPC connections to them """ - if extra_args is None: extra_args = [ None for i in range(num_nodes) ] - if binary is None: binary = [ None for i in range(num_nodes) ] + if extra_args is None: extra_args = [ None for _ in range(num_nodes) ] + if binary is None: binary = [ None for _ in range(num_nodes) ] rpcs = [] try: for i in range(num_nodes): @@ -306,13 +334,19 @@ def log_filename(dirname, n_node, logname): return os.path.join(dirname, "node"+str(n_node), "regtest", logname) def stop_node(node, i): - node.stop() - bitcoind_processes[i].wait() + try: + node.stop() + except http.client.CannotSendRequest as e: + print("WARN: Unable to stop node: " + repr(e)) + bitcoind_processes[i].wait(timeout=BITCOIND_PROC_WAIT_TIMEOUT) del bitcoind_processes[i] def stop_nodes(nodes): for node in nodes: - node.stop() + try: + node.stop() + except http.client.CannotSendRequest as e: + print("WARN: Unable to stop node: " + repr(e)) del nodes[:] # Emptying array closes connections as a side effect def set_node_times(nodes, t): @@ -322,7 +356,7 @@ def set_node_times(nodes, t): def wait_bitcoinds(): # Wait for all bitcoinds to cleanly exit for bitcoind in bitcoind_processes.values(): - bitcoind.wait() + bitcoind.wait(timeout=BITCOIND_PROC_WAIT_TIMEOUT) bitcoind_processes.clear() def connect_nodes(from_connection, node_num): @@ -469,7 +503,7 @@ def assert_is_hex_string(string): "Couldn't interpret %r as hexadecimal; raised: %s" % (string, e)) def assert_is_hash_string(string, length=64): - if not isinstance(string, basestring): + if not isinstance(string, str): raise AssertionError("Expected a string, got type %r" % type(string)) elif length and len(string) != length: raise AssertionError( @@ -478,6 +512,35 @@ def assert_is_hash_string(string, length=64): raise AssertionError( "String %r contains invalid characters for a hash." % string) +def assert_array_result(object_array, to_match, expected, should_not_find = False): + """ + Pass in array of JSON objects, a dictionary with key/value pairs + to match against, and another dictionary with expected key/value + pairs. + If the should_not_find flag is true, to_match should not be found + in object_array + """ + if should_not_find == True: + assert_equal(expected, { }) + num_matched = 0 + for item in object_array: + all_match = True + for key,value in to_match.items(): + if item[key] != value: + all_match = False + if not all_match: + continue + elif should_not_find == True: + num_matched = num_matched+1 + for key,value in expected.items(): + if item[key] != value: + raise AssertionError("%s : expected %s=%s"%(str(item), str(key), str(value))) + num_matched = num_matched+1 + if num_matched == 0 and should_not_find != True: + raise AssertionError("No objects matched %s"%(str(to_match))) + if num_matched > 0 and should_not_find == True: + raise AssertionError("Objects were found %s"%(str(to_match))) + def satoshi_round(amount): return Decimal(amount).quantize(Decimal('0.00000001'), rounding=ROUND_DOWN) @@ -491,7 +554,7 @@ def create_confirmed_utxos(fee, node, count): addr2 = node.getnewaddress() if iterations <= 0: return utxos - for i in xrange(iterations): + for i in range(iterations): t = utxos.pop() inputs = [] inputs.append({ "txid" : t["txid"], "vout" : t["vout"]}) @@ -517,11 +580,11 @@ def gen_return_txouts(): # So we have big transactions (and therefore can't fit very many into each block) # create one script_pubkey script_pubkey = "6a4d0200" #OP_RETURN OP_PUSH2 512 bytes - for i in xrange (512): + for i in range (512): script_pubkey = script_pubkey + "01" # concatenate 128 txouts of above script_pubkey which we'll insert before the txout for change txouts = "81" - for k in xrange(128): + for k in range(128): # add txout value txouts = txouts + "0000000000000000" # add length of script_pubkey @@ -543,7 +606,7 @@ def create_tx(node, coinbase, to_address, amount): def create_lots_of_big_transactions(node, txouts, utxos, fee): addr = node.getnewaddress() txids = [] - for i in xrange(len(utxos)): + for i in range(len(utxos)): t = utxos.pop() inputs = [] inputs.append({ "txid" : t["txid"], "vout" : t["vout"]}) diff --git a/qa/rpc-tests/txn_clone.py b/qa/rpc-tests/txn_clone.py index b132aec4b6165..22f850ece64a2 100755 --- a/qa/rpc-tests/txn_clone.py +++ b/qa/rpc-tests/txn_clone.py @@ -1,5 +1,5 @@ -#!/usr/bin/env python2 -# Copyright (c) 2014-2015 The Bitcoin Core developers +#!/usr/bin/env python3 +# Copyright (c) 2014-2016 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -12,6 +12,11 @@ class TxnMallTest(BitcoinTestFramework): + def __init__(self): + super().__init__() + self.num_nodes = 4 + self.setup_clean_chain = False + def add_options(self, parser): parser.add_option("--mineblock", dest="mine_block", default=False, action="store_true", help="Test double-spend of 1-confirmed transaction") diff --git a/qa/rpc-tests/txn_doublespend.py b/qa/rpc-tests/txn_doublespend.py index 8d7f6e505d15b..84944c3c1943a 100755 --- a/qa/rpc-tests/txn_doublespend.py +++ b/qa/rpc-tests/txn_doublespend.py @@ -1,5 +1,5 @@ -#!/usr/bin/env python2 -# Copyright (c) 2014-2015 The Bitcoin Core developers +#!/usr/bin/env python3 +# Copyright (c) 2014-2016 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -12,6 +12,11 @@ class TxnMallTest(BitcoinTestFramework): + def __init__(self): + super().__init__() + self.num_nodes = 4 + self.setup_clean_chain = False + def add_options(self, parser): parser.add_option("--mineblock", dest="mine_block", default=False, action="store_true", help="Test double-spend of 1-confirmed transaction") diff --git a/qa/rpc-tests/wallet.py b/qa/rpc-tests/wallet.py index 8fdcea50b6329..9dda712f4ff08 100755 --- a/qa/rpc-tests/wallet.py +++ b/qa/rpc-tests/wallet.py @@ -1,5 +1,5 @@ -#!/usr/bin/env python2 -# Copyright (c) 2014-2015 The Bitcoin Core developers +#!/usr/bin/env python3 +# Copyright (c) 2014-2016 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -19,9 +19,10 @@ def check_fee_amount(self, curr_balance, balance_with_fee, fee_per_byte, tx_size raise AssertionError("Fee of %s BTC too high! (Should be %s BTC)"%(str(fee), str(target_fee))) return curr_balance - def setup_chain(self): - print("Initializing test directory "+self.options.tmpdir) - initialize_chain_clean(self.options.tmpdir, 4) + def __init__(self): + super().__init__() + self.setup_clean_chain = True + self.num_nodes = 4 def setup_network(self, split=False): self.nodes = start_nodes(3, self.options.tmpdir) @@ -32,7 +33,13 @@ def setup_network(self, split=False): self.sync_all() def run_test (self): - print "Mining blocks..." + + # Check that there's no UTXO on none of the nodes + assert_equal(len(self.nodes[0].listunspent()), 0) + assert_equal(len(self.nodes[1].listunspent()), 0) + assert_equal(len(self.nodes[2].listunspent()), 0) + + print("Mining blocks...") self.nodes[0].generate(1) @@ -48,6 +55,11 @@ def run_test (self): assert_equal(self.nodes[1].getbalance(), 50) assert_equal(self.nodes[2].getbalance(), 0) + # Check that only first and second nodes have UTXOs + assert_equal(len(self.nodes[0].listunspent()), 1) + assert_equal(len(self.nodes[1].listunspent()), 1) + assert_equal(len(self.nodes[2].listunspent()), 0) + # Send 21 BTC from 0 to 2 using sendtoaddress call. self.nodes[0].sendtoaddress(self.nodes[2].getnewaddress(), 11) self.nodes[0].sendtoaddress(self.nodes[2].getnewaddress(), 10) @@ -259,6 +271,32 @@ def run_test (self): except JSONRPCException as e: assert("not an integer" in e.error['message']) + # Import address and private key to check correct behavior of spendable unspents + # 1. Send some coins to generate new UTXO + address_to_import = self.nodes[2].getnewaddress() + txid = self.nodes[0].sendtoaddress(address_to_import, 1) + self.nodes[0].generate(1) + self.sync_all() + + # 2. Import address from node2 to node1 + self.nodes[1].importaddress(address_to_import) + + # 3. Validate that the imported address is watch-only on node1 + assert(self.nodes[1].validateaddress(address_to_import)["iswatchonly"]) + + # 4. Check that the unspents after import are not spendable + assert_array_result(self.nodes[1].listunspent(), + {"address": address_to_import}, + {"spendable": False}) + + # 5. Import private key of the previously imported address on node1 + priv_key = self.nodes[2].dumpprivkey(address_to_import) + self.nodes[1].importprivkey(priv_key) + + # 6. Check that the unspents are now spendable on node1 + assert_array_result(self.nodes[1].listunspent(), + {"address": address_to_import}, + {"spendable": True}) # Mine a block from node0 to an address from node1 cbAddr = self.nodes[1].getnewaddress() @@ -269,7 +307,7 @@ def run_test (self): # Check that the txid and balance is found by node1 self.nodes[1].gettransaction(cbTxId) - #check if wallet or blochchain maintenance changes the balance + # check if wallet or blockchain maintenance changes the balance self.sync_all() blocks = self.nodes[0].generate(2) self.sync_all() @@ -281,10 +319,11 @@ def run_test (self): '-reindex', '-zapwallettxes=1', '-zapwallettxes=2', - '-salvagewallet', + # disabled until issue is fixed: https://github.com/bitcoin/bitcoin/issues/7463 + # '-salvagewallet', ] for m in maintenance: - print "check " + m + print("check " + m) stop_nodes(self.nodes) wait_bitcoinds() self.nodes = start_nodes(3, self.options.tmpdir, [[m]] * 3) @@ -301,4 +340,4 @@ def run_test (self): assert_equal(len(self.nodes[0].listsinceblock(blocks[1])["transactions"]), 0) if __name__ == '__main__': - WalletTest ().main () + WalletTest().main() diff --git a/qa/rpc-tests/walletbackup.py b/qa/rpc-tests/walletbackup.py index 1221a09116791..b991d5c761862 100755 --- a/qa/rpc-tests/walletbackup.py +++ b/qa/rpc-tests/walletbackup.py @@ -1,5 +1,5 @@ -#!/usr/bin/env python2 -# Copyright (c) 2014-2015 The Bitcoin Core developers +#!/usr/bin/env python3 +# Copyright (c) 2014-2016 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -37,19 +37,20 @@ from test_framework.util import * from random import randint import logging -logging.basicConfig(format='%(levelname)s:%(message)s', level=logging.INFO) +logging.basicConfig(format='%(levelname)s: %(message)s', level=logging.INFO, stream=sys.stdout) class WalletBackupTest(BitcoinTestFramework): - def setup_chain(self): - logging.info("Initializing test directory "+self.options.tmpdir) - initialize_chain_clean(self.options.tmpdir, 4) + def __init__(self): + super().__init__() + self.setup_clean_chain = True + self.num_nodes = 4 # This mirrors how the network was setup in the bash test def setup_network(self, split=False): # nodes 1, 2,3 are spenders, let's give them a keypool=100 extra_args = [["-keypool=100"], ["-keypool=100"], ["-keypool=100"], []] - self.nodes = start_nodes(4, self.options.tmpdir, extra_args) + self.nodes = start_nodes(self.num_nodes, self.options.tmpdir, extra_args) connect_nodes(self.nodes[0], 3) connect_nodes(self.nodes[1], 3) connect_nodes(self.nodes[2], 3) diff --git a/qa/rpc-tests/zapwallettxes.py b/qa/rpc-tests/zapwallettxes.py index 1ba4ded249aad..17ba53a844b4f 100755 --- a/qa/rpc-tests/zapwallettxes.py +++ b/qa/rpc-tests/zapwallettxes.py @@ -1,5 +1,5 @@ -#!/usr/bin/env python2 -# Copyright (c) 2014-2015 The Bitcoin Core developers +#!/usr/bin/env python3 +# Copyright (c) 2014-2016 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -9,12 +9,13 @@ class ZapWalletTXesTest (BitcoinTestFramework): - def setup_chain(self): - print("Initializing test directory "+self.options.tmpdir) - initialize_chain_clean(self.options.tmpdir, 3) + def __init__(self): + super().__init__() + self.setup_clean_chain = True + self.num_nodes = 3 def setup_network(self, split=False): - self.nodes = start_nodes(3, self.options.tmpdir) + self.nodes = start_nodes(self.num_nodes, self.options.tmpdir) connect_nodes_bi(self.nodes,0,1) connect_nodes_bi(self.nodes,1,2) connect_nodes_bi(self.nodes,0,2) @@ -22,7 +23,7 @@ def setup_network(self, split=False): self.sync_all() def run_test (self): - print "Mining blocks..." + print("Mining blocks...") self.nodes[0].generate(1) self.sync_all() self.nodes[1].generate(101) diff --git a/qa/rpc-tests/zmq_test.py b/qa/rpc-tests/zmq_test.py index 3a8d62ef2eddc..3a116317fed96 100755 --- a/qa/rpc-tests/zmq_test.py +++ b/qa/rpc-tests/zmq_test.py @@ -1,5 +1,5 @@ -#!/usr/bin/env python2 -# Copyright (c) 2015 The Bitcoin Core developers +#!/usr/bin/env python3 +# Copyright (c) 2015-2016 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -10,19 +10,17 @@ from test_framework.test_framework import BitcoinTestFramework from test_framework.util import * import zmq -import binascii +import struct -try: - import http.client as httplib -except ImportError: - import httplib -try: - import urllib.parse as urlparse -except ImportError: - import urlparse +import http.client +import urllib.parse class ZMQTest (BitcoinTestFramework): + def __init__(self): + super().__init__() + self.num_nodes = 4 + port = 28332 def setup_nodes(self): @@ -31,7 +29,7 @@ def setup_nodes(self): self.zmqSubSocket.setsockopt(zmq.SUBSCRIBE, b"hashblock") self.zmqSubSocket.setsockopt(zmq.SUBSCRIBE, b"hashtx") self.zmqSubSocket.connect("tcp://127.0.0.1:%i" % self.port) - return start_nodes(4, self.options.tmpdir, extra_args=[ + return start_nodes(self.num_nodes, self.options.tmpdir, extra_args=[ ['-zmqpubhashtx=tcp://127.0.0.1:'+str(self.port), '-zmqpubhashblock=tcp://127.0.0.1:'+str(self.port)], [], [], @@ -44,14 +42,20 @@ def run_test(self): genhashes = self.nodes[0].generate(1) self.sync_all() - print "listen..." + print("listen...") msg = self.zmqSubSocket.recv_multipart() topic = msg[0] + assert_equal(topic, b"hashtx") body = msg[1] + nseq = msg[2] + msgSequence = struct.unpack('/dev/null)" -a "$(git rev-parse --is-inside-work-tree 2>/dev/null)" = "true" ]; then # clean 'dirty' status of touched files that haven't been modified git diff >/dev/null 2>/dev/null @@ -29,9 +28,6 @@ if [ -e "$(which git 2>/dev/null)" -a "$(git rev-parse --is-inside-work-tree 2>/ # otherwise generate suffix from git, i.e. string like "59887e8-dirty" SUFFIX=$(git rev-parse --short HEAD) git diff-index --quiet HEAD -- || SUFFIX="$SUFFIX-dirty" - - # get a string like "2012-04-10 16:27:19 +0200" - LAST_COMMIT_DATE="$(git log -n 1 --format="%ci")" fi if [ -n "$DESC" ]; then @@ -45,7 +41,4 @@ fi # only update build.h if necessary if [ "$INFO" != "$NEWINFO" ]; then echo "$NEWINFO" >"$FILE" - if [ -n "$LAST_COMMIT_DATE" ]; then - echo "#define BUILD_DATE \"$LAST_COMMIT_DATE\"" >> "$FILE" - fi fi diff --git a/share/qt/extract_strings_qt.py b/share/qt/extract_strings_qt.py index 7728a4377590b..2ba8bb9b3acb7 100755 --- a/share/qt/extract_strings_qt.py +++ b/share/qt/extract_strings_qt.py @@ -32,7 +32,7 @@ def parse_po(text): in_msgstr = False # message start in_msgid = True - + msgid = [line[6:]] elif line.startswith('msgstr '): in_msgid = False @@ -67,7 +67,7 @@ def parse_po(text): #include -// Automatically generated by extract_strings.py +// Automatically generated by extract_strings_qt.py #ifdef __GNUC__ #define UNUSED __attribute__((unused)) #else diff --git a/src/Makefile.am b/src/Makefile.am index 24744835f50d6..3df8e267bb335 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -3,6 +3,7 @@ DIST_SUBDIRS = secp256k1 univalue AM_LDFLAGS = $(PTHREAD_CFLAGS) $(LIBTOOL_LDFLAGS) $(HARDENED_LDFLAGS) AM_CXXFLAGS = $(HARDENED_CXXFLAGS) AM_CPPFLAGS = $(HARDENED_CPPFLAGS) +EXTRA_LIBRARIES = if EMBEDDED_UNIVALUE LIBUNIVALUE = univalue/libunivalue.la @@ -13,29 +14,13 @@ else LIBUNIVALUE = $(UNIVALUE_LIBS) endif -if EMBEDDED_LEVELDB -LEVELDB_CPPFLAGS += -I$(srcdir)/leveldb/include -LEVELDB_CPPFLAGS += -I$(srcdir)/leveldb/helpers/memenv -LIBLEVELDB += $(builddir)/leveldb/libleveldb.a -LIBMEMENV += $(builddir)/leveldb/libmemenv.a - -# NOTE: This dependency is not strictly necessary, but without it make may try to build both in parallel, which breaks the LevelDB build system in a race -$(LIBLEVELDB): $(LIBMEMENV) - -$(LIBLEVELDB) $(LIBMEMENV): - @echo "Building LevelDB ..." && $(MAKE) -C $(@D) $(@F) CXX="$(CXX)" \ - CC="$(CC)" PLATFORM=$(TARGET_OS) AR="$(AR)" $(LEVELDB_TARGET_FLAGS) \ - OPT="$(AM_CXXFLAGS) $(PIE_FLAGS) $(CXXFLAGS) $(AM_CPPFLAGS) $(CPPFLAGS) -D__STDC_LIMIT_MACROS" -endif - BITCOIN_CONFIG_INCLUDES=-I$(builddir)/config -BITCOIN_INCLUDES=-I$(builddir) -I$(builddir)/obj $(BOOST_CPPFLAGS) $(LEVELDB_CPPFLAGS) $(CRYPTO_CFLAGS) $(SSL_CFLAGS) +BITCOIN_INCLUDES=-I$(builddir) -I$(builddir)/obj $(BDB_CPPFLAGS) $(BOOST_CPPFLAGS) $(LEVELDB_CPPFLAGS) $(CRYPTO_CFLAGS) $(SSL_CFLAGS) BITCOIN_INCLUDES += -I$(srcdir)/secp256k1/include BITCOIN_INCLUDES += $(UNIVALUE_CFLAGS) LIBBITCOIN_SERVER=libbitcoin_server.a -LIBBITCOIN_WALLET=libbitcoin_wallet.a LIBBITCOIN_COMMON=libbitcoin_common.a LIBBITCOIN_CONSENSUS=libbitcoin_consensus.a LIBBITCOIN_CLI=libbitcoin_cli.a @@ -44,32 +29,32 @@ LIBBITCOIN_CRYPTO=crypto/libbitcoin_crypto.a LIBBITCOINQT=qt/libbitcoinqt.a LIBSECP256K1=secp256k1/libsecp256k1.la +if ENABLE_ZMQ +LIBBITCOIN_ZMQ=libbitcoin_zmq.a +endif +if BUILD_BITCOIN_LIBS +LIBBITCOINCONSENSUS=libbitcoinconsensus.la +endif +if ENABLE_WALLET +LIBBITCOIN_WALLET=libbitcoin_wallet.a +endif + $(LIBSECP256K1): $(wildcard secp256k1/src/*) $(wildcard secp256k1/include/*) $(AM_V_at)$(MAKE) $(AM_MAKEFLAGS) -C $(@D) $(@F) # Make is not made aware of per-object dependencies to avoid limiting building parallelization # But to build the less dependent modules first, we manually select their order here: -EXTRA_LIBRARIES = \ - crypto/libbitcoin_crypto.a \ - libbitcoin_util.a \ - libbitcoin_common.a \ - libbitcoin_consensus.a \ - libbitcoin_server.a \ - libbitcoin_cli.a -if ENABLE_WALLET -BITCOIN_INCLUDES += $(BDB_CPPFLAGS) -EXTRA_LIBRARIES += libbitcoin_wallet.a -endif -if ENABLE_ZMQ -EXTRA_LIBRARIES += libbitcoin_zmq.a -endif +EXTRA_LIBRARIES += \ + $(LIBBITCOIN_CRYPTO) \ + $(LIBBITCOIN_UTIL) \ + $(LIBBITCOIN_COMMON) \ + $(LIBBITCOIN_CONSENSUS) \ + $(LIBBITCOIN_SERVER) \ + $(LIBBITCOIN_CLI) \ + $(LIBBITCOIN_WALLET) \ + $(LIBBITCOIN_ZMQ) -if BUILD_BITCOIN_LIBS -lib_LTLIBRARIES = libbitcoinconsensus.la -LIBBITCOINCONSENSUS=libbitcoinconsensus.la -else -LIBBITCOINCONSENSUS= -endif +lib_LTLIBRARIES = $(LIBBITCOINCONSENSUS) bin_PROGRAMS = TESTS = @@ -108,6 +93,7 @@ BITCOIN_CORE_H = \ core_memusage.h \ httprpc.h \ httpserver.h \ + indirectmap.h \ init.h \ key.h \ keystore.h \ @@ -204,13 +190,12 @@ libbitcoin_server_a_SOURCES = \ torcontrol.cpp \ txdb.cpp \ txmempool.cpp \ + ui_interface.cpp \ validationinterface.cpp \ versionbits.cpp \ $(BITCOIN_CORE_H) if ENABLE_ZMQ -LIBBITCOIN_ZMQ=libbitcoin_zmq.a - libbitcoin_zmq_a_CPPFLAGS = $(BITCOIN_INCLUDES) $(ZMQ_CFLAGS) libbitcoin_zmq_a_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) libbitcoin_zmq_a_SOURCES = \ @@ -238,6 +223,8 @@ libbitcoin_wallet_a_SOURCES = \ crypto_libbitcoin_crypto_a_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_CONFIG_INCLUDES) crypto_libbitcoin_crypto_a_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) crypto_libbitcoin_crypto_a_SOURCES = \ + crypto/aes.cpp \ + crypto/aes.h \ crypto/common.h \ crypto/hmac_sha256.cpp \ crypto/hmac_sha256.h \ @@ -358,21 +345,15 @@ bitcoind_LDADD = \ $(LIBBITCOIN_COMMON) \ $(LIBUNIVALUE) \ $(LIBBITCOIN_UTIL) \ + $(LIBBITCOIN_WALLET) \ + $(LIBBITCOIN_ZMQ) \ $(LIBBITCOIN_CONSENSUS) \ $(LIBBITCOIN_CRYPTO) \ $(LIBLEVELDB) \ $(LIBMEMENV) \ $(LIBSECP256K1) -if ENABLE_ZMQ -bitcoind_LDADD += $(LIBBITCOIN_ZMQ) $(ZMQ_LIBS) -endif - -if ENABLE_WALLET -bitcoind_LDADD += libbitcoin_wallet.a -endif - -bitcoind_LDADD += $(BOOST_LIBS) $(BDB_LIBS) $(SSL_LIBS) $(CRYPTO_LIBS) $(MINIUPNPC_LIBS) $(EVENT_PTHREADS_LIBS) $(EVENT_LIBS) +bitcoind_LDADD += $(BOOST_LIBS) $(BDB_LIBS) $(SSL_LIBS) $(CRYPTO_LIBS) $(MINIUPNPC_LIBS) $(EVENT_PTHREADS_LIBS) $(EVENT_LIBS) $(ZMQ_LIBS) # bitcoin-cli binary # bitcoin_cli_SOURCES = bitcoin-cli.cpp @@ -387,7 +368,8 @@ endif bitcoin_cli_LDADD = \ $(LIBBITCOIN_CLI) \ $(LIBUNIVALUE) \ - $(LIBBITCOIN_UTIL) + $(LIBBITCOIN_UTIL) \ + $(LIBBITCOIN_CRYPTO) bitcoin_cli_LDADD += $(BOOST_LIBS) $(SSL_LIBS) $(CRYPTO_LIBS) $(EVENT_LIBS) # @@ -430,8 +412,14 @@ libbitcoinconsensus_la_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) endif # -CLEANFILES = leveldb/libleveldb.a leveldb/libmemenv.a -CLEANFILES += $(EXTRA_LIBRARIES) +CTAES_DIST = crypto/ctaes/bench.c +CTAES_DIST += crypto/ctaes/ctaes.c +CTAES_DIST += crypto/ctaes/ctaes.h +CTAES_DIST += crypto/ctaes/README.md +CTAES_DIST += crypto/ctaes/test.c + +CLEANFILES = $(EXTRA_LIBRARIES) + CLEANFILES += *.gcda *.gcno CLEANFILES += compat/*.gcda compat/*.gcno CLEANFILES += consensus/*.gcda consensus/*.gcno @@ -447,14 +435,14 @@ CLEANFILES += zmq/*.gcda zmq/*.gcno DISTCLEANFILES = obj/build.h -EXTRA_DIST = leveldb +EXTRA_DIST = $(CTAES_DIST) clean-local: - -$(MAKE) -C leveldb clean -$(MAKE) -C secp256k1 clean -$(MAKE) -C univalue clean -rm -f leveldb/*/*.gcda leveldb/*/*.gcno leveldb/helpers/memenv/*.gcda leveldb/helpers/memenv/*.gcno -rm -f config.h + -rm -rf test/__pycache__ .rc.o: @test -f $(WINDRES) @@ -481,6 +469,10 @@ endif @test -f $(PROTOC) $(AM_V_GEN) $(PROTOC) --cpp_out=$(@D) --proto_path=$( $@ + @rm $(@D)/temp_$( $@ moc_%.cpp: %.h - $(AM_V_GEN) QT_SELECT=$(QT_SELECT) $(MOC) $(QT_INCLUDES) $(MOC_DEFS) $< | \ + $(AM_V_GEN) QT_SELECT=$(QT_SELECT) $(MOC) $(DEFAULT_INCLUDES) $(QT_INCLUDES) $(MOC_DEFS) $< | \ $(SED) -e '/^\*\*.*Created:/d' -e '/^\*\*.*by:/d' > $@ %.qm: %.ts diff --git a/src/Makefile.test.include b/src/Makefile.test.include index 08e2f6af4db73..41d811fb5400f 100644 --- a/src/Makefile.test.include +++ b/src/Makefile.test.include @@ -17,7 +17,9 @@ EXTRA_DIST += \ test/data/txcreate2.hex \ test/data/txcreatedata1.hex \ test/data/txcreatedata2.hex \ - test/data/txcreatesign.hex + test/data/txcreatesign.hex \ + test/data/txcreatedata_seq0.hex \ + test/data/txcreatedata_seq1.hex JSON_TEST_FILES = \ test/data/script_tests.json \ @@ -59,6 +61,7 @@ BITCOIN_TESTS =\ test/merkle_tests.cpp \ test/miner_tests.cpp \ test/multisig_tests.cpp \ + test/net_tests.cpp \ test/netbase_tests.cpp \ test/pmt_tests.cpp \ test/policyestimator_tests.cpp \ @@ -94,6 +97,7 @@ BITCOIN_TESTS += \ wallet/test/wallet_test_fixture.h \ wallet/test/accounting_tests.cpp \ wallet/test/wallet_tests.cpp \ + wallet/test/crypto_tests.cpp \ wallet/test/rpc_wallet_tests.cpp endif @@ -121,6 +125,9 @@ CLEAN_BITCOIN_TEST = test/*.gcda test/*.gcno $(GENERATED_TEST_FILES) CLEANFILES += $(CLEAN_BITCOIN_TEST) +# This file is problematic for out-of-tree builds if it exists. +DISTCLEANFILES += test/buildenv.pyc + bitcoin_test: $(TEST_BINARY) bitcoin_test_check: $(TEST_BINARY) FORCE diff --git a/src/addrman.cpp b/src/addrman.cpp index 2de005df37234..cebb1c8e5e708 100644 --- a/src/addrman.cpp +++ b/src/addrman.cpp @@ -197,6 +197,9 @@ void CAddrMan::MakeTried(CAddrInfo& info, int nId) void CAddrMan::Good_(const CService& addr, int64_t nTime) { int nId; + + nLastGood = nTime; + CAddrInfo* pinfo = Find(addr, &nId); // if not found, bail out @@ -260,7 +263,7 @@ bool CAddrMan::Add_(const CAddress& addr, const CNetAddr& source, int64_t nTimeP pinfo->nTime = std::max((int64_t)0, addr.nTime - nTimePenalty); // add services - pinfo->nServices |= addr.nServices; + pinfo->nServices = ServiceFlags(pinfo->nServices | addr.nServices); // do not update if no new information is present if (!addr.nTime || (pinfo->nTime && addr.nTime <= pinfo->nTime)) @@ -311,7 +314,7 @@ bool CAddrMan::Add_(const CAddress& addr, const CNetAddr& source, int64_t nTimeP return fNew; } -void CAddrMan::Attempt_(const CService& addr, int64_t nTime) +void CAddrMan::Attempt_(const CService& addr, bool fCountFailure, int64_t nTime) { CAddrInfo* pinfo = Find(addr); @@ -327,7 +330,10 @@ void CAddrMan::Attempt_(const CService& addr, int64_t nTime) // update info info.nLastTry = nTime; - info.nAttempts++; + if (fCountFailure && info.nLastCountAttempt < nLastGood) { + info.nLastCountAttempt = nTime; + info.nAttempts++; + } } CAddrInfo CAddrMan::Select_(bool newOnly) @@ -496,11 +502,7 @@ void CAddrMan::Connected_(const CService& addr, int64_t nTime) info.nTime = nTime; } -int CAddrMan::RandomInt(int nMax){ - return GetRandInt(nMax); -} - -void CAddrMan::SetServices_(const CService& addr, uint64_t nServices) +void CAddrMan::SetServices_(const CService& addr, ServiceFlags nServices) { CAddrInfo* pinfo = Find(addr); @@ -517,3 +519,7 @@ void CAddrMan::SetServices_(const CService& addr, uint64_t nServices) // update info info.nServices = nServices; } + +int CAddrMan::RandomInt(int nMax){ + return GetRandInt(nMax); +} diff --git a/src/addrman.h b/src/addrman.h index 646d0a1b348dc..1caf540758797 100644 --- a/src/addrman.h +++ b/src/addrman.h @@ -29,6 +29,9 @@ class CAddrInfo : public CAddress //! last try whatsoever by us (memory only) int64_t nLastTry; + //! last counted attempt (memory only) + int64_t nLastCountAttempt; + private: //! where knowledge about this address first came from CNetAddr source; @@ -66,6 +69,7 @@ class CAddrInfo : public CAddress { nLastSuccess = 0; nLastTry = 0; + nLastCountAttempt = 0; nAttempts = 0; nRefCount = 0; fInTried = false; @@ -200,6 +204,9 @@ class CAddrMan //! list of "new" buckets int vvNew[ADDRMAN_NEW_BUCKET_COUNT][ADDRMAN_BUCKET_SIZE]; + //! last time Good was called (memory only) + int64_t nLastGood; + protected: //! secret key to randomize bucket select with uint256 nKey; @@ -230,7 +237,7 @@ class CAddrMan bool Add_(const CAddress &addr, const CNetAddr& source, int64_t nTimePenalty); //! Mark an entry as attempted to connect. - void Attempt_(const CService &addr, int64_t nTime); + void Attempt_(const CService &addr, bool fCountFailure, int64_t nTime); //! Select an address to connect to, if newOnly is set to true, only the new table is selected from. CAddrInfo Select_(bool newOnly); @@ -250,7 +257,7 @@ class CAddrMan void Connected_(const CService &addr, int64_t nTime); //! Update an entry's service bits. - void SetServices_(const CService &addr, uint64_t nServices); + void SetServices_(const CService &addr, ServiceFlags nServices); public: /** @@ -353,6 +360,14 @@ class CAddrMan nUBuckets ^= (1 << 30); } + if (nNew > ADDRMAN_NEW_BUCKET_COUNT * ADDRMAN_BUCKET_SIZE) { + throw std::ios_base::failure("Corrupt CAddrMan serialization, nNew exceeds limit."); + } + + if (nTried > ADDRMAN_TRIED_BUCKET_COUNT * ADDRMAN_BUCKET_SIZE) { + throw std::ios_base::failure("Corrupt CAddrMan serialization, nTried exceeds limit."); + } + // Deserialize entries from the new table. for (int n = 0; n < nNew; n++) { CAddrInfo &info = mapInfo[n]; @@ -453,6 +468,7 @@ class CAddrMan nIdCount = 0; nTried = 0; nNew = 0; + nLastGood = 1; //Initially at 1 so that "never" is strictly worse. } CAddrMan() @@ -527,12 +543,12 @@ class CAddrMan } //! Mark an entry as connection attempted to. - void Attempt(const CService &addr, int64_t nTime = GetAdjustedTime()) + void Attempt(const CService &addr, bool fCountFailure, int64_t nTime = GetAdjustedTime()) { { LOCK(cs); Check(); - Attempt_(addr, nTime); + Attempt_(addr, fCountFailure, nTime); Check(); } } @@ -576,7 +592,7 @@ class CAddrMan } } - void SetServices(const CService &addr, uint64_t nServices) + void SetServices(const CService &addr, ServiceFlags nServices) { LOCK(cs); Check(); diff --git a/src/base58.cpp b/src/base58.cpp index d81c26092c305..d1d60a6f1dcf5 100644 --- a/src/base58.cpp +++ b/src/base58.cpp @@ -177,7 +177,7 @@ bool CBase58Data::SetString(const char* psz, unsigned int nVersionBytes) vchData.resize(vchTemp.size() - nVersionBytes); if (!vchData.empty()) memcpy(&vchData[0], &vchTemp[nVersionBytes], vchData.size()); - memory_cleanse(&vchTemp[0], vchData.size()); + memory_cleanse(&vchTemp[0], vchTemp.size()); return true; } diff --git a/src/bench/base58.cpp b/src/bench/base58.cpp new file mode 100644 index 0000000000000..1279c3e7df5dd --- /dev/null +++ b/src/bench/base58.cpp @@ -0,0 +1,56 @@ +// Copyright (c) 2016 the Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include "bench.h" + +#include "main.h" +#include "base58.h" + +#include +#include + + +static void Base58Encode(benchmark::State& state) +{ + unsigned char buff[32] = { + 17, 79, 8, 99, 150, 189, 208, 162, 22, 23, 203, 163, 36, 58, 147, + 227, 139, 2, 215, 100, 91, 38, 11, 141, 253, 40, 117, 21, 16, 90, + 200, 24 + }; + unsigned char* b = buff; + while (state.KeepRunning()) { + EncodeBase58(b, b + 32); + } +} + + +static void Base58CheckEncode(benchmark::State& state) +{ + unsigned char buff[32] = { + 17, 79, 8, 99, 150, 189, 208, 162, 22, 23, 203, 163, 36, 58, 147, + 227, 139, 2, 215, 100, 91, 38, 11, 141, 253, 40, 117, 21, 16, 90, + 200, 24 + }; + unsigned char* b = buff; + std::vector vch; + vch.assign(b, b + 32); + while (state.KeepRunning()) { + EncodeBase58Check(vch); + } +} + + +static void Base58Decode(benchmark::State& state) +{ + const char* addr = "17VZNX1SN5NtKa8UQFxwQbFeFc3iqRYhem"; + std::vector vch; + while (state.KeepRunning()) { + DecodeBase58(addr, vch); + } +} + + +BENCHMARK(Base58Encode); +BENCHMARK(Base58CheckEncode); +BENCHMARK(Base58Decode); diff --git a/src/bench/bench.cpp b/src/bench/bench.cpp index 6ee3cdc27a49e..227546a7a78a3 100644 --- a/src/bench/bench.cpp +++ b/src/bench/bench.cpp @@ -5,6 +5,7 @@ #include "bench.h" #include +#include #include using namespace benchmark; @@ -25,7 +26,7 @@ BenchRunner::BenchRunner(std::string name, BenchFunction func) void BenchRunner::RunAll(double elapsedTimeForOne) { - std::cout << "Benchmark" << "," << "count" << "," << "min" << "," << "max" << "," << "average" << "\n"; + std::cout << "#Benchmark" << "," << "count" << "," << "min" << "," << "max" << "," << "average" << "\n"; for (std::map::iterator it = benchmarks.begin(); it != benchmarks.end(); ++it) { @@ -38,22 +39,34 @@ BenchRunner::RunAll(double elapsedTimeForOne) bool State::KeepRunning() { + if (count & countMask) { + ++count; + return true; + } double now; if (count == 0) { - beginTime = now = gettimedouble(); + lastTime = beginTime = now = gettimedouble(); } else { - // timeCheckCount is used to avoid calling gettime most of the time, - // so benchmarks that run very quickly get consistent results. - if ((count+1)%timeCheckCount != 0) { - ++count; - return true; // keep going - } now = gettimedouble(); - double elapsedOne = (now - lastTime)/timeCheckCount; + double elapsed = now - lastTime; + double elapsedOne = elapsed * countMaskInv; if (elapsedOne < minTime) minTime = elapsedOne; if (elapsedOne > maxTime) maxTime = elapsedOne; - if (elapsedOne*timeCheckCount < maxElapsed/16) timeCheckCount *= 2; + if (elapsed*128 < maxElapsed) { + // If the execution was much too fast (1/128th of maxElapsed), increase the count mask by 8x and restart timing. + // The restart avoids including the overhead of this code in the measurement. + countMask = ((countMask<<3)|7) & ((1LL<<60)-1); + countMaskInv = 1./(countMask+1); + count = 0; + minTime = std::numeric_limits::max(); + maxTime = std::numeric_limits::min(); + return true; + } + if (elapsed*16 < maxElapsed) { + countMask = ((countMask<<1)|1) & ((1LL<<60)-1); + countMaskInv = 1./(countMask+1); + } } lastTime = now; ++count; @@ -64,7 +77,7 @@ bool State::KeepRunning() // Output results double average = (now-beginTime)/count; - std::cout << name << "," << count << "," << minTime << "," << maxTime << "," << average << "\n"; + std::cout << std::fixed << std::setprecision(15) << name << "," << count << "," << minTime << "," << maxTime << "," << average << "\n"; return false; } diff --git a/src/bench/bench.h b/src/bench/bench.h index 5ce13c642b59b..f13b145aaf0c7 100644 --- a/src/bench/bench.h +++ b/src/bench/bench.h @@ -40,14 +40,15 @@ namespace benchmark { std::string name; double maxElapsed; double beginTime; - double lastTime, minTime, maxTime; + double lastTime, minTime, maxTime, countMaskInv; int64_t count; - int64_t timeCheckCount; + int64_t countMask; public: State(std::string _name, double _maxElapsed) : name(_name), maxElapsed(_maxElapsed), count(0) { minTime = std::numeric_limits::max(); maxTime = std::numeric_limits::min(); - timeCheckCount = 1; + countMask = 1; + countMaskInv = 1./(countMask + 1); } bool KeepRunning(); }; diff --git a/src/bench/crypto_hash.cpp b/src/bench/crypto_hash.cpp new file mode 100644 index 0000000000000..168006154f8fe --- /dev/null +++ b/src/bench/crypto_hash.cpp @@ -0,0 +1,78 @@ +// Copyright (c) 2016 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include + +#include "bench.h" +#include "bloom.h" +#include "hash.h" +#include "uint256.h" +#include "utiltime.h" +#include "crypto/ripemd160.h" +#include "crypto/sha1.h" +#include "crypto/sha256.h" +#include "crypto/sha512.h" + +/* Number of bytes to hash per iteration */ +static const uint64_t BUFFER_SIZE = 1000*1000; + +static void RIPEMD160(benchmark::State& state) +{ + uint8_t hash[CRIPEMD160::OUTPUT_SIZE]; + std::vector in(BUFFER_SIZE,0); + while (state.KeepRunning()) + CRIPEMD160().Write(begin_ptr(in), in.size()).Finalize(hash); +} + +static void SHA1(benchmark::State& state) +{ + uint8_t hash[CSHA1::OUTPUT_SIZE]; + std::vector in(BUFFER_SIZE,0); + while (state.KeepRunning()) + CSHA1().Write(begin_ptr(in), in.size()).Finalize(hash); +} + +static void SHA256(benchmark::State& state) +{ + uint8_t hash[CSHA256::OUTPUT_SIZE]; + std::vector in(BUFFER_SIZE,0); + while (state.KeepRunning()) + CSHA256().Write(begin_ptr(in), in.size()).Finalize(hash); +} + +static void SHA256_32b(benchmark::State& state) +{ + std::vector in(32,0); + while (state.KeepRunning()) { + for (int i = 0; i < 1000000; i++) { + CSHA256().Write(begin_ptr(in), in.size()).Finalize(&in[0]); + } + } +} + +static void SHA512(benchmark::State& state) +{ + uint8_t hash[CSHA512::OUTPUT_SIZE]; + std::vector in(BUFFER_SIZE,0); + while (state.KeepRunning()) + CSHA512().Write(begin_ptr(in), in.size()).Finalize(hash); +} + +static void SipHash_32b(benchmark::State& state) +{ + uint256 x; + while (state.KeepRunning()) { + for (int i = 0; i < 1000000; i++) { + *((uint64_t*)x.begin()) = SipHashUint256(0, i, x); + } + } +} + +BENCHMARK(RIPEMD160); +BENCHMARK(SHA1); +BENCHMARK(SHA256); +BENCHMARK(SHA512); + +BENCHMARK(SHA256_32b); +BENCHMARK(SipHash_32b); diff --git a/src/bench/rollingbloom.cpp b/src/bench/rollingbloom.cpp new file mode 100644 index 0000000000000..73c02cf7189eb --- /dev/null +++ b/src/bench/rollingbloom.cpp @@ -0,0 +1,43 @@ +// Copyright (c) 2016 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include + +#include "bench.h" +#include "bloom.h" +#include "utiltime.h" + +static void RollingBloom(benchmark::State& state) +{ + CRollingBloomFilter filter(120000, 0.000001); + std::vector data(32); + uint32_t count = 0; + uint32_t nEntriesPerGeneration = (120000 + 1) / 2; + uint32_t countnow = 0; + uint64_t match = 0; + while (state.KeepRunning()) { + count++; + data[0] = count; + data[1] = count >> 8; + data[2] = count >> 16; + data[3] = count >> 24; + if (countnow == nEntriesPerGeneration) { + int64_t b = GetTimeMicros(); + filter.insert(data); + int64_t e = GetTimeMicros(); + std::cout << "RollingBloom-refresh,1," << (e-b)*0.000001 << "," << (e-b)*0.000001 << "," << (e-b)*0.000001 << "\n"; + countnow = 0; + } else { + filter.insert(data); + } + countnow++; + data[0] = count >> 24; + data[1] = count >> 16; + data[2] = count >> 8; + data[3] = count; + match += filter.contains(data); + } +} + +BENCHMARK(RollingBloom); diff --git a/src/bitcoin-cli.cpp b/src/bitcoin-cli.cpp index 49935699fd5d8..a04101d3ed847 100644 --- a/src/bitcoin-cli.cpp +++ b/src/bitcoin-cli.cpp @@ -314,7 +314,7 @@ int main(int argc, char* argv[]) SetupEnvironment(); if (!SetupNetworking()) { fprintf(stderr, "Error: Initializing networking failed\n"); - exit(1); + return EXIT_FAILURE; } try { diff --git a/src/bitcoin-tx.cpp b/src/bitcoin-tx.cpp index 09e1d548643a8..8e8ac47455cbe 100644 --- a/src/bitcoin-tx.cpp +++ b/src/bitcoin-tx.cpp @@ -71,7 +71,7 @@ static bool AppInitRawTx(int argc, char* argv[]) strUsage = HelpMessageGroup(_("Commands:")); strUsage += HelpMessageOpt("delin=N", _("Delete input N from TX")); strUsage += HelpMessageOpt("delout=N", _("Delete output N from TX")); - strUsage += HelpMessageOpt("in=TXID:VOUT", _("Add input to TX")); + strUsage += HelpMessageOpt("in=TXID:VOUT(:SEQUENCE_NUMBER)", _("Add input to TX")); strUsage += HelpMessageOpt("locktime=N", _("Set TX lock time to N")); strUsage += HelpMessageOpt("nversion=N", _("Set TX version to N")); strUsage += HelpMessageOpt("outaddr=VALUE:ADDRESS", _("Add address-based output to TX")); @@ -181,15 +181,15 @@ static void MutateTxLocktime(CMutableTransaction& tx, const string& cmdVal) static void MutateTxAddInput(CMutableTransaction& tx, const string& strInput) { + std::vector vStrInputParts; + boost::split(vStrInputParts, strInput, boost::is_any_of(":")); + // separate TXID:VOUT in string - size_t pos = strInput.find(':'); - if ((pos == string::npos) || - (pos == 0) || - (pos == (strInput.size() - 1))) + if (vStrInputParts.size()<2) throw runtime_error("TX input missing separator"); // extract and validate TXID - string strTxid = strInput.substr(0, pos); + string strTxid = vStrInputParts[0]; if ((strTxid.size() != 64) || !IsHex(strTxid)) throw runtime_error("invalid TX input txid"); uint256 txid(uint256S(strTxid)); @@ -198,13 +198,18 @@ static void MutateTxAddInput(CMutableTransaction& tx, const string& strInput) static const unsigned int maxVout = MAX_BLOCK_BASE_SIZE / minTxOutSz; // extract and validate vout - string strVout = strInput.substr(pos + 1, string::npos); + string strVout = vStrInputParts[1]; int vout = atoi(strVout); if ((vout < 0) || (vout > (int)maxVout)) throw runtime_error("invalid TX input vout"); + // extract the optional sequence number + uint32_t nSequenceIn=std::numeric_limits::max(); + if (vStrInputParts.size() > 2) + nSequenceIn = std::stoul(vStrInputParts[2]); + // append to transaction input list - CTxIn txin(txid, vout); + CTxIn txin(txid, vout, CScript(), nSequenceIn); tx.vin.push_back(txin); } diff --git a/src/bloom.cpp b/src/bloom.cpp index 4e6f0e5d2ddaa..fd328e8e96e53 100644 --- a/src/bloom.cpp +++ b/src/bloom.cpp @@ -234,14 +234,18 @@ CRollingBloomFilter::CRollingBloomFilter(unsigned int nElements, double fpRate) */ uint32_t nFilterBits = (uint32_t)ceil(-1.0 * nHashFuncs * nMaxElements / log(1.0 - exp(logFpRate / nHashFuncs))); data.clear(); - /* We store up to 16 'bits' per data element. */ - data.resize((nFilterBits + 15) / 16); + /* For each data element we need to store 2 bits. If both bits are 0, the + * bit is treated as unset. If the bits are (01), (10), or (11), the bit is + * treated as set in generation 1, 2, or 3 respectively. + * These bits are stored in separate integers: position P corresponds to bit + * (P & 63) of the integers data[(P >> 6) * 2] and data[(P >> 6) * 2 + 1]. */ + data.resize(((nFilterBits + 63) / 64) << 1); reset(); } /* Similar to CBloomFilter::Hash */ -inline unsigned int CRollingBloomFilter::Hash(unsigned int nHashNum, const std::vector& vDataToHash) const { - return MurmurHash3(nHashNum * 0xFBA4C795 + nTweak, vDataToHash) % (data.size() * 16); +static inline uint32_t RollingBloomHash(unsigned int nHashNum, uint32_t nTweak, const std::vector& vDataToHash) { + return MurmurHash3(nHashNum * 0xFBA4C795 + nTweak, vDataToHash); } void CRollingBloomFilter::insert(const std::vector& vKey) @@ -252,18 +256,25 @@ void CRollingBloomFilter::insert(const std::vector& vKey) if (nGeneration == 4) { nGeneration = 1; } + uint64_t nGenerationMask1 = -(uint64_t)(nGeneration & 1); + uint64_t nGenerationMask2 = -(uint64_t)(nGeneration >> 1); /* Wipe old entries that used this generation number. */ - for (uint32_t p = 0; p < data.size() * 16; p++) { - if (get(p) == nGeneration) { - put(p, 0); - } + for (uint32_t p = 0; p < data.size(); p += 2) { + uint64_t p1 = data[p], p2 = data[p + 1]; + uint64_t mask = (p1 ^ nGenerationMask1) | (p2 ^ nGenerationMask2); + data[p] = p1 & mask; + data[p + 1] = p2 & mask; } } nEntriesThisGeneration++; for (int n = 0; n < nHashFuncs; n++) { - uint32_t h = Hash(n, vKey); - put(h, nGeneration); + uint32_t h = RollingBloomHash(n, nTweak, vKey); + int bit = h & 0x3F; + uint32_t pos = (h >> 6) % data.size(); + /* The lowest bit of pos is ignored, and set to zero for the first bit, and to one for the second. */ + data[pos & ~1] = (data[pos & ~1] & ~(((uint64_t)1) << bit)) | ((uint64_t)(nGeneration & 1)) << bit; + data[pos | 1] = (data[pos | 1] & ~(((uint64_t)1) << bit)) | ((uint64_t)(nGeneration >> 1)) << bit; } } @@ -276,8 +287,11 @@ void CRollingBloomFilter::insert(const uint256& hash) bool CRollingBloomFilter::contains(const std::vector& vKey) const { for (int n = 0; n < nHashFuncs; n++) { - uint32_t h = Hash(n, vKey); - if (get(h) == 0) { + uint32_t h = RollingBloomHash(n, nTweak, vKey); + int bit = h & 0x3F; + uint32_t pos = (h >> 6) % data.size(); + /* If the relevant bit is not set in either data[pos & ~1] or data[pos | 1], the filter does not contain vKey */ + if (!(((data[pos & ~1] | data[pos | 1]) >> bit) & 1)) { return false; } } @@ -295,7 +309,7 @@ void CRollingBloomFilter::reset() nTweak = GetRand(std::numeric_limits::max()); nEntriesThisGeneration = 0; nGeneration = 1; - for (std::vector::iterator it = data.begin(); it != data.end(); it++) { + for (std::vector::iterator it = data.begin(); it != data.end(); it++) { *it = 0; } } diff --git a/src/bloom.h b/src/bloom.h index b0ad8b875d04b..ad6de625d8f8a 100644 --- a/src/bloom.h +++ b/src/bloom.h @@ -135,20 +135,9 @@ class CRollingBloomFilter int nEntriesPerGeneration; int nEntriesThisGeneration; int nGeneration; - std::vector data; + std::vector data; unsigned int nTweak; int nHashFuncs; - - unsigned int Hash(unsigned int nHashNum, const std::vector& vDataToHash) const; - - inline int get(uint32_t position) const { - return (data[(position >> 4) % data.size()] >> (2 * (position & 0xF))) & 0x3; - } - - inline void put(uint32_t position, uint32_t val) { - uint32_t& cell = data[(position >> 4) % data.size()]; - cell = (cell & ~(((uint32_t)3) << (2 * (position & 0xF)))) | (val << (2 * (position & 0xF))); - } }; #endif // BITCOIN_BLOOM_H diff --git a/src/chain.cpp b/src/chain.cpp index 32f6480f849da..77e924e703043 100644 --- a/src/chain.cpp +++ b/src/chain.cpp @@ -93,6 +93,7 @@ CBlockIndex* CBlockIndex::GetAncestor(int height) pindexWalk = pindexWalk->pskip; heightWalk = heightSkip; } else { + assert(pindexWalk->pprev); pindexWalk = pindexWalk->pprev; heightWalk--; } diff --git a/src/chain.h b/src/chain.h index cc94edb2b0dfe..76a774c1238f1 100644 --- a/src/chain.h +++ b/src/chain.h @@ -14,6 +14,60 @@ #include +class CBlockFileInfo +{ +public: + unsigned int nBlocks; //!< number of blocks stored in file + unsigned int nSize; //!< number of used bytes of block file + unsigned int nUndoSize; //!< number of used bytes in the undo file + unsigned int nHeightFirst; //!< lowest height of block in file + unsigned int nHeightLast; //!< highest height of block in file + uint64_t nTimeFirst; //!< earliest time of block in file + uint64_t nTimeLast; //!< latest time of block in file + + ADD_SERIALIZE_METHODS; + + template + inline void SerializationOp(Stream& s, Operation ser_action, int nType, int nVersion) { + READWRITE(VARINT(nBlocks)); + READWRITE(VARINT(nSize)); + READWRITE(VARINT(nUndoSize)); + READWRITE(VARINT(nHeightFirst)); + READWRITE(VARINT(nHeightLast)); + READWRITE(VARINT(nTimeFirst)); + READWRITE(VARINT(nTimeLast)); + } + + void SetNull() { + nBlocks = 0; + nSize = 0; + nUndoSize = 0; + nHeightFirst = 0; + nHeightLast = 0; + nTimeFirst = 0; + nTimeLast = 0; + } + + CBlockFileInfo() { + SetNull(); + } + + std::string ToString() const; + + /** update statistics (does not update nSize) */ + void AddBlock(unsigned int nHeightIn, uint64_t nTimeIn) { + if (nBlocks==0 || nHeightFirst > nHeightIn) + nHeightFirst = nHeightIn; + if (nBlocks==0 || nTimeFirst > nTimeIn) + nTimeFirst = nTimeIn; + nBlocks++; + if (nHeightIn > nHeightLast) + nHeightLast = nHeightIn; + if (nTimeIn > nTimeLast) + nTimeLast = nTimeIn; + } +}; + struct CDiskBlockPos { int nFile; @@ -54,7 +108,7 @@ struct CDiskBlockPos }; -enum BlockStatus { +enum BlockStatus: uint32_t { //! Unused. BLOCK_VALID_UNKNOWN = 0, diff --git a/src/chainparams.cpp b/src/chainparams.cpp index 445616b313045..44b2cc0254920 100644 --- a/src/chainparams.cpp +++ b/src/chainparams.cpp @@ -16,6 +16,14 @@ #include "chainparamsseeds.h" +std::string CDNSSeedData::getHost(uint64_t requiredServiceBits) const { + //use default host for non-filter-capable seeds or if we use the default service bits (NODE_NETWORK) + if (!supportsServiceBitsFiltering || requiredServiceBits == NODE_NETWORK) + return host; + + return strprintf("x%x.%s", requiredServiceBits, host); +} + static CBlock CreateGenesisBlock(const char* pszTimestamp, const CScript& genesisOutputScript, uint32_t nTime, uint32_t nNonce, uint32_t nBits, int32_t nVersion, const CAmount& genesisReward) { CMutableTransaction txNew; @@ -207,6 +215,8 @@ class CTestNetParams : public CChainParams { vFixedSeeds.clear(); vSeeds.clear(); + // nodes with support for servicebits filtering should be at the top + vSeeds.push_back(CDNSSeedData("testnetbitcoin.jonasschnelli.ch", "testnet-seed.bitcoin.jonasschnelli.ch", true)); vSeeds.push_back(CDNSSeedData("bitcoin.petertodd.org", "testnet-seed.bitcoin.petertodd.org")); vSeeds.push_back(CDNSSeedData("bluematt.me", "testnet-seed.bluematt.me")); vSeeds.push_back(CDNSSeedData("bitcoin.schildbach.de", "testnet-seed.bitcoin.schildbach.de")); diff --git a/src/chainparams.h b/src/chainparams.h index 59202f548a3ca..7168daaf4347c 100644 --- a/src/chainparams.h +++ b/src/chainparams.h @@ -13,9 +13,12 @@ #include -struct CDNSSeedData { +class CDNSSeedData { +public: std::string name, host; - CDNSSeedData(const std::string &strName, const std::string &strHost) : name(strName), host(strHost) {} + bool supportsServiceBitsFiltering; + std::string getHost(uint64_t requiredServiceBits) const; + CDNSSeedData(const std::string &strName, const std::string &strHost, bool supportsServiceBitsFilteringIn = false) : name(strName), host(strHost), supportsServiceBitsFiltering(supportsServiceBitsFilteringIn) {} }; struct SeedSpec6 { diff --git a/src/clientversion.cpp b/src/clientversion.cpp index aae0569bba211..bfe9e16f80ee7 100644 --- a/src/clientversion.cpp +++ b/src/clientversion.cpp @@ -67,16 +67,7 @@ const std::string CLIENT_NAME("Satoshi"); #endif #endif -#ifndef BUILD_DATE -#ifdef GIT_COMMIT_DATE -#define BUILD_DATE GIT_COMMIT_DATE -#else -#define BUILD_DATE __DATE__ ", " __TIME__ -#endif -#endif - const std::string CLIENT_BUILD(BUILD_DESC CLIENT_VERSION_SUFFIX); -const std::string CLIENT_DATE(BUILD_DATE); static std::string FormatVersion(int nVersion) { diff --git a/src/clientversion.h b/src/clientversion.h index 6f255d69c96ef..47263d5344f3c 100644 --- a/src/clientversion.h +++ b/src/clientversion.h @@ -59,7 +59,6 @@ static const int CLIENT_VERSION = extern const std::string CLIENT_NAME; extern const std::string CLIENT_BUILD; -extern const std::string CLIENT_DATE; std::string FormatFullVersion(); diff --git a/src/coincontrol.h b/src/coincontrol.h index 12fe9ce219c27..e33adc4d2b00c 100644 --- a/src/coincontrol.h +++ b/src/coincontrol.h @@ -18,6 +18,10 @@ class CCoinControl bool fAllowWatchOnly; //! Minimum absolute fee (not per kilobyte) CAmount nMinimumTotalFee; + //! Override estimated feerate + bool fOverrideFeeRate; + //! Feerate to use if overrideFeeRate is true + CFeeRate nFeeRate; CCoinControl() { @@ -31,6 +35,8 @@ class CCoinControl fAllowWatchOnly = false; setSelected.clear(); nMinimumTotalFee = 0; + nFeeRate = CFeeRate(0); + fOverrideFeeRate = false; } bool HasSelected() const diff --git a/src/coins.cpp b/src/coins.cpp index 1c329740b45f6..39db7dedfbbe1 100644 --- a/src/coins.cpp +++ b/src/coins.cpp @@ -56,7 +56,7 @@ void CCoinsViewBacked::SetBackend(CCoinsView &viewIn) { base = &viewIn; } bool CCoinsViewBacked::BatchWrite(CCoinsMap &mapCoins, const uint256 &hashBlock) { return base->BatchWrite(mapCoins, hashBlock); } CCoinsViewCursor *CCoinsViewBacked::Cursor() const { return base->Cursor(); } -CCoinsKeyHasher::CCoinsKeyHasher() : salt(GetRandHash()) {} +SaltedTxidHasher::SaltedTxidHasher() : k0(GetRand(std::numeric_limits::max())), k1(GetRand(std::numeric_limits::max())) {} CCoinsViewCache::CCoinsViewCache(CCoinsView *baseIn) : CCoinsViewBacked(baseIn), hasModifier(false), cachedCoinsUsage(0) { } diff --git a/src/coins.h b/src/coins.h index d72f8854731eb..033651a435c04 100644 --- a/src/coins.h +++ b/src/coins.h @@ -8,6 +8,7 @@ #include "compressor.h" #include "core_memusage.h" +#include "hash.h" #include "memusage.h" #include "serialize.h" #include "uint256.h" @@ -264,21 +265,22 @@ class CCoins } }; -class CCoinsKeyHasher +class SaltedTxidHasher { private: - uint256 salt; + /** Salt */ + const uint64_t k0, k1; public: - CCoinsKeyHasher(); + SaltedTxidHasher(); /** * This *must* return size_t. With Boost 1.46 on 32-bit systems the * unordered_map will behave unpredictably if the custom hasher returns a * uint64_t, resulting in failures when syncing the chain (#4634). */ - size_t operator()(const uint256& key) const { - return key.GetHash(salt); + size_t operator()(const uint256& txid) const { + return SipHashUint256(k0, k1, txid); } }; @@ -295,7 +297,7 @@ struct CCoinsCacheEntry CCoinsCacheEntry() : coins(), flags(0) {} }; -typedef boost::unordered_map CCoinsMap; +typedef boost::unordered_map CCoinsMap; /** Cursor for iterating over CoinsView state */ class CCoinsViewCursor diff --git a/src/compat.h b/src/compat.h index 1225ea18edb85..79a297e5e4010 100644 --- a/src/compat.h +++ b/src/compat.h @@ -78,17 +78,6 @@ typedef u_int SOCKET; #define MSG_NOSIGNAL 0 #endif -#ifndef WIN32 -// PRIO_MAX is not defined on Solaris -#ifndef PRIO_MAX -#define PRIO_MAX 20 -#endif -#define THREAD_PRIORITY_LOWEST PRIO_MAX -#define THREAD_PRIORITY_BELOW_NORMAL 2 -#define THREAD_PRIORITY_NORMAL 0 -#define THREAD_PRIORITY_ABOVE_NORMAL (-2) -#endif - #if HAVE_DECL_STRNLEN == 0 size_t strnlen( const char *start, size_t max_len); #endif // HAVE_DECL_STRNLEN diff --git a/src/compressor.h b/src/compressor.h index 4a72090830587..fa702f0dfa900 100644 --- a/src/compressor.h +++ b/src/compressor.h @@ -86,8 +86,14 @@ class CScriptCompressor return; } nSize -= nSpecialScripts; - script.resize(nSize); - s >> REF(CFlatData(script)); + if (nSize > MAX_SCRIPT_SIZE) { + // Overly long script, replace with a short invalid one + script << OP_RETURN; + s.ignore(nSize); + } else { + script.resize(nSize); + s >> REF(CFlatData(script)); + } } }; diff --git a/src/consensus/params.h b/src/consensus/params.h index 5ddfba8209e0b..48e65307b8312 100644 --- a/src/consensus/params.h +++ b/src/consensus/params.h @@ -17,6 +17,7 @@ enum DeploymentPos DEPLOYMENT_TESTDUMMY, DEPLOYMENT_CSV, // Deployment of BIP68, BIP112, and BIP113. DEPLOYMENT_WITNESS, // Deployment of BIP141 and BIP143 + // NOTE: Also add new deployments to VersionBitsDeploymentInfo in versionbits.cpp MAX_VERSION_BITS_DEPLOYMENTS }; diff --git a/src/crypto/aes.cpp b/src/crypto/aes.cpp new file mode 100644 index 0000000000000..1d469d0fb4622 --- /dev/null +++ b/src/crypto/aes.cpp @@ -0,0 +1,217 @@ +// Copyright (c) 2016 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include "aes.h" +#include "crypto/common.h" + +#include +#include + +extern "C" { +#include "crypto/ctaes/ctaes.c" +} + +AES128Encrypt::AES128Encrypt(const unsigned char key[16]) +{ + AES128_init(&ctx, key); +} + +AES128Encrypt::~AES128Encrypt() +{ + memset(&ctx, 0, sizeof(ctx)); +} + +void AES128Encrypt::Encrypt(unsigned char ciphertext[16], const unsigned char plaintext[16]) const +{ + AES128_encrypt(&ctx, 1, ciphertext, plaintext); +} + +AES128Decrypt::AES128Decrypt(const unsigned char key[16]) +{ + AES128_init(&ctx, key); +} + +AES128Decrypt::~AES128Decrypt() +{ + memset(&ctx, 0, sizeof(ctx)); +} + +void AES128Decrypt::Decrypt(unsigned char plaintext[16], const unsigned char ciphertext[16]) const +{ + AES128_decrypt(&ctx, 1, plaintext, ciphertext); +} + +AES256Encrypt::AES256Encrypt(const unsigned char key[32]) +{ + AES256_init(&ctx, key); +} + +AES256Encrypt::~AES256Encrypt() +{ + memset(&ctx, 0, sizeof(ctx)); +} + +void AES256Encrypt::Encrypt(unsigned char ciphertext[16], const unsigned char plaintext[16]) const +{ + AES256_encrypt(&ctx, 1, ciphertext, plaintext); +} + +AES256Decrypt::AES256Decrypt(const unsigned char key[32]) +{ + AES256_init(&ctx, key); +} + +AES256Decrypt::~AES256Decrypt() +{ + memset(&ctx, 0, sizeof(ctx)); +} + +void AES256Decrypt::Decrypt(unsigned char plaintext[16], const unsigned char ciphertext[16]) const +{ + AES256_decrypt(&ctx, 1, plaintext, ciphertext); +} + + +template +static int CBCEncrypt(const T& enc, const unsigned char iv[AES_BLOCKSIZE], const unsigned char* data, int size, bool pad, unsigned char* out) +{ + int written = 0; + int padsize = size % AES_BLOCKSIZE; + unsigned char mixed[AES_BLOCKSIZE]; + + if (!data || !size || !out) + return 0; + + if (!pad && padsize != 0) + return 0; + + memcpy(mixed, iv, AES_BLOCKSIZE); + + // Write all but the last block + while (written + AES_BLOCKSIZE <= size) { + for (int i = 0; i != AES_BLOCKSIZE; i++) + mixed[i] ^= *data++; + enc.Encrypt(out + written, mixed); + memcpy(mixed, out + written, AES_BLOCKSIZE); + written += AES_BLOCKSIZE; + } + if (pad) { + // For all that remains, pad each byte with the value of the remaining + // space. If there is none, pad by a full block. + for (int i = 0; i != padsize; i++) + mixed[i] ^= *data++; + for (int i = padsize; i != AES_BLOCKSIZE; i++) + mixed[i] ^= AES_BLOCKSIZE - padsize; + enc.Encrypt(out + written, mixed); + written += AES_BLOCKSIZE; + } + return written; +} + +template +static int CBCDecrypt(const T& dec, const unsigned char iv[AES_BLOCKSIZE], const unsigned char* data, int size, bool pad, unsigned char* out) +{ + unsigned char padsize = 0; + int written = 0; + bool fail = false; + const unsigned char* prev = iv; + + if (!data || !size || !out) + return 0; + + if (size % AES_BLOCKSIZE != 0) + return 0; + + // Decrypt all data. Padding will be checked in the output. + while (written != size) { + dec.Decrypt(out, data + written); + for (int i = 0; i != AES_BLOCKSIZE; i++) + *out++ ^= prev[i]; + prev = data + written; + written += AES_BLOCKSIZE; + } + + // When decrypting padding, attempt to run in constant-time + if (pad) { + // If used, padding size is the value of the last decrypted byte. For + // it to be valid, It must be between 1 and AES_BLOCKSIZE. + padsize = *--out; + fail = !padsize | (padsize > AES_BLOCKSIZE); + + // If not well-formed, treat it as though there's no padding. + padsize *= !fail; + + // All padding must equal the last byte otherwise it's not well-formed + for (int i = AES_BLOCKSIZE; i != 0; i--) + fail |= ((i > AES_BLOCKSIZE - padsize) & (*out-- != padsize)); + + written -= padsize; + } + return written * !fail; +} + +AES256CBCEncrypt::AES256CBCEncrypt(const unsigned char key[AES256_KEYSIZE], const unsigned char ivIn[AES_BLOCKSIZE], bool padIn) + : enc(key), pad(padIn) +{ + memcpy(iv, ivIn, AES_BLOCKSIZE); +} + +int AES256CBCEncrypt::Encrypt(const unsigned char* data, int size, unsigned char* out) const +{ + return CBCEncrypt(enc, iv, data, size, pad, out); +} + +AES256CBCEncrypt::~AES256CBCEncrypt() +{ + memset(iv, 0, sizeof(iv)); +} + +AES256CBCDecrypt::AES256CBCDecrypt(const unsigned char key[AES256_KEYSIZE], const unsigned char ivIn[AES_BLOCKSIZE], bool padIn) + : dec(key), pad(padIn) +{ + memcpy(iv, ivIn, AES_BLOCKSIZE); +} + + +int AES256CBCDecrypt::Decrypt(const unsigned char* data, int size, unsigned char* out) const +{ + return CBCDecrypt(dec, iv, data, size, pad, out); +} + +AES256CBCDecrypt::~AES256CBCDecrypt() +{ + memset(iv, 0, sizeof(iv)); +} + +AES128CBCEncrypt::AES128CBCEncrypt(const unsigned char key[AES128_KEYSIZE], const unsigned char ivIn[AES_BLOCKSIZE], bool padIn) + : enc(key), pad(padIn) +{ + memcpy(iv, ivIn, AES_BLOCKSIZE); +} + +AES128CBCEncrypt::~AES128CBCEncrypt() +{ + memset(iv, 0, AES_BLOCKSIZE); +} + +int AES128CBCEncrypt::Encrypt(const unsigned char* data, int size, unsigned char* out) const +{ + return CBCEncrypt(enc, iv, data, size, pad, out); +} + +AES128CBCDecrypt::AES128CBCDecrypt(const unsigned char key[AES128_KEYSIZE], const unsigned char ivIn[AES_BLOCKSIZE], bool padIn) + : dec(key), pad(padIn) +{ + memcpy(iv, ivIn, AES_BLOCKSIZE); +} + +AES128CBCDecrypt::~AES128CBCDecrypt() +{ + memset(iv, 0, AES_BLOCKSIZE); +} + +int AES128CBCDecrypt::Decrypt(const unsigned char* data, int size, unsigned char* out) const +{ + return CBCDecrypt(dec, iv, data, size, pad, out); +} diff --git a/src/crypto/aes.h b/src/crypto/aes.h new file mode 100644 index 0000000000000..8cae357c1287e --- /dev/null +++ b/src/crypto/aes.h @@ -0,0 +1,118 @@ +// Copyright (c) 2015 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. +// +// C++ wrapper around ctaes, a constant-time AES implementation + +#ifndef BITCOIN_CRYPTO_AES_H +#define BITCOIN_CRYPTO_AES_H + +extern "C" { +#include "crypto/ctaes/ctaes.h" +} + +static const int AES_BLOCKSIZE = 16; +static const int AES128_KEYSIZE = 16; +static const int AES256_KEYSIZE = 32; + +/** An encryption class for AES-128. */ +class AES128Encrypt +{ +private: + AES128_ctx ctx; + +public: + AES128Encrypt(const unsigned char key[16]); + ~AES128Encrypt(); + void Encrypt(unsigned char ciphertext[16], const unsigned char plaintext[16]) const; +}; + +/** A decryption class for AES-128. */ +class AES128Decrypt +{ +private: + AES128_ctx ctx; + +public: + AES128Decrypt(const unsigned char key[16]); + ~AES128Decrypt(); + void Decrypt(unsigned char plaintext[16], const unsigned char ciphertext[16]) const; +}; + +/** An encryption class for AES-256. */ +class AES256Encrypt +{ +private: + AES256_ctx ctx; + +public: + AES256Encrypt(const unsigned char key[32]); + ~AES256Encrypt(); + void Encrypt(unsigned char ciphertext[16], const unsigned char plaintext[16]) const; +}; + +/** A decryption class for AES-256. */ +class AES256Decrypt +{ +private: + AES256_ctx ctx; + +public: + AES256Decrypt(const unsigned char key[32]); + ~AES256Decrypt(); + void Decrypt(unsigned char plaintext[16], const unsigned char ciphertext[16]) const; +}; + +class AES256CBCEncrypt +{ +public: + AES256CBCEncrypt(const unsigned char key[AES256_KEYSIZE], const unsigned char ivIn[AES_BLOCKSIZE], bool padIn); + ~AES256CBCEncrypt(); + int Encrypt(const unsigned char* data, int size, unsigned char* out) const; + +private: + const AES256Encrypt enc; + const bool pad; + unsigned char iv[AES_BLOCKSIZE]; +}; + +class AES256CBCDecrypt +{ +public: + AES256CBCDecrypt(const unsigned char key[AES256_KEYSIZE], const unsigned char ivIn[AES_BLOCKSIZE], bool padIn); + ~AES256CBCDecrypt(); + int Decrypt(const unsigned char* data, int size, unsigned char* out) const; + +private: + const AES256Decrypt dec; + const bool pad; + unsigned char iv[AES_BLOCKSIZE]; +}; + +class AES128CBCEncrypt +{ +public: + AES128CBCEncrypt(const unsigned char key[AES128_KEYSIZE], const unsigned char ivIn[AES_BLOCKSIZE], bool padIn); + ~AES128CBCEncrypt(); + int Encrypt(const unsigned char* data, int size, unsigned char* out) const; + +private: + const AES128Encrypt enc; + const bool pad; + unsigned char iv[AES_BLOCKSIZE]; +}; + +class AES128CBCDecrypt +{ +public: + AES128CBCDecrypt(const unsigned char key[AES128_KEYSIZE], const unsigned char ivIn[AES_BLOCKSIZE], bool padIn); + ~AES128CBCDecrypt(); + int Decrypt(const unsigned char* data, int size, unsigned char* out) const; + +private: + const AES128Decrypt dec; + const bool pad; + unsigned char iv[AES_BLOCKSIZE]; +}; + +#endif // BITCOIN_CRYPTO_AES_H diff --git a/src/crypto/ctaes/COPYING b/src/crypto/ctaes/COPYING new file mode 100644 index 0000000000000..415b202a2a54e --- /dev/null +++ b/src/crypto/ctaes/COPYING @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2016 Pieter Wuille + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/src/crypto/ctaes/README.md b/src/crypto/ctaes/README.md new file mode 100644 index 0000000000000..0e7fe1775153e --- /dev/null +++ b/src/crypto/ctaes/README.md @@ -0,0 +1,41 @@ +ctaes +===== + +Simple C module for constant-time AES encryption and decryption. + +Features: +* Simple, pure C code without any dependencies. +* No tables or data-dependent branches whatsoever, but using bit sliced approach from https://eprint.iacr.org/2009/129.pdf. +* Very small object code: slightly over 4k of executable code when compiled with -Os. +* Slower than implementations based on precomputed tables or specialized instructions, but can do ~15 MB/s on modern CPUs. + +Performance +----------- + +Compiled with GCC 5.3.1 with -O3, on an Intel(R) Core(TM) i7-4800MQ CPU, numbers in CPU cycles: + +| Algorithm | Key schedule | Encryption per byte | Decryption per byte | +| --------- | ------------:| -------------------:| -------------------:| +| AES-128 | 2.8k | 154 | 161 | +| AES-192 | 3.1k | 169 | 181 | +| AES-256 | 4.0k | 191 | 203 | + +Build steps +----------- + +Object code: + + $ gcc -O3 ctaes.c -c -o ctaes.o + +Tests: + + $ gcc -O3 ctaes.c test.c -o test + +Benchmark: + + $ gcc -O3 ctaes.c bench.c -o bench + +Review +------ + +Results of a formal review of the code can be found in http://bitcoin.sipa.be/ctaes/review.zip diff --git a/src/crypto/ctaes/bench.c b/src/crypto/ctaes/bench.c new file mode 100644 index 0000000000000..a86df496c8300 --- /dev/null +++ b/src/crypto/ctaes/bench.c @@ -0,0 +1,170 @@ +#include +#include +#include "sys/time.h" + +#include "ctaes.h" + +static double gettimedouble(void) { + struct timeval tv; + gettimeofday(&tv, NULL); + return tv.tv_usec * 0.000001 + tv.tv_sec; +} + +static void print_number(double x) { + double y = x; + int c = 0; + if (y < 0.0) { + y = -y; + } + while (y < 100.0) { + y *= 10.0; + c++; + } + printf("%.*f", c, x); +} + +static void run_benchmark(char *name, void (*benchmark)(void*), void (*setup)(void*), void (*teardown)(void*), void* data, int count, int iter) { + int i; + double min = HUGE_VAL; + double sum = 0.0; + double max = 0.0; + for (i = 0; i < count; i++) { + double begin, total; + if (setup != NULL) { + setup(data); + } + begin = gettimedouble(); + benchmark(data); + total = gettimedouble() - begin; + if (teardown != NULL) { + teardown(data); + } + if (total < min) { + min = total; + } + if (total > max) { + max = total; + } + sum += total; + } + printf("%s: min ", name); + print_number(min * 1000000000.0 / iter); + printf("ns / avg "); + print_number((sum / count) * 1000000000.0 / iter); + printf("ns / max "); + print_number(max * 1000000000.0 / iter); + printf("ns\n"); +} + +static void bench_AES128_init(void* data) { + AES128_ctx* ctx = (AES128_ctx*)data; + int i; + for (i = 0; i < 50000; i++) { + AES128_init(ctx, (unsigned char*)ctx); + } +} + +static void bench_AES128_encrypt_setup(void* data) { + AES128_ctx* ctx = (AES128_ctx*)data; + static const unsigned char key[16] = {0}; + AES128_init(ctx, key); +} + +static void bench_AES128_encrypt(void* data) { + const AES128_ctx* ctx = (const AES128_ctx*)data; + unsigned char scratch[16] = {0}; + int i; + for (i = 0; i < 4000000 / 16; i++) { + AES128_encrypt(ctx, 1, scratch, scratch); + } +} + +static void bench_AES128_decrypt(void* data) { + const AES128_ctx* ctx = (const AES128_ctx*)data; + unsigned char scratch[16] = {0}; + int i; + for (i = 0; i < 4000000 / 16; i++) { + AES128_decrypt(ctx, 1, scratch, scratch); + } +} + +static void bench_AES192_init(void* data) { + AES192_ctx* ctx = (AES192_ctx*)data; + int i; + for (i = 0; i < 50000; i++) { + AES192_init(ctx, (unsigned char*)ctx); + } +} + +static void bench_AES192_encrypt_setup(void* data) { + AES192_ctx* ctx = (AES192_ctx*)data; + static const unsigned char key[16] = {0}; + AES192_init(ctx, key); +} + +static void bench_AES192_encrypt(void* data) { + const AES192_ctx* ctx = (const AES192_ctx*)data; + unsigned char scratch[16] = {0}; + int i; + for (i = 0; i < 4000000 / 16; i++) { + AES192_encrypt(ctx, 1, scratch, scratch); + } +} + +static void bench_AES192_decrypt(void* data) { + const AES192_ctx* ctx = (const AES192_ctx*)data; + unsigned char scratch[16] = {0}; + int i; + for (i = 0; i < 4000000 / 16; i++) { + AES192_decrypt(ctx, 1, scratch, scratch); + } +} + +static void bench_AES256_init(void* data) { + AES256_ctx* ctx = (AES256_ctx*)data; + int i; + for (i = 0; i < 50000; i++) { + AES256_init(ctx, (unsigned char*)ctx); + } +} + + +static void bench_AES256_encrypt_setup(void* data) { + AES256_ctx* ctx = (AES256_ctx*)data; + static const unsigned char key[16] = {0}; + AES256_init(ctx, key); +} + +static void bench_AES256_encrypt(void* data) { + const AES256_ctx* ctx = (const AES256_ctx*)data; + unsigned char scratch[16] = {0}; + int i; + for (i = 0; i < 4000000 / 16; i++) { + AES256_encrypt(ctx, 1, scratch, scratch); + } +} + +static void bench_AES256_decrypt(void* data) { + const AES256_ctx* ctx = (const AES256_ctx*)data; + unsigned char scratch[16] = {0}; + int i; + for (i = 0; i < 4000000 / 16; i++) { + AES256_decrypt(ctx, 1, scratch, scratch); + } +} + +int main(void) { + AES128_ctx ctx128; + AES192_ctx ctx192; + AES256_ctx ctx256; + run_benchmark("aes128_init", bench_AES128_init, NULL, NULL, &ctx128, 20, 50000); + run_benchmark("aes128_encrypt_byte", bench_AES128_encrypt, bench_AES128_encrypt_setup, NULL, &ctx128, 20, 4000000); + run_benchmark("aes128_decrypt_byte", bench_AES128_decrypt, bench_AES128_encrypt_setup, NULL, &ctx128, 20, 4000000); + run_benchmark("aes192_init", bench_AES192_init, NULL, NULL, &ctx192, 20, 50000); + run_benchmark("aes192_encrypt_byte", bench_AES192_encrypt, bench_AES192_encrypt_setup, NULL, &ctx192, 20, 4000000); + run_benchmark("aes192_decrypt_byte", bench_AES192_decrypt, bench_AES192_encrypt_setup, NULL, &ctx192, 20, 4000000); + run_benchmark("aes256_init", bench_AES256_init, NULL, NULL, &ctx256, 20, 50000); + run_benchmark("aes256_encrypt_byte", bench_AES256_encrypt, bench_AES256_encrypt_setup, NULL, &ctx256, 20, 4000000); + run_benchmark("aes256_decrypt_byte", bench_AES256_decrypt, bench_AES256_encrypt_setup, NULL, &ctx256, 20, 4000000); + return 0; +} diff --git a/src/crypto/ctaes/ctaes.c b/src/crypto/ctaes/ctaes.c new file mode 100644 index 0000000000000..2389fc0bb254f --- /dev/null +++ b/src/crypto/ctaes/ctaes.c @@ -0,0 +1,556 @@ + /********************************************************************* + * Copyright (c) 2016 Pieter Wuille * + * Distributed under the MIT software license, see the accompanying * + * file COPYING or http://www.opensource.org/licenses/mit-license.php.* + **********************************************************************/ + +/* Constant time, unoptimized, concise, plain C, AES implementation + * Based On: + * Emilia Kasper and Peter Schwabe, Faster and Timing-Attack Resistant AES-GCM + * http://www.iacr.org/archive/ches2009/57470001/57470001.pdf + * But using 8 16-bit integers representing a single AES state rather than 8 128-bit + * integers representing 8 AES states. + */ + +#include "ctaes.h" + +/* Slice variable slice_i contains the i'th bit of the 16 state variables in this order: + * 0 1 2 3 + * 4 5 6 7 + * 8 9 10 11 + * 12 13 14 15 + */ + +/** Convert a byte to sliced form, storing it corresponding to given row and column in s */ +static void LoadByte(AES_state* s, unsigned char byte, int r, int c) { + int i; + for (i = 0; i < 8; i++) { + s->slice[i] |= (byte & 1) << (r * 4 + c); + byte >>= 1; + } +} + +/** Load 16 bytes of data into 8 sliced integers */ +static void LoadBytes(AES_state *s, const unsigned char* data16) { + int c; + for (c = 0; c < 4; c++) { + int r; + for (r = 0; r < 4; r++) { + LoadByte(s, *(data16++), r, c); + } + } +} + +/** Convert 8 sliced integers into 16 bytes of data */ +static void SaveBytes(unsigned char* data16, const AES_state *s) { + int c; + for (c = 0; c < 4; c++) { + int r; + for (r = 0; r < 4; r++) { + int b; + uint8_t v = 0; + for (b = 0; b < 8; b++) { + v |= ((s->slice[b] >> (r * 4 + c)) & 1) << b; + } + *(data16++) = v; + } + } +} + +/* S-box implementation based on the gate logic from: + * Joan Boyar and Rene Peralta, A depth-16 circuit for the AES S-box. + * https://eprint.iacr.org/2011/332.pdf +*/ +static void SubBytes(AES_state *s, int inv) { + /* Load the bit slices */ + uint16_t U0 = s->slice[7], U1 = s->slice[6], U2 = s->slice[5], U3 = s->slice[4]; + uint16_t U4 = s->slice[3], U5 = s->slice[2], U6 = s->slice[1], U7 = s->slice[0]; + + uint16_t T1, T2, T3, T4, T5, T6, T7, T8, T9, T10, T11, T12, T13, T14, T15, T16; + uint16_t T17, T18, T19, T20, T21, T22, T23, T24, T25, T26, T27, D; + uint16_t M1, M6, M11, M13, M15, M20, M21, M22, M23, M25, M37, M38, M39, M40; + uint16_t M41, M42, M43, M44, M45, M46, M47, M48, M49, M50, M51, M52, M53, M54; + uint16_t M55, M56, M57, M58, M59, M60, M61, M62, M63; + + if (inv) { + uint16_t R5, R13, R17, R18, R19; + /* Undo linear postprocessing */ + T23 = U0 ^ U3; + T22 = ~(U1 ^ U3); + T2 = ~(U0 ^ U1); + T1 = U3 ^ U4; + T24 = ~(U4 ^ U7); + R5 = U6 ^ U7; + T8 = ~(U1 ^ T23); + T19 = T22 ^ R5; + T9 = ~(U7 ^ T1); + T10 = T2 ^ T24; + T13 = T2 ^ R5; + T3 = T1 ^ R5; + T25 = ~(U2 ^ T1); + R13 = U1 ^ U6; + T17 = ~(U2 ^ T19); + T20 = T24 ^ R13; + T4 = U4 ^ T8; + R17 = ~(U2 ^ U5); + R18 = ~(U5 ^ U6); + R19 = ~(U2 ^ U4); + D = U0 ^ R17; + T6 = T22 ^ R17; + T16 = R13 ^ R19; + T27 = T1 ^ R18; + T15 = T10 ^ T27; + T14 = T10 ^ R18; + T26 = T3 ^ T16; + } else { + /* Linear preprocessing. */ + T1 = U0 ^ U3; + T2 = U0 ^ U5; + T3 = U0 ^ U6; + T4 = U3 ^ U5; + T5 = U4 ^ U6; + T6 = T1 ^ T5; + T7 = U1 ^ U2; + T8 = U7 ^ T6; + T9 = U7 ^ T7; + T10 = T6 ^ T7; + T11 = U1 ^ U5; + T12 = U2 ^ U5; + T13 = T3 ^ T4; + T14 = T6 ^ T11; + T15 = T5 ^ T11; + T16 = T5 ^ T12; + T17 = T9 ^ T16; + T18 = U3 ^ U7; + T19 = T7 ^ T18; + T20 = T1 ^ T19; + T21 = U6 ^ U7; + T22 = T7 ^ T21; + T23 = T2 ^ T22; + T24 = T2 ^ T10; + T25 = T20 ^ T17; + T26 = T3 ^ T16; + T27 = T1 ^ T12; + D = U7; + } + + /* Non-linear transformation (identical to the code in SubBytes) */ + M1 = T13 & T6; + M6 = T3 & T16; + M11 = T1 & T15; + M13 = (T4 & T27) ^ M11; + M15 = (T2 & T10) ^ M11; + M20 = T14 ^ M1 ^ (T23 & T8) ^ M13; + M21 = (T19 & D) ^ M1 ^ T24 ^ M15; + M22 = T26 ^ M6 ^ (T22 & T9) ^ M13; + M23 = (T20 & T17) ^ M6 ^ M15 ^ T25; + M25 = M22 & M20; + M37 = M21 ^ ((M20 ^ M21) & (M23 ^ M25)); + M38 = M20 ^ M25 ^ (M21 | (M20 & M23)); + M39 = M23 ^ ((M22 ^ M23) & (M21 ^ M25)); + M40 = M22 ^ M25 ^ (M23 | (M21 & M22)); + M41 = M38 ^ M40; + M42 = M37 ^ M39; + M43 = M37 ^ M38; + M44 = M39 ^ M40; + M45 = M42 ^ M41; + M46 = M44 & T6; + M47 = M40 & T8; + M48 = M39 & D; + M49 = M43 & T16; + M50 = M38 & T9; + M51 = M37 & T17; + M52 = M42 & T15; + M53 = M45 & T27; + M54 = M41 & T10; + M55 = M44 & T13; + M56 = M40 & T23; + M57 = M39 & T19; + M58 = M43 & T3; + M59 = M38 & T22; + M60 = M37 & T20; + M61 = M42 & T1; + M62 = M45 & T4; + M63 = M41 & T2; + + if (inv){ + /* Undo linear preprocessing */ + uint16_t P0 = M52 ^ M61; + uint16_t P1 = M58 ^ M59; + uint16_t P2 = M54 ^ M62; + uint16_t P3 = M47 ^ M50; + uint16_t P4 = M48 ^ M56; + uint16_t P5 = M46 ^ M51; + uint16_t P6 = M49 ^ M60; + uint16_t P7 = P0 ^ P1; + uint16_t P8 = M50 ^ M53; + uint16_t P9 = M55 ^ M63; + uint16_t P10 = M57 ^ P4; + uint16_t P11 = P0 ^ P3; + uint16_t P12 = M46 ^ M48; + uint16_t P13 = M49 ^ M51; + uint16_t P14 = M49 ^ M62; + uint16_t P15 = M54 ^ M59; + uint16_t P16 = M57 ^ M61; + uint16_t P17 = M58 ^ P2; + uint16_t P18 = M63 ^ P5; + uint16_t P19 = P2 ^ P3; + uint16_t P20 = P4 ^ P6; + uint16_t P22 = P2 ^ P7; + uint16_t P23 = P7 ^ P8; + uint16_t P24 = P5 ^ P7; + uint16_t P25 = P6 ^ P10; + uint16_t P26 = P9 ^ P11; + uint16_t P27 = P10 ^ P18; + uint16_t P28 = P11 ^ P25; + uint16_t P29 = P15 ^ P20; + s->slice[7] = P13 ^ P22; + s->slice[6] = P26 ^ P29; + s->slice[5] = P17 ^ P28; + s->slice[4] = P12 ^ P22; + s->slice[3] = P23 ^ P27; + s->slice[2] = P19 ^ P24; + s->slice[1] = P14 ^ P23; + s->slice[0] = P9 ^ P16; + } else { + /* Linear postprocessing */ + uint16_t L0 = M61 ^ M62; + uint16_t L1 = M50 ^ M56; + uint16_t L2 = M46 ^ M48; + uint16_t L3 = M47 ^ M55; + uint16_t L4 = M54 ^ M58; + uint16_t L5 = M49 ^ M61; + uint16_t L6 = M62 ^ L5; + uint16_t L7 = M46 ^ L3; + uint16_t L8 = M51 ^ M59; + uint16_t L9 = M52 ^ M53; + uint16_t L10 = M53 ^ L4; + uint16_t L11 = M60 ^ L2; + uint16_t L12 = M48 ^ M51; + uint16_t L13 = M50 ^ L0; + uint16_t L14 = M52 ^ M61; + uint16_t L15 = M55 ^ L1; + uint16_t L16 = M56 ^ L0; + uint16_t L17 = M57 ^ L1; + uint16_t L18 = M58 ^ L8; + uint16_t L19 = M63 ^ L4; + uint16_t L20 = L0 ^ L1; + uint16_t L21 = L1 ^ L7; + uint16_t L22 = L3 ^ L12; + uint16_t L23 = L18 ^ L2; + uint16_t L24 = L15 ^ L9; + uint16_t L25 = L6 ^ L10; + uint16_t L26 = L7 ^ L9; + uint16_t L27 = L8 ^ L10; + uint16_t L28 = L11 ^ L14; + uint16_t L29 = L11 ^ L17; + s->slice[7] = L6 ^ L24; + s->slice[6] = ~(L16 ^ L26); + s->slice[5] = ~(L19 ^ L28); + s->slice[4] = L6 ^ L21; + s->slice[3] = L20 ^ L22; + s->slice[2] = L25 ^ L29; + s->slice[1] = ~(L13 ^ L27); + s->slice[0] = ~(L6 ^ L23); + } +} + +#define BIT_RANGE(from,to) (((1 << ((to) - (from))) - 1) << (from)) + +#define BIT_RANGE_LEFT(x,from,to,shift) (((x) & BIT_RANGE((from), (to))) << (shift)) +#define BIT_RANGE_RIGHT(x,from,to,shift) (((x) & BIT_RANGE((from), (to))) >> (shift)) + +static void ShiftRows(AES_state* s) { + int i; + for (i = 0; i < 8; i++) { + uint16_t v = s->slice[i]; + s->slice[i] = + (v & BIT_RANGE(0, 4)) | + BIT_RANGE_LEFT(v, 4, 5, 3) | BIT_RANGE_RIGHT(v, 5, 8, 1) | + BIT_RANGE_LEFT(v, 8, 10, 2) | BIT_RANGE_RIGHT(v, 10, 12, 2) | + BIT_RANGE_LEFT(v, 12, 15, 1) | BIT_RANGE_RIGHT(v, 15, 16, 3); + } +} + +static void InvShiftRows(AES_state* s) { + int i; + for (i = 0; i < 8; i++) { + uint16_t v = s->slice[i]; + s->slice[i] = + (v & BIT_RANGE(0, 4)) | + BIT_RANGE_LEFT(v, 4, 7, 1) | BIT_RANGE_RIGHT(v, 7, 8, 3) | + BIT_RANGE_LEFT(v, 8, 10, 2) | BIT_RANGE_RIGHT(v, 10, 12, 2) | + BIT_RANGE_LEFT(v, 12, 13, 3) | BIT_RANGE_RIGHT(v, 13, 16, 1); + } +} + +#define ROT(x,b) (((x) >> ((b) * 4)) | ((x) << ((4-(b)) * 4))) + +static void MixColumns(AES_state* s, int inv) { + /* The MixColumns transform treats the bytes of the columns of the state as + * coefficients of a 3rd degree polynomial over GF(2^8) and multiplies them + * by the fixed polynomial a(x) = {03}x^3 + {01}x^2 + {01}x + {02}, modulo + * x^4 + {01}. + * + * In the inverse transform, we multiply by the inverse of a(x), + * a^-1(x) = {0b}x^3 + {0d}x^2 + {09}x + {0e}. This is equal to + * a(x) * ({04}x^2 + {05}), so we can reuse the forward transform's code + * (found in OpenSSL's bsaes-x86_64.pl, attributed to Jussi Kivilinna) + * + * In the bitsliced representation, a multiplication of every column by x + * mod x^4 + 1 is simply a right rotation. + */ + + /* Shared for both directions is a multiplication by a(x), which can be + * rewritten as (x^3 + x^2 + x) + {02}*(x^3 + {01}). + * + * First compute s into the s? variables, (x^3 + {01}) * s into the s?_01 + * variables and (x^3 + x^2 + x)*s into the s?_123 variables. + */ + uint16_t s0 = s->slice[0], s1 = s->slice[1], s2 = s->slice[2], s3 = s->slice[3]; + uint16_t s4 = s->slice[4], s5 = s->slice[5], s6 = s->slice[6], s7 = s->slice[7]; + uint16_t s0_01 = s0 ^ ROT(s0, 1), s0_123 = ROT(s0_01, 1) ^ ROT(s0, 3); + uint16_t s1_01 = s1 ^ ROT(s1, 1), s1_123 = ROT(s1_01, 1) ^ ROT(s1, 3); + uint16_t s2_01 = s2 ^ ROT(s2, 1), s2_123 = ROT(s2_01, 1) ^ ROT(s2, 3); + uint16_t s3_01 = s3 ^ ROT(s3, 1), s3_123 = ROT(s3_01, 1) ^ ROT(s3, 3); + uint16_t s4_01 = s4 ^ ROT(s4, 1), s4_123 = ROT(s4_01, 1) ^ ROT(s4, 3); + uint16_t s5_01 = s5 ^ ROT(s5, 1), s5_123 = ROT(s5_01, 1) ^ ROT(s5, 3); + uint16_t s6_01 = s6 ^ ROT(s6, 1), s6_123 = ROT(s6_01, 1) ^ ROT(s6, 3); + uint16_t s7_01 = s7 ^ ROT(s7, 1), s7_123 = ROT(s7_01, 1) ^ ROT(s7, 3); + /* Now compute s = s?_123 + {02} * s?_01. */ + s->slice[0] = s7_01 ^ s0_123; + s->slice[1] = s7_01 ^ s0_01 ^ s1_123; + s->slice[2] = s1_01 ^ s2_123; + s->slice[3] = s7_01 ^ s2_01 ^ s3_123; + s->slice[4] = s7_01 ^ s3_01 ^ s4_123; + s->slice[5] = s4_01 ^ s5_123; + s->slice[6] = s5_01 ^ s6_123; + s->slice[7] = s6_01 ^ s7_123; + if (inv) { + /* In the reverse direction, we further need to multiply by + * {04}x^2 + {05}, which can be written as {04} * (x^2 + {01}) + {01}. + * + * First compute (x^2 + {01}) * s into the t?_02 variables: */ + uint16_t t0_02 = s->slice[0] ^ ROT(s->slice[0], 2); + uint16_t t1_02 = s->slice[1] ^ ROT(s->slice[1], 2); + uint16_t t2_02 = s->slice[2] ^ ROT(s->slice[2], 2); + uint16_t t3_02 = s->slice[3] ^ ROT(s->slice[3], 2); + uint16_t t4_02 = s->slice[4] ^ ROT(s->slice[4], 2); + uint16_t t5_02 = s->slice[5] ^ ROT(s->slice[5], 2); + uint16_t t6_02 = s->slice[6] ^ ROT(s->slice[6], 2); + uint16_t t7_02 = s->slice[7] ^ ROT(s->slice[7], 2); + /* And then update s += {04} * t?_02 */ + s->slice[0] ^= t6_02; + s->slice[1] ^= t6_02 ^ t7_02; + s->slice[2] ^= t0_02 ^ t7_02; + s->slice[3] ^= t1_02 ^ t6_02; + s->slice[4] ^= t2_02 ^ t6_02 ^ t7_02; + s->slice[5] ^= t3_02 ^ t7_02; + s->slice[6] ^= t4_02; + s->slice[7] ^= t5_02; + } +} + +static void AddRoundKey(AES_state* s, const AES_state* round) { + int b; + for (b = 0; b < 8; b++) { + s->slice[b] ^= round->slice[b]; + } +} + +/** column_0(s) = column_c(a) */ +static void GetOneColumn(AES_state* s, const AES_state* a, int c) { + int b; + for (b = 0; b < 8; b++) { + s->slice[b] = (a->slice[b] >> c) & 0x1111; + } +} + +/** column_c1(r) |= (column_0(s) ^= column_c2(a)) */ +static void KeySetupColumnMix(AES_state* s, AES_state* r, const AES_state* a, int c1, int c2) { + int b; + for (b = 0; b < 8; b++) { + r->slice[b] |= ((s->slice[b] ^= ((a->slice[b] >> c2) & 0x1111)) & 0x1111) << c1; + } +} + +/** Rotate the rows in s one position upwards, and xor in r */ +static void KeySetupTransform(AES_state* s, const AES_state* r) { + int b; + for (b = 0; b < 8; b++) { + s->slice[b] = ((s->slice[b] >> 4) | (s->slice[b] << 12)) ^ r->slice[b]; + } +} + +/* Multiply the cells in s by x, as polynomials over GF(2) mod x^8 + x^4 + x^3 + x + 1 */ +static void MultX(AES_state* s) { + uint16_t top = s->slice[7]; + s->slice[7] = s->slice[6]; + s->slice[6] = s->slice[5]; + s->slice[5] = s->slice[4]; + s->slice[4] = s->slice[3] ^ top; + s->slice[3] = s->slice[2] ^ top; + s->slice[2] = s->slice[1]; + s->slice[1] = s->slice[0] ^ top; + s->slice[0] = top; +} + +/** Expand the cipher key into the key schedule. + * + * state must be a pointer to an array of size nrounds + 1. + * key must be a pointer to 4 * nkeywords bytes. + * + * AES128 uses nkeywords = 4, nrounds = 10 + * AES192 uses nkeywords = 6, nrounds = 12 + * AES256 uses nkeywords = 8, nrounds = 14 + */ +static void AES_setup(AES_state* rounds, const uint8_t* key, int nkeywords, int nrounds) +{ + int i; + + /* The one-byte round constant */ + AES_state rcon = {{1,0,0,0,0,0,0,0}}; + /* The number of the word being generated, modulo nkeywords */ + int pos = 0; + /* The column representing the word currently being processed */ + AES_state column; + + for (i = 0; i < nrounds + 1; i++) { + int b; + for (b = 0; b < 8; b++) { + rounds[i].slice[b] = 0; + } + } + + /* The first nkeywords round columns are just taken from the key directly. */ + for (i = 0; i < nkeywords; i++) { + int r; + for (r = 0; r < 4; r++) { + LoadByte(&rounds[i >> 2], *(key++), r, i & 3); + } + } + + GetOneColumn(&column, &rounds[(nkeywords - 1) >> 2], (nkeywords - 1) & 3); + + for (i = nkeywords; i < 4 * (nrounds + 1); i++) { + /* Transform column */ + if (pos == 0) { + SubBytes(&column, 0); + KeySetupTransform(&column, &rcon); + MultX(&rcon); + } else if (nkeywords > 6 && pos == 4) { + SubBytes(&column, 0); + } + if (++pos == nkeywords) pos = 0; + KeySetupColumnMix(&column, &rounds[i >> 2], &rounds[(i - nkeywords) >> 2], i & 3, (i - nkeywords) & 3); + } +} + +static void AES_encrypt(const AES_state* rounds, int nrounds, unsigned char* cipher16, const unsigned char* plain16) { + AES_state s = {{0}}; + int round; + + LoadBytes(&s, plain16); + AddRoundKey(&s, rounds++); + + for (round = 1; round < nrounds; round++) { + SubBytes(&s, 0); + ShiftRows(&s); + MixColumns(&s, 0); + AddRoundKey(&s, rounds++); + } + + SubBytes(&s, 0); + ShiftRows(&s); + AddRoundKey(&s, rounds); + + SaveBytes(cipher16, &s); +} + +static void AES_decrypt(const AES_state* rounds, int nrounds, unsigned char* plain16, const unsigned char* cipher16) { + /* Most AES decryption implementations use the alternate scheme + * (the Equivalent Inverse Cipher), which looks more like encryption, but + * needs different round constants. We can't reuse any code here anyway, so + * don't bother. */ + AES_state s = {{0}}; + int round; + + rounds += nrounds; + + LoadBytes(&s, cipher16); + AddRoundKey(&s, rounds--); + + for (round = 1; round < nrounds; round++) { + InvShiftRows(&s); + SubBytes(&s, 1); + AddRoundKey(&s, rounds--); + MixColumns(&s, 1); + } + + InvShiftRows(&s); + SubBytes(&s, 1); + AddRoundKey(&s, rounds); + + SaveBytes(plain16, &s); +} + +void AES128_init(AES128_ctx* ctx, const unsigned char* key16) { + AES_setup(ctx->rk, key16, 4, 10); +} + +void AES128_encrypt(const AES128_ctx* ctx, size_t blocks, unsigned char* cipher16, const unsigned char* plain16) { + while (blocks--) { + AES_encrypt(ctx->rk, 10, cipher16, plain16); + cipher16 += 16; + plain16 += 16; + } +} + +void AES128_decrypt(const AES128_ctx* ctx, size_t blocks, unsigned char* plain16, const unsigned char* cipher16) { + while (blocks--) { + AES_decrypt(ctx->rk, 10, plain16, cipher16); + cipher16 += 16; + plain16 += 16; + } +} + +void AES192_init(AES192_ctx* ctx, const unsigned char* key24) { + AES_setup(ctx->rk, key24, 6, 12); +} + +void AES192_encrypt(const AES192_ctx* ctx, size_t blocks, unsigned char* cipher16, const unsigned char* plain16) { + while (blocks--) { + AES_encrypt(ctx->rk, 12, cipher16, plain16); + cipher16 += 16; + plain16 += 16; + } + +} + +void AES192_decrypt(const AES192_ctx* ctx, size_t blocks, unsigned char* plain16, const unsigned char* cipher16) { + while (blocks--) { + AES_decrypt(ctx->rk, 12, plain16, cipher16); + cipher16 += 16; + plain16 += 16; + } +} + +void AES256_init(AES256_ctx* ctx, const unsigned char* key32) { + AES_setup(ctx->rk, key32, 8, 14); +} + +void AES256_encrypt(const AES256_ctx* ctx, size_t blocks, unsigned char* cipher16, const unsigned char* plain16) { + while (blocks--) { + AES_encrypt(ctx->rk, 14, cipher16, plain16); + cipher16 += 16; + plain16 += 16; + } +} + +void AES256_decrypt(const AES256_ctx* ctx, size_t blocks, unsigned char* plain16, const unsigned char* cipher16) { + while (blocks--) { + AES_decrypt(ctx->rk, 14, plain16, cipher16); + cipher16 += 16; + plain16 += 16; + } +} diff --git a/src/crypto/ctaes/ctaes.h b/src/crypto/ctaes/ctaes.h new file mode 100644 index 0000000000000..2f0af04216053 --- /dev/null +++ b/src/crypto/ctaes/ctaes.h @@ -0,0 +1,41 @@ + /********************************************************************* + * Copyright (c) 2016 Pieter Wuille * + * Distributed under the MIT software license, see the accompanying * + * file COPYING or http://www.opensource.org/licenses/mit-license.php.* + **********************************************************************/ + +#ifndef _CTAES_H_ +#define _CTAES_H_ 1 + +#include +#include + +typedef struct { + uint16_t slice[8]; +} AES_state; + +typedef struct { + AES_state rk[11]; +} AES128_ctx; + +typedef struct { + AES_state rk[13]; +} AES192_ctx; + +typedef struct { + AES_state rk[15]; +} AES256_ctx; + +void AES128_init(AES128_ctx* ctx, const unsigned char* key16); +void AES128_encrypt(const AES128_ctx* ctx, size_t blocks, unsigned char* cipher16, const unsigned char* plain16); +void AES128_decrypt(const AES128_ctx* ctx, size_t blocks, unsigned char* plain16, const unsigned char* cipher16); + +void AES192_init(AES192_ctx* ctx, const unsigned char* key24); +void AES192_encrypt(const AES192_ctx* ctx, size_t blocks, unsigned char* cipher16, const unsigned char* plain16); +void AES192_decrypt(const AES192_ctx* ctx, size_t blocks, unsigned char* plain16, const unsigned char* cipher16); + +void AES256_init(AES256_ctx* ctx, const unsigned char* key32); +void AES256_encrypt(const AES256_ctx* ctx, size_t blocks, unsigned char* cipher16, const unsigned char* plain16); +void AES256_decrypt(const AES256_ctx* ctx, size_t blocks, unsigned char* plain16, const unsigned char* cipher16); + +#endif diff --git a/src/crypto/ctaes/test.c b/src/crypto/ctaes/test.c new file mode 100644 index 0000000000000..fce1696acdc2d --- /dev/null +++ b/src/crypto/ctaes/test.c @@ -0,0 +1,110 @@ + /********************************************************************* + * Copyright (c) 2016 Pieter Wuille * + * Distributed under the MIT software license, see the accompanying * + * file COPYING or http://www.opensource.org/licenses/mit-license.php.* + **********************************************************************/ + +#include "ctaes.h" + +#include +#include +#include + +typedef struct { + int keysize; + const char* key; + const char* plain; + const char* cipher; +} ctaes_test; + +static const ctaes_test ctaes_tests[] = { + /* AES test vectors from FIPS 197. */ + {128, "000102030405060708090a0b0c0d0e0f", "00112233445566778899aabbccddeeff", "69c4e0d86a7b0430d8cdb78070b4c55a"}, + {192, "000102030405060708090a0b0c0d0e0f1011121314151617", "00112233445566778899aabbccddeeff", "dda97ca4864cdfe06eaf70a0ec0d7191"}, + {256, "000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f", "00112233445566778899aabbccddeeff", "8ea2b7ca516745bfeafc49904b496089"}, + + /* AES-ECB test vectors from NIST sp800-38a. */ + {128, "2b7e151628aed2a6abf7158809cf4f3c", "6bc1bee22e409f96e93d7e117393172a", "3ad77bb40d7a3660a89ecaf32466ef97"}, + {128, "2b7e151628aed2a6abf7158809cf4f3c", "ae2d8a571e03ac9c9eb76fac45af8e51", "f5d3d58503b9699de785895a96fdbaaf"}, + {128, "2b7e151628aed2a6abf7158809cf4f3c", "30c81c46a35ce411e5fbc1191a0a52ef", "43b1cd7f598ece23881b00e3ed030688"}, + {128, "2b7e151628aed2a6abf7158809cf4f3c", "f69f2445df4f9b17ad2b417be66c3710", "7b0c785e27e8ad3f8223207104725dd4"}, + {192, "8e73b0f7da0e6452c810f32b809079e562f8ead2522c6b7b", "6bc1bee22e409f96e93d7e117393172a", "bd334f1d6e45f25ff712a214571fa5cc"}, + {192, "8e73b0f7da0e6452c810f32b809079e562f8ead2522c6b7b", "ae2d8a571e03ac9c9eb76fac45af8e51", "974104846d0ad3ad7734ecb3ecee4eef"}, + {192, "8e73b0f7da0e6452c810f32b809079e562f8ead2522c6b7b", "30c81c46a35ce411e5fbc1191a0a52ef", "ef7afd2270e2e60adce0ba2face6444e"}, + {192, "8e73b0f7da0e6452c810f32b809079e562f8ead2522c6b7b", "f69f2445df4f9b17ad2b417be66c3710", "9a4b41ba738d6c72fb16691603c18e0e"}, + {256, "603deb1015ca71be2b73aef0857d77811f352c073b6108d72d9810a30914dff4", "6bc1bee22e409f96e93d7e117393172a", "f3eed1bdb5d2a03c064b5a7e3db181f8"}, + {256, "603deb1015ca71be2b73aef0857d77811f352c073b6108d72d9810a30914dff4", "ae2d8a571e03ac9c9eb76fac45af8e51", "591ccb10d410ed26dc5ba74a31362870"}, + {256, "603deb1015ca71be2b73aef0857d77811f352c073b6108d72d9810a30914dff4", "30c81c46a35ce411e5fbc1191a0a52ef", "b6ed21b99ca6f4f9f153e7b1beafed1d"}, + {256, "603deb1015ca71be2b73aef0857d77811f352c073b6108d72d9810a30914dff4", "f69f2445df4f9b17ad2b417be66c3710", "23304b7a39f9f3ff067d8d8f9e24ecc7"} +}; + +static void from_hex(unsigned char* data, int len, const char* hex) { + int p; + for (p = 0; p < len; p++) { + int v = 0; + int n; + for (n = 0; n < 2; n++) { + assert((*hex >= '0' && *hex <= '9') || (*hex >= 'a' && *hex <= 'f')); + if (*hex >= '0' && *hex <= '9') { + v |= (*hex - '0') << (4 * (1 - n)); + } else { + v |= (*hex - 'a' + 10) << (4 * (1 - n)); + } + hex++; + } + *(data++) = v; + } + assert(*hex == 0); +} + +int main(void) { + int i; + int fail = 0; + for (i = 0; i < sizeof(ctaes_tests) / sizeof(ctaes_tests[0]); i++) { + unsigned char key[32], plain[16], cipher[16], ciphered[16], deciphered[16]; + const ctaes_test* test = &ctaes_tests[i]; + assert(test->keysize == 128 || test->keysize == 192 || test->keysize == 256); + from_hex(plain, 16, test->plain); + from_hex(cipher, 16, test->cipher); + switch (test->keysize) { + case 128: { + AES128_ctx ctx; + from_hex(key, 16, test->key); + AES128_init(&ctx, key); + AES128_encrypt(&ctx, 1, ciphered, plain); + AES128_decrypt(&ctx, 1, deciphered, cipher); + break; + } + case 192: { + AES192_ctx ctx; + from_hex(key, 24, test->key); + AES192_init(&ctx, key); + AES192_encrypt(&ctx, 1, ciphered, plain); + AES192_decrypt(&ctx, 1, deciphered, cipher); + break; + } + case 256: { + AES256_ctx ctx; + from_hex(key, 32, test->key); + AES256_init(&ctx, key); + AES256_encrypt(&ctx, 1, ciphered, plain); + AES256_decrypt(&ctx, 1, deciphered, cipher); + break; + } + } + if (memcmp(cipher, ciphered, 16)) { + fprintf(stderr, "E(key=\"%s\", plain=\"%s\") != \"%s\"\n", test->key, test->plain, test->cipher); + fail++; + } + if (memcmp(plain, deciphered, 16)) { + fprintf(stderr, "D(key=\"%s\", cipher=\"%s\") != \"%s\"\n", test->key, test->cipher, test->plain); + fail++; + } + } + if (fail == 0) { + fprintf(stderr, "All tests succesful\n"); + } else { + fprintf(stderr, "%i tests failed\n", fail); + } + return (fail != 0); +} diff --git a/src/dbwrapper.cpp b/src/dbwrapper.cpp index 1907e2fa78439..09c68fbe550c8 100644 --- a/src/dbwrapper.cpp +++ b/src/dbwrapper.cpp @@ -15,20 +15,6 @@ #include #include -void HandleError(const leveldb::Status& status) throw(dbwrapper_error) -{ - if (status.ok()) - return; - LogPrintf("%s\n", status.ToString()); - if (status.IsCorruption()) - throw dbwrapper_error("Database corrupted"); - if (status.IsIOError()) - throw dbwrapper_error("Database I/O error"); - if (status.IsNotFound()) - throw dbwrapper_error("Database entry missing"); - throw dbwrapper_error("Unknown database error"); -} - static leveldb::Options GetOptions(size_t nCacheSize) { leveldb::Options options; @@ -61,13 +47,13 @@ CDBWrapper::CDBWrapper(const boost::filesystem::path& path, size_t nCacheSize, b if (fWipe) { LogPrintf("Wiping LevelDB in %s\n", path.string()); leveldb::Status result = leveldb::DestroyDB(path.string(), options); - HandleError(result); + dbwrapper_private::HandleError(result); } TryCreateDirectory(path); LogPrintf("Opening LevelDB in %s\n", path.string()); } leveldb::Status status = leveldb::DB::Open(options, path.string(), &pdb); - HandleError(status); + dbwrapper_private::HandleError(status); LogPrintf("Opened LevelDB successfully\n"); // The base-case obfuscation key, which is a noop. @@ -84,10 +70,10 @@ CDBWrapper::CDBWrapper(const boost::filesystem::path& path, size_t nCacheSize, b Write(OBFUSCATE_KEY_KEY, new_key); obfuscate_key = new_key; - LogPrintf("Wrote new obfuscate key for %s: %s\n", path.string(), GetObfuscateKeyHex()); + LogPrintf("Wrote new obfuscate key for %s: %s\n", path.string(), HexStr(obfuscate_key)); } - LogPrintf("Using obfuscation key for %s: %s\n", path.string(), GetObfuscateKeyHex()); + LogPrintf("Using obfuscation key for %s: %s\n", path.string(), HexStr(obfuscate_key)); } CDBWrapper::~CDBWrapper() @@ -102,10 +88,10 @@ CDBWrapper::~CDBWrapper() options.env = NULL; } -bool CDBWrapper::WriteBatch(CDBBatch& batch, bool fSync) throw(dbwrapper_error) +bool CDBWrapper::WriteBatch(CDBBatch& batch, bool fSync) { leveldb::Status status = pdb->Write(fSync ? syncoptions : writeoptions, &batch.batch); - HandleError(status); + dbwrapper_private::HandleError(status); return true; } @@ -136,17 +122,30 @@ bool CDBWrapper::IsEmpty() return !(it->Valid()); } -const std::vector& CDBWrapper::GetObfuscateKey() const +CDBIterator::~CDBIterator() { delete piter; } +bool CDBIterator::Valid() { return piter->Valid(); } +void CDBIterator::SeekToFirst() { piter->SeekToFirst(); } +void CDBIterator::Next() { piter->Next(); } + +namespace dbwrapper_private { + +void HandleError(const leveldb::Status& status) { - return obfuscate_key; + if (status.ok()) + return; + LogPrintf("%s\n", status.ToString()); + if (status.IsCorruption()) + throw dbwrapper_error("Database corrupted"); + if (status.IsIOError()) + throw dbwrapper_error("Database I/O error"); + if (status.IsNotFound()) + throw dbwrapper_error("Database entry missing"); + throw dbwrapper_error("Unknown database error"); } -std::string CDBWrapper::GetObfuscateKeyHex() const +const std::vector& GetObfuscateKey(const CDBWrapper &w) { - return HexStr(obfuscate_key); + return w.obfuscate_key; } -CDBIterator::~CDBIterator() { delete piter; } -bool CDBIterator::Valid() { return piter->Valid(); } -void CDBIterator::SeekToFirst() { piter->SeekToFirst(); } -void CDBIterator::Next() { piter->Next(); } +}; diff --git a/src/dbwrapper.h b/src/dbwrapper.h index 5e7313f7eb544..a0779d3ab9918 100644 --- a/src/dbwrapper.h +++ b/src/dbwrapper.h @@ -23,7 +23,23 @@ class dbwrapper_error : public std::runtime_error dbwrapper_error(const std::string& msg) : std::runtime_error(msg) {} }; -void HandleError(const leveldb::Status& status) throw(dbwrapper_error); +class CDBWrapper; + +/** These should be considered an implementation detail of the specific database. + */ +namespace dbwrapper_private { + +/** Handle database error by throwing dbwrapper_error exception. + */ +void HandleError(const leveldb::Status& status); + +/** Work around circular dependency, as well as for testing in dbwrapper_tests. + * Database obfuscation should be considered an implementation detail of the + * specific database. + */ +const std::vector& GetObfuscateKey(const CDBWrapper &w); + +}; /** Batch of changes queued to be written to a CDBWrapper */ class CDBBatch @@ -31,14 +47,14 @@ class CDBBatch friend class CDBWrapper; private: + const CDBWrapper &parent; leveldb::WriteBatch batch; - const std::vector *obfuscate_key; public: /** - * @param[in] obfuscate_key If passed, XOR data with this key. + * @param[in] parent CDBWrapper that this batch is to be submitted to */ - CDBBatch(const std::vector *obfuscate_key) : obfuscate_key(obfuscate_key) { }; + CDBBatch(const CDBWrapper &parent) : parent(parent) { }; template void Write(const K& key, const V& value) @@ -51,7 +67,7 @@ class CDBBatch CDataStream ssValue(SER_DISK, CLIENT_VERSION); ssValue.reserve(ssValue.GetSerializeSize(value)); ssValue << value; - ssValue.Xor(*obfuscate_key); + ssValue.Xor(dbwrapper_private::GetObfuscateKey(parent)); leveldb::Slice slValue(&ssValue[0], ssValue.size()); batch.Put(slKey, slValue); @@ -72,17 +88,17 @@ class CDBBatch class CDBIterator { private: + const CDBWrapper &parent; leveldb::Iterator *piter; - const std::vector *obfuscate_key; public: /** + * @param[in] parent Parent CDBWrapper instance. * @param[in] piterIn The original leveldb iterator. - * @param[in] obfuscate_key If passed, XOR data with this key. */ - CDBIterator(leveldb::Iterator *piterIn, const std::vector* obfuscate_key) : - piter(piterIn), obfuscate_key(obfuscate_key) { }; + CDBIterator(const CDBWrapper &parent, leveldb::Iterator *piterIn) : + parent(parent), piter(piterIn) { }; ~CDBIterator(); bool Valid(); @@ -118,7 +134,7 @@ class CDBIterator leveldb::Slice slValue = piter->value(); try { CDataStream ssValue(slValue.data(), slValue.data() + slValue.size(), SER_DISK, CLIENT_VERSION); - ssValue.Xor(*obfuscate_key); + ssValue.Xor(dbwrapper_private::GetObfuscateKey(parent)); ssValue >> value; } catch (const std::exception&) { return false; @@ -134,6 +150,7 @@ class CDBIterator class CDBWrapper { + friend const std::vector& dbwrapper_private::GetObfuscateKey(const CDBWrapper &w); private: //! custom environment this database is using (may be NULL in case of default environment) leveldb::Env* penv; @@ -180,7 +197,7 @@ class CDBWrapper ~CDBWrapper(); template - bool Read(const K& key, V& value) const throw(dbwrapper_error) + bool Read(const K& key, V& value) const { CDataStream ssKey(SER_DISK, CLIENT_VERSION); ssKey.reserve(ssKey.GetSerializeSize(key)); @@ -193,7 +210,7 @@ class CDBWrapper if (status.IsNotFound()) return false; LogPrintf("LevelDB read failure: %s\n", status.ToString()); - HandleError(status); + dbwrapper_private::HandleError(status); } try { CDataStream ssValue(strValue.data(), strValue.data() + strValue.size(), SER_DISK, CLIENT_VERSION); @@ -206,15 +223,15 @@ class CDBWrapper } template - bool Write(const K& key, const V& value, bool fSync = false) throw(dbwrapper_error) + bool Write(const K& key, const V& value, bool fSync = false) { - CDBBatch batch(&obfuscate_key); + CDBBatch batch(*this); batch.Write(key, value); return WriteBatch(batch, fSync); } template - bool Exists(const K& key) const throw(dbwrapper_error) + bool Exists(const K& key) const { CDataStream ssKey(SER_DISK, CLIENT_VERSION); ssKey.reserve(ssKey.GetSerializeSize(key)); @@ -227,20 +244,20 @@ class CDBWrapper if (status.IsNotFound()) return false; LogPrintf("LevelDB read failure: %s\n", status.ToString()); - HandleError(status); + dbwrapper_private::HandleError(status); } return true; } template - bool Erase(const K& key, bool fSync = false) throw(dbwrapper_error) + bool Erase(const K& key, bool fSync = false) { - CDBBatch batch(&obfuscate_key); + CDBBatch batch(*this); batch.Erase(key); return WriteBatch(batch, fSync); } - bool WriteBatch(CDBBatch& batch, bool fSync = false) throw(dbwrapper_error); + bool WriteBatch(CDBBatch& batch, bool fSync = false); // not available for LevelDB; provide for compatibility with BDB bool Flush() @@ -248,32 +265,21 @@ class CDBWrapper return true; } - bool Sync() throw(dbwrapper_error) + bool Sync() { - CDBBatch batch(&obfuscate_key); + CDBBatch batch(*this); return WriteBatch(batch, true); } CDBIterator *NewIterator() { - return new CDBIterator(pdb->NewIterator(iteroptions), &obfuscate_key); + return new CDBIterator(*this, pdb->NewIterator(iteroptions)); } /** * Return true if the database managed by this class contains no entries. */ bool IsEmpty(); - - /** - * Accessor for obfuscate_key. - */ - const std::vector& GetObfuscateKey() const; - - /** - * Return the obfuscate_key as a hex-formatted string. - */ - std::string GetObfuscateKeyHex() const; - }; #endif // BITCOIN_DBWRAPPER_H diff --git a/src/hash.cpp b/src/hash.cpp index 7f3cf1a1faa74..20a83342db4c9 100644 --- a/src/hash.cpp +++ b/src/hash.cpp @@ -81,3 +81,130 @@ void BIP32Hash(const ChainCode &chainCode, unsigned int nChild, unsigned char he num[3] = (nChild >> 0) & 0xFF; CHMAC_SHA512(chainCode.begin(), chainCode.size()).Write(&header, 1).Write(data, 32).Write(num, 4).Finalize(output); } + +#define ROTL(x, b) (uint64_t)(((x) << (b)) | ((x) >> (64 - (b)))) + +#define SIPROUND do { \ + v0 += v1; v1 = ROTL(v1, 13); v1 ^= v0; \ + v0 = ROTL(v0, 32); \ + v2 += v3; v3 = ROTL(v3, 16); v3 ^= v2; \ + v0 += v3; v3 = ROTL(v3, 21); v3 ^= v0; \ + v2 += v1; v1 = ROTL(v1, 17); v1 ^= v2; \ + v2 = ROTL(v2, 32); \ +} while (0) + +CSipHasher::CSipHasher(uint64_t k0, uint64_t k1) +{ + v[0] = 0x736f6d6570736575ULL ^ k0; + v[1] = 0x646f72616e646f6dULL ^ k1; + v[2] = 0x6c7967656e657261ULL ^ k0; + v[3] = 0x7465646279746573ULL ^ k1; + count = 0; + tmp = 0; +} + +CSipHasher& CSipHasher::Write(uint64_t data) +{ + uint64_t v0 = v[0], v1 = v[1], v2 = v[2], v3 = v[3]; + + assert(count % 8 == 0); + + v3 ^= data; + SIPROUND; + SIPROUND; + v0 ^= data; + + v[0] = v0; + v[1] = v1; + v[2] = v2; + v[3] = v3; + + count += 8; + return *this; +} + +CSipHasher& CSipHasher::Write(const unsigned char* data, size_t size) +{ + uint64_t v0 = v[0], v1 = v[1], v2 = v[2], v3 = v[3]; + uint64_t t = tmp; + int c = count; + + while (size--) { + t |= ((uint64_t)(*(data++))) << (8 * (c % 8)); + c++; + if ((c & 7) == 0) { + v3 ^= t; + SIPROUND; + SIPROUND; + v0 ^= t; + t = 0; + } + } + + v[0] = v0; + v[1] = v1; + v[2] = v2; + v[3] = v3; + count = c; + tmp = t; + + return *this; +} + +uint64_t CSipHasher::Finalize() const +{ + uint64_t v0 = v[0], v1 = v[1], v2 = v[2], v3 = v[3]; + + uint64_t t = tmp | (((uint64_t)count) << 56); + + v3 ^= t; + SIPROUND; + SIPROUND; + v0 ^= t; + v2 ^= 0xFF; + SIPROUND; + SIPROUND; + SIPROUND; + SIPROUND; + return v0 ^ v1 ^ v2 ^ v3; +} + +uint64_t SipHashUint256(uint64_t k0, uint64_t k1, const uint256& val) +{ + /* Specialized implementation for efficiency */ + uint64_t d = val.GetUint64(0); + + uint64_t v0 = 0x736f6d6570736575ULL ^ k0; + uint64_t v1 = 0x646f72616e646f6dULL ^ k1; + uint64_t v2 = 0x6c7967656e657261ULL ^ k0; + uint64_t v3 = 0x7465646279746573ULL ^ k1 ^ d; + + SIPROUND; + SIPROUND; + v0 ^= d; + d = val.GetUint64(1); + v3 ^= d; + SIPROUND; + SIPROUND; + v0 ^= d; + d = val.GetUint64(2); + v3 ^= d; + SIPROUND; + SIPROUND; + v0 ^= d; + d = val.GetUint64(3); + v3 ^= d; + SIPROUND; + SIPROUND; + v0 ^= d; + v3 ^= ((uint64_t)4) << 59; + SIPROUND; + SIPROUND; + v0 ^= ((uint64_t)4) << 59; + v2 ^= 0xFF; + SIPROUND; + SIPROUND; + SIPROUND; + SIPROUND; + return v0 ^ v1 ^ v2 ^ v3; +} diff --git a/src/hash.h b/src/hash.h index 97955c8d5ad6d..db4e130ae7fec 100644 --- a/src/hash.h +++ b/src/hash.h @@ -171,4 +171,38 @@ unsigned int MurmurHash3(unsigned int nHashSeed, const std::vector // for to_lower() #include -#include /** Maximum size of http request (request line + headers) */ static const size_t MAX_HEADERS_SIZE = 8192; @@ -45,8 +44,8 @@ static const size_t MAX_HEADERS_SIZE = 8192; class HTTPWorkItem : public HTTPClosure { public: - HTTPWorkItem(HTTPRequest* req, const std::string &path, const HTTPRequestHandler& func): - req(req), path(path), func(func) + HTTPWorkItem(std::unique_ptr req, const std::string &path, const HTTPRequestHandler& func): + req(std::move(req)), path(path), func(func) { } void operator()() @@ -54,7 +53,7 @@ class HTTPWorkItem : public HTTPClosure func(req.get(), path); } - boost::scoped_ptr req; + std::unique_ptr req; private: std::string path; @@ -71,8 +70,7 @@ class WorkQueue /** Mutex protects entire object */ CWaitableCriticalSection cs; CConditionVariable cond; - /* XXX in C++11 we can use std::unique_ptr here and avoid manual cleanup */ - std::deque queue; + std::deque> queue; bool running; size_t maxDepth; int numThreads; @@ -101,15 +99,11 @@ class WorkQueue numThreads(0) { } - /*( Precondition: worker threads have all stopped + /** Precondition: worker threads have all stopped * (call WaitExit) */ ~WorkQueue() { - while (!queue.empty()) { - delete queue.front(); - queue.pop_front(); - } } /** Enqueue a work item */ bool Enqueue(WorkItem* item) @@ -118,7 +112,7 @@ class WorkQueue if (queue.size() >= maxDepth) { return false; } - queue.push_back(item); + queue.emplace_back(std::unique_ptr(item)); cond.notify_one(); return true; } @@ -127,18 +121,17 @@ class WorkQueue { ThreadCounter count(*this); while (running) { - WorkItem* i = 0; + std::unique_ptr i; { boost::unique_lock lock(cs); while (running && queue.empty()) cond.wait(lock); if (!running) break; - i = queue.front(); + i = std::move(queue.front()); queue.pop_front(); } (*i)(); - delete i; } } /** Interrupt and exit loops */ @@ -252,7 +245,7 @@ static std::string RequestMethodString(HTTPRequest::RequestMethod m) /** HTTP request callback */ static void http_request_cb(struct evhttp_request* req, void* arg) { - std::auto_ptr hreq(new HTTPRequest(req)); + std::unique_ptr hreq(new HTTPRequest(req)); LogPrint("http", "Received a %s request for %s from %s\n", RequestMethodString(hreq->GetRequestMethod()), hreq->GetURI(), hreq->GetPeer().ToString()); @@ -288,12 +281,14 @@ static void http_request_cb(struct evhttp_request* req, void* arg) // Dispatch to worker thread if (i != iend) { - std::auto_ptr item(new HTTPWorkItem(hreq.release(), path, i->handler)); + std::unique_ptr item(new HTTPWorkItem(std::move(hreq), path, i->handler)); assert(workQueue); if (workQueue->Enqueue(item.get())) item.release(); /* if true, queue took ownership */ - else + else { + LogPrintf("WARNING: request rejected because http work queue depth exceeded, it can be increased with the -rpcworkqueue= setting\n"); item->req->WriteReply(HTTP_INTERNAL, "Work queue depth exceeded"); + } } else { hreq->WriteReply(HTTP_NOTFOUND); } diff --git a/src/indirectmap.h b/src/indirectmap.h new file mode 100644 index 0000000000000..28e1e8deddbde --- /dev/null +++ b/src/indirectmap.h @@ -0,0 +1,52 @@ +#ifndef BITCOIN_INDIRECTMAP_H +#define BITCOIN_INDIRECTMAP_H + +template +struct DereferencingComparator { bool operator()(const T a, const T b) const { return *a < *b; } }; + +/* Map whose keys are pointers, but are compared by their dereferenced values. + * + * Differs from a plain std::map > in + * that methods that take a key for comparison take a K rather than taking a K* + * (taking a K* would be confusing, since it's the value rather than the address + * of the object for comparison that matters due to the dereferencing comparator). + * + * Objects pointed to by keys must not be modified in any way that changes the + * result of DereferencingComparator. + */ +template +class indirectmap { +private: + typedef std::map > base; + base m; +public: + typedef typename base::iterator iterator; + typedef typename base::const_iterator const_iterator; + typedef typename base::size_type size_type; + typedef typename base::value_type value_type; + + // passthrough (pointer interface) + std::pair insert(const value_type& value) { return m.insert(value); } + + // pass address (value interface) + iterator find(const K& key) { return m.find(&key); } + const_iterator find(const K& key) const { return m.find(&key); } + iterator lower_bound(const K& key) { return m.lower_bound(&key); } + const_iterator lower_bound(const K& key) const { return m.lower_bound(&key); } + size_type erase(const K& key) { return m.erase(&key); } + size_type count(const K& key) const { return m.count(&key); } + + // passthrough + bool empty() const { return m.empty(); } + size_type size() const { return m.size(); } + size_type max_size() const { return m.max_size(); } + void clear() { m.clear(); } + iterator begin() { return m.begin(); } + iterator end() { return m.end(); } + const_iterator begin() const { return m.begin(); } + const_iterator end() const { return m.end(); } + const_iterator cbegin() const { return m.cbegin(); } + const_iterator cend() const { return m.cend(); } +}; + +#endif // BITCOIN_INDIRECTMAP_H diff --git a/src/init.cpp b/src/init.cpp index a799673ae88b9..21f0b4ee594dc 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -37,9 +37,7 @@ #include "utilmoneystr.h" #include "validationinterface.h" #ifdef ENABLE_WALLET -#include "wallet/db.h" #include "wallet/wallet.h" -#include "wallet/walletdb.h" #endif #include #include @@ -94,7 +92,6 @@ enum BindFlags { }; static const char* FEE_ESTIMATES_FILENAME="fee_estimates.dat"; -CClientUIInterface uiInterface; // Declared but not defined in ui_interface.h ////////////////////////////////////////////////////////////////////////////// // @@ -126,7 +123,7 @@ CClientUIInterface uiInterface; // Declared but not defined in ui_interface.h // shutdown thing. // -volatile bool fRequestShutdown = false; +std::atomic fRequestShutdown(false); void StartShutdown() { @@ -137,6 +134,11 @@ bool ShutdownRequested() return fRequestShutdown; } +/** + * This is a minimally invasive approach to shutdown on LevelDB read errors from the + * chainstate, while keeping user interface out of the common library, which is shared + * between bitcoind, and bitcoin-qt and non-server tools. +*/ class CCoinsViewErrorCatcher : public CCoinsViewBacked { public: @@ -266,18 +268,6 @@ void HandleSIGHUP(int) fReopenDebugLog = true; } -bool static InitError(const std::string &str) -{ - uiInterface.ThreadSafeMessageBox(str, "", CClientUIInterface::MSG_ERROR); - return false; -} - -bool static InitWarning(const std::string &str) -{ - uiInterface.ThreadSafeMessageBox(str, "", CClientUIInterface::MSG_WARNING); - return true; -} - bool static Bind(const CService &addr, unsigned int flags) { if (!(flags & BF_EXPLICIT) && IsLimited(addr)) return false; @@ -329,7 +319,8 @@ std::string HelpMessage(HelpMessageMode mode) } strUsage += HelpMessageOpt("-datadir=", _("Specify data directory")); strUsage += HelpMessageOpt("-dbcache=", strprintf(_("Set database cache size in megabytes (%d to %d, default: %d)"), nMinDbCache, nMaxDbCache, nDefaultDbCache)); - strUsage += HelpMessageOpt("-feefilter", strprintf(_("Tell other nodes to filter invs to us by our mempool min fee (default: %u)"), DEFAULT_FEEFILTER)); + if (showDebug) + strUsage += HelpMessageOpt("-feefilter", strprintf("Tell other nodes to filter invs to us by our mempool min fee (default: %u)", DEFAULT_FEEFILTER)); strUsage += HelpMessageOpt("-loadblock=", _("Imports blocks from external blk000??.dat file on startup")); strUsage += HelpMessageOpt("-maxorphantx=", strprintf(_("Keep at most unconnectable transactions in memory (default: %u)"), DEFAULT_MAX_ORPHAN_TRANSACTIONS)); strUsage += HelpMessageOpt("-maxmempool=", strprintf(_("Keep the transaction memory pool below megabytes (default: %u)"), DEFAULT_MAX_MEMPOOL_SIZE)); @@ -342,7 +333,8 @@ std::string HelpMessage(HelpMessageMode mode) strUsage += HelpMessageOpt("-prune=", strprintf(_("Reduce storage requirements by pruning (deleting) old blocks. This mode is incompatible with -txindex and -rescan. " "Warning: Reverting this setting requires re-downloading the entire blockchain. " "(default: 0 = disable pruning blocks, >%u = target size in MiB to use for block files)"), MIN_DISK_SPACE_FOR_BLOCK_FILES / 1024 / 1024)); - strUsage += HelpMessageOpt("-reindex", _("Rebuild block chain index from current blk000??.dat files on startup")); + strUsage += HelpMessageOpt("-reindex-chainstate", _("Rebuild chain state from the currently indexed blocks")); + strUsage += HelpMessageOpt("-reindex", _("Rebuild chain state and block index from the blk*.dat files on disk")); #ifndef WIN32 strUsage += HelpMessageOpt("-sysperms", _("Create new files with system default permissions, instead of umask 077 (only effective with disabled wallet functionality)")); #endif @@ -419,7 +411,7 @@ std::string HelpMessage(HelpMessageMode mode) strUsage += HelpMessageOpt("-limitdescendantcount=", strprintf("Do not accept transactions if any ancestor would have or more in-mempool descendants (default: %u)", DEFAULT_DESCENDANT_LIMIT)); strUsage += HelpMessageOpt("-limitdescendantsize=", strprintf("Do not accept transactions if any ancestor would have more than kilobytes of in-mempool descendants (default: %u).", DEFAULT_DESCENDANT_SIZE_LIMIT)); } - string debugCategories = "addrman, alert, bench, coindb, db, lock, rand, rpc, selectcoins, mempool, mempoolrej, net, proxy, prune, http, libevent, tor, zmq"; // Don't translate these and qt below + string debugCategories = "addrman, alert, bench, coindb, db, http, libevent, lock, mempool, mempoolrej, net, proxy, prune, rand, reindex, rpc, selectcoins, tor, zmq"; // Don't translate these and qt below if (mode == HMM_BITCOIN_QT) debugCategories += ", qt"; strUsage += HelpMessageOpt("-debug=", strprintf(_("Output debugging information (default: %u, supplying is optional)"), 0) + ". " + @@ -570,9 +562,10 @@ void ThreadImport(std::vector vImportFiles) { const CChainParams& chainparams = Params(); RenameThread("bitcoin-loadblk"); + CImportingNow imp; + // -reindex if (fReindex) { - CImportingNow imp; int nFile = 0; while (true) { CDiskBlockPos pos(nFile, 0); @@ -597,7 +590,6 @@ void ThreadImport(std::vector vImportFiles) if (boost::filesystem::exists(pathBootstrap)) { FILE *file = fopen(pathBootstrap.string().c_str(), "rb"); if (file) { - CImportingNow imp; boost::filesystem::path pathBootstrapOld = GetDataDir() / "bootstrap.dat.old"; LogPrintf("Importing bootstrap.dat...\n"); LoadExternalBlockFile(chainparams, file); @@ -611,7 +603,6 @@ void ThreadImport(std::vector vImportFiles) BOOST_FOREACH(const boost::filesystem::path& path, vImportFiles) { FILE *file = fopen(path.string().c_str(), "rb"); if (file) { - CImportingNow imp; LogPrintf("Importing blocks file %s...\n", path.string()); LoadExternalBlockFile(chainparams, file); } else { @@ -619,6 +610,13 @@ void ThreadImport(std::vector vImportFiles) } } + // scan for better chains in the block chain database, that are not yet connected in the active best chain + CValidationState state; + if (!ActivateBestChain(state, chainparams)) { + LogPrintf("Failed to connect best block"); + StartShutdown(); + } + if (GetBoolArg("-stopafterblockimport", DEFAULT_STOPAFTERBLOCKIMPORT)) { LogPrintf("Stopping after block import\n"); StartShutdown(); @@ -743,11 +741,6 @@ static std::string ResolveErrMsg(const char * const optname, const std::string& return strprintf(_("Cannot resolve -%s address: '%s'"), optname, strBind); } -static std::string AmountErrMsg(const char * const optname, const std::string& strValue) -{ - return strprintf(_("Invalid amount for -%s=: '%s'"), optname, strValue); -} - void InitLogging() { fPrintToConsole = GetBoolArg("-printtoconsole", false); @@ -756,7 +749,7 @@ void InitLogging() fLogIPs = GetBoolArg("-logips", DEFAULT_LOGIPS); LogPrintf("\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n"); - LogPrintf("Bitcoin version %s (%s)\n", FormatFullVersion(), CLIENT_DATE); + LogPrintf("Bitcoin version %s\n", FormatFullVersion()); } /** Initialize bitcoin. @@ -958,7 +951,7 @@ bool AppInit2(boost::thread_group& threadGroup, CScheduler& scheduler) SetMockTime(GetArg("-mocktime", 0)); // SetMockTime(0) is a no-op if (GetBoolArg("-peerbloomfilters", DEFAULT_PEERBLOOMFILTERS)) - nLocalServices |= NODE_BLOOM; + nLocalServices = ServiceFlags(nLocalServices | NODE_BLOOM); nMaxTipAge = GetArg("-maxtipage", DEFAULT_MAX_TIP_AGE); @@ -1005,9 +998,6 @@ bool AppInit2(boost::thread_group& threadGroup, CScheduler& scheduler) if (fPrintToDebugLog) OpenDebugLog(); -#ifdef ENABLE_WALLET - LogPrintf("Using BerkeleyDB version %s\n", DbEnv::version(0, 0, 0)); -#endif if (!fLogTimestamps) LogPrintf("Startup time: %s\n", DateTimeStrFormat("%Y-%m-%d %H:%M:%S", GetTime())); LogPrintf("Default data directory %s\n", GetDefaultDataDir().string()); @@ -1126,6 +1116,7 @@ bool AppInit2(boost::thread_group& threadGroup, CScheduler& scheduler) fListen = GetBoolArg("-listen", DEFAULT_LISTEN); fDiscover = GetBoolArg("-discover", true); fNameLookup = GetBoolArg("-dns", DEFAULT_NAME_LOOKUP); + fRelayTxes = !GetBoolArg("-blocksonly", DEFAULT_BLOCKSONLY); bool fBound = false; if (fListen) { @@ -1157,10 +1148,11 @@ bool AppInit2(boost::thread_group& threadGroup, CScheduler& scheduler) if (mapArgs.count("-externalip")) { BOOST_FOREACH(const std::string& strAddr, mapMultiArgs["-externalip"]) { - CService addrLocal(strAddr, GetListenPort(), fNameLookup); - if (!addrLocal.IsValid()) + CService addrLocal; + if (Lookup(strAddr.c_str(), addrLocal, GetListenPort(), fNameLookup) && addrLocal.IsValid()) + AddLocal(addrLocal, LOCAL_MANUAL); + else return InitError(ResolveErrMsg("externalip", strAddr)); - AddLocal(CService(strAddr, GetListenPort(), fNameLookup), LOCAL_MANUAL); } } @@ -1181,6 +1173,7 @@ bool AppInit2(boost::thread_group& threadGroup, CScheduler& scheduler) // ********************************************************* Step 7: load block chain fReindex = GetBoolArg("-reindex", false); + bool fReindexChainState = GetBoolArg("-reindex-chainstate", false); // Upgrading to 0.8; hard-link the old blknnnn.dat files into /blocks/ boost::filesystem::path blocksDir = GetDataDir() / "blocks"; @@ -1242,7 +1235,7 @@ bool AppInit2(boost::thread_group& threadGroup, CScheduler& scheduler) delete pblocktree; pblocktree = new CBlockTreeDB(nBlockTreeDBCache, false, fReindex); - pcoinsdbview = new CCoinsViewDB(nCoinDBCache, false, fReindex); + pcoinsdbview = new CCoinsViewDB(nCoinDBCache, false, fReindex || fReindexChainState); pcoinscatcher = new CCoinsViewErrorCatcher(pcoinsdbview); pcoinsTip = new CCoinsViewCache(pcoinscatcher); @@ -1271,7 +1264,7 @@ bool AppInit2(boost::thread_group& threadGroup, CScheduler& scheduler) // Check for changed -txindex state if (fTxIndex != GetBoolArg("-txindex", DEFAULT_TXINDEX)) { - strLoadError = _("You need to rebuild the database using -reindex to change -txindex"); + strLoadError = _("You need to rebuild the database using -reindex-chainstate to change -txindex"); break; } @@ -1377,7 +1370,7 @@ bool AppInit2(boost::thread_group& threadGroup, CScheduler& scheduler) // after any wallet rescanning has taken place. if (fPruneMode) { LogPrintf("Unsetting NODE_NETWORK on prune mode\n"); - nLocalServices &= ~NODE_NETWORK; + nLocalServices = ServiceFlags(nLocalServices & ~NODE_NETWORK); if (!fReindex) { uiInterface.InitMessage(_("Pruning blockstore...")); PruneAndFlush(); @@ -1389,12 +1382,6 @@ bool AppInit2(boost::thread_group& threadGroup, CScheduler& scheduler) if (mapArgs.count("-blocknotify")) uiInterface.NotifyBlockTip.connect(BlockNotifyCallback); - uiInterface.InitMessage(_("Activating best chain...")); - // scan for better chains in the block chain database, that are not yet connected in the active best chain - CValidationState state; - if (!ActivateBestChain(state, chainparams)) - strErrors << "Failed to connect best block"; - std::vector vImportFiles; if (mapArgs.count("-loadblock")) { @@ -1402,10 +1389,18 @@ bool AppInit2(boost::thread_group& threadGroup, CScheduler& scheduler) vImportFiles.push_back(strFile); } threadGroup.create_thread(boost::bind(&ThreadImport, vImportFiles)); - if (chainActive.Tip() == NULL) { - LogPrintf("Waiting for genesis block to be imported...\n"); - while (!fRequestShutdown && chainActive.Tip() == NULL) + + // Wait for genesis block to be processed + bool fHaveGenesis = false; + while (!fHaveGenesis && !fRequestShutdown) { + { + LOCK(cs_main); + fHaveGenesis = (chainActive.Tip() != NULL); + } + + if (!fHaveGenesis) { MilliSleep(10); + } } // ********************************************************* Step 11: start node @@ -1416,8 +1411,6 @@ bool AppInit2(boost::thread_group& threadGroup, CScheduler& scheduler) if (!strErrors.str().empty()) return InitError(strErrors.str()); - RandAddSeedPerfmon(); - //// debug print LogPrintf("mapBlockIndex.size() = %u\n", mapBlockIndex.size()); LogPrintf("nBestHeight = %d\n", chainActive.Height()); diff --git a/src/key.cpp b/src/key.cpp index 6a3d9aa140906..79023566c3e61 100644 --- a/src/key.cpp +++ b/src/key.cpp @@ -124,9 +124,8 @@ bool CKey::Check(const unsigned char *vch) { } void CKey::MakeNewKey(bool fCompressedIn) { - RandAddSeedPerfmon(); do { - GetRandBytes(vch, sizeof(vch)); + GetStrongRandBytes(vch, sizeof(vch)); } while (!Check(vch)); fValid = true; fCompressed = fCompressedIn; diff --git a/src/keystore.cpp b/src/keystore.cpp index cc8a5733671da..d568a7435098a 100644 --- a/src/keystore.cpp +++ b/src/keystore.cpp @@ -19,6 +19,7 @@ bool CBasicKeyStore::GetPubKey(const CKeyID &address, CPubKey &vchPubKeyOut) con { CKey key; if (!GetKey(address, key)) { + LOCK(cs_KeyStore); WatchKeyMap::const_iterator it = mapWatchKeys.find(address); if (it != mapWatchKeys.end()) { vchPubKeyOut = it->second; diff --git a/src/main.cpp b/src/main.cpp index 87e5362d2e36c..897d6db13f6d0 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -37,9 +37,11 @@ #include "validationinterface.h" #include "versionbits.h" +#include #include #include +#include #include #include #include @@ -79,6 +81,7 @@ uint64_t nPruneTarget = 0; int64_t nMaxTipAge = DEFAULT_MAX_TIP_AGE; bool fEnableReplacement = DEFAULT_ENABLE_REPLACEMENT; + CFeeRate minRelayTxFee = CFeeRate(DEFAULT_MIN_RELAY_TX_FEE); CAmount maxTxFee = DEFAULT_TRANSACTION_MAXFEE; @@ -210,6 +213,12 @@ namespace { /** Number of peers from which we're downloading blocks. */ int nPeersWithValidatedDownloads = 0; + + /** Relay map, protected by cs_main. */ + typedef std::map> MapRelay; + MapRelay mapRelay; + /** Expiration-time ordered list of (expire time, relay map entry) pairs, protected by cs_main). */ + std::deque> vRelayExpiration; } // anon namespace ////////////////////////////////////////////////////////////////////////////// @@ -1027,7 +1036,7 @@ std::string FormatStateMessage(const CValidationState &state) } bool AcceptToMemoryPoolWorker(CTxMemPool& pool, CValidationState& state, const CTransaction& tx, bool fLimitFree, - bool* pfMissingInputs, CFeeRate* txFeeRate, bool fOverrideMempoolLimit, const CAmount& nAbsurdFee, + bool* pfMissingInputs, bool fOverrideMempoolLimit, const CAmount& nAbsurdFee, std::vector& vHashTxnToUncache) { const uint256 hash = tx.GetHash(); @@ -1076,9 +1085,10 @@ bool AcceptToMemoryPoolWorker(CTxMemPool& pool, CValidationState& state, const C LOCK(pool.cs); // protect pool.mapNextTx BOOST_FOREACH(const CTxIn &txin, tx.vin) { - if (pool.mapNextTx.count(txin.prevout)) + auto itConflicting = pool.mapNextTx.find(txin.prevout); + if (itConflicting != pool.mapNextTx.end()) { - const CTransaction *ptxConflicting = pool.mapNextTx[txin.prevout].ptx; + const CTransaction *ptxConflicting = itConflicting->second; if (!setConflicts.count(ptxConflicting->GetHash())) { // Allow opt-out of transaction replacement by setting @@ -1196,9 +1206,6 @@ bool AcceptToMemoryPoolWorker(CTxMemPool& pool, CValidationState& state, const C CTxMemPoolEntry entry(tx, nFees, GetTime(), dPriority, chainActive.Height(), pool.HasNoInputsOf(tx), inChainInputValue, fSpendsCoinbase, nSigOpsCost, lp); unsigned int nSize = entry.GetTxSize(); - if (txFeeRate) { - *txFeeRate = CFeeRate(nFees, nSize); - } // Check that the transaction doesn't have an excessive number of // sigops, making it impossible to mine. Since the coinbase transaction @@ -1461,10 +1468,10 @@ bool AcceptToMemoryPoolWorker(CTxMemPool& pool, CValidationState& state, const C } bool AcceptToMemoryPool(CTxMemPool& pool, CValidationState &state, const CTransaction &tx, bool fLimitFree, - bool* pfMissingInputs, CFeeRate* txFeeRate, bool fOverrideMempoolLimit, const CAmount nAbsurdFee) + bool* pfMissingInputs, bool fOverrideMempoolLimit, const CAmount nAbsurdFee) { std::vector vHashTxToUncache; - bool res = AcceptToMemoryPoolWorker(pool, state, tx, fLimitFree, pfMissingInputs, txFeeRate, fOverrideMempoolLimit, nAbsurdFee, vHashTxToUncache); + bool res = AcceptToMemoryPoolWorker(pool, state, tx, fLimitFree, pfMissingInputs, fOverrideMempoolLimit, nAbsurdFee, vHashTxToUncache); if (!res) { BOOST_FOREACH(const uint256& hashTx, vHashTxToUncache) pcoinsTip->Uncache(hashTx); @@ -1479,8 +1486,10 @@ bool GetTransaction(const uint256 &hash, CTransaction &txOut, const Consensus::P LOCK(cs_main); - if (mempool.lookup(hash, txOut)) + std::shared_ptr ptx = mempool.get(hash); + if (ptx) { + txOut = *ptx; return true; } @@ -1508,7 +1517,7 @@ bool GetTransaction(const uint256 &hash, CTransaction &txOut, const Consensus::P if (fAllowSlow) { // use coin database to locate block that contains transaction, and scan it int nHeight = -1; { - CCoinsViewCache &view = *pcoinsTip; + const CCoinsViewCache& view = *pcoinsTip; const CCoins* coins = view.AccessCoins(hash); if (coins) nHeight = coins->nHeight; @@ -1614,18 +1623,24 @@ CAmount GetBlockSubsidy(int nHeight, const Consensus::Params& consensusParams) bool IsInitialBlockDownload() { const CChainParams& chainParams = Params(); + + // Once this function has returned false, it must remain false. + static std::atomic latchToFalse{false}; + // Optimization: pre-test latch before taking the lock. + if (latchToFalse.load(std::memory_order_relaxed)) + return false; + LOCK(cs_main); + if (latchToFalse.load(std::memory_order_relaxed)) + return false; if (fImporting || fReindex) return true; if (fCheckpointsEnabled && chainActive.Height() < Checkpoints::GetTotalBlocksEstimate(chainParams.Checkpoints())) return true; - static bool lockIBDState = false; - if (lockIBDState) - return false; bool state = (chainActive.Height() < pindexBestHeader->nHeight - 24 * 6 || - pindexBestHeader->GetBlockTime() < GetTime() - nMaxTipAge); + std::max(chainActive.Tip()->GetBlockTime(), pindexBestHeader->GetBlockTime()) < GetTime() - nMaxTipAge); if (!state) - lockIBDState = true; + latchToFalse.store(true, std::memory_order_relaxed); return state; } @@ -1633,7 +1648,7 @@ bool fLargeWorkForkFound = false; bool fLargeWorkInvalidChainFound = false; CBlockIndex *pindexBestForkTip = NULL, *pindexBestForkBase = NULL; -static void AlertNotify(const std::string& strMessage, bool fThread) +static void AlertNotify(const std::string& strMessage) { uiInterface.NotifyAlertChanged(); std::string strCmd = GetArg("-alertnotify", ""); @@ -1647,10 +1662,7 @@ static void AlertNotify(const std::string& strMessage, bool fThread) safeStatus = singleQuote+safeStatus+singleQuote; boost::replace_all(strCmd, "%s", safeStatus); - if (fThread) - boost::thread t(runCommand, strCmd); // thread runs free - else - runCommand(strCmd); + boost::thread t(runCommand, strCmd); // thread runs free } void CheckForkWarningConditions() @@ -1672,7 +1684,7 @@ void CheckForkWarningConditions() { std::string warning = std::string("'Warning: Large-work fork detected, forking after block ") + pindexBestForkBase->phashBlock->ToString() + std::string("'"); - AlertNotify(warning, true); + AlertNotify(warning); } if (pindexBestForkTip && pindexBestForkBase) { @@ -1784,7 +1796,7 @@ void static InvalidBlockFound(CBlockIndex *pindex, const CValidationState &state } } -void UpdateCoins(const CTransaction& tx, CValidationState &state, CCoinsViewCache &inputs, CTxUndo &txundo, int nHeight) +void UpdateCoins(const CTransaction& tx, CCoinsViewCache& inputs, CTxUndo &txundo, int nHeight) { // mark inputs spent if (!tx.IsCoinBase()) { @@ -1810,10 +1822,10 @@ void UpdateCoins(const CTransaction& tx, CValidationState &state, CCoinsViewCach inputs.ModifyNewCoins(tx.GetHash(), tx.IsCoinBase())->FromTx(tx, nHeight); } -void UpdateCoins(const CTransaction& tx, CValidationState &state, CCoinsViewCache &inputs, int nHeight) +void UpdateCoins(const CTransaction& tx, CCoinsViewCache& inputs, int nHeight) { CTxUndo txundo; - UpdateCoins(tx, state, inputs, txundo, nHeight); + UpdateCoins(tx, inputs, txundo, nHeight); } bool CScriptCheck::operator()() { @@ -1927,8 +1939,8 @@ bool CheckInputs(const CTransaction& tx, CValidationState &state, const CCoinsVi // such nodes as they are not following the protocol. That // said during an upgrade careful thought should be taken // as to the correct behavior - we may want to continue - // peering with non-upgraded nodes even after a soft-fork - // super-majority vote has passed. + // peering with non-upgraded nodes even after soft-fork + // super-majority signaling has occurred. return state.DoS(100,false, REJECT_INVALID, strprintf("mandatory-script-verify-flag-failed (%s)", ScriptErrorString(check.GetScriptError()))); } } @@ -2204,13 +2216,13 @@ void PartitionCheck(bool (*initialDownloadCheck)(), CCriticalSection& cs, const if (!strWarning.empty()) { strMiscWarning = strWarning; - AlertNotify(strWarning, true); + AlertNotify(strWarning); lastAlertTime = now; } } // Protected by cs_main -static VersionBitsCache versionbitscache; +VersionBitsCache versionbitscache; int32_t ComputeBlockVersion(const CBlockIndex* pindexPrev, const Consensus::Params& params) { @@ -2262,15 +2274,15 @@ static int64_t nTimeIndex = 0; static int64_t nTimeCallbacks = 0; static int64_t nTimeTotal = 0; -bool ConnectBlock(const CBlock& block, CValidationState& state, CBlockIndex* pindex, CCoinsViewCache& view, bool fJustCheck) +bool ConnectBlock(const CBlock& block, CValidationState& state, CBlockIndex* pindex, + CCoinsViewCache& view, const CChainParams& chainparams, bool fJustCheck) { - const CChainParams& chainparams = Params(); AssertLockHeld(cs_main); int64_t nTimeStart = GetTimeMicros(); // Check it again in case a previous version let a bad block in - if (!CheckBlock(block, state, !fJustCheck, !fJustCheck)) + if (!CheckBlock(block, state, chainparams.GetConsensus(), GetAdjustedTime(), !fJustCheck, !fJustCheck)) return error("%s: Consensus::CheckBlock: %s", __func__, FormatStateMessage(state)); // verify that the view's current state corresponds to the previous block @@ -2428,7 +2440,7 @@ bool ConnectBlock(const CBlock& block, CValidationState& state, CBlockIndex* pin if (i > 0) { blockundo.vtxundo.push_back(CTxUndo()); } - UpdateCoins(tx, state, view, i == 0 ? undoDummy : blockundo.vtxundo.back(), pindex->nHeight); + UpdateCoins(tx, view, i == 0 ? undoDummy : blockundo.vtxundo.back(), pindex->nHeight); vPos.push_back(std::make_pair(tx.GetHash(), pos)); pos.nTxOffset += ::GetSerializeSize(tx, SER_DISK, CLIENT_VERSION); @@ -2613,24 +2625,17 @@ void PruneAndFlush() { } /** Update chainActive and related internal data structures. */ -void static UpdateTip(CBlockIndex *pindexNew) { - const CChainParams& chainParams = Params(); +void static UpdateTip(CBlockIndex *pindexNew, const CChainParams& chainParams) { chainActive.SetTip(pindexNew); // New best block nTimeBestReceived = GetTime(); mempool.AddTransactionsUpdated(1); - LogPrintf("%s: new best=%s height=%d version=0x%08x log2_work=%.8g tx=%lu date='%s' progress=%f cache=%.1fMiB(%utx)\n", __func__, - chainActive.Tip()->GetBlockHash().ToString(), chainActive.Height(), chainActive.Tip()->nVersion, - log(chainActive.Tip()->nChainWork.getdouble())/log(2.0), (unsigned long)chainActive.Tip()->nChainTx, - DateTimeStrFormat("%Y-%m-%d %H:%M:%S", chainActive.Tip()->GetBlockTime()), - Checkpoints::GuessVerificationProgress(chainParams.Checkpoints(), chainActive.Tip()), pcoinsTip->DynamicMemoryUsage() * (1.0 / (1<<20)), pcoinsTip->GetCacheSize()); - cvBlockChange.notify_all(); - // Check the version of the last 100 blocks to see if we need to upgrade: static bool fWarned = false; + std::vector warningMessages; if (!IsInitialBlockDownload()) { int nUpgraded = 0; @@ -2642,14 +2647,15 @@ void static UpdateTip(CBlockIndex *pindexNew) { if (state == THRESHOLD_ACTIVE) { strMiscWarning = strprintf(_("Warning: unknown new rules activated (versionbit %i)"), bit); if (!fWarned) { - AlertNotify(strMiscWarning, true); + AlertNotify(strMiscWarning); fWarned = true; } } else { - LogPrintf("%s: unknown new rules are about to activate (versionbit %i)\n", __func__, bit); + warningMessages.push_back(strprintf("unknown new rules are about to activate (versionbit %i)", bit)); } } } + // Check the version of the last 100 blocks to see if we need to upgrade: for (int i = 0; i < 100 && pindex != NULL; i++) { int32_t nExpectedVersion = ComputeBlockVersion(pindex->pprev, chainParams.GetConsensus()); @@ -2658,27 +2664,36 @@ void static UpdateTip(CBlockIndex *pindexNew) { pindex = pindex->pprev; } if (nUpgraded > 0) - LogPrintf("%s: %d of last 100 blocks have unexpected version\n", __func__, nUpgraded); + warningMessages.push_back(strprintf("%d of last 100 blocks have unexpected version", nUpgraded)); if (nUpgraded > 100/2) { // strMiscWarning is read by GetWarnings(), called by Qt and the JSON-RPC code to warn the user: strMiscWarning = _("Warning: Unknown block versions being mined! It's possible unknown rules are in effect"); if (!fWarned) { - AlertNotify(strMiscWarning, true); + AlertNotify(strMiscWarning); fWarned = true; } } } + LogPrintf("%s: new best=%s height=%d version=0x%08x log2_work=%.8g tx=%lu date='%s' progress=%f cache=%.1fMiB(%utx)", __func__, + chainActive.Tip()->GetBlockHash().ToString(), chainActive.Height(), chainActive.Tip()->nVersion, + log(chainActive.Tip()->nChainWork.getdouble())/log(2.0), (unsigned long)chainActive.Tip()->nChainTx, + DateTimeStrFormat("%Y-%m-%d %H:%M:%S", chainActive.Tip()->GetBlockTime()), + Checkpoints::GuessVerificationProgress(chainParams.Checkpoints(), chainActive.Tip()), pcoinsTip->DynamicMemoryUsage() * (1.0 / (1<<20)), pcoinsTip->GetCacheSize()); + if (!warningMessages.empty()) + LogPrintf(" warning='%s'", boost::algorithm::join(warningMessages, ", ")); + LogPrintf("\n"); + } /** Disconnect chainActive's tip. You probably want to call mempool.removeForReorg and manually re-limit mempool size after this, with cs_main held. */ -bool static DisconnectTip(CValidationState& state, const Consensus::Params& consensusParams, bool fBare = false) +bool static DisconnectTip(CValidationState& state, const CChainParams& chainparams, bool fBare = false) { CBlockIndex *pindexDelete = chainActive.Tip(); assert(pindexDelete); // Read block from disk. CBlock block; - if (!ReadBlockFromDisk(block, pindexDelete, consensusParams)) + if (!ReadBlockFromDisk(block, pindexDelete, chainparams.GetConsensus())) return AbortNode(state, "Failed to read block"); // Apply the block atomically to the chain state. int64_t nStart = GetTimeMicros(); @@ -2700,7 +2715,7 @@ bool static DisconnectTip(CValidationState& state, const Consensus::Params& cons // ignore validation errors in resurrected transactions list removed; CValidationState stateDummy; - if (tx.IsCoinBase() || !AcceptToMemoryPool(mempool, stateDummy, tx, false, NULL, NULL, true)) { + if (tx.IsCoinBase() || !AcceptToMemoryPool(mempool, stateDummy, tx, false, NULL, true)) { mempool.removeRecursive(tx, removed); } else if (mempool.exists(tx.GetHash())) { vHashUpdate.push_back(tx.GetHash()); @@ -2715,7 +2730,7 @@ bool static DisconnectTip(CValidationState& state, const Consensus::Params& cons } // Update chainActive and related variables. - UpdateTip(pindexDelete->pprev); + UpdateTip(pindexDelete->pprev, chainparams); // Let wallets know transactions went from 1-confirmed to // 0-confirmed or conflicted: BOOST_FOREACH(const CTransaction &tx, block.vtx) { @@ -2751,7 +2766,7 @@ bool static ConnectTip(CValidationState& state, const CChainParams& chainparams, LogPrint("bench", " - Load block from disk: %.2fms [%.2fs]\n", (nTime2 - nTime1) * 0.001, nTimeReadFromDisk * 0.000001); { CCoinsViewCache view(pcoinsTip); - bool rv = ConnectBlock(*pblock, state, pindexNew, view); + bool rv = ConnectBlock(*pblock, state, pindexNew, view, chainparams); GetMainSignals().BlockChecked(*pblock, state); if (!rv) { if (state.IsInvalid()) @@ -2774,7 +2789,7 @@ bool static ConnectTip(CValidationState& state, const CChainParams& chainparams, list txConflicted; mempool.removeForBlock(pblock->vtx, pindexNew->nHeight, txConflicted, !IsInitialBlockDownload()); // Update chainActive & related variables. - UpdateTip(pindexNew); + UpdateTip(pindexNew, chainparams); // Tell wallet about transactions that went from mempool // to conflicted: BOOST_FOREACH(const CTransaction &tx, txConflicted) { @@ -2865,17 +2880,16 @@ static void PruneBlockIndexCandidates() { * Try to make some progress towards making pindexMostWork the active block. * pblock is either NULL or a pointer to a CBlock corresponding to pindexMostWork. */ -static bool ActivateBestChainStep(CValidationState& state, const CChainParams& chainparams, CBlockIndex* pindexMostWork, const CBlock* pblock) +static bool ActivateBestChainStep(CValidationState& state, const CChainParams& chainparams, CBlockIndex* pindexMostWork, const CBlock* pblock, bool& fInvalidFound) { AssertLockHeld(cs_main); - bool fInvalidFound = false; const CBlockIndex *pindexOldTip = chainActive.Tip(); const CBlockIndex *pindexFork = chainActive.FindFork(pindexMostWork); // Disconnect active blocks which are no longer in the best chain. bool fBlocksDisconnected = false; while (chainActive.Tip() && chainActive.Tip() != pindexFork) { - if (!DisconnectTip(state, chainparams.GetConsensus())) + if (!DisconnectTip(state, chainparams)) return false; fBlocksDisconnected = true; } @@ -2938,6 +2952,28 @@ static bool ActivateBestChainStep(CValidationState& state, const CChainParams& c return true; } +static void NotifyHeaderTip() { + bool fNotify = false; + bool fInitialBlockDownload = false; + static CBlockIndex* pindexHeaderOld = NULL; + CBlockIndex* pindexHeader = NULL; + { + LOCK(cs_main); + if (!setBlockIndexCandidates.empty()) { + pindexHeader = *setBlockIndexCandidates.rbegin(); + } + if (pindexHeader != pindexHeaderOld) { + fNotify = true; + fInitialBlockDownload = IsInitialBlockDownload(); + pindexHeaderOld = pindexHeader; + } + } + // Send block tip changed notifications without cs_main + if (fNotify) { + uiInterface.NotifyHeaderTip(fInitialBlockDownload, pindexHeader); + } +} + /** * Make the best chain active, in multiple steps. The result is either failure * or an activated best chain. pblock is either NULL or a pointer to a block @@ -2945,29 +2981,38 @@ static bool ActivateBestChainStep(CValidationState& state, const CChainParams& c */ bool ActivateBestChain(CValidationState &state, const CChainParams& chainparams, const CBlock *pblock) { CBlockIndex *pindexMostWork = NULL; + CBlockIndex *pindexNewTip = NULL; do { boost::this_thread::interruption_point(); if (ShutdownRequested()) break; - CBlockIndex *pindexNewTip = NULL; const CBlockIndex *pindexFork; bool fInitialDownload; + int nNewHeight; { LOCK(cs_main); CBlockIndex *pindexOldTip = chainActive.Tip(); - pindexMostWork = FindMostWorkChain(); + if (pindexMostWork == NULL) { + pindexMostWork = FindMostWorkChain(); + } // Whether we have anything to do at all. if (pindexMostWork == NULL || pindexMostWork == chainActive.Tip()) return true; - if (!ActivateBestChainStep(state, chainparams, pindexMostWork, pblock && pblock->GetHash() == pindexMostWork->GetBlockHash() ? pblock : NULL)) + bool fInvalidFound = false; + if (!ActivateBestChainStep(state, chainparams, pindexMostWork, pblock && pblock->GetHash() == pindexMostWork->GetBlockHash() ? pblock : NULL, fInvalidFound)) return false; + if (fInvalidFound) { + // Wipe cache, we may need another branch now. + pindexMostWork = NULL; + } pindexNewTip = chainActive.Tip(); pindexFork = chainActive.FindFork(pindexOldTip); fInitialDownload = IsInitialBlockDownload(); + nNewHeight = chainActive.Height(); } // When we reach this point, we switched to a new tip (stored in pindexNewTip). @@ -2996,7 +3041,7 @@ bool ActivateBestChain(CValidationState &state, const CChainParams& chainparams, { LOCK(cs_vNodes); BOOST_FOREACH(CNode* pnode, vNodes) { - if (chainActive.Height() > (pnode->nStartingHeight != -1 ? pnode->nStartingHeight - 2000 : nBlockEstimate)) { + if (nNewHeight > (pnode->nStartingHeight != -1 ? pnode->nStartingHeight - 2000 : nBlockEstimate)) { BOOST_REVERSE_FOREACH(const uint256& hash, vHashes) { pnode->PushBlockHash(hash); } @@ -3009,7 +3054,7 @@ bool ActivateBestChain(CValidationState &state, const CChainParams& chainparams, } } } - } while(pindexMostWork != chainActive.Tip()); + } while (pindexNewTip != pindexMostWork); CheckBlockIndex(chainparams.GetConsensus()); // Write changes periodically to disk, after relay. @@ -3020,7 +3065,7 @@ bool ActivateBestChain(CValidationState &state, const CChainParams& chainparams, return true; } -bool InvalidateBlock(CValidationState& state, const Consensus::Params& consensusParams, CBlockIndex *pindex) +bool InvalidateBlock(CValidationState& state, const CChainParams& chainparams, CBlockIndex *pindex) { AssertLockHeld(cs_main); @@ -3036,7 +3081,7 @@ bool InvalidateBlock(CValidationState& state, const Consensus::Params& consensus setBlockIndexCandidates.erase(pindexWalk); // ActivateBestChain considers blocks already in chainActive // unconditionally valid already, so force disconnect away from it. - if (!DisconnectTip(state, consensusParams)) { + if (!DisconnectTip(state, chainparams)) { mempool.removeForReorg(pcoinsTip, chainActive.Tip()->nHeight + 1, STANDARD_LOCKTIME_VERIFY_FLAGS); return false; } @@ -3059,7 +3104,7 @@ bool InvalidateBlock(CValidationState& state, const Consensus::Params& consensus return true; } -bool ReconsiderBlock(CValidationState& state, CBlockIndex *pindex) { +bool ResetBlockFailureFlags(CBlockIndex *pindex) { AssertLockHeld(cs_main); int nHeight = pindex->nHeight; @@ -3260,20 +3305,20 @@ bool FindUndoPos(CValidationState &state, int nFile, CDiskBlockPos &pos, unsigne return true; } -bool CheckBlockHeader(const CBlockHeader& block, CValidationState& state, bool fCheckPOW) +bool CheckBlockHeader(const CBlockHeader& block, CValidationState& state, const Consensus::Params& consensusParams, int64_t nAdjustedTime, bool fCheckPOW) { // Check proof of work matches claimed amount - if (fCheckPOW && !CheckProofOfWork(block.GetHash(), block.nBits, Params().GetConsensus())) + if (fCheckPOW && !CheckProofOfWork(block.GetHash(), block.nBits, consensusParams)) return state.DoS(50, false, REJECT_INVALID, "high-hash", false, "proof of work failed"); // Check timestamp - if (block.GetBlockTime() > GetAdjustedTime() + 2 * 60 * 60) + if (block.GetBlockTime() > nAdjustedTime + 2 * 60 * 60) return state.Invalid(false, REJECT_INVALID, "time-too-new", "block timestamp too far in the future"); return true; } -bool CheckBlock(const CBlock& block, CValidationState& state, bool fCheckPOW, bool fCheckMerkleRoot) +bool CheckBlock(const CBlock& block, CValidationState& state, const Consensus::Params& consensusParams, int64_t nAdjustedTime, bool fCheckPOW, bool fCheckMerkleRoot) { // These are checks that are independent of context. @@ -3282,7 +3327,7 @@ bool CheckBlock(const CBlock& block, CValidationState& state, bool fCheckPOW, bo // Check that the header is valid (particularly PoW). This is mostly // redundant with the call in AcceptBlockHeader. - if (!CheckBlockHeader(block, state, fCheckPOW)) + if (!CheckBlockHeader(block, state, consensusParams, nAdjustedTime, fCheckPOW)) return false; // Check the merkle root. @@ -3415,9 +3460,8 @@ std::vector GenerateCoinbaseCommitment(CBlock& block, const CBloc return commitment; } -bool ContextualCheckBlockHeader(const CBlockHeader& block, CValidationState& state, CBlockIndex * const pindexPrev) +bool ContextualCheckBlockHeader(const CBlockHeader& block, CValidationState& state, const Consensus::Params& consensusParams, CBlockIndex * const pindexPrev) { - const Consensus::Params& consensusParams = Params().GetConsensus(); // Check proof of work if (block.nBits != GetNextWorkRequired(pindexPrev, &block, consensusParams)) return state.DoS(100, false, REJECT_INVALID, "bad-diffbits", false, "incorrect proof of work"); @@ -3533,11 +3577,11 @@ static bool AcceptBlockHeader(const CBlockHeader& block, CValidationState& state if (ppindex) *ppindex = pindex; if (pindex->nStatus & BLOCK_FAILED_MASK) - return state.Invalid(error("%s: block is marked invalid", __func__), 0, "duplicate"); + return state.Invalid(error("%s: block %s is marked invalid", __func__, hash.ToString()), 0, "duplicate"); return true; } - if (!CheckBlockHeader(block, state)) + if (!CheckBlockHeader(block, state, chainparams.GetConsensus(), GetAdjustedTime())) return error("%s: Consensus::CheckBlockHeader: %s, %s", __func__, hash.ToString(), FormatStateMessage(state)); // Get prev block index @@ -3553,7 +3597,7 @@ static bool AcceptBlockHeader(const CBlockHeader& block, CValidationState& state if (fCheckpointsEnabled && !CheckIndexAgainstCheckpoint(pindexPrev, state, chainparams, hash)) return error("%s: CheckIndexAgainstCheckpoint(): %s", __func__, state.GetRejectReason().c_str()); - if (!ContextualCheckBlockHeader(block, state, pindexPrev)) + if (!ContextualCheckBlockHeader(block, state, chainparams.GetConsensus(), pindexPrev)) return error("%s: Consensus::ContextualCheckBlockHeader: %s, %s", __func__, hash.ToString(), FormatStateMessage(state)); } if (pindex == NULL) @@ -3566,11 +3610,12 @@ static bool AcceptBlockHeader(const CBlockHeader& block, CValidationState& state } /** Store block on disk. If dbp is non-NULL, the file is known to already reside on disk */ -static bool AcceptBlock(const CBlock& block, CValidationState& state, const CChainParams& chainparams, CBlockIndex** ppindex, bool fRequested, CDiskBlockPos* dbp) +static bool AcceptBlock(const CBlock& block, CValidationState& state, const CChainParams& chainparams, CBlockIndex** ppindex, bool fRequested, const CDiskBlockPos* dbp) { AssertLockHeld(cs_main); - CBlockIndex *&pindex = *ppindex; + CBlockIndex *pindexDummy = NULL; + CBlockIndex *&pindex = ppindex ? *ppindex : pindexDummy; if (!AcceptBlockHeader(block, state, chainparams, &pindex)) return false; @@ -3596,7 +3641,7 @@ static bool AcceptBlock(const CBlock& block, CValidationState& state, const CCha if (fTooFarAhead) return true; // Block height is too high } - if ((!CheckBlock(block, state)) || !ContextualCheckBlock(block, state, pindex->pprev)) { + if ((!CheckBlock(block, state, chainparams.GetConsensus(), GetAdjustedTime())) || !ContextualCheckBlock(block, state, pindex->pprev)) { if (state.IsInvalid() && !state.CorruptionPossible()) { pindex->nStatus |= BLOCK_FAILED_VALID; setDirtyBlockIndex.insert(pindex); @@ -3642,7 +3687,7 @@ static bool IsSuperMajority(int minVersion, const CBlockIndex* pstart, unsigned } -bool ProcessNewBlock(CValidationState& state, const CChainParams& chainparams, const CNode* pfrom, const CBlock* pblock, bool fForceProcessing, CDiskBlockPos* dbp) +bool ProcessNewBlock(CValidationState& state, const CChainParams& chainparams, const CNode* pfrom, const CBlock* pblock, bool fForceProcessing, const CDiskBlockPos* dbp) { { LOCK(cs_main); @@ -3660,6 +3705,8 @@ bool ProcessNewBlock(CValidationState& state, const CChainParams& chainparams, c return error("%s: AcceptBlock FAILED", __func__); } + NotifyHeaderTip(); + if (!ActivateBestChain(state, chainparams, pblock)) return error("%s: ActivateBestChain failed", __func__); @@ -3679,13 +3726,13 @@ bool TestBlockValidity(CValidationState& state, const CChainParams& chainparams, indexDummy.nHeight = pindexPrev->nHeight + 1; // NOTE: CheckBlockHeader is called by CheckBlock - if (!ContextualCheckBlockHeader(block, state, pindexPrev)) + if (!ContextualCheckBlockHeader(block, state, chainparams.GetConsensus(), pindexPrev)) return error("%s: Consensus::ContextualCheckBlockHeader: %s", __func__, FormatStateMessage(state)); - if (!CheckBlock(block, state, fCheckPOW, fCheckMerkleRoot)) + if (!CheckBlock(block, state, chainparams.GetConsensus(), GetAdjustedTime(), fCheckPOW, fCheckMerkleRoot)) return error("%s: Consensus::CheckBlock: %s", __func__, FormatStateMessage(state)); if (!ContextualCheckBlock(block, state, pindexPrev)) return error("%s: Consensus::ContextualCheckBlock: %s", __func__, FormatStateMessage(state)); - if (!ConnectBlock(block, state, &indexDummy, viewNew, true)) + if (!ConnectBlock(block, state, &indexDummy, viewNew, chainparams, true)) return false; assert(state.IsValid()); @@ -3867,7 +3914,7 @@ CBlockIndex * InsertBlockIndex(uint256 hash) bool static LoadBlockIndexDB() { const CChainParams& chainparams = Params(); - if (!pblocktree->LoadBlockIndexGuts()) + if (!pblocktree->LoadBlockIndexGuts(InsertBlockIndex)) return false; boost::this_thread::interruption_point(); @@ -4002,10 +4049,18 @@ bool CVerifyDB::VerifyDB(const CChainParams& chainparams, CCoinsView *coinsview, CBlockIndex* pindexFailure = NULL; int nGoodTransactions = 0; CValidationState state; + int reportDone = 0; + LogPrintf("[0%]..."); for (CBlockIndex* pindex = chainActive.Tip(); pindex && pindex->pprev; pindex = pindex->pprev) { boost::this_thread::interruption_point(); - uiInterface.ShowProgress(_("Verifying blocks..."), std::max(1, std::min(99, (int)(((double)(chainActive.Height() - pindex->nHeight)) / (double)nCheckDepth * (nCheckLevel >= 4 ? 50 : 100))))); + int percentageDone = std::max(1, std::min(99, (int)(((double)(chainActive.Height() - pindex->nHeight)) / (double)nCheckDepth * (nCheckLevel >= 4 ? 50 : 100)))); + if (reportDone < percentageDone/10) { + // report every 10% step + LogPrintf("[%d%%]...", percentageDone); + reportDone = percentageDone/10; + } + uiInterface.ShowProgress(_("Verifying blocks..."), percentageDone); if (pindex->nHeight < chainActive.Height()-nCheckDepth) break; if (fPruneMode && !(pindex->nStatus & BLOCK_HAVE_DATA)) { @@ -4018,7 +4073,7 @@ bool CVerifyDB::VerifyDB(const CChainParams& chainparams, CCoinsView *coinsview, if (!ReadBlockFromDisk(block, pindex, chainparams.GetConsensus())) return error("VerifyDB(): *** ReadBlockFromDisk failed at %d, hash=%s", pindex->nHeight, pindex->GetBlockHash().ToString()); // check level 1: verify block validity - if (nCheckLevel >= 1 && !CheckBlock(block, state)) + if (nCheckLevel >= 1 && !CheckBlock(block, state, chainparams.GetConsensus(), GetAdjustedTime())) return error("%s: *** found bad block at %d, hash=%s (%s)\n", __func__, pindex->nHeight, pindex->GetBlockHash().ToString(), FormatStateMessage(state)); // check level 2: verify undo validity @@ -4058,11 +4113,12 @@ bool CVerifyDB::VerifyDB(const CChainParams& chainparams, CCoinsView *coinsview, CBlock block; if (!ReadBlockFromDisk(block, pindex, chainparams.GetConsensus())) return error("VerifyDB(): *** ReadBlockFromDisk failed at %d, hash=%s", pindex->nHeight, pindex->GetBlockHash().ToString()); - if (!ConnectBlock(block, state, pindex, coins)) + if (!ConnectBlock(block, state, pindex, coins, chainparams)) return error("VerifyDB(): *** found unconnectable block at %d, hash=%s", pindex->nHeight, pindex->GetBlockHash().ToString()); } } + LogPrintf("[DONE].\n"); LogPrintf("No coin database inconsistencies in last %i blocks (%i transactions)\n", chainActive.Height() - pindexState->nHeight, nGoodTransactions); return true; @@ -4092,7 +4148,7 @@ bool RewindBlockIndex(const CChainParams& params) // of the blockchain). break; } - if (!DisconnectTip(state, params.GetConsensus(), true)) { + if (!DisconnectTip(state, params, true)) { return error("RewindBlockIndex: unable to disconnect block at height %i", pindex->nHeight); } // Occasionally flush state to disk. @@ -4294,15 +4350,26 @@ bool LoadExternalBlockFile(const CChainParams& chainparams, FILE* fileIn, CDiskB // process in case the block isn't known yet if (mapBlockIndex.count(hash) == 0 || (mapBlockIndex[hash]->nStatus & BLOCK_HAVE_DATA) == 0) { + LOCK(cs_main); CValidationState state; - if (ProcessNewBlock(state, chainparams, NULL, &block, true, dbp)) + if (AcceptBlock(block, state, chainparams, NULL, true, dbp)) nLoaded++; if (state.IsError()) break; } else if (hash != chainparams.GetConsensus().hashGenesisBlock && mapBlockIndex[hash]->nHeight % 1000 == 0) { - LogPrintf("Block Import: already had block %s at height %d\n", hash.ToString(), mapBlockIndex[hash]->nHeight); + LogPrint("reindex", "Block Import: already had block %s at height %d\n", hash.ToString(), mapBlockIndex[hash]->nHeight); + } + + // Activate the genesis block so normal node progress can continue + if (hash == chainparams.GetConsensus().hashGenesisBlock) { + CValidationState state; + if (!ActivateBestChain(state, chainparams)) { + break; + } } + NotifyHeaderTip(); + // Recursively process earlier encountered successors of this block deque queue; queue.push_back(hash); @@ -4314,10 +4381,11 @@ bool LoadExternalBlockFile(const CChainParams& chainparams, FILE* fileIn, CDiskB std::multimap::iterator it = range.first; if (ReadBlockFromDisk(block, it->second, chainparams.GetConsensus())) { - LogPrintf("%s: Processing out of order child %s of %s\n", __func__, block.GetHash().ToString(), + LogPrint("reindex", "%s: Processing out of order child %s of %s\n", __func__, block.GetHash().ToString(), head.ToString()); + LOCK(cs_main); CValidationState dummy; - if (ProcessNewBlock(dummy, chainparams, NULL, &block, true, &it->second)) + if (AcceptBlock(block, dummy, chainparams, NULL, true, &it->second)) { nLoaded++; queue.push_back(block.GetHash()); @@ -4325,6 +4393,7 @@ bool LoadExternalBlockFile(const CChainParams& chainparams, FILE* fileIn, CDiskB } range.first++; mapBlocksUnknownParent.erase(it); + NotifyHeaderTip(); } } } catch (const std::exception& e) { @@ -4709,23 +4778,21 @@ void static ProcessGetData(CNode* pfrom, const Consensus::Params& consensusParam else if (inv.type == MSG_TX || inv.type == MSG_WITNESS_TX) { // Send stream from relay memory - bool pushed = false; - { - LOCK(cs_mapRelay); - map::iterator mi = mapRelay.find(inv.hash); - if (mi != mapRelay.end()) { - pfrom->PushMessageWithFlag(inv.type == MSG_WITNESS_TX ? 0 : SERIALIZE_TRANSACTION_NO_WITNESS, NetMsgType::TX, (*mi).second); - pushed = true; + bool push = false; + auto mi = mapRelay.find(inv.hash); + if (mi != mapRelay.end()) { + pfrom->PushMessageWithFlag(inv.type == MSG_TX ? SERIALIZE_TRANSACTION_NO_WITNESS : 0, NetMsgType::TX, *mi->second); + push = true; + } else if (pfrom->timeLastMempoolReq) { + auto txinfo = mempool.info(inv.hash); + // To protect privacy, do not answer getdata using the mempool when + // that TX couldn't have been INVed in reply to a MEMPOOL request. + if (txinfo.tx && txinfo.nTime <= pfrom->timeLastMempoolReq) { + pfrom->PushMessageWithFlag(inv.type == MSG_TX ? SERIALIZE_TRANSACTION_NO_WITNESS : 0, NetMsgType::TX, *txinfo.tx); + push = true; } } - if (!pushed && (inv.type == MSG_TX || inv.type == MSG_WITNESS_TX)) { - CTransaction tx; - if (mempool.lookup(inv.hash, tx)) { - pfrom->PushMessageWithFlag(inv.type == MSG_WITNESS_TX ? 0 : SERIALIZE_TRANSACTION_NO_WITNESS, NetMsgType::TX, tx); - pushed = true; - } - } - if (!pushed) { + if (!push) { vNotFound.push_back(inv); } } @@ -4754,7 +4821,6 @@ void static ProcessGetData(CNode* pfrom, const Consensus::Params& consensusParam bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv, int64_t nTimeReceived, const CChainParams& chainparams) { - RandAddSeedPerfmon(); LogPrint("net", "received: %s (%u bytes) peer=%d\n", SanitizeString(strCommand), vRecv.size(), pfrom->id); if (mapArgs.count("-dropmessagestest") && GetRand(atoi(mapArgs["-dropmessagestest"])) == 0) { @@ -4769,6 +4835,7 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv, strCommand == NetMsgType::FILTERCLEAR)) { if (pfrom->nVersion >= NO_BLOOM_VERSION) { + LOCK(cs_main); Misbehaving(pfrom->GetId(), 100); return false; } else { @@ -4784,6 +4851,7 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv, if (pfrom->nVersion != 0) { pfrom->PushMessage(NetMsgType::REJECT, strCommand, REJECT_DUPLICATE, string("Duplicate version message")); + LOCK(cs_main); Misbehaving(pfrom->GetId(), 1); return false; } @@ -4792,7 +4860,9 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv, CAddress addrMe; CAddress addrFrom; uint64_t nNonce = 1; - vRecv >> pfrom->nVersion >> pfrom->nServices >> nTime >> addrMe; + uint64_t nServiceInt; + vRecv >> pfrom->nVersion >> nServiceInt >> nTime >> addrMe; + pfrom->nServices = ServiceFlags(nServiceInt); if (!pfrom->fInbound) { addrman.SetServices(pfrom->addr, pfrom->nServices); @@ -4803,6 +4873,7 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv, pfrom->PushMessage(NetMsgType::REJECT, strCommand, REJECT_NONSTANDARD, strprintf("Expected to offer services %08x", pfrom->nServicesExpected)); pfrom->fDisconnect = true; + return false; } if (pfrom->nVersion < MIN_PEER_PROTO_VERSION) @@ -4823,12 +4894,16 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv, vRecv >> LIMITED_STRING(pfrom->strSubVer, MAX_SUBVERSION_LENGTH); pfrom->cleanSubVer = SanitizeString(pfrom->strSubVer); } - if (!vRecv.empty()) + if (!vRecv.empty()) { vRecv >> pfrom->nStartingHeight; - if (!vRecv.empty()) - vRecv >> pfrom->fRelayTxes; // set to true after we get the first filter* message - else - pfrom->fRelayTxes = true; + } + { + LOCK(pfrom->cs_filter); + if (!vRecv.empty()) + vRecv >> pfrom->fRelayTxes; // set to true after we get the first filter* message + else + pfrom->fRelayTxes = true; + } // Disconnect if we connected to ourself if (nNonce == nLocalHostNonce && nNonce > 1) @@ -4857,7 +4932,10 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv, } // Potentially mark this peer as a preferred download peer. + { + LOCK(cs_main); UpdatePreferredDownload(pfrom, State(pfrom->GetId())); + } // Change version pfrom->PushMessage(NetMsgType::VERACK); @@ -4915,6 +4993,7 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv, else if (pfrom->nVersion == 0) { // Must have a version message before anything else + LOCK(cs_main); Misbehaving(pfrom->GetId(), 1); return false; } @@ -4950,6 +5029,7 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv, return true; if (vAddr.size() > 1000) { + LOCK(cs_main); Misbehaving(pfrom->GetId(), 20); return error("message addr size() = %u", vAddr.size()); } @@ -4962,7 +5042,7 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv, { boost::this_thread::interruption_point(); - if (!(addr.nServices & NODE_NETWORK)) + if ((addr.nServices & REQUIRED_SERVICES) != REQUIRED_SERVICES) continue; if (addr.nTime <= 100000000 || addr.nTime > nNow + 10 * 60) @@ -4976,25 +5056,20 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv, LOCK(cs_vNodes); // Use deterministic randomness to send to the same nodes for 24 hours // at a time so the addrKnowns of the chosen nodes prevent repeats - static uint256 hashSalt; - if (hashSalt.IsNull()) - hashSalt = GetRandHash(); + static const uint64_t salt0 = GetRand(std::numeric_limits::max()); + static const uint64_t salt1 = GetRand(std::numeric_limits::max()); uint64_t hashAddr = addr.GetHash(); - uint256 hashRand = ArithToUint256(UintToArith256(hashSalt) ^ (hashAddr<<32) ^ ((GetTime()+hashAddr)/(24*60*60))); - hashRand = Hash(BEGIN(hashRand), END(hashRand)); - multimap mapMix; + multimap mapMix; + const CSipHasher hasher = CSipHasher(salt0, salt1).Write(hashAddr << 32).Write((GetTime() + hashAddr) / (24*60*60)); BOOST_FOREACH(CNode* pnode, vNodes) { if (pnode->nVersion < CADDR_TIME_VERSION) continue; - unsigned int nPointer; - memcpy(&nPointer, &pnode, sizeof(nPointer)); - uint256 hashKey = ArithToUint256(UintToArith256(hashRand) ^ nPointer); - hashKey = Hash(BEGIN(hashKey), END(hashKey)); + uint64_t hashKey = CSipHasher(hasher).Write(pnode->id).Finalize(); mapMix.insert(make_pair(hashKey, pnode)); } int nRelayNodes = fReachable ? 2 : 1; // limited relaying of addresses outside our network(s) - for (multimap::iterator mi = mapMix.begin(); mi != mapMix.end() && nRelayNodes-- > 0; ++mi) + for (multimap::iterator mi = mapMix.begin(); mi != mapMix.end() && nRelayNodes-- > 0; ++mi) ((*mi).second)->PushAddress(addr); } } @@ -5022,11 +5097,12 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv, vRecv >> vInv; if (vInv.size() > MAX_INV_SZ) { + LOCK(cs_main); Misbehaving(pfrom->GetId(), 20); return error("message inv size() = %u", vInv.size()); } - bool fBlocksOnly = GetBoolArg("-blocksonly", DEFAULT_BLOCKSONLY); + bool fBlocksOnly = !fRelayTxes; // Allow whitelisted peers to send data other than blocks in blocks only mode if whitelistrelay is true if (pfrom->fWhitelisted && GetBoolArg("-whitelistrelay", DEFAULT_WHITELISTRELAY)) @@ -5041,7 +5117,6 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv, CInv &inv = vInv[nInv]; boost::this_thread::interruption_point(); - pfrom->AddInventoryKnown(inv); bool fAlreadyHave = AlreadyHave(inv); LogPrint("net", "got inv: %s %s peer=%d\n", inv.ToString(), fAlreadyHave ? "have" : "new", pfrom->id); @@ -5079,6 +5154,7 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv, } else { + pfrom->AddInventoryKnown(inv); if (fBlocksOnly) LogPrint("net", "transaction (%s) inv sent in violation of protocol peer=%d\n", inv.hash.ToString(), pfrom->id); else if (!fAlreadyHave && !fImporting && !fReindex && !IsInitialBlockDownload()) @@ -5105,6 +5181,7 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv, vRecv >> vInv; if (vInv.size() > MAX_INV_SZ) { + LOCK(cs_main); Misbehaving(pfrom->GetId(), 20); return error("message getdata size() = %u", vInv.size()); } @@ -5217,7 +5294,7 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv, { // Stop processing the transaction early if // We are in blocks only mode and peer is either not whitelisted or whitelistrelay is off - if (GetBoolArg("-blocksonly", DEFAULT_BLOCKSONLY) && (!pfrom->fWhitelisted || !GetBoolArg("-whitelistrelay", DEFAULT_WHITELISTRELAY))) + if (!fRelayTxes && (!pfrom->fWhitelisted || !GetBoolArg("-whitelistrelay", DEFAULT_WHITELISTRELAY))) { LogPrint("net", "transaction sent in violation of protocol peer=%d\n", pfrom->id); return true; @@ -5239,10 +5316,9 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv, pfrom->setAskFor.erase(inv.hash); mapAlreadyAskedFor.erase(inv.hash); - CFeeRate txFeeRate = CFeeRate(0); - if (!AlreadyHave(inv) && AcceptToMemoryPool(mempool, state, tx, true, &fMissingInputs, &txFeeRate)) { + if (!AlreadyHave(inv) && AcceptToMemoryPool(mempool, state, tx, true, &fMissingInputs)) { mempool.check(pcoinsTip); - RelayTransaction(tx, txFeeRate); + RelayTransaction(tx); vWorkQueue.push_back(inv.hash); LogPrint("mempool", "AcceptToMemoryPool: peer=%d: accepted %s (poolsz %u txn, %u kB)\n", @@ -5273,10 +5349,9 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv, if (setMisbehaving.count(fromPeer)) continue; - CFeeRate orphanFeeRate = CFeeRate(0); - if (AcceptToMemoryPool(mempool, stateDummy, orphanTx, true, &fMissingInputs2, &orphanFeeRate)) { + if (AcceptToMemoryPool(mempool, stateDummy, orphanTx, true, &fMissingInputs2)) { LogPrint("mempool", " accepted orphan tx %s\n", orphanHash.ToString()); - RelayTransaction(orphanTx, orphanFeeRate); + RelayTransaction(orphanTx); vWorkQueue.push_back(orphanHash); vEraseQueue.push_back(orphanHash); } @@ -5333,7 +5408,7 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv, int nDoS = 0; if (!state.IsInvalid(nDoS) || nDoS == 0) { LogPrintf("Force relaying tx %s from whitelisted peer=%d\n", tx.GetHash().ToString(), pfrom->id); - RelayTransaction(tx, txFeeRate); + RelayTransaction(tx); } else { LogPrintf("Not relaying invalid transaction %s from whitelisted peer=%d (%s)\n", tx.GetHash().ToString(), pfrom->id, FormatStateMessage(state)); } @@ -5365,6 +5440,7 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv, // Bypass the normal CBlock deserialization, as we don't want to risk deserializing 2000 full blocks. unsigned int nCount = ReadCompactSize(vRecv); if (nCount > MAX_HEADERS_RESULTS) { + LOCK(cs_main); Misbehaving(pfrom->GetId(), 20); return error("headers message size = %u", nCount); } @@ -5374,6 +5450,7 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv, ReadCompactSize(vRecv); // ignore tx count; assume it is 0. } + { LOCK(cs_main); if (nCount == 0) { @@ -5381,6 +5458,13 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv, return true; } + // If we already know the last header in the message, then it contains + // no new information for us. In this case, we do not request + // more headers later. This prevents multiple chains of redundant + // getheader requests from running in parallel if triggered by incoming + // blocks while the node is still in initial headers sync. + const bool hasNewHeaders = (mapBlockIndex.count(headers.back().GetHash()) == 0); + CBlockIndex *pindexLast = NULL; BOOST_FOREACH(const CBlockHeader& header, headers) { CValidationState state; @@ -5401,7 +5485,7 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv, if (pindexLast) UpdateBlockAvailability(pfrom->GetId(), pindexLast->GetBlockHash()); - if (nCount == MAX_HEADERS_RESULTS && pindexLast) { + if (nCount == MAX_HEADERS_RESULTS && pindexLast && hasNewHeaders) { // Headers message had its maximum size; the peer may have more headers. // TODO: optimize: if pindexLast is an ancestor of chainActive.Tip or pindexBestHeader, continue // from there instead. @@ -5458,6 +5542,9 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv, } CheckBlockIndex(chainparams.GetConsensus()); + } + + NotifyHeaderTip(); } else if (strCommand == NetMsgType::BLOCK && !fImporting && !fReindex) // Ignore blocks received while importing @@ -5465,10 +5552,7 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv, CBlock block; vRecv >> block; - CInv inv(MSG_BLOCK, block.GetHash()); - LogPrint("net", "received block %s peer=%d\n", inv.hash.ToString(), pfrom->id); - - pfrom->AddInventoryKnown(inv); + LogPrint("net", "received block %s peer=%d\n", block.GetHash().ToString(), pfrom->id); CValidationState state; // Process all blocks from whitelisted peers, even if not requested, @@ -5481,7 +5565,7 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv, if (state.IsInvalid(nDoS)) { assert (state.GetRejectCode() < REJECT_INTERNAL); // Blocks are never rejected with internal reject codes pfrom->PushMessage(NetMsgType::REJECT, strCommand, (unsigned char)state.GetRejectCode(), - state.GetRejectReason().substr(0, MAX_REJECT_MESSAGE_LENGTH), inv.hash); + state.GetRejectReason().substr(0, MAX_REJECT_MESSAGE_LENGTH), block.GetHash()); if (nDoS > 0) { LOCK(cs_main); Misbehaving(pfrom->GetId(), nDoS); @@ -5520,40 +5604,22 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv, else if (strCommand == NetMsgType::MEMPOOL) { + if (!(nLocalServices & NODE_BLOOM) && !pfrom->fWhitelisted) + { + LogPrint("net", "mempool request with bloom filters disabled, disconnect peer=%d\n", pfrom->GetId()); + pfrom->fDisconnect = true; + return true; + } + if (CNode::OutboundTargetReached(false) && !pfrom->fWhitelisted) { LogPrint("net", "mempool request with bandwidth limit reached, disconnect peer=%d\n", pfrom->GetId()); pfrom->fDisconnect = true; return true; } - LOCK2(cs_main, pfrom->cs_filter); - std::vector vtxid; - mempool.queryHashes(vtxid); - vector vInv; - BOOST_FOREACH(uint256& hash, vtxid) { - CInv inv(MSG_TX, hash); - if (pfrom->pfilter) { - CTransaction tx; - bool fInMemPool = mempool.lookup(hash, tx); - if (!fInMemPool) continue; // another thread removed since queryHashes, maybe... - if (!pfrom->pfilter->IsRelevantAndUpdate(tx)) continue; - } - if (pfrom->minFeeFilter) { - CFeeRate feeRate; - mempool.lookupFeeRate(hash, feeRate); - LOCK(pfrom->cs_feeFilter); - if (feeRate.GetFeePerK() < pfrom->minFeeFilter) - continue; - } - vInv.push_back(inv); - if (vInv.size() == MAX_INV_SZ) { - pfrom->PushMessage(NetMsgType::INV, vInv); - vInv.clear(); - } - } - if (vInv.size() > 0) - pfrom->PushMessage(NetMsgType::INV, vInv); + LOCK(pfrom->cs_inventory); + pfrom->fSendMempool = true; } @@ -5641,12 +5707,16 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv, CBloomFilter filter; vRecv >> filter; + LOCK(pfrom->cs_filter); + if (!filter.IsWithinSizeConstraints()) + { // There is no excuse for sending a too-large filter + LOCK(cs_main); Misbehaving(pfrom->GetId(), 100); + } else { - LOCK(pfrom->cs_filter); delete pfrom->pfilter; pfrom->pfilter = new CBloomFilter(filter); pfrom->pfilter->UpdateEmptyFull(); @@ -5664,13 +5734,17 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv, // and thus, the maximum size any matched object can have) in a filteradd message if (vData.size() > MAX_SCRIPT_ELEMENT_SIZE) { + LOCK(cs_main); Misbehaving(pfrom->GetId(), 100); } else { LOCK(pfrom->cs_filter); if (pfrom->pfilter) pfrom->pfilter->insert(vData); else + { + LOCK(cs_main); Misbehaving(pfrom->GetId(), 100); + } } } @@ -5824,6 +5898,11 @@ bool ProcessMessages(CNode* pfrom) // Allow exceptions from over-long size LogPrintf("%s(%s, %u bytes): Exception '%s' caught\n", __func__, SanitizeString(strCommand), nMessageSize, e.what()); } + else if (strstr(e.what(), "non-canonical ReadCompactSize()")) + { + // Allow exceptions from non-canonical encoding + LogPrintf("%s(%s, %u bytes): Exception '%s' caught\n", __func__, SanitizeString(strCommand), nMessageSize, e.what()); + } else { PrintExceptionContinue(&e, "ProcessMessages()"); @@ -5851,6 +5930,22 @@ bool ProcessMessages(CNode* pfrom) return fOk; } +class CompareInvMempoolOrder +{ + CTxMemPool *mp; +public: + CompareInvMempoolOrder(CTxMemPool *mempool) + { + mp = mempool; + } + + bool operator()(std::set::iterator a, std::set::iterator b) + { + /* As std::make_heap produces a max-heap, we want the entries with the + * fewest ancestors/highest fee to sort later. */ + return mp->CompareDepthAndScore(*b, *a); + } +}; bool SendMessages(CNode* pto) { @@ -5924,6 +6019,9 @@ bool SendMessages(CNode* pto) pto->vAddrToSend.clear(); if (!vAddr.empty()) pto->PushMessage(NetMsgType::ADDR, vAddr); + // we only send the big addr message once + if (pto->vAddrToSend.capacity() > 40) + pto->vAddrToSend.shrink_to_fit(); } CNodeState &state = *State(pto->GetId()); @@ -6009,7 +6107,21 @@ bool SendMessages(CNode* pto) fRevertToInv = true; break; } - assert(pBestIndex == NULL || pindex->pprev == pBestIndex); + if (pBestIndex != NULL && pindex->pprev != pBestIndex) { + // This means that the list of blocks to announce don't + // connect to each other. + // This shouldn't really be possible to hit during + // regular operation (because reorgs should take us to + // a chain that has some block not on the prior chain, + // which should be caught by the prior check), but one + // way this could happen is by using invalidateblock / + // reconsiderblock repeatedly on the tip, causing it to + // be added multiple times to vBlockHashesToAnnounce. + // Robustly deal with this rare situation by reverting + // to an inv. + fRevertToInv = true; + break; + } pBestIndex = pindex; if (fFoundStartingHeader) { // add this to the headers message @@ -6047,9 +6159,7 @@ bool SendMessages(CNode* pto) hashToAnnounce.ToString(), chainActive.Tip()->GetBlockHash().ToString()); } - // If the peer announced this block to us, don't inv it back. - // (Since block announcements may not be via inv's, we can't solely rely on - // setInventoryKnown to track this.) + // If the peer's chain has this block, don't inv it back. if (!PeerHasHeader(&state, pindex)) { pto->PushInventory(CInv(MSG_BLOCK, hashToAnnounce)); LogPrint("net", "%s: sending inv peer=%d hash=%s\n", __func__, @@ -6076,49 +6186,132 @@ bool SendMessages(CNode* pto) // Message: inventory // vector vInv; - vector vInvWait; { + LOCK(pto->cs_inventory); + vInv.reserve(std::max(pto->vInventoryBlockToSend.size(), INVENTORY_BROADCAST_MAX)); + + // Add blocks + BOOST_FOREACH(const uint256& hash, pto->vInventoryBlockToSend) { + vInv.push_back(CInv(MSG_BLOCK, hash)); + if (vInv.size() == MAX_INV_SZ) { + pto->PushMessage(NetMsgType::INV, vInv); + vInv.clear(); + } + } + pto->vInventoryBlockToSend.clear(); + + // Check whether periodic sends should happen bool fSendTrickle = pto->fWhitelisted; if (pto->nNextInvSend < nNow) { fSendTrickle = true; - pto->nNextInvSend = PoissonNextSend(nNow, AVG_INVENTORY_BROADCAST_INTERVAL); + // Use half the delay for outbound peers, as there is less privacy concern for them. + pto->nNextInvSend = PoissonNextSend(nNow, INVENTORY_BROADCAST_INTERVAL >> !pto->fInbound); + } + + // Time to send but the peer has requested we not relay transactions. + if (fSendTrickle) { + LOCK(pto->cs_filter); + if (!pto->fRelayTxes) pto->setInventoryTxToSend.clear(); } - LOCK(pto->cs_inventory); - vInv.reserve(std::min(1000, pto->vInventoryToSend.size())); - vInvWait.reserve(pto->vInventoryToSend.size()); - BOOST_FOREACH(const CInv& inv, pto->vInventoryToSend) - { - if ((inv.type == MSG_TX || inv.type == MSG_WITNESS_TX) && pto->filterInventoryKnown.contains(inv.hash)) - continue; - // trickle out tx inv to protect privacy - if ((inv.type == MSG_TX || inv.type == MSG_WITNESS_TX) && !fSendTrickle) + // Respond to BIP35 mempool requests + if (fSendTrickle && pto->fSendMempool) { + auto vtxinfo = mempool.infoAll(); + pto->fSendMempool = false; + CAmount filterrate = 0; { - // 1/4 of tx invs blast to all immediately - static uint256 hashSalt; - if (hashSalt.IsNull()) - hashSalt = GetRandHash(); - uint256 hashRand = ArithToUint256(UintToArith256(inv.hash) ^ UintToArith256(hashSalt)); - hashRand = Hash(BEGIN(hashRand), END(hashRand)); - bool fTrickleWait = ((UintToArith256(hashRand) & 3) != 0); - - if (fTrickleWait) - { - vInvWait.push_back(inv); - continue; - } + LOCK(pto->cs_feeFilter); + filterrate = pto->minFeeFilter; } - pto->filterInventoryKnown.insert(inv.hash); + LOCK(pto->cs_filter); + + for (const auto& txinfo : vtxinfo) { + const uint256& hash = txinfo.tx->GetHash(); + CInv inv(MSG_TX, hash); + pto->setInventoryTxToSend.erase(hash); + if (filterrate) { + if (txinfo.feeRate.GetFeePerK() < filterrate) + continue; + } + if (pto->pfilter) { + if (!pto->pfilter->IsRelevantAndUpdate(*txinfo.tx)) continue; + } + pto->filterInventoryKnown.insert(hash); + vInv.push_back(inv); + if (vInv.size() == MAX_INV_SZ) { + pto->PushMessage(NetMsgType::INV, vInv); + vInv.clear(); + } + } + pto->timeLastMempoolReq = GetTime(); + } - vInv.push_back(inv); - if (vInv.size() >= 1000) + // Determine transactions to relay + if (fSendTrickle) { + // Produce a vector with all candidates for sending + vector::iterator> vInvTx; + vInvTx.reserve(pto->setInventoryTxToSend.size()); + for (std::set::iterator it = pto->setInventoryTxToSend.begin(); it != pto->setInventoryTxToSend.end(); it++) { + vInvTx.push_back(it); + } + CAmount filterrate = 0; { - pto->PushMessage(NetMsgType::INV, vInv); - vInv.clear(); + LOCK(pto->cs_feeFilter); + filterrate = pto->minFeeFilter; + } + // Topologically and fee-rate sort the inventory we send for privacy and priority reasons. + // A heap is used so that not all items need sorting if only a few are being sent. + CompareInvMempoolOrder compareInvMempoolOrder(&mempool); + std::make_heap(vInvTx.begin(), vInvTx.end(), compareInvMempoolOrder); + // No reason to drain out at many times the network's capacity, + // especially since we have many peers and some will draw much shorter delays. + unsigned int nRelayedTransactions = 0; + LOCK(pto->cs_filter); + while (!vInvTx.empty() && nRelayedTransactions < INVENTORY_BROADCAST_MAX) { + // Fetch the top element from the heap + std::pop_heap(vInvTx.begin(), vInvTx.end(), compareInvMempoolOrder); + std::set::iterator it = vInvTx.back(); + vInvTx.pop_back(); + uint256 hash = *it; + // Remove it from the to-be-sent set + pto->setInventoryTxToSend.erase(it); + // Check if not in the filter already + if (pto->filterInventoryKnown.contains(hash)) { + continue; + } + // Not in the mempool anymore? don't bother sending it. + auto txinfo = mempool.info(hash); + if (!txinfo.tx) { + continue; + } + if (filterrate && txinfo.feeRate.GetFeePerK() < filterrate) { + continue; + } + if (pto->pfilter && !pto->pfilter->IsRelevantAndUpdate(*txinfo.tx)) continue; + // Send + vInv.push_back(CInv(MSG_TX, hash)); + nRelayedTransactions++; + { + // Expire old relay messages + while (!vRelayExpiration.empty() && vRelayExpiration.front().first < nNow) + { + mapRelay.erase(vRelayExpiration.front().second); + vRelayExpiration.pop_front(); + } + + auto ret = mapRelay.insert(std::make_pair(hash, std::move(txinfo.tx))); + if (ret.second) { + vRelayExpiration.push_back(std::make_pair(nNow + 15 * 60 * 1000000, ret.first)); + } + } + if (vInv.size() == MAX_INV_SZ) { + pto->PushMessage(NetMsgType::INV, vInv); + vInv.clear(); + } + pto->filterInventoryKnown.insert(hash); } } - pto->vInventoryToSend = vInvWait; } if (!vInv.empty()) pto->PushMessage(NetMsgType::INV, vInv); diff --git a/src/main.h b/src/main.h index 3087f20387baa..b0cb849827b6b 100644 --- a/src/main.h +++ b/src/main.h @@ -99,9 +99,12 @@ static const unsigned int MAX_REJECT_MESSAGE_LENGTH = 111; static const unsigned int AVG_LOCAL_ADDRESS_BROADCAST_INTERVAL = 24 * 24 * 60; /** Average delay between peer address broadcasts in seconds. */ static const unsigned int AVG_ADDRESS_BROADCAST_INTERVAL = 30; -/** Average delay between trickled inventory broadcasts in seconds. - * Blocks, whitelisted receivers, and a random 25% of transactions bypass this. */ -static const unsigned int AVG_INVENTORY_BROADCAST_INTERVAL = 5; +/** Average delay between trickled inventory transmissions in seconds. + * Blocks and whitelisted receivers bypass this, outbound peers get half this delay. */ +static const unsigned int INVENTORY_BROADCAST_INTERVAL = 5; +/** Maximum number of inventory items to send per transmission. + * Limits the impact of low-fee transaction floods. */ +static const unsigned int INVENTORY_BROADCAST_MAX = 7 * INVENTORY_BROADCAST_INTERVAL; /** Average delay between feefilter broadcasts in seconds. */ static const unsigned int AVG_FEEFILTER_BROADCAST_INTERVAL = 10 * 60; /** Maximum feefilter broadcast delay after significant change. */ @@ -210,10 +213,10 @@ void UnregisterNodeSignals(CNodeSignals& nodeSignals); * @param[in] pfrom The node which we are receiving the block from; it is added to mapBlockSource and may be penalised if the block is invalid. * @param[in] pblock The block we want to process. * @param[in] fForceProcessing Process this block even if unrequested; used for non-network block sources and whitelisted peers. - * @param[out] dbp If pblock is stored to disk (or already there), this will be set to its location. + * @param[out] dbp The already known disk position of pblock, or NULL if not yet stored. * @return True if state.IsValid() */ -bool ProcessNewBlock(CValidationState& state, const CChainParams& chainparams, const CNode* pfrom, const CBlock* pblock, bool fForceProcessing, CDiskBlockPos* dbp); +bool ProcessNewBlock(CValidationState& state, const CChainParams& chainparams, const CNode* pfrom, const CBlock* pblock, bool fForceProcessing, const CDiskBlockPos* dbp); /** Check whether enough disk space is available for an incoming block */ bool CheckDiskSpace(uint64_t nAdditionalBytes = 0); /** Open a block file (blk?????.dat) */ @@ -293,7 +296,7 @@ void PruneAndFlush(); /** (try to) add transaction to memory pool **/ bool AcceptToMemoryPool(CTxMemPool& pool, CValidationState &state, const CTransaction &tx, bool fLimitFree, - bool* pfMissingInputs, CFeeRate* txFeeRate, bool fOverrideMempoolLimit=false, const CAmount nAbsurdFee=0); + bool* pfMissingInputs, bool fOverrideMempoolLimit=false, const CAmount nAbsurdFee=0); /** Convert CValidationState to a human-readable message for logging */ std::string FormatStateMessage(const CValidationState &state); @@ -308,30 +311,6 @@ struct CNodeStateStats { std::vector vHeightInFlight; }; -struct CDiskTxPos : public CDiskBlockPos -{ - unsigned int nTxOffset; // after header - - ADD_SERIALIZE_METHODS; - - template - inline void SerializationOp(Stream& s, Operation ser_action, int nType, int nVersion) { - READWRITE(*(CDiskBlockPos*)this); - READWRITE(VARINT(nTxOffset)); - } - - CDiskTxPos(const CDiskBlockPos &blockIn, unsigned int nTxOffsetIn) : CDiskBlockPos(blockIn.nFile, blockIn.nPos), nTxOffset(nTxOffsetIn) { - } - - CDiskTxPos() { - SetNull(); - } - - void SetNull() { - CDiskBlockPos::SetNull(); - nTxOffset = 0; - } -}; /** @@ -368,7 +347,7 @@ bool CheckInputs(const CTransaction& tx, CValidationState &state, const CCoinsVi unsigned int flags, bool cacheStore, std::vector *pvChecks = NULL); /** Apply the effects of this transaction on the UTXO set represented by view */ -void UpdateCoins(const CTransaction& tx, CValidationState &state, CCoinsViewCache &inputs, int nHeight); +void UpdateCoins(const CTransaction& tx, CCoinsViewCache& inputs, int nHeight); /** Context-independent validity checks */ bool CheckTransaction(const CTransaction& tx, CValidationState& state); @@ -457,19 +436,20 @@ bool ReadBlockFromDisk(CBlock& block, const CBlockIndex* pindex, const Consensus /** Functions for validating blocks and updating the block tree */ /** Context-independent validity checks */ -bool CheckBlockHeader(const CBlockHeader& block, CValidationState& state, bool fCheckPOW = true); -bool CheckBlock(const CBlock& block, CValidationState& state, bool fCheckPOW = true, bool fCheckMerkleRoot = true); +bool CheckBlockHeader(const CBlockHeader& block, CValidationState& state, const Consensus::Params& consensusParams, int64_t nAdjustedTime, bool fCheckPOW = true); +bool CheckBlock(const CBlock& block, CValidationState& state, const Consensus::Params& consensusParams, int64_t nAdjustedTime, bool fCheckPOW = true, bool fCheckMerkleRoot = true); /** Context-dependent validity checks. * By "context", we mean only the previous block headers, but not the UTXO * set; UTXO-related validity checks are done in ConnectBlock(). */ -bool ContextualCheckBlockHeader(const CBlockHeader& block, CValidationState& state, CBlockIndex *pindexPrev); +bool ContextualCheckBlockHeader(const CBlockHeader& block, CValidationState& state, const Consensus::Params& consensusParams, CBlockIndex* pindexPrev); bool ContextualCheckBlock(const CBlock& block, CValidationState& state, CBlockIndex *pindexPrev); /** Apply the effects of this block (with given index) on the UTXO set represented by coins. * Validity checks that depend on the UTXO set are also done; ConnectBlock() * can fail if those validity checks fail (among other reasons). */ -bool ConnectBlock(const CBlock& block, CValidationState& state, CBlockIndex* pindex, CCoinsViewCache& coins, bool fJustCheck = false); +bool ConnectBlock(const CBlock& block, CValidationState& state, CBlockIndex* pindex, CCoinsViewCache& coins, + const CChainParams& chainparams, bool fJustCheck = false); /** Undo the effects of this block (with given index) on the UTXO set represented by coins. * In case pfClean is provided, operation will try to be tolerant about errors, and *pfClean @@ -492,61 +472,6 @@ void UpdateUncommitedBlockStructures(CBlock& block, const CBlockIndex* pindexPre /** Produce the necessary coinbase commitment for a block (modifies the hash, don't call for mined blocks). */ std::vector GenerateCoinbaseCommitment(CBlock& block, const CBlockIndex* pindexPrev, const Consensus::Params& consensusParams); - -class CBlockFileInfo -{ -public: - unsigned int nBlocks; //!< number of blocks stored in file - unsigned int nSize; //!< number of used bytes of block file - unsigned int nUndoSize; //!< number of used bytes in the undo file - unsigned int nHeightFirst; //!< lowest height of block in file - unsigned int nHeightLast; //!< highest height of block in file - uint64_t nTimeFirst; //!< earliest time of block in file - uint64_t nTimeLast; //!< latest time of block in file - - ADD_SERIALIZE_METHODS; - - template - inline void SerializationOp(Stream& s, Operation ser_action, int nType, int nVersion) { - READWRITE(VARINT(nBlocks)); - READWRITE(VARINT(nSize)); - READWRITE(VARINT(nUndoSize)); - READWRITE(VARINT(nHeightFirst)); - READWRITE(VARINT(nHeightLast)); - READWRITE(VARINT(nTimeFirst)); - READWRITE(VARINT(nTimeLast)); - } - - void SetNull() { - nBlocks = 0; - nSize = 0; - nUndoSize = 0; - nHeightFirst = 0; - nHeightLast = 0; - nTimeFirst = 0; - nTimeLast = 0; - } - - CBlockFileInfo() { - SetNull(); - } - - std::string ToString() const; - - /** update statistics (does not update nSize) */ - void AddBlock(unsigned int nHeightIn, uint64_t nTimeIn) { - if (nBlocks==0 || nHeightFirst > nHeightIn) - nHeightFirst = nHeightIn; - if (nBlocks==0 || nTimeFirst > nTimeIn) - nTimeFirst = nTimeIn; - nBlocks++; - if (nHeightIn > nHeightLast) - nHeightLast = nHeightIn; - if (nTimeIn > nTimeLast) - nTimeLast = nTimeIn; - } -}; - /** RAII wrapper for VerifyDB: Verify consistency of the block and coin databases */ class CVerifyDB { public: @@ -559,10 +484,10 @@ class CVerifyDB { CBlockIndex* FindForkInGlobalIndex(const CChain& chain, const CBlockLocator& locator); /** Mark a block as invalid. */ -bool InvalidateBlock(CValidationState& state, const Consensus::Params& consensusParams, CBlockIndex *pindex); +bool InvalidateBlock(CValidationState& state, const CChainParams& chainparams, CBlockIndex *pindex); /** Remove invalidity status from a block and its descendants. */ -bool ReconsiderBlock(CValidationState& state, CBlockIndex *pindex); +bool ResetBlockFailureFlags(CBlockIndex *pindex); /** The currently-connected chain of blocks (protected by cs_main). */ extern CChain chainActive; @@ -580,6 +505,8 @@ extern CBlockTreeDB *pblocktree; */ int GetSpendHeight(const CCoinsViewCache& inputs); +extern VersionBitsCache versionbitscache; + /** * Determine what nVersion a new block should use. */ diff --git a/src/memusage.h b/src/memusage.h index 49760e64c78cd..3810bfad07a91 100644 --- a/src/memusage.h +++ b/src/memusage.h @@ -5,6 +5,8 @@ #ifndef BITCOIN_MEMUSAGE_H #define BITCOIN_MEMUSAGE_H +#include "indirectmap.h" + #include #include @@ -70,6 +72,15 @@ struct stl_tree_node X x; }; +struct stl_shared_counter +{ + /* Various platforms use different sized counters here. + * Conservatively assume that they won't be larger than size_t. */ + void* class_type; + size_t use_count; + size_t weak_count; +}; + template static inline size_t DynamicUsage(const std::vector& v) { @@ -106,6 +117,35 @@ static inline size_t IncrementalDynamicUsage(const std::map& m) return MallocUsage(sizeof(stl_tree_node >)); } +// indirectmap has underlying map with pointer as key + +template +static inline size_t DynamicUsage(const indirectmap& m) +{ + return MallocUsage(sizeof(stl_tree_node >)) * m.size(); +} + +template +static inline size_t IncrementalDynamicUsage(const indirectmap& m) +{ + return MallocUsage(sizeof(stl_tree_node >)); +} + +template +static inline size_t DynamicUsage(const std::unique_ptr& p) +{ + return p ? MallocUsage(sizeof(X)) : 0; +} + +template +static inline size_t DynamicUsage(const std::shared_ptr& p) +{ + // A shared_ptr can either use a single continuous memory block for both + // the counter and the storage (when using std::make_shared), or separate. + // We can't observe the difference, however, so assume the worst. + return p ? MallocUsage(sizeof(X)) + MallocUsage(sizeof(stl_shared_counter)) : 0; +} + // Boost data structures template diff --git a/src/miner.cpp b/src/miner.cpp index 87eb1b3e4510b..50f6018939401 100644 --- a/src/miner.cpp +++ b/src/miner.cpp @@ -72,32 +72,15 @@ int64_t UpdateTime(CBlockHeader* pblock, const Consensus::Params& consensusParam return nNewTime - nOldTime; } -CBlockTemplate* CreateNewBlock(const CChainParams& chainparams, const CScript& scriptPubKeyIn) +BlockAssembler::BlockAssembler(const CChainParams& _chainparams) + : chainparams(_chainparams) { - // Create new block - auto_ptr pblocktemplate(new CBlockTemplate()); - if(!pblocktemplate.get()) - return NULL; - CBlock *pblock = &pblocktemplate->block; // pointer for convenience - - // Create coinbase tx - CMutableTransaction txNew; - txNew.vin.resize(1); - txNew.vin[0].prevout.SetNull(); - txNew.vout.resize(1); - txNew.vout[0].scriptPubKey = scriptPubKeyIn; - - // Add dummy coinbase tx as first transaction - pblock->vtx.push_back(CTransaction()); - pblocktemplate->vTxFees.push_back(-1); // updated at end - pblocktemplate->vTxSigOpsCost.push_back(-1); // updated at end - // Block resource limits // If neither -blockmaxsize or -blockmaxcost is given, limit to DEFAULT_BLOCK_MAX_* // If only one is given, only restrict the specified resource. // If both are given, restrict both. - unsigned int nBlockMaxCost = DEFAULT_BLOCK_MAX_COST; - unsigned int nBlockMaxSize = DEFAULT_BLOCK_MAX_SIZE; + nBlockMaxCost = DEFAULT_BLOCK_MAX_COST; + nBlockMaxSize = DEFAULT_BLOCK_MAX_SIZE; bool fCostSet = false; if (mapArgs.count("-blockmaxcost")) { nBlockMaxCost = GetArg("-blockmaxcost", DEFAULT_BLOCK_MAX_COST); @@ -115,225 +98,337 @@ CBlockTemplate* CreateNewBlock(const CChainParams& chainparams, const CScript& s // Limit size to between 1K and MAX_BLOCK_SERIALIZED_SIZE-1K for sanity: nBlockMaxSize = std::max((unsigned int)1000, std::min((unsigned int)(MAX_BLOCK_SERIALIZED_SIZE-1000), nBlockMaxSize)); - // How much of the block should be dedicated to high-priority transactions, - // included regardless of the fees they pay - unsigned int nBlockPrioritySize = GetArg("-blockprioritysize", DEFAULT_BLOCK_PRIORITY_SIZE); - nBlockPrioritySize = std::min(nBlockMaxSize, nBlockPrioritySize); - // Minimum block size you want to create; block will be filled with free transactions // until there are no more or the block reaches this size: - unsigned int nBlockMinSize = GetArg("-blockminsize", DEFAULT_BLOCK_MIN_SIZE); + nBlockMinSize = GetArg("-blockminsize", DEFAULT_BLOCK_MIN_SIZE); nBlockMinSize = std::min(nBlockMaxSize, nBlockMinSize); // Whether we need to account for byte usage (in addition to cost usage) - bool fNeedSizeAccounting = (nBlockMaxSize < MAX_BLOCK_SERIALIZED_SIZE-1000) || (nBlockPrioritySize > 0) || (nBlockMinSize > 0); + fNeedSizeAccounting = (nBlockMaxSize < MAX_BLOCK_SERIALIZED_SIZE-1000) || (nBlockMinSize > 0); +} - // Collect memory pool transactions into the block - CTxMemPool::setEntries inBlock; - CTxMemPool::setEntries waitSet; +void BlockAssembler::resetBlock() +{ + inBlock.clear(); - // This vector will be sorted into a priority queue: - vector vecPriority; - TxCoinAgePriorityCompare pricomparer; - std::map waitPriMap; - typedef std::map::iterator waitPriIter; - double actualPriority = -1; + // Reserve space for coinbase tx + nBlockSize = 1000; + nBlockCost = 4000; + nBlockSigOpsCost = 400; + fIncludeWitness = false; - std::priority_queue, ScoreCompare> clearedTxs; - bool fPrintPriority = GetBoolArg("-printpriority", DEFAULT_PRINTPRIORITY); - uint64_t nBlockSize = 1000; - uint64_t nBlockCost = nBlockSize * WITNESS_SCALE_FACTOR; - uint64_t nBlockTx = 0; - int64_t nBlockSigOpsCost = 400; - int lastFewTxs = 0; - CAmount nFees = 0; + // These counters do not include coinbase tx + nBlockTx = 0; + nFees = 0; + + lastFewTxs = 0; + blockFinished = false; +} +CBlockTemplate* BlockAssembler::CreateNewBlock(const CScript& scriptPubKeyIn) +{ + resetBlock(); + + pblocktemplate.reset(new CBlockTemplate()); + + if(!pblocktemplate.get()) + return NULL; + pblock = &pblocktemplate->block; // pointer for convenience + + // Add dummy coinbase tx as first transaction + pblock->vtx.push_back(CTransaction()); + pblocktemplate->vTxFees.push_back(-1); // updated at end + pblocktemplate->vTxSigOpsCost.push_back(-1); // updated at end + + LOCK2(cs_main, mempool.cs); + CBlockIndex* pindexPrev = chainActive.Tip(); + nHeight = pindexPrev->nHeight + 1; + + pblock->nVersion = ComputeBlockVersion(pindexPrev, chainparams.GetConsensus()); + // -regtest only: allow overriding block.nVersion with + // -blockversion=N to test forking scenarios + if (chainparams.MineBlocksOnDemand()) + pblock->nVersion = GetArg("-blockversion", pblock->nVersion); + + pblock->nTime = GetAdjustedTime(); + const int64_t nMedianTimePast = pindexPrev->GetMedianTimePast(); + + nLockTimeCutoff = (STANDARD_LOCKTIME_VERIFY_FLAGS & LOCKTIME_MEDIAN_TIME_PAST) + ? nMedianTimePast + : pblock->GetBlockTime(); + + // Decide whether to include witness transactions + // This is only needed in case the witness softfork activation is reverted + // (which would require a very deep reorganization) or when + // -promiscuousmempoolflags is used. + // TODO: replace this with a call to main to assess validity of a mempool + // transaction (which in most cases can be a no-op). + fIncludeWitness = IsWitnessEnabled(pindexPrev, chainparams.GetConsensus()); + + addPriorityTxs(); + addScoreTxs(); + + nLastBlockTx = nBlockTx; + nLastBlockSize = nBlockSize; + nLastBlockCost = nBlockCost; + LogPrintf("CreateNewBlock(): total size %u txs: %u fees: %ld sigops %d\n", nBlockSize, nBlockTx, nFees, nBlockSigOpsCost); + + // Create coinbase transaction. + CMutableTransaction coinbaseTx; + coinbaseTx.vin.resize(1); + coinbaseTx.vin[0].prevout.SetNull(); + coinbaseTx.vout.resize(1); + coinbaseTx.vout[0].scriptPubKey = scriptPubKeyIn; + coinbaseTx.vout[0].nValue = nFees + GetBlockSubsidy(nHeight, chainparams.GetConsensus()); + coinbaseTx.vin[0].scriptSig = CScript() << nHeight << OP_0; + pblock->vtx[0] = coinbaseTx; + pblocktemplate->vchCoinbaseCommitment = GenerateCoinbaseCommitment(*pblock, pindexPrev, chainparams.GetConsensus()); + pblocktemplate->vTxFees[0] = -nFees; + + // Fill in header + pblock->hashPrevBlock = pindexPrev->GetBlockHash(); + UpdateTime(pblock, chainparams.GetConsensus(), pindexPrev); + pblock->nBits = GetNextWorkRequired(pindexPrev, pblock, chainparams.GetConsensus()); + pblock->nNonce = 0; + pblocktemplate->vTxSigOpsCost[0] = GetLegacySigOpCount(pblock->vtx[0]); + + CValidationState state; + if (!TestBlockValidity(state, chainparams, *pblock, pindexPrev, false, false)) { + throw std::runtime_error(strprintf("%s: TestBlockValidity failed: %s", __func__, FormatStateMessage(state))); + } + + return pblocktemplate.release(); +} + +bool BlockAssembler::isStillDependent(CTxMemPool::txiter iter) +{ + BOOST_FOREACH(CTxMemPool::txiter parent, mempool.GetMemPoolParents(iter)) { - LOCK2(cs_main, mempool.cs); - CBlockIndex* pindexPrev = chainActive.Tip(); - const int nHeight = pindexPrev->nHeight + 1; - pblock->nTime = GetAdjustedTime(); - const int64_t nMedianTimePast = pindexPrev->GetMedianTimePast(); - - pblock->nVersion = ComputeBlockVersion(pindexPrev, chainparams.GetConsensus()); - // -regtest only: allow overriding block.nVersion with - // -blockversion=N to test forking scenarios - if (chainparams.MineBlocksOnDemand()) - pblock->nVersion = GetArg("-blockversion", pblock->nVersion); - - // Decide whether to include witness transactions - // This is only needed in case the witness softfork activation is reverted - // (which would require a very deep reorganization) or when - // -promiscuousmempoolflags is used. - // TODO: replace this with a call to main to assess validity of a mempool - // transaction (which in most cases can be a no-op). - bool fIncludeWitness = IsWitnessEnabled(pindexPrev, chainparams.GetConsensus()); - - int64_t nLockTimeCutoff = (STANDARD_LOCKTIME_VERIFY_FLAGS & LOCKTIME_MEDIAN_TIME_PAST) - ? nMedianTimePast - : pblock->GetBlockTime(); - - bool fPriorityBlock = nBlockPrioritySize > 0; - if (fPriorityBlock) { - vecPriority.reserve(mempool.mapTx.size()); - for (CTxMemPool::indexed_transaction_set::iterator mi = mempool.mapTx.begin(); - mi != mempool.mapTx.end(); ++mi) - { - double dPriority = mi->GetPriority(nHeight); - CAmount dummy; - mempool.ApplyDeltas(mi->GetTx().GetHash(), dPriority, dummy); - vecPriority.push_back(TxCoinAgePriority(dPriority, mi)); - } - std::make_heap(vecPriority.begin(), vecPriority.end(), pricomparer); + if (!inBlock.count(parent)) { + return true; } + } + return false; +} - CTxMemPool::indexed_transaction_set::index::type::iterator mi = mempool.mapTx.get().begin(); - CTxMemPool::txiter iter; - - while (mi != mempool.mapTx.get().end() || !clearedTxs.empty()) - { - bool priorityTx = false; - if (fPriorityBlock && !vecPriority.empty()) { // add a tx from priority queue to fill the blockprioritysize - priorityTx = true; - iter = vecPriority.front().second; - actualPriority = vecPriority.front().first; - std::pop_heap(vecPriority.begin(), vecPriority.end(), pricomparer); - vecPriority.pop_back(); - } - else if (clearedTxs.empty()) { // add tx with next highest score - iter = mempool.mapTx.project<0>(mi); - mi++; + + +bool BlockAssembler::TestForBlock(CTxMemPool::txiter iter) +{ + if (nBlockCost + iter->GetTxCost() >= nBlockMaxCost) { + // If the block is so close to full that no more txs will fit + // or if we've tried more than 50 times to fill remaining space + // then flag that the block is finished + if (nBlockCost > nBlockMaxCost - 400 || lastFewTxs > 50) { + blockFinished = true; + return false; + } + // Once we're within 4000 cost of a full block, only look at 50 more txs + // to try to fill the remaining space. + if (nBlockCost > nBlockMaxCost - 4000) { + lastFewTxs++; + } + return false; + } + + if (fNeedSizeAccounting) { + if (nBlockSize + ::GetSerializeSize(iter->GetTx(), SER_NETWORK, PROTOCOL_VERSION) >= nBlockMaxSize) { + if (nBlockSize > nBlockMaxSize - 100 || lastFewTxs > 50) { + blockFinished = true; + return false; } - else { // try to add a previously postponed child tx - iter = clearedTxs.top(); - clearedTxs.pop(); + if (nBlockSize > nBlockMaxSize - 1000) { + lastFewTxs++; } + return false; + } + } + + if (nBlockSigOpsCost + iter->GetSigOpCost() >= MAX_BLOCK_SIGOPS_COST) { + // If the block has room for no more sig ops then + // flag that the block is finished + if (nBlockSigOpsCost > MAX_BLOCK_SIGOPS_COST - 8) { + blockFinished = true; + return false; + } + // Otherwise attempt to find another tx with fewer sigops + // to put in the block. + return false; + } - if (inBlock.count(iter)) - continue; // could have been added to the priorityBlock + // Must check that lock times are still valid + // This can be removed once MTP is always enforced + // as long as reorgs keep the mempool consistent. + if (!IsFinalTx(iter->GetTx(), nHeight, nLockTimeCutoff)) + return false; - const CTransaction& tx = iter->GetTx(); + return true; +} + +void BlockAssembler::AddToBlock(CTxMemPool::txiter iter) +{ + pblock->vtx.push_back(iter->GetTx()); + pblocktemplate->vTxFees.push_back(iter->GetFee()); + pblocktemplate->vTxSigOpsCost.push_back(iter->GetSigOpCost()); + if (fNeedSizeAccounting) { + nBlockSize += ::GetSerializeSize(iter->GetTx(), SER_NETWORK, PROTOCOL_VERSION); + } + nBlockCost += iter->GetTxCost(); + ++nBlockTx; + nBlockSigOpsCost += iter->GetSigOpCost(); + nFees += iter->GetFee(); + inBlock.insert(iter); + + bool fPrintPriority = GetBoolArg("-printpriority", DEFAULT_PRINTPRIORITY); + if (fPrintPriority) { + double dPriority = iter->GetPriority(nHeight); + CAmount dummy; + mempool.ApplyDeltas(iter->GetTx().GetHash(), dPriority, dummy); + LogPrintf("priority %.1f fee %s txid %s\n", + dPriority, + CFeeRate(iter->GetModifiedFee(), iter->GetTxSize()).ToString(), + iter->GetTx().GetHash().ToString()); + } +} - if (!fIncludeWitness && !tx.wit.IsNull()) - continue; // cannot accept witness transactions into a non-witness block +void BlockAssembler::addScoreTxs() +{ + std::priority_queue, ScoreCompare> clearedTxs; + CTxMemPool::setEntries waitSet; + CTxMemPool::indexed_transaction_set::index::type::iterator mi = mempool.mapTx.get().begin(); + CTxMemPool::txiter iter; + while (!blockFinished && (mi != mempool.mapTx.get().end() || !clearedTxs.empty())) + { + // If no txs that were previously postponed are available to try + // again, then try the next highest score tx + if (clearedTxs.empty()) { + iter = mempool.mapTx.project<0>(mi); + mi++; + } + // If a previously postponed tx is available to try again, then it + // has higher score than all untried so far txs + else { + iter = clearedTxs.top(); + clearedTxs.pop(); + } + + // If tx already in block, skip (added by addPriorityTxs) + if (inBlock.count(iter)) { + continue; + } - bool fOrphan = false; - BOOST_FOREACH(CTxMemPool::txiter parent, mempool.GetMemPoolParents(iter)) + // cannot accept witness transactions into a non-witness block + if (!fIncludeWitness && !iter->GetTx().wit.IsNull()) + continue; + + // If tx is dependent on other mempool txs which haven't yet been included + // then put it in the waitSet + if (isStillDependent(iter)) { + waitSet.insert(iter); + continue; + } + + // If the fee rate is below the min fee rate for mining, then we're done + // adding txs based on score (fee rate) + if (iter->GetModifiedFee() < ::minRelayTxFee.GetFee(iter->GetTxSize()) && nBlockSize >= nBlockMinSize) { + return; + } + + // If this tx fits in the block add it, otherwise keep looping + if (TestForBlock(iter)) { + AddToBlock(iter); + + // This tx was successfully added, so + // add transactions that depend on this one to the priority queue to try again + BOOST_FOREACH(CTxMemPool::txiter child, mempool.GetMemPoolChildren(iter)) { - if (!inBlock.count(parent)) { - fOrphan = true; - break; + if (waitSet.count(child)) { + clearedTxs.push(child); + waitSet.erase(child); } } - if (fOrphan) { - if (priorityTx) - waitPriMap.insert(std::make_pair(iter,actualPriority)); - else - waitSet.insert(iter); - continue; - } + } + } +} - unsigned int nTxSize = fNeedSizeAccounting ? ::GetSerializeSize(iter->GetTx(), SER_NETWORK, PROTOCOL_VERSION) : 0; - int64_t nTxCost = iter->GetTxCost(); - if (fPriorityBlock && - (nBlockSize + nTxSize >= nBlockPrioritySize || !AllowFree(actualPriority))) { - fPriorityBlock = false; - waitPriMap.clear(); - } - if (!priorityTx && - (iter->GetModifiedFee() < ::minRelayTxFee.GetFee(nTxSize) && nBlockSize >= nBlockMinSize)) { - break; - } - if (nBlockSize + nTxSize >= nBlockMaxSize || nBlockCost + nTxCost > nBlockMaxCost) { - if (nBlockSize > nBlockMaxSize - 100 || nBlockCost > nBlockMaxCost - 400 || lastFewTxs > 50) { - break; - } - // Once we're within 1000 bytes (or 4000 cost) of a full block, - // only look at 50 more txs to try to fill the remaining space. - if (nBlockSize > nBlockMaxSize - 1000 || nBlockCost > nBlockMaxCost - 4000) { - lastFewTxs++; - } - continue; - } +void BlockAssembler::addPriorityTxs() +{ + // How much of the block should be dedicated to high-priority transactions, + // included regardless of the fees they pay + unsigned int nBlockPrioritySize = GetArg("-blockprioritysize", DEFAULT_BLOCK_PRIORITY_SIZE); + nBlockPrioritySize = std::min(nBlockMaxSize, nBlockPrioritySize); - if (!IsFinalTx(tx, nHeight, nLockTimeCutoff)) - continue; + if (nBlockPrioritySize == 0) { + return; + } - int64_t nTxSigOpsCost = iter->GetSigOpCost(); - if (nBlockSigOpsCost + nTxSigOpsCost >= MAX_BLOCK_SIGOPS_COST) { - if (nBlockSigOpsCost > MAX_BLOCK_SIGOPS_COST - 8) { - break; - } - continue; - } + fNeedSizeAccounting = true; - CAmount nTxFees = iter->GetFee(); - // Added - pblock->vtx.push_back(tx); - pblocktemplate->vTxFees.push_back(nTxFees); - pblocktemplate->vTxSigOpsCost.push_back(nTxSigOpsCost); - nBlockSize += nTxSize; - nBlockCost += nTxCost; - ++nBlockTx; - nBlockSigOpsCost += nTxSigOpsCost; - nFees += nTxFees; - - if (fPrintPriority) - { - double dPriority = iter->GetPriority(nHeight); - CAmount dummy; - mempool.ApplyDeltas(tx.GetHash(), dPriority, dummy); - LogPrintf("priority %.1f fee %s txid %s\n", - dPriority , CFeeRate(iter->GetModifiedFee(), nTxSize).ToString(), tx.GetHash().ToString()); - } + // This vector will be sorted into a priority queue: + vector vecPriority; + TxCoinAgePriorityCompare pricomparer; + std::map waitPriMap; + typedef std::map::iterator waitPriIter; + double actualPriority = -1; + + vecPriority.reserve(mempool.mapTx.size()); + for (CTxMemPool::indexed_transaction_set::iterator mi = mempool.mapTx.begin(); + mi != mempool.mapTx.end(); ++mi) + { + double dPriority = mi->GetPriority(nHeight); + CAmount dummy; + mempool.ApplyDeltas(mi->GetTx().GetHash(), dPriority, dummy); + vecPriority.push_back(TxCoinAgePriority(dPriority, mi)); + } + std::make_heap(vecPriority.begin(), vecPriority.end(), pricomparer); + + CTxMemPool::txiter iter; + while (!vecPriority.empty() && !blockFinished) { // add a tx from priority queue to fill the blockprioritysize + iter = vecPriority.front().second; + actualPriority = vecPriority.front().first; + std::pop_heap(vecPriority.begin(), vecPriority.end(), pricomparer); + vecPriority.pop_back(); + + // If tx already in block, skip + if (inBlock.count(iter)) { + assert(false); // shouldn't happen for priority txs + continue; + } - inBlock.insert(iter); + // cannot accept witness transactions into a non-witness block + if (!fIncludeWitness && !iter->GetTx().wit.IsNull()) + continue; + + // If tx is dependent on other mempool txs which haven't yet been included + // then put it in the waitSet + if (isStillDependent(iter)) { + waitPriMap.insert(std::make_pair(iter, actualPriority)); + continue; + } - // Add transactions that depend on this one to the priority queue + // If this tx fits in the block add it, otherwise keep looping + if (TestForBlock(iter)) { + AddToBlock(iter); + + // If now that this txs is added we've surpassed our desired priority size + // or have dropped below the AllowFreeThreshold, then we're done adding priority txs + if (nBlockSize >= nBlockPrioritySize || !AllowFree(actualPriority)) { + return; + } + + // This tx was successfully added, so + // add transactions that depend on this one to the priority queue to try again BOOST_FOREACH(CTxMemPool::txiter child, mempool.GetMemPoolChildren(iter)) { - if (fPriorityBlock) { - waitPriIter wpiter = waitPriMap.find(child); - if (wpiter != waitPriMap.end()) { - vecPriority.push_back(TxCoinAgePriority(wpiter->second,child)); - std::push_heap(vecPriority.begin(), vecPriority.end(), pricomparer); - waitPriMap.erase(wpiter); - } - } - else { - if (waitSet.count(child)) { - clearedTxs.push(child); - waitSet.erase(child); - } + waitPriIter wpiter = waitPriMap.find(child); + if (wpiter != waitPriMap.end()) { + vecPriority.push_back(TxCoinAgePriority(wpiter->second,child)); + std::push_heap(vecPriority.begin(), vecPriority.end(), pricomparer); + waitPriMap.erase(wpiter); } } } - nLastBlockTx = nBlockTx; - nLastBlockSize = nBlockSize; - nLastBlockCost = nBlockCost; - LogPrintf("CreateNewBlock(): total size %u cost %u txs: %u fees: %ld sigops %d\n", nBlockSize, nBlockCost, nBlockTx, nFees, nBlockSigOpsCost); - - // Compute final coinbase transaction. - txNew.vout[0].nValue = nFees + GetBlockSubsidy(nHeight, chainparams.GetConsensus()); - txNew.vin[0].scriptSig = CScript() << nHeight << OP_0; - pblock->vtx[0] = txNew; - pblocktemplate->vchCoinbaseCommitment = GenerateCoinbaseCommitment(*pblock, pindexPrev, chainparams.GetConsensus()); - pblocktemplate->vTxFees[0] = -nFees; - - // Fill in header - pblock->hashPrevBlock = pindexPrev->GetBlockHash(); - UpdateTime(pblock, chainparams.GetConsensus(), pindexPrev); - pblock->nBits = GetNextWorkRequired(pindexPrev, pblock, chainparams.GetConsensus()); - pblock->nNonce = 0; - pblocktemplate->vTxSigOpsCost[0] = GetLegacySigOpCount(pblock->vtx[0]) * WITNESS_SCALE_FACTOR; - - CValidationState state; - if (!TestBlockValidity(state, chainparams, *pblock, pindexPrev, false, false)) { - throw std::runtime_error(strprintf("%s: TestBlockValidity failed: %s", __func__, FormatStateMessage(state))); - } } - - return pblocktemplate.release(); } void IncrementExtraNonce(CBlock* pblock, const CBlockIndex* pindexPrev, unsigned int& nExtraNonce) diff --git a/src/miner.h b/src/miner.h index 525f7ca085724..de3174f0d339e 100644 --- a/src/miner.h +++ b/src/miner.h @@ -7,14 +7,17 @@ #define BITCOIN_MINER_H #include "primitives/block.h" +#include "txmempool.h" #include +#include class CBlockIndex; class CChainParams; class CReserveKey; class CScript; class CWallet; + namespace Consensus { struct Params; }; static const bool DEFAULT_PRINTPRIORITY = false; @@ -28,7 +31,61 @@ struct CBlockTemplate }; /** Generate a new block, without valid proof-of-work */ -CBlockTemplate* CreateNewBlock(const CChainParams& chainparams, const CScript& scriptPubKeyIn); +class BlockAssembler +{ +private: + // The constructed block template + std::unique_ptr pblocktemplate; + // A convenience pointer that always refers to the CBlock in pblocktemplate + CBlock* pblock; + + // Configuration parameters for the block size + bool fIncludeWitness; + unsigned int nBlockMaxCost, nBlockMaxSize, nBlockMinSize; + bool fNeedSizeAccounting; + + // Information on the current status of the block + uint64_t nBlockCost; + uint64_t nBlockSize; + uint64_t nBlockTx; + uint64_t nBlockSigOpsCost; + CAmount nFees; + CTxMemPool::setEntries inBlock; + + // Chain context for the block + int nHeight; + int64_t nLockTimeCutoff; + const CChainParams& chainparams; + + // Variables used for addScoreTxs and addPriorityTxs + int lastFewTxs; + bool blockFinished; + +public: + BlockAssembler(const CChainParams& chainparams); + /** Construct a new block template with coinbase to scriptPubKeyIn */ + CBlockTemplate* CreateNewBlock(const CScript& scriptPubKeyIn); + +private: + // utility functions + /** Clear the block's state and prepare for assembling a new block */ + void resetBlock(); + /** Add a tx to the block */ + void AddToBlock(CTxMemPool::txiter iter); + + // Methods for how to add transactions to a block. + /** Add transactions based on modified feerate */ + void addScoreTxs(); + /** Add transactions based on tx "priority" */ + void addPriorityTxs(); + + // helper function for addScoreTxs and addPriorityTxs + /** Test if tx will still "fit" in the block */ + bool TestForBlock(CTxMemPool::txiter iter); + /** Test if tx still has unconfirmed parents not yet in block */ + bool isStillDependent(CTxMemPool::txiter iter); +}; + /** Modify the extranonce in a block */ void IncrementExtraNonce(CBlock* pblock, const CBlockIndex* pindexPrev, unsigned int& nExtraNonce); int64_t UpdateTime(CBlockHeader* pblock, const Consensus::Params& consensusParams, const CBlockIndex* pindexPrev); diff --git a/src/net.cpp b/src/net.cpp index 57b9099cdade9..cb0f5f7527b12 100644 --- a/src/net.cpp +++ b/src/net.cpp @@ -14,6 +14,7 @@ #include "clientversion.h" #include "consensus/consensus.h" #include "crypto/common.h" +#include "crypto/sha256.h" #include "hash.h" #include "primitives/transaction.h" #include "scheduler.h" @@ -56,7 +57,6 @@ #endif #endif -using namespace std; namespace { const int MAX_OUTBOUND_CONNECTIONS = 8; @@ -72,16 +72,17 @@ namespace { const static std::string NET_MESSAGE_COMMAND_OTHER = "*other*"; /** Services this node implementation cares about */ -static const uint64_t nRelevantServices = NODE_NETWORK | NODE_WITNESS; +static const ServiceFlags nRelevantServices = ServiceFlags(NODE_NETWORK | NODE_WITNESS); // // Global state variables // bool fDiscover = true; bool fListen = true; -uint64_t nLocalServices = NODE_NETWORK | NODE_WITNESS; +ServiceFlags nLocalServices = ServiceFlags(NODE_NETWORK | NODE_WITNESS); +bool fRelayTxes = true; CCriticalSection cs_mapLocalHost; -map mapLocalHost; +std::map mapLocalHost; static bool vfLimited[NET_MAX] = {}; static CNode* pnodeLocalHost = NULL; uint64_t nLocalHostNonce = 0; @@ -91,20 +92,14 @@ int nMaxConnections = DEFAULT_MAX_PEER_CONNECTIONS; bool fAddressesInitialized = false; std::string strSubVersion; -vector vNodes; +std::vector vNodes; CCriticalSection cs_vNodes; -map mapRelay; -deque > vRelayExpiration; -CCriticalSection cs_mapRelay; limitedmap mapAlreadyAskedFor(MAX_INV_SZ); -static deque vOneShots; +static std::deque vOneShots; CCriticalSection cs_vOneShots; -set setservAddNodeAddresses; -CCriticalSection cs_setservAddNodeAddresses; - -vector vAddedNodes; +std::vector vAddedNodes; CCriticalSection cs_vAddedNodes; NodeId nLastNodeId = 0; @@ -138,7 +133,7 @@ bool GetLocal(CService& addr, const CNetAddr *paddrPeer) int nBestReachability = -1; { LOCK(cs_mapLocalHost); - for (map::iterator it = mapLocalHost.begin(); it != mapLocalHost.end(); it++) + for (std::map::iterator it = mapLocalHost.begin(); it != mapLocalHost.end(); it++) { int nScore = (*it).second.nScore; int nReachability = (*it).first.GetReachabilityFrom(paddrPeer); @@ -167,7 +162,7 @@ static std::vector convertSeed6(const std::vector &vSeedsIn { struct in6_addr ip; memcpy(&ip, i->addr, sizeof(ip)); - CAddress addr(CService(ip, i->port)); + CAddress addr(CService(ip, i->port), NODE_NETWORK); addr.nTime = GetTime() - GetRand(nOneWeek) - nOneWeek; vSeedsOut.push_back(addr); } @@ -180,13 +175,12 @@ static std::vector convertSeed6(const std::vector &vSeedsIn // one by discovery. CAddress GetLocalAddress(const CNetAddr *paddrPeer) { - CAddress ret(CService("0.0.0.0",GetListenPort()),0); + CAddress ret(CService("0.0.0.0",GetListenPort()), NODE_NONE); CService addr; if (GetLocal(addr, paddrPeer)) { - ret = CAddress(addr); + ret = CAddress(addr, nLocalServices); } - ret.nServices = nLocalServices; ret.nTime = GetAdjustedTime(); return ret; } @@ -374,7 +368,7 @@ CNode* FindNode(const CService& addr) return NULL; } -CNode* ConnectNode(CAddress addrConnect, const char *pszDest) +CNode* ConnectNode(CAddress addrConnect, const char *pszDest, bool fCountFailure) { if (pszDest == NULL) { if (IsLocal(addrConnect)) @@ -406,7 +400,7 @@ CNode* ConnectNode(CAddress addrConnect, const char *pszDest) return NULL; } - addrman.Attempt(addrConnect); + addrman.Attempt(addrConnect, fCountFailure); // Add node CNode* pnode = new CNode(hSocket, addrConnect, pszDest ? pszDest : "", false); @@ -417,19 +411,39 @@ CNode* ConnectNode(CAddress addrConnect, const char *pszDest) vNodes.push_back(pnode); } - pnode->nServicesExpected = addrConnect.nServices & nRelevantServices; + pnode->nServicesExpected = ServiceFlags(addrConnect.nServices & nRelevantServices); pnode->nTimeConnected = GetTime(); return pnode; } else if (!proxyConnectionFailed) { // If connecting to the node failed, and failure is not caused by a problem connecting to // the proxy, mark this as an attempt. - addrman.Attempt(addrConnect); + addrman.Attempt(addrConnect, fCountFailure); } return NULL; } +static void DumpBanlist() +{ + CNode::SweepBanned(); // clean unused entries (if bantime has expired) + + if (!CNode::BannedSetIsDirty()) + return; + + int64_t nStart = GetTimeMillis(); + + CBanDB bandb; + banmap_t banmap; + CNode::SetBannedSetDirty(false); + CNode::GetBanned(banmap); + if (!bandb.Write(banmap)) + CNode::SetBannedSetDirty(true); + + LogPrint("net", "Flushed %d banned node ips/subnets to banlist.dat %dms\n", + banmap.size(), GetTimeMillis() - nStart); +} + void CNode::CloseSocketDisconnect() { fDisconnect = true; @@ -447,18 +461,18 @@ void CNode::CloseSocketDisconnect() void CNode::PushVersion() { - int nBestHeight = g_signals.GetHeight().get_value_or(0); + int nBestHeight = GetNodeSignals().GetHeight().get_value_or(0); int64_t nTime = (fInbound ? GetAdjustedTime() : GetTime()); - CAddress addrYou = (addr.IsRoutable() && !IsProxy(addr) ? addr : CAddress(CService("0.0.0.0",0))); + CAddress addrYou = (addr.IsRoutable() && !IsProxy(addr) ? addr : CAddress(CService("0.0.0.0", 0), addr.nServices)); CAddress addrMe = GetLocalAddress(&addr); GetRandBytes((unsigned char*)&nLocalHostNonce, sizeof(nLocalHostNonce)); if (fLogIPs) LogPrint("net", "send version message: version %d, blocks=%d, us=%s, them=%s, peer=%d\n", PROTOCOL_VERSION, nBestHeight, addrMe.ToString(), addrYou.ToString(), id); else LogPrint("net", "send version message: version %d, blocks=%d, us=%s, peer=%d\n", PROTOCOL_VERSION, nBestHeight, addrMe.ToString(), id); - PushMessage(NetMsgType::VERSION, PROTOCOL_VERSION, nLocalServices, nTime, addrYou, addrMe, - nLocalHostNonce, strSubVersion, nBestHeight, !GetBoolArg("-blocksonly", DEFAULT_BLOCKSONLY)); + PushMessage(NetMsgType::VERSION, PROTOCOL_VERSION, (uint64_t)nLocalServices, nTime, addrYou, addrMe, + nLocalHostNonce, strSubVersion, nBestHeight, ::fRelayTxes); } @@ -471,9 +485,13 @@ bool CNode::setBannedIsDirty; void CNode::ClearBanned() { - LOCK(cs_setBanned); - setBanned.clear(); - setBannedIsDirty = true; + { + LOCK(cs_setBanned); + setBanned.clear(); + setBannedIsDirty = true; + } + DumpBanlist(); //store banlist to disk + uiInterface.BannedListChanged(); } bool CNode::IsBanned(CNetAddr ip) @@ -524,11 +542,25 @@ void CNode::Ban(const CSubNet& subNet, const BanReason &banReason, int64_t banti } banEntry.nBanUntil = (sinceUnixEpoch ? 0 : GetTime() )+bantimeoffset; - LOCK(cs_setBanned); - if (setBanned[subNet].nBanUntil < banEntry.nBanUntil) - setBanned[subNet] = banEntry; - - setBannedIsDirty = true; + { + LOCK(cs_setBanned); + if (setBanned[subNet].nBanUntil < banEntry.nBanUntil) { + setBanned[subNet] = banEntry; + setBannedIsDirty = true; + } + else + return; + } + uiInterface.BannedListChanged(); + { + LOCK(cs_vNodes); + BOOST_FOREACH(CNode* pnode, vNodes) { + if (subNet.Match((CNetAddr)pnode->addr)) + pnode->fDisconnect = true; + } + } + if(banReason == BanReasonManuallyAdded) + DumpBanlist(); //store banlist to disk immediately if user requested ban } bool CNode::Unban(const CNetAddr &addr) { @@ -537,13 +569,15 @@ bool CNode::Unban(const CNetAddr &addr) { } bool CNode::Unban(const CSubNet &subNet) { - LOCK(cs_setBanned); - if (setBanned.erase(subNet)) { + LOCK(cs_setBanned); + if (!setBanned.erase(subNet)) + return false; setBannedIsDirty = true; - return true; } - return false; + uiInterface.BannedListChanged(); + DumpBanlist(); //store banlist to disk immediately + return true; } void CNode::GetBanned(banmap_t &banMap) @@ -800,89 +834,41 @@ void SocketSendData(CNode *pnode) pnode->vSendMsg.erase(pnode->vSendMsg.begin(), it); } -static list vNodesDisconnected; +static std::list vNodesDisconnected; -class CNodeRef { -public: - CNodeRef(CNode *pnode) : _pnode(pnode) { - LOCK(cs_vNodes); - _pnode->AddRef(); - } - - ~CNodeRef() { - LOCK(cs_vNodes); - _pnode->Release(); - } - - CNode& operator *() const {return *_pnode;}; - CNode* operator ->() const {return _pnode;}; - - CNodeRef& operator =(const CNodeRef& other) - { - if (this != &other) { - LOCK(cs_vNodes); - - _pnode->Release(); - _pnode = other._pnode; - _pnode->AddRef(); - } - return *this; - } - - CNodeRef(const CNodeRef& other): - _pnode(other._pnode) - { - LOCK(cs_vNodes); - _pnode->AddRef(); - } -private: - CNode *_pnode; +struct NodeEvictionCandidate +{ + NodeId id; + int64_t nTimeConnected; + int64_t nMinPingUsecTime; + CAddress addr; + uint64_t nKeyedNetGroup; }; -static bool ReverseCompareNodeMinPingTime(const CNodeRef &a, const CNodeRef &b) +static bool ReverseCompareNodeMinPingTime(const NodeEvictionCandidate &a, const NodeEvictionCandidate &b) { - return a->nMinPingUsecTime > b->nMinPingUsecTime; + return a.nMinPingUsecTime > b.nMinPingUsecTime; } -static bool ReverseCompareNodeTimeConnected(const CNodeRef &a, const CNodeRef &b) +static bool ReverseCompareNodeTimeConnected(const NodeEvictionCandidate &a, const NodeEvictionCandidate &b) { - return a->nTimeConnected > b->nTimeConnected; + return a.nTimeConnected > b.nTimeConnected; } -class CompareNetGroupKeyed -{ - std::vector vchSecretKey; -public: - CompareNetGroupKeyed() - { - vchSecretKey.resize(32, 0); - GetRandBytes(vchSecretKey.data(), vchSecretKey.size()); - } - - bool operator()(const CNodeRef &a, const CNodeRef &b) - { - std::vector vchGroupA, vchGroupB; - CSHA256 hashA, hashB; - std::vector vchA(32), vchB(32); - - vchGroupA = a->addr.GetGroup(); - vchGroupB = b->addr.GetGroup(); - - hashA.Write(begin_ptr(vchGroupA), vchGroupA.size()); - hashB.Write(begin_ptr(vchGroupB), vchGroupB.size()); - - hashA.Write(begin_ptr(vchSecretKey), vchSecretKey.size()); - hashB.Write(begin_ptr(vchSecretKey), vchSecretKey.size()); - - hashA.Finalize(begin_ptr(vchA)); - hashB.Finalize(begin_ptr(vchB)); - - return vchA < vchB; - } +static bool CompareNetGroupKeyed(const NodeEvictionCandidate &a, const NodeEvictionCandidate &b) { + return a.nKeyedNetGroup < b.nKeyedNetGroup; }; +/** Try to find a connection to evict when the node is full. + * Extreme care must be taken to avoid opening the node to attacker + * triggered network partitioning. + * The strategy used here is to protect a small number of peers + * for each of several distinct characteristics which are difficult + * to forge. In order to partition a node the attacker must be + * simultaneously better at all of them than honest peers. + */ static bool AttemptToEvictConnection(bool fPreferNewConnection) { - std::vector vEvictionCandidates; + std::vector vEvictionCandidates; { LOCK(cs_vNodes); @@ -893,7 +879,8 @@ static bool AttemptToEvictConnection(bool fPreferNewConnection) { continue; if (node->fDisconnect) continue; - vEvictionCandidates.push_back(CNodeRef(node)); + NodeEvictionCandidate candidate = {node->id, node->nTimeConnected, node->nMinPingUsecTime, node->addr, node->nKeyedNetGroup}; + vEvictionCandidates.push_back(candidate); } } @@ -902,14 +889,13 @@ static bool AttemptToEvictConnection(bool fPreferNewConnection) { // Protect connections with certain characteristics // Deterministically select 4 peers to protect by netgroup. - // An attacker cannot predict which netgroups will be protected. - static CompareNetGroupKeyed comparerNetGroupKeyed; - std::sort(vEvictionCandidates.begin(), vEvictionCandidates.end(), comparerNetGroupKeyed); + // An attacker cannot predict which netgroups will be protected + std::sort(vEvictionCandidates.begin(), vEvictionCandidates.end(), CompareNetGroupKeyed); vEvictionCandidates.erase(vEvictionCandidates.end() - std::min(4, static_cast(vEvictionCandidates.size())), vEvictionCandidates.end()); if (vEvictionCandidates.empty()) return false; - // Protect the 8 nodes with the best ping times. + // Protect the 8 nodes with the lowest minimum ping time. // An attacker cannot manipulate this metric without physically moving nodes closer to the target. std::sort(vEvictionCandidates.begin(), vEvictionCandidates.end(), ReverseCompareNodeMinPingTime); vEvictionCandidates.erase(vEvictionCandidates.end() - std::min(8, static_cast(vEvictionCandidates.size())), vEvictionCandidates.end()); @@ -917,7 +903,7 @@ static bool AttemptToEvictConnection(bool fPreferNewConnection) { if (vEvictionCandidates.empty()) return false; // Protect the half of the remaining nodes which have been connected the longest. - // This replicates the existing implicit behavior. + // This replicates the non-eviction implicit behavior, and precludes attacks that start later. std::sort(vEvictionCandidates.begin(), vEvictionCandidates.end(), ReverseCompareNodeTimeConnected); vEvictionCandidates.erase(vEvictionCandidates.end() - static_cast(vEvictionCandidates.size() / 2), vEvictionCandidates.end()); @@ -925,35 +911,42 @@ static bool AttemptToEvictConnection(bool fPreferNewConnection) { // Identify the network group with the most connections and youngest member. // (vEvictionCandidates is already sorted by reverse connect time) - std::vector naMostConnections; + uint64_t naMostConnections; unsigned int nMostConnections = 0; int64_t nMostConnectionsTime = 0; - std::map, std::vector > mapAddrCounts; - BOOST_FOREACH(const CNodeRef &node, vEvictionCandidates) { - mapAddrCounts[node->addr.GetGroup()].push_back(node); - int64_t grouptime = mapAddrCounts[node->addr.GetGroup()][0]->nTimeConnected; - size_t groupsize = mapAddrCounts[node->addr.GetGroup()].size(); + std::map > mapAddrCounts; + BOOST_FOREACH(const NodeEvictionCandidate &node, vEvictionCandidates) { + mapAddrCounts[node.nKeyedNetGroup].push_back(node); + int64_t grouptime = mapAddrCounts[node.nKeyedNetGroup][0].nTimeConnected; + size_t groupsize = mapAddrCounts[node.nKeyedNetGroup].size(); if (groupsize > nMostConnections || (groupsize == nMostConnections && grouptime > nMostConnectionsTime)) { nMostConnections = groupsize; nMostConnectionsTime = grouptime; - naMostConnections = node->addr.GetGroup(); + naMostConnections = node.nKeyedNetGroup; } } // Reduce to the network group with the most connections - vEvictionCandidates = mapAddrCounts[naMostConnections]; + vEvictionCandidates = std::move(mapAddrCounts[naMostConnections]); // Do not disconnect peers if there is only one unprotected connection from their network group. + // This step excessively favors netgroup diversity, and should be removed once more protective criteria are established. if (vEvictionCandidates.size() <= 1) // unless we prefer the new connection (for whitelisted peers) if (!fPreferNewConnection) return false; // Disconnect from the network group with the most connections - vEvictionCandidates[0]->fDisconnect = true; - - return true; + NodeId evicted = vEvictionCandidates.front().id; + LOCK(cs_vNodes); + for(std::vector::const_iterator it(vNodes.begin()); it != vNodes.end(); ++it) { + if ((*it)->GetId() == evicted) { + (*it)->fDisconnect = true; + return true; + } + } + return false; } static void AcceptConnection(const ListenSocket& hListenSocket) { @@ -1040,7 +1033,7 @@ void ThreadSocketHandler() { LOCK(cs_vNodes); // Disconnect unused nodes - vector vNodesCopy = vNodes; + std::vector vNodesCopy = vNodes; BOOST_FOREACH(CNode* pnode, vNodesCopy) { if (pnode->fDisconnect || @@ -1064,7 +1057,7 @@ void ThreadSocketHandler() } { // Delete disconnected nodes - list vNodesDisconnectedCopy = vNodesDisconnected; + std::list vNodesDisconnectedCopy = vNodesDisconnected; BOOST_FOREACH(CNode* pnode, vNodesDisconnectedCopy) { // wait until threads are done using it @@ -1115,7 +1108,7 @@ void ThreadSocketHandler() BOOST_FOREACH(const ListenSocket& hListenSocket, vhListenSocket) { FD_SET(hListenSocket.socket, &fdsetRecv); - hSocketMax = max(hSocketMax, hListenSocket.socket); + hSocketMax = std::max(hSocketMax, hListenSocket.socket); have_fds = true; } @@ -1126,7 +1119,7 @@ void ThreadSocketHandler() if (pnode->hSocket == INVALID_SOCKET) continue; FD_SET(pnode->hSocket, &fdsetError); - hSocketMax = max(hSocketMax, pnode->hSocket); + hSocketMax = std::max(hSocketMax, pnode->hSocket); have_fds = true; // Implement the following logic: @@ -1193,7 +1186,7 @@ void ThreadSocketHandler() // // Service each socket // - vector vNodesCopy; + std::vector vNodesCopy; { LOCK(cs_vNodes); vNodesCopy = vNodes; @@ -1350,7 +1343,7 @@ void ThreadMapPort() } } - string strDesc = "Bitcoin " + FormatFullVersion(); + std::string strDesc = "Bitcoin " + FormatFullVersion(); try { while (true) { @@ -1436,7 +1429,7 @@ void ThreadDNSAddressSeed() } } - const vector &vSeeds = Params().DNSSeeds(); + const std::vector &vSeeds = Params().DNSSeeds(); int found = 0; LogPrintf("Loading addresses from DNS seeds (could take a while)\n"); @@ -1445,20 +1438,29 @@ void ThreadDNSAddressSeed() if (HaveNameProxy()) { AddOneShot(seed.host); } else { - vector vIPs; - vector vAdd; - if (LookupHost(seed.host.c_str(), vIPs)) + std::vector vIPs; + std::vector vAdd; + ServiceFlags requiredServiceBits = nRelevantServices; + if (LookupHost(seed.getHost(requiredServiceBits).c_str(), vIPs, 0, true)) { BOOST_FOREACH(const CNetAddr& ip, vIPs) { int nOneDay = 24*3600; - CAddress addr = CAddress(CService(ip, Params().GetDefaultPort())); + CAddress addr = CAddress(CService(ip, Params().GetDefaultPort()), requiredServiceBits); addr.nTime = GetTime() - 3*nOneDay - GetRand(4*nOneDay); // use a random age between 3 and 7 days old vAdd.push_back(addr); found++; } } - addrman.Add(vAdd, CNetAddr(seed.name, true)); + // TODO: The seed name resolve may fail, yielding an IP of [::], which results in + // addrman assigning the same source to results from different seeds. + // This should switch to a hard-coded stable dummy IP for each seed name, so that the + // resolve is not required at all. + if (!vIPs.empty()) { + CService seedSource; + Lookup(seed.name.c_str(), seedSource, 0, true); + addrman.Add(vAdd, seedSource); + } } } @@ -1495,7 +1497,7 @@ void DumpData() void static ProcessOneShot() { - string strDest; + std::string strDest; { LOCK(cs_vOneShots); if (vOneShots.empty()) @@ -1506,7 +1508,7 @@ void static ProcessOneShot() CAddress addr; CSemaphoreGrant grant(*semOutbound, true); if (grant) { - if (!OpenNetworkConnection(addr, &grant, strDest.c_str(), true)) + if (!OpenNetworkConnection(addr, false, &grant, strDest.c_str(), true)) AddOneShot(strDest); } } @@ -1521,8 +1523,8 @@ void ThreadOpenConnections() ProcessOneShot(); BOOST_FOREACH(const std::string& strAddr, mapMultiArgs["-connect"]) { - CAddress addr; - OpenNetworkConnection(addr, NULL, strAddr.c_str()); + CAddress addr(CService(), NODE_NONE); + OpenNetworkConnection(addr, false, NULL, strAddr.c_str()); for (int i = 0; i < 10 && i < nLoop; i++) { MilliSleep(500); @@ -1561,7 +1563,7 @@ void ThreadOpenConnections() // Only connect out to one peer per network group (/16 for IPv4). // Do this here so we don't have to critsect vNodes inside mapAddresses critsect. int nOutbound = 0; - set > setConnected; + std::set > setConnected; { LOCK(cs_vNodes); BOOST_FOREACH(CNode* pnode, vNodes) { @@ -1594,7 +1596,7 @@ void ThreadOpenConnections() continue; // only connect to full nodes - if (!(addr.nServices & NODE_NETWORK)) + if ((addr.nServices & REQUIRED_SERVICES) != REQUIRED_SERVICES) continue; // only consider very recently tried nodes after 30 failed attempts @@ -1614,7 +1616,7 @@ void ThreadOpenConnections() } if (addrConnect.IsValid()) - OpenNetworkConnection(addrConnect, &grant); + OpenNetworkConnection(addrConnect, (int)setConnected.size() >= std::min(nMaxConnections - 1, 2), &grant); } } @@ -1627,7 +1629,7 @@ void ThreadOpenAddedConnections() if (HaveNameProxy()) { while(true) { - list lAddresses(0); + std::list lAddresses(0); { LOCK(cs_vAddedNodes); BOOST_FOREACH(const std::string& strAddNode, vAddedNodes) @@ -1636,7 +1638,7 @@ void ThreadOpenAddedConnections() BOOST_FOREACH(const std::string& strAddNode, lAddresses) { CAddress addr; CSemaphoreGrant grant(*semOutbound); - OpenNetworkConnection(addr, &grant, strAddNode.c_str()); + OpenNetworkConnection(addr, false, &grant, strAddNode.c_str()); MilliSleep(500); } MilliSleep(120000); // Retry every 2 minutes @@ -1645,32 +1647,25 @@ void ThreadOpenAddedConnections() for (unsigned int i = 0; true; i++) { - list lAddresses(0); + std::list lAddresses(0); { LOCK(cs_vAddedNodes); BOOST_FOREACH(const std::string& strAddNode, vAddedNodes) lAddresses.push_back(strAddNode); } - list > lservAddressesToAdd(0); + std::list > lservAddressesToAdd(0); BOOST_FOREACH(const std::string& strAddNode, lAddresses) { - vector vservNode(0); + std::vector vservNode(0); if(Lookup(strAddNode.c_str(), vservNode, Params().GetDefaultPort(), fNameLookup, 0)) - { lservAddressesToAdd.push_back(vservNode); - { - LOCK(cs_setservAddNodeAddresses); - BOOST_FOREACH(const CService& serv, vservNode) - setservAddNodeAddresses.insert(serv); - } - } } // Attempt to connect to each IP for each addnode entry until at least one is successful per addnode entry // (keeping in mind that addnode entries can have many IPs if fNameLookup) { LOCK(cs_vNodes); BOOST_FOREACH(CNode* pnode, vNodes) - for (list >::iterator it = lservAddressesToAdd.begin(); it != lservAddressesToAdd.end(); it++) + for (std::list >::iterator it = lservAddressesToAdd.begin(); it != lservAddressesToAdd.end(); it++) BOOST_FOREACH(const CService& addrNode, *(it)) if (pnode->addr == addrNode) { @@ -1679,10 +1674,12 @@ void ThreadOpenAddedConnections() break; } } - BOOST_FOREACH(vector& vserv, lservAddressesToAdd) + BOOST_FOREACH(std::vector& vserv, lservAddressesToAdd) { CSemaphoreGrant grant(*semOutbound); - OpenNetworkConnection(CAddress(vserv[i % vserv.size()]), &grant); + /* We want -addnode to work even for nodes that don't provide all + * wanted services, so pass in nServices=NODE_NONE to CAddress. */ + OpenNetworkConnection(CAddress(vserv[i % vserv.size()], NODE_NONE), false, &grant); MilliSleep(500); } MilliSleep(120000); // Retry every 2 minutes @@ -1690,7 +1687,7 @@ void ThreadOpenAddedConnections() } // if successful, this moves the passed grant to the constructed node -bool OpenNetworkConnection(const CAddress& addrConnect, CSemaphoreGrant *grantOutbound, const char *pszDest, bool fOneShot) +bool OpenNetworkConnection(const CAddress& addrConnect, bool fCountFailure, CSemaphoreGrant *grantOutbound, const char *pszDest, bool fOneShot) { // // Initiate outbound network connection @@ -1704,7 +1701,7 @@ bool OpenNetworkConnection(const CAddress& addrConnect, CSemaphoreGrant *grantOu } else if (FindNode(std::string(pszDest))) return false; - CNode* pnode = ConnectNode(addrConnect, pszDest); + CNode* pnode = ConnectNode(addrConnect, pszDest, fCountFailure); boost::this_thread::interruption_point(); if (!pnode) @@ -1724,10 +1721,9 @@ void ThreadMessageHandler() boost::mutex condition_mutex; boost::unique_lock lock(condition_mutex); - SetThreadPriority(THREAD_PRIORITY_BELOW_NORMAL); while (true) { - vector vNodesCopy; + std::vector vNodesCopy; { LOCK(cs_vNodes); vNodesCopy = vNodes; @@ -1748,7 +1744,7 @@ void ThreadMessageHandler() TRY_LOCK(pnode->cs_vRecvMsg, lockRecv); if (lockRecv) { - if (!g_signals.ProcessMessages(pnode)) + if (!GetNodeSignals().ProcessMessages(pnode)) pnode->CloseSocketDisconnect(); if (pnode->nSendSize < SendBufferSize()) @@ -1766,7 +1762,7 @@ void ThreadMessageHandler() { TRY_LOCK(pnode->cs_vSend, lockSend); if (lockSend) - g_signals.SendMessages(pnode); + GetNodeSignals().SendMessages(pnode); } boost::this_thread::interruption_point(); } @@ -1787,7 +1783,7 @@ void ThreadMessageHandler() -bool BindListenPort(const CService &addrBind, string& strError, bool fWhitelisted) +bool BindListenPort(const CService &addrBind, std::string& strError, bool fWhitelisted) { strError = ""; int nOne = 1; @@ -1895,8 +1891,8 @@ void static Discover(boost::thread_group& threadGroup) char pszHostName[256] = ""; if (gethostname(pszHostName, sizeof(pszHostName)) != SOCKET_ERROR) { - vector vaddr; - if (LookupHost(pszHostName, vaddr)) + std::vector vaddr; + if (LookupHost(pszHostName, vaddr, 0, true)) { BOOST_FOREACH (const CNetAddr &addr, vaddr) { @@ -1946,6 +1942,7 @@ void StartNode(boost::thread_group& threadGroup, CScheduler& scheduler) if (adb.Read(addrman)) LogPrintf("Loaded %i addresses from peers.dat %dms\n", addrman.size(), GetTimeMillis() - nStart); else { + addrman.Clear(); // Addrman can be in an inconsistent state after failure, reset it LogPrintf("Invalid or missing peers.dat; recreating\n"); DumpAddresses(); } @@ -2065,38 +2062,13 @@ class CNetCleanup instance_of_cnetcleanup; -void RelayTransaction(const CTransaction& tx, CFeeRate feerate) +void RelayTransaction(const CTransaction& tx) { CInv inv(MSG_TX, tx.GetHash()); - { - LOCK(cs_mapRelay); - // Expire old relay messages - while (!vRelayExpiration.empty() && vRelayExpiration.front().first < GetTime()) - { - mapRelay.erase(vRelayExpiration.front().second); - vRelayExpiration.pop_front(); - } - - mapRelay.insert(std::make_pair(inv.hash, tx)); - vRelayExpiration.push_back(std::make_pair(GetTime() + 15 * 60, inv.hash)); - } LOCK(cs_vNodes); BOOST_FOREACH(CNode* pnode, vNodes) { - if(!pnode->fRelayTxes) - continue; - { - LOCK(pnode->cs_feeFilter); - if (feerate.GetFeePerK() < pnode->minFeeFilter) - continue; - } - LOCK(pnode->cs_filter); - if (pnode->pfilter) - { - if (pnode->pfilter->IsRelevantAndUpdate(tx)) - pnode->PushInventory(inv); - } else - pnode->PushInventory(inv); + pnode->PushInventory(inv); } } @@ -2308,7 +2280,7 @@ bool CAddrDB::Read(CAddrMan& addr) // Don't try to resize to a negative number if file is small if (fileSize >= sizeof(uint256)) dataSize = fileSize - sizeof(uint256); - vector vchData; + std::vector vchData; vchData.resize(dataSize); uint256 hashIn; @@ -2329,6 +2301,11 @@ bool CAddrDB::Read(CAddrMan& addr) if (hashIn != hashTmp) return error("%s: Checksum mismatch, data corrupted", __func__); + return Read(addr, ssPeers); +} + +bool CAddrDB::Read(CAddrMan& addr, CDataStream& ssPeers) +{ unsigned char pchMsgTmp[4]; try { // de-serialize file header (network specific magic number) and .. @@ -2342,6 +2319,8 @@ bool CAddrDB::Read(CAddrMan& addr) ssPeers >> addr; } catch (const std::exception& e) { + // de-serialization has failed, ensure addrman is left in a clean state + addr.Clear(); return error("%s: Deserialize or I/O error - %s", __func__, e.what()); } @@ -2353,11 +2332,13 @@ unsigned int SendBufferSize() { return 1000*GetArg("-maxsendbuffer", DEFAULT_MAX CNode::CNode(SOCKET hSocketIn, const CAddress& addrIn, const std::string& addrNameIn, bool fInboundIn) : ssSend(SER_NETWORK, INIT_PROTO_VERSION), + addr(addrIn), + nKeyedNetGroup(CalculateKeyedNetGroup(addrIn)), addrKnown(5000, 0.001), filterInventoryKnown(50000, 0.000001) { - nServices = 0; - nServicesExpected = 0; + nServices = NODE_NONE; + nServicesExpected = NODE_NONE; hSocket = hSocketIn; nRecvVersion = INIT_PROTO_VERSION; nLastSend = 0; @@ -2366,7 +2347,6 @@ CNode::CNode(SOCKET hSocketIn, const CAddress& addrIn, const std::string& addrNa nRecvBytes = 0; nTimeConnected = GetTime(); nTimeOffset = 0; - addr = addrIn; addrName = addrNameIn == "" ? addr.ToStringIPPort() : addrNameIn; nVersion = 0; strSubVer = ""; @@ -2383,6 +2363,7 @@ CNode::CNode(SOCKET hSocketIn, const CAddress& addrIn, const std::string& addrNa hashContinue = uint256(); nStartingHeight = -1; filterInventoryKnown.reset(); + fSendMempool = false; fGetAddr = false; nNextLocalAddrSend = 0; nNextAddrSend = 0; @@ -2390,6 +2371,7 @@ CNode::CNode(SOCKET hSocketIn, const CAddress& addrIn, const std::string& addrNa fRelayTxes = false; fSentAddr = false; pfilter = new CBloomFilter(); + timeLastMempoolReq = 0; nPingNonceSent = 0; nPingUsecStart = 0; nPingUsecTime = 0; @@ -2588,7 +2570,7 @@ bool CBanDB::Read(banmap_t& banSet) // Don't try to resize to a negative number if file is small if (fileSize >= sizeof(uint256)) dataSize = fileSize - sizeof(uint256); - vector vchData; + std::vector vchData; vchData.resize(dataSize); uint256 hashIn; @@ -2628,25 +2610,16 @@ bool CBanDB::Read(banmap_t& banSet) return true; } -void DumpBanlist() -{ - CNode::SweepBanned(); // clean unused entries (if bantime has expired) - - if (!CNode::BannedSetIsDirty()) - return; - - int64_t nStart = GetTimeMillis(); +int64_t PoissonNextSend(int64_t nNow, int average_interval_seconds) { + return nNow + (int64_t)(log1p(GetRand(1ULL << 48) * -0.0000000000000035527136788 /* -1/2^48 */) * average_interval_seconds * -1000000.0 + 0.5); +} - CBanDB bandb; - banmap_t banmap; - CNode::GetBanned(banmap); - if (bandb.Write(banmap)) - CNode::SetBannedSetDirty(false); +/* static */ uint64_t CNode::CalculateKeyedNetGroup(const CAddress& ad) +{ + static const uint64_t k0 = GetRand(std::numeric_limits::max()); + static const uint64_t k1 = GetRand(std::numeric_limits::max()); - LogPrint("net", "Flushed %d banned node ips/subnets to banlist.dat %dms\n", - banmap.size(), GetTimeMillis() - nStart); -} + std::vector vchNetGroup(ad.GetGroup()); -int64_t PoissonNextSend(int64_t nNow, int average_interval_seconds) { - return nNow + (int64_t)(log1p(GetRand(1ULL << 48) * -0.0000000000000035527136788 /* -1/2^48 */) * average_interval_seconds * -1000000.0 + 0.5); + return CSipHasher(k0, k1).Write(&vchNetGroup[0], vchNetGroup.size()).Finalize(); } diff --git a/src/net.h b/src/net.h index a577ce42104e3..60c8be6f751e2 100644 --- a/src/net.h +++ b/src/net.h @@ -17,6 +17,7 @@ #include "sync.h" #include "uint256.h" +#include #include #include @@ -71,6 +72,8 @@ static const bool DEFAULT_FORCEDNSSEED = false; static const size_t DEFAULT_MAXRECEIVEBUFFER = 5 * 1000; static const size_t DEFAULT_MAXSENDBUFFER = 1 * 1000; +static const ServiceFlags REQUIRED_SERVICES = NODE_NETWORK; + // NOTE: When adjusting this, update rpcnet:setban's help ("24h") static const unsigned int DEFAULT_MISBEHAVING_BANTIME = 60 * 60 * 24; // Default 24-hour ban @@ -83,8 +86,7 @@ CNode* FindNode(const CNetAddr& ip); CNode* FindNode(const CSubNet& subNet); CNode* FindNode(const std::string& addrName); CNode* FindNode(const CService& ip); -CNode* ConnectNode(CAddress addrConnect, const char *pszDest = NULL); -bool OpenNetworkConnection(const CAddress& addrConnect, CSemaphoreGrant *grantOutbound = NULL, const char *strDest = NULL, bool fOneShot = false); +bool OpenNetworkConnection(const CAddress& addrConnect, bool fCountFailure, CSemaphoreGrant *grantOutbound = NULL, const char *strDest = NULL, bool fOneShot = false); void MapPort(bool fUseUPnP); unsigned short GetListenPort(); bool BindListenPort(const CService &bindAddr, std::string& strError, bool fWhitelisted = false); @@ -152,7 +154,8 @@ CAddress GetLocalAddress(const CNetAddr *paddrPeer = NULL); extern bool fDiscover; extern bool fListen; -extern uint64_t nLocalServices; +extern ServiceFlags nLocalServices; +extern bool fRelayTxes; extern uint64_t nLocalHostNonce; extern CAddrMan addrman; @@ -161,9 +164,6 @@ extern int nMaxConnections; extern std::vector vNodes; extern CCriticalSection cs_vNodes; -extern std::map mapRelay; -extern std::deque > vRelayExpiration; -extern CCriticalSection cs_mapRelay; extern limitedmap mapAlreadyAskedFor; extern std::vector vAddedNodes; @@ -188,7 +188,7 @@ class CNodeStats { public: NodeId nodeid; - uint64_t nServices; + ServiceFlags nServices; bool fRelayTxes; int64_t nLastSend; int64_t nLastRecv; @@ -318,8 +318,8 @@ class CNode { public: // socket - uint64_t nServices; - uint64_t nServicesExpected; + ServiceFlags nServices; + ServiceFlags nServicesExpected; SOCKET hSocket; CDataStream ssSend; size_t nSendSize; // total size of all vSendMsg entries @@ -338,7 +338,7 @@ class CNode int64_t nLastRecv; int64_t nTimeConnected; int64_t nTimeOffset; - CAddress addr; + const CAddress addr; std::string addrName; CService addrLocal; int nVersion; @@ -358,13 +358,15 @@ class CNode // a) it allows us to not relay tx invs before receiving the peer's version message // b) the peer may tell us in its version message that we should not relay tx invs // unless it loads a bloom filter. - bool fRelayTxes; + bool fRelayTxes; //protected by cs_filter bool fSentAddr; CSemaphoreGrant grantOutbound; CCriticalSection cs_filter; CBloomFilter* pfilter; int nRefCount; NodeId id; + + const uint64_t nKeyedNetGroup; protected: // Denial-of-service detection/prevention @@ -398,7 +400,13 @@ class CNode // inventory based relay CRollingBloomFilter filterInventoryKnown; - std::vector vInventoryToSend; + // Set of transaction ids we still have to announce. + // They are sorted by the mempool before relay, so the order is not important. + std::set setInventoryTxToSend; + // List of block ids we still have announce. + // There is no final sorting before sending, as they are always sent immediately + // and in the order requested. + std::vector vInventoryBlockToSend; CCriticalSection cs_inventory; std::set setAskFor; std::multimap mapAskFor; @@ -406,7 +414,11 @@ class CNode // Used for headers announcements - unfiltered blocks to relay // Also protected by cs_inventory std::vector vBlockHashesToAnnounce; + // Used for BIP35 mempool sending, also protected by cs_inventory + bool fSendMempool; + // Last time a "MEMPOOL" request was serviced. + std::atomic timeLastMempoolReq; // Ping time measurement: // The pong reply we're expecting, or 0 if no pong expected. uint64_t nPingNonceSent; @@ -443,6 +455,8 @@ class CNode CNode(const CNode&); void operator=(const CNode&); + static uint64_t CalculateKeyedNetGroup(const CAddress& ad); + public: NodeId GetId() const { @@ -518,11 +532,13 @@ class CNode void PushInventory(const CInv& inv) { - { - LOCK(cs_inventory); - if ((inv.type == MSG_TX || inv.type == MSG_WITNESS_TX) && filterInventoryKnown.contains(inv.hash)) - return; - vInventoryToSend.push_back(inv); + LOCK(cs_inventory); + if (inv.type == MSG_TX) { + if (!filterInventoryKnown.contains(inv.hash)) { + setInventoryTxToSend.insert(inv.hash); + } + } else if (inv.type == MSG_BLOCK) { + vInventoryBlockToSend.push_back(inv.hash); } } @@ -791,7 +807,7 @@ class CNode class CTransaction; -void RelayTransaction(const CTransaction& tx, CFeeRate feerate); +void RelayTransaction(const CTransaction& tx); /** Access to the (IP) address database (peers.dat) */ class CAddrDB @@ -802,6 +818,7 @@ class CAddrDB CAddrDB(); bool Write(const CAddrMan& addr); bool Read(CAddrMan& addr); + bool Read(CAddrMan& addr, CDataStream& ssPeers); }; /** Access to the banlist database (banlist.dat) */ @@ -815,8 +832,6 @@ class CBanDB bool Read(banmap_t& banSet); }; -void DumpBanlist(); - /** Return a timestamp in the future (in microseconds) for exponentially distributed events. */ int64_t PoissonNextSend(int64_t nNow, int average_interval_seconds); diff --git a/src/netbase.cpp b/src/netbase.cpp index 281c6bcb7fe79..572ae70871d40 100644 --- a/src/netbase.cpp +++ b/src/netbase.cpp @@ -291,10 +291,25 @@ struct ProxyCredentials std::string password; }; +std::string Socks5ErrorString(int err) +{ + switch(err) { + case 0x01: return "general failure"; + case 0x02: return "connection not allowed"; + case 0x03: return "network unreachable"; + case 0x04: return "host unreachable"; + case 0x05: return "connection refused"; + case 0x06: return "TTL expired"; + case 0x07: return "protocol error"; + case 0x08: return "address type not supported"; + default: return "unknown"; + } +} + /** Connect using SOCKS5 (as described in RFC1928) */ static bool Socks5(const std::string& strDest, int port, const ProxyCredentials *auth, SOCKET& hSocket) { - LogPrintf("SOCKS5 connecting %s\n", strDest); + LogPrint("net", "SOCKS5 connecting %s\n", strDest); if (strDest.size() > 255) { CloseSocket(hSocket); return error("Hostname too long"); @@ -318,7 +333,8 @@ static bool Socks5(const std::string& strDest, int port, const ProxyCredentials char pchRet1[2]; if (!InterruptibleRecv(pchRet1, 2, SOCKS5_RECV_TIMEOUT, hSocket)) { CloseSocket(hSocket); - return error("Error reading proxy response"); + LogPrintf("Socks5() connect to %s:%d failed: InterruptibleRecv() timeout or other failure\n", strDest, port); + return false; } if (pchRet1[0] != 0x05) { CloseSocket(hSocket); @@ -379,19 +395,10 @@ static bool Socks5(const std::string& strDest, int port, const ProxyCredentials return error("Proxy failed to accept request"); } if (pchRet2[1] != 0x00) { + // Failures to connect to a peer that are not proxy errors CloseSocket(hSocket); - switch (pchRet2[1]) - { - case 0x01: return error("Proxy error: general failure"); - case 0x02: return error("Proxy error: connection not allowed"); - case 0x03: return error("Proxy error: network unreachable"); - case 0x04: return error("Proxy error: host unreachable"); - case 0x05: return error("Proxy error: connection refused"); - case 0x06: return error("Proxy error: TTL expired"); - case 0x07: return error("Proxy error: protocol error"); - case 0x08: return error("Proxy error: address type not supported"); - default: return error("Proxy error: unknown"); - } + LogPrintf("Socks5() connect to %s:%d failed: %s\n", strDest, port, Socks5ErrorString(pchRet2[1])); + return false; } if (pchRet2[2] != 0x00) { CloseSocket(hSocket); @@ -423,7 +430,7 @@ static bool Socks5(const std::string& strDest, int port, const ProxyCredentials CloseSocket(hSocket); return error("Error reading from proxy"); } - LogPrintf("SOCKS5 connected %s\n", strDest); + LogPrint("net", "SOCKS5 connected %s\n", strDest); return true; } @@ -614,10 +621,12 @@ bool ConnectSocketByName(CService &addr, SOCKET& hSocketRet, const char *pszDest proxyType nameProxy; GetNameProxy(nameProxy); - CService addrResolved(CNetAddr(strDest, fNameLookup && !HaveNameProxy()), port); - if (addrResolved.IsValid()) { - addr = addrResolved; - return ConnectSocket(addr, hSocketRet, nTimeout); + CService addrResolved; + if (Lookup(strDest.c_str(), addrResolved, port, fNameLookup && !HaveNameProxy())) { + if (addrResolved.IsValid()) { + addr = addrResolved; + return ConnectSocket(addr, hSocketRet, nTimeout); + } } addr = CService("0.0.0.0:0"); @@ -686,19 +695,19 @@ CNetAddr::CNetAddr(const struct in6_addr& ipv6Addr, const uint32_t scope) scopeId = scope; } -CNetAddr::CNetAddr(const char *pszIp, bool fAllowLookup) +CNetAddr::CNetAddr(const char *pszIp) { Init(); std::vector vIP; - if (LookupHost(pszIp, vIP, 1, fAllowLookup)) + if (LookupHost(pszIp, vIP, 1, false)) *this = vIP[0]; } -CNetAddr::CNetAddr(const std::string &strIp, bool fAllowLookup) +CNetAddr::CNetAddr(const std::string &strIp) { Init(); std::vector vIP; - if (LookupHost(strIp.c_str(), vIP, 1, fAllowLookup)) + if (LookupHost(strIp.c_str(), vIP, 1, false)) *this = vIP[0]; } @@ -1121,35 +1130,35 @@ bool CService::SetSockAddr(const struct sockaddr *paddr) } } -CService::CService(const char *pszIpPort, bool fAllowLookup) +CService::CService(const char *pszIpPort) { Init(); CService ip; - if (Lookup(pszIpPort, ip, 0, fAllowLookup)) + if (Lookup(pszIpPort, ip, 0, false)) *this = ip; } -CService::CService(const char *pszIpPort, int portDefault, bool fAllowLookup) +CService::CService(const char *pszIpPort, int portDefault) { Init(); CService ip; - if (Lookup(pszIpPort, ip, portDefault, fAllowLookup)) + if (Lookup(pszIpPort, ip, portDefault, false)) *this = ip; } -CService::CService(const std::string &strIpPort, bool fAllowLookup) +CService::CService(const std::string &strIpPort) { Init(); CService ip; - if (Lookup(strIpPort.c_str(), ip, 0, fAllowLookup)) + if (Lookup(strIpPort.c_str(), ip, 0, false)) *this = ip; } -CService::CService(const std::string &strIpPort, int portDefault, bool fAllowLookup) +CService::CService(const std::string &strIpPort, int portDefault) { Init(); CService ip; - if (Lookup(strIpPort.c_str(), ip, portDefault, fAllowLookup)) + if (Lookup(strIpPort.c_str(), ip, portDefault, false)) *this = ip; } @@ -1243,7 +1252,7 @@ CSubNet::CSubNet(): memset(netmask, 0, sizeof(netmask)); } -CSubNet::CSubNet(const std::string &strSubnet, bool fAllowLookup) +CSubNet::CSubNet(const std::string &strSubnet) { size_t slash = strSubnet.find_last_of('/'); std::vector vIP; @@ -1253,7 +1262,7 @@ CSubNet::CSubNet(const std::string &strSubnet, bool fAllowLookup) memset(netmask, 255, sizeof(netmask)); std::string strAddress = strSubnet.substr(0, slash); - if (LookupHost(strAddress.c_str(), vIP, 1, fAllowLookup)) + if (LookupHost(strAddress.c_str(), vIP, 1, false)) { network = vIP[0]; if (slash != strSubnet.npos) diff --git a/src/netbase.h b/src/netbase.h index db736154fa684..65187a17cfd54 100644 --- a/src/netbase.h +++ b/src/netbase.h @@ -49,8 +49,8 @@ class CNetAddr public: CNetAddr(); CNetAddr(const struct in_addr& ipv4Addr); - explicit CNetAddr(const char *pszIp, bool fAllowLookup = false); - explicit CNetAddr(const std::string &strIp, bool fAllowLookup = false); + explicit CNetAddr(const char *pszIp); + explicit CNetAddr(const std::string &strIp); void Init(); void SetIP(const CNetAddr& ip); @@ -119,7 +119,7 @@ class CSubNet public: CSubNet(); - explicit CSubNet(const std::string &strSubnet, bool fAllowLookup = false); + explicit CSubNet(const std::string &strSubnet); //constructor for single ip subnet (/32 or /128) explicit CSubNet(const CNetAddr &addr); @@ -154,10 +154,10 @@ class CService : public CNetAddr CService(const CNetAddr& ip, unsigned short port); CService(const struct in_addr& ipv4Addr, unsigned short port); CService(const struct sockaddr_in& addr); - explicit CService(const char *pszIpPort, int portDefault, bool fAllowLookup = false); - explicit CService(const char *pszIpPort, bool fAllowLookup = false); - explicit CService(const std::string& strIpPort, int portDefault, bool fAllowLookup = false); - explicit CService(const std::string& strIpPort, bool fAllowLookup = false); + explicit CService(const char *pszIpPort, int portDefault); + explicit CService(const char *pszIpPort); + explicit CService(const std::string& strIpPort, int portDefault); + explicit CService(const std::string& strIpPort); void Init(); void SetPort(unsigned short portIn); unsigned short GetPort() const; @@ -206,9 +206,9 @@ bool GetProxy(enum Network net, proxyType &proxyInfoOut); bool IsProxy(const CNetAddr &addr); bool SetNameProxy(const proxyType &addrProxy); bool HaveNameProxy(); -bool LookupHost(const char *pszName, std::vector& vIP, unsigned int nMaxSolutions = 0, bool fAllowLookup = true); -bool Lookup(const char *pszName, CService& addr, int portDefault = 0, bool fAllowLookup = true); -bool Lookup(const char *pszName, std::vector& vAddr, int portDefault = 0, bool fAllowLookup = true, unsigned int nMaxSolutions = 0); +bool LookupHost(const char *pszName, std::vector& vIP, unsigned int nMaxSolutions, bool fAllowLookup); +bool Lookup(const char *pszName, CService& addr, int portDefault, bool fAllowLookup); +bool Lookup(const char *pszName, std::vector& vAddr, int portDefault, bool fAllowLookup, unsigned int nMaxSolutions); bool LookupNumeric(const char *pszName, CService& addr, int portDefault = 0); bool ConnectSocket(const CService &addr, SOCKET& hSocketRet, int nTimeout, bool *outProxyConnectionFailed = 0); bool ConnectSocketByName(CService &addr, SOCKET& hSocketRet, const char *pszDest, int portDefault, int nTimeout, bool *outProxyConnectionFailed = 0); diff --git a/src/pow.cpp b/src/pow.cpp index 59d597e0492f3..6df9d71d401db 100644 --- a/src/pow.cpp +++ b/src/pow.cpp @@ -64,9 +64,7 @@ unsigned int CalculateNextWorkRequired(const CBlockIndex* pindexLast, int64_t nF // Retarget const arith_uint256 bnPowLimit = UintToArith256(params.powLimit); arith_uint256 bnNew; - arith_uint256 bnOld; bnNew.SetCompact(pindexLast->nBits); - bnOld = bnNew; bnNew *= nActualTimespan; bnNew /= params.nPowTargetTimespan; diff --git a/src/protocol.cpp b/src/protocol.cpp index e6a52fbd24a41..925229e8eb3fd 100644 --- a/src/protocol.cpp +++ b/src/protocol.cpp @@ -125,7 +125,7 @@ CAddress::CAddress() : CService() Init(); } -CAddress::CAddress(CService ipIn, uint64_t nServicesIn) : CService(ipIn) +CAddress::CAddress(CService ipIn, ServiceFlags nServicesIn) : CService(ipIn) { Init(); nServices = nServicesIn; @@ -133,7 +133,7 @@ CAddress::CAddress(CService ipIn, uint64_t nServicesIn) : CService(ipIn) void CAddress::Init() { - nServices = NODE_NETWORK; + nServices = NODE_NONE; nTime = 100000000; } diff --git a/src/protocol.h b/src/protocol.h index 1b8d22e93e416..8aaae27912be4 100644 --- a/src/protocol.h +++ b/src/protocol.h @@ -223,7 +223,9 @@ extern const char *FEEFILTER; const std::vector &getAllNetMessageTypes(); /** nServices flags */ -enum { +enum ServiceFlags : uint64_t { + // Nothing + NODE_NONE = 0, // NODE_NETWORK means that the node is capable of serving the block chain. It is currently // set by all Bitcoin Core nodes, and is unset by SPV clients or other peers that just want // network services but don't provide them. @@ -254,7 +256,7 @@ class CAddress : public CService { public: CAddress(); - explicit CAddress(CService ipIn, uint64_t nServicesIn = NODE_NETWORK); + explicit CAddress(CService ipIn, ServiceFlags nServicesIn); void Init(); @@ -270,13 +272,15 @@ class CAddress : public CService if ((nType & SER_DISK) || (nVersion >= CADDR_TIME_VERSION && !(nType & SER_GETHASH))) READWRITE(nTime); - READWRITE(nServices); + uint64_t nServicesInt = nServices; + READWRITE(nServicesInt); + nServices = (ServiceFlags)nServicesInt; READWRITE(*(CService*)this); } // TODO: make private (improves encapsulation) public: - uint64_t nServices; + ServiceFlags nServices; // disk and network only unsigned int nTime; diff --git a/src/qt/askpassphrasedialog.cpp b/src/qt/askpassphrasedialog.cpp index 415bffb991457..e8aa79679c04a 100644 --- a/src/qt/askpassphrasedialog.cpp +++ b/src/qt/askpassphrasedialog.cpp @@ -77,10 +77,7 @@ AskPassphraseDialog::AskPassphraseDialog(Mode mode, QWidget *parent) : AskPassphraseDialog::~AskPassphraseDialog() { - // Attempt to overwrite text so that they do not linger around in memory - ui->passEdit1->setText(QString(" ").repeated(ui->passEdit1->text().size())); - ui->passEdit2->setText(QString(" ").repeated(ui->passEdit2->text().size())); - ui->passEdit3->setText(QString(" ").repeated(ui->passEdit3->text().size())); + secureClearPassFields(); delete ui; } @@ -103,6 +100,8 @@ void AskPassphraseDialog::accept() newpass1.assign(ui->passEdit2->text().toStdString().c_str()); newpass2.assign(ui->passEdit3->text().toStdString().c_str()); + secureClearPassFields(); + switch(mode) { case Encrypt: { @@ -260,3 +259,17 @@ bool AskPassphraseDialog::eventFilter(QObject *object, QEvent *event) } return QDialog::eventFilter(object, event); } + +static void SecureClearQLineEdit(QLineEdit* edit) +{ + // Attempt to overwrite text so that they do not linger around in memory + edit->setText(QString(" ").repeated(edit->text().size())); + edit->clear(); +} + +void AskPassphraseDialog::secureClearPassFields() +{ + SecureClearQLineEdit(ui->passEdit1); + SecureClearQLineEdit(ui->passEdit2); + SecureClearQLineEdit(ui->passEdit3); +} diff --git a/src/qt/askpassphrasedialog.h b/src/qt/askpassphrasedialog.h index 727b5a1ada18a..34bf7ccb31f6f 100644 --- a/src/qt/askpassphrasedialog.h +++ b/src/qt/askpassphrasedialog.h @@ -42,6 +42,7 @@ class AskPassphraseDialog : public QDialog private Q_SLOTS: void textChanged(); + void secureClearPassFields(); protected: bool event(QEvent *event); diff --git a/src/qt/bitcoin.cpp b/src/qt/bitcoin.cpp index 9c21bb24cee28..6218ab6ab0542 100644 --- a/src/qt/bitcoin.cpp +++ b/src/qt/bitcoin.cpp @@ -370,6 +370,7 @@ void BitcoinApplication::createSplashScreen(const NetworkStyle *networkStyle) splash->setAttribute(Qt::WA_DeleteOnClose); splash->show(); connect(this, SIGNAL(splashFinished(QWidget*)), splash, SLOT(slotFinish(QWidget*))); + connect(this, SIGNAL(requestedShutdown()), splash, SLOT(close())); } void BitcoinApplication::startThread() diff --git a/src/qt/bitcoingui.cpp b/src/qt/bitcoingui.cpp index b00cdfcaf2bdd..50c19c3848046 100644 --- a/src/qt/bitcoingui.cpp +++ b/src/qt/bitcoingui.cpp @@ -320,12 +320,14 @@ void BitcoinGUI::createActions() aboutAction = new QAction(platformStyle->TextColorIcon(":/icons/about"), tr("&About %1").arg(tr(PACKAGE_NAME)), this); aboutAction->setStatusTip(tr("Show information about %1").arg(tr(PACKAGE_NAME))); aboutAction->setMenuRole(QAction::AboutRole); + aboutAction->setEnabled(false); aboutQtAction = new QAction(platformStyle->TextColorIcon(":/icons/about_qt"), tr("About &Qt"), this); aboutQtAction->setStatusTip(tr("Show information about Qt")); aboutQtAction->setMenuRole(QAction::AboutQtRole); optionsAction = new QAction(platformStyle->TextColorIcon(":/icons/options"), tr("&Options..."), this); optionsAction->setStatusTip(tr("Modify configuration options for %1").arg(tr(PACKAGE_NAME))); optionsAction->setMenuRole(QAction::PreferencesRole); + optionsAction->setEnabled(false); toggleHideAction = new QAction(platformStyle->TextColorIcon(":/icons/about"), tr("&Show / Hide"), this); toggleHideAction->setStatusTip(tr("Show or hide the main Window")); @@ -343,6 +345,8 @@ void BitcoinGUI::createActions() openRPCConsoleAction = new QAction(platformStyle->TextColorIcon(":/icons/debugwindow"), tr("&Debug window"), this); openRPCConsoleAction->setStatusTip(tr("Open debugging and diagnostic console")); + // initially disable the debug window menu item + openRPCConsoleAction->setEnabled(false); usedSendingAddressesAction = new QAction(platformStyle->TextColorIcon(":/icons/address-book"), tr("&Sending addresses..."), this); usedSendingAddressesAction->setStatusTip(tr("Show the list of used sending addresses and labels")); @@ -457,8 +461,8 @@ void BitcoinGUI::setClientModel(ClientModel *clientModel) setNumConnections(clientModel->getNumConnections()); connect(clientModel, SIGNAL(numConnectionsChanged(int)), this, SLOT(setNumConnections(int))); - setNumBlocks(clientModel->getNumBlocks(), clientModel->getLastBlockDate(), clientModel->getVerificationProgress(NULL)); - connect(clientModel, SIGNAL(numBlocksChanged(int,QDateTime,double)), this, SLOT(setNumBlocks(int,QDateTime,double))); + setNumBlocks(clientModel->getNumBlocks(), clientModel->getLastBlockDate(), clientModel->getVerificationProgress(NULL), false); + connect(clientModel, SIGNAL(numBlocksChanged(int,QDateTime,double,bool)), this, SLOT(setNumBlocks(int,QDateTime,double,bool))); // Receive and report messages from client model connect(clientModel, SIGNAL(message(QString,QString,unsigned int)), this, SLOT(message(QString,QString,unsigned int))); @@ -474,6 +478,16 @@ void BitcoinGUI::setClientModel(ClientModel *clientModel) } #endif // ENABLE_WALLET unitDisplayControl->setOptionsModel(clientModel->getOptionsModel()); + + OptionsModel* optionsModel = clientModel->getOptionsModel(); + if(optionsModel) + { + // be aware of the tray icon disable state change reported by the OptionsModel object. + connect(optionsModel,SIGNAL(hideTrayIconChanged(bool)),this,SLOT(setTrayIconVisible(bool))); + + // initialize the disable state of the tray icon with the current value in the model. + setTrayIconVisible(optionsModel->getHideTrayIcon()); + } } else { // Disable possibility to show main window via action toggleHideAction->setEnabled(false); @@ -535,7 +549,7 @@ void BitcoinGUI::createTrayIcon(const NetworkStyle *networkStyle) QString toolTip = tr("%1 client").arg(tr(PACKAGE_NAME)) + " " + networkStyle->getTitleAddText(); trayIcon->setToolTip(toolTip); trayIcon->setIcon(networkStyle->getTrayAndWindowIcon()); - trayIcon->show(); + trayIcon->hide(); #endif notificator = new Notificator(QApplication::applicationName(), trayIcon, this); @@ -686,7 +700,7 @@ void BitcoinGUI::setNumConnections(int count) labelConnectionsIcon->setToolTip(tr("%n active connection(s) to Bitcoin network", "", count)); } -void BitcoinGUI::setNumBlocks(int count, const QDateTime& blockDate, double nVerificationProgress) +void BitcoinGUI::setNumBlocks(int count, const QDateTime& blockDate, double nVerificationProgress, bool header) { if(!clientModel) return; @@ -698,15 +712,25 @@ void BitcoinGUI::setNumBlocks(int count, const QDateTime& blockDate, double nVer enum BlockSource blockSource = clientModel->getBlockSource(); switch (blockSource) { case BLOCK_SOURCE_NETWORK: + if (header) { + return; + } progressBarLabel->setText(tr("Synchronizing with network...")); break; case BLOCK_SOURCE_DISK: - progressBarLabel->setText(tr("Importing blocks from disk...")); + if (header) { + progressBarLabel->setText(tr("Indexing blocks on disk...")); + } else { + progressBarLabel->setText(tr("Processing blocks on disk...")); + } break; case BLOCK_SOURCE_REINDEX: progressBarLabel->setText(tr("Reindexing blocks on disk...")); break; case BLOCK_SOURCE_NONE: + if (header) { + return; + } // Case: not Importing, not Reindexing and no network connection progressBarLabel->setText(tr("No block source available...")); break; @@ -891,6 +915,14 @@ void BitcoinGUI::closeEvent(QCloseEvent *event) QMainWindow::closeEvent(event); } +void BitcoinGUI::showEvent(QShowEvent *event) +{ + // enable the debug window when the main window shows up + openRPCConsoleAction->setEnabled(true); + aboutAction->setEnabled(true); + optionsAction->setEnabled(true); +} + #ifdef ENABLE_WALLET void BitcoinGUI::incomingTransaction(const QString& date, int unit, const CAmount& amount, const QString& type, const QString& address, const QString& label) { @@ -1044,6 +1076,14 @@ void BitcoinGUI::showProgress(const QString &title, int nProgress) progressDialog->setValue(nProgress); } +void BitcoinGUI::setTrayIconVisible(bool fHideTrayIcon) +{ + if (trayIcon) + { + trayIcon->setVisible(!fHideTrayIcon); + } +} + static bool ThreadSafeMessageBox(BitcoinGUI *gui, const std::string& message, const std::string& caption, unsigned int style) { bool modal = (style & CClientUIInterface::MODAL); diff --git a/src/qt/bitcoingui.h b/src/qt/bitcoingui.h index 871ca1ba347e9..12e7702ed82d6 100644 --- a/src/qt/bitcoingui.h +++ b/src/qt/bitcoingui.h @@ -72,6 +72,7 @@ class BitcoinGUI : public QMainWindow protected: void changeEvent(QEvent *e); void closeEvent(QCloseEvent *event); + void showEvent(QShowEvent *event); void dragEnterEvent(QDragEnterEvent *event); void dropEvent(QDropEvent *event); bool eventFilter(QObject *object, QEvent *event); @@ -150,7 +151,7 @@ public Q_SLOTS: /** Set number of connections shown in the UI */ void setNumConnections(int count); /** Set number of blocks and last block date shown in the UI */ - void setNumBlocks(int count, const QDateTime& blockDate, double nVerificationProgress); + void setNumBlocks(int count, const QDateTime& blockDate, double nVerificationProgress, bool headers); /** Notify the user of an event from the core network or transaction handling code. @param[in] title the message box / notification title @@ -218,6 +219,9 @@ private Q_SLOTS: /** Show progress dialog e.g. for verifychain */ void showProgress(const QString &title, int nProgress); + + /** When hideTrayIcon setting is changed in OptionsModel hide or show the icon accordingly. */ + void setTrayIconVisible(bool); }; class UnitDisplayStatusBarControl : public QLabel diff --git a/src/qt/bitcoinstrings.cpp b/src/qt/bitcoinstrings.cpp index 41f1d5841a659..9e53f19591070 100644 --- a/src/qt/bitcoinstrings.cpp +++ b/src/qt/bitcoinstrings.cpp @@ -1,26 +1,35 @@ -// Copyright (c) 2013-2015 The Bitcoin Core developers -// Distributed under the MIT software license, see the accompanying -// file COPYING or http://www.opensource.org/licenses/mit-license.php. + #include -// Automatically generated by extract_strings.py +// Automatically generated by extract_strings_qt.py #ifdef __GNUC__ #define UNUSED __attribute__((unused)) #else #define UNUSED #endif static const char UNUSED *bitcoin_strings[] = { +QT_TRANSLATE_NOOP("bitcoin-core", "Bitcoin Core"), +QT_TRANSLATE_NOOP("bitcoin-core", "The %s developers"), QT_TRANSLATE_NOOP("bitcoin-core", "" "(1 = keep tx meta data e.g. account owner and payment request information, 2 " "= drop tx meta data)"), QT_TRANSLATE_NOOP("bitcoin-core", "" +"-fallbackfee is set very high! This is the transaction fee you may pay when " +"fee estimates are not available."), +QT_TRANSLATE_NOOP("bitcoin-core", "" "-maxtxfee is set very high! Fees this large could be paid on a single " "transaction."), QT_TRANSLATE_NOOP("bitcoin-core", "" "-paytxfee is set very high! This is the transaction fee you will pay if you " "send a transaction."), QT_TRANSLATE_NOOP("bitcoin-core", "" +"A fee rate (in %s/kB) that will be used when fee estimation has insufficient " +"data (default: %s)"), +QT_TRANSLATE_NOOP("bitcoin-core", "" +"Accept relayed transactions received from whitelisted peers even when not " +"relaying transactions (default: %d)"), +QT_TRANSLATE_NOOP("bitcoin-core", "" "Allow JSON-RPC connections from specified source. Valid for are a " "single IP (e.g. 1.2.3.4), a network/netmask (e.g. 1.2.3.4/255.255.255.0) or " "a network/CIDR (e.g. 1.2.3.4/24). This option can be specified multiple times"), @@ -35,8 +44,7 @@ QT_TRANSLATE_NOOP("bitcoin-core", "" "notation for IPv6. This option can be specified multiple times (default: " "bind to all interfaces)"), QT_TRANSLATE_NOOP("bitcoin-core", "" -"Cannot obtain a lock on data directory %s. Bitcoin Core is probably already " -"running."), +"Cannot obtain a lock on data directory %s. %s is probably already running."), QT_TRANSLATE_NOOP("bitcoin-core", "" "Create new files with system default permissions, instead of umask 077 (only " "effective with disabled wallet functionality)"), @@ -52,8 +60,8 @@ QT_TRANSLATE_NOOP("bitcoin-core", "" QT_TRANSLATE_NOOP("bitcoin-core", "" "Do not keep transactions in the mempool longer than hours (default: %u)"), QT_TRANSLATE_NOOP("bitcoin-core", "" -"Error reading wallet.dat! All keys read correctly, but transaction data or " -"address book entries might be missing or incorrect."), +"Error reading %s! All keys read correctly, but transaction data or address " +"book entries might be missing or incorrect."), QT_TRANSLATE_NOOP("bitcoin-core", "" "Error: Listening for incoming connections failed (listen returned error %s)"), QT_TRANSLATE_NOOP("bitcoin-core", "" @@ -72,6 +80,9 @@ QT_TRANSLATE_NOOP("bitcoin-core", "" "Fees (in %s/kB) smaller than this are considered zero fee for transaction " "creation (default: %s)"), QT_TRANSLATE_NOOP("bitcoin-core", "" +"Force relay of transactions from whitelisted peers even they violate local " +"relay policy (default: %d)"), +QT_TRANSLATE_NOOP("bitcoin-core", "" "How thorough the block verification of -checkblocks is (0-4, default: %u)"), QT_TRANSLATE_NOOP("bitcoin-core", "" "If is not supplied or if = 1, output all debugging " @@ -86,18 +97,22 @@ QT_TRANSLATE_NOOP("bitcoin-core", "" "Maintain a full transaction index, used by the getrawtransaction rpc call " "(default: %u)"), QT_TRANSLATE_NOOP("bitcoin-core", "" +"Maximum allowed median peer time offset adjustment. Local perspective of " +"time may be influenced by peers forward or backward by this amount. " +"(default: %u seconds)"), +QT_TRANSLATE_NOOP("bitcoin-core", "" "Maximum size of data in data carrier transactions we relay and mine " "(default: %u)"), QT_TRANSLATE_NOOP("bitcoin-core", "" -"Maximum total fees (in %s) to use in a single wallet transaction; setting " -"this too low may abort large transactions (default: %s)"), +"Maximum total fees (in %s) to use in a single wallet transaction or raw " +"transaction; setting this too low may abort large transactions (default: %s)"), QT_TRANSLATE_NOOP("bitcoin-core", "" "Number of seconds to keep misbehaving peers from reconnecting (default: %u)"), QT_TRANSLATE_NOOP("bitcoin-core", "" "Output debugging information (default: %u, supplying is optional)"), QT_TRANSLATE_NOOP("bitcoin-core", "" "Please check that your computer's date and time are correct! If your clock " -"is wrong Bitcoin Core will not work properly."), +"is wrong, %s will not work properly."), QT_TRANSLATE_NOOP("bitcoin-core", "" "Prune configured below the minimum of %d MiB. Please use a higher number."), QT_TRANSLATE_NOOP("bitcoin-core", "" @@ -123,9 +138,6 @@ QT_TRANSLATE_NOOP("bitcoin-core", "" "Set the number of script verification threads (%u to %d, 0 = auto, <0 = " "leave that many cores free, default: %d)"), QT_TRANSLATE_NOOP("bitcoin-core", "" -"Set the number of threads for coin generation if enabled (-1 = all cores, " -"default: %d)"), -QT_TRANSLATE_NOOP("bitcoin-core", "" "Support filtering of blocks and transaction with bloom filters (default: %u)"), QT_TRANSLATE_NOOP("bitcoin-core", "" "The block database contains a block which appears to be from the future. " @@ -148,12 +160,12 @@ QT_TRANSLATE_NOOP("bitcoin-core", "" "Tries to keep outbound traffic under the given target (in MiB per 24h), 0 = " "no limit (default: %d)"), QT_TRANSLATE_NOOP("bitcoin-core", "" -"Unable to bind to %s on this computer. Bitcoin Core is probably already " -"running."), -QT_TRANSLATE_NOOP("bitcoin-core", "" "Unsupported argument -socks found. Setting SOCKS version isn't possible " "anymore, only SOCKS5 proxies are supported."), QT_TRANSLATE_NOOP("bitcoin-core", "" +"Unsupported argument -whitelistalwaysrelay ignored, use -whitelistrelay and/" +"or -whitelistforcerelay."), +QT_TRANSLATE_NOOP("bitcoin-core", "" "Use UPnP to map the listening port (default: 1 when listening and no -proxy)"), QT_TRANSLATE_NOOP("bitcoin-core", "" "Use separate SOCKS5 proxy to reach peers via Tor hidden services (default: " @@ -172,13 +184,16 @@ QT_TRANSLATE_NOOP("bitcoin-core", "" "Warning: The network does not appear to fully agree! Some miners appear to " "be experiencing issues."), QT_TRANSLATE_NOOP("bitcoin-core", "" +"Warning: Unknown block versions being mined! It's possible unknown rules are " +"in effect"), +QT_TRANSLATE_NOOP("bitcoin-core", "" +"Warning: Wallet file corrupt, data salvaged! Original %s saved as %s in %s; " +"if your balance or transactions are incorrect you should restore from a " +"backup."), +QT_TRANSLATE_NOOP("bitcoin-core", "" "Warning: We do not appear to fully agree with our peers! You may need to " "upgrade, or other nodes may need to upgrade."), QT_TRANSLATE_NOOP("bitcoin-core", "" -"Warning: wallet.dat corrupt, data salvaged! Original wallet.dat saved as " -"wallet.{timestamp}.bak in %s; if your balance or transactions are incorrect " -"you should restore from a backup."), -QT_TRANSLATE_NOOP("bitcoin-core", "" "Whitelist peers connecting from the given netmask or IP address. Can be " "specified multiple times."), QT_TRANSLATE_NOOP("bitcoin-core", "" @@ -187,6 +202,9 @@ QT_TRANSLATE_NOOP("bitcoin-core", "" QT_TRANSLATE_NOOP("bitcoin-core", "" "You need to rebuild the database using -reindex to go back to unpruned " "mode. This will redownload the entire blockchain"), +QT_TRANSLATE_NOOP("bitcoin-core", "" +"You need to rebuild the database using -reindex-chainstate to change -txindex"), +QT_TRANSLATE_NOOP("bitcoin-core", "%s corrupt, salvage failed"), QT_TRANSLATE_NOOP("bitcoin-core", "(default: %s)"), QT_TRANSLATE_NOOP("bitcoin-core", "(default: %u)"), QT_TRANSLATE_NOOP("bitcoin-core", "-maxmempool must be at least %d MB"), @@ -194,24 +212,22 @@ QT_TRANSLATE_NOOP("bitcoin-core", " can be:"), QT_TRANSLATE_NOOP("bitcoin-core", "Accept command line and JSON-RPC commands"), QT_TRANSLATE_NOOP("bitcoin-core", "Accept connections from outside (default: 1 if no -proxy or -connect)"), QT_TRANSLATE_NOOP("bitcoin-core", "Accept public REST requests (default: %u)"), -QT_TRANSLATE_NOOP("bitcoin-core", "Activating best chain..."), QT_TRANSLATE_NOOP("bitcoin-core", "Add a node to connect to and attempt to keep the connection open"), QT_TRANSLATE_NOOP("bitcoin-core", "Allow DNS lookups for -addnode, -seednode and -connect"), QT_TRANSLATE_NOOP("bitcoin-core", "Always query for peer addresses via DNS lookup (default: %u)"), -QT_TRANSLATE_NOOP("bitcoin-core", "Always relay transactions received from whitelisted peers (default: %d)"), -QT_TRANSLATE_NOOP("bitcoin-core", "Attempt to recover private keys from a corrupt wallet.dat on startup"), +QT_TRANSLATE_NOOP("bitcoin-core", "Append comment to the user agent string"), +QT_TRANSLATE_NOOP("bitcoin-core", "Attempt to recover private keys from a corrupt wallet on startup"), QT_TRANSLATE_NOOP("bitcoin-core", "Automatically create Tor hidden service (default: %d)"), QT_TRANSLATE_NOOP("bitcoin-core", "Block creation options:"), QT_TRANSLATE_NOOP("bitcoin-core", "Cannot downgrade wallet"), -QT_TRANSLATE_NOOP("bitcoin-core", "Cannot resolve -bind address: '%s'"), -QT_TRANSLATE_NOOP("bitcoin-core", "Cannot resolve -externalip address: '%s'"), -QT_TRANSLATE_NOOP("bitcoin-core", "Cannot resolve -whitebind address: '%s'"), +QT_TRANSLATE_NOOP("bitcoin-core", "Cannot resolve -%s address: '%s'"), QT_TRANSLATE_NOOP("bitcoin-core", "Cannot write default address"), +QT_TRANSLATE_NOOP("bitcoin-core", "Change index out of range"), QT_TRANSLATE_NOOP("bitcoin-core", "Connect only to the specified node(s)"), QT_TRANSLATE_NOOP("bitcoin-core", "Connect through SOCKS5 proxy"), QT_TRANSLATE_NOOP("bitcoin-core", "Connect to a node to retrieve peer addresses, and disconnect"), QT_TRANSLATE_NOOP("bitcoin-core", "Connection options:"), -QT_TRANSLATE_NOOP("bitcoin-core", "Copyright (C) 2009-%i The Bitcoin Core Developers"), +QT_TRANSLATE_NOOP("bitcoin-core", "Copyright (C) %i-%i"), QT_TRANSLATE_NOOP("bitcoin-core", "Corrupted block database detected"), QT_TRANSLATE_NOOP("bitcoin-core", "Debugging/Testing options:"), QT_TRANSLATE_NOOP("bitcoin-core", "Do not load the wallet and disable wallet RPC calls"), @@ -221,12 +237,13 @@ QT_TRANSLATE_NOOP("bitcoin-core", "Enable publish hash block in
"), QT_TRANSLATE_NOOP("bitcoin-core", "Enable publish hash transaction in
"), QT_TRANSLATE_NOOP("bitcoin-core", "Enable publish raw block in
"), QT_TRANSLATE_NOOP("bitcoin-core", "Enable publish raw transaction in
"), +QT_TRANSLATE_NOOP("bitcoin-core", "Enable transaction replacement in the memory pool (default: %u)"), QT_TRANSLATE_NOOP("bitcoin-core", "Error initializing block database"), QT_TRANSLATE_NOOP("bitcoin-core", "Error initializing wallet database environment %s!"), +QT_TRANSLATE_NOOP("bitcoin-core", "Error loading %s"), +QT_TRANSLATE_NOOP("bitcoin-core", "Error loading %s: Wallet corrupted"), +QT_TRANSLATE_NOOP("bitcoin-core", "Error loading %s: Wallet requires newer version of %s"), QT_TRANSLATE_NOOP("bitcoin-core", "Error loading block database"), -QT_TRANSLATE_NOOP("bitcoin-core", "Error loading wallet.dat"), -QT_TRANSLATE_NOOP("bitcoin-core", "Error loading wallet.dat: Wallet corrupted"), -QT_TRANSLATE_NOOP("bitcoin-core", "Error loading wallet.dat: Wallet requires newer version of Bitcoin Core"), QT_TRANSLATE_NOOP("bitcoin-core", "Error opening block database"), QT_TRANSLATE_NOOP("bitcoin-core", "Error reading from database, shutting down."), QT_TRANSLATE_NOOP("bitcoin-core", "Error"), @@ -234,34 +251,34 @@ QT_TRANSLATE_NOOP("bitcoin-core", "Error: A fatal internal error occurred, see d QT_TRANSLATE_NOOP("bitcoin-core", "Error: Disk space is low!"), QT_TRANSLATE_NOOP("bitcoin-core", "Failed to listen on any port. Use -listen=0 if you want this."), QT_TRANSLATE_NOOP("bitcoin-core", "Fee (in %s/kB) to add to transactions you send (default: %s)"), -QT_TRANSLATE_NOOP("bitcoin-core", "Generate coins (default: %u)"), QT_TRANSLATE_NOOP("bitcoin-core", "How many blocks to check at startup (default: %u, 0 = all)"), QT_TRANSLATE_NOOP("bitcoin-core", "Importing..."), QT_TRANSLATE_NOOP("bitcoin-core", "Imports blocks from external blk000??.dat file on startup"), QT_TRANSLATE_NOOP("bitcoin-core", "Include IP addresses in debug output (default: %u)"), QT_TRANSLATE_NOOP("bitcoin-core", "Incorrect or no genesis block found. Wrong datadir for network?"), QT_TRANSLATE_NOOP("bitcoin-core", "Information"), -QT_TRANSLATE_NOOP("bitcoin-core", "Initialization sanity check failed. Bitcoin Core is shutting down."), +QT_TRANSLATE_NOOP("bitcoin-core", "Initialization sanity check failed. %s is shutting down."), QT_TRANSLATE_NOOP("bitcoin-core", "Insufficient funds"), QT_TRANSLATE_NOOP("bitcoin-core", "Invalid -onion address: '%s'"), QT_TRANSLATE_NOOP("bitcoin-core", "Invalid -proxy address: '%s'"), -QT_TRANSLATE_NOOP("bitcoin-core", "Invalid amount for -maxtxfee=: '%s'"), -QT_TRANSLATE_NOOP("bitcoin-core", "Invalid amount for -minrelaytxfee=: '%s'"), -QT_TRANSLATE_NOOP("bitcoin-core", "Invalid amount for -mintxfee=: '%s'"), +QT_TRANSLATE_NOOP("bitcoin-core", "Invalid amount for -%s=: '%s'"), +QT_TRANSLATE_NOOP("bitcoin-core", "Invalid amount for -fallbackfee=: '%s'"), QT_TRANSLATE_NOOP("bitcoin-core", "Invalid amount for -paytxfee=: '%s' (must be at least %s)"), -QT_TRANSLATE_NOOP("bitcoin-core", "Invalid amount for -paytxfee=: '%s'"), QT_TRANSLATE_NOOP("bitcoin-core", "Invalid netmask specified in -whitelist: '%s'"), QT_TRANSLATE_NOOP("bitcoin-core", "Keep at most unconnectable transactions in memory (default: %u)"), QT_TRANSLATE_NOOP("bitcoin-core", "Keep the transaction memory pool below megabytes (default: %u)"), QT_TRANSLATE_NOOP("bitcoin-core", "Listen for JSON-RPC connections on (default: %u or testnet: %u)"), QT_TRANSLATE_NOOP("bitcoin-core", "Listen for connections on (default: %u or testnet: %u)"), QT_TRANSLATE_NOOP("bitcoin-core", "Loading addresses..."), +QT_TRANSLATE_NOOP("bitcoin-core", "Loading banlist..."), QT_TRANSLATE_NOOP("bitcoin-core", "Loading block index..."), QT_TRANSLATE_NOOP("bitcoin-core", "Loading wallet..."), +QT_TRANSLATE_NOOP("bitcoin-core", "Location of the auth cookie (default: data dir)"), QT_TRANSLATE_NOOP("bitcoin-core", "Maintain at most connections to peers (default: %u)"), QT_TRANSLATE_NOOP("bitcoin-core", "Make the wallet broadcast transactions"), QT_TRANSLATE_NOOP("bitcoin-core", "Maximum per-connection receive buffer, *1000 bytes (default: %u)"), QT_TRANSLATE_NOOP("bitcoin-core", "Maximum per-connection send buffer, *1000 bytes (default: %u)"), +QT_TRANSLATE_NOOP("bitcoin-core", "Minimum bytes per sigop in transactions we relay and mine (default: %u)"), QT_TRANSLATE_NOOP("bitcoin-core", "Need to specify a port with -whitebind: '%s'"), QT_TRANSLATE_NOOP("bitcoin-core", "Node relay options:"), QT_TRANSLATE_NOOP("bitcoin-core", "Not enough file descriptors available."), @@ -269,12 +286,14 @@ QT_TRANSLATE_NOOP("bitcoin-core", "Only connect to nodes in network (ipv4, QT_TRANSLATE_NOOP("bitcoin-core", "Options:"), QT_TRANSLATE_NOOP("bitcoin-core", "Password for JSON-RPC connections"), QT_TRANSLATE_NOOP("bitcoin-core", "Prepend debug output with timestamp (default: %u)"), +QT_TRANSLATE_NOOP("bitcoin-core", "Print this help message and exit"), +QT_TRANSLATE_NOOP("bitcoin-core", "Print version and exit"), QT_TRANSLATE_NOOP("bitcoin-core", "Prune cannot be configured with a negative value."), QT_TRANSLATE_NOOP("bitcoin-core", "Prune mode is incompatible with -txindex."), QT_TRANSLATE_NOOP("bitcoin-core", "Pruning blockstore..."), QT_TRANSLATE_NOOP("bitcoin-core", "RPC server options:"), -QT_TRANSLATE_NOOP("bitcoin-core", "Rebuild block chain index from current blk000??.dat files on startup"), -QT_TRANSLATE_NOOP("bitcoin-core", "Receive and display P2P network alerts (default: %u)"), +QT_TRANSLATE_NOOP("bitcoin-core", "Rebuild chain state and block index from the blk*.dat files on disk"), +QT_TRANSLATE_NOOP("bitcoin-core", "Rebuild chain state from the currently indexed blocks"), QT_TRANSLATE_NOOP("bitcoin-core", "Reducing -maxconnections from %d to %d, because of system limitations."), QT_TRANSLATE_NOOP("bitcoin-core", "Relay and mine data carrier transactions (default: %u)"), QT_TRANSLATE_NOOP("bitcoin-core", "Relay non-P2SH multisig (default: %u)"), @@ -299,7 +318,6 @@ QT_TRANSLATE_NOOP("bitcoin-core", "Specify wallet file (within data directory)") QT_TRANSLATE_NOOP("bitcoin-core", "Specify your own public address"), QT_TRANSLATE_NOOP("bitcoin-core", "Spend unconfirmed change when sending transactions (default: %u)"), QT_TRANSLATE_NOOP("bitcoin-core", "The transaction amount is too small to pay the fee"), -QT_TRANSLATE_NOOP("bitcoin-core", "This help message"), QT_TRANSLATE_NOOP("bitcoin-core", "This is experimental software."), QT_TRANSLATE_NOOP("bitcoin-core", "Threshold for disconnecting misbehaving peers (default: %u)"), QT_TRANSLATE_NOOP("bitcoin-core", "Tor control port password (default: empty)"), @@ -309,6 +327,7 @@ QT_TRANSLATE_NOOP("bitcoin-core", "Transaction amounts must be positive"), QT_TRANSLATE_NOOP("bitcoin-core", "Transaction too large for fee policy"), QT_TRANSLATE_NOOP("bitcoin-core", "Transaction too large"), QT_TRANSLATE_NOOP("bitcoin-core", "Unable to bind to %s on this computer (bind returned error %s)"), +QT_TRANSLATE_NOOP("bitcoin-core", "Unable to bind to %s on this computer. %s is probably already running."), QT_TRANSLATE_NOOP("bitcoin-core", "Unable to start HTTP server. See debug log for details."), QT_TRANSLATE_NOOP("bitcoin-core", "Unknown network specified in -onlynet: '%s'"), QT_TRANSLATE_NOOP("bitcoin-core", "Unsupported argument -benchmark ignored, use -debug=bench."), @@ -321,13 +340,12 @@ QT_TRANSLATE_NOOP("bitcoin-core", "Username for JSON-RPC connections"), QT_TRANSLATE_NOOP("bitcoin-core", "Verifying blocks..."), QT_TRANSLATE_NOOP("bitcoin-core", "Verifying wallet..."), QT_TRANSLATE_NOOP("bitcoin-core", "Wallet %s resides outside data directory %s"), -QT_TRANSLATE_NOOP("bitcoin-core", "Wallet needed to be rewritten: restart Bitcoin Core to complete"), +QT_TRANSLATE_NOOP("bitcoin-core", "Wallet debugging/testing options:"), +QT_TRANSLATE_NOOP("bitcoin-core", "Wallet needed to be rewritten: restart %s to complete"), QT_TRANSLATE_NOOP("bitcoin-core", "Wallet options:"), QT_TRANSLATE_NOOP("bitcoin-core", "Warning"), -QT_TRANSLATE_NOOP("bitcoin-core", "Warning: This version is obsolete; upgrade required!"), +QT_TRANSLATE_NOOP("bitcoin-core", "Warning: unknown new rules activated (versionbit %i)"), QT_TRANSLATE_NOOP("bitcoin-core", "Whether to operate in a blocks only mode (default: %u)"), -QT_TRANSLATE_NOOP("bitcoin-core", "You need to rebuild the database using -reindex to change -txindex"), QT_TRANSLATE_NOOP("bitcoin-core", "Zapping all transactions from wallet..."), QT_TRANSLATE_NOOP("bitcoin-core", "ZeroMQ notification options:"), -QT_TRANSLATE_NOOP("bitcoin-core", "wallet.dat corrupt, salvage failed"), }; diff --git a/src/qt/clientmodel.cpp b/src/qt/clientmodel.cpp index 697736cc880bb..108500654b2da 100644 --- a/src/qt/clientmodel.cpp +++ b/src/qt/clientmodel.cpp @@ -24,6 +24,7 @@ class CBlockIndex; static const int64_t nClientStartupTime = GetTime(); +static int64_t nLastHeaderTipUpdateNotification = 0; static int64_t nLastBlockTipUpdateNotification = 0; ClientModel::ClientModel(OptionsModel *optionsModel, QObject *parent) : @@ -226,7 +227,7 @@ static void BannedListChanged(ClientModel *clientmodel) QMetaObject::invokeMethod(clientmodel, "updateBanlist", Qt::QueuedConnection); } -static void BlockTipChanged(ClientModel *clientmodel, bool initialSync, const CBlockIndex *pIndex) +static void BlockTipChanged(ClientModel *clientmodel, bool initialSync, const CBlockIndex *pIndex, bool fHeader) { // lock free async UI updates in case we have a new block tip // during initial sync, only update the UI if the last update @@ -235,14 +236,17 @@ static void BlockTipChanged(ClientModel *clientmodel, bool initialSync, const CB if (initialSync) now = GetTimeMillis(); + int64_t& nLastUpdateNotification = fHeader ? nLastHeaderTipUpdateNotification : nLastBlockTipUpdateNotification; + // if we are in-sync, update the UI regardless of last update time - if (!initialSync || now - nLastBlockTipUpdateNotification > MODEL_UPDATE_DELAY) { + if (!initialSync || now - nLastUpdateNotification > MODEL_UPDATE_DELAY) { //pass a async signal to the UI thread QMetaObject::invokeMethod(clientmodel, "numBlocksChanged", Qt::QueuedConnection, Q_ARG(int, pIndex->nHeight), Q_ARG(QDateTime, QDateTime::fromTime_t(pIndex->GetBlockTime())), - Q_ARG(double, clientmodel->getVerificationProgress(pIndex))); - nLastBlockTipUpdateNotification = now; + Q_ARG(double, clientmodel->getVerificationProgress(pIndex)), + Q_ARG(bool, fHeader)); + nLastUpdateNotification = now; } } @@ -253,7 +257,8 @@ void ClientModel::subscribeToCoreSignals() uiInterface.NotifyNumConnectionsChanged.connect(boost::bind(NotifyNumConnectionsChanged, this, _1)); uiInterface.NotifyAlertChanged.connect(boost::bind(NotifyAlertChanged, this)); uiInterface.BannedListChanged.connect(boost::bind(BannedListChanged, this)); - uiInterface.NotifyBlockTip.connect(boost::bind(BlockTipChanged, this, _1, _2)); + uiInterface.NotifyBlockTip.connect(boost::bind(BlockTipChanged, this, _1, _2, false)); + uiInterface.NotifyHeaderTip.connect(boost::bind(BlockTipChanged, this, _1, _2, true)); } void ClientModel::unsubscribeFromCoreSignals() @@ -263,5 +268,6 @@ void ClientModel::unsubscribeFromCoreSignals() uiInterface.NotifyNumConnectionsChanged.disconnect(boost::bind(NotifyNumConnectionsChanged, this, _1)); uiInterface.NotifyAlertChanged.disconnect(boost::bind(NotifyAlertChanged, this)); uiInterface.BannedListChanged.disconnect(boost::bind(BannedListChanged, this)); - uiInterface.NotifyBlockTip.disconnect(boost::bind(BlockTipChanged, this, _1, _2)); + uiInterface.NotifyBlockTip.disconnect(boost::bind(BlockTipChanged, this, _1, _2, false)); + uiInterface.NotifyHeaderTip.disconnect(boost::bind(BlockTipChanged, this, _1, _2, true)); } diff --git a/src/qt/clientmodel.h b/src/qt/clientmodel.h index 109f95a2a7112..439680431963e 100644 --- a/src/qt/clientmodel.h +++ b/src/qt/clientmodel.h @@ -89,7 +89,7 @@ class ClientModel : public QObject Q_SIGNALS: void numConnectionsChanged(int count); - void numBlocksChanged(int count, const QDateTime& blockDate, double nVerificationProgress); + void numBlocksChanged(int count, const QDateTime& blockDate, double nVerificationProgress, bool header); void mempoolSizeChanged(long count, size_t mempoolSizeInBytes); void alertsChanged(const QString &warnings); void bytesChanged(quint64 totalBytesIn, quint64 totalBytesOut); diff --git a/src/qt/forms/optionsdialog.ui b/src/qt/forms/optionsdialog.ui index c712e6ea01793..0b292018727ed 100644 --- a/src/qt/forms/optionsdialog.ui +++ b/src/qt/forms/optionsdialog.ui @@ -504,6 +504,16 @@ &Window + + + + &Hide the icon from the system tray. + + + Hide tray icon + + + diff --git a/src/qt/forms/transactiondescdialog.ui b/src/qt/forms/transactiondescdialog.ui index 5ae1e12856631..3a698cfd1d408 100644 --- a/src/qt/forms/transactiondescdialog.ui +++ b/src/qt/forms/transactiondescdialog.ui @@ -11,7 +11,7 @@ - Transaction details + Transaction details diff --git a/src/qt/locale/bitcoin_en.ts b/src/qt/locale/bitcoin_en.ts index 00411741f1b9a..b90221f2c2535 100644 --- a/src/qt/locale/bitcoin_en.ts +++ b/src/qt/locale/bitcoin_en.ts @@ -165,7 +165,7 @@ Repeat new passphrase - + Encrypt wallet Encrypt wallet @@ -195,7 +195,7 @@ Change passphrase - + Confirm wallet encryption Confirm wallet encryption @@ -210,12 +210,7 @@ Are you sure you wish to encrypt your wallet? - - Bitcoin Core will close now to finish the encryption process. Remember that encrypting your wallet cannot fully protect your bitcoins from being stolen by malware infecting your computer. - - - - + IMPORTANT: Any previous backups you have made of your wallet file should be replaced with the newly generated, encrypted wallet file. For security reasons, previous backups of the unencrypted wallet file will become useless as soon as you start using the new, encrypted wallet. IMPORTANT: Any previous backups you have made of your wallet file should be replaced with the newly generated, encrypted wallet file. For security reasons, previous backups of the unencrypted wallet file will become useless as soon as you start using the new, encrypted wallet. @@ -232,7 +227,7 @@ Wallet encrypted - + Enter the new passphrase to the wallet.<br/>Please use a passphrase of <b>ten or more random characters</b>, or <b>eight or more words</b>. @@ -242,7 +237,12 @@ - + + %1 will close now to finish the encryption process. Remember that encrypting your wallet cannot fully protect your bitcoins from being stolen by malware infecting your computer. + + + + @@ -299,17 +299,17 @@ BitcoinGUI - + Sign &message... Sign &message... - + Synchronizing with network... Synchronizing with network... - + &Overview &Overview @@ -344,7 +344,17 @@ Quit application - + + &About %1 + + + + + Show information about %1 + + + + About &Qt About &Qt @@ -358,6 +368,11 @@ &Options... &Options... + + + Modify configuration options for %1 + + &Encrypt Wallet... @@ -374,7 +389,7 @@ &Change Passphrase... - + &Sending addresses... @@ -389,27 +404,17 @@ - - Bitcoin Core client - - - - - Importing blocks from disk... - Importing blocks from disk... - - - + Reindexing blocks on disk... Reindexing blocks on disk... - + Send coins to a Bitcoin address Send coins to a Bitcoin address - + Backup wallet to another location Backup wallet to another location @@ -434,12 +439,12 @@ &Verify message... - + Bitcoin Bitcoin - + Wallet Wallet @@ -454,12 +459,7 @@ &Receive - - Show information about Bitcoin Core - - - - + &Show / Hide &Show / Hide @@ -484,7 +484,7 @@ Verify messages to ensure they were signed with specified Bitcoin addresses - + &File &File @@ -504,27 +504,12 @@ Tabs toolbar - - Bitcoin Core - Bitcoin Core - - - + Request payments (generates QR codes and bitcoin: URIs) - - &About Bitcoin Core - - - - - Modify configuration options for Bitcoin Core - - - - + Show the list of used sending addresses and labels @@ -543,13 +528,8 @@ &Command-line options - - - Show the Bitcoin Core help message to get a list with possible Bitcoin command-line options - - - + %n active connection(s) to Bitcoin network %n active connection to Bitcoin network @@ -557,7 +537,17 @@ - + + Indexing blocks on disk... + + + + + Processing blocks on disk... + + + + No block source available... No block source available... @@ -643,12 +633,22 @@ Up to date - + + Show the %1 help message to get a list with possible Bitcoin command-line options + + + + + %1 client + + + + Catching up... Catching up... - + Date: %1 @@ -698,14 +698,6 @@ Wallet is <b>encrypted</b> and currently <b>locked</b> - - ClientModel - - - Network Alert - Network Alert - - CoinControlDialog @@ -870,7 +862,7 @@ - + highest @@ -935,12 +927,7 @@ - - This label turns red if any recipient receives an amount smaller than %1. - - - - + Can vary +/- %1 satoshi(s) per input. @@ -971,7 +958,12 @@ - + + This label turns red if any recipient receives an amount smaller than the current dust threshold. + + + + (no label) (no label) @@ -1058,7 +1050,7 @@ FreespaceChecker - + A new data directory will be created. A new data directory will be created. @@ -1086,12 +1078,7 @@ HelpMessageDialog - - Bitcoin Core - Bitcoin Core - - - + version version @@ -1103,7 +1090,7 @@ - About Bitcoin Core + About %1 @@ -1166,17 +1153,17 @@ - Welcome to Bitcoin Core. + Welcome to %1. - As this is the first time the program is launched, you can choose where Bitcoin Core will store its data. + As this is the first time the program is launched, you can choose where %1 will store its data. - Bitcoin Core will download and store a copy of the Bitcoin block chain. At least %1GB of data will be stored in this directory, and it will grow over time. The wallet will also be stored in this directory. + %1 will download and store a copy of the Bitcoin block chain. At least %2GB of data will be stored in this directory, and it will grow over time. The wallet will also be stored in this directory. @@ -1190,12 +1177,7 @@ Use a custom data directory: - - Bitcoin Core - Bitcoin Core - - - + Error: Specified data directory "%1" cannot be created. @@ -1262,7 +1244,17 @@ &Main - + + Automatically start %1 after logging in to the system. + + + + + &Start %1 on system login + + + + Size of &database cache @@ -1293,17 +1285,12 @@ - + Minimize instead of exit the application when the window is closed. When this option is enabled, the application will be closed only after selecting Exit in the menu. - - The user interface language can be set here. This setting will take effect after restarting Bitcoin Core. - - - - + Third party URLs (e.g. a block explorer) that appear in the transactions tab as context menu items. %s in the URL is replaced by transaction hash. Multiple URLs are separated by vertical bar |. @@ -1329,22 +1316,12 @@ &Reset Options - + &Network &Network - - Automatically start Bitcoin Core after logging in to the system. - - - - - &Start Bitcoin Core on system login - - - - + (0 = auto, <0 = leave that many cores free) @@ -1455,6 +1432,16 @@ + &Hide the icon from the system tray. + + + + + Hide tray icon + + + + Show only a tray icon after minimizing the window. Show only a tray icon after minimizing the window. @@ -1479,7 +1466,12 @@ User Interface &language: - + + The user interface language can be set here. This setting will take effect after restarting %1. + + + + &Unit to show amounts in: &Unit to show amounts in: @@ -1489,12 +1481,12 @@ Choose the default subdivision unit to show in the interface and when sending coins. - + Whether to show coin control features or not. - + &OK &OK @@ -1504,7 +1496,7 @@ &Cancel - + default default @@ -1514,23 +1506,23 @@ - + Confirm options reset Confirm options reset - + Client restart required to activate changes. - + Client will be shut down. Do you want to proceed? - + This change would require a client restart. @@ -1781,12 +1773,12 @@ QObject - + Amount Amount - + Enter a Bitcoin address (e.g. %1) @@ -1807,7 +1799,7 @@ - + %1 s @@ -1862,16 +1854,15 @@ - - - - + + - + + @@ -1892,7 +1883,7 @@ N/A - + Client version Client version @@ -1913,26 +1904,26 @@ - Using OpenSSL version - Using OpenSSL version + Using BerkeleyDB version + - Using BerkeleyDB version + Datadir - + Startup time Startup time - + Network Network - + Name @@ -1962,41 +1953,36 @@ - + Memory usage - - Open the Bitcoin Core debug log file from the current data directory. This can take a few seconds for large log files. - - - - - + + Received - - + + Sent - + &Peers - + Banned peers - - - + + + Select a peer to view detailed information. @@ -2031,13 +2017,28 @@ - - + + User Agent - + + Open the %1 debug log file from the current data directory. This can take a few seconds for large log files. + + + + + Decrease font size + + + + + Increase font size + + + + Services @@ -2082,12 +2083,12 @@ - + Last block time Last block time - + &Open &Open @@ -2097,7 +2098,7 @@ &Console - + &Network Traffic @@ -2112,7 +2113,7 @@ - + In: @@ -2122,22 +2123,17 @@ - - Build date - Build date - - - + Debug log file Debug log file - + Clear console Clear console - + &Disconnect Node @@ -2175,8 +2171,8 @@ - - Welcome to the Bitcoin Core RPC console. + + Welcome to the %1 RPC console. @@ -2190,7 +2186,7 @@ Type <b>help</b> for an overview of available commands. - + %1 B @@ -2444,9 +2440,14 @@ Message - - Amount - Amount + + (no amount requested) + + + + + Requested + @@ -2458,17 +2459,12 @@ (no message) - - - (no amount) - - SendCoinsDialog - + Send Coins Send Coins @@ -2615,17 +2611,7 @@ - - Send as zero-fee transaction if possible - - - - - (confirmation may take longer) - - - - + Send to multiple recipients at once Send to multiple recipients at once @@ -2640,12 +2626,12 @@ - + Dust: - + Clear &All Clear &All @@ -2678,7 +2664,7 @@ - + Copy quantity @@ -2713,7 +2699,7 @@ - + Total Amount %1 @@ -2758,12 +2744,12 @@ - + Pay only the required fee of %1 - + Estimated to begin confirmation within %n block(s). Estimated to begin confirmation within %n block. @@ -2796,12 +2782,12 @@ - + Copy dust - + Are you sure you want to send? @@ -2920,11 +2906,20 @@ + + SendConfirmationDialog + + + + Yes + + + ShutdownWindow - Bitcoin Core is shutting down... + %1 is shutting down... @@ -3117,16 +3112,6 @@ SplashScreen - - - Bitcoin Core - Bitcoin Core - - - - The Bitcoin Core developers - - [testnet] @@ -3149,14 +3134,29 @@ Open until %1 - - conflicted - + + %1/offline + %1/offline - %1/offline - %1/offline + 0/unconfirmed, %1 + + + + + in memory pool + + + + + not in memory pool + + + + + abandoned + @@ -3231,12 +3231,12 @@ - + Credit Credit - + matures in %n more block(s) matures in %n more block @@ -3251,12 +3251,12 @@ - + Debit Debit - + Total debit @@ -3277,12 +3277,12 @@ - + Message Message - + Comment Comment @@ -3291,6 +3291,11 @@ Transaction ID Transaction ID + + + Output index + + Merchant @@ -3334,12 +3339,12 @@ false - + , has not been successfully broadcast yet , has not been successfully broadcast yet - + Open for %n more block(s) Open for %n more block @@ -3347,7 +3352,12 @@ - + + conflicted with a transaction with %1 confirmations + + + + unknown unknown @@ -3355,15 +3365,15 @@ TransactionDescDialog - - Transaction details - Transaction details - - - + This pane shows a detailed description of the transaction This pane shows a detailed description of the transaction + + + Details for %1 + + TransactionTableModel @@ -3378,12 +3388,12 @@ Type - + Immature (%1 confirmations, will be available after %2) - + Open for %n more block(s) Open for %n more block @@ -3396,7 +3406,7 @@ Open until %1 - + Confirmed (%1 confirmations) Confirmed (%1 confirmations) @@ -3411,7 +3421,7 @@ Generated but not accepted - + Offline @@ -3425,6 +3435,11 @@ Unconfirmed + + + Abandoned + + Confirming (%1 of %2 recommended confirmations) @@ -3471,7 +3486,12 @@ (n/a) - + + (no label) + (no label) + + + Transaction status. Hover over this field to show number of confirmations. Transaction status. Hover over this field to show number of confirmations. @@ -3576,6 +3596,11 @@ + Abandon transaction + + + + Copy address Copy address @@ -3599,6 +3624,11 @@ Copy raw transaction + + + Copy full transaction details + + Edit label @@ -3610,7 +3640,7 @@ Show transaction details - + Export Transaction History @@ -3675,7 +3705,7 @@ ID - + Range: Range: @@ -3688,7 +3718,7 @@ UnitDisplayStatusBarControl - + Unit to show amounts in. Click to select another unit. @@ -3755,47 +3785,37 @@ bitcoin-core - + Options: Options: - + Specify data directory Specify data directory - + Connect to a node to retrieve peer addresses, and disconnect Connect to a node to retrieve peer addresses, and disconnect - + Specify your own public address Specify your own public address - + Accept command line and JSON-RPC commands Accept command line and JSON-RPC commands - + If <category> is not supplied or if <category> = 1, output all debugging information. - - Maximum total fees (in %s) to use in a single wallet transaction; setting this too low may abort large transactions (default: %s) - - - - - Please check that your computer's date and time are correct! If your clock is wrong Bitcoin Core will not work properly. - - - - + Prune configured below the minimum of %d MiB. Please use a higher number. @@ -3815,7 +3835,7 @@ - + Error: A fatal internal error occurred, see debug.log for details @@ -3825,7 +3845,7 @@ - + Pruning blockstore... @@ -3840,17 +3860,47 @@ - + Accept connections from outside (default: 1 if no -proxy or -connect) Accept connections from outside (default: 1 if no -proxy or -connect) - + + Bitcoin Core + Bitcoin Core + + + + The %s developers + + + + + -fallbackfee is set very high! This is the transaction fee you may pay when fee estimates are not available. + + + + + A fee rate (in %s/kB) that will be used when fee estimation has insufficient data (default: %s) + + + + + Accept relayed transactions received from whitelisted peers even when not relaying transactions (default: %d) + + + + Bind to given address and always listen on it. Use [host]:port notation for IPv6 Bind to given address and always listen on it. Use [host]:port notation for IPv6 - + + Cannot obtain a lock on data directory %s. %s is probably already running. + + + + Delete all wallet transactions and only recover those parts of the blockchain through -rescan on startup @@ -3860,17 +3910,42 @@ - + + Error reading %s! All keys read correctly, but transaction data or address book entries might be missing or incorrect. + + + + Execute command when a wallet transaction changes (%s in cmd is replaced by TxID) Execute command when a wallet transaction changes (%s in cmd is replaced by TxID) - + + Force relay of transactions from whitelisted peers even they violate local relay policy (default: %d) + + + + + Maximum allowed median peer time offset adjustment. Local perspective of time may be influenced by peers forward or backward by this amount. (default: %u seconds) + + + + + Maximum total fees (in %s) to use in a single wallet transaction or raw transaction; setting this too low may abort large transactions (default: %s) + + + + + Please check that your computer's date and time are correct! If your clock is wrong, %s will not work properly. + + + + Set the number of script verification threads (%u to %d, 0 = auto, <0 = leave that many cores free, default: %d) - + The block database contains a block which appears to be from the future. This may be due to your computer's date and time being set incorrectly. Only rebuild the block database if you are sure that your computer's date and time are correct @@ -3880,12 +3955,7 @@ This is a pre-release test build - use at your own risk - do not use for mining or merchant applications - - Unable to bind to %s on this computer. Bitcoin Core is probably already running. - - - - + Use UPnP to map the listening port (default: 1 when listening and no -proxy) @@ -3905,22 +3975,27 @@ Warning: The network does not appear to fully agree! Some miners appear to be experiencing issues. - + Warning: We do not appear to fully agree with our peers! You may need to upgrade, or other nodes may need to upgrade. Warning: We do not appear to fully agree with our peers! You may need to upgrade, or other nodes may need to upgrade. - Warning: wallet.dat corrupt, data salvaged! Original wallet.dat saved as wallet.{timestamp}.bak in %s; if your balance or transactions are incorrect you should restore from a backup. - Warning: wallet.dat corrupt, data salvaged! Original wallet.dat saved as wallet.{timestamp}.bak in %s; if your balance or transactions are incorrect you should restore from a backup. + Whitelist peers connecting from the given netmask or IP address. Can be specified multiple times. + - - Whitelist peers connecting from the given netmask or IP address. Can be specified multiple times. + + You need to rebuild the database using -reindex-chainstate to change -txindex - + + %s corrupt, salvage failed + + + + -maxmempool must be at least %d MB @@ -3930,12 +4005,32 @@ - + + Append comment to the user agent string + + + + + Attempt to recover private keys from a corrupt wallet on startup + + + + Block creation options: Block creation options: - + + Cannot resolve -%s address: '%s' + + + + + Change index out of range + + + + Connect only to the specified node(s) Connect only to the specified node(s) @@ -3945,7 +4040,12 @@ - + + Copyright (C) %i-%i + + + + Corrupted block database detected Corrupted block database detected @@ -3984,6 +4084,11 @@ Enable publish raw transaction in <address> + + + Enable transaction replacement in the memory pool (default: %u) + + Error initializing block database @@ -3994,13 +4099,28 @@ Error initializing wallet database environment %s! Error initializing wallet database environment %s! + + + Error loading %s + + + + + Error loading %s: Wallet corrupted + + + + + Error loading %s: Wallet requires newer version of %s + + Error loading block database Error loading block database - + Error opening block database Error opening block database @@ -4015,7 +4135,7 @@ Failed to listen on any port. Use -listen=0 if you want this. - + Importing... @@ -4025,17 +4145,47 @@ Incorrect or no genesis block found. Wrong datadir for network? - + + Initialization sanity check failed. %s is shutting down. + + + + Invalid -onion address: '%s' - + + Invalid amount for -%s=<amount>: '%s' + + + + + Invalid amount for -fallbackfee=<amount>: '%s' + + + + Keep the transaction memory pool below <n> megabytes (default: %u) - + + Loading banlist... + + + + + Location of the auth cookie (default: data dir) + + + + + Minimum bytes per sigop in transactions we relay and mine (default: %u) + + + + Not enough file descriptors available. Not enough file descriptors available. @@ -4046,6 +4196,16 @@ + Print this help message and exit + + + + + Print version and exit + + + + Prune cannot be configured with a negative value. @@ -4055,7 +4215,17 @@ - + + Rebuild chain state and block index from the blk*.dat files on disk + + + + + Rebuild chain state from the currently indexed blocks + + + + Set database cache size in megabytes (%d to %d, default: %d) @@ -4070,7 +4240,12 @@ Specify wallet file (within data directory) - + + Unable to bind to %s on this computer. %s is probably already running. + + + + Unsupported argument -benchmark ignored, use -debug=bench. @@ -4110,22 +4285,22 @@ Wallet %s resides outside data directory %s - - Wallet options: + + Wallet debugging/testing options: - - Warning: This version is obsolete; upgrade required! + + Wallet needed to be rewritten: restart %s to complete - - You need to rebuild the database using -reindex to change -txindex - You need to rebuild the database using -reindex to change -txindex + + Wallet options: + - + Allow JSON-RPC connections from specified source. Valid for <ip> are a single IP (e.g. 1.2.3.4), a network/netmask (e.g. 1.2.3.4/255.255.255.0) or a network/CIDR (e.g. 1.2.3.4/24). This option can be specified multiple times @@ -4140,12 +4315,7 @@ - - Cannot obtain a lock on data directory %s. Bitcoin Core is probably already running. - - - - + Create new files with system default permissions, instead of umask 077 (only effective with disabled wallet functionality) @@ -4170,7 +4340,7 @@ - + If paytxfee is not set, include enough fee so transactions begin confirmation on average within n blocks (default: %u) @@ -4180,7 +4350,7 @@ - + Maximum size of data in data carrier transactions we relay and mine (default: %u) @@ -4200,12 +4370,7 @@ - - Set the number of threads for coin generation if enabled (-1 = all cores, default: %d) - - - - + The transaction amount is too small to send after the fee has been deducted @@ -4215,7 +4380,7 @@ - + Whitelisted peers cannot be DoS banned and their transactions are always relayed, even if they are already in the mempool, useful e.g. for a gateway @@ -4225,7 +4390,7 @@ - + (default: %u) @@ -4235,52 +4400,22 @@ - - Activating best chain... - - - - - Always relay transactions received from whitelisted peers (default: %d) - - - - - Attempt to recover private keys from a corrupt wallet.dat on startup - - - - + Automatically create Tor hidden service (default: %d) - - Cannot resolve -whitebind address: '%s' - - - - + Connect through SOCKS5 proxy - - Copyright (C) 2009-%i The Bitcoin Core Developers - - - - - Error loading wallet.dat: Wallet requires newer version of Bitcoin Core - - - - + Error reading from database, shutting down. - + Imports blocks from external blk000??.dat file on startup @@ -4290,32 +4425,12 @@ Information - - Initialization sanity check failed. Bitcoin Core is shutting down. - - - - - Invalid amount for -maxtxfee=<amount>: '%s' - - - - - Invalid amount for -minrelaytxfee=<amount>: '%s' - Invalid amount for -minrelaytxfee=<amount>: '%s' - - - - Invalid amount for -mintxfee=<amount>: '%s' - Invalid amount for -mintxfee=<amount>: '%s' - - - + Invalid amount for -paytxfee=<amount>: '%s' (must be at least %s) - + Invalid netmask specified in -whitelist: '%s' @@ -4325,7 +4440,7 @@ - + Need to specify a port with -whitebind: '%s' @@ -4335,22 +4450,12 @@ - + RPC server options: - - Rebuild block chain index from current blk000??.dat files on startup - - - - - Receive and display P2P network alerts (default: %u) - - - - + Reducing -maxconnections from %d to %d, because of system limitations. @@ -4390,7 +4495,7 @@ - + This is experimental software. @@ -4430,7 +4535,7 @@ - + Upgrade wallet to latest format on startup @@ -4440,72 +4545,57 @@ Username for JSON-RPC connections - - Wallet needed to be rewritten: restart Bitcoin Core to complete - - - - + Warning Warning - - Whether to operate in a blocks only mode (default: %u) + + Warning: unknown new rules activated (versionbit %i) - - Zapping all transactions from wallet... + + Whether to operate in a blocks only mode (default: %u) - ZeroMQ notification options: + Zapping all transactions from wallet... - wallet.dat corrupt, salvage failed - wallet.dat corrupt, salvage failed + ZeroMQ notification options: + - + Password for JSON-RPC connections Password for JSON-RPC connections - + Execute command when the best block changes (%s in cmd is replaced by block hash) Execute command when the best block changes (%s in cmd is replaced by block hash) - - This help message - This help message - - - + Allow DNS lookups for -addnode, -seednode and -connect Allow DNS lookups for -addnode, -seednode and -connect - + Loading addresses... Loading addresses... - - Error loading wallet.dat: Wallet corrupted - Error loading wallet.dat: Wallet corrupted - - - + (1 = keep tx meta data e.g. account owner and payment request information, 2 = drop tx meta data) - + -maxtxfee is set very high! Fees this large could be paid on a single transaction. @@ -4515,22 +4605,17 @@ - + Do not keep transactions in the mempool longer than <n> hours (default: %u) - - Error reading wallet.dat! All keys read correctly, but transaction data or address book entries might be missing or incorrect. - - - - + Fees (in %s/kB) smaller than this are considered zero fee for transaction creation (default: %s) - + How thorough the block verification of -checkblocks is (0-4, default: %u) @@ -4540,7 +4625,7 @@ - + Number of seconds to keep misbehaving peers from reconnecting (default: %u) @@ -4550,7 +4635,7 @@ - + Support filtering of blocks and transaction with bloom filters (default: %u) @@ -4565,10 +4650,15 @@ - + Unsupported argument -socks found. Setting SOCKS version isn't possible anymore, only SOCKS5 proxies are supported. + + + Unsupported argument -whitelistalwaysrelay ignored, use -whitelistrelay and/or -whitelistforcerelay. + + Use separate SOCKS5 proxy to reach peers via Tor hidden services (default: %s) @@ -4580,27 +4670,27 @@ - - (default: %s) + + Warning: Unknown block versions being mined! It's possible unknown rules are in effect - - Always query for peer addresses via DNS lookup (default: %u) + + Warning: Wallet file corrupt, data salvaged! Original %s saved as %s in %s; if your balance or transactions are incorrect you should restore from a backup. - - Error loading wallet.dat - Error loading wallet.dat + + (default: %s) + - - Generate coins (default: %u) + + Always query for peer addresses via DNS lookup (default: %u) - + How many blocks to check at startup (default: %u, 0 = all) @@ -4615,7 +4705,7 @@ Invalid -proxy address: '%s' - + Listen for JSON-RPC connections on <port> (default: %u or testnet: %u) @@ -4625,7 +4715,7 @@ - + Maintain at most <n> connections to peers (default: %u) @@ -4645,12 +4735,12 @@ - + Prepend debug output with timestamp (default: %u) - + Relay and mine data carrier transactions (default: %u) @@ -4695,72 +4785,57 @@ - + Threshold for disconnecting misbehaving peers (default: %u) - + Unknown network specified in -onlynet: '%s' Unknown network specified in -onlynet: '%s' - - Cannot resolve -bind address: '%s' - Cannot resolve -bind address: '%s' - - - - Cannot resolve -externalip address: '%s' - Cannot resolve -externalip address: '%s' - - - - Invalid amount for -paytxfee=<amount>: '%s' - Invalid amount for -paytxfee=<amount>: '%s' - - - + Insufficient funds Insufficient funds - + Loading block index... Loading block index... - + Add a node to connect to and attempt to keep the connection open Add a node to connect to and attempt to keep the connection open - + Loading wallet... Loading wallet... - + Cannot downgrade wallet Cannot downgrade wallet - + Cannot write default address Cannot write default address - + Rescanning... Rescanning... - + Done loading Done loading - + Error Error diff --git a/src/qt/optionsdialog.cpp b/src/qt/optionsdialog.cpp index 95a3fa8d21a4d..f2db3988991a0 100644 --- a/src/qt/optionsdialog.cpp +++ b/src/qt/optionsdialog.cpp @@ -198,6 +198,7 @@ void OptionsDialog::setMapper() /* Window */ #ifndef Q_OS_MAC + mapper->addMapping(ui->hideTrayIcon, OptionsModel::HideTrayIcon); mapper->addMapping(ui->minimizeToTray, OptionsModel::MinimizeToTray); mapper->addMapping(ui->minimizeOnClose, OptionsModel::MinimizeOnClose); #endif @@ -243,6 +244,19 @@ void OptionsDialog::on_cancelButton_clicked() reject(); } +void OptionsDialog::on_hideTrayIcon_stateChanged(int fState) +{ + if(fState) + { + ui->minimizeToTray->setChecked(false); + ui->minimizeToTray->setEnabled(false); + } + else + { + ui->minimizeToTray->setEnabled(true); + } +} + void OptionsDialog::showRestartWarning(bool fPersistent) { ui->statusLabel->setStyleSheet("QLabel { color: red; }"); diff --git a/src/qt/optionsdialog.h b/src/qt/optionsdialog.h index e944fb9ee9b2a..41b56d1386257 100644 --- a/src/qt/optionsdialog.h +++ b/src/qt/optionsdialog.h @@ -49,6 +49,8 @@ private Q_SLOTS: void on_resetButton_clicked(); void on_okButton_clicked(); void on_cancelButton_clicked(); + + void on_hideTrayIcon_stateChanged(int fState); void showRestartWarning(bool fPersistent = false); void clearStatusLabel(); diff --git a/src/qt/optionsmodel.cpp b/src/qt/optionsmodel.cpp index d091bb9e610aa..cc2cbc0e66419 100644 --- a/src/qt/optionsmodel.cpp +++ b/src/qt/optionsmodel.cpp @@ -51,9 +51,14 @@ void OptionsModel::Init(bool resetSettings) // These are Qt-only settings: // Window + if (!settings.contains("fHideTrayIcon")) + settings.setValue("fHideTrayIcon", false); + fHideTrayIcon = settings.value("fHideTrayIcon").toBool(); + Q_EMIT hideTrayIconChanged(fHideTrayIcon); + if (!settings.contains("fMinimizeToTray")) settings.setValue("fMinimizeToTray", false); - fMinimizeToTray = settings.value("fMinimizeToTray").toBool(); + fMinimizeToTray = settings.value("fMinimizeToTray").toBool() && !fHideTrayIcon; if (!settings.contains("fMinimizeOnClose")) settings.setValue("fMinimizeOnClose", false); @@ -166,6 +171,8 @@ QVariant OptionsModel::data(const QModelIndex & index, int role) const { case StartAtStartup: return GUIUtil::GetStartOnSystemStartup(); + case HideTrayIcon: + return fHideTrayIcon; case MinimizeToTray: return fMinimizeToTray; case MapPortUPnP: @@ -242,6 +249,11 @@ bool OptionsModel::setData(const QModelIndex & index, const QVariant & value, in case StartAtStartup: successful = GUIUtil::SetStartOnSystemStartup(value.toBool()); break; + case HideTrayIcon: + fHideTrayIcon = value.toBool(); + settings.setValue("fHideTrayIcon", fHideTrayIcon); + Q_EMIT hideTrayIconChanged(fHideTrayIcon); + break; case MinimizeToTray: fMinimizeToTray = value.toBool(); settings.setValue("fMinimizeToTray", fMinimizeToTray); diff --git a/src/qt/optionsmodel.h b/src/qt/optionsmodel.h index 841711dd2d978..3b491ceac2cde 100644 --- a/src/qt/optionsmodel.h +++ b/src/qt/optionsmodel.h @@ -28,6 +28,7 @@ class OptionsModel : public QAbstractListModel enum OptionID { StartAtStartup, // bool + HideTrayIcon, // bool MinimizeToTray, // bool MapPortUPnP, // bool MinimizeOnClose, // bool @@ -58,6 +59,7 @@ class OptionsModel : public QAbstractListModel void setDisplayUnit(const QVariant &value); /* Explicit getters */ + bool getHideTrayIcon() { return fHideTrayIcon; } bool getMinimizeToTray() { return fMinimizeToTray; } bool getMinimizeOnClose() { return fMinimizeOnClose; } int getDisplayUnit() { return nDisplayUnit; } @@ -72,6 +74,7 @@ class OptionsModel : public QAbstractListModel private: /* Qt-only settings */ + bool fHideTrayIcon; bool fMinimizeToTray; bool fMinimizeOnClose; QString language; @@ -87,6 +90,7 @@ class OptionsModel : public QAbstractListModel Q_SIGNALS: void displayUnitChanged(int unit); void coinControlFeaturesChanged(bool); + void hideTrayIconChanged(bool); }; #endif // BITCOIN_QT_OPTIONSMODEL_H diff --git a/src/qt/overviewpage.cpp b/src/qt/overviewpage.cpp index d577345e49097..6a0404cbf702f 100644 --- a/src/qt/overviewpage.cpp +++ b/src/qt/overviewpage.cpp @@ -219,7 +219,7 @@ void OverviewPage::setWalletModel(WalletModel *model) filter->setDynamicSortFilter(true); filter->setSortRole(Qt::EditRole); filter->setShowInactive(false); - filter->sort(TransactionTableModel::Status, Qt::DescendingOrder); + filter->sort(TransactionTableModel::Date, Qt::DescendingOrder); ui->listTransactions->setModel(filter); ui->listTransactions->setModelColumn(TransactionTableModel::ToAddress); diff --git a/src/qt/rpcconsole.cpp b/src/qt/rpcconsole.cpp index d8647d902a0e7..11f3e49a06d9b 100644 --- a/src/qt/rpcconsole.cpp +++ b/src/qt/rpcconsole.cpp @@ -327,6 +327,14 @@ bool RPCConsole::eventFilter(QObject* obj, QEvent *event) return true; } break; + case Qt::Key_Return: + case Qt::Key_Enter: + // forward these events to lineEdit + if(obj == autoCompleter->popup()) { + QApplication::postEvent(ui->lineEdit, new QKeyEvent(*keyevt)); + return true; + } + break; default: // Typing in messages widget brings focus to line edit, and redirects key there // Exclude most combinations and keys that emit no text, except paste shortcuts @@ -353,8 +361,8 @@ void RPCConsole::setClientModel(ClientModel *model) setNumConnections(model->getNumConnections()); connect(model, SIGNAL(numConnectionsChanged(int)), this, SLOT(setNumConnections(int))); - setNumBlocks(model->getNumBlocks(), model->getLastBlockDate(), model->getVerificationProgress(NULL)); - connect(model, SIGNAL(numBlocksChanged(int,QDateTime,double)), this, SLOT(setNumBlocks(int,QDateTime,double))); + setNumBlocks(model->getNumBlocks(), model->getLastBlockDate(), model->getVerificationProgress(NULL), false); + connect(model, SIGNAL(numBlocksChanged(int,QDateTime,double,bool)), this, SLOT(setNumBlocks(int,QDateTime,double,bool))); updateTrafficStats(model->getTotalBytesRecv(), model->getTotalBytesSent()); connect(model, SIGNAL(bytesChanged(quint64,quint64)), this, SLOT(updateTrafficStats(quint64, quint64))); @@ -458,9 +466,7 @@ void RPCConsole::setClientModel(ClientModel *model) autoCompleter = new QCompleter(wordList, this); ui->lineEdit->setCompleter(autoCompleter); - - // clear the lineEdit after activating from QCompleter - connect(autoCompleter, SIGNAL(activated(const QString&)), ui->lineEdit, SLOT(clear()), Qt::QueuedConnection); + autoCompleter->popup()->installEventFilter(this); } } @@ -585,10 +591,12 @@ void RPCConsole::setNumConnections(int count) ui->numberOfConnections->setText(connections); } -void RPCConsole::setNumBlocks(int count, const QDateTime& blockDate, double nVerificationProgress) +void RPCConsole::setNumBlocks(int count, const QDateTime& blockDate, double nVerificationProgress, bool headers) { - ui->numberOfBlocks->setText(QString::number(count)); - ui->lastBlockTime->setText(blockDate.toString()); + if (!headers) { + ui->numberOfBlocks->setText(QString::number(count)); + ui->lastBlockTime->setText(blockDate.toString()); + } } void RPCConsole::setMempoolSize(long numberOfTxs, size_t dynUsage) @@ -885,15 +893,13 @@ void RPCConsole::banSelectedNode(int bantime) // Get currently selected peer address QString strNode = GUIUtil::getEntryData(ui->peerWidget, 0, PeerTableModel::Address); // Find possible nodes, ban it and clear the selected node - if (CNode *bannedNode = FindNode(strNode.toStdString())) { + if (FindNode(strNode.toStdString())) { std::string nStr = strNode.toStdString(); std::string addr; int port = 0; SplitHostPort(nStr, port, addr); CNode::Ban(CNetAddr(addr), BanReasonManuallyAdded, bantime); - bannedNode->fDisconnect = true; - DumpBanlist(); clearSelectedNode(); clientModel->getBanTableModel()->refresh(); @@ -912,7 +918,6 @@ void RPCConsole::unbanSelectedNode() if (possibleSubnet.IsValid()) { CNode::Unban(possibleSubnet); - DumpBanlist(); clientModel->getBanTableModel()->refresh(); } } diff --git a/src/qt/rpcconsole.h b/src/qt/rpcconsole.h index 2923587bc82c5..28affa954dca2 100644 --- a/src/qt/rpcconsole.h +++ b/src/qt/rpcconsole.h @@ -87,7 +87,7 @@ public Q_SLOTS: /** Set number of connections shown in the UI */ void setNumConnections(int count); /** Set number of blocks and last block date shown in the UI */ - void setNumBlocks(int count, const QDateTime& blockDate, double nVerificationProgress); + void setNumBlocks(int count, const QDateTime& blockDate, double nVerificationProgress, bool headers); /** Set size (number of transactions and memory usage) of the mempool in the UI */ void setMempoolSize(long numberOfTxs, size_t dynUsage); /** Go forward or back in history */ diff --git a/src/qt/sendcoinsdialog.cpp b/src/qt/sendcoinsdialog.cpp index 780a6c9709ed8..6d50be56ecba7 100644 --- a/src/qt/sendcoinsdialog.cpp +++ b/src/qt/sendcoinsdialog.cpp @@ -26,6 +26,9 @@ #include #include #include +#include + +#define SEND_CONFIRM_DELAY 3 SendCoinsDialog::SendCoinsDialog(const PlatformStyle *platformStyle, QWidget *parent) : QDialog(parent), @@ -121,7 +124,7 @@ void SendCoinsDialog::setClientModel(ClientModel *clientModel) this->clientModel = clientModel; if (clientModel) { - connect(clientModel, SIGNAL(numBlocksChanged(int,QDateTime,double)), this, SLOT(updateSmartFeeLabel())); + connect(clientModel, SIGNAL(numBlocksChanged(int,QDateTime,double,bool)), this, SLOT(updateSmartFeeLabel())); } } @@ -311,10 +314,10 @@ void SendCoinsDialog::on_sendButton_clicked() questionString.append(QString("
(=%2)") .arg(alternativeUnits.join(" " + tr("or") + "
"))); - QMessageBox::StandardButton retval = QMessageBox::question(this, tr("Confirm send coins"), - questionString.arg(formatted.join("
")), - QMessageBox::Yes | QMessageBox::Cancel, - QMessageBox::Cancel); + SendConfirmationDialog confirmationDialog(tr("Confirm send coins"), + questionString.arg(formatted.join("
")), SEND_CONFIRM_DELAY, this); + confirmationDialog.exec(); + QMessageBox::StandardButton retval = (QMessageBox::StandardButton)confirmationDialog.result(); if(retval != QMessageBox::Yes) { @@ -828,3 +831,45 @@ void SendCoinsDialog::coinControlUpdateLabels() ui->labelCoinControlInsuffFunds->hide(); } } + +SendConfirmationDialog::SendConfirmationDialog(const QString &title, const QString &text, int secDelay, + QWidget *parent) : + QMessageBox(QMessageBox::Question, title, text, QMessageBox::Yes | QMessageBox::Cancel, parent), secDelay(secDelay) +{ + setDefaultButton(QMessageBox::Cancel); + yesButton = button(QMessageBox::Yes); + updateYesButton(); + connect(&countDownTimer, SIGNAL(timeout()), this, SLOT(countDown())); +} + +int SendConfirmationDialog::exec() +{ + updateYesButton(); + countDownTimer.start(1000); + return QMessageBox::exec(); +} + +void SendConfirmationDialog::countDown() +{ + secDelay--; + updateYesButton(); + + if(secDelay <= 0) + { + countDownTimer.stop(); + } +} + +void SendConfirmationDialog::updateYesButton() +{ + if(secDelay > 0) + { + yesButton->setEnabled(false); + yesButton->setText(tr("Yes") + " (" + QString::number(secDelay) + ")"); + } + else + { + yesButton->setEnabled(true); + yesButton->setText(tr("Yes")); + } +} diff --git a/src/qt/sendcoinsdialog.h b/src/qt/sendcoinsdialog.h index ec171734faa20..be4f2ee44ba03 100644 --- a/src/qt/sendcoinsdialog.h +++ b/src/qt/sendcoinsdialog.h @@ -8,7 +8,9 @@ #include "walletmodel.h" #include +#include #include +#include class ClientModel; class OptionsModel; @@ -100,4 +102,24 @@ private Q_SLOTS: void message(const QString &title, const QString &message, unsigned int style); }; + + +class SendConfirmationDialog : public QMessageBox +{ + Q_OBJECT + +public: + SendConfirmationDialog(const QString &title, const QString &text, int secDelay = 0, QWidget *parent = 0); + int exec(); + +private Q_SLOTS: + void countDown(); + void updateYesButton(); + +private: + QAbstractButton *yesButton; + QTimer countDownTimer; + int secDelay; +}; + #endif // BITCOIN_QT_SENDCOINSDIALOG_H diff --git a/src/qt/transactiondescdialog.cpp b/src/qt/transactiondescdialog.cpp index f7b6995b239c3..65adcc4f023e3 100644 --- a/src/qt/transactiondescdialog.cpp +++ b/src/qt/transactiondescdialog.cpp @@ -14,6 +14,7 @@ TransactionDescDialog::TransactionDescDialog(const QModelIndex &idx, QWidget *pa ui(new Ui::TransactionDescDialog) { ui->setupUi(this); + setWindowTitle(tr("Details for %1").arg(idx.data(TransactionTableModel::TxIDRole).toString())); QString desc = idx.data(TransactionTableModel::LongDescriptionRole).toString(); ui->detailText->setHtml(desc); } diff --git a/src/qt/transactionview.cpp b/src/qt/transactionview.cpp index a352228c36fb5..199a7b2d77b1b 100644 --- a/src/qt/transactionview.cpp +++ b/src/qt/transactionview.cpp @@ -203,7 +203,7 @@ void TransactionView::setModel(WalletModel *model) transactionView->setSelectionBehavior(QAbstractItemView::SelectRows); transactionView->setSelectionMode(QAbstractItemView::ExtendedSelection); transactionView->setSortingEnabled(true); - transactionView->sortByColumn(TransactionTableModel::Status, Qt::DescendingOrder); + transactionView->sortByColumn(TransactionTableModel::Date, Qt::DescendingOrder); transactionView->verticalHeader()->hide(); transactionView->setColumnWidth(TransactionTableModel::Status, STATUS_COLUMN_WIDTH); @@ -478,8 +478,9 @@ void TransactionView::showDetails() QModelIndexList selection = transactionView->selectionModel()->selectedRows(); if(!selection.isEmpty()) { - TransactionDescDialog dlg(selection.at(0)); - dlg.exec(); + TransactionDescDialog *dlg = new TransactionDescDialog(selection.at(0)); + dlg->setAttribute(Qt::WA_DeleteOnClose); + dlg->show(); } } diff --git a/src/qt/walletmodel.cpp b/src/qt/walletmodel.cpp index ce230d6aedbce..3867310cd6857 100644 --- a/src/qt/walletmodel.cpp +++ b/src/qt/walletmodel.cpp @@ -447,7 +447,7 @@ bool WalletModel::changePassphrase(const SecureString &oldPass, const SecureStri bool WalletModel::backupWallet(const QString &filename) { - return BackupWallet(*wallet, filename.toLocal8Bit().data()); + return wallet->BackupWallet(filename.toLocal8Bit().data()); } // Handlers for core signals @@ -572,7 +572,7 @@ void WalletModel::getOutputs(const std::vector& vOutpoints, std::vect if (!wallet->mapWallet.count(outpoint.hash)) continue; int nDepth = wallet->mapWallet[outpoint.hash].GetDepthInMainChain(); if (nDepth < 0) continue; - COutput out(&wallet->mapWallet[outpoint.hash], outpoint.n, nDepth, true); + COutput out(&wallet->mapWallet[outpoint.hash], outpoint.n, nDepth, true, true); vOutputs.push_back(out); } } @@ -599,7 +599,7 @@ void WalletModel::listCoins(std::map >& mapCoins) if (!wallet->mapWallet.count(outpoint.hash)) continue; int nDepth = wallet->mapWallet[outpoint.hash].GetDepthInMainChain(); if (nDepth < 0) continue; - COutput out(&wallet->mapWallet[outpoint.hash], outpoint.n, nDepth, true); + COutput out(&wallet->mapWallet[outpoint.hash], outpoint.n, nDepth, true, true); if (outpoint.n < out.tx->vout.size() && wallet->IsMine(out.tx->vout[outpoint.n]) == ISMINE_SPENDABLE) vCoins.push_back(out); } @@ -611,7 +611,7 @@ void WalletModel::listCoins(std::map >& mapCoins) while (wallet->IsChange(cout.tx->vout[cout.i]) && cout.tx->vin.size() > 0 && wallet->IsMine(cout.tx->vin[0])) { if (!wallet->mapWallet.count(cout.tx->vin[0].prevout.hash)) break; - cout = COutput(&wallet->mapWallet[cout.tx->vin[0].prevout.hash], cout.tx->vin[0].prevout.n, 0, true); + cout = COutput(&wallet->mapWallet[cout.tx->vin[0].prevout.hash], cout.tx->vin[0].prevout.n, 0, true, true); } CTxDestination address; diff --git a/src/random.cpp b/src/random.cpp index 6155c0d8cf23a..d9a8cc145ea91 100644 --- a/src/random.cpp +++ b/src/random.cpp @@ -5,14 +5,17 @@ #include "random.h" +#include "crypto/sha512.h" #include "support/cleanse.h" #ifdef WIN32 #include "compat.h" // for Windows API +#include #endif #include "serialize.h" // for begin_ptr(vec) #include "util.h" // for LogPrint() #include "utilstrencodings.h" // for GetTime() +#include #include #ifndef WIN32 @@ -22,6 +25,12 @@ #include #include +static void RandFailure() +{ + LogPrintf("Failed to read randomness, aborting\n"); + abort(); +} + static inline int64_t GetPerformanceCounter() { int64_t nCounter = 0; @@ -43,7 +52,7 @@ void RandAddSeed() memory_cleanse((void*)&nCounter, sizeof(nCounter)); } -void RandAddSeedPerfmon() +static void RandAddSeedPerfmon() { RandAddSeed(); @@ -83,14 +92,65 @@ void RandAddSeedPerfmon() #endif } +/** Get 32 bytes of system entropy. */ +static void GetOSRand(unsigned char *ent32) +{ +#ifdef WIN32 + HCRYPTPROV hProvider; + int ret = CryptAcquireContextW(&hProvider, NULL, NULL, PROV_RSA_FULL, CRYPT_VERIFYCONTEXT); + if (!ret) { + RandFailure(); + } + ret = CryptGenRandom(hProvider, 32, ent32); + if (!ret) { + RandFailure(); + } + CryptReleaseContext(hProvider, 0); +#else + int f = open("/dev/urandom", O_RDONLY); + if (f == -1) { + RandFailure(); + } + int have = 0; + do { + ssize_t n = read(f, ent32 + have, 32 - have); + if (n <= 0 || n + have > 32) { + RandFailure(); + } + have += n; + } while (have < 32); + close(f); +#endif +} + void GetRandBytes(unsigned char* buf, int num) { if (RAND_bytes(buf, num) != 1) { - LogPrintf("%s: OpenSSL RAND_bytes() failed with error: %s\n", __func__, ERR_error_string(ERR_get_error(), NULL)); - assert(false); + RandFailure(); } } +void GetStrongRandBytes(unsigned char* out, int num) +{ + assert(num <= 32); + CSHA512 hasher; + unsigned char buf[64]; + + // First source: OpenSSL's RNG + RandAddSeedPerfmon(); + GetRandBytes(buf, 32); + hasher.Write(buf, 32); + + // Second source: OS RNG + GetOSRand(buf); + hasher.Write(buf, 32); + + // Produce output + hasher.Finalize(buf); + memcpy(out, buf, num); + memory_cleanse(buf, 64); +} + uint64_t GetRand(uint64_t nMax) { if (nMax == 0) diff --git a/src/random.h b/src/random.h index 1a2d3e8ee24ca..31b80bd565bdc 100644 --- a/src/random.h +++ b/src/random.h @@ -10,11 +10,8 @@ #include -/** - * Seed OpenSSL PRNG with additional entropy data - */ +/* Seed OpenSSL PRNG with additional entropy data */ void RandAddSeed(); -void RandAddSeedPerfmon(); /** * Functions to gather random data via the OpenSSL PRNG @@ -24,6 +21,12 @@ uint64_t GetRand(uint64_t nMax); int GetRandInt(int nMax); uint256 GetRandHash(); +/** + * Function to gather random data from multiple sources, failing whenever any + * of those source fail to provide a result. + */ +void GetStrongRandBytes(unsigned char* buf, int num); + /** * Seed insecure_rand using the random pool. * @param Deterministic Use a deterministic seed diff --git a/src/rpc/blockchain.cpp b/src/rpc/blockchain.cpp index 1beb332a36f2d..f4c5fb3f3e710 100644 --- a/src/rpc/blockchain.cpp +++ b/src/rpc/blockchain.cpp @@ -185,6 +185,60 @@ UniValue getdifficulty(const UniValue& params, bool fHelp) return GetDifficulty(); } +std::string EntryDescriptionString() +{ + return " \"size\" : n, (numeric) transaction size in bytes\n" + " \"fee\" : n, (numeric) transaction fee in " + CURRENCY_UNIT + "\n" + " \"modifiedfee\" : n, (numeric) transaction fee with fee deltas used for mining priority\n" + " \"time\" : n, (numeric) local time transaction entered pool in seconds since 1 Jan 1970 GMT\n" + " \"height\" : n, (numeric) block height when transaction entered pool\n" + " \"startingpriority\" : n, (numeric) priority when transaction entered pool\n" + " \"currentpriority\" : n, (numeric) transaction priority now\n" + " \"descendantcount\" : n, (numeric) number of in-mempool descendant transactions (including this one)\n" + " \"descendantsize\" : n, (numeric) size of in-mempool descendants (including this one)\n" + " \"descendantfees\" : n, (numeric) modified fees (see above) of in-mempool descendants (including this one)\n" + " \"ancestorcount\" : n, (numeric) number of in-mempool ancestor transactions (including this one)\n" + " \"ancestorsize\" : n, (numeric) size of in-mempool ancestors (including this one)\n" + " \"ancestorfees\" : n, (numeric) modified fees (see above) of in-mempool ancestors (including this one)\n" + " \"depends\" : [ (array) unconfirmed transactions used as inputs for this transaction\n" + " \"transactionid\", (string) parent transaction id\n" + " ... ]\n"; +} + +void entryToJSON(UniValue &info, const CTxMemPoolEntry &e) +{ + AssertLockHeld(mempool.cs); + + info.push_back(Pair("size", (int)e.GetTxSize())); + info.push_back(Pair("fee", ValueFromAmount(e.GetFee()))); + info.push_back(Pair("modifiedfee", ValueFromAmount(e.GetModifiedFee()))); + info.push_back(Pair("time", e.GetTime())); + info.push_back(Pair("height", (int)e.GetHeight())); + info.push_back(Pair("startingpriority", e.GetPriority(e.GetHeight()))); + info.push_back(Pair("currentpriority", e.GetPriority(chainActive.Height()))); + info.push_back(Pair("descendantcount", e.GetCountWithDescendants())); + info.push_back(Pair("descendantsize", e.GetSizeWithDescendants())); + info.push_back(Pair("descendantfees", e.GetModFeesWithDescendants())); + info.push_back(Pair("ancestorcount", e.GetCountWithAncestors())); + info.push_back(Pair("ancestorsize", e.GetSizeWithAncestors())); + info.push_back(Pair("ancestorfees", e.GetModFeesWithAncestors())); + const CTransaction& tx = e.GetTx(); + set setDepends; + BOOST_FOREACH(const CTxIn& txin, tx.vin) + { + if (mempool.exists(txin.prevout.hash)) + setDepends.insert(txin.prevout.hash.ToString()); + } + + UniValue depends(UniValue::VARR); + BOOST_FOREACH(const string& dep, setDepends) + { + depends.push_back(dep); + } + + info.push_back(Pair("depends", depends)); +} + UniValue mempoolToJSON(bool fVerbose = false) { if (fVerbose) @@ -195,31 +249,7 @@ UniValue mempoolToJSON(bool fVerbose = false) { const uint256& hash = e.GetTx().GetHash(); UniValue info(UniValue::VOBJ); - info.push_back(Pair("size", (int)e.GetTxSize())); - info.push_back(Pair("fee", ValueFromAmount(e.GetFee()))); - info.push_back(Pair("modifiedfee", ValueFromAmount(e.GetModifiedFee()))); - info.push_back(Pair("time", e.GetTime())); - info.push_back(Pair("height", (int)e.GetHeight())); - info.push_back(Pair("startingpriority", e.GetPriority(e.GetHeight()))); - info.push_back(Pair("currentpriority", e.GetPriority(chainActive.Height()))); - info.push_back(Pair("descendantcount", e.GetCountWithDescendants())); - info.push_back(Pair("descendantsize", e.GetSizeWithDescendants())); - info.push_back(Pair("descendantfees", e.GetModFeesWithDescendants())); - const CTransaction& tx = e.GetTx(); - set setDepends; - BOOST_FOREACH(const CTxIn& txin, tx.vin) - { - if (mempool.exists(txin.prevout.hash)) - setDepends.insert(txin.prevout.hash.ToString()); - } - - UniValue depends(UniValue::VARR); - BOOST_FOREACH(const string& dep, setDepends) - { - depends.push_back(dep); - } - - info.push_back(Pair("depends", depends)); + entryToJSON(info, e); o.push_back(Pair(hash.ToString(), info)); } return o; @@ -253,20 +283,8 @@ UniValue getrawmempool(const UniValue& params, bool fHelp) "\nResult: (for verbose = true):\n" "{ (json object)\n" " \"transactionid\" : { (json object)\n" - " \"size\" : n, (numeric) transaction size in bytes\n" - " \"fee\" : n, (numeric) transaction fee in " + CURRENCY_UNIT + "\n" - " \"modifiedfee\" : n, (numeric) transaction fee with fee deltas used for mining priority\n" - " \"time\" : n, (numeric) local time transaction entered pool in seconds since 1 Jan 1970 GMT\n" - " \"height\" : n, (numeric) block height when transaction entered pool\n" - " \"startingpriority\" : n, (numeric) priority when transaction entered pool\n" - " \"currentpriority\" : n, (numeric) transaction priority now\n" - " \"descendantcount\" : n, (numeric) number of in-mempool descendant transactions (including this one)\n" - " \"descendantsize\" : n, (numeric) size of in-mempool descendants (including this one)\n" - " \"descendantfees\" : n, (numeric) modified fees (see above) of in-mempool descendants (including this one)\n" - " \"depends\" : [ (array) unconfirmed transactions used as inputs for this transaction\n" - " \"transactionid\", (string) parent transaction id\n" - " ... ]\n" - " }, ...\n" + + EntryDescriptionString() + + " }, ...\n" "}\n" "\nExamples\n" + HelpExampleCli("getrawmempool", "true") @@ -282,6 +300,167 @@ UniValue getrawmempool(const UniValue& params, bool fHelp) return mempoolToJSON(fVerbose); } +UniValue getmempoolancestors(const UniValue& params, bool fHelp) +{ + if (fHelp || params.size() < 1 || params.size() > 2) { + throw runtime_error( + "getmempoolancestors txid (verbose)\n" + "\nIf txid is in the mempool, returns all in-mempool ancestors.\n" + "\nArguments:\n" + "1. \"txid\" (string, required) The transaction id (must be in mempool)\n" + "2. verbose (boolean, optional, default=false) true for a json object, false for array of transaction ids\n" + "\nResult (for verbose=false):\n" + "[ (json array of strings)\n" + " \"transactionid\" (string) The transaction id of an in-mempool ancestor transaction\n" + " ,...\n" + "]\n" + "\nResult (for verbose=true):\n" + "{ (json object)\n" + " \"transactionid\" : { (json object)\n" + + EntryDescriptionString() + + " }, ...\n" + "}\n" + "\nExamples\n" + + HelpExampleCli("getmempoolancestors", "\"mytxid\"") + + HelpExampleRpc("getmempoolancestors", "\"mytxid\"") + ); + } + + bool fVerbose = false; + if (params.size() > 1) + fVerbose = params[1].get_bool(); + + uint256 hash = ParseHashV(params[0], "parameter 1"); + + LOCK(mempool.cs); + + CTxMemPool::txiter it = mempool.mapTx.find(hash); + if (it == mempool.mapTx.end()) { + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Transaction not in mempool"); + } + + CTxMemPool::setEntries setAncestors; + uint64_t noLimit = std::numeric_limits::max(); + std::string dummy; + mempool.CalculateMemPoolAncestors(*it, setAncestors, noLimit, noLimit, noLimit, noLimit, dummy, false); + + if (!fVerbose) { + UniValue o(UniValue::VARR); + BOOST_FOREACH(CTxMemPool::txiter ancestorIt, setAncestors) { + o.push_back(ancestorIt->GetTx().GetHash().ToString()); + } + + return o; + } else { + UniValue o(UniValue::VOBJ); + BOOST_FOREACH(CTxMemPool::txiter ancestorIt, setAncestors) { + const CTxMemPoolEntry &e = *ancestorIt; + const uint256& hash = e.GetTx().GetHash(); + UniValue info(UniValue::VOBJ); + entryToJSON(info, e); + o.push_back(Pair(hash.ToString(), info)); + } + return o; + } +} + +UniValue getmempooldescendants(const UniValue& params, bool fHelp) +{ + if (fHelp || params.size() < 1 || params.size() > 2) { + throw runtime_error( + "getmempooldescendants txid (verbose)\n" + "\nIf txid is in the mempool, returns all in-mempool descendants.\n" + "\nArguments:\n" + "1. \"txid\" (string, required) The transaction id (must be in mempool)\n" + "2. verbose (boolean, optional, default=false) true for a json object, false for array of transaction ids\n" + "\nResult (for verbose=false):\n" + "[ (json array of strings)\n" + " \"transactionid\" (string) The transaction id of an in-mempool descendant transaction\n" + " ,...\n" + "]\n" + "\nResult (for verbose=true):\n" + "{ (json object)\n" + " \"transactionid\" : { (json object)\n" + + EntryDescriptionString() + + " }, ...\n" + "}\n" + "\nExamples\n" + + HelpExampleCli("getmempooldescendants", "\"mytxid\"") + + HelpExampleRpc("getmempooldescendants", "\"mytxid\"") + ); + } + + bool fVerbose = false; + if (params.size() > 1) + fVerbose = params[1].get_bool(); + + uint256 hash = ParseHashV(params[0], "parameter 1"); + + LOCK(mempool.cs); + + CTxMemPool::txiter it = mempool.mapTx.find(hash); + if (it == mempool.mapTx.end()) { + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Transaction not in mempool"); + } + + CTxMemPool::setEntries setDescendants; + mempool.CalculateDescendants(it, setDescendants); + // CTxMemPool::CalculateDescendants will include the given tx + setDescendants.erase(it); + + if (!fVerbose) { + UniValue o(UniValue::VARR); + BOOST_FOREACH(CTxMemPool::txiter descendantIt, setDescendants) { + o.push_back(descendantIt->GetTx().GetHash().ToString()); + } + + return o; + } else { + UniValue o(UniValue::VOBJ); + BOOST_FOREACH(CTxMemPool::txiter descendantIt, setDescendants) { + const CTxMemPoolEntry &e = *descendantIt; + const uint256& hash = e.GetTx().GetHash(); + UniValue info(UniValue::VOBJ); + entryToJSON(info, e); + o.push_back(Pair(hash.ToString(), info)); + } + return o; + } +} + +UniValue getmempoolentry(const UniValue& params, bool fHelp) +{ + if (fHelp || params.size() != 1) { + throw runtime_error( + "getmempoolentry txid\n" + "\nReturns mempool data for given transaction\n" + "\nArguments:\n" + "1. \"txid\" (string, required) The transaction id (must be in mempool)\n" + "\nResult:\n" + "{ (json object)\n" + + EntryDescriptionString() + + "}\n" + "\nExamples\n" + + HelpExampleCli("getmempoolentry", "\"mytxid\"") + + HelpExampleRpc("getmempoolentry", "\"mytxid\"") + ); + } + + uint256 hash = ParseHashV(params[0], "parameter 1"); + + LOCK(mempool.cs); + + CTxMemPool::txiter it = mempool.mapTx.find(hash); + if (it == mempool.mapTx.end()) { + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Transaction not in mempool"); + } + + const CTxMemPoolEntry &e = *it; + UniValue info(UniValue::VOBJ); + entryToJSON(info, e); + return info; +} + UniValue getblockhash(const UniValue& params, bool fHelp) { if (fHelp || params.size() != 1) @@ -947,7 +1126,7 @@ UniValue invalidateblock(const UniValue& params, bool fHelp) throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Block not found"); CBlockIndex* pblockindex = mapBlockIndex[hash]; - InvalidateBlock(state, Params().GetConsensus(), pblockindex); + InvalidateBlock(state, Params(), pblockindex); } if (state.IsValid()) { @@ -978,7 +1157,6 @@ UniValue reconsiderblock(const UniValue& params, bool fHelp) std::string strHash = params[0].get_str(); uint256 hash(uint256S(strHash)); - CValidationState state; { LOCK(cs_main); @@ -986,12 +1164,11 @@ UniValue reconsiderblock(const UniValue& params, bool fHelp) throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Block not found"); CBlockIndex* pblockindex = mapBlockIndex[hash]; - ReconsiderBlock(state, pblockindex); + ResetBlockFailureFlags(pblockindex); } - if (state.IsValid()) { - ActivateBestChain(state, Params()); - } + CValidationState state; + ActivateBestChain(state, Params()); if (!state.IsValid()) { throw JSONRPCError(RPC_DATABASE_ERROR, state.GetRejectReason()); @@ -1011,6 +1188,9 @@ static const CRPCCommand commands[] = { "blockchain", "getblockheader", &getblockheader, true }, { "blockchain", "getchaintips", &getchaintips, true }, { "blockchain", "getdifficulty", &getdifficulty, true }, + { "blockchain", "getmempoolancestors", &getmempoolancestors, true }, + { "blockchain", "getmempooldescendants", &getmempooldescendants, true }, + { "blockchain", "getmempoolentry", &getmempoolentry, true }, { "blockchain", "getmempoolinfo", &getmempoolinfo, true }, { "blockchain", "getrawmempool", &getrawmempool, true }, { "blockchain", "gettxout", &gettxout, true }, diff --git a/src/rpc/client.cpp b/src/rpc/client.cpp index c89af6bfa7c52..d0675fdb49506 100644 --- a/src/rpc/client.cpp +++ b/src/rpc/client.cpp @@ -102,6 +102,8 @@ static const CRPCConvertParam vRPCConvertParams[] = { "prioritisetransaction", 2 }, { "setban", 2 }, { "setban", 3 }, + { "getmempoolancestors", 1 }, + { "getmempooldescendants", 1 }, }; class CRPCConvertTable diff --git a/src/rpc/mining.cpp b/src/rpc/mining.cpp index 43d455b098b89..6ec90e70a1585 100644 --- a/src/rpc/mining.cpp +++ b/src/rpc/mining.cpp @@ -8,6 +8,7 @@ #include "chain.h" #include "chainparams.h" #include "consensus/consensus.h" +#include "consensus/params.h" #include "consensus/validation.h" #include "core_io.h" #include "init.h" @@ -111,7 +112,7 @@ UniValue generateBlocks(boost::shared_ptr coinbaseScript, int nG UniValue blockHashes(UniValue::VARR); while (nHeight < nHeightEnd) { - auto_ptr pblocktemplate(CreateNewBlock(Params(), coinbaseScript->reserveScript)); + std::unique_ptr pblocktemplate(BlockAssembler(Params()).CreateNewBlock(coinbaseScript->reserveScript)); if (!pblocktemplate.get()) throw JSONRPCError(RPC_INTERNAL_ERROR, "Couldn't create new block"); CBlock *pblock = &pblocktemplate->block; @@ -305,6 +306,15 @@ static UniValue BIP22ValidationResult(const CValidationState& state) return "valid?"; } +std::string gbt_vb_name(const Consensus::DeploymentPos pos) { + const struct BIP9DeploymentInfo& vbinfo = VersionBitsDeploymentInfo[pos]; + std::string s = vbinfo.name; + if (!vbinfo.gbt_force) { + s.insert(s.begin(), '!'); + } + return s; +} + UniValue getblocktemplate(const UniValue& params, bool fHelp) { if (fHelp || params.size() > 1) @@ -312,9 +322,9 @@ UniValue getblocktemplate(const UniValue& params, bool fHelp) "getblocktemplate ( \"jsonrequestobject\" )\n" "\nIf the request parameters include a 'mode' key, that is used to explicitly select between the default 'template' request or a 'proposal'.\n" "It returns data needed to construct a block to work on.\n" - "For full specification, see BIP 22 and BIP 145:\n" + "For full specification, see BIPs 22 and 9:\n" " https://github.com/bitcoin/bips/blob/master/bip-0022.mediawiki\n" - " https://github.com/bitcoin/bips/blob/master/bip-0145.mediawiki\n" + " https://github.com/bitcoin/bips/blob/master/bip-0009.mediawiki#getblocktemplate_changes\n" "\nArguments:\n" "1. \"jsonrequestobject\" (string, optional) A json object in the following spec\n" @@ -330,6 +340,12 @@ UniValue getblocktemplate(const UniValue& params, bool fHelp) "\nResult:\n" "{\n" " \"version\" : n, (numeric) The block version\n" + " \"rules\" : [ \"rulename\", ... ], (array of strings) specific block rules that are to be enforced\n" + " \"vbavailable\" : { (json object) set of pending, supported versionbit (BIP 9) softfork deployments\n" + " \"rulename\" : bitnumber (numeric) identifies the bit number as indicating acceptance and readiness for the named softfork rule\n" + " ,...\n" + " },\n" + " \"vbrequired\" : n, (numeric) bit mask of versionbits the server requires set in submissions\n" " \"previousblockhash\" : \"xxxx\", (string) The hash of current highest block\n" " \"transactions\" : [ (array) contents of non-coinbase transactions that should be included in the next block\n" " {\n" @@ -376,6 +392,8 @@ UniValue getblocktemplate(const UniValue& params, bool fHelp) std::string strMode = "template"; UniValue lpval = NullUniValue; + std::set setClientRules; + int64_t nMaxVersionPreVB = -1; if (params.size() > 0) { const UniValue& oparam = params[0].get_obj(); @@ -419,6 +437,20 @@ UniValue getblocktemplate(const UniValue& params, bool fHelp) TestBlockValidity(state, Params(), block, pindexPrev, false, true); return BIP22ValidationResult(state); } + + const UniValue& aClientRules = find_value(oparam, "rules"); + if (aClientRules.isArray()) { + for (unsigned int i = 0; i < aClientRules.size(); ++i) { + const UniValue& v = aClientRules[i]; + setClientRules.insert(v.get_str()); + } + } else { + // NOTE: It is important that this NOT be read if versionbits is supported + const UniValue& uvMaxVersion = find_value(oparam, "maxversion"); + if (uvMaxVersion.isNum()) { + nMaxVersionPreVB = uvMaxVersion.get_int64(); + } + } } if (strMode != "template") @@ -500,7 +532,7 @@ UniValue getblocktemplate(const UniValue& params, bool fHelp) pblocktemplate = NULL; } CScript scriptDummy = CScript() << OP_TRUE; - pblocktemplate = CreateNewBlock(Params(), scriptDummy); + pblocktemplate = BlockAssembler(Params()).CreateNewBlock(scriptDummy); if (!pblocktemplate) throw JSONRPCError(RPC_OUT_OF_MEMORY, "Out of memory"); @@ -508,9 +540,10 @@ UniValue getblocktemplate(const UniValue& params, bool fHelp) pindexPrev = pindexPrevNew; } CBlock* pblock = &pblocktemplate->block; // pointer for convenience + const Consensus::Params& consensusParams = Params().GetConsensus(); // Update nTime - UpdateTime(pblock, Params().GetConsensus(), pindexPrev); + UpdateTime(pblock, consensusParams, pindexPrev); pblock->nNonce = 0; UniValue aCaps(UniValue::VARR); aCaps.push_back("proposal"); @@ -552,17 +585,69 @@ UniValue getblocktemplate(const UniValue& params, bool fHelp) arith_uint256 hashTarget = arith_uint256().SetCompact(pblock->nBits); - static UniValue aMutable(UniValue::VARR); - if (aMutable.empty()) - { - aMutable.push_back("time"); - aMutable.push_back("transactions"); - aMutable.push_back("prevblock"); - } + UniValue aMutable(UniValue::VARR); + aMutable.push_back("time"); + aMutable.push_back("transactions"); + aMutable.push_back("prevblock"); UniValue result(UniValue::VOBJ); result.push_back(Pair("capabilities", aCaps)); + + UniValue aRules(UniValue::VARR); + UniValue vbavailable(UniValue::VOBJ); + for (int i = 0; i < (int)Consensus::MAX_VERSION_BITS_DEPLOYMENTS; ++i) { + Consensus::DeploymentPos pos = Consensus::DeploymentPos(i); + ThresholdState state = VersionBitsState(pindexPrev, consensusParams, pos, versionbitscache); + switch (state) { + case THRESHOLD_DEFINED: + case THRESHOLD_FAILED: + // Not exposed to GBT at all + break; + case THRESHOLD_LOCKED_IN: + // Ensure bit is set in block version + pblock->nVersion |= VersionBitsMask(consensusParams, pos); + // FALL THROUGH to get vbavailable set... + case THRESHOLD_STARTED: + { + const struct BIP9DeploymentInfo& vbinfo = VersionBitsDeploymentInfo[pos]; + vbavailable.push_back(Pair(gbt_vb_name(pos), consensusParams.vDeployments[pos].bit)); + if (setClientRules.find(vbinfo.name) == setClientRules.end()) { + if (!vbinfo.gbt_force) { + // If the client doesn't support this, don't indicate it in the [default] version + pblock->nVersion &= ~VersionBitsMask(consensusParams, pos); + } + } + break; + } + case THRESHOLD_ACTIVE: + { + // Add to rules only + const struct BIP9DeploymentInfo& vbinfo = VersionBitsDeploymentInfo[pos]; + aRules.push_back(gbt_vb_name(pos)); + if (setClientRules.find(vbinfo.name) == setClientRules.end()) { + // Not supported by the client; make sure it's safe to proceed + if (!vbinfo.gbt_force) { + // If we do anything other than throw an exception here, be sure version/force isn't sent to old clients + throw JSONRPCError(RPC_INVALID_PARAMETER, strprintf("Support for '%s' rule requires explicit client support", vbinfo.name)); + } + } + break; + } + } + } result.push_back(Pair("version", pblock->nVersion)); + result.push_back(Pair("rules", aRules)); + result.push_back(Pair("vbavailable", vbavailable)); + result.push_back(Pair("vbrequired", int(0))); + + if (nMaxVersionPreVB >= 2) { + // If VB is supported by the client, nMaxVersionPreVB is -1, so we won't get here + // Because BIP 34 changed how the generation transaction is serialised, we can only use version/force back to v2 blocks + // This is safe to do [otherwise-]unconditionally only because we are throwing an exception above if a non-force deployment gets activated + // Note that this can probably also be removed entirely after the first BIP9 non-force deployment (ie, probably segwit) gets activated + aMutable.push_back("version/force"); + } + result.push_back(Pair("previousblockhash", pblock->hashPrevBlock.GetHex())); result.push_back(Pair("transactions", transactions)); result.push_back(Pair("coinbaseaux", aux)); diff --git a/src/rpc/misc.cpp b/src/rpc/misc.cpp index c58c6f1df3d27..f2a29416e675d 100644 --- a/src/rpc/misc.cpp +++ b/src/rpc/misc.cpp @@ -403,6 +403,48 @@ UniValue verifymessage(const UniValue& params, bool fHelp) return (pubkey.GetID() == keyID); } +UniValue signmessagewithprivkey(const UniValue& params, bool fHelp) +{ + if (fHelp || params.size() != 2) + throw runtime_error( + "signmessagewithprivkey \"privkey\" \"message\"\n" + "\nSign a message with the private key of an address\n" + "\nArguments:\n" + "1. \"privkey\" (string, required) The private key to sign the message with.\n" + "2. \"message\" (string, required) The message to create a signature of.\n" + "\nResult:\n" + "\"signature\" (string) The signature of the message encoded in base 64\n" + "\nExamples:\n" + "\nCreate the signature\n" + + HelpExampleCli("signmessagewithprivkey", "\"privkey\" \"my message\"") + + "\nVerify the signature\n" + + HelpExampleCli("verifymessage", "\"1D1ZrZNe3JUo7ZycKEYQQiQAWd9y54F4XZ\" \"signature\" \"my message\"") + + "\nAs json rpc\n" + + HelpExampleRpc("signmessagewithprivkey", "\"privkey\", \"my message\"") + ); + + string strPrivkey = params[0].get_str(); + string strMessage = params[1].get_str(); + + CBitcoinSecret vchSecret; + bool fGood = vchSecret.SetString(strPrivkey); + if (!fGood) + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Invalid private key"); + CKey key = vchSecret.GetKey(); + if (!key.IsValid()) + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Private key outside allowed range"); + + CHashWriter ss(SER_GETHASH, 0); + ss << strMessageMagic; + ss << strMessage; + + vector vchSig; + if (!key.SignCompact(ss.GetHash(), vchSig)) + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Sign failed"); + + return EncodeBase64(&vchSig[0], vchSig.size()); +} + UniValue setmocktime(const UniValue& params, bool fHelp) { if (fHelp || params.size() != 1) @@ -442,6 +484,7 @@ static const CRPCCommand commands[] = { "util", "createmultisig", &createmultisig, true }, { "util", "createwitnessaddress", &createwitnessaddress, true }, { "util", "verifymessage", &verifymessage, true }, + { "util", "signmessagewithprivkey", &signmessagewithprivkey, true }, /* Not shown in help */ { "hidden", "setmocktime", &setmocktime, true }, diff --git a/src/rpc/net.cpp b/src/rpc/net.cpp index ce14d034ce1fd..cae964e46df1e 100644 --- a/src/rpc/net.cpp +++ b/src/rpc/net.cpp @@ -219,7 +219,7 @@ UniValue addnode(const UniValue& params, bool fHelp) if (strCommand == "onetry") { CAddress addr; - OpenNetworkConnection(addr, NULL, strNode.c_str()); + OpenNetworkConnection(addr, false, NULL, strNode.c_str()); return NullUniValue; } @@ -347,6 +347,7 @@ UniValue getaddednodeinfo(const UniValue& params, bool fHelp) obj.push_back(Pair("connected", false)); UniValue addresses(UniValue::VARR); obj.push_back(Pair("addresses", addresses)); + ret.push_back(obj); } } @@ -459,6 +460,7 @@ UniValue getnetworkinfo(const UniValue& params, bool fHelp) " \"subversion\": \"/Satoshi:x.x.x/\", (string) the server subversion string\n" " \"protocolversion\": xxxxx, (numeric) the protocol version\n" " \"localservices\": \"xxxxxxxxxxxxxxxx\", (string) the services we offer to the network\n" + " \"localrelay\": true|false, (bool) true if transaction relay is requested from peers\n" " \"timeoffset\": xxxxx, (numeric) the time offset\n" " \"connections\": xxxxx, (numeric) the number of connections\n" " \"networks\": [ (array) information per network\n" @@ -493,6 +495,7 @@ UniValue getnetworkinfo(const UniValue& params, bool fHelp) obj.push_back(Pair("subversion", strSubVersion)); obj.push_back(Pair("protocolversion",PROTOCOL_VERSION)); obj.push_back(Pair("localservices", strprintf("%016x", nLocalServices))); + obj.push_back(Pair("localrelay", fRelayTxes)); obj.push_back(Pair("timeoffset", GetTimeOffset())); obj.push_back(Pair("connections", (int)vNodes.size())); obj.push_back(Pair("networks", GetNetworksInfo())); @@ -564,20 +567,12 @@ UniValue setban(const UniValue& params, bool fHelp) absolute = true; isSubnet ? CNode::Ban(subNet, BanReasonManuallyAdded, banTime, absolute) : CNode::Ban(netAddr, BanReasonManuallyAdded, banTime, absolute); - - //disconnect possible nodes - while(CNode *bannedNode = (isSubnet ? FindNode(subNet) : FindNode(netAddr))) - bannedNode->fDisconnect = true; } else if(strCommand == "remove") { if (!( isSubnet ? CNode::Unban(subNet) : CNode::Unban(netAddr) )) throw JSONRPCError(RPC_MISC_ERROR, "Error: Unban failed"); } - - DumpBanlist(); //store banlist to disk - uiInterface.BannedListChanged(); - return NullUniValue; } @@ -623,8 +618,6 @@ UniValue clearbanned(const UniValue& params, bool fHelp) ); CNode::ClearBanned(); - DumpBanlist(); //store banlist to disk - uiInterface.BannedListChanged(); return NullUniValue; } diff --git a/src/rpc/rawtransaction.cpp b/src/rpc/rawtransaction.cpp index 8ed260ed7a372..cb8e071aaceca 100644 --- a/src/rpc/rawtransaction.cpp +++ b/src/rpc/rawtransaction.cpp @@ -229,7 +229,7 @@ UniValue gettxoutproof(const UniValue& params, bool fHelp) "\nNOTE: By default this function only works sometimes. This is when there is an\n" "unspent output in the utxo for this transaction. To make it always work,\n" "you need to maintain a transaction index, using the -txindex command line option or\n" - "specify the block in which the transaction is included in manually (by blockhash).\n" + "specify the block in which the transaction is included manually (by blockhash).\n" "\nReturn the raw transaction data.\n" "\nArguments:\n" "1. \"txids\" (string) A json array of txids to filter\n" @@ -352,6 +352,7 @@ UniValue createrawtransaction(const UniValue& params, bool fHelp) " {\n" " \"txid\":\"id\", (string, required) The transaction id\n" " \"vout\":n (numeric, required) The output number\n" + " \"sequence\":n (numeric, optional) The sequence number\n" " }\n" " ,...\n" " ]\n" @@ -402,6 +403,12 @@ UniValue createrawtransaction(const UniValue& params, bool fHelp) throw JSONRPCError(RPC_INVALID_PARAMETER, "Invalid parameter, vout must be positive"); uint32_t nSequence = (rawTx.nLockTime ? std::numeric_limits::max() - 1 : std::numeric_limits::max()); + + // set the sequence number if passed in the parameters object + const UniValue& sequenceObj = find_value(o, "sequence"); + if (sequenceObj.isNum()) + nSequence = sequenceObj.get_int(); + CTxIn in(COutPoint(txid, nOutput), CScript(), nSequence); rawTx.vin.push_back(in); @@ -697,7 +704,12 @@ UniValue signrawtransaction(const UniValue& params, bool fHelp) UniValue prevOut = p.get_obj(); - RPCTypeCheckObj(prevOut, boost::assign::map_list_of("txid", UniValue::VSTR)("vout", UniValue::VNUM)("scriptPubKey", UniValue::VSTR)); + RPCTypeCheckObj(prevOut, + { + {"txid", UniValueType(UniValue::VSTR)}, + {"vout", UniValueType(UniValue::VNUM)}, + {"scriptPubKey", UniValueType(UniValue::VSTR)}, + }); uint256 txid = ParseHashO(prevOut, "txid"); @@ -728,7 +740,13 @@ UniValue signrawtransaction(const UniValue& params, bool fHelp) // if redeemScript given and not using the local wallet (private keys // given), add redeemScript to the tempKeystore so it can be signed: if (fGivenKeys && (scriptPubKey.IsPayToScriptHash() || scriptPubKey.IsPayToWitnessScriptHash())) { - RPCTypeCheckObj(prevOut, boost::assign::map_list_of("txid", UniValue::VSTR)("vout", UniValue::VNUM)("scriptPubKey", UniValue::VSTR)("redeemScript",UniValue::VSTR)); + RPCTypeCheckObj(prevOut, + { + {"txid", UniValueType(UniValue::VSTR)}, + {"vout", UniValueType(UniValue::VNUM)}, + {"scriptPubKey", UniValueType(UniValue::VSTR)}, + {"redeemScript", UniValueType(UniValue::VSTR)}, + }); UniValue v = find_value(prevOut, "redeemScript"); if (!v.isNull()) { vector rsData(ParseHexV(v, "redeemScript")); @@ -768,6 +786,9 @@ UniValue signrawtransaction(const UniValue& params, bool fHelp) // Script verification errors UniValue vErrors(UniValue::VARR); + // Use CTransaction for the constant parts of the + // transaction to avoid rehashing. + const CTransaction txConst(mergedTx); // Sign what we can: for (unsigned int i = 0; i < mergedTx.vin.size(); i++) { CTxIn& txin = mergedTx.vin[i]; @@ -786,13 +807,13 @@ UniValue signrawtransaction(const UniValue& params, bool fHelp) // ... and merge in other signatures: BOOST_FOREACH(const CMutableTransaction& txv, txVariants) { - sigdata = CombineSignatures(prevPubKey, MutableTransactionSignatureChecker(&mergedTx, i, amount), sigdata, DataFromTransaction(txv, i)); + sigdata = CombineSignatures(prevPubKey, TransactionSignatureChecker(&txConst, i, amount), sigdata, DataFromTransaction(txv, i)); } UpdateTransaction(mergedTx, i, sigdata); ScriptError serror = SCRIPT_ERR_OK; - if (!VerifyScript(txin.scriptSig, prevPubKey, mergedTx.wit.vtxinwit.size() > i ? &mergedTx.wit.vtxinwit[i].scriptWitness : NULL, STANDARD_SCRIPT_VERIFY_FLAGS, MutableTransactionSignatureChecker(&mergedTx, i, amount), &serror)) { + if (!VerifyScript(txin.scriptSig, prevPubKey, mergedTx.wit.vtxinwit.size() > i ? &mergedTx.wit.vtxinwit[i].scriptWitness : NULL, STANDARD_SCRIPT_VERIFY_FLAGS, TransactionSignatureChecker(&txConst, i, amount), &serror)) { TxInErrorToJSON(txin, vErrors, ScriptErrorString(serror)); } } @@ -848,12 +869,11 @@ UniValue sendrawtransaction(const UniValue& params, bool fHelp) const CCoins* existingCoins = view.AccessCoins(hashTx); bool fHaveMempool = mempool.exists(hashTx); bool fHaveChain = existingCoins && existingCoins->nHeight < 1000000000; - CFeeRate txFeeRate = CFeeRate(0); if (!fHaveMempool && !fHaveChain) { // push to local node and sync with wallets CValidationState state; bool fMissingInputs; - if (!AcceptToMemoryPool(mempool, state, tx, false, &fMissingInputs, &txFeeRate, false, nMaxRawTxFee)) { + if (!AcceptToMemoryPool(mempool, state, tx, false, &fMissingInputs, false, nMaxRawTxFee)) { if (state.IsInvalid()) { throw JSONRPCError(RPC_TRANSACTION_REJECTED, strprintf("%i: %s", state.GetRejectCode(), state.GetRejectReason())); } else { @@ -866,7 +886,7 @@ UniValue sendrawtransaction(const UniValue& params, bool fHelp) } else if (fHaveChain) { throw JSONRPCError(RPC_TRANSACTION_ALREADY_IN_CHAIN, "transaction already in block chain"); } - RelayTransaction(tx, txFeeRate); + RelayTransaction(tx); return hashTx.GetHex(); } diff --git a/src/rpc/server.cpp b/src/rpc/server.cpp index d06a9142b6819..23149baa6d945 100644 --- a/src/rpc/server.cpp +++ b/src/rpc/server.cpp @@ -88,20 +88,18 @@ void RPCTypeCheck(const UniValue& params, } void RPCTypeCheckObj(const UniValue& o, - const map& typesExpected, - bool fAllowNull, - bool fStrict) + const map& typesExpected, + bool fAllowNull, + bool fStrict) { - BOOST_FOREACH(const PAIRTYPE(string, UniValue::VType)& t, typesExpected) - { + for (const auto& t : typesExpected) { const UniValue& v = find_value(o, t.first); if (!fAllowNull && v.isNull()) throw JSONRPCError(RPC_TYPE_ERROR, strprintf("Missing %s", t.first)); - if (!((v.type() == t.second) || (fAllowNull && (v.isNull())))) - { + if (!(t.second.typeAny || v.type() == t.second.type || (fAllowNull && v.isNull()))) { string err = strprintf("Expected type %s for %s, got %s", - uvTypeName(t.second), t.first, uvTypeName(v.type())); + uvTypeName(t.second.type), t.first, uvTypeName(v.type())); throw JSONRPCError(RPC_TYPE_ERROR, err); } } diff --git a/src/rpc/server.h b/src/rpc/server.h index b4713366172ae..b5ccc153d07ef 100644 --- a/src/rpc/server.h +++ b/src/rpc/server.h @@ -32,6 +32,15 @@ namespace RPCServer class CBlockIndex; class CNetAddr; +/** Wrapper for UniValue::VType, which includes typeAny: + * Used to denote don't care type. Only used by RPCTypeCheckObj */ +struct UniValueType { + UniValueType(UniValue::VType _type) : typeAny(false), type(_type) {} + UniValueType() : typeAny(true) {} + bool typeAny; + UniValue::VType type; +}; + class JSONRequest { public: @@ -60,17 +69,17 @@ bool RPCIsInWarmup(std::string *statusOut); /** * Type-check arguments; throws JSONRPCError if wrong type given. Does not check that * the right number of arguments are passed, just that any passed are the correct type. - * Use like: RPCTypeCheck(params, boost::assign::list_of(str_type)(int_type)(obj_type)); */ void RPCTypeCheck(const UniValue& params, const std::list& typesExpected, bool fAllowNull=false); /* Check for expected keys/value types in an Object. - Use like: RPCTypeCheckObj(object, boost::assign::map_list_of("name", str_type)("value", int_type)); */ void RPCTypeCheckObj(const UniValue& o, - const std::map& typesExpected, bool fAllowNull=false, bool fStrict=false); + const std::map& typesExpected, + bool fAllowNull = false, + bool fStrict = false); /** Opaque base class for timers returned by NewTimerFunc. * This provides no methods at the moment, but makes sure that delete diff --git a/src/scheduler.cpp b/src/scheduler.cpp index 184ddc28abbff..52777b61f96c8 100644 --- a/src/scheduler.cpp +++ b/src/scheduler.cpp @@ -79,6 +79,7 @@ void CScheduler::serviceQueue() } } --nThreadsServicingQueue; + newTaskScheduled.notify_one(); } void CScheduler::stop(bool drain) diff --git a/src/script/interpreter.cpp b/src/script/interpreter.cpp index 691afca6ff663..bc027e9f0c113 100644 --- a/src/script/interpreter.cpp +++ b/src/script/interpreter.cpp @@ -247,7 +247,7 @@ bool EvalScript(vector >& stack, const CScript& script, un vector vfExec; vector altstack; set_error(serror, SCRIPT_ERR_UNKNOWN_ERROR); - if (script.size() > 10000) + if (script.size() > MAX_SCRIPT_SIZE) return set_error(serror, SCRIPT_ERR_SCRIPT_SIZE); int nOpCount = 0; bool fRequireMinimal = (flags & SCRIPT_VERIFY_MINIMALDATA) != 0; diff --git a/src/script/script.h b/src/script/script.h index ff6de47384902..71af3754bcd17 100644 --- a/src/script/script.h +++ b/src/script/script.h @@ -27,6 +27,9 @@ static const int MAX_OPS_PER_SCRIPT = 201; // Maximum number of public keys per multisig static const int MAX_PUBKEYS_PER_MULTISIG = 20; +// Maximum script length in bytes +static const int MAX_SCRIPT_SIZE = 10000; + // Threshold for nLockTime: below this value it is interpreted as block number, // otherwise as UNIX timestamp. static const unsigned int LOCKTIME_THRESHOLD = 500000000; // Tue Nov 5 00:53:20 1985 UTC @@ -570,17 +573,26 @@ class CScript : public CScriptBase int nFound = 0; if (b.empty()) return nFound; - iterator pc = begin(); + CScript result; + iterator pc = begin(), pc2 = begin(); opcodetype opcode; do { - while (end() - pc >= (long)b.size() && memcmp(&pc[0], &b[0], b.size()) == 0) + result.insert(result.end(), pc2, pc); + while (static_cast(end() - pc) >= b.size() && std::equal(b.begin(), b.end(), pc)) { - pc = erase(pc, pc + b.size()); + pc = pc + b.size(); ++nFound; } + pc2 = pc; } while (GetOp(pc, opcode)); + + if (nFound > 0) { + result.insert(result.end(), pc2, end()); + *this = result; + } + return nFound; } int Find(opcodetype op) const @@ -623,7 +635,7 @@ class CScript : public CScriptBase */ bool IsUnspendable() const { - return (size() > 0 && *begin() == OP_RETURN); + return (size() > 0 && *begin() == OP_RETURN) || (size() > MAX_SCRIPT_SIZE); } void clear() diff --git a/src/streams.h b/src/streams.h index e5b0f06cde475..7132364eb1951 100644 --- a/src/streams.h +++ b/src/streams.h @@ -273,7 +273,9 @@ class CDataStream CDataStream& ignore(int nSize) { // Ignore from the beginning of the buffer - assert(nSize >= 0); + if (nSize < 0) { + throw std::ios_base::failure("CDataStream::ignore(): nSize negative"); + } unsigned int nReadPosNext = nReadPos + nSize; if (nReadPosNext >= vch.size()) { @@ -437,6 +439,20 @@ class CAutoFile return (*this); } + CAutoFile& ignore(size_t nSize) + { + if (!file) + throw std::ios_base::failure("CAutoFile::ignore: file handle is NULL"); + unsigned char data[4096]; + while (nSize > 0) { + size_t nNow = std::min(nSize, sizeof(data)); + if (fread(data, 1, nNow, file) != nNow) + throw std::ios_base::failure(feof(file) ? "CAutoFile::ignore: end of file" : "CAutoFile::read: fread failed"); + nSize -= nNow; + } + return (*this); + } + CAutoFile& write(const char* pch, size_t nSize) { if (!file) diff --git a/src/test/DoS_tests.cpp b/src/test/DoS_tests.cpp index 0f9158bd06ee8..fa8deaa0c2042 100644 --- a/src/test/DoS_tests.cpp +++ b/src/test/DoS_tests.cpp @@ -45,7 +45,7 @@ BOOST_FIXTURE_TEST_SUITE(DoS_tests, TestingSetup) BOOST_AUTO_TEST_CASE(DoS_banning) { CNode::ClearBanned(); - CAddress addr1(ip(0xa0b0c001)); + CAddress addr1(ip(0xa0b0c001), NODE_NONE); CNode dummyNode1(INVALID_SOCKET, addr1, "", true); dummyNode1.nVersion = 1; Misbehaving(dummyNode1.GetId(), 100); // Should get banned @@ -53,7 +53,7 @@ BOOST_AUTO_TEST_CASE(DoS_banning) BOOST_CHECK(CNode::IsBanned(addr1)); BOOST_CHECK(!CNode::IsBanned(ip(0xa0b0c001|0x0000ff00))); // Different IP, not banned - CAddress addr2(ip(0xa0b0c002)); + CAddress addr2(ip(0xa0b0c002), NODE_NONE); CNode dummyNode2(INVALID_SOCKET, addr2, "", true); dummyNode2.nVersion = 1; Misbehaving(dummyNode2.GetId(), 50); @@ -69,7 +69,7 @@ BOOST_AUTO_TEST_CASE(DoS_banscore) { CNode::ClearBanned(); mapArgs["-banscore"] = "111"; // because 11 is my favorite number - CAddress addr1(ip(0xa0b0c001)); + CAddress addr1(ip(0xa0b0c001), NODE_NONE); CNode dummyNode1(INVALID_SOCKET, addr1, "", true); dummyNode1.nVersion = 1; Misbehaving(dummyNode1.GetId(), 100); @@ -90,7 +90,7 @@ BOOST_AUTO_TEST_CASE(DoS_bantime) int64_t nStartTime = GetTime(); SetMockTime(nStartTime); // Overrides future calls to GetTime() - CAddress addr(ip(0xa0b0c001)); + CAddress addr(ip(0xa0b0c001), NODE_NONE); CNode dummyNode(INVALID_SOCKET, addr, "", true); dummyNode.nVersion = 1; diff --git a/src/test/addrman_tests.cpp b/src/test/addrman_tests.cpp index 767b653e47734..b6cec24b5752a 100644 --- a/src/test/addrman_tests.cpp +++ b/src/test/addrman_tests.cpp @@ -68,7 +68,7 @@ BOOST_AUTO_TEST_CASE(addrman_simple) // Test 2: Does Addrman::Add work as expected. CService addr1 = CService("250.1.1.1", 8333); - addrman.Add(CAddress(addr1), source); + addrman.Add(CAddress(addr1, NODE_NONE), source); BOOST_CHECK(addrman.size() == 1); CAddrInfo addr_ret1 = addrman.Select(); BOOST_CHECK(addr_ret1.ToString() == "250.1.1.1:8333"); @@ -76,14 +76,14 @@ BOOST_AUTO_TEST_CASE(addrman_simple) // Test 3: Does IP address deduplication work correctly. // Expected dup IP should not be added. CService addr1_dup = CService("250.1.1.1", 8333); - addrman.Add(CAddress(addr1_dup), source); + addrman.Add(CAddress(addr1_dup, NODE_NONE), source); BOOST_CHECK(addrman.size() == 1); // Test 5: New table has one addr and we add a diff addr we should // have two addrs. CService addr2 = CService("250.1.1.2", 8333); - addrman.Add(CAddress(addr2), source); + addrman.Add(CAddress(addr2, NODE_NONE), source); BOOST_CHECK(addrman.size() == 2); // Test 6: AddrMan::Clear() should empty the new table. @@ -106,18 +106,18 @@ BOOST_AUTO_TEST_CASE(addrman_ports) // Test 7; Addr with same IP but diff port does not replace existing addr. CService addr1 = CService("250.1.1.1", 8333); - addrman.Add(CAddress(addr1), source); + addrman.Add(CAddress(addr1, NODE_NONE), source); BOOST_CHECK(addrman.size() == 1); CService addr1_port = CService("250.1.1.1", 8334); - addrman.Add(CAddress(addr1_port), source); + addrman.Add(CAddress(addr1_port, NODE_NONE), source); BOOST_CHECK(addrman.size() == 1); CAddrInfo addr_ret2 = addrman.Select(); BOOST_CHECK(addr_ret2.ToString() == "250.1.1.1:8333"); // Test 8: Add same IP but diff port to tried table, it doesn't get added. // Perhaps this is not ideal behavior but it is the current behavior. - addrman.Good(CAddress(addr1_port)); + addrman.Good(CAddress(addr1_port, NODE_NONE)); BOOST_CHECK(addrman.size() == 1); bool newOnly = true; CAddrInfo addr_ret3 = addrman.Select(newOnly); @@ -136,7 +136,7 @@ BOOST_AUTO_TEST_CASE(addrman_select) // Test 9: Select from new with 1 addr in new. CService addr1 = CService("250.1.1.1", 8333); - addrman.Add(CAddress(addr1), source); + addrman.Add(CAddress(addr1, NODE_NONE), source); BOOST_CHECK(addrman.size() == 1); bool newOnly = true; @@ -144,7 +144,7 @@ BOOST_AUTO_TEST_CASE(addrman_select) BOOST_CHECK(addr_ret1.ToString() == "250.1.1.1:8333"); // Test 10: move addr to tried, select from new expected nothing returned. - addrman.Good(CAddress(addr1)); + addrman.Good(CAddress(addr1, NODE_NONE)); BOOST_CHECK(addrman.size() == 1); CAddrInfo addr_ret2 = addrman.Select(newOnly); BOOST_CHECK(addr_ret2.ToString() == "[::]:0"); @@ -160,21 +160,21 @@ BOOST_AUTO_TEST_CASE(addrman_select) CService addr3 = CService("250.3.2.2", 9999); CService addr4 = CService("250.3.3.3", 9999); - addrman.Add(CAddress(addr2), CService("250.3.1.1", 8333)); - addrman.Add(CAddress(addr3), CService("250.3.1.1", 8333)); - addrman.Add(CAddress(addr4), CService("250.4.1.1", 8333)); + addrman.Add(CAddress(addr2, NODE_NONE), CService("250.3.1.1", 8333)); + addrman.Add(CAddress(addr3, NODE_NONE), CService("250.3.1.1", 8333)); + addrman.Add(CAddress(addr4, NODE_NONE), CService("250.4.1.1", 8333)); // Add three addresses to tried table. CService addr5 = CService("250.4.4.4", 8333); CService addr6 = CService("250.4.5.5", 7777); CService addr7 = CService("250.4.6.6", 8333); - addrman.Add(CAddress(addr5), CService("250.3.1.1", 8333)); - addrman.Good(CAddress(addr5)); - addrman.Add(CAddress(addr6), CService("250.3.1.1", 8333)); - addrman.Good(CAddress(addr6)); - addrman.Add(CAddress(addr7), CService("250.1.1.3", 8333)); - addrman.Good(CAddress(addr7)); + addrman.Add(CAddress(addr5, NODE_NONE), CService("250.3.1.1", 8333)); + addrman.Good(CAddress(addr5, NODE_NONE)); + addrman.Add(CAddress(addr6, NODE_NONE), CService("250.3.1.1", 8333)); + addrman.Good(CAddress(addr6, NODE_NONE)); + addrman.Add(CAddress(addr7, NODE_NONE), CService("250.1.1.3", 8333)); + addrman.Good(CAddress(addr7, NODE_NONE)); // Test 11: 6 addrs + 1 addr from last test = 7. BOOST_CHECK(addrman.size() == 7); @@ -199,7 +199,7 @@ BOOST_AUTO_TEST_CASE(addrman_new_collisions) for (unsigned int i = 1; i < 18; i++) { CService addr = CService("250.1.1." + boost::to_string(i)); - addrman.Add(CAddress(addr), source); + addrman.Add(CAddress(addr, NODE_NONE), source); //Test 13: No collision in new table yet. BOOST_CHECK(addrman.size() == i); @@ -207,11 +207,11 @@ BOOST_AUTO_TEST_CASE(addrman_new_collisions) //Test 14: new table collision! CService addr1 = CService("250.1.1.18"); - addrman.Add(CAddress(addr1), source); + addrman.Add(CAddress(addr1, NODE_NONE), source); BOOST_CHECK(addrman.size() == 17); CService addr2 = CService("250.1.1.19"); - addrman.Add(CAddress(addr2), source); + addrman.Add(CAddress(addr2, NODE_NONE), source); BOOST_CHECK(addrman.size() == 18); } @@ -228,8 +228,8 @@ BOOST_AUTO_TEST_CASE(addrman_tried_collisions) for (unsigned int i = 1; i < 80; i++) { CService addr = CService("250.1.1." + boost::to_string(i)); - addrman.Add(CAddress(addr), source); - addrman.Good(CAddress(addr)); + addrman.Add(CAddress(addr, NODE_NONE), source); + addrman.Good(CAddress(addr, NODE_NONE)); //Test 15: No collision in tried table yet. BOOST_TEST_MESSAGE(addrman.size()); @@ -238,11 +238,11 @@ BOOST_AUTO_TEST_CASE(addrman_tried_collisions) //Test 16: tried table collision! CService addr1 = CService("250.1.1.80"); - addrman.Add(CAddress(addr1), source); + addrman.Add(CAddress(addr1, NODE_NONE), source); BOOST_CHECK(addrman.size() == 79); CService addr2 = CService("250.1.1.81"); - addrman.Add(CAddress(addr2), source); + addrman.Add(CAddress(addr2, NODE_NONE), source); BOOST_CHECK(addrman.size() == 80); } @@ -255,9 +255,9 @@ BOOST_AUTO_TEST_CASE(addrman_find) BOOST_CHECK(addrman.size() == 0); - CAddress addr1 = CAddress(CService("250.1.2.1", 8333)); - CAddress addr2 = CAddress(CService("250.1.2.1", 9999)); - CAddress addr3 = CAddress(CService("251.255.2.1", 8333)); + CAddress addr1 = CAddress(CService("250.1.2.1", 8333), NODE_NONE); + CAddress addr2 = CAddress(CService("250.1.2.1", 9999), NODE_NONE); + CAddress addr3 = CAddress(CService("251.255.2.1", 8333), NODE_NONE); CNetAddr source1 = CNetAddr("250.1.2.1"); CNetAddr source2 = CNetAddr("250.1.2.2"); @@ -294,7 +294,7 @@ BOOST_AUTO_TEST_CASE(addrman_create) BOOST_CHECK(addrman.size() == 0); - CAddress addr1 = CAddress(CService("250.1.2.1", 8333)); + CAddress addr1 = CAddress(CService("250.1.2.1", 8333), NODE_NONE); CNetAddr source1 = CNetAddr("250.1.2.1"); int nId; @@ -317,7 +317,7 @@ BOOST_AUTO_TEST_CASE(addrman_delete) BOOST_CHECK(addrman.size() == 0); - CAddress addr1 = CAddress(CService("250.1.2.1", 8333)); + CAddress addr1 = CAddress(CService("250.1.2.1", 8333), NODE_NONE); CNetAddr source1 = CNetAddr("250.1.2.1"); int nId; @@ -344,15 +344,15 @@ BOOST_AUTO_TEST_CASE(addrman_getaddr) vector vAddr1 = addrman.GetAddr(); BOOST_CHECK(vAddr1.size() == 0); - CAddress addr1 = CAddress(CService("250.250.2.1", 8333)); + CAddress addr1 = CAddress(CService("250.250.2.1", 8333), NODE_NONE); addr1.nTime = GetAdjustedTime(); // Set time so isTerrible = false - CAddress addr2 = CAddress(CService("250.251.2.2", 9999)); + CAddress addr2 = CAddress(CService("250.251.2.2", 9999), NODE_NONE); addr2.nTime = GetAdjustedTime(); - CAddress addr3 = CAddress(CService("251.252.2.3", 8333)); + CAddress addr3 = CAddress(CService("251.252.2.3", 8333), NODE_NONE); addr3.nTime = GetAdjustedTime(); - CAddress addr4 = CAddress(CService("252.253.3.4", 8333)); + CAddress addr4 = CAddress(CService("252.253.3.4", 8333), NODE_NONE); addr4.nTime = GetAdjustedTime(); - CAddress addr5 = CAddress(CService("252.254.4.5", 8333)); + CAddress addr5 = CAddress(CService("252.254.4.5", 8333), NODE_NONE); addr5.nTime = GetAdjustedTime(); CNetAddr source1 = CNetAddr("250.1.2.1"); CNetAddr source2 = CNetAddr("250.2.3.3"); @@ -368,8 +368,8 @@ BOOST_AUTO_TEST_CASE(addrman_getaddr) BOOST_CHECK(addrman.GetAddr().size() == 1); // Test 24: Ensure GetAddr works with new and tried addresses. - addrman.Good(CAddress(addr1)); - addrman.Good(CAddress(addr2)); + addrman.Good(CAddress(addr1, NODE_NONE)); + addrman.Good(CAddress(addr2, NODE_NONE)); BOOST_CHECK(addrman.GetAddr().size() == 1); // Test 25: Ensure GetAddr still returns 23% when addrman has many addrs. @@ -378,7 +378,7 @@ BOOST_AUTO_TEST_CASE(addrman_getaddr) int octet2 = (i / 256) % 256; int octet3 = (i / (256 * 2)) % 256; string strAddr = boost::to_string(octet1) + "." + boost::to_string(octet2) + "." + boost::to_string(octet3) + ".23"; - CAddress addr = CAddress(CService(strAddr)); + CAddress addr = CAddress(CService(strAddr), NODE_NONE); // Ensure that for all addrs in addrman, isTerrible == false. addr.nTime = GetAdjustedTime(); @@ -403,8 +403,8 @@ BOOST_AUTO_TEST_CASE(caddrinfo_get_tried_bucket) // Set addrman addr placement to be deterministic. addrman.MakeDeterministic(); - CAddress addr1 = CAddress(CService("250.1.1.1", 8333)); - CAddress addr2 = CAddress(CService("250.1.1.1", 9999)); + CAddress addr1 = CAddress(CService("250.1.1.1", 8333), NODE_NONE); + CAddress addr2 = CAddress(CService("250.1.1.1", 9999), NODE_NONE); CNetAddr source1 = CNetAddr("250.1.1.1"); @@ -431,7 +431,7 @@ BOOST_AUTO_TEST_CASE(caddrinfo_get_tried_bucket) set buckets; for (int i = 0; i < 255; i++) { CAddrInfo infoi = CAddrInfo( - CAddress(CService("250.1.1." + boost::to_string(i))), + CAddress(CService("250.1.1." + boost::to_string(i)), NODE_NONE), CNetAddr("250.1.1." + boost::to_string(i))); int bucket = infoi.GetTriedBucket(nKey1); buckets.insert(bucket); @@ -443,7 +443,7 @@ BOOST_AUTO_TEST_CASE(caddrinfo_get_tried_bucket) buckets.clear(); for (int j = 0; j < 255; j++) { CAddrInfo infoj = CAddrInfo( - CAddress(CService("250." + boost::to_string(j) + ".1.1")), + CAddress(CService("250." + boost::to_string(j) + ".1.1"), NODE_NONE), CNetAddr("250." + boost::to_string(j) + ".1.1")); int bucket = infoj.GetTriedBucket(nKey1); buckets.insert(bucket); @@ -460,8 +460,8 @@ BOOST_AUTO_TEST_CASE(caddrinfo_get_new_bucket) // Set addrman addr placement to be deterministic. addrman.MakeDeterministic(); - CAddress addr1 = CAddress(CService("250.1.2.1", 8333)); - CAddress addr2 = CAddress(CService("250.1.2.1", 9999)); + CAddress addr1 = CAddress(CService("250.1.2.1", 8333), NODE_NONE); + CAddress addr2 = CAddress(CService("250.1.2.1", 9999), NODE_NONE); CNetAddr source1 = CNetAddr("250.1.2.1"); @@ -484,7 +484,7 @@ BOOST_AUTO_TEST_CASE(caddrinfo_get_new_bucket) set buckets; for (int i = 0; i < 255; i++) { CAddrInfo infoi = CAddrInfo( - CAddress(CService("250.1.1." + boost::to_string(i))), + CAddress(CService("250.1.1." + boost::to_string(i)), NODE_NONE), CNetAddr("250.1.1." + boost::to_string(i))); int bucket = infoi.GetNewBucket(nKey1); buckets.insert(bucket); @@ -497,7 +497,7 @@ BOOST_AUTO_TEST_CASE(caddrinfo_get_new_bucket) for (int j = 0; j < 4 * 255; j++) { CAddrInfo infoj = CAddrInfo(CAddress( CService( - boost::to_string(250 + (j / 255)) + "." + boost::to_string(j % 256) + ".1.1")), + boost::to_string(250 + (j / 255)) + "." + boost::to_string(j % 256) + ".1.1"), NODE_NONE), CNetAddr("251.4.1.1")); int bucket = infoj.GetNewBucket(nKey1); buckets.insert(bucket); @@ -509,7 +509,7 @@ BOOST_AUTO_TEST_CASE(caddrinfo_get_new_bucket) buckets.clear(); for (int p = 0; p < 255; p++) { CAddrInfo infoj = CAddrInfo( - CAddress(CService("250.1.1.1")), + CAddress(CService("250.1.1.1"), NODE_NONE), CNetAddr("250." + boost::to_string(p) + ".1.1")); int bucket = infoj.GetNewBucket(nKey1); buckets.insert(bucket); diff --git a/src/test/base58_tests.cpp b/src/test/base58_tests.cpp index e5a2e28b2e9de..01eb2aee9ed02 100644 --- a/src/test/base58_tests.cpp +++ b/src/test/base58_tests.cpp @@ -79,7 +79,7 @@ class TestAddrTypeVisitor : public boost::static_visitor private: std::string exp_addrType; public: - TestAddrTypeVisitor(const std::string &exp_addrType) : exp_addrType(exp_addrType) { } + TestAddrTypeVisitor(const std::string &_exp_addrType) : exp_addrType(_exp_addrType) { } bool operator()(const CKeyID &id) const { return (exp_addrType == "pubkey"); @@ -100,7 +100,7 @@ class TestPayloadVisitor : public boost::static_visitor private: std::vector exp_payload; public: - TestPayloadVisitor(std::vector &exp_payload) : exp_payload(exp_payload) { } + TestPayloadVisitor(std::vector &_exp_payload) : exp_payload(_exp_payload) { } bool operator()(const CKeyID &id) const { uint160 exp_key(exp_payload); diff --git a/src/test/bloom_tests.cpp b/src/test/bloom_tests.cpp index 9557000ddcc11..042fad42dae89 100644 --- a/src/test/bloom_tests.cpp +++ b/src/test/bloom_tests.cpp @@ -514,11 +514,14 @@ BOOST_AUTO_TEST_CASE(rolling_bloom) if (i >= 100) BOOST_CHECK(rb1.contains(data[i-100])); rb1.insert(data[i]); + BOOST_CHECK(rb1.contains(data[i])); } // Insert 999 more random entries: for (int i = 0; i < 999; i++) { - rb1.insert(RandomData()); + std::vector d = RandomData(); + rb1.insert(d); + BOOST_CHECK(rb1.contains(d)); } // Sanity check to make sure the filter isn't just filling up: nHits = 0; diff --git a/src/test/coins_tests.cpp b/src/test/coins_tests.cpp index 48e3c8ed8e07f..e6923265592d7 100644 --- a/src/test/coins_tests.cpp +++ b/src/test/coins_tests.cpp @@ -4,7 +4,9 @@ #include "coins.h" #include "random.h" +#include "script/standard.h" #include "uint256.h" +#include "utilstrencodings.h" #include "test/test_bitcoin.h" #include "main.h" #include "consensus/validation.h" @@ -295,8 +297,7 @@ BOOST_AUTO_TEST_CASE(updatecoins_simulation_test) CCoins &coins = result[tx.GetHash()]; coins.FromTx(tx, height); - CValidationState dummy; - UpdateCoins(tx, dummy, *(stack.back()), height); + UpdateCoins(tx, *(stack.back()), height); } // Once every 1000 iterations and at the end, verify the full cache. @@ -345,4 +346,73 @@ BOOST_AUTO_TEST_CASE(updatecoins_simulation_test) BOOST_CHECK(spent_a_duplicate_coinbase); } +BOOST_AUTO_TEST_CASE(ccoins_serialization) +{ + // Good example + CDataStream ss1(ParseHex("0104835800816115944e077fe7c803cfa57f29b36bf87c1d358bb85e"), SER_DISK, CLIENT_VERSION); + CCoins cc1; + ss1 >> cc1; + BOOST_CHECK_EQUAL(cc1.nVersion, 1); + BOOST_CHECK_EQUAL(cc1.fCoinBase, false); + BOOST_CHECK_EQUAL(cc1.nHeight, 203998); + BOOST_CHECK_EQUAL(cc1.vout.size(), 2); + BOOST_CHECK_EQUAL(cc1.IsAvailable(0), false); + BOOST_CHECK_EQUAL(cc1.IsAvailable(1), true); + BOOST_CHECK_EQUAL(cc1.vout[1].nValue, 60000000000ULL); + BOOST_CHECK_EQUAL(HexStr(cc1.vout[1].scriptPubKey), HexStr(GetScriptForDestination(CKeyID(uint160(ParseHex("816115944e077fe7c803cfa57f29b36bf87c1d35")))))); + + // Good example + CDataStream ss2(ParseHex("0109044086ef97d5790061b01caab50f1b8e9c50a5057eb43c2d9563a4eebbd123008c988f1a4a4de2161e0f50aac7f17e7f9555caa486af3b"), SER_DISK, CLIENT_VERSION); + CCoins cc2; + ss2 >> cc2; + BOOST_CHECK_EQUAL(cc2.nVersion, 1); + BOOST_CHECK_EQUAL(cc2.fCoinBase, true); + BOOST_CHECK_EQUAL(cc2.nHeight, 120891); + BOOST_CHECK_EQUAL(cc2.vout.size(), 17); + for (int i = 0; i < 17; i++) { + BOOST_CHECK_EQUAL(cc2.IsAvailable(i), i == 4 || i == 16); + } + BOOST_CHECK_EQUAL(cc2.vout[4].nValue, 234925952); + BOOST_CHECK_EQUAL(HexStr(cc2.vout[4].scriptPubKey), HexStr(GetScriptForDestination(CKeyID(uint160(ParseHex("61b01caab50f1b8e9c50a5057eb43c2d9563a4ee")))))); + BOOST_CHECK_EQUAL(cc2.vout[16].nValue, 110397); + BOOST_CHECK_EQUAL(HexStr(cc2.vout[16].scriptPubKey), HexStr(GetScriptForDestination(CKeyID(uint160(ParseHex("8c988f1a4a4de2161e0f50aac7f17e7f9555caa4")))))); + + // Smallest possible example + CDataStream ssx(SER_DISK, CLIENT_VERSION); + BOOST_CHECK_EQUAL(HexStr(ssx.begin(), ssx.end()), ""); + + CDataStream ss3(ParseHex("0002000600"), SER_DISK, CLIENT_VERSION); + CCoins cc3; + ss3 >> cc3; + BOOST_CHECK_EQUAL(cc3.nVersion, 0); + BOOST_CHECK_EQUAL(cc3.fCoinBase, false); + BOOST_CHECK_EQUAL(cc3.nHeight, 0); + BOOST_CHECK_EQUAL(cc3.vout.size(), 1); + BOOST_CHECK_EQUAL(cc3.IsAvailable(0), true); + BOOST_CHECK_EQUAL(cc3.vout[0].nValue, 0); + BOOST_CHECK_EQUAL(cc3.vout[0].scriptPubKey.size(), 0); + + // scriptPubKey that ends beyond the end of the stream + CDataStream ss4(ParseHex("0002000800"), SER_DISK, CLIENT_VERSION); + try { + CCoins cc4; + ss4 >> cc4; + BOOST_CHECK_MESSAGE(false, "We should have thrown"); + } catch (const std::ios_base::failure& e) { + } + + // Very large scriptPubKey (3*10^9 bytes) past the end of the stream + CDataStream tmp(SER_DISK, CLIENT_VERSION); + uint64_t x = 3000000000ULL; + tmp << VARINT(x); + BOOST_CHECK_EQUAL(HexStr(tmp.begin(), tmp.end()), "8a95c0bb00"); + CDataStream ss5(ParseHex("0002008a95c0bb0000"), SER_DISK, CLIENT_VERSION); + try { + CCoins cc5; + ss5 >> cc5; + BOOST_CHECK_MESSAGE(false, "We should have thrown"); + } catch (const std::ios_base::failure& e) { + } +} + BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/crypto_tests.cpp b/src/test/crypto_tests.cpp index 0b46d718d1d13..58a62ee02262d 100644 --- a/src/test/crypto_tests.cpp +++ b/src/test/crypto_tests.cpp @@ -2,6 +2,7 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. +#include "crypto/aes.h" #include "crypto/ripemd160.h" #include "crypto/sha1.h" #include "crypto/sha256.h" @@ -16,6 +17,8 @@ #include #include +#include +#include BOOST_FIXTURE_TEST_SUITE(crypto_tests, BasicTestingSetup) @@ -63,6 +66,127 @@ void TestHMACSHA512(const std::string &hexkey, const std::string &hexin, const s TestVector(CHMAC_SHA512(&key[0], key.size()), ParseHex(hexin), ParseHex(hexout)); } +void TestAES128(const std::string &hexkey, const std::string &hexin, const std::string &hexout) +{ + std::vector key = ParseHex(hexkey); + std::vector in = ParseHex(hexin); + std::vector correctout = ParseHex(hexout); + std::vector buf, buf2; + + assert(key.size() == 16); + assert(in.size() == 16); + assert(correctout.size() == 16); + AES128Encrypt enc(&key[0]); + buf.resize(correctout.size()); + buf2.resize(correctout.size()); + enc.Encrypt(&buf[0], &in[0]); + BOOST_CHECK_EQUAL(HexStr(buf), HexStr(correctout)); + AES128Decrypt dec(&key[0]); + dec.Decrypt(&buf2[0], &buf[0]); + BOOST_CHECK_EQUAL(HexStr(buf2), HexStr(in)); +} + +void TestAES256(const std::string &hexkey, const std::string &hexin, const std::string &hexout) +{ + std::vector key = ParseHex(hexkey); + std::vector in = ParseHex(hexin); + std::vector correctout = ParseHex(hexout); + std::vector buf; + + assert(key.size() == 32); + assert(in.size() == 16); + assert(correctout.size() == 16); + AES256Encrypt enc(&key[0]); + buf.resize(correctout.size()); + enc.Encrypt(&buf[0], &in[0]); + BOOST_CHECK(buf == correctout); + AES256Decrypt dec(&key[0]); + dec.Decrypt(&buf[0], &buf[0]); + BOOST_CHECK(buf == in); +} + +void TestAES128CBC(const std::string &hexkey, const std::string &hexiv, bool pad, const std::string &hexin, const std::string &hexout) +{ + std::vector key = ParseHex(hexkey); + std::vector iv = ParseHex(hexiv); + std::vector in = ParseHex(hexin); + std::vector correctout = ParseHex(hexout); + std::vector realout(in.size() + AES_BLOCKSIZE); + + // Encrypt the plaintext and verify that it equals the cipher + AES128CBCEncrypt enc(&key[0], &iv[0], pad); + int size = enc.Encrypt(&in[0], in.size(), &realout[0]); + realout.resize(size); + BOOST_CHECK(realout.size() == correctout.size()); + BOOST_CHECK_MESSAGE(realout == correctout, HexStr(realout) + std::string(" != ") + hexout); + + // Decrypt the cipher and verify that it equals the plaintext + std::vector decrypted(correctout.size()); + AES128CBCDecrypt dec(&key[0], &iv[0], pad); + size = dec.Decrypt(&correctout[0], correctout.size(), &decrypted[0]); + decrypted.resize(size); + BOOST_CHECK(decrypted.size() == in.size()); + BOOST_CHECK_MESSAGE(decrypted == in, HexStr(decrypted) + std::string(" != ") + hexin); + + // Encrypt and re-decrypt substrings of the plaintext and verify that they equal each-other + for(std::vector::iterator i(in.begin()); i != in.end(); ++i) + { + std::vector sub(i, in.end()); + std::vector subout(sub.size() + AES_BLOCKSIZE); + int size = enc.Encrypt(&sub[0], sub.size(), &subout[0]); + if (size != 0) + { + subout.resize(size); + std::vector subdecrypted(subout.size()); + size = dec.Decrypt(&subout[0], subout.size(), &subdecrypted[0]); + subdecrypted.resize(size); + BOOST_CHECK(decrypted.size() == in.size()); + BOOST_CHECK_MESSAGE(subdecrypted == sub, HexStr(subdecrypted) + std::string(" != ") + HexStr(sub)); + } + } +} + +void TestAES256CBC(const std::string &hexkey, const std::string &hexiv, bool pad, const std::string &hexin, const std::string &hexout) +{ + std::vector key = ParseHex(hexkey); + std::vector iv = ParseHex(hexiv); + std::vector in = ParseHex(hexin); + std::vector correctout = ParseHex(hexout); + std::vector realout(in.size() + AES_BLOCKSIZE); + + // Encrypt the plaintext and verify that it equals the cipher + AES256CBCEncrypt enc(&key[0], &iv[0], pad); + int size = enc.Encrypt(&in[0], in.size(), &realout[0]); + realout.resize(size); + BOOST_CHECK(realout.size() == correctout.size()); + BOOST_CHECK_MESSAGE(realout == correctout, HexStr(realout) + std::string(" != ") + hexout); + + // Decrypt the cipher and verify that it equals the plaintext + std::vector decrypted(correctout.size()); + AES256CBCDecrypt dec(&key[0], &iv[0], pad); + size = dec.Decrypt(&correctout[0], correctout.size(), &decrypted[0]); + decrypted.resize(size); + BOOST_CHECK(decrypted.size() == in.size()); + BOOST_CHECK_MESSAGE(decrypted == in, HexStr(decrypted) + std::string(" != ") + hexin); + + // Encrypt and re-decrypt substrings of the plaintext and verify that they equal each-other + for(std::vector::iterator i(in.begin()); i != in.end(); ++i) + { + std::vector sub(i, in.end()); + std::vector subout(sub.size() + AES_BLOCKSIZE); + int size = enc.Encrypt(&sub[0], sub.size(), &subout[0]); + if (size != 0) + { + subout.resize(size); + std::vector subdecrypted(subout.size()); + size = dec.Decrypt(&subout[0], subout.size(), &subdecrypted[0]); + subdecrypted.resize(size); + BOOST_CHECK(decrypted.size() == in.size()); + BOOST_CHECK_MESSAGE(subdecrypted == sub, HexStr(subdecrypted) + std::string(" != ") + HexStr(sub)); + } + } +} + std::string LongTestString(void) { std::string ret; for (int i=0; i<200000; i++) { @@ -248,4 +372,71 @@ BOOST_AUTO_TEST_CASE(hmac_sha512_testvectors) { "b6022cac3c4982b10d5eeb55c3e4de15134676fb6de0446065c97440fa8c6a58"); } +BOOST_AUTO_TEST_CASE(aes_testvectors) { + // AES test vectors from FIPS 197. + TestAES128("000102030405060708090a0b0c0d0e0f", "00112233445566778899aabbccddeeff", "69c4e0d86a7b0430d8cdb78070b4c55a"); + TestAES256("000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f", "00112233445566778899aabbccddeeff", "8ea2b7ca516745bfeafc49904b496089"); + + // AES-ECB test vectors from NIST sp800-38a. + TestAES128("2b7e151628aed2a6abf7158809cf4f3c", "6bc1bee22e409f96e93d7e117393172a", "3ad77bb40d7a3660a89ecaf32466ef97"); + TestAES128("2b7e151628aed2a6abf7158809cf4f3c", "ae2d8a571e03ac9c9eb76fac45af8e51", "f5d3d58503b9699de785895a96fdbaaf"); + TestAES128("2b7e151628aed2a6abf7158809cf4f3c", "30c81c46a35ce411e5fbc1191a0a52ef", "43b1cd7f598ece23881b00e3ed030688"); + TestAES128("2b7e151628aed2a6abf7158809cf4f3c", "f69f2445df4f9b17ad2b417be66c3710", "7b0c785e27e8ad3f8223207104725dd4"); + TestAES256("603deb1015ca71be2b73aef0857d77811f352c073b6108d72d9810a30914dff4", "6bc1bee22e409f96e93d7e117393172a", "f3eed1bdb5d2a03c064b5a7e3db181f8"); + TestAES256("603deb1015ca71be2b73aef0857d77811f352c073b6108d72d9810a30914dff4", "ae2d8a571e03ac9c9eb76fac45af8e51", "591ccb10d410ed26dc5ba74a31362870"); + TestAES256("603deb1015ca71be2b73aef0857d77811f352c073b6108d72d9810a30914dff4", "30c81c46a35ce411e5fbc1191a0a52ef", "b6ed21b99ca6f4f9f153e7b1beafed1d"); + TestAES256("603deb1015ca71be2b73aef0857d77811f352c073b6108d72d9810a30914dff4", "f69f2445df4f9b17ad2b417be66c3710", "23304b7a39f9f3ff067d8d8f9e24ecc7"); +} + +BOOST_AUTO_TEST_CASE(aes_cbc_testvectors) { + + // NIST AES CBC 128-bit encryption test-vectors + TestAES128CBC("2b7e151628aed2a6abf7158809cf4f3c", "000102030405060708090A0B0C0D0E0F", false, \ + "6bc1bee22e409f96e93d7e117393172a", "7649abac8119b246cee98e9b12e9197d"); + TestAES128CBC("2b7e151628aed2a6abf7158809cf4f3c", "7649ABAC8119B246CEE98E9B12E9197D", false, \ + "ae2d8a571e03ac9c9eb76fac45af8e51", "5086cb9b507219ee95db113a917678b2"); + TestAES128CBC("2b7e151628aed2a6abf7158809cf4f3c", "5086cb9b507219ee95db113a917678b2", false, \ + "30c81c46a35ce411e5fbc1191a0a52ef", "73bed6b8e3c1743b7116e69e22229516"); + TestAES128CBC("2b7e151628aed2a6abf7158809cf4f3c", "73bed6b8e3c1743b7116e69e22229516", false, \ + "f69f2445df4f9b17ad2b417be66c3710", "3ff1caa1681fac09120eca307586e1a7"); + + // The same vectors with padding enabled + TestAES128CBC("2b7e151628aed2a6abf7158809cf4f3c", "000102030405060708090A0B0C0D0E0F", true, \ + "6bc1bee22e409f96e93d7e117393172a", "7649abac8119b246cee98e9b12e9197d8964e0b149c10b7b682e6e39aaeb731c"); + TestAES128CBC("2b7e151628aed2a6abf7158809cf4f3c", "7649ABAC8119B246CEE98E9B12E9197D", true, \ + "ae2d8a571e03ac9c9eb76fac45af8e51", "5086cb9b507219ee95db113a917678b255e21d7100b988ffec32feeafaf23538"); + TestAES128CBC("2b7e151628aed2a6abf7158809cf4f3c", "5086cb9b507219ee95db113a917678b2", true, \ + "30c81c46a35ce411e5fbc1191a0a52ef", "73bed6b8e3c1743b7116e69e22229516f6eccda327bf8e5ec43718b0039adceb"); + TestAES128CBC("2b7e151628aed2a6abf7158809cf4f3c", "73bed6b8e3c1743b7116e69e22229516", true, \ + "f69f2445df4f9b17ad2b417be66c3710", "3ff1caa1681fac09120eca307586e1a78cb82807230e1321d3fae00d18cc2012"); + + // NIST AES CBC 256-bit encryption test-vectors + TestAES256CBC("603deb1015ca71be2b73aef0857d77811f352c073b6108d72d9810a30914dff4", \ + "000102030405060708090A0B0C0D0E0F", false, "6bc1bee22e409f96e93d7e117393172a", \ + "f58c4c04d6e5f1ba779eabfb5f7bfbd6"); + TestAES256CBC("603deb1015ca71be2b73aef0857d77811f352c073b6108d72d9810a30914dff4", \ + "F58C4C04D6E5F1BA779EABFB5F7BFBD6", false, "ae2d8a571e03ac9c9eb76fac45af8e51", \ + "9cfc4e967edb808d679f777bc6702c7d"); + TestAES256CBC("603deb1015ca71be2b73aef0857d77811f352c073b6108d72d9810a30914dff4", \ + "9CFC4E967EDB808D679F777BC6702C7D", false, "30c81c46a35ce411e5fbc1191a0a52ef", + "39f23369a9d9bacfa530e26304231461"); + TestAES256CBC("603deb1015ca71be2b73aef0857d77811f352c073b6108d72d9810a30914dff4", \ + "39F23369A9D9BACFA530E26304231461", false, "f69f2445df4f9b17ad2b417be66c3710", \ + "b2eb05e2c39be9fcda6c19078c6a9d1b"); + + // The same vectors with padding enabled + TestAES256CBC("603deb1015ca71be2b73aef0857d77811f352c073b6108d72d9810a30914dff4", \ + "000102030405060708090A0B0C0D0E0F", true, "6bc1bee22e409f96e93d7e117393172a", \ + "f58c4c04d6e5f1ba779eabfb5f7bfbd6485a5c81519cf378fa36d42b8547edc0"); + TestAES256CBC("603deb1015ca71be2b73aef0857d77811f352c073b6108d72d9810a30914dff4", \ + "F58C4C04D6E5F1BA779EABFB5F7BFBD6", true, "ae2d8a571e03ac9c9eb76fac45af8e51", \ + "9cfc4e967edb808d679f777bc6702c7d3a3aa5e0213db1a9901f9036cf5102d2"); + TestAES256CBC("603deb1015ca71be2b73aef0857d77811f352c073b6108d72d9810a30914dff4", \ + "9CFC4E967EDB808D679F777BC6702C7D", true, "30c81c46a35ce411e5fbc1191a0a52ef", + "39f23369a9d9bacfa530e263042314612f8da707643c90a6f732b3de1d3f5cee"); + TestAES256CBC("603deb1015ca71be2b73aef0857d77811f352c073b6108d72d9810a30914dff4", \ + "39F23369A9D9BACFA530E26304231461", true, "f69f2445df4f9b17ad2b417be66c3710", \ + "b2eb05e2c39be9fcda6c19078c6a9d1b3f461796d6b0d6b2e0c2a72b4d80e644"); +} + BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/data/bitcoin-util-test.json b/src/test/data/bitcoin-util-test.json index 3bf80ca4348a2..5cb383de85a06 100644 --- a/src/test/data/bitcoin-util-test.json +++ b/src/test/data/bitcoin-util-test.json @@ -86,5 +86,18 @@ "outaddr=0.18:13tuJJDR2RgArmgfv6JScSdreahzgc4T6o", "outdata=54686973204f505f52455455524e207472616e73616374696f6e206f7574707574207761732063726561746564206279206d6f646966696564206372656174657261777472616e73616374696f6e2e"], "output_cmp": "txcreatedata2.hex" + }, + { "exec": "./bitcoin-tx", + "args": + ["-create", + "in=5897de6bd6027a475eadd57019d4e6872c396d0716c4875a5f1a6fcfdf385c1f:0:4294967293", + "outaddr=0.18:13tuJJDR2RgArmgfv6JScSdreahzgc4T6o"], + "output_cmp": "txcreatedata_seq0.hex" + }, + { "exec": "./bitcoin-tx", + "args": + ["01000000011f5c38dfcf6f1a5f5a87c416076d392c87e6d41970d5ad5e477a02d66bde97580000000000fdffffff0180a81201000000001976a9141fc11f39be1729bf973a7ab6a615ca4729d6457488ac00000000", + "in=5897de6bd6027a475eadd57019d4e6872c396d0716c4875a5f1a6fcfdf385c1f:0:1"], + "output_cmp": "txcreatedata_seq1.hex" } ] diff --git a/src/test/data/script_tests.json b/src/test/data/script_tests.json index 89b7a557981a9..9b81e0c77ddf7 100644 --- a/src/test/data/script_tests.json +++ b/src/test/data/script_tests.json @@ -700,6 +700,8 @@ ["0x17 0x3014020002107777777777777777777777777777777701", "0 CHECKSIG NOT", "", "OK", "Zero-length R is correctly encoded"], ["0x17 0x3014021077777777777777777777777777777777020001", "0 CHECKSIG NOT", "", "OK", "Zero-length S is correctly encoded for DERSIG"], ["0x27 0x302402107777777777777777777777777777777702108777777777777777777777777777777701", "0 CHECKSIG NOT", "", "OK", "Negative S is correctly encoded"], + +["2147483648", "NOP3", "CHECKSEQUENCEVERIFY", "OK", "CSV passes if stack top bit 1 << 31 is set"], ["", "DEPTH", "P2SH,STRICTENC", "EVAL_FALSE", "Test the test: we should have an empty stack after scriptSig evaluation"], [" ", "DEPTH", "P2SH,STRICTENC", "EVAL_FALSE", "and multiple spaces should not change that."], @@ -855,7 +857,7 @@ ["2 2 LSHIFT", "8 EQUAL", "P2SH,STRICTENC", "DISABLED_OPCODE", "disabled"], ["2 1 RSHIFT", "1 EQUAL", "P2SH,STRICTENC", "DISABLED_OPCODE", "disabled"], -["1","NOP1 CHECKLOCKTIMEVERIFY NOP3 NOP4 NOP5 NOP6 NOP7 NOP8 NOP9 NOP10 2 EQUAL", "P2SH,STRICTENC", "EVAL_FALSE"], +["1", "NOP1 CHECKLOCKTIMEVERIFY NOP3 NOP4 NOP5 NOP6 NOP7 NOP8 NOP9 NOP10 2 EQUAL", "P2SH,STRICTENC", "EVAL_FALSE"], ["'NOP_1_to_10' NOP1 CHECKLOCKTIMEVERIFY NOP3 NOP4 NOP5 NOP6 NOP7 NOP8 NOP9 NOP10","'NOP_1_to_11' EQUAL", "P2SH,STRICTENC", "EVAL_FALSE"], ["Ensure 100% coverage of discouraged NOPS"], @@ -1098,7 +1100,7 @@ "NOP NOP NOP NOP NOP NOP NOP NOP NOP NOP NOP NOP NOP 0 0 'a' 'b' 'c' 'd' 'e' 'f' 'g' 'h' 'i' 'j' 'k' 'l' 'm' 'n' 'o' 'p' 'q' 'r' 's' 't' 20 CHECKMULTISIG 0 0 'a' 'b' 'c' 'd' 'e' 'f' 'g' 'h' 'i' 'j' 'k' 'l' 'm' 'n' 'o' 'p' 'q' 'r' 's' 't' 20 CHECKMULTISIG 0 0 'a' 'b' 'c' 'd' 'e' 'f' 'g' 'h' 'i' 'j' 'k' 'l' 'm' 'n' 'o' 'p' 'q' 'r' 's' 't' 20 CHECKMULTISIG 0 0 'a' 'b' 'c' 'd' 'e' 'f' 'g' 'h' 'i' 'j' 'k' 'l' 'm' 'n' 'o' 'p' 'q' 'r' 's' 't' 20 CHECKMULTISIG 0 0 'a' 'b' 'c' 'd' 'e' 'f' 'g' 'h' 'i' 'j' 'k' 'l' 'm' 'n' 'o' 'p' 'q' 'r' 's' 't' 20 CHECKMULTISIG 0 0 'a' 'b' 'c' 'd' 'e' 'f' 'g' 'h' 'i' 'j' 'k' 'l' 'm' 'n' 'o' 'p' 'q' 'r' 's' 't' 20 CHECKMULTISIG 0 0 'a' 'b' 'c' 'd' 'e' 'f' 'g' 'h' 'i' 'j' 'k' 'l' 'm' 'n' 'o' 'p' 'q' 'r' 's' 't' 20 CHECKMULTISIG 0 0 'a' 'b' 'c' 'd' 'e' 'f' 'g' 'h' 'i' 'j' 'k' 'l' 'm' 'n' 'o' 'p' 'q' 'r' 's' 't' 20 CHECKMULTISIG 0 0 'a' 'b' 'c' 'd' 'e' 'f' 'g' 'h' 'i' 'j' 'k' 'l' 'm' 'n' 'o' 'p' 'q' 'r' 's' 't' 20 CHECKMULTISIG", "P2SH,STRICTENC", "OP_COUNT", -"Fails due to 201 sig op limit"], +"Fails due to 201 script operation limit"], ["1", "NOP NOP NOP NOP NOP NOP NOP NOP NOP NOP NOP NOP NOP 0 0 'a' 'b' 'c' 'd' 'e' 'f' 'g' 'h' 'i' 'j' 'k' 'l' 'm' 'n' 'o' 'p' 'q' 'r' 's' 't' 20 CHECKMULTISIGVERIFY 0 0 'a' 'b' 'c' 'd' 'e' 'f' 'g' 'h' 'i' 'j' 'k' 'l' 'm' 'n' 'o' 'p' 'q' 'r' 's' 't' 20 CHECKMULTISIGVERIFY 0 0 'a' 'b' 'c' 'd' 'e' 'f' 'g' 'h' 'i' 'j' 'k' 'l' 'm' 'n' 'o' 'p' 'q' 'r' 's' 't' 20 CHECKMULTISIGVERIFY 0 0 'a' 'b' 'c' 'd' 'e' 'f' 'g' 'h' 'i' 'j' 'k' 'l' 'm' 'n' 'o' 'p' 'q' 'r' 's' 't' 20 CHECKMULTISIGVERIFY 0 0 'a' 'b' 'c' 'd' 'e' 'f' 'g' 'h' 'i' 'j' 'k' 'l' 'm' 'n' 'o' 'p' 'q' 'r' 's' 't' 20 CHECKMULTISIGVERIFY 0 0 'a' 'b' 'c' 'd' 'e' 'f' 'g' 'h' 'i' 'j' 'k' 'l' 'm' 'n' 'o' 'p' 'q' 'r' 's' 't' 20 CHECKMULTISIGVERIFY 0 0 'a' 'b' 'c' 'd' 'e' 'f' 'g' 'h' 'i' 'j' 'k' 'l' 'm' 'n' 'o' 'p' 'q' 'r' 's' 't' 20 CHECKMULTISIGVERIFY 0 0 'a' 'b' 'c' 'd' 'e' 'f' 'g' 'h' 'i' 'j' 'k' 'l' 'm' 'n' 'o' 'p' 'q' 'r' 's' 't' 20 CHECKMULTISIGVERIFY 0 0 'a' 'b' 'c' 'd' 'e' 'f' 'g' 'h' 'i' 'j' 'k' 'l' 'm' 'n' 'o' 'p' 'q' 'r' 's' 't' 20 CHECKMULTISIGVERIFY", @@ -1314,6 +1316,13 @@ "EVAL_FALSE", "P2SH(P2PK), bad redeemscript" ], +[ + "0x47 0x30440220781ba4f59a7b207a10db87628bc2168df4d59b844b397d2dbc9a5835fb2f2b7602206ed8fbcc1072fe2dfc5bb25909269e5dc42ffcae7ec2bc81d59692210ff30c2b01 0x41 0x0479be667ef9dcbbac55a06295ce870b07029bfcdb2dce28d959f2815b16f81798483ada7726a3c4655da4fbfc0e1108a8fd17b448a68554199c47d08ffb10d4b8 0x19 0x76a91491b24bf9f5288532960ac687abb035127b1d28a588ac", + "HASH160 0x14 0x7f67f0521934a57d3039f77f9f32cf313f3ac74b EQUAL", + "P2SH", + "OK", + "P2SH(P2PKH)" +], [ "0x47 0x304402204e2eb034be7b089534ac9e798cf6a2c79f38bcb34d1b179efd6f2de0841735db022071461beb056b5a7be1819da6a3e3ce3662831ecc298419ca101eb6887b5dd6a401 0x19 0x76a9147cf9c846cd4882efec4bf07e44ebdad495c94f4b88ac", "HASH160 0x14 0x2df519943d5acc0ef5222091f9dfe3543f489a82 EQUAL", @@ -2109,5 +2118,12 @@ "P2PK with witness" ], +["CHECKSEQUENCEVERIFY tests"], +["", "NOP3", "CHECKSEQUENCEVERIFY", "INVALID_STACK_OPERATION", "CSV automatically fails on a empty stack"], +["-1", "NOP3", "CHECKSEQUENCEVERIFY", "NEGATIVE_LOCKTIME", "CSV automatically fails if stack top is negative"], +["0x0100", "NOP3", "CHECKSEQUENCEVERIFY,MINIMALDATA", "UNKNOWN_ERROR", "CSV fails if stack top is not minimally encoded"], +["0", "NOP3", "CHECKSEQUENCEVERIFY", "UNSATISFIED_LOCKTIME", "CSV fails if stack top bit 1 << 31 is set and the tx version < 2"], +["4294967296", "NOP3", "CHECKSEQUENCEVERIFY", "UNSATISFIED_LOCKTIME", + "CSV fails if stack top bit 1 << 31 is not set, and tx version < 2"], ["The End"] ] diff --git a/src/test/data/txcreatedata_seq0.hex b/src/test/data/txcreatedata_seq0.hex new file mode 100644 index 0000000000000..db02b5e4a46a7 --- /dev/null +++ b/src/test/data/txcreatedata_seq0.hex @@ -0,0 +1 @@ +01000000011f5c38dfcf6f1a5f5a87c416076d392c87e6d41970d5ad5e477a02d66bde97580000000000fdffffff0180a81201000000001976a9141fc11f39be1729bf973a7ab6a615ca4729d6457488ac00000000 diff --git a/src/test/data/txcreatedata_seq1.hex b/src/test/data/txcreatedata_seq1.hex new file mode 100644 index 0000000000000..4cedcd975cc21 --- /dev/null +++ b/src/test/data/txcreatedata_seq1.hex @@ -0,0 +1 @@ +01000000021f5c38dfcf6f1a5f5a87c416076d392c87e6d41970d5ad5e477a02d66bde97580000000000fdffffff1f5c38dfcf6f1a5f5a87c416076d392c87e6d41970d5ad5e477a02d66bde97580000000000010000000180a81201000000001976a9141fc11f39be1729bf973a7ab6a615ca4729d6457488ac00000000 diff --git a/src/test/dbwrapper_tests.cpp b/src/test/dbwrapper_tests.cpp index e399315870e79..a0bdcf4afbc70 100644 --- a/src/test/dbwrapper_tests.cpp +++ b/src/test/dbwrapper_tests.cpp @@ -39,7 +39,7 @@ BOOST_AUTO_TEST_CASE(dbwrapper) uint256 res; // Ensure that we're doing real obfuscation when obfuscate=true - BOOST_CHECK(obfuscate != is_null_key(dbw.GetObfuscateKey())); + BOOST_CHECK(obfuscate != is_null_key(dbwrapper_private::GetObfuscateKey(dbw))); BOOST_CHECK(dbw.Write(key, in)); BOOST_CHECK(dbw.Read(key, res)); @@ -64,7 +64,7 @@ BOOST_AUTO_TEST_CASE(dbwrapper_batch) uint256 in3 = GetRandHash(); uint256 res; - CDBBatch batch(&dbw.GetObfuscateKey()); + CDBBatch batch(dbw); batch.Write(key, in); batch.Write(key2, in2); @@ -156,7 +156,7 @@ BOOST_AUTO_TEST_CASE(existing_data_no_obfuscate) BOOST_CHECK_EQUAL(res2.ToString(), in.ToString()); BOOST_CHECK(!odbw.IsEmpty()); // There should be existing data - BOOST_CHECK(is_null_key(odbw.GetObfuscateKey())); // The key should be an empty string + BOOST_CHECK(is_null_key(dbwrapper_private::GetObfuscateKey(odbw))); // The key should be an empty string uint256 in2 = GetRandHash(); uint256 res3; @@ -193,7 +193,7 @@ BOOST_AUTO_TEST_CASE(existing_data_reindex) // Check that the key/val we wrote with unobfuscated wrapper doesn't exist uint256 res2; BOOST_CHECK(!odbw.Read(key, res2)); - BOOST_CHECK(!is_null_key(odbw.GetObfuscateKey())); + BOOST_CHECK(!is_null_key(dbwrapper_private::GetObfuscateKey(odbw))); uint256 in2 = GetRandHash(); uint256 res3; @@ -203,5 +203,125 @@ BOOST_AUTO_TEST_CASE(existing_data_reindex) BOOST_CHECK(odbw.Read(key, res3)); BOOST_CHECK_EQUAL(res3.ToString(), in2.ToString()); } - + +BOOST_AUTO_TEST_CASE(iterator_ordering) +{ + path ph = temp_directory_path() / unique_path(); + CDBWrapper dbw(ph, (1 << 20), true, false, false); + for (int x=0x00; x<256; ++x) { + uint8_t key = x; + uint32_t value = x*x; + BOOST_CHECK(dbw.Write(key, value)); + } + + boost::scoped_ptr it(const_cast(&dbw)->NewIterator()); + for (int c=0; c<2; ++c) { + int seek_start; + if (c == 0) + seek_start = 0x00; + else + seek_start = 0x80; + it->Seek((uint8_t)seek_start); + for (int x=seek_start; x<256; ++x) { + uint8_t key; + uint32_t value; + BOOST_CHECK(it->Valid()); + if (!it->Valid()) // Avoid spurious errors about invalid iterator's key and value in case of failure + break; + BOOST_CHECK(it->GetKey(key)); + BOOST_CHECK(it->GetValue(value)); + BOOST_CHECK_EQUAL(key, x); + BOOST_CHECK_EQUAL(value, x*x); + it->Next(); + } + BOOST_CHECK(!it->Valid()); + } +} + +struct StringContentsSerializer { + // Used to make two serialized objects the same while letting them have a different lengths + // This is a terrible idea + string str; + StringContentsSerializer() {} + StringContentsSerializer(const string& inp) : str(inp) {} + + StringContentsSerializer& operator+=(const string& s) { + str += s; + return *this; + } + StringContentsSerializer& operator+=(const StringContentsSerializer& s) { return *this += s.str; } + + ADD_SERIALIZE_METHODS; + + template + inline void SerializationOp(Stream& s, Operation ser_action, int nType, int nVersion) { + if (ser_action.ForRead()) { + str.clear(); + char c = 0; + while (true) { + try { + READWRITE(c); + str.push_back(c); + } catch (const std::ios_base::failure& e) { + break; + } + } + } else { + for (size_t i = 0; i < str.size(); i++) + READWRITE(str[i]); + } + } +}; + +BOOST_AUTO_TEST_CASE(iterator_string_ordering) +{ + char buf[10]; + + path ph = temp_directory_path() / unique_path(); + CDBWrapper dbw(ph, (1 << 20), true, false, false); + for (int x=0x00; x<10; ++x) { + for (int y = 0; y < 10; y++) { + sprintf(buf, "%d", x); + StringContentsSerializer key(buf); + for (int z = 0; z < y; z++) + key += key; + uint32_t value = x*x; + BOOST_CHECK(dbw.Write(key, value)); + } + } + + boost::scoped_ptr it(const_cast(&dbw)->NewIterator()); + for (int c=0; c<2; ++c) { + int seek_start; + if (c == 0) + seek_start = 0; + else + seek_start = 5; + sprintf(buf, "%d", seek_start); + StringContentsSerializer seek_key(buf); + it->Seek(seek_key); + for (int x=seek_start; x<10; ++x) { + for (int y = 0; y < 10; y++) { + sprintf(buf, "%d", x); + string exp_key(buf); + for (int z = 0; z < y; z++) + exp_key += exp_key; + StringContentsSerializer key; + uint32_t value; + BOOST_CHECK(it->Valid()); + if (!it->Valid()) // Avoid spurious errors about invalid iterator's key and value in case of failure + break; + BOOST_CHECK(it->GetKey(key)); + BOOST_CHECK(it->GetValue(value)); + BOOST_CHECK_EQUAL(key.str, exp_key); + BOOST_CHECK_EQUAL(value, x*x); + it->Next(); + } + } + BOOST_CHECK(!it->Valid()); + } +} + + + BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/hash_tests.cpp b/src/test/hash_tests.cpp index 35079d1614345..82d61209b5e50 100644 --- a/src/test/hash_tests.cpp +++ b/src/test/hash_tests.cpp @@ -47,4 +47,81 @@ BOOST_AUTO_TEST_CASE(murmurhash3) #undef T } +/* + SipHash-2-4 output with + k = 00 01 02 ... + and + in = (empty string) + in = 00 (1 byte) + in = 00 01 (2 bytes) + in = 00 01 02 (3 bytes) + ... + in = 00 01 02 ... 3e (63 bytes) + + from: https://131002.net/siphash/siphash24.c +*/ +uint64_t siphash_4_2_testvec[] = { + 0x726fdb47dd0e0e31, 0x74f839c593dc67fd, 0x0d6c8009d9a94f5a, 0x85676696d7fb7e2d, + 0xcf2794e0277187b7, 0x18765564cd99a68d, 0xcbc9466e58fee3ce, 0xab0200f58b01d137, + 0x93f5f5799a932462, 0x9e0082df0ba9e4b0, 0x7a5dbbc594ddb9f3, 0xf4b32f46226bada7, + 0x751e8fbc860ee5fb, 0x14ea5627c0843d90, 0xf723ca908e7af2ee, 0xa129ca6149be45e5, + 0x3f2acc7f57c29bdb, 0x699ae9f52cbe4794, 0x4bc1b3f0968dd39c, 0xbb6dc91da77961bd, + 0xbed65cf21aa2ee98, 0xd0f2cbb02e3b67c7, 0x93536795e3a33e88, 0xa80c038ccd5ccec8, + 0xb8ad50c6f649af94, 0xbce192de8a85b8ea, 0x17d835b85bbb15f3, 0x2f2e6163076bcfad, + 0xde4daaaca71dc9a5, 0xa6a2506687956571, 0xad87a3535c49ef28, 0x32d892fad841c342, + 0x7127512f72f27cce, 0xa7f32346f95978e3, 0x12e0b01abb051238, 0x15e034d40fa197ae, + 0x314dffbe0815a3b4, 0x027990f029623981, 0xcadcd4e59ef40c4d, 0x9abfd8766a33735c, + 0x0e3ea96b5304a7d0, 0xad0c42d6fc585992, 0x187306c89bc215a9, 0xd4a60abcf3792b95, + 0xf935451de4f21df2, 0xa9538f0419755787, 0xdb9acddff56ca510, 0xd06c98cd5c0975eb, + 0xe612a3cb9ecba951, 0xc766e62cfcadaf96, 0xee64435a9752fe72, 0xa192d576b245165a, + 0x0a8787bf8ecb74b2, 0x81b3e73d20b49b6f, 0x7fa8220ba3b2ecea, 0x245731c13ca42499, + 0xb78dbfaf3a8d83bd, 0xea1ad565322a1a0b, 0x60e61c23a3795013, 0x6606d7e446282b93, + 0x6ca4ecb15c5f91e1, 0x9f626da15c9625f3, 0xe51b38608ef25f57, 0x958a324ceb064572 +}; + +BOOST_AUTO_TEST_CASE(siphash) +{ + CSipHasher hasher(0x0706050403020100ULL, 0x0F0E0D0C0B0A0908ULL); + BOOST_CHECK_EQUAL(hasher.Finalize(), 0x726fdb47dd0e0e31ull); + static const unsigned char t0[1] = {0}; + hasher.Write(t0, 1); + BOOST_CHECK_EQUAL(hasher.Finalize(), 0x74f839c593dc67fdull); + static const unsigned char t1[7] = {1,2,3,4,5,6,7}; + hasher.Write(t1, 7); + BOOST_CHECK_EQUAL(hasher.Finalize(), 0x93f5f5799a932462ull); + hasher.Write(0x0F0E0D0C0B0A0908ULL); + BOOST_CHECK_EQUAL(hasher.Finalize(), 0x3f2acc7f57c29bdbull); + static const unsigned char t2[2] = {16,17}; + hasher.Write(t2, 2); + BOOST_CHECK_EQUAL(hasher.Finalize(), 0x4bc1b3f0968dd39cull); + static const unsigned char t3[9] = {18,19,20,21,22,23,24,25,26}; + hasher.Write(t3, 9); + BOOST_CHECK_EQUAL(hasher.Finalize(), 0x2f2e6163076bcfadull); + static const unsigned char t4[5] = {27,28,29,30,31}; + hasher.Write(t4, 5); + BOOST_CHECK_EQUAL(hasher.Finalize(), 0x7127512f72f27cceull); + hasher.Write(0x2726252423222120ULL); + BOOST_CHECK_EQUAL(hasher.Finalize(), 0x0e3ea96b5304a7d0ull); + hasher.Write(0x2F2E2D2C2B2A2928ULL); + BOOST_CHECK_EQUAL(hasher.Finalize(), 0xe612a3cb9ecba951ull); + + BOOST_CHECK_EQUAL(SipHashUint256(0x0706050403020100ULL, 0x0F0E0D0C0B0A0908ULL, uint256S("1f1e1d1c1b1a191817161514131211100f0e0d0c0b0a09080706050403020100")), 0x7127512f72f27cceull); + + // Check test vectors from spec, one byte at a time + CSipHasher hasher2(0x0706050403020100ULL, 0x0F0E0D0C0B0A0908ULL); + for (uint8_t x=0; x limit: 1000 CHECKMULTISIG + 1 tx.vin.resize(1); // NOTE: OP_NOP is used to force 20 SigOps for the CHECKMULTISIG @@ -131,31 +136,31 @@ BOOST_AUTO_TEST_CASE(CreateNewBlock_validity) tx.vin[0].prevout.hash = txFirst[0]->GetHash(); tx.vin[0].prevout.n = 0; tx.vout.resize(1); - tx.vout[0].nValue = 5000000000LL; + tx.vout[0].nValue = BLOCKSUBSIDY; for (unsigned int i = 0; i < 1001; ++i) { - tx.vout[0].nValue -= 1000000; + tx.vout[0].nValue -= LOWFEE; hash = tx.GetHash(); bool spendsCoinbase = (i == 0) ? true : false; // only first tx spends coinbase // If we don't set the # of sig ops in the CTxMemPoolEntry, template creation fails - mempool.addUnchecked(hash, entry.Fee(1000000).Time(GetTime()).SpendsCoinbase(spendsCoinbase).FromTx(tx)); + mempool.addUnchecked(hash, entry.Fee(LOWFEE).Time(GetTime()).SpendsCoinbase(spendsCoinbase).FromTx(tx)); tx.vin[0].prevout.hash = hash; } - BOOST_CHECK_THROW(CreateNewBlock(chainparams, scriptPubKey), std::runtime_error); + BOOST_CHECK_THROW(BlockAssembler(chainparams).CreateNewBlock(scriptPubKey), std::runtime_error); mempool.clear(); tx.vin[0].prevout.hash = txFirst[0]->GetHash(); - tx.vout[0].nValue = 5000000000LL; + tx.vout[0].nValue = BLOCKSUBSIDY; for (unsigned int i = 0; i < 1001; ++i) { - tx.vout[0].nValue -= 1000000; + tx.vout[0].nValue -= LOWFEE; hash = tx.GetHash(); bool spendsCoinbase = (i == 0) ? true : false; // only first tx spends coinbase // If we do set the # of sig ops in the CTxMemPoolEntry, template creation passes - mempool.addUnchecked(hash, entry.Fee(1000000).Time(GetTime()).SpendsCoinbase(spendsCoinbase).SigOpsCost(80).FromTx(tx)); + mempool.addUnchecked(hash, entry.Fee(LOWFEE).Time(GetTime()).SpendsCoinbase(spendsCoinbase).SigOpsCost(80).FromTx(tx)); tx.vin[0].prevout.hash = hash; } - BOOST_CHECK(pblocktemplate = CreateNewBlock(chainparams, scriptPubKey)); + BOOST_CHECK(pblocktemplate = BlockAssembler(chainparams).CreateNewBlock(scriptPubKey)); delete pblocktemplate; mempool.clear(); @@ -167,40 +172,40 @@ BOOST_AUTO_TEST_CASE(CreateNewBlock_validity) tx.vin[0].scriptSig << vchData << OP_DROP; tx.vin[0].scriptSig << OP_1; tx.vin[0].prevout.hash = txFirst[0]->GetHash(); - tx.vout[0].nValue = 5000000000LL; + tx.vout[0].nValue = BLOCKSUBSIDY; for (unsigned int i = 0; i < 128; ++i) { - tx.vout[0].nValue -= 10000000; + tx.vout[0].nValue -= LOWFEE; hash = tx.GetHash(); bool spendsCoinbase = (i == 0) ? true : false; // only first tx spends coinbase - mempool.addUnchecked(hash, entry.Fee(1000000).Time(GetTime()).SpendsCoinbase(spendsCoinbase).FromTx(tx)); + mempool.addUnchecked(hash, entry.Fee(LOWFEE).Time(GetTime()).SpendsCoinbase(spendsCoinbase).FromTx(tx)); tx.vin[0].prevout.hash = hash; } - BOOST_CHECK(pblocktemplate = CreateNewBlock(chainparams, scriptPubKey)); + BOOST_CHECK(pblocktemplate = BlockAssembler(chainparams).CreateNewBlock(scriptPubKey)); delete pblocktemplate; mempool.clear(); // orphan in mempool, template creation fails hash = tx.GetHash(); - mempool.addUnchecked(hash, entry.Fee(1000000).Time(GetTime()).FromTx(tx)); - BOOST_CHECK_THROW(CreateNewBlock(chainparams, scriptPubKey), std::runtime_error); + mempool.addUnchecked(hash, entry.Fee(LOWFEE).Time(GetTime()).FromTx(tx)); + BOOST_CHECK_THROW(BlockAssembler(chainparams).CreateNewBlock(scriptPubKey), std::runtime_error); mempool.clear(); // child with higher priority than parent tx.vin[0].scriptSig = CScript() << OP_1; tx.vin[0].prevout.hash = txFirst[1]->GetHash(); - tx.vout[0].nValue = 4900000000LL; + tx.vout[0].nValue = BLOCKSUBSIDY-HIGHFEE; hash = tx.GetHash(); - mempool.addUnchecked(hash, entry.Fee(100000000LL).Time(GetTime()).SpendsCoinbase(true).FromTx(tx)); + mempool.addUnchecked(hash, entry.Fee(HIGHFEE).Time(GetTime()).SpendsCoinbase(true).FromTx(tx)); tx.vin[0].prevout.hash = hash; tx.vin.resize(2); tx.vin[1].scriptSig = CScript() << OP_1; tx.vin[1].prevout.hash = txFirst[0]->GetHash(); tx.vin[1].prevout.n = 0; - tx.vout[0].nValue = 5900000000LL; + tx.vout[0].nValue = tx.vout[0].nValue+BLOCKSUBSIDY-HIGHERFEE; //First txn output + fresh coinbase - new txn fee hash = tx.GetHash(); - mempool.addUnchecked(hash, entry.Fee(400000000LL).Time(GetTime()).SpendsCoinbase(true).FromTx(tx)); - BOOST_CHECK(pblocktemplate = CreateNewBlock(chainparams, scriptPubKey)); + mempool.addUnchecked(hash, entry.Fee(HIGHERFEE).Time(GetTime()).SpendsCoinbase(true).FromTx(tx)); + BOOST_CHECK(pblocktemplate = BlockAssembler(chainparams).CreateNewBlock(scriptPubKey)); delete pblocktemplate; mempool.clear(); @@ -211,38 +216,38 @@ BOOST_AUTO_TEST_CASE(CreateNewBlock_validity) tx.vout[0].nValue = 0; hash = tx.GetHash(); // give it a fee so it'll get mined - mempool.addUnchecked(hash, entry.Fee(100000).Time(GetTime()).SpendsCoinbase(false).FromTx(tx)); - BOOST_CHECK_THROW(CreateNewBlock(chainparams, scriptPubKey), std::runtime_error); + mempool.addUnchecked(hash, entry.Fee(LOWFEE).Time(GetTime()).SpendsCoinbase(false).FromTx(tx)); + BOOST_CHECK_THROW(BlockAssembler(chainparams).CreateNewBlock(scriptPubKey), std::runtime_error); mempool.clear(); // invalid (pre-p2sh) txn in mempool, template creation fails tx.vin[0].prevout.hash = txFirst[0]->GetHash(); tx.vin[0].prevout.n = 0; tx.vin[0].scriptSig = CScript() << OP_1; - tx.vout[0].nValue = 4900000000LL; + tx.vout[0].nValue = BLOCKSUBSIDY-LOWFEE; script = CScript() << OP_0; tx.vout[0].scriptPubKey = GetScriptForDestination(CScriptID(script)); hash = tx.GetHash(); - mempool.addUnchecked(hash, entry.Fee(10000000L).Time(GetTime()).SpendsCoinbase(true).FromTx(tx)); + mempool.addUnchecked(hash, entry.Fee(LOWFEE).Time(GetTime()).SpendsCoinbase(true).FromTx(tx)); tx.vin[0].prevout.hash = hash; tx.vin[0].scriptSig = CScript() << std::vector(script.begin(), script.end()); - tx.vout[0].nValue -= 1000000; + tx.vout[0].nValue -= LOWFEE; hash = tx.GetHash(); - mempool.addUnchecked(hash, entry.Fee(1000000).Time(GetTime()).SpendsCoinbase(false).FromTx(tx)); - BOOST_CHECK_THROW(CreateNewBlock(chainparams, scriptPubKey), std::runtime_error); + mempool.addUnchecked(hash, entry.Fee(LOWFEE).Time(GetTime()).SpendsCoinbase(false).FromTx(tx)); + BOOST_CHECK_THROW(BlockAssembler(chainparams).CreateNewBlock(scriptPubKey), std::runtime_error); mempool.clear(); // double spend txn pair in mempool, template creation fails tx.vin[0].prevout.hash = txFirst[0]->GetHash(); tx.vin[0].scriptSig = CScript() << OP_1; - tx.vout[0].nValue = 4900000000LL; + tx.vout[0].nValue = BLOCKSUBSIDY-HIGHFEE; tx.vout[0].scriptPubKey = CScript() << OP_1; hash = tx.GetHash(); - mempool.addUnchecked(hash, entry.Fee(100000000L).Time(GetTime()).SpendsCoinbase(true).FromTx(tx)); + mempool.addUnchecked(hash, entry.Fee(HIGHFEE).Time(GetTime()).SpendsCoinbase(true).FromTx(tx)); tx.vout[0].scriptPubKey = CScript() << OP_2; hash = tx.GetHash(); - mempool.addUnchecked(hash, entry.Fee(100000000L).Time(GetTime()).SpendsCoinbase(true).FromTx(tx)); - BOOST_CHECK_THROW(CreateNewBlock(chainparams, scriptPubKey), std::runtime_error); + mempool.addUnchecked(hash, entry.Fee(HIGHFEE).Time(GetTime()).SpendsCoinbase(true).FromTx(tx)); + BOOST_CHECK_THROW(BlockAssembler(chainparams).CreateNewBlock(scriptPubKey), std::runtime_error); mempool.clear(); // subsidy changing @@ -258,7 +263,7 @@ BOOST_AUTO_TEST_CASE(CreateNewBlock_validity) next->BuildSkip(); chainActive.SetTip(next); } - BOOST_CHECK(pblocktemplate = CreateNewBlock(chainparams, scriptPubKey)); + BOOST_CHECK(pblocktemplate = BlockAssembler(chainparams).CreateNewBlock(scriptPubKey)); delete pblocktemplate; // Extend to a 210000-long block chain. while (chainActive.Tip()->nHeight < 210000) { @@ -271,7 +276,7 @@ BOOST_AUTO_TEST_CASE(CreateNewBlock_validity) next->BuildSkip(); chainActive.SetTip(next); } - BOOST_CHECK(pblocktemplate = CreateNewBlock(chainparams, scriptPubKey)); + BOOST_CHECK(pblocktemplate = BlockAssembler(chainparams).CreateNewBlock(scriptPubKey)); delete pblocktemplate; // Delete the dummy blocks again. while (chainActive.Tip()->nHeight > nHeight) { @@ -298,11 +303,11 @@ BOOST_AUTO_TEST_CASE(CreateNewBlock_validity) tx.vin[0].nSequence = chainActive.Tip()->nHeight + 1; // txFirst[0] is the 2nd block prevheights[0] = baseheight + 1; tx.vout.resize(1); - tx.vout[0].nValue = 4900000000LL; + tx.vout[0].nValue = BLOCKSUBSIDY-HIGHFEE; tx.vout[0].scriptPubKey = CScript() << OP_1; tx.nLockTime = 0; hash = tx.GetHash(); - mempool.addUnchecked(hash, entry.Fee(100000000L).Time(GetTime()).SpendsCoinbase(true).FromTx(tx)); + mempool.addUnchecked(hash, entry.Fee(HIGHFEE).Time(GetTime()).SpendsCoinbase(true).FromTx(tx)); BOOST_CHECK(CheckFinalTx(tx, flags)); // Locktime passes BOOST_CHECK(!TestSequenceLocks(tx, flags)); // Sequence locks fail BOOST_CHECK(SequenceLocks(tx, flags, &prevheights, CreateBlockIndex(chainActive.Tip()->nHeight + 2))); // Sequence locks pass on 2nd block @@ -358,7 +363,7 @@ BOOST_AUTO_TEST_CASE(CreateNewBlock_validity) tx.vin[0].nSequence = CTxIn::SEQUENCE_LOCKTIME_TYPE_FLAG | 1; BOOST_CHECK(!TestSequenceLocks(tx, flags)); // Sequence locks fail - BOOST_CHECK(pblocktemplate = CreateNewBlock(chainparams, scriptPubKey)); + BOOST_CHECK(pblocktemplate = BlockAssembler(chainparams).CreateNewBlock(scriptPubKey)); // None of the of the absolute height/time locked tx should have made // it into the template because we still check IsFinalTx in CreateNewBlock, @@ -372,7 +377,7 @@ BOOST_AUTO_TEST_CASE(CreateNewBlock_validity) chainActive.Tip()->nHeight++; SetMockTime(chainActive.Tip()->GetMedianTimePast() + 1); - BOOST_CHECK(pblocktemplate = CreateNewBlock(chainparams, scriptPubKey)); + BOOST_CHECK(pblocktemplate = BlockAssembler(chainparams).CreateNewBlock(scriptPubKey)); BOOST_CHECK_EQUAL(pblocktemplate->block.vtx.size(), 5); delete pblocktemplate; @@ -380,8 +385,8 @@ BOOST_AUTO_TEST_CASE(CreateNewBlock_validity) SetMockTime(0); mempool.clear(); - BOOST_FOREACH(CTransaction *tx, txFirst) - delete tx; + BOOST_FOREACH(CTransaction *_tx, txFirst) + delete _tx; fCheckpointsEnabled = true; } diff --git a/src/test/net_tests.cpp b/src/test/net_tests.cpp new file mode 100644 index 0000000000000..d005d6a1633e9 --- /dev/null +++ b/src/test/net_tests.cpp @@ -0,0 +1,145 @@ +// Copyright (c) 2012-2016 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. +#include "addrman.h" +#include "test/test_bitcoin.h" +#include +#include +#include "hash.h" +#include "serialize.h" +#include "streams.h" +#include "net.h" +#include "chainparams.h" + +using namespace std; + +class CAddrManSerializationMock : public CAddrMan +{ +public: + virtual void Serialize(CDataStream& s, int nType, int nVersionDummy) const = 0; + + //! Ensure that bucket placement is always the same for testing purposes. + void MakeDeterministic() + { + nKey.SetNull(); + seed_insecure_rand(true); + } +}; + +class CAddrManUncorrupted : public CAddrManSerializationMock +{ +public: + void Serialize(CDataStream& s, int nType, int nVersionDummy) const + { + CAddrMan::Serialize(s, nType, nVersionDummy); + } +}; + +class CAddrManCorrupted : public CAddrManSerializationMock +{ +public: + void Serialize(CDataStream& s, int nType, int nVersionDummy) const + { + // Produces corrupt output that claims addrman has 20 addrs when it only has one addr. + unsigned char nVersion = 1; + s << nVersion; + s << ((unsigned char)32); + s << nKey; + s << 10; // nNew + s << 10; // nTried + + int nUBuckets = ADDRMAN_NEW_BUCKET_COUNT ^ (1 << 30); + s << nUBuckets; + + CAddress addr = CAddress(CService("252.1.1.1", 7777), NODE_NONE); + CAddrInfo info = CAddrInfo(addr, CNetAddr("252.2.2.2")); + s << info; + } +}; + +CDataStream AddrmanToStream(CAddrManSerializationMock& addrman) +{ + CDataStream ssPeersIn(SER_DISK, CLIENT_VERSION); + ssPeersIn << FLATDATA(Params().MessageStart()); + ssPeersIn << addrman; + std::string str = ssPeersIn.str(); + vector vchData(str.begin(), str.end()); + return CDataStream(vchData, SER_DISK, CLIENT_VERSION); +} + +BOOST_FIXTURE_TEST_SUITE(net_tests, BasicTestingSetup) + +BOOST_AUTO_TEST_CASE(caddrdb_read) +{ + CAddrManUncorrupted addrmanUncorrupted; + addrmanUncorrupted.MakeDeterministic(); + + CService addr1 = CService("250.7.1.1", 8333); + CService addr2 = CService("250.7.2.2", 9999); + CService addr3 = CService("250.7.3.3", 9999); + + // Add three addresses to new table. + addrmanUncorrupted.Add(CAddress(addr1, NODE_NONE), CService("252.5.1.1", 8333)); + addrmanUncorrupted.Add(CAddress(addr2, NODE_NONE), CService("252.5.1.1", 8333)); + addrmanUncorrupted.Add(CAddress(addr3, NODE_NONE), CService("252.5.1.1", 8333)); + + // Test that the de-serialization does not throw an exception. + CDataStream ssPeers1 = AddrmanToStream(addrmanUncorrupted); + bool exceptionThrown = false; + CAddrMan addrman1; + + BOOST_CHECK(addrman1.size() == 0); + try { + unsigned char pchMsgTmp[4]; + ssPeers1 >> FLATDATA(pchMsgTmp); + ssPeers1 >> addrman1; + } catch (const std::exception& e) { + exceptionThrown = true; + } + + BOOST_CHECK(addrman1.size() == 3); + BOOST_CHECK(exceptionThrown == false); + + // Test that CAddrDB::Read creates an addrman with the correct number of addrs. + CDataStream ssPeers2 = AddrmanToStream(addrmanUncorrupted); + + CAddrMan addrman2; + CAddrDB adb; + BOOST_CHECK(addrman2.size() == 0); + adb.Read(addrman2, ssPeers2); + BOOST_CHECK(addrman2.size() == 3); +} + + +BOOST_AUTO_TEST_CASE(caddrdb_read_corrupted) +{ + CAddrManCorrupted addrmanCorrupted; + addrmanCorrupted.MakeDeterministic(); + + // Test that the de-serialization of corrupted addrman throws an exception. + CDataStream ssPeers1 = AddrmanToStream(addrmanCorrupted); + bool exceptionThrown = false; + CAddrMan addrman1; + BOOST_CHECK(addrman1.size() == 0); + try { + unsigned char pchMsgTmp[4]; + ssPeers1 >> FLATDATA(pchMsgTmp); + ssPeers1 >> addrman1; + } catch (const std::exception& e) { + exceptionThrown = true; + } + // Even through de-serialization failed addrman is not left in a clean state. + BOOST_CHECK(addrman1.size() == 1); + BOOST_CHECK(exceptionThrown); + + // Test that CAddrDB::Read leaves addrman in a clean state if de-serialization fails. + CDataStream ssPeers2 = AddrmanToStream(addrmanCorrupted); + + CAddrMan addrman2; + CAddrDB adb; + BOOST_CHECK(addrman2.size() == 0); + adb.Read(addrman2, ssPeers2); + BOOST_CHECK(addrman2.size() == 0); +} + +BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/pmt_tests.cpp b/src/test/pmt_tests.cpp index 2f3f607889498..74ffe0cc74045 100644 --- a/src/test/pmt_tests.cpp +++ b/src/test/pmt_tests.cpp @@ -37,8 +37,8 @@ BOOST_AUTO_TEST_CASE(pmt_test1) seed_insecure_rand(false); static const unsigned int nTxCounts[] = {1, 4, 7, 17, 56, 100, 127, 256, 312, 513, 1000, 4095}; - for (int n = 0; n < 12; n++) { - unsigned int nTx = nTxCounts[n]; + for (int i = 0; i < 12; i++) { + unsigned int nTx = nTxCounts[i]; // build a block with some dummy transactions CBlock block; diff --git a/src/test/policyestimator_tests.cpp b/src/test/policyestimator_tests.cpp index 4dd1906944729..5c902387f1214 100644 --- a/src/test/policyestimator_tests.cpp +++ b/src/test/policyestimator_tests.cpp @@ -75,9 +75,9 @@ BOOST_AUTO_TEST_CASE(BlockPolicyEstimates) // 9/10 blocks add 2nd highest and so on until ... // 1/10 blocks add lowest fee/pri transactions while (txHashes[9-h].size()) { - CTransaction btx; - if (mpool.lookup(txHashes[9-h].back(), btx)) - block.push_back(btx); + std::shared_ptr ptx = mpool.get(txHashes[9-h].back()); + if (ptx) + block.push_back(*ptx); txHashes[9-h].pop_back(); } } @@ -161,9 +161,9 @@ BOOST_AUTO_TEST_CASE(BlockPolicyEstimates) // Estimates should still not be below original for (int j = 0; j < 10; j++) { while(txHashes[j].size()) { - CTransaction btx; - if (mpool.lookup(txHashes[j].back(), btx)) - block.push_back(btx); + std::shared_ptr ptx = mpool.get(txHashes[j].back()); + if (ptx) + block.push_back(*ptx); txHashes[j].pop_back(); } } @@ -182,9 +182,9 @@ BOOST_AUTO_TEST_CASE(BlockPolicyEstimates) tx.vin[0].prevout.n = 10000*blocknum+100*j+k; uint256 hash = tx.GetHash(); mpool.addUnchecked(hash, entry.Fee(feeV[k/4][j]).Time(GetTime()).Priority(priV[k/4][j]).Height(blocknum).FromTx(tx, &mpool)); - CTransaction btx; - if (mpool.lookup(hash, btx)) - block.push_back(btx); + std::shared_ptr ptx = mpool.get(hash); + if (ptx) + block.push_back(*ptx); } } mpool.removeForBlock(block, ++blocknum, dummyConflicted); diff --git a/src/test/prevector_tests.cpp b/src/test/prevector_tests.cpp index b39b90353053c..d1407c1da9339 100644 --- a/src/test/prevector_tests.cpp +++ b/src/test/prevector_tests.cpp @@ -192,8 +192,8 @@ BOOST_AUTO_TEST_CASE(PrevectorTestInt) if (((r >> 21) % 32) == 7) { int values[4]; int num = 1 + (insecure_rand() % 4); - for (int i = 0; i < num; i++) { - values[i] = insecure_rand(); + for (int k = 0; k < num; k++) { + values[k] = insecure_rand(); } test.insert_range(insecure_rand() % (test.size() + 1), values, values + num); } diff --git a/src/test/scheduler_tests.cpp b/src/test/scheduler_tests.cpp index 9acd0e2430a30..aa12dfbd54f8f 100644 --- a/src/test/scheduler_tests.cpp +++ b/src/test/scheduler_tests.cpp @@ -40,7 +40,6 @@ static void MicroSleep(uint64_t n) #endif } -#if 0 /* Disabled for now because there is a race condition issue in this test - see #6540 */ BOOST_AUTO_TEST_CASE(manythreads) { seed_insecure_rand(false); @@ -116,6 +115,5 @@ BOOST_AUTO_TEST_CASE(manythreads) } BOOST_CHECK_EQUAL(counterSum, 200); } -#endif BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/script_tests.cpp b/src/test/script_tests.cpp index fee5e57692f44..5a9aaf9bc0738 100644 --- a/src/test/script_tests.cpp +++ b/src/test/script_tests.cpp @@ -505,7 +505,10 @@ BOOST_AUTO_TEST_CASE(script_build) tests.push_back(TestBuilder(CScript() << ToByteVector(keys.pubkey0C) << OP_CHECKSIG, "P2SH(P2PK), bad redeemscript", SCRIPT_VERIFY_P2SH, true ).PushSig(keys.key0).PushRedeem().DamagePush(10).ScriptError(SCRIPT_ERR_EVAL_FALSE)); - + + tests.push_back(TestBuilder(CScript() << OP_DUP << OP_HASH160 << ToByteVector(keys.pubkey0.GetID()) << OP_EQUALVERIFY << OP_CHECKSIG, + "P2SH(P2PKH)", SCRIPT_VERIFY_P2SH, true + ).PushSig(keys.key0).Push(keys.pubkey0).PushRedeem()); tests.push_back(TestBuilder(CScript() << OP_DUP << OP_HASH160 << ToByteVector(keys.pubkey1.GetID()) << OP_EQUALVERIFY << OP_CHECKSIG, "P2SH(P2PKH), bad sig but no VERIFY_P2SH", 0, true ).PushSig(keys.key0).DamagePush(10).PushRedeem()); @@ -1227,4 +1230,121 @@ BOOST_AUTO_TEST_CASE(script_GetScriptAsm) BOOST_CHECK_EQUAL(derSig + "83 " + pubKey, ScriptToAsmStr(CScript() << ToByteVector(ParseHex(derSig + "83")) << vchPubKey)); } +static CScript +ScriptFromHex(const char* hex) +{ + std::vector data = ParseHex(hex); + return CScript(data.begin(), data.end()); +} + + +BOOST_AUTO_TEST_CASE(script_FindAndDelete) +{ + // Exercise the FindAndDelete functionality + CScript s; + CScript d; + CScript expect; + + s = CScript() << OP_1 << OP_2; + d = CScript(); // delete nothing should be a no-op + expect = s; + BOOST_CHECK_EQUAL(s.FindAndDelete(d), 0); + BOOST_CHECK(s == expect); + + s = CScript() << OP_1 << OP_2 << OP_3; + d = CScript() << OP_2; + expect = CScript() << OP_1 << OP_3; + BOOST_CHECK_EQUAL(s.FindAndDelete(d), 1); + BOOST_CHECK(s == expect); + + s = CScript() << OP_3 << OP_1 << OP_3 << OP_3 << OP_4 << OP_3; + d = CScript() << OP_3; + expect = CScript() << OP_1 << OP_4; + BOOST_CHECK_EQUAL(s.FindAndDelete(d), 4); + BOOST_CHECK(s == expect); + + s = ScriptFromHex("0302ff03"); // PUSH 0x02ff03 onto stack + d = ScriptFromHex("0302ff03"); + expect = CScript(); + BOOST_CHECK_EQUAL(s.FindAndDelete(d), 1); + BOOST_CHECK(s == expect); + + s = ScriptFromHex("0302ff030302ff03"); // PUSH 0x2ff03 PUSH 0x2ff03 + d = ScriptFromHex("0302ff03"); + expect = CScript(); + BOOST_CHECK_EQUAL(s.FindAndDelete(d), 2); + BOOST_CHECK(s == expect); + + s = ScriptFromHex("0302ff030302ff03"); + d = ScriptFromHex("02"); + expect = s; // FindAndDelete matches entire opcodes + BOOST_CHECK_EQUAL(s.FindAndDelete(d), 0); + BOOST_CHECK(s == expect); + + s = ScriptFromHex("0302ff030302ff03"); + d = ScriptFromHex("ff"); + expect = s; + BOOST_CHECK_EQUAL(s.FindAndDelete(d), 0); + BOOST_CHECK(s == expect); + + // This is an odd edge case: strip of the push-three-bytes + // prefix, leaving 02ff03 which is push-two-bytes: + s = ScriptFromHex("0302ff030302ff03"); + d = ScriptFromHex("03"); + expect = CScript() << ParseHex("ff03") << ParseHex("ff03"); + BOOST_CHECK_EQUAL(s.FindAndDelete(d), 2); + BOOST_CHECK(s == expect); + + // Byte sequence that spans multiple opcodes: + s = ScriptFromHex("02feed5169"); // PUSH(0xfeed) OP_1 OP_VERIFY + d = ScriptFromHex("feed51"); + expect = s; + BOOST_CHECK_EQUAL(s.FindAndDelete(d), 0); // doesn't match 'inside' opcodes + BOOST_CHECK(s == expect); + + s = ScriptFromHex("02feed5169"); // PUSH(0xfeed) OP_1 OP_VERIFY + d = ScriptFromHex("02feed51"); + expect = ScriptFromHex("69"); + BOOST_CHECK_EQUAL(s.FindAndDelete(d), 1); + BOOST_CHECK(s == expect); + + s = ScriptFromHex("516902feed5169"); + d = ScriptFromHex("feed51"); + expect = s; + BOOST_CHECK_EQUAL(s.FindAndDelete(d), 0); + BOOST_CHECK(s == expect); + + s = ScriptFromHex("516902feed5169"); + d = ScriptFromHex("02feed51"); + expect = ScriptFromHex("516969"); + BOOST_CHECK_EQUAL(s.FindAndDelete(d), 1); + BOOST_CHECK(s == expect); + + s = CScript() << OP_0 << OP_0 << OP_1 << OP_1; + d = CScript() << OP_0 << OP_1; + expect = CScript() << OP_0 << OP_1; // FindAndDelete is single-pass + BOOST_CHECK_EQUAL(s.FindAndDelete(d), 1); + BOOST_CHECK(s == expect); + + s = CScript() << OP_0 << OP_0 << OP_1 << OP_0 << OP_1 << OP_1; + d = CScript() << OP_0 << OP_1; + expect = CScript() << OP_0 << OP_1; // FindAndDelete is single-pass + BOOST_CHECK_EQUAL(s.FindAndDelete(d), 2); + BOOST_CHECK(s == expect); + + // Another weird edge case: + // End with invalid push (not enough data)... + s = ScriptFromHex("0003feed"); + d = ScriptFromHex("03feed"); // ... can remove the invalid push + expect = ScriptFromHex("00"); + BOOST_CHECK_EQUAL(s.FindAndDelete(d), 1); + BOOST_CHECK(s == expect); + + s = ScriptFromHex("0003feed"); + d = ScriptFromHex("00"); + expect = ScriptFromHex("03feed"); + BOOST_CHECK_EQUAL(s.FindAndDelete(d), 1); + BOOST_CHECK(s == expect); +} + BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/sighash_tests.cpp b/src/test/sighash_tests.cpp index 44d7b7cc66e45..4a48347b702dd 100644 --- a/src/test/sighash_tests.cpp +++ b/src/test/sighash_tests.cpp @@ -195,7 +195,6 @@ BOOST_AUTO_TEST_CASE(sighash_from_data) nHashType = test[3].get_int(); sigHashHex = test[4].get_str(); - uint256 sh; CDataStream stream(ParseHex(raw_tx), SER_NETWORK, PROTOCOL_VERSION); stream >> tx; diff --git a/src/test/test_bitcoin.cpp b/src/test/test_bitcoin.cpp index db989e393b549..3feb473d69fbe 100644 --- a/src/test/test_bitcoin.cpp +++ b/src/test/test_bitcoin.cpp @@ -26,8 +26,6 @@ #include #include -CClientUIInterface uiInterface; // Declared but not defined in ui_interface.h - extern bool fPrintToConsole; extern void noui_connect(); @@ -100,7 +98,7 @@ CBlock TestChain100Setup::CreateAndProcessBlock(const std::vector& txns, const CScript& scriptPubKey) { const CChainParams& chainparams = Params(); - CBlockTemplate *pblocktemplate = CreateNewBlock(chainparams, scriptPubKey); + CBlockTemplate *pblocktemplate = BlockAssembler(chainparams).CreateNewBlock(scriptPubKey); CBlock& block = pblocktemplate->block; // Replace mempool-selected txns with just coinbase plus passed-in txns: diff --git a/src/test/txvalidationcache_tests.cpp b/src/test/txvalidationcache_tests.cpp index 000eca1821e0b..76e4e7a4be865 100644 --- a/src/test/txvalidationcache_tests.cpp +++ b/src/test/txvalidationcache_tests.cpp @@ -23,7 +23,7 @@ ToMemPool(CMutableTransaction& tx) LOCK(cs_main); CValidationState state; - return AcceptToMemoryPool(mempool, state, tx, false, NULL, NULL, true, 0); + return AcceptToMemoryPool(mempool, state, tx, false, NULL, true, 0); } BOOST_FIXTURE_TEST_CASE(tx_mempool_block_doublespend, TestChain100Setup) diff --git a/src/test/util_tests.cpp b/src/test/util_tests.cpp index b99f952a0dbc4..e467a4171d984 100644 --- a/src/test/util_tests.cpp +++ b/src/test/util_tests.cpp @@ -376,6 +376,69 @@ BOOST_AUTO_TEST_CASE(test_ParseInt64) BOOST_CHECK(!ParseInt64("32482348723847471234", NULL)); } +BOOST_AUTO_TEST_CASE(test_ParseUInt32) +{ + uint32_t n; + // Valid values + BOOST_CHECK(ParseUInt32("1234", NULL)); + BOOST_CHECK(ParseUInt32("0", &n) && n == 0); + BOOST_CHECK(ParseUInt32("1234", &n) && n == 1234); + BOOST_CHECK(ParseUInt32("01234", &n) && n == 1234); // no octal + BOOST_CHECK(ParseUInt32("2147483647", &n) && n == 2147483647); + BOOST_CHECK(ParseUInt32("2147483648", &n) && n == (uint32_t)2147483648); + BOOST_CHECK(ParseUInt32("4294967295", &n) && n == (uint32_t)4294967295); + // Invalid values + BOOST_CHECK(!ParseUInt32("", &n)); + BOOST_CHECK(!ParseUInt32(" 1", &n)); // no padding inside + BOOST_CHECK(!ParseUInt32(" -1", &n)); + BOOST_CHECK(!ParseUInt32("1 ", &n)); + BOOST_CHECK(!ParseUInt32("1a", &n)); + BOOST_CHECK(!ParseUInt32("aap", &n)); + BOOST_CHECK(!ParseUInt32("0x1", &n)); // no hex + BOOST_CHECK(!ParseUInt32("0x1", &n)); // no hex + const char test_bytes[] = {'1', 0, '1'}; + std::string teststr(test_bytes, sizeof(test_bytes)); + BOOST_CHECK(!ParseUInt32(teststr, &n)); // no embedded NULs + // Overflow and underflow + BOOST_CHECK(!ParseUInt32("-2147483648", &n)); + BOOST_CHECK(!ParseUInt32("4294967296", &n)); + BOOST_CHECK(!ParseUInt32("-1234", &n)); + BOOST_CHECK(!ParseUInt32("-32482348723847471234", NULL)); + BOOST_CHECK(!ParseUInt32("32482348723847471234", NULL)); +} + +BOOST_AUTO_TEST_CASE(test_ParseUInt64) +{ + uint64_t n; + // Valid values + BOOST_CHECK(ParseUInt64("1234", NULL)); + BOOST_CHECK(ParseUInt64("0", &n) && n == 0LL); + BOOST_CHECK(ParseUInt64("1234", &n) && n == 1234LL); + BOOST_CHECK(ParseUInt64("01234", &n) && n == 1234LL); // no octal + BOOST_CHECK(ParseUInt64("2147483647", &n) && n == 2147483647LL); + BOOST_CHECK(ParseUInt64("9223372036854775807", &n) && n == 9223372036854775807ULL); + BOOST_CHECK(ParseUInt64("9223372036854775808", &n) && n == 9223372036854775808ULL); + BOOST_CHECK(ParseUInt64("18446744073709551615", &n) && n == 18446744073709551615ULL); + // Invalid values + BOOST_CHECK(!ParseUInt64("", &n)); + BOOST_CHECK(!ParseUInt64(" 1", &n)); // no padding inside + BOOST_CHECK(!ParseUInt64(" -1", &n)); + BOOST_CHECK(!ParseUInt64("1 ", &n)); + BOOST_CHECK(!ParseUInt64("1a", &n)); + BOOST_CHECK(!ParseUInt64("aap", &n)); + BOOST_CHECK(!ParseUInt64("0x1", &n)); // no hex + const char test_bytes[] = {'1', 0, '1'}; + std::string teststr(test_bytes, sizeof(test_bytes)); + BOOST_CHECK(!ParseUInt64(teststr, &n)); // no embedded NULs + // Overflow and underflow + BOOST_CHECK(!ParseUInt64("-9223372036854775809", NULL)); + BOOST_CHECK(!ParseUInt64("18446744073709551616", NULL)); + BOOST_CHECK(!ParseUInt64("-32482348723847471234", NULL)); + BOOST_CHECK(!ParseUInt64("-2147483648", &n)); + BOOST_CHECK(!ParseUInt64("-9223372036854775808", &n)); + BOOST_CHECK(!ParseUInt64("-1234", &n)); +} + BOOST_AUTO_TEST_CASE(test_ParseDouble) { double n; diff --git a/src/tinyformat.h b/src/tinyformat.h index 73d49a1fe4d79..c6ec0419b3943 100644 --- a/src/tinyformat.h +++ b/src/tinyformat.h @@ -113,7 +113,7 @@ namespace tfm = tinyformat; // Define for C++11 variadic templates which make the code shorter & more // general. If you don't define this, C++11 support is autodetected below. -// #define TINYFORMAT_USE_VARIADIC_TEMPLATES +#define TINYFORMAT_USE_VARIADIC_TEMPLATES //------------------------------------------------------------------------------ diff --git a/src/torcontrol.cpp b/src/torcontrol.cpp index 1c7bc2dbeee6b..0d6b655675ded 100644 --- a/src/torcontrol.cpp +++ b/src/torcontrol.cpp @@ -438,7 +438,7 @@ void TorController::add_onion_cb(TorControlConnection& conn, const TorControlRep private_key = i->second; } - service = CService(service_id+".onion", GetListenPort(), false); + service = CService(service_id+".onion", GetListenPort()); LogPrintf("tor: Got service ID %s, advertising service %s\n", service_id, service.ToString()); if (WriteBinaryFile(GetPrivateKeyFile(), private_key)) { LogPrint("tor", "tor: Cached service private key to %s\n", GetPrivateKeyFile()); @@ -574,7 +574,15 @@ void TorController::protocolinfo_cb(TorControlConnection& conn, const TorControl * password: "password" */ std::string torpassword = GetArg("-torpassword", ""); - if (methods.count("NULL")) { + if (!torpassword.empty()) { + if (methods.count("HASHEDPASSWORD")) { + LogPrint("tor", "tor: Using HASHEDPASSWORD authentication\n"); + boost::replace_all(torpassword, "\"", "\\\""); + conn.Command("AUTHENTICATE \"" + torpassword + "\"", boost::bind(&TorController::auth_cb, this, _1, _2)); + } else { + LogPrintf("tor: Password provided with -torpassword, but HASHEDPASSWORD authentication is not available\n"); + } + } else if (methods.count("NULL")) { LogPrint("tor", "tor: Using NULL authentication\n"); conn.Command("AUTHENTICATE", boost::bind(&TorController::auth_cb, this, _1, _2)); } else if (methods.count("SAFECOOKIE")) { @@ -595,13 +603,7 @@ void TorController::protocolinfo_cb(TorControlConnection& conn, const TorControl } } } else if (methods.count("HASHEDPASSWORD")) { - if (!torpassword.empty()) { - LogPrint("tor", "tor: Using HASHEDPASSWORD authentication\n"); - boost::replace_all(torpassword, "\"", "\\\""); - conn.Command("AUTHENTICATE \"" + torpassword + "\"", boost::bind(&TorController::auth_cb, this, _1, _2)); - } else { - LogPrintf("tor: Password authentication required, but no password provided with -torpassword\n"); - } + LogPrintf("tor: The only supported authentication mechanism left is password, but no password provided with -torpassword\n"); } else { LogPrintf("tor: No supported authentication method\n"); } diff --git a/src/txdb.cpp b/src/txdb.cpp index 19ca1786549ee..078c29def319b 100644 --- a/src/txdb.cpp +++ b/src/txdb.cpp @@ -5,10 +5,8 @@ #include "txdb.h" -#include "chain.h" #include "chainparams.h" #include "hash.h" -#include "main.h" #include "pow.h" #include "uint256.h" @@ -49,7 +47,7 @@ uint256 CCoinsViewDB::GetBestBlock() const { } bool CCoinsViewDB::BatchWrite(CCoinsMap &mapCoins, const uint256 &hashBlock) { - CDBBatch batch(&db.GetObfuscateKey()); + CDBBatch batch(db); size_t count = 0; size_t changed = 0; for (CCoinsMap::iterator it = mapCoins.begin(); it != mapCoins.end();) { @@ -139,7 +137,7 @@ void CCoinsViewDBCursor::Next() } bool CBlockTreeDB::WriteBatchSync(const std::vector >& fileInfo, int nLastFile, const std::vector& blockinfo) { - CDBBatch batch(&GetObfuscateKey()); + CDBBatch batch(*this); for (std::vector >::const_iterator it=fileInfo.begin(); it != fileInfo.end(); it++) { batch.Write(make_pair(DB_BLOCK_FILES, it->first), *it->second); } @@ -155,7 +153,7 @@ bool CBlockTreeDB::ReadTxIndex(const uint256 &txid, CDiskTxPos &pos) { } bool CBlockTreeDB::WriteTxIndex(const std::vector >&vect) { - CDBBatch batch(&GetObfuscateKey()); + CDBBatch batch(*this); for (std::vector >::const_iterator it=vect.begin(); it!=vect.end(); it++) batch.Write(make_pair(DB_TXINDEX, it->first), it->second); return WriteBatch(batch); @@ -173,7 +171,7 @@ bool CBlockTreeDB::ReadFlag(const std::string &name, bool &fValue) { return true; } -bool CBlockTreeDB::LoadBlockIndexGuts() +bool CBlockTreeDB::LoadBlockIndexGuts(boost::function insertBlockIndex) { boost::scoped_ptr pcursor(NewIterator()); @@ -187,8 +185,8 @@ bool CBlockTreeDB::LoadBlockIndexGuts() CDiskBlockIndex diskindex; if (pcursor->GetValue(diskindex)) { // Construct block index object - CBlockIndex* pindexNew = InsertBlockIndex(diskindex.GetBlockHash()); - pindexNew->pprev = InsertBlockIndex(diskindex.hashPrev); + CBlockIndex* pindexNew = insertBlockIndex(diskindex.GetBlockHash()); + pindexNew->pprev = insertBlockIndex(diskindex.hashPrev); pindexNew->nHeight = diskindex.nHeight; pindexNew->nFile = diskindex.nFile; pindexNew->nDataPos = diskindex.nDataPos; diff --git a/src/txdb.h b/src/txdb.h index 749802f0e5fd0..ce3c39d7fec11 100644 --- a/src/txdb.h +++ b/src/txdb.h @@ -8,15 +8,17 @@ #include "coins.h" #include "dbwrapper.h" +#include "chain.h" #include #include #include #include -class CBlockFileInfo; +#include + class CBlockIndex; -struct CDiskTxPos; +class CCoinsViewDBCursor; class uint256; //! -dbcache default (MiB) @@ -26,7 +28,30 @@ static const int64_t nMaxDbCache = sizeof(void*) > 4 ? 16384 : 1024; //! min. -dbcache in (MiB) static const int64_t nMinDbCache = 4; -class CCoinsViewDBCursor; +struct CDiskTxPos : public CDiskBlockPos +{ + unsigned int nTxOffset; // after header + + ADD_SERIALIZE_METHODS; + + template + inline void SerializationOp(Stream& s, Operation ser_action, int nType, int nVersion) { + READWRITE(*(CDiskBlockPos*)this); + READWRITE(VARINT(nTxOffset)); + } + + CDiskTxPos(const CDiskBlockPos &blockIn, unsigned int nTxOffsetIn) : CDiskBlockPos(blockIn.nFile, blockIn.nPos), nTxOffset(nTxOffsetIn) { + } + + CDiskTxPos() { + SetNull(); + } + + void SetNull() { + CDiskBlockPos::SetNull(); + nTxOffset = 0; + } +}; /** CCoinsView backed by the coin database (chainstate/) */ class CCoinsViewDB : public CCoinsView @@ -83,7 +108,7 @@ class CBlockTreeDB : public CDBWrapper bool WriteTxIndex(const std::vector > &list); bool WriteFlag(const std::string &name, bool fValue); bool ReadFlag(const std::string &name, bool &fValue); - bool LoadBlockIndexGuts(); + bool LoadBlockIndexGuts(boost::function insertBlockIndex); }; #endif // BITCOIN_TXDB_H diff --git a/src/txmempool.cpp b/src/txmempool.cpp index 23abd0623df88..aae3c5ae77314 100644 --- a/src/txmempool.cpp +++ b/src/txmempool.cpp @@ -24,18 +24,18 @@ CTxMemPoolEntry::CTxMemPoolEntry(const CTransaction& _tx, const CAmount& _nFee, int64_t _nTime, double _entryPriority, unsigned int _entryHeight, bool poolHasNoInputsOf, CAmount _inChainInputValue, bool _spendsCoinbase, int64_t _sigOpsCost, LockPoints lp): - tx(_tx), nFee(_nFee), nTime(_nTime), entryPriority(_entryPriority), entryHeight(_entryHeight), + tx(std::make_shared(_tx)), nFee(_nFee), nTime(_nTime), entryPriority(_entryPriority), entryHeight(_entryHeight), hadNoDependencies(poolHasNoInputsOf), inChainInputValue(_inChainInputValue), spendsCoinbase(_spendsCoinbase), sigOpCost(_sigOpsCost), lockPoints(lp) { nTxCost = GetTransactionCost(_tx); nModSize = _tx.CalculateModifiedSize(GetTxSize()); - nUsageSize = RecursiveDynamicUsage(tx); + nUsageSize = RecursiveDynamicUsage(*tx) + memusage::DynamicUsage(tx); nCountWithDescendants = 1; nSizeWithDescendants = GetTxSize(); nModFeesWithDescendants = nFee; - CAmount nValueIn = tx.GetValueOut()+nFee; + CAmount nValueIn = _tx.GetValueOut()+nFee; assert(inChainInputValue <= nValueIn); feeDelta = 0; @@ -153,11 +153,11 @@ void CTxMemPool::UpdateTransactionsFromBlock(const std::vector &vHashes if (it == mapTx.end()) { continue; } - std::map::iterator iter = mapNextTx.lower_bound(COutPoint(hash, 0)); + auto iter = mapNextTx.lower_bound(COutPoint(hash, 0)); // First calculate the children, and update setMemPoolChildren to // include them, and update their setMemPoolParents to include this tx. - for (; iter != mapNextTx.end() && iter->first.hash == hash; ++iter) { - const uint256 &childHash = iter->second.ptx->GetHash(); + for (; iter != mapNextTx.end() && iter->first->hash == hash; ++iter) { + const uint256 &childHash = iter->second->GetHash(); txiter childIter = mapTx.find(childHash); assert(childIter != mapTx.end()); // We can skip updating entries we've encountered before or that @@ -371,11 +371,11 @@ void CTxMemPool::pruneSpent(const uint256 &hashTx, CCoins &coins) { LOCK(cs); - std::map::iterator it = mapNextTx.lower_bound(COutPoint(hashTx, 0)); + auto it = mapNextTx.lower_bound(COutPoint(hashTx, 0)); // iterate over all COutPoints in mapNextTx whose hash equals the provided hashTx - while (it != mapNextTx.end() && it->first.hash == hashTx) { - coins.Spend(it->first.n); // and remove those outputs from coins + while (it != mapNextTx.end() && it->first->hash == hashTx) { + coins.Spend(it->first->n); // and remove those outputs from coins it++; } } @@ -420,7 +420,7 @@ bool CTxMemPool::addUnchecked(const uint256& hash, const CTxMemPoolEntry &entry, const CTransaction& tx = newit->GetTx(); std::set setParentTransactions; for (unsigned int i = 0; i < tx.vin.size(); i++) { - mapNextTx[tx.vin[i].prevout] = CInPoint(&tx, i); + mapNextTx.insert(std::make_pair(&tx.vin[i].prevout, &tx)); setParentTransactions.insert(tx.vin[i].prevout.hash); } // Don't bother worrying about child transactions of this one. @@ -506,10 +506,10 @@ void CTxMemPool::removeRecursive(const CTransaction &origTx, std::list::iterator it = mapNextTx.find(COutPoint(origTx.GetHash(), i)); + auto it = mapNextTx.find(COutPoint(origTx.GetHash(), i)); if (it == mapNextTx.end()) continue; - txiter nextit = mapTx.find(it->second.ptx->GetHash()); + txiter nextit = mapTx.find(it->second->GetHash()); assert(nextit != mapTx.end()); txToRemove.insert(nextit); } @@ -567,9 +567,9 @@ void CTxMemPool::removeConflicts(const CTransaction &tx, std::list list result; LOCK(cs); BOOST_FOREACH(const CTxIn &txin, tx.vin) { - std::map::iterator it = mapNextTx.find(txin.prevout); + auto it = mapNextTx.find(txin.prevout); if (it != mapNextTx.end()) { - const CTransaction &txConflict = *it->second.ptx; + const CTransaction &txConflict = *it->second; if (txConflict != tx) { removeRecursive(txConflict, removed); @@ -677,10 +677,10 @@ void CTxMemPool::check(const CCoinsViewCache *pcoins) const assert(coins && coins->IsAvailable(txin.prevout.n)); } // Check whether its inputs are marked in mapNextTx. - std::map::const_iterator it3 = mapNextTx.find(txin.prevout); + auto it3 = mapNextTx.find(txin.prevout); assert(it3 != mapNextTx.end()); - assert(it3->second.ptx == &tx); - assert(it3->second.n == i); + assert(it3->first == &txin.prevout); + assert(it3->second == &tx); i++; } assert(setParentCheck == GetMemPoolParents(it)); @@ -707,10 +707,10 @@ void CTxMemPool::check(const CCoinsViewCache *pcoins) const // Check children against mapNextTx CTxMemPool::setEntries setChildrenCheck; - std::map::const_iterator iter = mapNextTx.lower_bound(COutPoint(it->GetTx().GetHash(), 0)); + auto iter = mapNextTx.lower_bound(COutPoint(it->GetTx().GetHash(), 0)); int64_t childSizes = 0; - for (; iter != mapNextTx.end() && iter->first.hash == it->GetTx().GetHash(); ++iter) { - txiter childit = mapTx.find(iter->second.ptx->GetHash()); + for (; iter != mapNextTx.end() && iter->first->hash == it->GetTx().GetHash(); ++iter) { + txiter childit = mapTx.find(iter->second->GetHash()); assert(childit != mapTx.end()); // mapNextTx points to in-mempool transactions if (setChildrenCheck.insert(childit).second) { childSizes += childit->GetTxSize(); @@ -726,7 +726,7 @@ void CTxMemPool::check(const CCoinsViewCache *pcoins) const else { CValidationState state; assert(CheckInputs(tx, state, mempoolDuplicate, false, 0, false, NULL)); - UpdateCoins(tx, state, mempoolDuplicate, 1000000); + UpdateCoins(tx, mempoolDuplicate, 1000000); } } unsigned int stepsSinceLastRemove = 0; @@ -740,51 +740,110 @@ void CTxMemPool::check(const CCoinsViewCache *pcoins) const assert(stepsSinceLastRemove < waitingOnDependants.size()); } else { assert(CheckInputs(entry->GetTx(), state, mempoolDuplicate, false, 0, false, NULL)); - UpdateCoins(entry->GetTx(), state, mempoolDuplicate, 1000000); + UpdateCoins(entry->GetTx(), mempoolDuplicate, 1000000); stepsSinceLastRemove = 0; } } - for (std::map::const_iterator it = mapNextTx.begin(); it != mapNextTx.end(); it++) { - uint256 hash = it->second.ptx->GetHash(); + for (auto it = mapNextTx.cbegin(); it != mapNextTx.cend(); it++) { + uint256 hash = it->second->GetHash(); indexed_transaction_set::const_iterator it2 = mapTx.find(hash); const CTransaction& tx = it2->GetTx(); assert(it2 != mapTx.end()); - assert(&tx == it->second.ptx); - assert(tx.vin.size() > it->second.n); - assert(it->first == it->second.ptx->vin[it->second.n].prevout); + assert(&tx == it->second); } assert(totalTxSize == checkTotal); assert(innerUsage == cachedInnerUsage); } +bool CTxMemPool::CompareDepthAndScore(const uint256& hasha, const uint256& hashb) +{ + LOCK(cs); + indexed_transaction_set::const_iterator i = mapTx.find(hasha); + if (i == mapTx.end()) return false; + indexed_transaction_set::const_iterator j = mapTx.find(hashb); + if (j == mapTx.end()) return true; + uint64_t counta = i->GetCountWithAncestors(); + uint64_t countb = j->GetCountWithAncestors(); + if (counta == countb) { + return CompareTxMemPoolEntryByScore()(*i, *j); + } + return counta < countb; +} + +namespace { +class DepthAndScoreComparator +{ +public: + bool operator()(const CTxMemPool::indexed_transaction_set::const_iterator& a, const CTxMemPool::indexed_transaction_set::const_iterator& b) + { + uint64_t counta = a->GetCountWithAncestors(); + uint64_t countb = b->GetCountWithAncestors(); + if (counta == countb) { + return CompareTxMemPoolEntryByScore()(*a, *b); + } + return counta < countb; + } +}; +} + +std::vector CTxMemPool::GetSortedDepthAndScore() const +{ + std::vector iters; + AssertLockHeld(cs); + + iters.reserve(mapTx.size()); + + for (indexed_transaction_set::iterator mi = mapTx.begin(); mi != mapTx.end(); ++mi) { + iters.push_back(mi); + } + std::sort(iters.begin(), iters.end(), DepthAndScoreComparator()); + return iters; +} + void CTxMemPool::queryHashes(vector& vtxid) { + LOCK(cs); + auto iters = GetSortedDepthAndScore(); + vtxid.clear(); + vtxid.reserve(mapTx.size()); + for (auto it : iters) { + vtxid.push_back(it->GetTx().GetHash()); + } +} + +std::vector CTxMemPool::infoAll() const +{ LOCK(cs); - vtxid.reserve(mapTx.size()); - for (indexed_transaction_set::iterator mi = mapTx.begin(); mi != mapTx.end(); ++mi) - vtxid.push_back(mi->GetTx().GetHash()); + auto iters = GetSortedDepthAndScore(); + + std::vector ret; + ret.reserve(mapTx.size()); + for (auto it : iters) { + ret.push_back(TxMempoolInfo{it->GetSharedTx(), it->GetTime(), CFeeRate(it->GetFee(), it->GetTxSize())}); + } + + return ret; } -bool CTxMemPool::lookup(uint256 hash, CTransaction& result) const +std::shared_ptr CTxMemPool::get(const uint256& hash) const { LOCK(cs); indexed_transaction_set::const_iterator i = mapTx.find(hash); - if (i == mapTx.end()) return false; - result = i->GetTx(); - return true; + if (i == mapTx.end()) + return nullptr; + return i->GetSharedTx(); } -bool CTxMemPool::lookupFeeRate(const uint256& hash, CFeeRate& feeRate) const +TxMempoolInfo CTxMemPool::info(const uint256& hash) const { LOCK(cs); indexed_transaction_set::const_iterator i = mapTx.find(hash); if (i == mapTx.end()) - return false; - feeRate = CFeeRate(i->GetFee(), i->GetTxSize()); - return true; + return TxMempoolInfo(); + return TxMempoolInfo{i->GetSharedTx(), i->GetTime(), CFeeRate(i->GetFee(), i->GetTxSize())}; } CFeeRate CTxMemPool::estimateFee(int nBlocks) const @@ -891,15 +950,15 @@ bool CTxMemPool::HasNoInputsOf(const CTransaction &tx) const return true; } -CCoinsViewMemPool::CCoinsViewMemPool(CCoinsView *baseIn, CTxMemPool &mempoolIn) : CCoinsViewBacked(baseIn), mempool(mempoolIn) { } +CCoinsViewMemPool::CCoinsViewMemPool(CCoinsView* baseIn, const CTxMemPool& mempoolIn) : CCoinsViewBacked(baseIn), mempool(mempoolIn) { } bool CCoinsViewMemPool::GetCoins(const uint256 &txid, CCoins &coins) const { // If an entry in the mempool exists, always return that one, as it's guaranteed to never // conflict with the underlying cache, and it cannot have pruned entries (as it contains full) // transactions. First checking the underlying cache risks returning a pruned entry instead. - CTransaction tx; - if (mempool.lookup(txid, tx)) { - coins = CCoins(tx, MEMPOOL_HEIGHT); + shared_ptr ptx = mempool.get(txid); + if (ptx) { + coins = CCoins(*ptx, MEMPOOL_HEIGHT); return true; } return (base->GetCoins(txid, coins) && !coins.IsPruned()); @@ -1050,8 +1109,8 @@ void CTxMemPool::TrimToSize(size_t sizelimit, std::vector* pvNoSpendsRe BOOST_FOREACH(const CTxIn& txin, tx.vin) { if (exists(txin.prevout.hash)) continue; - std::map::iterator it = mapNextTx.lower_bound(COutPoint(txin.prevout.hash, 0)); - if (it == mapNextTx.end() || it->first.hash != txin.prevout.hash) + auto it = mapNextTx.lower_bound(COutPoint(txin.prevout.hash, 0)); + if (it == mapNextTx.end() || it->first->hash != txin.prevout.hash) pvNoSpendsRemaining->push_back(txin.prevout.hash); } } diff --git a/src/txmempool.h b/src/txmempool.h index 7ce9c747e9680..02a6cc1c62305 100644 --- a/src/txmempool.h +++ b/src/txmempool.h @@ -7,16 +7,19 @@ #define BITCOIN_TXMEMPOOL_H #include +#include #include #include "amount.h" #include "coins.h" +#include "indirectmap.h" #include "primitives/transaction.h" #include "sync.h" #undef foreach #include "boost/multi_index_container.hpp" #include "boost/multi_index/ordered_index.hpp" +#include "boost/multi_index/hashed_index.hpp" class CAutoFile; class CBlockIndex; @@ -73,7 +76,7 @@ class CTxMemPool; class CTxMemPoolEntry { private: - CTransaction tx; + std::shared_ptr tx; CAmount nFee; //!< Cached to avoid expensive parent-transaction lookups size_t nTxCost; //!< ... and avoid recomputing tx cost (also used for GetTxSize()) size_t nModSize; //!< ... and modified size for priority @@ -110,7 +113,8 @@ class CTxMemPoolEntry int64_t nSigOpsCost, LockPoints lp); CTxMemPoolEntry(const CTxMemPoolEntry& other); - const CTransaction& GetTx() const { return this->tx; } + const CTransaction& GetTx() const { return *this->tx; } + std::shared_ptr GetSharedTx() const { return this->tx; } /** * Fast calculation of lower bound of current priority as update * from entry priority. Only inputs that were originally in-chain will age. @@ -306,18 +310,19 @@ struct ancestor_score {}; class CBlockPolicyEstimator; -/** An inpoint - a combination of a transaction and an index n into its vin */ -class CInPoint +/** + * Information about a mempool transaction. + */ +struct TxMempoolInfo { -public: - const CTransaction* ptx; - uint32_t n; - - CInPoint() { SetNull(); } - CInPoint(const CTransaction* ptxIn, uint32_t nIn) { ptx = ptxIn; n = nIn; } - void SetNull() { ptx = NULL; n = (uint32_t) -1; } - bool IsNull() const { return (ptx == NULL && n == (uint32_t) -1); } - size_t DynamicMemoryUsage() const { return 0; } + /** The transaction itself */ + std::shared_ptr tx; + + /** Time the transaction entered the mempool. */ + int64_t nTime; + + /** Feerate of the transaction. */ + CFeeRate feeRate; }; /** @@ -423,7 +428,7 @@ class CTxMemPool CTxMemPoolEntry, boost::multi_index::indexed_by< // sorted by txid - boost::multi_index::ordered_unique, + boost::multi_index::hashed_unique, // sorted by fee rate boost::multi_index::ordered_non_unique< boost::multi_index::tag, @@ -477,8 +482,10 @@ class CTxMemPool void UpdateParent(txiter entry, txiter parent, bool add); void UpdateChild(txiter entry, txiter child, bool add); + std::vector GetSortedDepthAndScore() const; + public: - std::map mapNextTx; + indirectmap mapNextTx; std::map > mapDeltas; /** Create a new CTxMemPool. @@ -512,6 +519,7 @@ class CTxMemPool std::list& conflicts, bool fCurrentEstimate = true); void clear(); void _clear(); //lock free + bool CompareDepthAndScore(const uint256& hasha, const uint256& hashb); void queryHashes(std::vector& vtxid); void pruneSpent(const uint256& hash, CCoins &coins); unsigned int GetTransactionsUpdated() const; @@ -600,8 +608,9 @@ class CTxMemPool return (mapTx.count(hash) != 0); } - bool lookup(uint256 hash, CTransaction& result) const; - bool lookupFeeRate(const uint256& hash, CFeeRate& feeRate) const; + std::shared_ptr get(const uint256& hash) const; + TxMempoolInfo info(const uint256& hash) const; + std::vector infoAll() const; /** Estimate fee rate needed to get into the next nBlocks * If no answer can be given at nBlocks, return an estimate @@ -673,10 +682,10 @@ class CTxMemPool class CCoinsViewMemPool : public CCoinsViewBacked { protected: - CTxMemPool &mempool; + const CTxMemPool& mempool; public: - CCoinsViewMemPool(CCoinsView *baseIn, CTxMemPool &mempoolIn); + CCoinsViewMemPool(CCoinsView* baseIn, const CTxMemPool& mempoolIn); bool GetCoins(const uint256 &txid, CCoins &coins) const; bool HaveCoins(const uint256 &txid) const; }; diff --git a/src/ui_interface.cpp b/src/ui_interface.cpp new file mode 100644 index 0000000000000..c778e40a9053a --- /dev/null +++ b/src/ui_interface.cpp @@ -0,0 +1,24 @@ +// Copyright (c) 2010-2016 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include "ui_interface.h" +#include "util.h" + +CClientUIInterface uiInterface; + +bool InitError(const std::string& str) +{ + uiInterface.ThreadSafeMessageBox(str, "", CClientUIInterface::MSG_ERROR); + return false; +} + +void InitWarning(const std::string& str) +{ + uiInterface.ThreadSafeMessageBox(str, "", CClientUIInterface::MSG_WARNING); +} + +std::string AmountErrMsg(const char* const optname, const std::string& strValue) +{ + return strprintf(_("Invalid amount for -%s=: '%s'"), optname, strValue); +} diff --git a/src/ui_interface.h b/src/ui_interface.h index 0b51d52e6576e..7ebfc17e5dd52 100644 --- a/src/ui_interface.h +++ b/src/ui_interface.h @@ -96,10 +96,21 @@ class CClientUIInterface /** New block has been accepted */ boost::signals2::signal NotifyBlockTip; + /** Best header has changed */ + boost::signals2::signal NotifyHeaderTip; + /** Banlist did change. */ boost::signals2::signal BannedListChanged; }; +/** Show warning message **/ +void InitWarning(const std::string& str); + +/** Show error message **/ +bool InitError(const std::string& str); + +std::string AmountErrMsg(const char* const optname, const std::string& strValue); + extern CClientUIInterface uiInterface; #endif // BITCOIN_UI_INTERFACE_H diff --git a/src/uint256.cpp b/src/uint256.cpp index c58c88bf4a8f1..f22ddcd1effce 100644 --- a/src/uint256.cpp +++ b/src/uint256.cpp @@ -80,67 +80,3 @@ template std::string base_blob<256>::GetHex() const; template std::string base_blob<256>::ToString() const; template void base_blob<256>::SetHex(const char*); template void base_blob<256>::SetHex(const std::string&); - -static void inline HashMix(uint32_t& a, uint32_t& b, uint32_t& c) -{ - // Taken from lookup3, by Bob Jenkins. - a -= c; - a ^= ((c << 4) | (c >> 28)); - c += b; - b -= a; - b ^= ((a << 6) | (a >> 26)); - a += c; - c -= b; - c ^= ((b << 8) | (b >> 24)); - b += a; - a -= c; - a ^= ((c << 16) | (c >> 16)); - c += b; - b -= a; - b ^= ((a << 19) | (a >> 13)); - a += c; - c -= b; - c ^= ((b << 4) | (b >> 28)); - b += a; -} - -static void inline HashFinal(uint32_t& a, uint32_t& b, uint32_t& c) -{ - // Taken from lookup3, by Bob Jenkins. - c ^= b; - c -= ((b << 14) | (b >> 18)); - a ^= c; - a -= ((c << 11) | (c >> 21)); - b ^= a; - b -= ((a << 25) | (a >> 7)); - c ^= b; - c -= ((b << 16) | (b >> 16)); - a ^= c; - a -= ((c << 4) | (c >> 28)); - b ^= a; - b -= ((a << 14) | (a >> 18)); - c ^= b; - c -= ((b << 24) | (b >> 8)); -} - -uint64_t uint256::GetHash(const uint256& salt) const -{ - uint32_t a, b, c; - const uint32_t *pn = (const uint32_t*)data; - const uint32_t *salt_pn = (const uint32_t*)salt.data; - a = b = c = 0xdeadbeef + WIDTH; - - a += pn[0] ^ salt_pn[0]; - b += pn[1] ^ salt_pn[1]; - c += pn[2] ^ salt_pn[2]; - HashMix(a, b, c); - a += pn[3] ^ salt_pn[3]; - b += pn[4] ^ salt_pn[4]; - c += pn[5] ^ salt_pn[5]; - HashMix(a, b, c); - a += pn[6] ^ salt_pn[6]; - b += pn[7] ^ salt_pn[7]; - HashFinal(a, b, c); - - return ((((uint64_t)b) << 32) | c); -} diff --git a/src/uint256.h b/src/uint256.h index bcdb6dd7c2958..dd8432d74cab5 100644 --- a/src/uint256.h +++ b/src/uint256.h @@ -83,6 +83,19 @@ class base_blob return sizeof(data); } + uint64_t GetUint64(int pos) const + { + const uint8_t* ptr = data + pos * 8; + return ((uint64_t)ptr[0]) | \ + ((uint64_t)ptr[1]) << 8 | \ + ((uint64_t)ptr[2]) << 16 | \ + ((uint64_t)ptr[3]) << 24 | \ + ((uint64_t)ptr[4]) << 32 | \ + ((uint64_t)ptr[5]) << 40 | \ + ((uint64_t)ptr[6]) << 48 | \ + ((uint64_t)ptr[7]) << 56; + } + template void Serialize(Stream& s, int nType, int nVersion) const { @@ -127,11 +140,6 @@ class uint256 : public base_blob<256> { { return ReadLE64(data); } - - /** A more secure, salted hash function. - * @note This hash is not stable between little and big endian. - */ - uint64_t GetHash(const uint256& salt) const; }; /* uint256 from const char *. diff --git a/src/util.cpp b/src/util.cpp index 00b75fbdbe38e..9a9209c62140b 100644 --- a/src/util.cpp +++ b/src/util.cpp @@ -113,7 +113,7 @@ string strMiscWarning; bool fLogTimestamps = DEFAULT_LOGTIMESTAMPS; bool fLogTimeMicros = DEFAULT_LOGTIMEMICROS; bool fLogIPs = DEFAULT_LOGIPS; -volatile bool fReopenDebugLog = false; +std::atomic fReopenDebugLog(false); CTranslationInterface translationInterface; /** Init OpenSSL library multithreading support */ @@ -790,19 +790,6 @@ bool SetupNetworking() return true; } -void SetThreadPriority(int nPriority) -{ -#ifdef WIN32 - SetThreadPriority(GetCurrentThread(), nPriority); -#else // WIN32 -#ifdef PRIO_THREAD - setpriority(PRIO_THREAD, 0, nPriority); -#else // PRIO_THREAD - setpriority(PRIO_PROCESS, 0, nPriority); -#endif // PRIO_THREAD -#endif // WIN32 -} - int GetNumCores() { #if BOOST_VERSION >= 105600 diff --git a/src/util.h b/src/util.h index ac099f11845d3..ac4b947785f43 100644 --- a/src/util.h +++ b/src/util.h @@ -18,6 +18,7 @@ #include "tinyformat.h" #include "utiltime.h" +#include #include #include #include @@ -50,7 +51,7 @@ extern std::string strMiscWarning; extern bool fLogTimestamps; extern bool fLogTimeMicros; extern bool fLogIPs; -extern volatile bool fReopenDebugLog; +extern std::atomic fReopenDebugLog; extern CTranslationInterface translationInterface; extern const char * const BITCOIN_CONF_FILENAME; @@ -76,40 +77,33 @@ int LogPrintStr(const std::string &str); #define LogPrintf(...) LogPrint(NULL, __VA_ARGS__) -/** - * When we switch to C++11, this can be switched to variadic templates instead - * of this macro-based construction (see tinyformat.h). - */ -#define MAKE_ERROR_AND_LOG_FUNC(n) \ - /** Print to debug.log if -debug=category switch is given OR category is NULL. */ \ - template \ - static inline int LogPrint(const char* category, const char* format, TINYFORMAT_VARARGS(n)) \ - { \ - if(!LogAcceptCategory(category)) return 0; \ - return LogPrintStr(tfm::format(format, TINYFORMAT_PASSARGS(n))); \ - } \ - /** Log error and return false */ \ - template \ - static inline bool error(const char* format, TINYFORMAT_VARARGS(n)) \ - { \ - LogPrintStr("ERROR: " + tfm::format(format, TINYFORMAT_PASSARGS(n)) + "\n"); \ - return false; \ - } +template +static inline int LogPrint(const char* category, const char* fmt, const T1& v1, const Args&... args) +{ + if(!LogAcceptCategory(category)) return 0; \ + return LogPrintStr(tfm::format(fmt, v1, args...)); +} -TINYFORMAT_FOREACH_ARGNUM(MAKE_ERROR_AND_LOG_FUNC) +template +bool error(const char* fmt, const T1& v1, const Args&... args) +{ + LogPrintStr("ERROR: " + tfm::format(fmt, v1, args...) + "\n"); + return false; +} /** * Zero-arg versions of logging and error, these are not covered by - * TINYFORMAT_FOREACH_ARGNUM + * the variadic templates above (and don't take format arguments but + * bare strings). */ -static inline int LogPrint(const char* category, const char* format) +static inline int LogPrint(const char* category, const char* s) { if(!LogAcceptCategory(category)) return 0; - return LogPrintStr(format); + return LogPrintStr(s); } -static inline bool error(const char* format) +static inline bool error(const char* s) { - LogPrintStr(std::string("ERROR: ") + format + "\n"); + LogPrintStr(std::string("ERROR: ") + s + "\n"); return false; } @@ -215,7 +209,6 @@ std::string HelpMessageOpt(const std::string& option, const std::string& message */ int GetNumCores(); -void SetThreadPriority(int nPriority); void RenameThread(const char* name); /** diff --git a/src/utilstrencodings.cpp b/src/utilstrencodings.cpp index 0f9334cbe3738..5ffdb3be15cf4 100644 --- a/src/utilstrencodings.cpp +++ b/src/utilstrencodings.cpp @@ -461,6 +461,40 @@ bool ParseInt64(const std::string& str, int64_t *out) n <= std::numeric_limits::max(); } +bool ParseUInt32(const std::string& str, uint32_t *out) +{ + if (!ParsePrechecks(str)) + return false; + if (str.size() >= 1 && str[0] == '-') // Reject negative values, unfortunately strtoul accepts these by default if they fit in the range + return false; + char *endp = NULL; + errno = 0; // strtoul will not set errno if valid + unsigned long int n = strtoul(str.c_str(), &endp, 10); + if(out) *out = (uint32_t)n; + // Note that strtoul returns a *unsigned long int*, so even if it doesn't report a over/underflow + // we still have to check that the returned value is within the range of an *uint32_t*. On 64-bit + // platforms the size of these types may be different. + return endp && *endp == 0 && !errno && + n <= std::numeric_limits::max(); +} + +bool ParseUInt64(const std::string& str, uint64_t *out) +{ + if (!ParsePrechecks(str)) + return false; + if (str.size() >= 1 && str[0] == '-') // Reject negative values, unfortunately strtoull accepts these by default if they fit in the range + return false; + char *endp = NULL; + errno = 0; // strtoull will not set errno if valid + unsigned long long int n = strtoull(str.c_str(), &endp, 10); + if(out) *out = (uint64_t)n; + // Note that strtoull returns a *unsigned long long int*, so even if it doesn't report a over/underflow + // we still have to check that the returned value is within the range of an *uint64_t*. + return endp && *endp == 0 && !errno && + n <= std::numeric_limits::max(); +} + + bool ParseDouble(const std::string& str, double *out) { if (!ParsePrechecks(str)) diff --git a/src/utilstrencodings.h b/src/utilstrencodings.h index d40613cfc43bc..5744f78c6e631 100644 --- a/src/utilstrencodings.h +++ b/src/utilstrencodings.h @@ -70,6 +70,20 @@ bool ParseInt32(const std::string& str, int32_t *out); */ bool ParseInt64(const std::string& str, int64_t *out); +/** + * Convert decimal string to unsigned 32-bit integer with strict parse error feedback. + * @returns true if the entire string could be parsed as valid integer, + * false if not the entire string could be parsed or when overflow or underflow occurred. + */ +bool ParseUInt32(const std::string& str, uint32_t *out); + +/** + * Convert decimal string to unsigned 64-bit integer with strict parse error feedback. + * @returns true if the entire string could be parsed as valid integer, + * false if not the entire string could be parsed or when overflow or underflow occurred. + */ +bool ParseUInt64(const std::string& str, uint64_t *out); + /** * Convert string to double with strict parse error feedback. * @returns true if the entire string could be parsed as valid double, diff --git a/src/versionbits.cpp b/src/versionbits.cpp index 78feb8ab0c6d2..bf32ae6627945 100644 --- a/src/versionbits.cpp +++ b/src/versionbits.cpp @@ -4,6 +4,23 @@ #include "versionbits.h" +#include "consensus/params.h" + +const struct BIP9DeploymentInfo VersionBitsDeploymentInfo[Consensus::MAX_VERSION_BITS_DEPLOYMENTS] = { + { + /*.name =*/ "testdummy", + /*.gbt_force =*/ true, + }, + { + /*.name =*/ "csv", + /*.gbt_force =*/ true, + }, + { + /*.name =*/ "segwit", + /*.gbt_force =*/ false, + } +}; + ThresholdState AbstractThresholdConditionChecker::GetStateFor(const CBlockIndex* pindexPrev, const Consensus::Params& params, ThresholdConditionCache& cache) const { int nPeriod = Period(params); diff --git a/src/versionbits.h b/src/versionbits.h index 04f4738272796..ede2dcdda8a7a 100644 --- a/src/versionbits.h +++ b/src/versionbits.h @@ -30,6 +30,15 @@ enum ThresholdState { // will either be NULL or a block with (height + 1) % Period() == 0. typedef std::map ThresholdConditionCache; +struct BIP9DeploymentInfo { + /** Deployment name */ + const char *name; + /** Whether GBT clients can safely ignore this rule in simplified usage */ + bool gbt_force; +}; + +extern const struct BIP9DeploymentInfo VersionBitsDeploymentInfo[]; + /** * Abstract class that implements BIP9-style threshold logic, and caches results. */ diff --git a/src/wallet/crypter.cpp b/src/wallet/crypter.cpp index 95aa4c2593164..190f8ecf2ab49 100644 --- a/src/wallet/crypter.cpp +++ b/src/wallet/crypter.cpp @@ -4,6 +4,8 @@ #include "crypter.h" +#include "crypto/aes.h" +#include "crypto/sha512.h" #include "script/script.h" #include "script/standard.h" #include "util.h" @@ -11,8 +13,33 @@ #include #include #include -#include -#include + +int CCrypter::BytesToKeySHA512AES(const std::vector& chSalt, const SecureString& strKeyData, int count, unsigned char *key,unsigned char *iv) const +{ + // This mimics the behavior of openssl's EVP_BytesToKey with an aes256cbc + // cipher and sha512 message digest. Because sha512's output size (64b) is + // greater than the aes256 block size (16b) + aes256 key size (32b), + // there's no need to process more than once (D_0). + + if(!count || !key || !iv) + return 0; + + unsigned char buf[CSHA512::OUTPUT_SIZE]; + CSHA512 di; + + di.Write((const unsigned char*)strKeyData.c_str(), strKeyData.size()); + if(chSalt.size()) + di.Write(&chSalt[0], chSalt.size()); + di.Finalize(buf); + + for(int i = 0; i != count - 1; i++) + di.Reset().Write(buf, sizeof(buf)).Finalize(buf); + + memcpy(key, buf, WALLET_CRYPTO_KEY_SIZE); + memcpy(iv, buf + WALLET_CRYPTO_KEY_SIZE, WALLET_CRYPTO_IV_SIZE); + memory_cleanse(buf, sizeof(buf)); + return WALLET_CRYPTO_KEY_SIZE; +} bool CCrypter::SetKeyFromPassphrase(const SecureString& strKeyData, const std::vector& chSalt, const unsigned int nRounds, const unsigned int nDerivationMethod) { @@ -21,8 +48,7 @@ bool CCrypter::SetKeyFromPassphrase(const SecureString& strKeyData, const std::v int i = 0; if (nDerivationMethod == 0) - i = EVP_BytesToKey(EVP_aes_256_cbc(), EVP_sha512(), &chSalt[0], - (unsigned char *)&strKeyData[0], strKeyData.size(), nRounds, chKey, chIV); + i = BytesToKeySHA512AES(chSalt, strKeyData, nRounds, chKey, chIV); if (i != (int)WALLET_CRYPTO_KEY_SIZE) { @@ -37,7 +63,7 @@ bool CCrypter::SetKeyFromPassphrase(const SecureString& strKeyData, const std::v bool CCrypter::SetKey(const CKeyingMaterial& chNewKey, const std::vector& chNewIV) { - if (chNewKey.size() != WALLET_CRYPTO_KEY_SIZE || chNewIV.size() != WALLET_CRYPTO_KEY_SIZE) + if (chNewKey.size() != WALLET_CRYPTO_KEY_SIZE || chNewIV.size() != WALLET_CRYPTO_IV_SIZE) return false; memcpy(&chKey[0], &chNewKey[0], sizeof chKey); @@ -47,57 +73,39 @@ bool CCrypter::SetKey(const CKeyingMaterial& chNewKey, const std::vector &vchCiphertext) +bool CCrypter::Encrypt(const CKeyingMaterial& vchPlaintext, std::vector &vchCiphertext) const { if (!fKeySet) return false; // max ciphertext len for a n bytes of plaintext is - // n + AES_BLOCK_SIZE - 1 bytes - int nLen = vchPlaintext.size(); - int nCLen = nLen + AES_BLOCK_SIZE, nFLen = 0; - vchCiphertext = std::vector (nCLen); - - EVP_CIPHER_CTX ctx; + // n + AES_BLOCKSIZE bytes + vchCiphertext.resize(vchPlaintext.size() + AES_BLOCKSIZE); - bool fOk = true; - - EVP_CIPHER_CTX_init(&ctx); - if (fOk) fOk = EVP_EncryptInit_ex(&ctx, EVP_aes_256_cbc(), NULL, chKey, chIV) != 0; - if (fOk) fOk = EVP_EncryptUpdate(&ctx, &vchCiphertext[0], &nCLen, &vchPlaintext[0], nLen) != 0; - if (fOk) fOk = EVP_EncryptFinal_ex(&ctx, (&vchCiphertext[0]) + nCLen, &nFLen) != 0; - EVP_CIPHER_CTX_cleanup(&ctx); - - if (!fOk) return false; + AES256CBCEncrypt enc(chKey, chIV, true); + size_t nLen = enc.Encrypt(&vchPlaintext[0], vchPlaintext.size(), &vchCiphertext[0]); + if(nLen < vchPlaintext.size()) + return false; + vchCiphertext.resize(nLen); - vchCiphertext.resize(nCLen + nFLen); return true; } -bool CCrypter::Decrypt(const std::vector& vchCiphertext, CKeyingMaterial& vchPlaintext) +bool CCrypter::Decrypt(const std::vector& vchCiphertext, CKeyingMaterial& vchPlaintext) const { if (!fKeySet) return false; // plaintext will always be equal to or lesser than length of ciphertext int nLen = vchCiphertext.size(); - int nPLen = nLen, nFLen = 0; - - vchPlaintext = CKeyingMaterial(nPLen); - EVP_CIPHER_CTX ctx; + vchPlaintext.resize(nLen); - bool fOk = true; - - EVP_CIPHER_CTX_init(&ctx); - if (fOk) fOk = EVP_DecryptInit_ex(&ctx, EVP_aes_256_cbc(), NULL, chKey, chIV) != 0; - if (fOk) fOk = EVP_DecryptUpdate(&ctx, &vchPlaintext[0], &nPLen, &vchCiphertext[0], nLen) != 0; - if (fOk) fOk = EVP_DecryptFinal_ex(&ctx, (&vchPlaintext[0]) + nPLen, &nFLen) != 0; - EVP_CIPHER_CTX_cleanup(&ctx); - - if (!fOk) return false; - - vchPlaintext.resize(nPLen + nFLen); + AES256CBCDecrypt dec(chKey, chIV, true); + nLen = dec.Decrypt(&vchCiphertext[0], vchCiphertext.size(), &vchPlaintext[0]); + if(nLen == 0) + return false; + vchPlaintext.resize(nLen); return true; } @@ -105,8 +113,8 @@ bool CCrypter::Decrypt(const std::vector& vchCiphertext, CKeyingM static bool EncryptSecret(const CKeyingMaterial& vMasterKey, const CKeyingMaterial &vchPlaintext, const uint256& nIV, std::vector &vchCiphertext) { CCrypter cKeyCrypter; - std::vector chIV(WALLET_CRYPTO_KEY_SIZE); - memcpy(&chIV[0], &nIV, WALLET_CRYPTO_KEY_SIZE); + std::vector chIV(WALLET_CRYPTO_IV_SIZE); + memcpy(&chIV[0], &nIV, WALLET_CRYPTO_IV_SIZE); if(!cKeyCrypter.SetKey(vMasterKey, chIV)) return false; return cKeyCrypter.Encrypt(*((const CKeyingMaterial*)&vchPlaintext), vchCiphertext); @@ -115,8 +123,8 @@ static bool EncryptSecret(const CKeyingMaterial& vMasterKey, const CKeyingMateri static bool DecryptSecret(const CKeyingMaterial& vMasterKey, const std::vector& vchCiphertext, const uint256& nIV, CKeyingMaterial& vchPlaintext) { CCrypter cKeyCrypter; - std::vector chIV(WALLET_CRYPTO_KEY_SIZE); - memcpy(&chIV[0], &nIV, WALLET_CRYPTO_KEY_SIZE); + std::vector chIV(WALLET_CRYPTO_IV_SIZE); + memcpy(&chIV[0], &nIV, WALLET_CRYPTO_IV_SIZE); if(!cKeyCrypter.SetKey(vMasterKey, chIV)) return false; return cKeyCrypter.Decrypt(vchCiphertext, *((CKeyingMaterial*)&vchPlaintext)); diff --git a/src/wallet/crypter.h b/src/wallet/crypter.h index eb06a7866a96c..5d0a4a3305b8c 100644 --- a/src/wallet/crypter.h +++ b/src/wallet/crypter.h @@ -13,6 +13,7 @@ class uint256; const unsigned int WALLET_CRYPTO_KEY_SIZE = 32; const unsigned int WALLET_CRYPTO_SALT_SIZE = 8; +const unsigned int WALLET_CRYPTO_IV_SIZE = 16; /** * Private key encryption is done based on a CMasterKey, @@ -66,18 +67,26 @@ class CMasterKey typedef std::vector > CKeyingMaterial; +namespace wallet_crypto +{ + class TestCrypter; +} + /** Encryption/decryption context with key information */ class CCrypter { +friend class wallet_crypto::TestCrypter; // for test access to chKey/chIV private: unsigned char chKey[WALLET_CRYPTO_KEY_SIZE]; - unsigned char chIV[WALLET_CRYPTO_KEY_SIZE]; + unsigned char chIV[WALLET_CRYPTO_IV_SIZE]; bool fKeySet; + int BytesToKeySHA512AES(const std::vector& chSalt, const SecureString& strKeyData, int count, unsigned char *key,unsigned char *iv) const; + public: bool SetKeyFromPassphrase(const SecureString &strKeyData, const std::vector& chSalt, const unsigned int nRounds, const unsigned int nDerivationMethod); - bool Encrypt(const CKeyingMaterial& vchPlaintext, std::vector &vchCiphertext); - bool Decrypt(const std::vector& vchCiphertext, CKeyingMaterial& vchPlaintext); + bool Encrypt(const CKeyingMaterial& vchPlaintext, std::vector &vchCiphertext) const; + bool Decrypt(const std::vector& vchCiphertext, CKeyingMaterial& vchPlaintext) const; bool SetKey(const CKeyingMaterial& chNewKey, const std::vector& chNewIV); void CleanKey() diff --git a/src/wallet/rpcdump.cpp b/src/wallet/rpcdump.cpp index bb40cf724536a..14c2e31d956d6 100644 --- a/src/wallet/rpcdump.cpp +++ b/src/wallet/rpcdump.cpp @@ -590,7 +590,7 @@ UniValue dumpwallet(const UniValue& params, bool fHelp) std::sort(vKeyBirth.begin(), vKeyBirth.end()); // produce output - file << strprintf("# Wallet dump created by Bitcoin %s (%s)\n", CLIENT_BUILD, CLIENT_DATE); + file << strprintf("# Wallet dump created by Bitcoin %s\n", CLIENT_BUILD); file << strprintf("# * Created on %s\n", EncodeDumpTime(GetTime())); file << strprintf("# * Best block at time of backup was %i (%s),\n", chainActive.Height(), chainActive.Tip()->GetBlockHash().ToString()); file << strprintf("# mined on %s\n", EncodeDumpTime(chainActive.Tip()->GetBlockTime())); diff --git a/src/wallet/rpcwallet.cpp b/src/wallet/rpcwallet.cpp index 2d5c57771d557..8538f880ff636 100644 --- a/src/wallet/rpcwallet.cpp +++ b/src/wallet/rpcwallet.cpp @@ -146,38 +146,12 @@ UniValue getnewaddress(const UniValue& params, bool fHelp) CBitcoinAddress GetAccountAddress(string strAccount, bool bForceNew=false) { - CWalletDB walletdb(pwalletMain->strWalletFile); - - CAccount account; - walletdb.ReadAccount(strAccount, account); - - if (!bForceNew) { - if (!account.vchPubKey.IsValid()) - bForceNew = true; - else { - // Check if the current key has been used - CScript scriptPubKey = GetScriptForDestination(account.vchPubKey.GetID()); - for (map::iterator it = pwalletMain->mapWallet.begin(); - it != pwalletMain->mapWallet.end() && account.vchPubKey.IsValid(); - ++it) - BOOST_FOREACH(const CTxOut& txout, (*it).second.vout) - if (txout.scriptPubKey == scriptPubKey) { - bForceNew = true; - break; - } - } - } - - // Generate a new key - if (bForceNew) { - if (!pwalletMain->GetKeyFromPool(account.vchPubKey)) - throw JSONRPCError(RPC_WALLET_KEYPOOL_RAN_OUT, "Error: Keypool ran out, please call keypoolrefill first"); - - pwalletMain->SetAddressBook(account.vchPubKey.GetID(), strAccount, "receive"); - walletdb.WriteAccount(strAccount, account); + CPubKey pubKey; + if (!pwalletMain->GetAccountPubkey(pubKey, strAccount, bForceNew)) { + throw JSONRPCError(RPC_WALLET_KEYPOOL_RAN_OUT, "Error: Keypool ran out, please call keypoolrefill first"); } - return CBitcoinAddress(account.vchPubKey.GetID()); + return CBitcoinAddress(pubKey.GetID()); } UniValue getaccountaddress(const UniValue& params, bool fHelp) @@ -673,38 +647,6 @@ UniValue getreceivedbyaccount(const UniValue& params, bool fHelp) } -CAmount GetAccountBalance(CWalletDB& walletdb, const string& strAccount, int nMinDepth, const isminefilter& filter) -{ - CAmount nBalance = 0; - - // Tally wallet transactions - for (map::iterator it = pwalletMain->mapWallet.begin(); it != pwalletMain->mapWallet.end(); ++it) - { - const CWalletTx& wtx = (*it).second; - if (!CheckFinalTx(wtx) || wtx.GetBlocksToMaturity() > 0 || wtx.GetDepthInMainChain() < 0) - continue; - - CAmount nReceived, nSent, nFee; - wtx.GetAccountAmounts(strAccount, nReceived, nSent, nFee, filter); - - if (nReceived != 0 && wtx.GetDepthInMainChain() >= nMinDepth) - nBalance += nReceived; - nBalance -= nSent + nFee; - } - - // Tally internal accounting entries - nBalance += walletdb.GetAccountCreditDebit(strAccount); - - return nBalance; -} - -CAmount GetAccountBalance(const string& strAccount, int nMinDepth, const isminefilter& filter) -{ - CWalletDB walletdb(pwalletMain->strWalletFile); - return GetAccountBalance(walletdb, strAccount, nMinDepth, filter); -} - - UniValue getbalance(const UniValue& params, bool fHelp) { if (!EnsureWalletIsAvailable(fHelp)) @@ -775,7 +717,7 @@ UniValue getbalance(const UniValue& params, bool fHelp) string strAccount = AccountFromValue(params[0]); - CAmount nBalance = GetAccountBalance(strAccount, nMinDepth, filter); + CAmount nBalance = pwalletMain->GetAccountBalance(strAccount, nMinDepth, filter); return ValueFromAmount(nBalance); } @@ -836,33 +778,7 @@ UniValue movecmd(const UniValue& params, bool fHelp) if (params.size() > 4) strComment = params[4].get_str(); - CWalletDB walletdb(pwalletMain->strWalletFile); - if (!walletdb.TxnBegin()) - throw JSONRPCError(RPC_DATABASE_ERROR, "database error"); - - int64_t nNow = GetAdjustedTime(); - - // Debit - CAccountingEntry debit; - debit.nOrderPos = pwalletMain->IncOrderPosNext(&walletdb); - debit.strAccount = strFrom; - debit.nCreditDebit = -nAmount; - debit.nTime = nNow; - debit.strOtherAccount = strTo; - debit.strComment = strComment; - pwalletMain->AddAccountingEntry(debit, walletdb); - - // Credit - CAccountingEntry credit; - credit.nOrderPos = pwalletMain->IncOrderPosNext(&walletdb); - credit.strAccount = strTo; - credit.nCreditDebit = nAmount; - credit.nTime = nNow; - credit.strOtherAccount = strFrom; - credit.strComment = strComment; - pwalletMain->AddAccountingEntry(credit, walletdb); - - if (!walletdb.TxnCommit()) + if (!pwalletMain->AccountMove(strFrom, strTo, nAmount, strComment)) throw JSONRPCError(RPC_DATABASE_ERROR, "database error"); return true; @@ -923,7 +839,7 @@ UniValue sendfrom(const UniValue& params, bool fHelp) EnsureWalletIsUnlocked(); // Check funds - CAmount nBalance = GetAccountBalance(strAccount, nMinDepth, ISMINE_SPENDABLE); + CAmount nBalance = pwalletMain->GetAccountBalance(strAccount, nMinDepth, ISMINE_SPENDABLE); if (nAmount > nBalance) throw JSONRPCError(RPC_WALLET_INSUFFICIENT_FUNDS, "Account has insufficient funds"); @@ -1026,7 +942,7 @@ UniValue sendmany(const UniValue& params, bool fHelp) EnsureWalletIsUnlocked(); // Check funds - CAmount nBalance = GetAccountBalance(strAccount, nMinDepth, ISMINE_SPENDABLE); + CAmount nBalance = pwalletMain->GetAccountBalance(strAccount, nMinDepth, ISMINE_SPENDABLE); if (totalAmount > nBalance) throw JSONRPCError(RPC_WALLET_INSUFFICIENT_FUNDS, "Account has insufficient funds"); @@ -1915,7 +1831,7 @@ UniValue backupwallet(const UniValue& params, bool fHelp) LOCK2(cs_main, pwalletMain->cs_wallet); string strDest = params[0].get_str(); - if (!BackupWallet(*pwalletMain, strDest)) + if (!pwalletMain->BackupWallet(strDest)) throw JSONRPCError(RPC_WALLET_ERROR, "Error: Wallet backup failed!"); return NullUniValue; @@ -2232,7 +2148,11 @@ UniValue lockunspent(const UniValue& params, bool fHelp) throw JSONRPCError(RPC_INVALID_PARAMETER, "Invalid parameter, expected object"); const UniValue& o = output.get_obj(); - RPCTypeCheckObj(o, boost::assign::map_list_of("txid", UniValue::VSTR)("vout", UniValue::VNUM)); + RPCTypeCheckObj(o, + { + {"txid", UniValueType(UniValue::VSTR)}, + {"vout", UniValueType(UniValue::VNUM)}, + }); string txid = find_value(o, "txid").get_str(); if (!IsHex(txid)) @@ -2312,7 +2232,7 @@ UniValue settxfee(const UniValue& params, bool fHelp) "settxfee amount\n" "\nSet the transaction fee per kB. Overwrites the paytxfee parameter.\n" "\nArguments:\n" - "1. amount (numeric or sting, required) The transaction fee in " + CURRENCY_UNIT + "/kB\n" + "1. amount (numeric or string, required) The transaction fee in " + CURRENCY_UNIT + "/kB\n" "\nResult\n" "true|false (boolean) Returns true if successful\n" "\nExamples:\n" @@ -2407,8 +2327,6 @@ UniValue listunspent(const UniValue& params, bool fHelp) "\nReturns array of unspent transaction outputs\n" "with between minconf and maxconf (inclusive) confirmations.\n" "Optionally filter to only include txouts paid to specified addresses.\n" - "Results are an array of Objects, each of which has:\n" - "{txid, vout, scriptPubKey, amount, confirmations}\n" "\nArguments:\n" "1. minconf (numeric, optional, default=1) The minimum confirmations to filter\n" "2. maxconf (numeric, optional, default=9999999) The maximum confirmations to filter\n" @@ -2420,13 +2338,16 @@ UniValue listunspent(const UniValue& params, bool fHelp) "\nResult\n" "[ (array of json object)\n" " {\n" - " \"txid\" : \"txid\", (string) the transaction id \n" + " \"txid\" : \"txid\", (string) the transaction id \n" " \"vout\" : n, (numeric) the vout value\n" - " \"address\" : \"address\", (string) the bitcoin address\n" - " \"account\" : \"account\", (string) DEPRECATED. The associated account, or \"\" for the default account\n" - " \"scriptPubKey\" : \"key\", (string) the script key\n" + " \"address\" : \"address\", (string) the bitcoin address\n" + " \"account\" : \"account\", (string) DEPRECATED. The associated account, or \"\" for the default account\n" + " \"scriptPubKey\" : \"key\", (string) the script key\n" " \"amount\" : x.xxx, (numeric) the transaction amount in " + CURRENCY_UNIT + "\n" - " \"confirmations\" : n (numeric) The number of confirmations\n" + " \"confirmations\" : n, (numeric) The number of confirmations\n" + " \"redeemScript\" : n (string) The redeemScript if scriptPubKey is P2SH\n" + " \"spendable\" : xxx, (bool) Whether we have the private keys to spend this output\n" + " \"solvable\" : xxx (bool) Whether we know how to spend this output, ignoring the lack of keys\n" " }\n" " ,...\n" "]\n" @@ -2470,39 +2391,36 @@ UniValue listunspent(const UniValue& params, bool fHelp) if (out.nDepth < nMinDepth || out.nDepth > nMaxDepth) continue; - if (setAddress.size()) { - CTxDestination address; - if (!ExtractDestination(out.tx->vout[out.i].scriptPubKey, address)) - continue; + CTxDestination address; + const CScript& scriptPubKey = out.tx->vout[out.i].scriptPubKey; + bool fValidAddress = ExtractDestination(scriptPubKey, address); - if (!setAddress.count(address)) - continue; - } + if (setAddress.size() && (!fValidAddress || !setAddress.count(address))) + continue; - CAmount nValue = out.tx->vout[out.i].nValue; - const CScript& pk = out.tx->vout[out.i].scriptPubKey; UniValue entry(UniValue::VOBJ); entry.push_back(Pair("txid", out.tx->GetHash().GetHex())); entry.push_back(Pair("vout", out.i)); - CTxDestination address; - if (ExtractDestination(out.tx->vout[out.i].scriptPubKey, address)) { + + if (fValidAddress) { entry.push_back(Pair("address", CBitcoinAddress(address).ToString())); + if (pwalletMain->mapAddressBook.count(address)) entry.push_back(Pair("account", pwalletMain->mapAddressBook[address].name)); - } - entry.push_back(Pair("scriptPubKey", HexStr(pk.begin(), pk.end()))); - if (pk.IsPayToScriptHash()) { - CTxDestination address; - if (ExtractDestination(pk, address)) { + + if (scriptPubKey.IsPayToScriptHash()) { const CScriptID& hash = boost::get(address); CScript redeemScript; if (pwalletMain->GetCScript(hash, redeemScript)) entry.push_back(Pair("redeemScript", HexStr(redeemScript.begin(), redeemScript.end()))); } } - entry.push_back(Pair("amount",ValueFromAmount(nValue))); - entry.push_back(Pair("confirmations",out.nDepth)); + + entry.push_back(Pair("scriptPubKey", HexStr(scriptPubKey.begin(), scriptPubKey.end()))); + entry.push_back(Pair("amount", ValueFromAmount(out.tx->vout[out.i].nValue))); + entry.push_back(Pair("confirmations", out.nDepth)); entry.push_back(Pair("spendable", out.fSpendable)); + entry.push_back(Pair("solvable", out.fSolvable)); results.push_back(entry); } @@ -2524,6 +2442,7 @@ UniValue fundrawtransaction(const UniValue& params, bool fHelp) "Note that all existing inputs must have their previous output transaction be in the wallet.\n" "Note that all inputs selected must be of standard form and P2SH scripts must be\n" "in the wallet using importaddress or addmultisigaddress (to calculate fees).\n" + "You can see whether this is the case by checking the \"solvable\" field in the listunspent output.\n" "Only pay-to-pubkey, multisig, and P2SH versions thereof are currently supported for watch-only\n" "\nArguments:\n" "1. \"hexstring\" (string, required) The hex string of the raw transaction\n" @@ -2533,12 +2452,13 @@ UniValue fundrawtransaction(const UniValue& params, bool fHelp) " \"changePosition\" (numeric, optional, default random) The index of the change output\n" " \"includeWatching\" (boolean, optional, default false) Also select inputs which are watch only\n" " \"lockUnspents\" (boolean, optional, default false) Lock selected unspent outputs\n" + " \"feeRate\" (numeric, optional, default not set: makes wallet determine the fee) Set a specific feerate (" + CURRENCY_UNIT + " per KB)\n" " }\n" " for backward compatibility: passing in a true instead of an object will result in {\"includeWatching\":true}\n" "\nResult:\n" "{\n" " \"hex\": \"value\", (string) The resulting raw transaction (hex-encoded string)\n" - " \"fee\": n, (numeric) Fee the resulting transaction pays\n" + " \"fee\": n, (numeric) Fee in " + CURRENCY_UNIT + " the resulting transaction pays\n" " \"changepos\": n (numeric) The position of the added change output, or -1\n" "}\n" "\"hex\" \n" @@ -2559,6 +2479,8 @@ UniValue fundrawtransaction(const UniValue& params, bool fHelp) int changePosition = -1; bool includeWatching = false; bool lockUnspents = false; + CFeeRate feeRate = CFeeRate(0); + bool overrideEstimatedFeerate = false; if (params.size() > 1) { if (params[1].type() == UniValue::VBOOL) { @@ -2570,7 +2492,15 @@ UniValue fundrawtransaction(const UniValue& params, bool fHelp) UniValue options = params[1]; - RPCTypeCheckObj(options, boost::assign::map_list_of("changeAddress", UniValue::VSTR)("changePosition", UniValue::VNUM)("includeWatching", UniValue::VBOOL)("lockUnspents", UniValue::VBOOL), true, true); + RPCTypeCheckObj(options, + { + {"changeAddress", UniValueType(UniValue::VSTR)}, + {"changePosition", UniValueType(UniValue::VNUM)}, + {"includeWatching", UniValueType(UniValue::VBOOL)}, + {"lockUnspents", UniValueType(UniValue::VBOOL)}, + {"feeRate", UniValueType()}, // will be checked below + }, + true, true); if (options.exists("changeAddress")) { CBitcoinAddress address(options["changeAddress"].get_str()); @@ -2589,6 +2519,12 @@ UniValue fundrawtransaction(const UniValue& params, bool fHelp) if (options.exists("lockUnspents")) lockUnspents = options["lockUnspents"].get_bool(); + + if (options.exists("feeRate")) + { + feeRate = CFeeRate(AmountFromValue(options["feeRate"])); + overrideEstimatedFeerate = true; + } } } @@ -2600,20 +2536,20 @@ UniValue fundrawtransaction(const UniValue& params, bool fHelp) if (origTx.vout.size() == 0) throw JSONRPCError(RPC_INVALID_PARAMETER, "TX must have at least one output"); - if (changePosition != -1 && (changePosition < 0 || changePosition > origTx.vout.size())) + if (changePosition != -1 && (changePosition < 0 || (unsigned int)changePosition > origTx.vout.size())) throw JSONRPCError(RPC_INVALID_PARAMETER, "changePosition out of bounds"); CMutableTransaction tx(origTx); - CAmount nFee; + CAmount nFeeOut; string strFailReason; - if(!pwalletMain->FundTransaction(tx, nFee, changePosition, strFailReason, includeWatching, lockUnspents, changeAddress)) + if(!pwalletMain->FundTransaction(tx, nFeeOut, overrideEstimatedFeerate, feeRate, changePosition, strFailReason, includeWatching, lockUnspents, changeAddress)) throw JSONRPCError(RPC_INTERNAL_ERROR, strFailReason); UniValue result(UniValue::VOBJ); result.push_back(Pair("hex", EncodeHexTx(tx))); result.push_back(Pair("changepos", changePosition)); - result.push_back(Pair("fee", ValueFromAmount(nFee))); + result.push_back(Pair("fee", ValueFromAmount(nFeeOut))); return result; } diff --git a/src/wallet/test/crypto_tests.cpp b/src/wallet/test/crypto_tests.cpp new file mode 100644 index 0000000000000..05387f5f2b5e6 --- /dev/null +++ b/src/wallet/test/crypto_tests.cpp @@ -0,0 +1,230 @@ +// Copyright (c) 2014 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include "random.h" +#include "utilstrencodings.h" +#include "test/test_bitcoin.h" +#include "wallet/crypter.h" + +#include + +#include +#include +#include + +BOOST_FIXTURE_TEST_SUITE(wallet_crypto, BasicTestingSetup) + +bool OldSetKeyFromPassphrase(const SecureString& strKeyData, const std::vector& chSalt, const unsigned int nRounds, const unsigned int nDerivationMethod, unsigned char* chKey, unsigned char* chIV) +{ + if (nRounds < 1 || chSalt.size() != WALLET_CRYPTO_SALT_SIZE) + return false; + + int i = 0; + if (nDerivationMethod == 0) + i = EVP_BytesToKey(EVP_aes_256_cbc(), EVP_sha512(), &chSalt[0], + (unsigned char *)&strKeyData[0], strKeyData.size(), nRounds, chKey, chIV); + + if (i != (int)WALLET_CRYPTO_KEY_SIZE) + { + memory_cleanse(chKey, sizeof(chKey)); + memory_cleanse(chIV, sizeof(chIV)); + return false; + } + return true; +} + +bool OldEncrypt(const CKeyingMaterial& vchPlaintext, std::vector &vchCiphertext, const unsigned char chKey[32], const unsigned char chIV[16]) +{ + // max ciphertext len for a n bytes of plaintext is + // n + AES_BLOCK_SIZE - 1 bytes + int nLen = vchPlaintext.size(); + int nCLen = nLen + AES_BLOCK_SIZE, nFLen = 0; + vchCiphertext = std::vector (nCLen); + + EVP_CIPHER_CTX ctx; + + bool fOk = true; + + EVP_CIPHER_CTX_init(&ctx); + if (fOk) fOk = EVP_EncryptInit_ex(&ctx, EVP_aes_256_cbc(), NULL, chKey, chIV) != 0; + if (fOk) fOk = EVP_EncryptUpdate(&ctx, &vchCiphertext[0], &nCLen, &vchPlaintext[0], nLen) != 0; + if (fOk) fOk = EVP_EncryptFinal_ex(&ctx, (&vchCiphertext[0]) + nCLen, &nFLen) != 0; + EVP_CIPHER_CTX_cleanup(&ctx); + + if (!fOk) return false; + + vchCiphertext.resize(nCLen + nFLen); + return true; +} + +bool OldDecrypt(const std::vector& vchCiphertext, CKeyingMaterial& vchPlaintext, const unsigned char chKey[32], const unsigned char chIV[16]) +{ + // plaintext will always be equal to or lesser than length of ciphertext + int nLen = vchCiphertext.size(); + int nPLen = nLen, nFLen = 0; + + vchPlaintext = CKeyingMaterial(nPLen); + + EVP_CIPHER_CTX ctx; + + bool fOk = true; + + EVP_CIPHER_CTX_init(&ctx); + if (fOk) fOk = EVP_DecryptInit_ex(&ctx, EVP_aes_256_cbc(), NULL, chKey, chIV) != 0; + if (fOk) fOk = EVP_DecryptUpdate(&ctx, &vchPlaintext[0], &nPLen, &vchCiphertext[0], nLen) != 0; + if (fOk) fOk = EVP_DecryptFinal_ex(&ctx, (&vchPlaintext[0]) + nPLen, &nFLen) != 0; + EVP_CIPHER_CTX_cleanup(&ctx); + + if (!fOk) return false; + + vchPlaintext.resize(nPLen + nFLen); + return true; +} + +class TestCrypter +{ +public: +static void TestPassphraseSingle(const std::vector& vchSalt, const SecureString& passphrase, uint32_t rounds, + const std::vector& correctKey = std::vector(), + const std::vector& correctIV=std::vector()) +{ + unsigned char chKey[WALLET_CRYPTO_KEY_SIZE]; + unsigned char chIV[WALLET_CRYPTO_IV_SIZE]; + + CCrypter crypt; + crypt.SetKeyFromPassphrase(passphrase, vchSalt, rounds, 0); + + OldSetKeyFromPassphrase(passphrase, vchSalt, rounds, 0, chKey, chIV); + + BOOST_CHECK_MESSAGE(memcmp(chKey, crypt.chKey, sizeof(chKey)) == 0, \ + HexStr(chKey, chKey+sizeof(chKey)) + std::string(" != ") + HexStr(crypt.chKey, crypt.chKey + (sizeof crypt.chKey))); + BOOST_CHECK_MESSAGE(memcmp(chIV, crypt.chIV, sizeof(chIV)) == 0, \ + HexStr(chIV, chIV+sizeof(chIV)) + std::string(" != ") + HexStr(crypt.chIV, crypt.chIV + (sizeof crypt.chIV))); + + if(!correctKey.empty()) + BOOST_CHECK_MESSAGE(memcmp(chKey, &correctKey[0], sizeof(chKey)) == 0, \ + HexStr(chKey, chKey+sizeof(chKey)) + std::string(" != ") + HexStr(correctKey.begin(), correctKey.end())); + if(!correctIV.empty()) + BOOST_CHECK_MESSAGE(memcmp(chIV, &correctIV[0], sizeof(chIV)) == 0, + HexStr(chIV, chIV+sizeof(chIV)) + std::string(" != ") + HexStr(correctIV.begin(), correctIV.end())); +} + +static void TestPassphrase(const std::vector& vchSalt, const SecureString& passphrase, uint32_t rounds, + const std::vector& correctKey = std::vector(), + const std::vector& correctIV=std::vector()) +{ + TestPassphraseSingle(vchSalt, passphrase, rounds, correctKey, correctIV); + for(SecureString::const_iterator i(passphrase.begin()); i != passphrase.end(); ++i) + TestPassphraseSingle(vchSalt, SecureString(i, passphrase.end()), rounds); +} + + +static void TestDecrypt(const CCrypter& crypt, const std::vector& vchCiphertext, \ + const std::vector& vchPlaintext = std::vector()) +{ + CKeyingMaterial vchDecrypted1; + CKeyingMaterial vchDecrypted2; + int result1, result2; + result1 = crypt.Decrypt(vchCiphertext, vchDecrypted1); + result2 = OldDecrypt(vchCiphertext, vchDecrypted2, crypt.chKey, crypt.chIV); + BOOST_CHECK(result1 == result2); + + // These two should be equal. However, OpenSSL 1.0.1j introduced a change + // that would zero all padding except for the last byte for failed decrypts. + // This behavior was reverted for 1.0.1k. + if (vchDecrypted1 != vchDecrypted2 && vchDecrypted1.size() >= AES_BLOCK_SIZE && SSLeay() == 0x100010afL) + { + for(CKeyingMaterial::iterator it = vchDecrypted1.end() - AES_BLOCK_SIZE; it != vchDecrypted1.end() - 1; it++) + *it = 0; + } + + BOOST_CHECK_MESSAGE(vchDecrypted1 == vchDecrypted2, HexStr(vchDecrypted1.begin(), vchDecrypted1.end()) + " != " + HexStr(vchDecrypted2.begin(), vchDecrypted2.end())); + + if (vchPlaintext.size()) + BOOST_CHECK(CKeyingMaterial(vchPlaintext.begin(), vchPlaintext.end()) == vchDecrypted2); +} + +static void TestEncryptSingle(const CCrypter& crypt, const CKeyingMaterial& vchPlaintext, + const std::vector& vchCiphertextCorrect = std::vector()) +{ + std::vector vchCiphertext1; + std::vector vchCiphertext2; + int result1 = crypt.Encrypt(vchPlaintext, vchCiphertext1); + + int result2 = OldEncrypt(vchPlaintext, vchCiphertext2, crypt.chKey, crypt.chIV); + BOOST_CHECK(result1 == result2); + BOOST_CHECK(vchCiphertext1 == vchCiphertext2); + + if (!vchCiphertextCorrect.empty()) + BOOST_CHECK(vchCiphertext2 == vchCiphertextCorrect); + + const std::vector vchPlaintext2(vchPlaintext.begin(), vchPlaintext.end()); + + if(vchCiphertext1 == vchCiphertext2) + TestDecrypt(crypt, vchCiphertext1, vchPlaintext2); +} + +static void TestEncrypt(const CCrypter& crypt, const std::vector& vchPlaintextIn, \ + const std::vector& vchCiphertextCorrect = std::vector()) +{ + TestEncryptSingle(crypt, CKeyingMaterial(vchPlaintextIn.begin(), vchPlaintextIn.end()), vchCiphertextCorrect); + for(std::vector::const_iterator i(vchPlaintextIn.begin()); i != vchPlaintextIn.end(); ++i) + TestEncryptSingle(crypt, CKeyingMaterial(i, vchPlaintextIn.end())); +} + +}; + +BOOST_AUTO_TEST_CASE(passphrase) { + // These are expensive. + + TestCrypter::TestPassphrase(ParseHex("0000deadbeef0000"), "test", 25000, \ + ParseHex("fc7aba077ad5f4c3a0988d8daa4810d0d4a0e3bcb53af662998898f33df0556a"), \ + ParseHex("cf2f2691526dd1aa220896fb8bf7c369")); + + std::string hash(GetRandHash().ToString()); + std::vector vchSalt(8); + GetRandBytes(&vchSalt[0], vchSalt.size()); + uint32_t rounds = insecure_rand(); + if (rounds > 30000) + rounds = 30000; + TestCrypter::TestPassphrase(vchSalt, SecureString(hash.begin(), hash.end()), rounds); +} + +BOOST_AUTO_TEST_CASE(encrypt) { + std::vector vchSalt = ParseHex("0000deadbeef0000"); + BOOST_CHECK(vchSalt.size() == WALLET_CRYPTO_SALT_SIZE); + CCrypter crypt; + crypt.SetKeyFromPassphrase("passphrase", vchSalt, 25000, 0); + TestCrypter::TestEncrypt(crypt, ParseHex("22bcade09ac03ff6386914359cfe885cfeb5f77ff0d670f102f619687453b29d")); + + for (int i = 0; i != 100; i++) + { + uint256 hash(GetRandHash()); + TestCrypter::TestEncrypt(crypt, std::vector(hash.begin(), hash.end())); + } + +} + +BOOST_AUTO_TEST_CASE(decrypt) { + std::vector vchSalt = ParseHex("0000deadbeef0000"); + BOOST_CHECK(vchSalt.size() == WALLET_CRYPTO_SALT_SIZE); + CCrypter crypt; + crypt.SetKeyFromPassphrase("passphrase", vchSalt, 25000, 0); + + // Some corner cases the came up while testing + TestCrypter::TestDecrypt(crypt,ParseHex("795643ce39d736088367822cdc50535ec6f103715e3e48f4f3b1a60a08ef59ca")); + TestCrypter::TestDecrypt(crypt,ParseHex("de096f4a8f9bd97db012aa9d90d74de8cdea779c3ee8bc7633d8b5d6da703486")); + TestCrypter::TestDecrypt(crypt,ParseHex("32d0a8974e3afd9c6c3ebf4d66aa4e6419f8c173de25947f98cf8b7ace49449c")); + TestCrypter::TestDecrypt(crypt,ParseHex("e7c055cca2faa78cb9ac22c9357a90b4778ded9b2cc220a14cea49f931e596ea")); + TestCrypter::TestDecrypt(crypt,ParseHex("b88efddd668a6801d19516d6830da4ae9811988ccbaf40df8fbb72f3f4d335fd")); + TestCrypter::TestDecrypt(crypt,ParseHex("8cae76aa6a43694e961ebcb28c8ca8f8540b84153d72865e8561ddd93fa7bfa9")); + + for (int i = 0; i != 100; i++) + { + uint256 hash(GetRandHash()); + TestCrypter::TestDecrypt(crypt, std::vector(hash.begin(), hash.end())); + } +} + +BOOST_AUTO_TEST_SUITE_END() diff --git a/src/wallet/test/wallet_tests.cpp b/src/wallet/test/wallet_tests.cpp index b759a6b2e8b75..387b223589340 100644 --- a/src/wallet/test/wallet_tests.cpp +++ b/src/wallet/test/wallet_tests.cpp @@ -48,7 +48,7 @@ static void add_coin(const CAmount& nValue, int nAge = 6*24, bool fIsFromMe = fa wtx->fDebitCached = true; wtx->nDebitCached = 1; } - COutput output(wtx, nInput, nAge, true); + COutput output(wtx, nInput, nAge, true, true); vCoins.push_back(output); } diff --git a/src/wallet/wallet.cpp b/src/wallet/wallet.cpp index 515e884d00581..4a5f13ada038c 100644 --- a/src/wallet/wallet.cpp +++ b/src/wallet/wallet.cpp @@ -23,6 +23,7 @@ #include "timedata.h" #include "txmempool.h" #include "util.h" +#include "ui_interface.h" #include "utilmoneystr.h" #include @@ -365,24 +366,9 @@ void CWallet::Flush(bool shutdown) bitdb.Flush(shutdown); } -bool static UIError(const std::string &str) -{ - uiInterface.ThreadSafeMessageBox(str, "", CClientUIInterface::MSG_ERROR); - return false; -} - -void static UIWarning(const std::string &str) -{ - uiInterface.ThreadSafeMessageBox(str, "", CClientUIInterface::MSG_WARNING); -} - -static std::string AmountErrMsg(const char * const optname, const std::string& strValue) -{ - return strprintf(_("Invalid amount for -%s=: '%s'"), optname, strValue); -} - bool CWallet::Verify() { + LogPrintf("Using BerkeleyDB version %s\n", DbEnv::version(0, 0, 0)); std::string walletFile = GetArg("-wallet", DEFAULT_WALLET_DAT); LogPrintf("Using wallet %s\n", walletFile); @@ -390,7 +376,7 @@ bool CWallet::Verify() // Wallet file must be a plain filename without a directory if (walletFile != boost::filesystem::basename(walletFile) + boost::filesystem::extension(walletFile)) - return UIError(strprintf(_("Wallet %s resides outside data directory %s"), walletFile, GetDataDir().string())); + return InitError(strprintf(_("Wallet %s resides outside data directory %s"), walletFile, GetDataDir().string())); if (!bitdb.Open(GetDataDir())) { @@ -407,7 +393,7 @@ bool CWallet::Verify() // try again if (!bitdb.Open(GetDataDir())) { // if it still fails, it probably means we can't even create the database env - return UIError(strprintf(_("Error initializing wallet database environment %s!"), GetDataDir())); + return InitError(strprintf(_("Error initializing wallet database environment %s!"), GetDataDir())); } } @@ -423,14 +409,14 @@ bool CWallet::Verify() CDBEnv::VerifyResult r = bitdb.Verify(walletFile, CWalletDB::Recover); if (r == CDBEnv::RECOVER_OK) { - UIWarning(strprintf(_("Warning: Wallet file corrupt, data salvaged!" + InitWarning(strprintf(_("Warning: Wallet file corrupt, data salvaged!" " Original %s saved as %s in %s; if" " your balance or transactions are incorrect you should" " restore from a backup."), walletFile, "wallet.{timestamp}.bak", GetDataDir())); } if (r == CDBEnv::RECOVER_FAIL) - return UIError(strprintf(_("%s corrupt, salvage failed"), walletFile)); + return InitError(strprintf(_("%s corrupt, salvage failed"), walletFile)); } return true; @@ -523,16 +509,14 @@ bool CWallet::EncryptWallet(const SecureString& strWalletPassphrase) return false; CKeyingMaterial vMasterKey; - RandAddSeedPerfmon(); vMasterKey.resize(WALLET_CRYPTO_KEY_SIZE); - GetRandBytes(&vMasterKey[0], WALLET_CRYPTO_KEY_SIZE); + GetStrongRandBytes(&vMasterKey[0], WALLET_CRYPTO_KEY_SIZE); CMasterKey kMasterKey; - RandAddSeedPerfmon(); kMasterKey.vchSalt.resize(WALLET_CRYPTO_SALT_SIZE); - GetRandBytes(&kMasterKey.vchSalt[0], WALLET_CRYPTO_SALT_SIZE); + GetStrongRandBytes(&kMasterKey.vchSalt[0], WALLET_CRYPTO_SALT_SIZE); CCrypter crypter; int64_t nStartTime = GetTimeMillis(); @@ -622,6 +606,78 @@ int64_t CWallet::IncOrderPosNext(CWalletDB *pwalletdb) return nRet; } +bool CWallet::AccountMove(std::string strFrom, std::string strTo, CAmount nAmount, std::string strComment) +{ + CWalletDB walletdb(strWalletFile); + if (!walletdb.TxnBegin()) + return false; + + int64_t nNow = GetAdjustedTime(); + + // Debit + CAccountingEntry debit; + debit.nOrderPos = IncOrderPosNext(&walletdb); + debit.strAccount = strFrom; + debit.nCreditDebit = -nAmount; + debit.nTime = nNow; + debit.strOtherAccount = strTo; + debit.strComment = strComment; + AddAccountingEntry(debit, walletdb); + + // Credit + CAccountingEntry credit; + credit.nOrderPos = IncOrderPosNext(&walletdb); + credit.strAccount = strTo; + credit.nCreditDebit = nAmount; + credit.nTime = nNow; + credit.strOtherAccount = strFrom; + credit.strComment = strComment; + AddAccountingEntry(credit, walletdb); + + if (!walletdb.TxnCommit()) + return false; + + return true; +} + +bool CWallet::GetAccountPubkey(CPubKey &pubKey, std::string strAccount, bool bForceNew) +{ + CWalletDB walletdb(strWalletFile); + + CAccount account; + walletdb.ReadAccount(strAccount, account); + + if (!bForceNew) { + if (!account.vchPubKey.IsValid()) + bForceNew = true; + else { + // Check if the current key has been used + CScript scriptPubKey = GetScriptForDestination(account.vchPubKey.GetID()); + for (map::iterator it = mapWallet.begin(); + it != mapWallet.end() && account.vchPubKey.IsValid(); + ++it) + BOOST_FOREACH(const CTxOut& txout, (*it).second.vout) + if (txout.scriptPubKey == scriptPubKey) { + bForceNew = true; + break; + } + } + } + + // Generate a new key + if (bForceNew) { + if (!GetKeyFromPool(account.vchPubKey)) + return false; + + SetAddressBook(account.vchPubKey.GetID(), strAccount, "receive"); + walletdb.WriteAccount(strAccount, account); + } + + pubKey = account.vchPubKey; + + return true; +} + void CWallet::MarkDirty() { { @@ -744,7 +800,7 @@ bool CWallet::AddToWallet(const CWalletTx& wtxIn, bool fFromLoadWallet, CWalletD // Write to disk if (fInsertedNew || fUpdated) - if (!wtx.WriteToDisk(pwalletdb)) + if (!pwalletdb->WriteTx(wtx)) return false; // Break debit/credit balance caches: @@ -844,7 +900,7 @@ bool CWallet::AbandonTransaction(const uint256& hashTx) wtx.nIndex = -1; wtx.setAbandoned(); wtx.MarkDirty(); - wtx.WriteToDisk(&walletdb); + walletdb.WriteTx(wtx); NotifyTransactionChanged(this, wtx.GetHash(), CT_UPDATED); // Iterate over all its outputs, and mark transactions in the wallet that spend them abandoned too TxSpends::const_iterator iter = mapTxSpends.lower_bound(COutPoint(hashTx, 0)); @@ -906,7 +962,7 @@ void CWallet::MarkConflicted(const uint256& hashBlock, const uint256& hashTx) wtx.nIndex = -1; wtx.hashBlock = hashBlock; wtx.MarkDirty(); - wtx.WriteToDisk(&walletdb); + walletdb.WriteTx(wtx); // Iterate over all its outputs, and mark transactions in the wallet that spend them conflicted too TxSpends::const_iterator iter = mapTxSpends.lower_bound(COutPoint(now, 0)); while (iter != mapTxSpends.end() && iter->first.hash == now) { @@ -1201,12 +1257,6 @@ void CWalletTx::GetAccountAmounts(const string& strAccount, CAmount& nReceived, } } - -bool CWalletTx::WriteToDisk(CWalletDB *pwalletdb) -{ - return pwalletdb->WriteTx(GetHash(), *this); -} - /** * Scan the block chain (starting in pindexStart) for transactions * from or to us. If fUpdate is true, found transactions that already @@ -1292,9 +1342,7 @@ bool CWalletTx::RelayWalletTransaction() { if (GetDepthInMainChain() == 0 && !isAbandoned() && InMempool()) { LogPrintf("Relaying wtx %s\n", GetHash().ToString()); - CFeeRate feeRate; - mempool.lookupFeeRate(GetHash(), feeRate); - RelayTransaction((CTransaction)*this, feeRate); + RelayTransaction((CTransaction)*this); return true; } } @@ -1706,7 +1754,8 @@ void CWallet::AvailableCoins(vector& vCoins, bool fOnlyConfirmed, const (!coinControl || !coinControl->HasSelected() || coinControl->fAllowOtherInputs || coinControl->IsSelected(COutPoint((*it).first, i)))) vCoins.push_back(COutput(pcoin, i, nDepth, ((mine & ISMINE_SPENDABLE) != ISMINE_NO) || - (coinControl && coinControl->fAllowWatchOnly && (mine & ISMINE_WATCH_SOLVABLE) != ISMINE_NO))); + (coinControl && coinControl->fAllowWatchOnly && (mine & ISMINE_WATCH_SOLVABLE) != ISMINE_NO), + (mine & (ISMINE_SPENDABLE | ISMINE_WATCH_SOLVABLE)) != ISMINE_NO)); } } } @@ -1932,7 +1981,7 @@ bool CWallet::SelectCoins(const vector& vAvailableCoins, const CAmount& return res; } -bool CWallet::FundTransaction(CMutableTransaction& tx, CAmount& nFeeRet, int& nChangePosInOut, std::string& strFailReason, bool includeWatching, bool lockUnspents, const CTxDestination& destChange) +bool CWallet::FundTransaction(CMutableTransaction& tx, CAmount& nFeeRet, bool overrideEstimatedFeeRate, const CFeeRate& specificFeeRate, int& nChangePosInOut, std::string& strFailReason, bool includeWatching, bool lockUnspents, const CTxDestination& destChange) { vector vecSend; @@ -1947,6 +1996,9 @@ bool CWallet::FundTransaction(CMutableTransaction& tx, CAmount& nFeeRet, int& nC coinControl.destChange = destChange; coinControl.fAllowOtherInputs = true; coinControl.fAllowWatchOnly = includeWatching; + coinControl.fOverrideFeeRate = overrideEstimatedFeeRate; + coinControl.nFeeRate = specificFeeRate; + BOOST_FOREACH(const CTxIn& txin, tx.vin) coinControl.Select(txin.prevout); @@ -2180,7 +2232,7 @@ bool CWallet::CreateTransaction(const vector& vecSend, CWalletTx& wt // Insert change txn at random position: nChangePosInOut = GetRandInt(txNew.vout.size()+1); } - else if (nChangePosInOut > txNew.vout.size()) + else if ((unsigned int)nChangePosInOut > txNew.vout.size()) { strFailReason = _("Change index out of range"); return false; @@ -2260,6 +2312,8 @@ bool CWallet::CreateTransaction(const vector& vecSend, CWalletTx& wt if (coinControl && nFeeNeeded > 0 && coinControl->nMinimumTotalFee > nFeeNeeded) { nFeeNeeded = coinControl->nMinimumTotalFee; } + if (coinControl && coinControl->fOverrideFeeRate) + nFeeNeeded = coinControl->nFeeRate.GetFee(nBytes); // If we made it here and we aren't even able to meet the relay fee on the next pass, give up // because we must be at the maximum allowed fee. @@ -2635,12 +2689,19 @@ bool CWallet::GetKeyFromPool(CPubKey& result) int64_t CWallet::GetOldestKeyPoolTime() { - int64_t nIndex = 0; - CKeyPool keypool; - ReserveKeyFromKeyPool(nIndex, keypool); - if (nIndex == -1) + LOCK(cs_wallet); + + // if the keypool is empty, return + if (setKeyPool.empty()) return GetTime(); - ReturnKey(nIndex); + + // load oldest key from keypool, get time and return + CKeyPool keypool; + CWalletDB walletdb(strWalletFile); + int64_t nIndex = *(setKeyPool.begin()); + if (!walletdb.ReadPool(nIndex, keypool)) + throw runtime_error("GetOldestKeyPoolTime(): read oldest key in keypool failed"); + assert(keypool.vchPubKey.IsValid()); return keypool.nTime; } @@ -2777,6 +2838,37 @@ set< set > CWallet::GetAddressGroupings() return ret; } +CAmount CWallet::GetAccountBalance(const std::string& strAccount, int nMinDepth, const isminefilter& filter) +{ + CWalletDB walletdb(strWalletFile); + return GetAccountBalance(walletdb, strAccount, nMinDepth, filter); +} + +CAmount CWallet::GetAccountBalance(CWalletDB& walletdb, const std::string& strAccount, int nMinDepth, const isminefilter& filter) +{ + CAmount nBalance = 0; + + // Tally wallet transactions + for (map::iterator it = mapWallet.begin(); it != mapWallet.end(); ++it) + { + const CWalletTx& wtx = (*it).second; + if (!CheckFinalTx(wtx) || wtx.GetBlocksToMaturity() > 0 || wtx.GetDepthInMainChain() < 0) + continue; + + CAmount nReceived, nSent, nFee; + wtx.GetAccountAmounts(strAccount, nReceived, nSent, nFee, filter); + + if (nReceived != 0 && wtx.GetDepthInMainChain() >= nMinDepth) + nBalance += nReceived; + nBalance -= nSent + nFee; + } + + // Tally internal accounting entries + nBalance += walletdb.GetAccountCreditDebit(strAccount); + + return nBalance; +} + std::set CWallet::GetAccountAddresses(const std::string& strAccount) const { LOCK(cs_wallet); @@ -3079,7 +3171,7 @@ bool CWallet::InitLoadWallet() CWallet *tempWallet = new CWallet(walletFile); DBErrors nZapWalletRet = tempWallet->ZapWalletTx(vWtx); if (nZapWalletRet != DB_LOAD_OK) { - return UIError(strprintf(_("Error loading %s: Wallet corrupted"), walletFile)); + return InitError(strprintf(_("Error loading %s: Wallet corrupted"), walletFile)); } delete tempWallet; @@ -3095,22 +3187,22 @@ bool CWallet::InitLoadWallet() if (nLoadWalletRet != DB_LOAD_OK) { if (nLoadWalletRet == DB_CORRUPT) - return UIError(strprintf(_("Error loading %s: Wallet corrupted"), walletFile)); + return InitError(strprintf(_("Error loading %s: Wallet corrupted"), walletFile)); else if (nLoadWalletRet == DB_NONCRITICAL_ERROR) { - UIWarning(strprintf(_("Error reading %s! All keys read correctly, but transaction data" + InitWarning(strprintf(_("Error reading %s! All keys read correctly, but transaction data" " or address book entries might be missing or incorrect."), walletFile)); } else if (nLoadWalletRet == DB_TOO_NEW) - return UIError(strprintf(_("Error loading %s: Wallet requires newer version of %s"), + return InitError(strprintf(_("Error loading %s: Wallet requires newer version of %s"), walletFile, _(PACKAGE_NAME))); else if (nLoadWalletRet == DB_NEED_REWRITE) { - return UIError(strprintf(_("Wallet needed to be rewritten: restart %s to complete"), _(PACKAGE_NAME))); + return InitError(strprintf(_("Wallet needed to be rewritten: restart %s to complete"), _(PACKAGE_NAME))); } else - return UIError(strprintf(_("Error loading %s"), walletFile)); + return InitError(strprintf(_("Error loading %s"), walletFile)); } if (GetBoolArg("-upgradewallet", fFirstRun)) @@ -3126,7 +3218,7 @@ bool CWallet::InitLoadWallet() LogPrintf("Allowing wallet upgrade up to %i\n", nMaxVersion); if (nMaxVersion < walletInstance->GetVersion()) { - return UIError(_("Cannot downgrade wallet")); + return InitError(_("Cannot downgrade wallet")); } walletInstance->SetMaxVersion(nMaxVersion); } @@ -3134,13 +3226,11 @@ bool CWallet::InitLoadWallet() if (fFirstRun) { // Create new keyUser and set as default key - RandAddSeedPerfmon(); - CPubKey newDefaultKey; if (walletInstance->GetKeyFromPool(newDefaultKey)) { walletInstance->SetDefaultKey(newDefaultKey); if (!walletInstance->SetAddressBook(walletInstance->vchDefaultKey.GetID(), "", "receive")) - return UIError(_("Cannot write default address") += "\n"); + return InitError(_("Cannot write default address") += "\n"); } walletInstance->SetBestChain(chainActive.GetLocator()); @@ -3174,7 +3264,7 @@ bool CWallet::InitLoadWallet() block = block->pprev; if (pindexRescan != block) - return UIError(_("Prune: last wallet synchronisation goes beyond pruned data. You need to -reindex (download the whole blockchain again in case of pruned node)")); + return InitError(_("Prune: last wallet synchronisation goes beyond pruned data. You need to -reindex (download the whole blockchain again in case of pruned node)")); } uiInterface.InitMessage(_("Rescanning...")); @@ -3205,7 +3295,7 @@ bool CWallet::InitLoadWallet() copyTo->fFromMe = copyFrom->fFromMe; copyTo->strFromAccount = copyFrom->strFromAccount; copyTo->nOrderPos = copyFrom->nOrderPos; - copyTo->WriteToDisk(&walletdb); + walletdb.WriteTx(*copyTo); } } } @@ -3224,28 +3314,28 @@ bool CWallet::ParameterInteraction() if (ParseMoney(mapArgs["-mintxfee"], n) && n > 0) CWallet::minTxFee = CFeeRate(n); else - return UIError(AmountErrMsg("mintxfee", mapArgs["-mintxfee"])); + return InitError(AmountErrMsg("mintxfee", mapArgs["-mintxfee"])); } if (mapArgs.count("-fallbackfee")) { CAmount nFeePerK = 0; if (!ParseMoney(mapArgs["-fallbackfee"], nFeePerK)) - return UIError(strprintf(_("Invalid amount for -fallbackfee=: '%s'"), mapArgs["-fallbackfee"])); + return InitError(strprintf(_("Invalid amount for -fallbackfee=: '%s'"), mapArgs["-fallbackfee"])); if (nFeePerK > HIGH_TX_FEE_PER_KB) - UIWarning(_("-fallbackfee is set very high! This is the transaction fee you may pay when fee estimates are not available.")); + InitWarning(_("-fallbackfee is set very high! This is the transaction fee you may pay when fee estimates are not available.")); CWallet::fallbackFee = CFeeRate(nFeePerK); } if (mapArgs.count("-paytxfee")) { CAmount nFeePerK = 0; if (!ParseMoney(mapArgs["-paytxfee"], nFeePerK)) - return UIError(AmountErrMsg("paytxfee", mapArgs["-paytxfee"])); + return InitError(AmountErrMsg("paytxfee", mapArgs["-paytxfee"])); if (nFeePerK > HIGH_TX_FEE_PER_KB) - UIWarning(_("-paytxfee is set very high! This is the transaction fee you will pay if you send a transaction.")); + InitWarning(_("-paytxfee is set very high! This is the transaction fee you will pay if you send a transaction.")); payTxFee = CFeeRate(nFeePerK, 1000); if (payTxFee < ::minRelayTxFee) { - return UIError(strprintf(_("Invalid amount for -paytxfee=: '%s' (must be at least %s)"), + return InitError(strprintf(_("Invalid amount for -paytxfee=: '%s' (must be at least %s)"), mapArgs["-paytxfee"], ::minRelayTxFee.ToString())); } } @@ -3253,13 +3343,13 @@ bool CWallet::ParameterInteraction() { CAmount nMaxFee = 0; if (!ParseMoney(mapArgs["-maxtxfee"], nMaxFee)) - return UIError(AmountErrMsg("maxtxfee", mapArgs["-maxtxfee"])); + return InitError(AmountErrMsg("maxtxfee", mapArgs["-maxtxfee"])); if (nMaxFee > HIGH_MAX_TX_FEE) - UIWarning(_("-maxtxfee is set very high! Fees this large could be paid on a single transaction.")); + InitWarning(_("-maxtxfee is set very high! Fees this large could be paid on a single transaction.")); maxTxFee = nMaxFee; if (CFeeRate(maxTxFee, 1000) < ::minRelayTxFee) { - return UIError(strprintf(_("Invalid amount for -maxtxfee=: '%s' (must be at least the minrelay fee of %s to prevent stuck transactions)"), + return InitError(strprintf(_("Invalid amount for -maxtxfee=: '%s' (must be at least the minrelay fee of %s to prevent stuck transactions)"), mapArgs["-maxtxfee"], ::minRelayTxFee.ToString())); } } @@ -3270,6 +3360,46 @@ bool CWallet::ParameterInteraction() return true; } +bool CWallet::BackupWallet(const std::string& strDest) +{ + if (!fFileBacked) + return false; + while (true) + { + { + LOCK(bitdb.cs_db); + if (!bitdb.mapFileUseCount.count(strWalletFile) || bitdb.mapFileUseCount[strWalletFile] == 0) + { + // Flush log data to the dat file + bitdb.CloseDb(strWalletFile); + bitdb.CheckpointLSN(strWalletFile); + bitdb.mapFileUseCount.erase(strWalletFile); + + // Copy wallet file + boost::filesystem::path pathSrc = GetDataDir() / strWalletFile; + boost::filesystem::path pathDest(strDest); + if (boost::filesystem::is_directory(pathDest)) + pathDest /= strWalletFile; + + try { +#if BOOST_VERSION >= 104000 + boost::filesystem::copy_file(pathSrc, pathDest, boost::filesystem::copy_option::overwrite_if_exists); +#else + boost::filesystem::copy_file(pathSrc, pathDest); +#endif + LogPrintf("copied %s to %s\n", strWalletFile, pathDest.string()); + return true; + } catch (const boost::filesystem::filesystem_error& e) { + LogPrintf("error copying %s to %s - %s\n", strWalletFile, pathDest.string(), e.what()); + return false; + } + } + } + MilliSleep(100); + } + return false; +} + CKeyPool::CKeyPool() { nTime = GetTime(); @@ -3347,5 +3477,5 @@ int CMerkleTx::GetBlocksToMaturity() const bool CMerkleTx::AcceptToMemoryPool(bool fLimitFree, CAmount nAbsurdFee) { CValidationState state; - return ::AcceptToMemoryPool(mempool, state, *this, fLimitFree, NULL, NULL, false, nAbsurdFee); + return ::AcceptToMemoryPool(mempool, state, *this, fLimitFree, NULL, false, nAbsurdFee); } diff --git a/src/wallet/wallet.h b/src/wallet/wallet.h index fa8740eb7fe38..683c901444c1b 100644 --- a/src/wallet/wallet.h +++ b/src/wallet/wallet.h @@ -394,8 +394,6 @@ class CWalletTx : public CMerkleTx bool InMempool() const; bool IsTrusted() const; - bool WriteToDisk(CWalletDB *pwalletdb); - int64_t GetTxTime() const; int GetRequestCount() const; @@ -414,10 +412,11 @@ class COutput int i; int nDepth; bool fSpendable; + bool fSolvable; - COutput(const CWalletTx *txIn, int iIn, int nDepthIn, bool fSpendableIn) + COutput(const CWalletTx *txIn, int iIn, int nDepthIn, bool fSpendableIn, bool fSolvableIn) { - tx = txIn; i = iIn; nDepth = nDepthIn; fSpendable = fSpendableIn; + tx = txIn; i = iIn; nDepth = nDepthIn; fSpendable = fSpendableIn; fSolvable = fSolvableIn; } std::string ToString() const; @@ -719,6 +718,8 @@ class CWallet : public CCryptoKeyStore, public CValidationInterface * @return next transaction order id */ int64_t IncOrderPosNext(CWalletDB *pwalletdb = NULL); + bool AccountMove(std::string strFrom, std::string strTo, CAmount nAmount, std::string strComment = ""); + bool GetAccountPubkey(CPubKey &pubKey, std::string strAccount, bool bForceNew = false); void MarkDirty(); bool AddToWallet(const CWalletTx& wtxIn, bool fFromLoadWallet, CWalletDB* pwalletdb); @@ -739,7 +740,7 @@ class CWallet : public CCryptoKeyStore, public CValidationInterface * Insert additional inputs into the transaction by * calling CreateTransaction(); */ - bool FundTransaction(CMutableTransaction& tx, CAmount& nFeeRet, int& nChangePosInOut, std::string& strFailReason, bool includeWatching, bool lockUnspents, const CTxDestination& destChange = CNoDestination()); + bool FundTransaction(CMutableTransaction& tx, CAmount& nFeeRet, bool overrideEstimatedFeeRate, const CFeeRate& specificFeeRate, int& nChangePosInOut, std::string& strFailReason, bool includeWatching, bool lockUnspents, const CTxDestination& destChange = CNoDestination()); /** * Create a new transaction paying the recipients with a set of coins @@ -777,6 +778,8 @@ class CWallet : public CCryptoKeyStore, public CValidationInterface std::set< std::set > GetAddressGroupings(); std::map GetAddressBalances(); + CAmount GetAccountBalance(const std::string& strAccount, int nMinDepth, const isminefilter& filter); + CAmount GetAccountBalance(CWalletDB& walletdb, const std::string& strAccount, int nMinDepth, const isminefilter& filter); std::set GetAccountAddresses(const std::string& strAccount) const; isminetype IsMine(const CTxIn& txin) const; @@ -884,6 +887,8 @@ class CWallet : public CCryptoKeyStore, public CValidationInterface /* Wallets parameter interaction */ static bool ParameterInteraction(); + + bool BackupWallet(const std::string& strDest); }; /** A key allocated from the key pool. */ diff --git a/src/wallet/walletdb.cpp b/src/wallet/walletdb.cpp index f2b5408e92b5d..b5037c9a65abf 100644 --- a/src/wallet/walletdb.cpp +++ b/src/wallet/walletdb.cpp @@ -55,10 +55,10 @@ bool CWalletDB::ErasePurpose(const string& strPurpose) return Erase(make_pair(string("purpose"), strPurpose)); } -bool CWalletDB::WriteTx(uint256 hash, const CWalletTx& wtx) +bool CWalletDB::WriteTx(const CWalletTx& wtx) { nWalletDBUpdated++; - return Write(std::make_pair(std::string("tx"), hash), wtx); + return Write(std::make_pair(std::string("tx"), wtx.GetHash()), wtx); } bool CWalletDB::EraseTx(uint256 hash) @@ -291,7 +291,7 @@ DBErrors CWalletDB::ReorderTransactions(CWallet* pwallet) if (pwtx) { - if (!WriteTx(pwtx->GetHash(), *pwtx)) + if (!WriteTx(*pwtx)) return DB_LOAD_FAIL; } else @@ -315,7 +315,7 @@ DBErrors CWalletDB::ReorderTransactions(CWallet* pwallet) // Since we're changing the order, write it back if (pwtx) { - if (!WriteTx(pwtx->GetHash(), *pwtx)) + if (!WriteTx(*pwtx)) return DB_LOAD_FAIL; } else @@ -698,7 +698,7 @@ DBErrors CWalletDB::LoadWallet(CWallet* pwallet) pwallet->nTimeFirstKey = 1; // 0 would be considered 'no value' BOOST_FOREACH(uint256 hash, wss.vWalletUpgrade) - WriteTx(hash, pwallet->mapWallet[hash]); + WriteTx(pwallet->mapWallet[hash]); // Rewrite encrypted wallets of versions 0.4.0 and 0.5.0rc: if (wss.fIsEncrypted && (wss.nFileVersion == 40000 || wss.nFileVersion == 50000)) @@ -903,46 +903,6 @@ void ThreadFlushWalletDB(const string& strFile) } } -bool BackupWallet(const CWallet& wallet, const string& strDest) -{ - if (!wallet.fFileBacked) - return false; - while (true) - { - { - LOCK(bitdb.cs_db); - if (!bitdb.mapFileUseCount.count(wallet.strWalletFile) || bitdb.mapFileUseCount[wallet.strWalletFile] == 0) - { - // Flush log data to the dat file - bitdb.CloseDb(wallet.strWalletFile); - bitdb.CheckpointLSN(wallet.strWalletFile); - bitdb.mapFileUseCount.erase(wallet.strWalletFile); - - // Copy wallet file - boost::filesystem::path pathSrc = GetDataDir() / wallet.strWalletFile; - boost::filesystem::path pathDest(strDest); - if (boost::filesystem::is_directory(pathDest)) - pathDest /= wallet.strWalletFile; - - try { -#if BOOST_VERSION >= 104000 - boost::filesystem::copy_file(pathSrc, pathDest, boost::filesystem::copy_option::overwrite_if_exists); -#else - boost::filesystem::copy_file(pathSrc, pathDest); -#endif - LogPrintf("copied %s to %s\n", wallet.strWalletFile, pathDest.string()); - return true; - } catch (const boost::filesystem::filesystem_error& e) { - LogPrintf("error copying %s to %s - %s\n", wallet.strWalletFile, pathDest.string(), e.what()); - return false; - } - } - } - MilliSleep(100); - } - return false; -} - // // Try to (very carefully!) recover wallet file if there is a problem. // diff --git a/src/wallet/walletdb.h b/src/wallet/walletdb.h index 0be0c1d94f665..f357ae227a64c 100644 --- a/src/wallet/walletdb.h +++ b/src/wallet/walletdb.h @@ -88,7 +88,7 @@ class CWalletDB : public CDB bool WritePurpose(const std::string& strAddress, const std::string& purpose); bool ErasePurpose(const std::string& strAddress); - bool WriteTx(uint256 hash, const CWalletTx& wtx); + bool WriteTx(const CWalletTx& wtx); bool EraseTx(uint256 hash); bool WriteKey(const CPubKey& vchPubKey, const CPrivKey& vchPrivKey, const CKeyMetadata &keyMeta); @@ -142,7 +142,6 @@ class CWalletDB : public CDB bool WriteAccountingEntry(const uint64_t nAccEntryNum, const CAccountingEntry& acentry); }; -bool BackupWallet(const CWallet& wallet, const std::string& strDest); void ThreadFlushWalletDB(const std::string& strFile); #endif // BITCOIN_WALLET_WALLETDB_H diff --git a/src/zmq/zmqpublishnotifier.cpp b/src/zmq/zmqpublishnotifier.cpp index f5839620ff227..b6c907980f0c3 100644 --- a/src/zmq/zmqpublishnotifier.cpp +++ b/src/zmq/zmqpublishnotifier.cpp @@ -9,6 +9,11 @@ static std::multimap mapPublishNotifiers; +static const char *MSG_HASHBLOCK = "hashblock"; +static const char *MSG_HASHTX = "hashtx"; +static const char *MSG_RAWBLOCK = "rawblock"; +static const char *MSG_RAWTX = "rawtx"; + // Internal function to send multipart message static int zmq_send_multipart(void *sock, const void* data, size_t size, ...) { @@ -118,6 +123,23 @@ void CZMQAbstractPublishNotifier::Shutdown() psocket = 0; } +bool CZMQAbstractPublishNotifier::SendMessage(const char *command, const void* data, size_t size) +{ + assert(psocket); + + /* send three parts, command & data & a LE 4byte sequence number */ + unsigned char msgseq[sizeof(uint32_t)]; + WriteLE32(&msgseq[0], nSequence); + int rc = zmq_send_multipart(psocket, command, strlen(command), data, size, msgseq, (size_t)sizeof(uint32_t), (void*)0); + if (rc == -1) + return false; + + /* increment memory only sequence number after sending */ + nSequence++; + + return true; +} + bool CZMQPublishHashBlockNotifier::NotifyBlock(const CBlockIndex *pindex) { uint256 hash = pindex->GetBlockHash(); @@ -125,8 +147,7 @@ bool CZMQPublishHashBlockNotifier::NotifyBlock(const CBlockIndex *pindex) char data[32]; for (unsigned int i = 0; i < 32; i++) data[31 - i] = hash.begin()[i]; - int rc = zmq_send_multipart(psocket, "hashblock", 9, data, 32, 0); - return rc == 0; + return SendMessage(MSG_HASHBLOCK, data, 32); } bool CZMQPublishHashTransactionNotifier::NotifyTransaction(const CTransaction &transaction) @@ -136,8 +157,7 @@ bool CZMQPublishHashTransactionNotifier::NotifyTransaction(const CTransaction &t char data[32]; for (unsigned int i = 0; i < 32; i++) data[31 - i] = hash.begin()[i]; - int rc = zmq_send_multipart(psocket, "hashtx", 6, data, 32, 0); - return rc == 0; + return SendMessage(MSG_HASHTX, data, 32); } bool CZMQPublishRawBlockNotifier::NotifyBlock(const CBlockIndex *pindex) @@ -158,8 +178,7 @@ bool CZMQPublishRawBlockNotifier::NotifyBlock(const CBlockIndex *pindex) ss << block; } - int rc = zmq_send_multipart(psocket, "rawblock", 8, &(*ss.begin()), ss.size(), 0); - return rc == 0; + return SendMessage(MSG_RAWBLOCK, &(*ss.begin()), ss.size()); } bool CZMQPublishRawTransactionNotifier::NotifyTransaction(const CTransaction &transaction) @@ -168,6 +187,5 @@ bool CZMQPublishRawTransactionNotifier::NotifyTransaction(const CTransaction &tr LogPrint("zmq", "zmq: Publish rawtx %s\n", hash.GetHex()); CDataStream ss(SER_NETWORK, PROTOCOL_VERSION); ss << transaction; - int rc = zmq_send_multipart(psocket, "rawtx", 5, &(*ss.begin()), ss.size(), 0); - return rc == 0; + return SendMessage(MSG_RAWTX, &(*ss.begin()), ss.size()); } diff --git a/src/zmq/zmqpublishnotifier.h b/src/zmq/zmqpublishnotifier.h index 44d5cbea67d20..22f02a3d0dbb8 100644 --- a/src/zmq/zmqpublishnotifier.h +++ b/src/zmq/zmqpublishnotifier.h @@ -11,7 +11,19 @@ class CBlockIndex; class CZMQAbstractPublishNotifier : public CZMQAbstractNotifier { +private: + uint32_t nSequence; //! upcounting per message sequence number + public: + + /* send zmq multipart message + parts: + * command + * data + * message sequence number + */ + bool SendMessage(const char *command, const void* data, size_t size); + bool Initialize(void *pcontext); void Shutdown(); };