build: gitian, linting and macOS-specific QoL improvements (part 1)

.gitlab-ci.yml

@@ -37,12 +37,12 @@ builder-image:
   image: $CI_REGISTRY_IMAGE:builder-$CI_COMMIT_REF_SLUG
   variables:
     SDK_URL: https://bitcoincore.org/depends-sources/sdks
-    OSX_SDK: "10.11"
+    OSX_SDK: "10.14"
     MAKEJOBS: -j4
   before_script:
     - echo HOST=$HOST
     - |
-      if [ "$HOST" = "x86_64-apple-darwin14" ]; then
+      if [ "$HOST" = "x86_64-apple-darwin16" ]; then
         echo "Downloading MacOS SDK"
         mkdir -p depends/SDKs
         mkdir -p depends/sdk-sources
@@ -187,12 +187,12 @@ x86_64-unknown-linux-gnu-release:
     HOST: x86_64-unknown-linux-gnu
     DEP_OPTS: "NO_UPNP=1"
 
-x86_64-apple-darwin14:
+x86_64-apple-darwin16:
   extends:
     - .build-depends-template
     - .skip-in-fast-mode-template
   variables:
-    HOST: x86_64-apple-darwin14
+    HOST: x86_64-apple-darwin16
 
 ###
 
@@ -267,7 +267,7 @@ mac-build:
     - .build-template
     - .skip-in-fast-mode-template
   needs:
-    - x86_64-apple-darwin14
+    - x86_64-apple-darwin16
   variables:
     BUILD_TARGET: mac
 

.python-version

@@ -1 +1 @@
-3.4.9
+3.5.6

.travis.yml

@@ -205,7 +205,7 @@ after_success:
       env:
       cache: false
       language: python
-      python: '3.4' # Oldest supported version according to doc/dependencies.md
+      python: '3.5' # Oldest supported version according to doc/dependencies.md
       install:
         - set -o errexit; source .travis/lint_04_install.sh
       before_script:
@@ -310,11 +310,11 @@ after_success:
         BITCOIN_CONFIG="--enable-glibc-back-compat --enable-reduce-exports"
 
     - stage: test
-      name: 'macOS 10.10  [GOAL: deploy]'
+      name: 'macOS 10.12  [GOAL: deploy]'
       env: >-
-        HOST=x86_64-apple-darwin14
-        PACKAGES="cmake imagemagick libcap-dev librsvg2-bin libz-dev libbz2-dev libtiff-tools python-dev python3-setuptools-git"
-        OSX_SDK=10.11
+        HOST=x86_64-apple-darwin16
+        PACKAGES="cmake imagemagick libcap-dev librsvg2-bin libz-dev libbz2-dev libtiff-tools python3-dev python3-setuptools"
+        OSX_SDK=10.14
         RUN_UNIT_TESTS=false
         RUN_FUNCTIONAL_TESTS=false
         GOAL="deploy"

CMakeLists.txt

@@ -20,7 +20,7 @@ include_directories(
 if(UNIX AND NOT APPLE)
     set(DEPENDS_PREFIX depends/x86_64-pc-linux-gnu)
 elseif(APPLE)
-    set(DEPENDS_PREFIX depends/x86_64-apple-darwin14)
+    set(DEPENDS_PREFIX depends/x86_64-apple-darwin16)
 elseif(WIN32)
     set(DEPENDS_PREFIX depends/x86_64-w64-mingw32)
 endif()

ci/matrix.sh

@@ -81,9 +81,9 @@ elif [ "$BUILD_TARGET" = "linux64_release" ]; then
   export BITCOIN_CONFIG="--enable-zmq --enable-glibc-back-compat --enable-reduce-exports"
   export PYZMQ=true
 elif [ "$BUILD_TARGET" = "mac" ]; then
-  export HOST=x86_64-apple-darwin14
+  export HOST=x86_64-apple-darwin16
   export BITCOIN_CONFIG="--enable-gui --enable-reduce-exports --disable-miner"
-  export OSX_SDK=10.11
+  export OSX_SDK=10.14
   export GOAL="all deploy"
   export RUN_UNITTESTS=false
   export RUN_INTEGRATIONTESTS=false

configure.ac

@@ -99,8 +99,8 @@ AC_PATH_TOOL(RANLIB, ranlib)
 AC_PATH_TOOL(STRIP, strip)
 AC_PATH_TOOL(GCOV, gcov)
 AC_PATH_PROG(LCOV, lcov)
-dnl Python 3.4 is specified in .python-version and should be used if available, see doc/dependencies.md
-AC_PATH_PROGS([PYTHON], [python3.4 python3.5 python3.6 python3.7 python3 python])
+dnl Python 3.5 is specified in .python-version and should be used if available, see doc/dependencies.md
+AC_PATH_PROGS([PYTHON], [python3.5 python3.6 python3.7 python3.8 python3 python])
 AC_PATH_PROG(GENHTML, genhtml)
 AC_PATH_PROG([GIT], [git])
 AC_PATH_PROG(CCACHE,ccache)

contrib/devtools/README.md

@@ -170,7 +170,7 @@ still compatible with the minimum supported Linux distribution versions.
 
 Example usage after a Gitian build:
 
-    find ../gitian-builder/build -type f -executable | xargs python contrib/devtools/symbol-check.py 
+    find ../gitian-builder/build -type f -executable | xargs python3 contrib/devtools/symbol-check.py
 
 If only supported symbols are used the return value will be 0 and the output will be empty.
 

contrib/devtools/symbol-check.py

@@ -9,7 +9,7 @@
 
 Example usage:
 
-    find ../gitian-builder/build -type f -executable | xargs python contrib/devtools/symbol-check.py
+    find ../gitian-builder/build -type f -executable | xargs python3 contrib/devtools/symbol-check.py
 '''
 import subprocess
 import re

contrib/gitian-build.py

@@ -70,8 +70,8 @@ def build():
 
     if args.macos:
         print('\nCompiling ' + args.version + ' MacOS')
-        subprocess.check_call(['wget', '-N', '-P', 'inputs', 'https://bitcoincore.org/depends-sources/sdks/MacOSX10.11.sdk.tar.gz'])
-        subprocess.check_output(["echo 'bec9d089ebf2e2dd59b1a811a38ec78ebd5da18cbbcd6ab39d1e59f64ac5033f inputs/MacOSX10.11.sdk.tar.gz' | sha256sum -c"], shell=True)
+        subprocess.check_call(['wget', '-N', '-P', 'inputs', 'https://bitcoincore.org/depends-sources/sdks/MacOSX10.14.sdk.tar.gz'])
+        subprocess.check_output(["echo '328aff47e28c17093d59a72712a9b2e62cd8a8b87bbe03f91abb32960b413f0f inputs/MacOSX10.14.sdk.tar.gz' | sha256sum -c"], shell=True)
         subprocess.check_call(['bin/gbuild', '-j', args.jobs, '-m', args.memory, '--commit', 'dash='+args.commit, '--url', 'dash='+args.url, '../dash/contrib/gitian-descriptors/gitian-osx.yml'])
         subprocess.check_call(['bin/gsign', '-p', args.sign_prog, '--signer', args.signer, '--release', args.version+'-osx-unsigned', '--destination', '../gitian.sigs/', '../dash/contrib/gitian-descriptors/gitian-osx.yml'])
         subprocess.check_call('mv build/out/dashcore-*-osx-unsigned.tar.gz inputs/dashcore-osx-unsigned.tar.gz', shell=True)
@@ -135,7 +135,7 @@ def verify():
 def main():
     global args, workdir
 
-    parser = argparse.ArgumentParser(usage='%(prog)s [options] signer version')
+    parser = argparse.ArgumentParser(description='Script for running full Gitian builds.')
     parser.add_argument('-c', '--commit', action='store_true', dest='commit', help='Indicate that the version argument is for a commit or branch')
     parser.add_argument('-p', '--pull', action='store_true', dest='pull', help='Indicate that the version argument is the number of a github repository pull request')
     parser.add_argument('-u', '--url', dest='url', default='https://github.com/dashpay/dash', help='Specify the URL of the repository. Default is %(default)s')
@@ -150,24 +150,14 @@ def main():
     parser.add_argument('-S', '--setup', action='store_true', dest='setup', help='Set up the Gitian building environment. Only works on Debian-based systems (Ubuntu, Debian)')
     parser.add_argument('-D', '--detach-sign', action='store_true', dest='detach_sign', help='Create the assert file for detached signing. Will not commit anything.')
     parser.add_argument('-n', '--no-commit', action='store_false', dest='commit_files', help='Do not commit anything to git')
-    parser.add_argument('signer', help='GPG signer to sign each build assert file')
-    parser.add_argument('version', help='Version number, commit, or branch to build. If building a commit or branch, the -c option must be specified')
+    parser.add_argument('signer', nargs='?', help='GPG signer to sign each build assert file')
+    parser.add_argument('version', nargs='?', help='Version number, commit, or branch to build. If building a commit or branch, the -c option must be specified')
 
     args = parser.parse_args()
     workdir = os.getcwd()
 
-    args.linux = 'l' in args.os
-    args.windows = 'w' in args.os
-    args.macos = 'm' in args.os
-
     args.is_bionic = b'bionic' in subprocess.check_output(['lsb_release', '-cs'])
 
-    if args.buildsign:
-        args.build = True
-        args.sign = True
-
-    args.sign_prog = 'true' if args.detach_sign else 'gpg --detach-sign'
-
     args.lxc = (args.virtualization == 'lxc')
     args.kvm = (args.virtualization == 'kvm')
     args.docker = (args.virtualization == 'docker')
@@ -193,13 +183,28 @@ def main():
         print('Try '+script_name+' --help for more information')
         exit(1)
 
-    # Signer and version shouldn't be empty
-    if args.signer == '':
-        print(script_name+': Missing signer.')
+    if args.setup:
+        setup()
+
+    if args.buildsign:
+        args.build = True
+        args.sign = True
+
+    if not args.build and not args.sign and not args.verify:
+        sys.exit(0)
+
+    args.linux = 'l' in args.os
+    args.windows = 'w' in args.os
+    args.macos = 'm' in args.os
+
+    args.sign_prog = 'true' if args.detach_sign else 'gpg --detach-sign'
+
+    if not args.signer:
+        print(script_name+': Missing signer')
         print('Try '+script_name+' --help for more information')
         exit(1)
-    if args.version == '':
-        print(script_name+': Missing version.')
+    if not args.version:
+        print(script_name+': Missing version')
         print('Try '+script_name+' --help for more information')
         exit(1)
 
@@ -208,12 +213,6 @@ def main():
         raise Exception('Cannot have both commit and pull')
     args.commit = ('' if args.commit else 'v') + args.version
 
-    if args.setup:
-        setup()
-
-    if not args.build and not args.sign and not args.verify:
-        exit(0)
-
     os.chdir('dash')
     if args.pull:
         subprocess.check_call(['git', 'fetch', args.url, 'refs/pull/'+args.version+'/merge'])

contrib/gitian-descriptors/gitian-linux.yml

@@ -26,7 +26,6 @@ packages:
 - "faketime"
 - "bsdmainutils"
 - "ca-certificates"
-- "python"
 - "python3"
 - "libxkbcommon0"
 - "ccache"

contrib/gitian-descriptors/gitian-osx.yml

@@ -22,8 +22,6 @@ packages:
 - "libcap-dev"
 - "libz-dev"
 - "libbz2-dev"
-- "python"
-- "python-dev"
 - "python3"
 - "python3-dev"
 - "python3-setuptools"
@@ -33,12 +31,12 @@ remotes:
 - "url": "https://github.com/dashpay/dash.git"
   "dir": "dash"
 files:
-- "MacOSX10.11.sdk.tar.gz"
+- "MacOSX10.14.sdk.tar.gz"
 script: |
   set -e -o pipefail
 
   WRAP_DIR=$HOME/wrapped
-  HOSTS="x86_64-apple-darwin14"
+  HOSTS="x86_64-apple-darwin16"
   CONFIGFLAGS="--enable-reduce-exports --disable-miner --disable-bench --disable-gui-tests GENISOIMAGE=$WRAP_DIR/genisoimage --enable-crash-hooks"
   FAKETIME_HOST_PROGS=""
   FAKETIME_PROGS="ar ranlib date dmg genisoimage"
@@ -109,7 +107,7 @@ script: |
   BASEPREFIX=`pwd`/depends
 
   mkdir -p ${BASEPREFIX}/SDKs
-  tar -C ${BASEPREFIX}/SDKs -xf ${BUILD_DIR}/MacOSX10.11.sdk.tar.gz
+  tar -C ${BASEPREFIX}/SDKs -xf ${BUILD_DIR}/MacOSX10.14.sdk.tar.gz
 
   # Build dependencies for each host
   for i in $HOSTS; do
@@ -188,8 +186,8 @@ script: |
   done
   mkdir -p $OUTDIR/src
   mv $SOURCEDIST $OUTDIR/src
-  mv ${OUTDIR}/${DISTNAME}-x86_64-apple-darwin14.tar.gz ${OUTDIR}/${DISTNAME}-osx64.tar.gz
-  mv ${OUTDIR}/${DISTNAME}-x86_64-apple-darwin14-debug.tar.gz ${OUTDIR}/${DISTNAME}-osx64-debug.tar.gz
+  mv ${OUTDIR}/${DISTNAME}-x86_64-apple-darwin16.tar.gz ${OUTDIR}/${DISTNAME}-osx64.tar.gz
+  mv ${OUTDIR}/${DISTNAME}-x86_64-apple-darwin16-debug.tar.gz ${OUTDIR}/${DISTNAME}-osx64-debug.tar.gz
 
   # Compress ccache (otherwise the assert file will get too huge)
   if [ "$CCACHE_DIR" != "" ]; then

contrib/gitian-descriptors/gitian-win.yml

@@ -20,7 +20,6 @@ packages:
 - "nsis"
 - "zip"
 - "ca-certificates"
-- "python"
 - "python3"
 - "rename"
 - "ccache"

contrib/linearize/README.md

@@ -1,6 +1,5 @@
 # Linearize
-Construct a linear, no-fork, best version of the Dash blockchain. The scripts
-run using Python 3 but are compatible with Python 2.
+Construct a linear, no-fork, best version of the Dash blockchain.
 
 ## Step 0: Install dash_hash
 

contrib/linearize/linearize-data.py

@@ -17,7 +17,7 @@
 import time
 import glob
 from collections import namedtuple
-from binascii import hexlify, unhexlify
+from binascii import unhexlify
 
 settings = {}
 
@@ -64,7 +64,7 @@ def calc_hash_str(blk_hdr):
     hash = calc_hdr_hash(blk_hdr)
     hash = bufreverse(hash)
     hash = wordreverse(hash)
-    hash_str = hexlify(hash).decode('utf-8')
+    hash_str = hash.hex()
     return hash_str
 
 def get_blk_dt(blk_hdr):
@@ -242,7 +242,7 @@ def run(self):
 
             inMagic = inhdr[:4]
             if (inMagic != self.settings['netmagic']):
-                print("Invalid magic: " + hexlify(inMagic).decode('utf-8'))
+                print("Invalid magic: " + inMagic.hex())
                 return
             inLenLE = inhdr[4:]
             su = struct.unpack("<I", inLenLE)