[Depends] Update macOS cross-compiling toolchain

.github/workflows/build-and-test.yml

@@ -166,11 +166,12 @@ jobs:
             apt_get: python3-zmq qtbase5-dev qttools5-dev-tools libqt5svg5-dev libqt5charts5-dev libqrencode-dev protobuf-compiler libdbus-1-dev libharfbuzz-dev libprotobuf-dev
             dep_opts: NO_QT=1 NO_UPNP=1 DEBUG=1 ALLOW_HOST_PACKAGES=1
 
-          - name: macOS 10.10
+          - name: macOS 10.12
             os: ubuntu-18.04
-            host: x86_64-apple-darwin14
+            host: x86_64-apple-darwin16
             apt_get: cmake imagemagick libcap-dev librsvg2-bin libz-dev libbz2-dev libtiff-tools python3-dev python3-setuptools
-            OSX_SDK: 10.11
+            XCODE_VERSION: 11.3.1
+            XCODE_BUILD_ID: 11C505
 
     steps:
       - name: Get Source
@@ -210,12 +211,15 @@ jobs:
 
           mkdir -p depends/SDKs depends/sdk-sources
 
-          if [ -n "${{ matrix.config.OSX_SDK }}" -a ! -f depends/sdk-sources/MacOSX${{ matrix.config.OSX_SDK }}.sdk.tar.gz ]; then
-            curl --location --fail $SDK_URL/MacOSX${{ matrix.config.OSX_SDK }}.sdk.tar.gz -o depends/sdk-sources/MacOSX${{ matrix.config.OSX_SDK }}.sdk.tar.gz
+          OSX_SDK_BASENAME="Xcode-${{ matrix.config.XCODE_VERSION }}-${{ matrix.config.XCODE_BUILD_ID }}-extracted-SDK-with-libcxx-headers.tar.gz"
+          OSX_SDK_PATH="depends/sdk-sources/${OSX_SDK_BASENAME}"
+
+          if [ -n "${{ matrix.config.XCODE_VERSION }}" ] && [ ! -f "$OSX_SDK_PATH" ]; then
+            curl --location --fail "${SDK_URL}/${OSX_SDK_BASENAME}" -o "$OSX_SDK_PATH"
           fi
 
-          if [ -n "${{ matrix.config.OSX_SDK }}" -a -f depends/sdk-sources/MacOSX${{ matrix.config.OSX_SDK }}.sdk.tar.gz ]; then
-            tar -C depends/SDKs -xf depends/sdk-sources/MacOSX${{ matrix.config.OSX_SDK }}.sdk.tar.gz
+          if [ -n "${{ matrix.config.XCODE_VERSION }}" ] && [ -f "$OSX_SDK_PATH" ]; then
+            tar -C "depends/SDKs" -xf "$OSX_SDK_PATH"
           fi
 
           if [[ ${{ matrix.config.host }} = *-mingw32 ]]; then
@@ -315,11 +319,12 @@ jobs:
             goal: install
             BITCOIN_CONFIG: "--enable-zmq --with-incompatible-bdb --with-gui=qt5 CPPFLAGS=-DDEBUG_LOCKORDER"
 
-          - name: macOS 10.10  [GOAL:deploy] [no functional tests]
+          - name: macOS 10.12  [GOAL:deploy] [no functional tests]
             os: ubuntu-18.04
-            host: x86_64-apple-darwin14
+            host: x86_64-apple-darwin16
             apt_get: cmake imagemagick libcap-dev librsvg2-bin libz-dev libbz2-dev libtiff-tools python3-dev python3-setuptools
-            OSX_SDK: 10.11
+            XCODE_VERSION: 11.3.1
+            XCODE_BUILD_ID: 11C505
             unit_tests: false
             functional_tests: false
             goal: deploy
@@ -377,12 +382,15 @@ jobs:
 
           mkdir -p depends/SDKs depends/sdk-sources
 
-          if [ -n "${{ matrix.config.OSX_SDK }}" -a ! -f depends/sdk-sources/MacOSX${{ matrix.config.OSX_SDK }}.sdk.tar.gz ]; then
-            curl --location --fail $SDK_URL/MacOSX${{ matrix.config.OSX_SDK }}.sdk.tar.gz -o depends/sdk-sources/MacOSX${{ matrix.config.OSX_SDK }}.sdk.tar.gz
+          OSX_SDK_BASENAME="Xcode-${{ matrix.config.XCODE_VERSION }}-${{ matrix.config.XCODE_BUILD_ID }}-extracted-SDK-with-libcxx-headers.tar.gz"
+          OSX_SDK_PATH="depends/sdk-sources/${OSX_SDK_BASENAME}"
+
+          if [ -n "${{ matrix.config.XCODE_VERSION }}" ] && [ ! -f "$OSX_SDK_PATH" ]; then
+            curl --location --fail "${SDK_URL}/${OSX_SDK_BASENAME}" -o "$OSX_SDK_PATH"
           fi
 
-          if [ -n "${{ matrix.config.OSX_SDK }}" -a -f depends/sdk-sources/MacOSX${{ matrix.config.OSX_SDK }}.sdk.tar.gz ]; then
-            tar -C depends/SDKs -xf depends/sdk-sources/MacOSX${{ matrix.config.OSX_SDK }}.sdk.tar.gz
+          if [ -n "${{ matrix.config.XCODE_VERSION }}" ] && [ -f "$OSX_SDK_PATH" ]; then
+            tar -C "depends/SDKs" -xf "$OSX_SDK_PATH"
           fi
 
           if [[ ${{ matrix.config.host }} = *-mingw32 ]]; then

contrib/gitian-build.py

@@ -334,7 +334,7 @@ def main():
     args.macos = 'm' in args.os
 
     # Disable for MacOS if no SDK found
-    if args.macos and not os.path.isfile('gitian-builder/inputs/MacOSX10.11.sdk.tar.gz'):
+    if args.macos and not os.path.isfile('gitian-builder/inputs/Xcode-11.3.1-11C505-extracted-SDK-with-libcxx-headers.tar.gz'):
         print('Cannot build for MacOS, SDK does not exist. Will build for other OSes')
         args.macos = False
 

contrib/gitian-descriptors/gitian-osx.yml

@@ -32,12 +32,12 @@ remotes:
 - "url": "https://github.com/pivx-project/pivx.git"
   "dir": "pivx"
 files:
-- "MacOSX10.11.sdk.tar.gz"
+- "Xcode-11.3.1-11C505-extracted-SDK-with-libcxx-headers.tar.gz"
 script: |
   set -e -o pipefail
 
   WRAP_DIR=$HOME/wrapped
-  HOSTS="x86_64-apple-darwin14"
+  HOSTS="x86_64-apple-darwin16"
   CONFIGFLAGS="--enable-reduce-exports --disable-bench --disable-gui-tests --disable-online-rust GENISOIMAGE=$WRAP_DIR/genisoimage"
   FAKETIME_HOST_PROGS=""
   FAKETIME_PROGS="ar ranlib date dmg genisoimage"
@@ -91,7 +91,7 @@ script: |
   BASEPREFIX=`pwd`/depends
 
   mkdir -p ${BASEPREFIX}/SDKs
-  tar -C ${BASEPREFIX}/SDKs -xf ${BUILD_DIR}/MacOSX10.11.sdk.tar.gz
+  tar -C ${BASEPREFIX}/SDKs -xf ${BUILD_DIR}/Xcode-11.3.1-11C505-extracted-SDK-with-libcxx-headers.tar.gz
 
   # Build dependencies for each host
   for i in $HOSTS; do

contrib/macdeploy/README.md

@@ -1,15 +1,128 @@
-### MacDeploy ###
+# MacOS Deployment
 
-For Snow Leopard (which uses [Python 2.6](http://www.python.org/download/releases/2.6/)), you will need the param_parser package:
+The `macdeployqtplus` script should not be run manually. Instead, after building as usual:
 
-	sudo easy_install argparse
+```bash
+make deploy
+```
 
-This script should not be run manually, instead, after building as usual:
+During the deployment process, the disk image window will pop up briefly
+when the fancy settings are applied. This is normal, please do not interfere,
+the process will unmount the DMG and cleanup before finishing.
 
-	make deploy
+When complete, it will have produced `PIVX-Qt.dmg`.
 
-During the process, the disk image window will pop up briefly where the fancy
-settings are applied. This is normal, please do not interfere.
+## SDK Extraction
 
-When finished, it will produce `Pivx-Qt.dmg`.
+### Step 1: Obtaining `Xcode.app`
 
+Our current macOS SDK
+(`Xcode-11.3.1-11C505-extracted-SDK-with-libcxx-headers.tar.gz`) can be
+extracted from
+[Xcode_11.3.1.xip](https://download.developer.apple.com/Developer_Tools/Xcode_11.3.1/Xcode_11.3.1.xip).
+An Apple ID is needed to download this.
+
+After Xcode version 7.x, Apple started shipping the `Xcode.app` in a `.xip`
+archive. This makes the SDK less-trivial to extract on non-macOS machines. One
+approach (tested on Debian Buster) is outlined below:
+
+```bash
+# Install/clone tools needed for extracting Xcode.app
+apt install cpio
+git clone https://github.com/bitcoin-core/apple-sdk-tools.git
+
+# Unpack Xcode_11.3.1.xip and place the resulting Xcode.app in your current
+# working directory
+python3 apple-sdk-tools/extract_xcode.py -f Xcode_11.3.1.xip | cpio -d -i
+```
+
+On macOS the process is more straightforward:
+
+```bash
+xip -x Xcode_11.3.1.xip
+```
+
+### Step 2: Generating `Xcode-11.3.1-11C505-extracted-SDK-with-libcxx-headers.tar.gz` from `Xcode.app`
+
+To generate `Xcode-11.3.1-11C505-extracted-SDK-with-libcxx-headers.tar.gz`, run
+the script [`gen-sdk`](./gen-sdk) with the path to `Xcode.app` (extracted in the
+previous stage) as the first argument.
+
+```bash
+# Generate a Xcode-11.3.1-11C505-extracted-SDK-with-libcxx-headers.tar.gz from
+# the supplied Xcode.app
+./contrib/macdeploy/gen-sdk '/path/to/Xcode.app'
+```
+
+## Deterministic macOS DMG Notes
+Working macOS DMGs are created in Linux by combining a recent `clang`, the Apple
+`binutils` (`ld`, `ar`, etc) and DMG authoring tools.
+
+Apple uses `clang` extensively for development and has upstreamed the necessary
+functionality so that a vanilla clang can take advantage. It supports the use of `-F`,
+`-target`, `-mmacosx-version-min`, and `--sysroot`, which are all necessary when
+building for macOS.
+
+Apple's version of `binutils` (called `cctools`) contains lots of functionality missing in the
+FSF's `binutils`. In addition to extra linker options for frameworks and sysroots, several
+other tools are needed as well such as `install_name_tool`, `lipo`, and `nmedit`. These
+do not build under Linux, so they have been patched to do so. The work here was used as
+a starting point: [mingwandroid/toolchain4](https://github.com/mingwandroid/toolchain4).
+
+In order to build a working toolchain, the following source packages are needed from
+Apple: `cctools`, `dyld`, and `ld64`.
+
+These tools inject timestamps by default, which produce non-deterministic binaries. The
+`ZERO_AR_DATE` environment variable is used to disable that.
+
+This version of `cctools` has been patched to use the current version of `clang`'s headers
+and its `libLTO.so` rather than those from `llvmgcc`, as it was originally done in `toolchain4`.
+
+To complicate things further, all builds must target an Apple SDK. These SDKs are free to
+download, but not redistributable. To obtain it, register for an Apple Developer Account,
+then download [Xcode_11.3.1](https://download.developer.apple.com/Developer_Tools/Xcode_11.3.1/Xcode_11.3.1.xip).
+
+This file is many gigabytes in size, but most (but not all) of what we need is
+contained only in a single directory:
+
+```bash
+Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX.sdk
+```
+
+See the SDK Extraction notes above for how to obtain it.
+
+The Gitian descriptors build 2 sets of files: Linux tools, then Apple binaries which are
+created using these tools. The build process has been designed to avoid including the
+SDK's files in Gitian's outputs. All interim tarballs are fully deterministic and may be freely
+redistributed.
+
+`genisoimage` is used to create the initial DMG. It is not deterministic as-is, so it has been
+patched. A system `genisoimage` will work fine, but it will not be deterministic because
+the file-order will change between invocations. The patch can be seen here: [cdrkit-deterministic.patch](https://github.com/pivx-project/pivx/blob/master/depends/patches/native_cdrkit/cdrkit-deterministic.patch).
+No effort was made to fix this cleanly, so it likely leaks memory badly, however it's only used for
+a single invocation, so that's no real concern.
+
+`genisoimage` cannot compress DMGs, so afterwards, the DMG tool from the
+`libdmg-hfsplus` project is used to compress it. There are several bugs in this tool and its
+maintainer has seemingly abandoned the project.
+
+The DMG tool has the ability to create DMGs from scratch as well, but this functionality is
+broken. Only the compression feature is currently used. Ideally, the creation could be fixed
+and `genisoimage` would no longer be necessary.
+
+Background images and other features can be added to DMG files by inserting a
+`.DS_Store` before creation. This is generated by the script `contrib/macdeploy/custom_dsstore.py`.
+
+As of OS X 10.9 Mavericks, using an Apple-blessed key to sign binaries is a requirement in
+order to satisfy the new Gatekeeper requirements. Because this private key cannot be
+shared, we'll have to be a bit creative in order for the build process to remain somewhat
+deterministic. Here's how it works:
+
+- Builders use Gitian to create an unsigned release. This outputs an unsigned DMG which
+  users may choose to bless and run. It also outputs an unsigned app structure in the form
+  of a tarball, which also contains all of the tools that have been previously (deterministically)
+  built in order to create a final DMG.
+- The Apple keyholder uses this unsigned app to create a detached signature, using the
+  script that is also included there. Detached signatures are available from this [repository](https://github.com/pivx-project/pivx-detached-sigs).
+- Builders feed the unsigned app + detached signature back into Gitian. It uses the
+  pre-built tools to recombine the pieces into a deterministic DMG.

contrib/macdeploy/custom_dsstore.py

@@ -2,7 +2,6 @@
 # Copyright (c) 2013-2017 The Bitcoin Core developers
 # Distributed under the MIT software license, see the accompanying
 # file COPYING or http://www.opensource.org/licenses/mit-license.php.
-import biplist
 from ds_store import DSStore
 from mac_alias import Alias
 import sys
@@ -47,7 +46,7 @@
 alias.volume.disk_image_alias.target.carbon_path = 'Macintosh HD:Users:\x00bitcoinuser:\x00Documents:\x00bitcoin:\x00bitcoin:\x00' + package_name_ns + '.temp.dmg'
 alias.volume.disk_image_alias.target.posix_path = 'Users/bitcoinuser/Documents/bitcoin/bitcoin/' + package_name_ns + '.temp.dmg'
 alias.target.carbon_path = package_name_ns + ':.background:\x00background.tiff'
-icvp['backgroundImageAlias'] = biplist.Data(alias.to_bytes())
+icvp['backgroundImageAlias'] = alias.to_bytes()
 ds['.']['icvp'] = icvp
 
 ds['.']['vSrn'] = ('long', 1)

contrib/macdeploy/gen-sdk

@@ -0,0 +1,94 @@
+#!/usr/bin/env python3
+import argparse
+import plistlib
+import pathlib
+import sys
+import tarfile
+import gzip
+import os
+import contextlib
+
+@contextlib.contextmanager
+def cd(path):
+    """Context manager that restores PWD even if an exception was raised."""
+    old_pwd = os.getcwd()
+    os.chdir(str(path))
+    try:
+        yield
+    finally:
+        os.chdir(old_pwd)
+
+def run():
+    parser = argparse.ArgumentParser(
+        description=__doc__, formatter_class=argparse.RawTextHelpFormatter)
+
+    parser.add_argument('xcode_app', metavar='XCODEAPP', nargs=1)
+    parser.add_argument("-o", metavar='OUTSDKTGZ', nargs=1, dest='out_sdktgz', required=False)
+
+    args = parser.parse_args()
+
+    xcode_app = pathlib.Path(args.xcode_app[0]).resolve()
+    assert xcode_app.is_dir(), "The supplied Xcode.app path '{}' either does not exist or is not a directory".format(xcode_app)
+
+    xcode_app_plist = xcode_app.joinpath("Contents/version.plist")
+    with xcode_app_plist.open('rb') as fp:
+        pl = plistlib.load(fp)
+        xcode_version = pl['CFBundleShortVersionString']
+        xcode_build_id = pl['ProductBuildVersion']
+    print("Found Xcode (version: {xcode_version}, build id: {xcode_build_id})".format(xcode_version=xcode_version, xcode_build_id=xcode_build_id))
+
+    sdk_dir = xcode_app.joinpath("Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX.sdk")
+    sdk_plist = sdk_dir.joinpath("System/Library/CoreServices/SystemVersion.plist")
+    with sdk_plist.open('rb') as fp:
+        pl = plistlib.load(fp)
+        sdk_version = pl['ProductVersion']
+        sdk_build_id = pl['ProductBuildVersion']
+    print("Found MacOSX SDK (version: {sdk_version}, build id: {sdk_build_id})".format(sdk_version=sdk_version, sdk_build_id=sdk_build_id))
+
+    out_name = "Xcode-{xcode_version}-{xcode_build_id}-extracted-SDK-with-libcxx-headers".format(xcode_version=xcode_version, xcode_build_id=xcode_build_id)
+
+    xcode_libcxx_dir = xcode_app.joinpath("Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/include/c++/v1")
+    assert xcode_libcxx_dir.is_dir()
+
+    if args.out_sdktgz:
+        out_sdktgz_path = pathlib.Path(args.out_sdktgz_path)
+    else:
+        # Construct our own out_sdktgz if not specified on the command line
+        out_sdktgz_path = pathlib.Path("./{}.tar.gz".format(out_name))
+
+    def tarfp_add_with_base_change(tarfp, dir_to_add, alt_base_dir):
+        """Add all files in dir_to_add to tarfp, but prepent MEMBERPREFIX to the files'
+        names
+
+        e.g. if the only file under /root/bazdir is /root/bazdir/qux, invoking:
+
+            tarfp_add_with_base_change(tarfp, "foo/bar", "/root/bazdir")
+
+        would result in the following members being added to tarfp:
+
+            foo/bar/             -> corresponding to /root/bazdir
+            foo/bar/qux          -> corresponding to /root/bazdir/qux
+
+        """
+        def change_tarinfo_base(tarinfo):
+            if tarinfo.name and tarinfo.name.startswith("./"):
+                tarinfo.name = str(pathlib.Path(alt_base_dir, tarinfo.name))
+            if tarinfo.linkname and tarinfo.linkname.startswith("./"):
+                tarinfo.linkname = str(pathlib.Path(alt_base_dir, tarinfo.linkname))
+            return tarinfo
+        with cd(dir_to_add):
+            tarfp.add(".", recursive=True, filter=change_tarinfo_base)
+
+    print("Creating output .tar.gz file...")
+    with out_sdktgz_path.open("wb") as fp:
+        with gzip.GzipFile(fileobj=fp, compresslevel=9, mtime=0) as gzf:
+            with tarfile.open(mode="w", fileobj=gzf) as tarfp:
+                print("Adding MacOSX SDK {} files...".format(sdk_version))
+                tarfp_add_with_base_change(tarfp, sdk_dir, out_name)
+                print("Adding libc++ headers...")
+                tarfp_add_with_base_change(tarfp, xcode_libcxx_dir, "{}/usr/include/c++/v1".format(out_name))
+    print("Done! Find the resulting gzipped tarball at:")
+    print(out_sdktgz_path.resolve())
+
+if __name__ == '__main__':
+    run()

contrib/macdeploy/macdeployqtplus

@@ -554,7 +554,8 @@ if len(config.fancy) == 1:
         sys.exit(1)
 
     try:
-        fancy = plistlib.readPlist(p)
+        with open(p, 'rb') as fp:
+            fancy = plistlib.load(fp, fmt=plistlib.FMT_XML)
     except:
         if verbose >= 1:
             sys.stderr.write("Error: Could not parse fancy disk image plist at \"%s\"\n" % (p))