Tests for Trezor and Coldcard implementations and enable Travis CI

.gitignore

@@ -3,3 +3,5 @@ __pycache__/
 build/
 dist/
 hwi.egg-info/
+test/emulator.img
+test/work

.travis.yml

@@ -0,0 +1,53 @@
+language: python
+os: linux
+dist: xenial
+python:
+    - '3.5'
+cache:
+    pip: true
+    ccache: true
+    directories:
+        - test/work
+addons:
+    apt:
+        sources:
+            - sourceline: 'ppa:bitcoin/bitcoin'
+        packages:
+            - libdb4.8-dev
+            - libdb4.8++-dev
+            - build-essential
+            - curl
+            - git
+            - libsdl2-dev
+            - libsdl2-image-dev
+            - gcc-arm-none-eabi
+            - libnewlib-arm-none-eabi
+            - libudev-dev
+            - libtool
+            - autotools-dev
+            - automake
+            - pkg-config
+            - bsdmainutils
+            - libssl-dev
+            - libevent-dev
+            - libboost-system-dev
+            - libboost-filesystem-dev
+            - libboost-chrono-dev
+            - libboost-test-dev
+            - libboost-thread-dev
+            - libusb-1.0-0-dev
+            - protobuf-compiler
+            - cython3
+            - ccache
+install:
+    - pip install pipenv pysdl2 python-bitcoinrpc
+    # From trezor-mcu to get the correct protobuf version
+    - curl -LO "https://github.com/google/protobuf/releases/download/v3.4.0/protoc-3.4.0-linux-x86_64.zip"
+    - unzip "protoc-3.4.0-linux-x86_64.zip" -d protoc
+    - export PATH="$(pwd)/protoc/bin:$PATH"
+    # Build emulators/simulators and bitcoind
+    - cd test; ./setup_environment.sh; cd ..
+    - pip uninstall -y trezor # Hack to get rid of master branch version of trezor that is installed for trezor-mcu build
+    - python setup.py install
+script:
+    - cd test; ./run_tests.py

README.md

@@ -1,5 +1,7 @@
 # Bitcoin Hardware Wallet Interaction scripts
 
+[![Build Status](https://travis-ci.org/achow101/HWI.svg?branch=master)](https://travis-ci.org/achow101/HWI)
+
 This project contains several scripts for interacting with Bitcoin hardware wallets.
 
 ## Prerequisites

hwilib/devices/trezor.py

@@ -2,6 +2,7 @@
 
 from ..hwwclient import HardwareWalletClient
 from trezorlib.client import TrezorClient as Trezor
+from trezorlib.client import TrezorClientDebugLink
 from trezorlib.transport import enumerate_devices, get_transport
 from trezorlib import protobuf, tools
 from trezorlib import messages as proto
@@ -12,6 +13,7 @@
 
 import binascii
 import json
+import logging
 
 class TxAPIPSBT(TxApi):
 
@@ -50,7 +52,14 @@ class TrezorClient(HardwareWalletClient):
 
     def __init__(self, path, password=''):
         super(TrezorClient, self).__init__(path, password)
-        self.client = Trezor(transport=get_transport(path))
+        if path.startswith('udp'):
+            logging.debug('Simulator found, using DebugLink')
+            transport = get_transport(path)
+            self.client = TrezorClientDebugLink(transport=transport)
+            debuglink = transport.find_debug()
+            self.client.set_debuglink(debuglink)
+        else:
+            self.client = Trezor(transport=get_transport(path))
 
         # if it wasn't able to find a client, throw an error
         if not self.client:

hwilib/serializations.py

@@ -503,7 +503,7 @@ def DeserializeHDKeypath(f, key, hd_keypaths):
 
 def SerializeHDKeypath(hd_keypaths, type):
     r = b""
-    for pubkey, path in hd_keypaths.items():
+    for pubkey, path in sorted(hd_keypaths.items()):
         r += ser_string(type + pubkey)
         packed = struct.pack("<" + "I" * len(path), *path)
         r += ser_string(packed)
@@ -638,7 +638,7 @@ def serialize(self):
             r += ser_string(tx)
 
         if len(self.final_script_sig) == 0 and self.final_script_witness.is_null():
-            for pubkey, sig in self.partial_sigs.items():
+            for pubkey, sig in sorted(self.partial_sigs.items()):
                 r += ser_string(b"\x02" + pubkey)
                 r += ser_string(sig)
 
@@ -664,7 +664,7 @@ def serialize(self):
             r += ser_string(b"\x08")
             r += self.final_script_witness.serialize()
 
-        for key, value in self.unknown:
+        for key, value in sorted(self.unknown.items()):
             r += ser_string(key)
             r += ser_string(value)
 
@@ -745,7 +745,7 @@ def serialize(self):
 
         r += SerializeHDKeypath(self.hd_keypaths, b"\x02")
 
-        for key, value in self.unknown:
+        for key, value in sorted(self.unknown.items()):
             r += ser_string(key)
             r += ser_string(value)
 
@@ -762,7 +762,7 @@ def __init__(self, tx = None):
             self.tx = CTransaction()
         self.inputs = []
         self.outputs = []
-        self.unknown = []
+        self.unknown = {}
 
     def deserialize(self, psbt):
         hexstring = Base64ToHex(psbt.strip())
@@ -860,7 +860,7 @@ def serialize(self):
         r += b"\x00"
 
         # unknowns
-        for key, value in self.unknown:
+        for key, value in sorted(self.unknown.items()):
             r += ser_string(key)
             r += ser_string(value)
 

setup.py

@@ -12,7 +12,7 @@
     long_description=long_description,
     long_description_content_type="text/markdown",
     url="https://github.com/achow101/hwi",
-    packages=setuptools.find_packages(exclude=['dopcs', 'test']),
+    packages=setuptools.find_packages(exclude=['docs', 'test']),
     install_requires=[
         'hidapi', # HID API needed in general
         'trezor[hidapi]', # Trezor One
@@ -27,4 +27,7 @@
         "License :: OSI Approved :: MIT License",
         "Operating System :: OS Independent",
     ],
+    extras_require={
+        'tests': ['python-bitcoinrpc']
+    }
 )

test/README.md

@@ -0,0 +1,102 @@
+# HWI Tests
+
+## Running the tests
+
+This folder contains test cases for HWI. To run these tests, `hwilib` will need to be installed to your python system. You can install it by doing `pip install -e .[tests]` in the root directory.
+
+- `test_bech32.py` tests the bech32 serialization.
+This is taken directly from the [python reference implementation](https://github.com/sipa/bech32/blob/master/ref/python/tests.py).
+- `test_psbt.py` tests the psbt serialization.
+It implements all of the [BIP 174 serialization test vectors](https://github.com/bitcoin/bips/blob/master/bip-0174.mediawiki#Test_Vectors).
+- `test_trezor.py` tests the command line interface and the Trezor implementation.
+It uses the [Trezor One firmware emulator](https://github.com/trezor/trezor-mcu/#building-for-development).
+It also tests usage with `bitcoind`, so the [patched Bitcoin Core](../docs/bitcoin-core-usage.md#bitcoin-core) is required.
+- `test_coldcard.py` tests the command line interface and Coldcard implementation.
+It uses the [Coldcard simulator](https://github.com/Coldcard/firmware/tree/master/unix#coldcard-desktop-simulator).
+It also tests usage with `bitcoind`, so the [patched Bitcoin Core](../docs/bitcoin-core-usage.md#bitcoin-core) is required.
+
+`setup_environment.sh` will build the Trezor emulator, the Coldcard simulator, and the patched `bitcoind`.
+if run in the `test/` directory, these will be built in `work/test/trezor-mcu`, `work/test/firmware`, and `work/test/bitcoin` respectively.
+
+`run_tests.py` runs the tests. If run from the `test/` directory, it will be able to find the Trezor emulator, Coldcard simulator, and bitcoind.
+Otherwise the paths to those will need to be specified on the command line.
+test_trezor.py` and `test_coldcard.py` can be disabled.
+
+If you are building the Trezor emulator, the Coldcard simulator, and `bitcoind` without `setup_environment.sh`, then you will need to make `work/` inside of `test/`.
+
+```
+$ cd test
+$ mkdir -p work
+$ cd work
+```
+
+## Trezor emulator
+
+### Dependencies
+
+In order to build the Trezor emulator, the following packages will need to be installed:
+
+```
+build-essential curl git python3 python3-pip libsdl2-dev libsdl2-image-dev gcc-arm-none-eabi libnewlib-arm-none-eabi gcc-multilib
+```
+
+The python packages can be installed with
+
+```
+pip install pipenv
+```
+
+### Building
+
+Clone the repository:
+
+```
+$ git clone https://github.com/trezor/trezor-mcu/
+```
+
+Build the emulator in headless mode:
+
+```
+$ cd trezor-mcu
+$ export EMULATOR=1 TREZOR_TRANSPORT_V1=1 DEBUG_LINK=1 HEADLESS=1
+$ script/setup
+$ pipenv install
+$ pipenv run script/cibuild
+```
+
+## Coldcard simulator
+
+### Dependencies
+
+In order to build the Coldcard simulator, the following packages will need to be installed:
+
+```
+build-essential git python3 python3-pip libudev-dev gcc-arm-none-eabi
+```
+
+After cloninig the Coldcard repo into this testing folder, the python packages can be installed with:
+
+```
+pip install -r ckcc_firmware/requirements.txt
+pip install -r ckcc_firmware/unix/requirements.txt
+```
+
+### Building
+
+Clone the repository:
+
+```
+$ git clone https://github.com/coldcard/firmware
+```
+
+Build the emulator in headless mode:
+
+```
+$ cd firmware/unix
+$ make setup
+$ make
+```
+
+## Bitcoin Core
+
+In order to build `bitcoind`, see [Bitcoin Core's build documentation](https://github.com/bitcoin/bitcoin/blob/master/doc/build-unix.md#linux-distribution-specific-instructions) to get all of the dependencies installed and for instructions on how to build.

test/data/bip32_vectors.json

@@ -0,0 +1,54 @@
+[
+    {
+        "xprv": "xprv9s21ZrQH143K3QTDL4LXw2F7HEK3wJUD2nW2nRk4stbPy6cq3jPPqjiChkVvvNKmPGJxWUtg6LnF5kejMRNNU3TGtRBeJgk33yuGBxrMPHi",
+        "master_xpub": "xpub6CDEarkRoiwWPj3n3gYygGwgoGchxYg3g6Zs5L2nB4B6wdojzcWCKKHMu9XuY1GyYygRfrVembjAko1T5xTsxj7ecKXxEPzDxx7nCK8Dxtx",
+        "vectors" : [
+            {
+                "path" : "m/0h",
+                "xpub" : "xpub68Gmy5EdvgibQVfPdqkBBCHxA5htiqg55crXYuXoQRKfDBFA1WEjWgP6LHhwBZeNK1VTsfTFUHCdrfp1bgwQ9xv5ski8PX9rL2dZXvgGDnw"
+            },
+            {
+                "path" : "m/0h/1",
+                "xpub" : "xpub6ASuArnXKPbfEwhqN6e3mwBcDTgzisQN1wXN9BJcM47sSikHjJf3UFHKkNAWbWMiGj7Wf5uMash7SyYq527Hqck2AxYysAA7xmALppuCkwQ"
+            },
+            {
+                "path" : "m/0h/1/2h",
+                "xpub" : "xpub6D4BDPcP2GT577Vvch3R8wDkScZWzQzMMUm3PWbmWvVJrZwQY4VUNgqFJPMM3No2dFDFGTsxxpG5uJh7n7epu4trkrX7x7DogT5Uv6fcLW5"
+            },
+            {
+                "path" : "m/0h/1/2h/2",
+                "xpub" : "xpub6FHa3pjLCk84BayeJxFW2SP4XRrFd1JYnxeLeU8EqN3vDfZmbqBqaGJAyiLjTAwm6ZLRQUMv1ZACTj37sR62cfN7fe5JnJ7dh8zL4fiyLHV"
+            },
+            {
+                "path" : "m/0h/1/2h/2/1000000000",
+                "xpub" : "xpub6H1LXWLaKsWFhvm6RVpEL9P4KfRZSW7abD2ttkWP3SSQvnyA8FSVqNTEcYFgJS2UaFcxupHiYkro49S8yGasTvXEYBVPamhGW6cFJodrTHy"
+            }
+        ]
+    },
+    {
+        "xprv": "xprv9s21ZrQH143K31xYSDQpPDxsXRTUcvj2iNHm5NUtrGiGG5e2DtALGdso3pGz6ssrdK4PFmM8NSpSBHNqPqm55Qn3LqFtT2emdEXVYsCzC2U",
+        "master_xpub": "xpub6DAiPJAHXi5oZE6cXrSgsWdMGKtHW6wCaWsGuYL1Wx9qMtRgJn2VekPQeZc1WwAoeuoytGozkCQnToL2PMw4deyhWGEu7Xou6gPYc1KqYuj",
+        "vectors" : [
+            {
+                "path" : "m/0",
+                "xpub" : "xpub69H7F5d8KSRgmmdJg2KhpAK8SR3DjMwAdkxj3ZuxV27CprR9LgpeyGmXUbC6wb7ERfvrnKZjXoUmmDznezpbZb7ap6r1D3tgFxHmwMkQTPH"
+            },
+            {
+                "path" : "m/0/2147483647h",
+                "xpub" : "xpub6ASAVgeehLbnwdqV6UKMHVzgqAG8Gr6riv3Fxxpj8ksbH9ebxaEyBLZ85ySDhKiLDBrQSARLq1uNRts8RuJiHjaDMBU4Zn9h8LZNnBC5y4a"
+            },
+            {
+                "path" : "m/0/2147483647h/1",
+                "xpub" : "xpub6DF8uhdarytz3FWdA8TvFSvvAh8dP3283MY7p2V4SeE2wyWmG5mg5EwVvmdMVCQcoNJxGoWaU9DCWh89LojfZ537wTfunKau47EL2dhHKon"
+            },
+            {
+                "path" : "m/0/2147483647h/1/2147483646h",
+                "xpub" : "xpub6ERApfZwUNrhLCkDtcHTcxd75RbzS1ed54G1LkBUHQVHQKqhMkhgbmJbZRkrgZw4koxb5JaHWkY4ALHY2grBGRjaDMzQLcgJvLJuZZvRcEL"
+            },
+            {
+                "path" : "m/0/2147483647h/1/2147483646h/2",
+                "xpub" : "xpub6FnCn6nSzZAw5Tw7cgR9bi15UV96gLZhjDstkXXxvCLsUXBGXPdSnLFbdpq8p9HmGsApME5hQTZ3emM2rnY5agb9rXpVGyy3bdW6EEgAtqt"
+            }
+        ]
+    }
+]

test/data/coldcard-linux-sock.patch

@@ -0,0 +1,30 @@
+From d1a3a1cef890ebe4ff72a8f89cd6c56dca89747e Mon Sep 17 00:00:00 2001
+From: Andrew Chow <achow101-github@achow101.com>
+Date: Tue, 27 Nov 2018 17:32:44 -0500
+Subject: [PATCH] Use linux unix socket address format
+
+---
+ unix/frozen-modules/pyb.py | 6 +++---
+ 1 file changed, 3 insertions(+), 3 deletions(-)
+
+diff --git a/unix/frozen-modules/pyb.py b/unix/frozen-modules/pyb.py
+index 39778a2..0108516 100644
+--- a/unix/frozen-modules/pyb.py
++++ b/unix/frozen-modules/pyb.py
+@@ -23,10 +23,10 @@ class USB_HID:
+         import usocket as socket
+         fn = b'/tmp/ckcc-simulator.sock'
+         self.pipe = socket.socket(socket.AF_UNIX, socket.SOCK_DGRAM)
+-        addr = bytes([len(fn)+2, socket.AF_UNIX] + list(fn))
++        # addr = bytes([len(fn)+2, socket.AF_UNIX] + list(fn))
+         # If on linux, try uncommenting the following two lines
+-        #import struct
+-        #addr = struct.pack('H108s', socket.AF_UNIX, fn)
++        import struct
++        addr = struct.pack('H108s', socket.AF_UNIX, fn)
+         while 1:
+             try:
+                 self.pipe.bind(addr)
+-- 
+2.11.0
+