From a15b4d9590f5642b6dbe3f1725e64324d3841441 Mon Sep 17 00:00:00 2001 From: Antoine Poinsot Date: Mon, 22 Jul 2024 12:51:51 +0200 Subject: [PATCH 1/7] Drop base58 dependency, copy over David Keijser's code directly There is no need to be remotely fetching code for a simple 100 lines file of code which should never change anymore. --- bip32/base58.py | 175 +++++++++++++++++++++++++++++++++++++++++++++++ bip32/bip32.py | 14 ++-- requirements.txt | 1 - 3 files changed, 182 insertions(+), 8 deletions(-) create mode 100644 bip32/base58.py diff --git a/bip32/base58.py b/bip32/base58.py new file mode 100644 index 0000000..a56f925 --- /dev/null +++ b/bip32/base58.py @@ -0,0 +1,175 @@ +'''Base58 encoding + +Implementations of Base58 and Base58Check encodings that are compatible +with the bitcoin network. + +This file was copied over and added to the bip32 project from David Keijser's https://github.com/keis/base58 (https://pypi.org/project/base58/). This +package is released under an MIT licensed. The code was copied in this file and left untouched. Here is a copy of the MIT license accompanying the +code: + Copyright (c) 2015 David Keijser + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in + all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + THE SOFTWARE. +''' + +# This module is based upon base58 snippets found scattered over many bitcoin +# tools written in python. From what I gather the original source is from a +# forum post by Gavin Andresen, so direct your praise to him. +# This module adds shiny packaging and support for python3. + +from functools import lru_cache +from hashlib import sha256 +from typing import Mapping, Union + +# 58 character alphabet used +BITCOIN_ALPHABET = \ + b'123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz' +RIPPLE_ALPHABET = b'rpshnaf39wBUDNEGHJKLM4PQRST7VWXYZ2bcdeCg65jkm8oFqi1tuvAxyz' +XRP_ALPHABET = RIPPLE_ALPHABET + +# Retro compatibility +alphabet = BITCOIN_ALPHABET + + +def scrub_input(v: Union[str, bytes]) -> bytes: + if isinstance(v, str): + v = v.encode('ascii') + + return v + + +def b58encode_int( + i: int, default_one: bool = True, alphabet: bytes = BITCOIN_ALPHABET +) -> bytes: + """ + Encode an integer using Base58 + """ + if not i and default_one: + return alphabet[0:1] + string = b"" + base = len(alphabet) + while i: + i, idx = divmod(i, base) + string = alphabet[idx:idx+1] + string + return string + + +def b58encode( + v: Union[str, bytes], alphabet: bytes = BITCOIN_ALPHABET +) -> bytes: + """ + Encode a string using Base58 + """ + v = scrub_input(v) + + origlen = len(v) + v = v.lstrip(b'\0') + newlen = len(v) + + acc = int.from_bytes(v, byteorder='big') # first byte is most significant + + result = b58encode_int(acc, default_one=False, alphabet=alphabet) + return alphabet[0:1] * (origlen - newlen) + result + + +@lru_cache() +def _get_base58_decode_map(alphabet: bytes, + autofix: bool) -> Mapping[int, int]: + invmap = {char: index for index, char in enumerate(alphabet)} + + if autofix: + groups = [b'0Oo', b'Il1'] + for group in groups: + pivots = [c for c in group if c in invmap] + if len(pivots) == 1: + for alternative in group: + invmap[alternative] = invmap[pivots[0]] + + return invmap + + +def b58decode_int( + v: Union[str, bytes], alphabet: bytes = BITCOIN_ALPHABET, *, + autofix: bool = False +) -> int: + """ + Decode a Base58 encoded string as an integer + """ + if b' ' not in alphabet: + v = v.rstrip() + v = scrub_input(v) + + map = _get_base58_decode_map(alphabet, autofix=autofix) + + decimal = 0 + base = len(alphabet) + try: + for char in v: + decimal = decimal * base + map[char] + except KeyError as e: + raise ValueError( + "Invalid character {!r}".format(chr(e.args[0])) + ) from None + return decimal + + +def b58decode( + v: Union[str, bytes], alphabet: bytes = BITCOIN_ALPHABET, *, + autofix: bool = False +) -> bytes: + """ + Decode a Base58 encoded string + """ + v = v.rstrip() + v = scrub_input(v) + + origlen = len(v) + v = v.lstrip(alphabet[0:1]) + newlen = len(v) + + acc = b58decode_int(v, alphabet=alphabet, autofix=autofix) + + return acc.to_bytes(origlen - newlen + (acc.bit_length() + 7) // 8, 'big') + + +def b58encode_check( + v: Union[str, bytes], alphabet: bytes = BITCOIN_ALPHABET +) -> bytes: + """ + Encode a string using Base58 with a 4 character checksum + """ + v = scrub_input(v) + + digest = sha256(sha256(v).digest()).digest() + return b58encode(v + digest[:4], alphabet=alphabet) + + +def b58decode_check( + v: Union[str, bytes], alphabet: bytes = BITCOIN_ALPHABET, *, + autofix: bool = False +) -> bytes: + '''Decode and verify the checksum of a Base58 encoded string''' + + result = b58decode(v, alphabet=alphabet, autofix=autofix) + result, check = result[:-4], result[-4:] + digest = sha256(sha256(result).digest()).digest() + + if check != digest[:4]: + raise ValueError("Invalid checksum") + + return result diff --git a/bip32/bip32.py b/bip32/bip32.py index 20b5b66..619461a 100644 --- a/bip32/bip32.py +++ b/bip32/bip32.py @@ -1,7 +1,7 @@ -import base58 import hashlib import hmac +from .base58 import b58encode_check, b58decode_check from .utils import ( HARDENED_INDEX, _derive_hardened_private_child, @@ -211,7 +211,7 @@ def get_xpriv_from_path(self, path): self.network, ) - return base58.b58encode_check(extended_key).decode() + return b58encode_check(extended_key).decode() def get_xpub_from_path(self, path): """Get an encoded extended pubkey from a derivation path. @@ -242,11 +242,11 @@ def get_xpub_from_path(self, path): self.network, ) - return base58.b58encode_check(extended_key).decode() + return b58encode_check(extended_key).decode() def get_xpriv(self): """Get the base58 encoded extended private key.""" - return base58.b58encode_check(self.get_xpriv_bytes()).decode() + return b58encode_check(self.get_xpriv_bytes()).decode() def get_xpriv_bytes(self): """Get the encoded extended private key.""" @@ -263,7 +263,7 @@ def get_xpriv_bytes(self): def get_xpub(self): """Get the encoded extended public key.""" - return base58.b58encode_check(self.get_xpub_bytes()).decode() + return b58encode_check(self.get_xpub_bytes()).decode() def get_xpub_bytes(self): """Get the encoded extended public key.""" @@ -285,7 +285,7 @@ def from_xpriv(cls, xpriv): if not isinstance(xpriv, str): raise InvalidInputError("'xpriv' must be a string") - extended_key = base58.b58decode_check(xpriv) + extended_key = b58decode_check(xpriv) ( network, depth, @@ -313,7 +313,7 @@ def from_xpub(cls, xpub): if not isinstance(xpub, str): raise InvalidInputError("'xpub' must be a string") - extended_key = base58.b58decode_check(xpub) + extended_key = b58decode_check(xpub) ( network, depth, diff --git a/requirements.txt b/requirements.txt index 87c5b1c..dfc363d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1 @@ coincurve>=15.0,<19 -base58~=2.0 From 4608fcc3c4c9888ced87b5fef14284a6feb6f57e Mon Sep 17 00:00:00 2001 From: Antoine Poinsot Date: Mon, 22 Jul 2024 13:21:12 +0200 Subject: [PATCH 2/7] base58: run black linter --- bip32/base58.py | 49 +++++++++++++++++++------------------------------ 1 file changed, 19 insertions(+), 30 deletions(-) diff --git a/bip32/base58.py b/bip32/base58.py index a56f925..bb0be21 100644 --- a/bip32/base58.py +++ b/bip32/base58.py @@ -1,4 +1,4 @@ -'''Base58 encoding +"""Base58 encoding Implementations of Base58 and Base58Check encodings that are compatible with the bitcoin network. @@ -25,7 +25,7 @@ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -''' +""" # This module is based upon base58 snippets found scattered over many bitcoin # tools written in python. From what I gather the original source is from a @@ -37,9 +37,8 @@ from typing import Mapping, Union # 58 character alphabet used -BITCOIN_ALPHABET = \ - b'123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz' -RIPPLE_ALPHABET = b'rpshnaf39wBUDNEGHJKLM4PQRST7VWXYZ2bcdeCg65jkm8oFqi1tuvAxyz' +BITCOIN_ALPHABET = b"123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz" +RIPPLE_ALPHABET = b"rpshnaf39wBUDNEGHJKLM4PQRST7VWXYZ2bcdeCg65jkm8oFqi1tuvAxyz" XRP_ALPHABET = RIPPLE_ALPHABET # Retro compatibility @@ -48,7 +47,7 @@ def scrub_input(v: Union[str, bytes]) -> bytes: if isinstance(v, str): - v = v.encode('ascii') + v = v.encode("ascii") return v @@ -65,35 +64,32 @@ def b58encode_int( base = len(alphabet) while i: i, idx = divmod(i, base) - string = alphabet[idx:idx+1] + string + string = alphabet[idx : idx + 1] + string return string -def b58encode( - v: Union[str, bytes], alphabet: bytes = BITCOIN_ALPHABET -) -> bytes: +def b58encode(v: Union[str, bytes], alphabet: bytes = BITCOIN_ALPHABET) -> bytes: """ Encode a string using Base58 """ v = scrub_input(v) origlen = len(v) - v = v.lstrip(b'\0') + v = v.lstrip(b"\0") newlen = len(v) - acc = int.from_bytes(v, byteorder='big') # first byte is most significant + acc = int.from_bytes(v, byteorder="big") # first byte is most significant result = b58encode_int(acc, default_one=False, alphabet=alphabet) return alphabet[0:1] * (origlen - newlen) + result @lru_cache() -def _get_base58_decode_map(alphabet: bytes, - autofix: bool) -> Mapping[int, int]: +def _get_base58_decode_map(alphabet: bytes, autofix: bool) -> Mapping[int, int]: invmap = {char: index for index, char in enumerate(alphabet)} if autofix: - groups = [b'0Oo', b'Il1'] + groups = [b"0Oo", b"Il1"] for group in groups: pivots = [c for c in group if c in invmap] if len(pivots) == 1: @@ -104,13 +100,12 @@ def _get_base58_decode_map(alphabet: bytes, def b58decode_int( - v: Union[str, bytes], alphabet: bytes = BITCOIN_ALPHABET, *, - autofix: bool = False + v: Union[str, bytes], alphabet: bytes = BITCOIN_ALPHABET, *, autofix: bool = False ) -> int: """ Decode a Base58 encoded string as an integer """ - if b' ' not in alphabet: + if b" " not in alphabet: v = v.rstrip() v = scrub_input(v) @@ -122,15 +117,12 @@ def b58decode_int( for char in v: decimal = decimal * base + map[char] except KeyError as e: - raise ValueError( - "Invalid character {!r}".format(chr(e.args[0])) - ) from None + raise ValueError("Invalid character {!r}".format(chr(e.args[0]))) from None return decimal def b58decode( - v: Union[str, bytes], alphabet: bytes = BITCOIN_ALPHABET, *, - autofix: bool = False + v: Union[str, bytes], alphabet: bytes = BITCOIN_ALPHABET, *, autofix: bool = False ) -> bytes: """ Decode a Base58 encoded string @@ -144,12 +136,10 @@ def b58decode( acc = b58decode_int(v, alphabet=alphabet, autofix=autofix) - return acc.to_bytes(origlen - newlen + (acc.bit_length() + 7) // 8, 'big') + return acc.to_bytes(origlen - newlen + (acc.bit_length() + 7) // 8, "big") -def b58encode_check( - v: Union[str, bytes], alphabet: bytes = BITCOIN_ALPHABET -) -> bytes: +def b58encode_check(v: Union[str, bytes], alphabet: bytes = BITCOIN_ALPHABET) -> bytes: """ Encode a string using Base58 with a 4 character checksum """ @@ -160,10 +150,9 @@ def b58encode_check( def b58decode_check( - v: Union[str, bytes], alphabet: bytes = BITCOIN_ALPHABET, *, - autofix: bool = False + v: Union[str, bytes], alphabet: bytes = BITCOIN_ALPHABET, *, autofix: bool = False ) -> bytes: - '''Decode and verify the checksum of a Base58 encoded string''' + """Decode and verify the checksum of a Base58 encoded string""" result = b58decode(v, alphabet=alphabet, autofix=autofix) result, check = result[:-4], result[-4:] From 0e0dcf60b5ce301a8ab844ba57829c5f00c4baef Mon Sep 17 00:00:00 2001 From: Antoine Poinsot Date: Mon, 22 Jul 2024 13:03:22 +0200 Subject: [PATCH 3/7] setup.py: source version dynamically to fix build from source We use the first method from the documentation at https://packaging.python.org/en/latest/guides/single-sourcing-package-version. Thanks to Weiliang Li (Github: https://github.com/kigawas) for pointing out the source build was broken and proposing a (different) fix. --- setup.py | 27 ++++++++++++++++++++++++--- 1 file changed, 24 insertions(+), 3 deletions(-) diff --git a/setup.py b/setup.py index f71ee85..21fc04a 100644 --- a/setup.py +++ b/setup.py @@ -1,6 +1,26 @@ -from setuptools import setup -import bip32 import io +import os + +from setuptools import setup + + +# Taken from https://github.com/pypa/pip/blob/003c7ac56b4da80235d4a147fbcef84b6fbc8248/setup.py#L7-L21 +def read(rel_path: str) -> str: + here = os.path.abspath(os.path.dirname(__file__)) + # intentionally *not* adding an encoding option to open, See: + # https://github.com/pypa/virtualenv/issues/201#issuecomment-3145690 + with open(os.path.join(here, rel_path)) as fp: + return fp.read() + + +# Taken from https://github.com/pypa/pip/blob/003c7ac56b4da80235d4a147fbcef84b6fbc8248/setup.py#L7-L21 +def get_version(rel_path: str) -> str: + for line in read(rel_path).splitlines(): + if line.startswith("__version__"): + # __version__ = "0.9" + delim = '"' if '"' in line else "'" + return line.split(delim)[1] + raise RuntimeError("Unable to find version string.") with io.open("README.md", encoding="utf-8") as f: @@ -10,7 +30,8 @@ requirements = [r for r in f.read().split('\n') if len(r)] setup(name="bip32", - version=bip32.__version__, + # We use the first approach from https://packaging.python.org/en/latest/guides/single-sourcing-package-version + version=get_version("bip32/__init__.py"), description="Minimalistic implementation of the BIP32 key derivation scheme", long_description=long_description, long_description_content_type="text/markdown", From cd1dd6bb4085d4c374f327390214432c36e32ab3 Mon Sep 17 00:00:00 2001 From: Antoine Poinsot Date: Mon, 22 Jul 2024 13:15:16 +0200 Subject: [PATCH 4/7] ci: run on both x86 and M1 for macos --- .github/workflows/python-package.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 9c1b140..d480e79 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -13,7 +13,8 @@ jobs: python-version: [3.7, 3.8, 3.9, '3.10'] os: - ubuntu-latest - - macOS-latest + - macos-13 # (non-M1) + - macos-latest # (M1) - windows-latest runs-on: ${{ matrix.os }} From 7d8e0cb5ca42506b04e0fe8144cfacc896924d01 Mon Sep 17 00:00:00 2001 From: Antoine Poinsot Date: Mon, 22 Jul 2024 13:16:15 +0200 Subject: [PATCH 5/7] ci: drop Python 3.7 support, test on 3.11 and 3.12 --- .github/workflows/python-package.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index d480e79..90dec94 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -10,7 +10,7 @@ jobs: tests: strategy: matrix: - python-version: [3.7, 3.8, 3.9, '3.10'] + python-version: [3.8, 3.9, '3.10', 3.11, 3.12] os: - ubuntu-latest - macos-13 # (non-M1) From 27a40367f4e1633514d04544466cb66c330c1f47 Mon Sep 17 00:00:00 2001 From: Antoine Poinsot Date: Mon, 22 Jul 2024 14:05:58 +0200 Subject: [PATCH 6/7] ci: test coincurve against latest Python version --- .github/workflows/python-package.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 90dec94..63b9228 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -59,11 +59,11 @@ jobs: - name: Set up Python 3.10 uses: actions/setup-python@v2 with: - python-version: '3.10' + python-version: 3.12 - name: Testing with Coincurve ${{ matrix.coincurve-version }} run: | python -m pip install --upgrade pip - pip install pytest - pip install -r requirements.txt -r tests/requirements.txt + pip install setuptools + pip install -r tests/requirements.txt pip install -I coincurve==${{ matrix.coincurve-version }} python setup.py install From a1cb75a061ddd9a9c280206425941d18639be867 Mon Sep 17 00:00:00 2001 From: Antoine Poinsot Date: Mon, 22 Jul 2024 13:08:40 +0200 Subject: [PATCH 7/7] ci: install bip32 from source and tame MacOS Install python-bip32 from source to avoid https://github.com/darosior/python-bip32/pull/41 from happening again. Unfortunately building coincurve on MacOS keeps failing for esoteric reasons and i don't have infinite time, so on MacOS first install coincurve wheels before installing python-bip32. --- .github/workflows/python-package.yml | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 63b9228..8bb6c62 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -24,14 +24,20 @@ jobs: uses: actions/setup-python@v2 with: python-version: ${{ matrix.python-version }} - - name: Installation (deps and package) + - name: Install pip and setuptools run: | python -m pip install --upgrade pip - pip install pytest - pip install -r requirements.txt -r tests/requirements.txt - python setup.py install + pip install setuptools + - name: On MacOS, install coincurve's dependencies and install it from wheels # FIXME: installing from source fails for some reason. + if: matrix.os == 'macos-latest' || matrix.os == 'macos-13' + run: | + brew install autoconf automake libffi libtool pkg-config python + pip install -r requirements.txt + - name: Install python-bip32 from source + run: python setup.py install - name: Test with pytest run: | + pip install -r tests/requirements.txt pytest -vvv linter: