From 4c10a830f3243d154b440f0250ec7399dd4cb733 Mon Sep 17 00:00:00 2001 From: SomberNight Date: Thu, 12 Mar 2020 01:44:42 +0100 Subject: [PATCH 01/17] lnmsg: rewrite LN msg encoding/decoding --- electrum/channel_db.py | 4 +- electrum/lightning.json | 903 --------------------------------- electrum/lnchannel.py | 5 +- electrum/lnmsg.py | 343 ++++++++----- electrum/lnpeer.py | 14 +- electrum/lnwire/README.md | 5 + electrum/lnwire/onion_wire.csv | 53 ++ electrum/lnwire/peer_wire.csv | 210 ++++++++ 8 files changed, 489 insertions(+), 1048 deletions(-) delete mode 100644 electrum/lightning.json create mode 100644 electrum/lnwire/README.md create mode 100644 electrum/lnwire/onion_wire.csv create mode 100644 electrum/lnwire/peer_wire.csv diff --git a/electrum/channel_db.py b/electrum/channel_db.py index 66dec951edad..54fc45460cb5 100644 --- a/electrum/channel_db.py +++ b/electrum/channel_db.py @@ -321,11 +321,12 @@ def get_recent_peers(self): return ret # note: currently channel announcements are trusted by default (trusted=True); - # they are not verified. Verifying them would make the gossip sync + # they are not SPV-verified. Verifying them would make the gossip sync # even slower; especially as servers will start throttling us. # It would probably put significant strain on servers if all clients # verified the complete gossip. def add_channel_announcement(self, msg_payloads, *, trusted=True): + # note: signatures have already been verified. if type(msg_payloads) is dict: msg_payloads = [msg_payloads] added = 0 @@ -499,6 +500,7 @@ def verify_channel_update(self, payload): raise Exception(f'failed verifying channel update for {short_channel_id}') def add_node_announcement(self, msg_payloads): + # note: signatures have already been verified. if type(msg_payloads) is dict: msg_payloads = [msg_payloads] new_nodes = {} diff --git a/electrum/lightning.json b/electrum/lightning.json deleted file mode 100644 index d6794e67cc89..000000000000 --- a/electrum/lightning.json +++ /dev/null @@ -1,903 +0,0 @@ -{ - "init": { - "type": "16", - "payload": { - "gflen": { - "position": "0", - "length": "2" - }, - "globalfeatures": { - "position": "2", - "length": "gflen" - }, - "lflen": { - "position": "2+gflen", - "length": "2" - }, - "localfeatures": { - "position": "4+gflen", - "length": "lflen" - } - } - }, - "error": { - "type": "17", - "payload": { - "channel_id": { - "position": "0", - "length": "32" - }, - "len": { - "position": "32", - "length": "2" - }, - "data": { - "position": "34", - "length": "len" - } - } - }, - "ping": { - "type": "18", - "payload": { - "num_pong_bytes": { - "position": "0", - "length": "2" - }, - "byteslen": { - "position": "2", - "length": "2" - }, - "ignored": { - "position": "4", - "length": "byteslen" - } - } - }, - "pong": { - "type": "19", - "payload": { - "byteslen": { - "position": "0", - "length": "2" - }, - "ignored": { - "position": "2", - "length": "byteslen" - } - } - }, - "open_channel": { - "type": "32", - "payload": { - "chain_hash": { - "position": "0", - "length": "32" - }, - "temporary_channel_id": { - "position": "32", - "length": "32" - }, - "funding_satoshis": { - "position": "64", - "length": "8" - }, - "push_msat": { - "position": "72", - "length": "8" - }, - "dust_limit_satoshis": { - "position": "80", - "length": "8" - }, - "max_htlc_value_in_flight_msat": { - "position": "88", - "length": "8" - }, - "channel_reserve_satoshis": { - "position": "96", - "length": "8" - }, - "htlc_minimum_msat": { - "position": "104", - "length": "8" - }, - "feerate_per_kw": { - "position": "112", - "length": "4" - }, - "to_self_delay": { - "position": "116", - "length": "2" - }, - "max_accepted_htlcs": { - "position": "118", - "length": "2" - }, - "funding_pubkey": { - "position": "120", - "length": "33" - }, - "revocation_basepoint": { - "position": "153", - "length": "33" - }, - "payment_basepoint": { - "position": "186", - "length": "33" - }, - "delayed_payment_basepoint": { - "position": "219", - "length": "33" - }, - "htlc_basepoint": { - "position": "252", - "length": "33" - }, - "first_per_commitment_point": { - "position": "285", - "length": "33" - }, - "channel_flags": { - "position": "318", - "length": "1" - }, - "shutdown_len": { - "position": "319", - "length": "2", - "feature": "option_upfront_shutdown_script" - }, - "shutdown_scriptpubkey": { - "position": "321", - "length": "shutdown_len", - "feature": "option_upfront_shutdown_script" - } - } - }, - "accept_channel": { - "type": "33", - "payload": { - "temporary_channel_id": { - "position": "0", - "length": "32" - }, - "dust_limit_satoshis": { - "position": "32", - "length": "8" - }, - "max_htlc_value_in_flight_msat": { - "position": "40", - "length": "8" - }, - "channel_reserve_satoshis": { - "position": "48", - "length": "8" - }, - "htlc_minimum_msat": { - "position": "56", - "length": "8" - }, - "minimum_depth": { - "position": "64", - "length": "4" - }, - "to_self_delay": { - "position": "68", - "length": "2" - }, - "max_accepted_htlcs": { - "position": "70", - "length": "2" - }, - "funding_pubkey": { - "position": "72", - "length": "33" - }, - "revocation_basepoint": { - "position": "105", - "length": "33" - }, - "payment_basepoint": { - "position": "138", - "length": "33" - }, - "delayed_payment_basepoint": { - "position": "171", - "length": "33" - }, - "htlc_basepoint": { - "position": "204", - "length": "33" - }, - "first_per_commitment_point": { - "position": "237", - "length": "33" - }, - "shutdown_len": { - "position": "270", - "length": "2", - "feature": "option_upfront_shutdown_script" - }, - "shutdown_scriptpubkey": { - "position": "272", - "length": "shutdown_len", - "feature": "option_upfront_shutdown_script" - } - } - }, - "funding_created": { - "type": "34", - "payload": { - "temporary_channel_id": { - "position": "0", - "length": "32" - }, - "funding_txid": { - "position": "32", - "length": "32" - }, - "funding_output_index": { - "position": "64", - "length": "2" - }, - "signature": { - "position": "66", - "length": "64" - } - } - }, - "funding_signed": { - "type": "35", - "payload": { - "channel_id": { - "position": "0", - "length": "32" - }, - "signature": { - "position": "32", - "length": "64" - } - } - }, - "funding_locked": { - "type": "36", - "payload": { - "channel_id": { - "position": "0", - "length": "32" - }, - "next_per_commitment_point": { - "position": "32", - "length": "33" - } - } - }, - "shutdown": { - "type": "38", - "payload": { - "channel_id": { - "position": "0", - "length": "32" - }, - "len": { - "position": "32", - "length": "2" - }, - "scriptpubkey": { - "position": "34", - "length": "len" - } - } - }, - "closing_signed": { - "type": "39", - "payload": { - "channel_id": { - "position": "0", - "length": "32" - }, - "fee_satoshis": { - "position": "32", - "length": "8" - }, - "signature": { - "position": "40", - "length": "64" - } - } - }, - "update_add_htlc": { - "type": "128", - "payload": { - "channel_id": { - "position": "0", - "length": "32" - }, - "id": { - "position": "32", - "length": "8" - }, - "amount_msat": { - "position": "40", - "length": "8" - }, - "payment_hash": { - "position": "48", - "length": "32" - }, - "cltv_expiry": { - "position": "80", - "length": "4" - }, - "onion_routing_packet": { - "position": "84", - "length": "1366" - } - } - }, - "update_fulfill_htlc": { - "type": "130", - "payload": { - "channel_id": { - "position": "0", - "length": "32" - }, - "id": { - "position": "32", - "length": "8" - }, - "payment_preimage": { - "position": "40", - "length": "32" - } - } - }, - "update_fail_htlc": { - "type": "131", - "payload": { - "channel_id": { - "position": "0", - "length": "32" - }, - "id": { - "position": "32", - "length": "8" - }, - "len": { - "position": "40", - "length": "2" - }, - "reason": { - "position": "42", - "length": "len" - } - } - }, - "update_fail_malformed_htlc": { - "type": "135", - "payload": { - "channel_id": { - "position": "0", - "length": "32" - }, - "id": { - "position": "32", - "length": "8" - }, - "sha256_of_onion": { - "position": "40", - "length": "32" - }, - "failure_code": { - "position": "72", - "length": "2" - } - } - }, - "commitment_signed": { - "type": "132", - "payload": { - "channel_id": { - "position": "0", - "length": "32" - }, - "signature": { - "position": "32", - "length": "64" - }, - "num_htlcs": { - "position": "96", - "length": "2" - }, - "htlc_signature": { - "position": "98", - "length": "num_htlcs*64" - } - } - }, - "revoke_and_ack": { - "type": "133", - "payload": { - "channel_id": { - "position": "0", - "length": "32" - }, - "per_commitment_secret": { - "position": "32", - "length": "32" - }, - "next_per_commitment_point": { - "position": "64", - "length": "33" - } - } - }, - "update_fee": { - "type": "134", - "payload": { - "channel_id": { - "position": "0", - "length": "32" - }, - "feerate_per_kw": { - "position": "32", - "length": "4" - } - } - }, - "channel_reestablish": { - "type": "136", - "payload": { - "channel_id": { - "position": "0", - "length": "32" - }, - "next_local_commitment_number": { - "position": "32", - "length": "8" - }, - "next_remote_revocation_number": { - "position": "40", - "length": "8" - }, - "your_last_per_commitment_secret": { - "position": "48", - "length": "32", - "feature": "option_data_loss_protect" - }, - "my_current_per_commitment_point": { - "position": "80", - "length": "33", - "feature": "option_data_loss_protect" - } - } - }, - "invalid_realm": { - "type": "PERM|1", - "payload": {} - }, - "temporary_node_failure": { - "type": "NODE|2", - "payload": {} - }, - "permanent_node_failure": { - "type": "PERM|NODE|2", - "payload": {} - }, - "required_node_feature_missing": { - "type": "PERM|NODE|3", - "payload": {} - }, - "invalid_onion_version": { - "type": "BADONION|PERM|4", - "payload": { - "sha256_of_onion": { - "position": "0", - "length": "32" - } - } - }, - "invalid_onion_hmac": { - "type": "BADONION|PERM|5", - "payload": { - "sha256_of_onion": { - "position": "0", - "length": "32" - } - } - }, - "invalid_onion_key": { - "type": "BADONION|PERM|6", - "payload": { - "sha256_of_onion": { - "position": "0", - "length": "32" - } - } - }, - "temporary_channel_failure": { - "type": "UPDATE|7", - "payload": { - "len": { - "position": "0", - "length": "2" - }, - "channel_update": { - "position": "2", - "length": "len" - } - } - }, - "permanent_channel_failure": { - "type": "PERM|8", - "payload": {} - }, - "required_channel_feature_missing": { - "type": "PERM|9", - "payload": {} - }, - "unknown_next_peer": { - "type": "PERM|10", - "payload": {} - }, - "amount_below_minimum": { - "type": "UPDATE|11", - "payload": { - "htlc_msat": { - "position": "0", - "length": "8" - }, - "len": { - "position": "8", - "length": "2" - }, - "channel_update": { - "position": "10", - "length": "len" - } - } - }, - "fee_insufficient": { - "type": "UPDATE|12", - "payload": { - "htlc_msat": { - "position": "0", - "length": "8" - }, - "len": { - "position": "8", - "length": "2" - }, - "channel_update": { - "position": "10", - "length": "len" - } - } - }, - "incorrect_cltv_expiry": { - "type": "UPDATE|13", - "payload": { - "cltv_expiry": { - "position": "0", - "length": "4" - }, - "len": { - "position": "4", - "length": "2" - }, - "channel_update": { - "position": "6", - "length": "len" - } - } - }, - "expiry_too_soon": { - "type": "UPDATE|14", - "payload": { - "len": { - "position": "0", - "length": "2" - }, - "channel_update": { - "position": "2", - "length": "len" - } - } - }, - "unknown_payment_hash": { - "type": "PERM|15", - "payload": {} - }, - "incorrect_payment_amount": { - "type": "PERM|16", - "payload": {} - }, - "final_expiry_too_soon": { - "type": "17", - "payload": {} - }, - "final_incorrect_cltv_expiry": { - "type": "18", - "payload": { - "cltv_expiry": { - "position": "0", - "length": "4" - } - } - }, - "final_incorrect_htlc_amount": { - "type": "19", - "payload": { - "incoming_htlc_amt": { - "position": "0", - "length": "8" - } - } - }, - "channel_disabled": { - "type": "UPDATE|20", - "payload": {} - }, - "expiry_too_far": { - "type": "21", - "payload": {} - }, - "announcement_signatures": { - "type": "259", - "payload": { - "channel_id": { - "position": "0", - "length": "32" - }, - "short_channel_id": { - "position": "32", - "length": "8" - }, - "node_signature": { - "position": "40", - "length": "64" - }, - "bitcoin_signature": { - "position": "104", - "length": "64" - } - } - }, - "channel_announcement": { - "type": "256", - "payload": { - "node_signature_1": { - "position": "0", - "length": "64" - }, - "node_signature_2": { - "position": "64", - "length": "64" - }, - "bitcoin_signature_1": { - "position": "128", - "length": "64" - }, - "bitcoin_signature_2": { - "position": "192", - "length": "64" - }, - "len": { - "position": "256", - "length": "2" - }, - "features": { - "position": "258", - "length": "len" - }, - "chain_hash": { - "position": "258+len", - "length": "32" - }, - "short_channel_id": { - "position": "290+len", - "length": "8" - }, - "node_id_1": { - "position": "298+len", - "length": "33" - }, - "node_id_2": { - "position": "331+len", - "length": "33" - }, - "bitcoin_key_1": { - "position": "364+len", - "length": "33" - }, - "bitcoin_key_2": { - "position": "397+len", - "length": "33" - } - } - }, - "node_announcement": { - "type": "257", - "payload": { - "signature": { - "position": "0", - "length": "64" - }, - "flen": { - "position": "64", - "length": "2" - }, - "features": { - "position": "66", - "length": "flen" - }, - "timestamp": { - "position": "66+flen", - "length": "4" - }, - "node_id": { - "position": "70+flen", - "length": "33" - }, - "rgb_color": { - "position": "103+flen", - "length": "3" - }, - "alias": { - "position": "106+flen", - "length": "32" - }, - "addrlen": { - "position": "138+flen", - "length": "2" - }, - "addresses": { - "position": "140+flen", - "length": "addrlen" - } - } - }, - "channel_update": { - "type": "258", - "payload": { - "signature": { - "position": "0", - "length": "64" - }, - "chain_hash": { - "position": "64", - "length": "32" - }, - "short_channel_id": { - "position": "96", - "length": "8" - }, - "timestamp": { - "position": "104", - "length": "4" - }, - "message_flags": { - "position": "108", - "length": "1" - }, - "channel_flags": { - "position": "109", - "length": "1" - }, - "cltv_expiry_delta": { - "position": "110", - "length": "2" - }, - "htlc_minimum_msat": { - "position": "112", - "length": "8" - }, - "fee_base_msat": { - "position": "120", - "length": "4" - }, - "fee_proportional_millionths": { - "position": "124", - "length": "4" - }, - "htlc_maximum_msat": { - "position": "128", - "length": "8", - "feature": "option_channel_htlc_max" - } - } - }, - "query_short_channel_ids": { - "type": "261", - "payload": { - "chain_hash": { - "position": "0", - "length": "32" - }, - "len": { - "position": "32", - "length": "2" - }, - "encoded_short_ids": { - "position": "34", - "length": "len" - } - } - }, - "reply_short_channel_ids_end": { - "type": "262", - "payload": { - "chain_hash": { - "position": "0", - "length": "32" - }, - "complete": { - "position": "32", - "length": "1" - } - } - }, - "query_channel_range": { - "type": "263", - "payload": { - "chain_hash": { - "position": "0", - "length": "32" - }, - "first_blocknum": { - "position": "32", - "length": "4" - }, - "number_of_blocks": { - "position": "36", - "length": "4" - } - } - }, - "reply_channel_range": { - "type": "264", - "payload": { - "chain_hash": { - "position": "0", - "length": "32" - }, - "first_blocknum": { - "position": "32", - "length": "4" - }, - "number_of_blocks": { - "position": "36", - "length": "4" - }, - "complete": { - "position": "40", - "length": "1" - }, - "len": { - "position": "41", - "length": "2" - }, - "encoded_short_ids": { - "position": "43", - "length": "len" - } - } - }, - "gossip_timestamp_filter": { - "type": "265", - "payload": { - "chain_hash": { - "position": "0", - "length": "32" - }, - "first_timestamp": { - "position": "32", - "length": "4" - }, - "timestamp_range": { - "position": "36", - "length": "4" - } - } - } -} diff --git a/electrum/lnchannel.py b/electrum/lnchannel.py index d38bd48243e3..bd52175b72ad 100644 --- a/electrum/lnchannel.py +++ b/electrum/lnchannel.py @@ -249,7 +249,8 @@ def construct_channel_announcement_without_sigs(self) -> bytes: node_ids = sorted_node_ids bitcoin_keys.reverse() - chan_ann = encode_msg("channel_announcement", + chan_ann = encode_msg( + "channel_announcement", len=0, features=b'', chain_hash=constants.net.rev_genesis_bytes(), @@ -257,7 +258,7 @@ def construct_channel_announcement_without_sigs(self) -> bytes: node_id_1=node_ids[0], node_id_2=node_ids[1], bitcoin_key_1=bitcoin_keys[0], - bitcoin_key_2=bitcoin_keys[1] + bitcoin_key_2=bitcoin_keys[1], ) self._chan_ann_without_sigs = chan_ann diff --git a/electrum/lnmsg.py b/electrum/lnmsg.py index a755756b4122..2ef48ae97b98 100644 --- a/electrum/lnmsg.py +++ b/electrum/lnmsg.py @@ -1,152 +1,225 @@ -import json import os -from typing import Callable, Tuple -from collections import OrderedDict - -def _eval_length_term(x, ma: dict) -> int: - """ - Evaluate a term of the simple language used - to specify lightning message field lengths. - - If `x` is an integer, it is returned as is, - otherwise it is treated as a variable and - looked up in `ma`. - - If the value in `ma` was no integer, it is - assumed big-endian bytes and decoded. - - Returns evaluated result as int - """ - try: - x = int(x) - except ValueError: - x = ma[x] - try: - x = int(x) - except ValueError: - x = int.from_bytes(x, byteorder='big') - return x - -def _eval_exp_with_ctx(exp, ctx: dict) -> int: - """ - Evaluate simple mathematical expression given - in `exp` with context (variables assigned) - from the dict `ctx`. - - Returns evaluated result as int - """ - exp = str(exp) - if "*" in exp: - assert "+" not in exp - result = 1 - for term in exp.split("*"): - result *= _eval_length_term(term, ctx) - return result - return sum(_eval_length_term(x, ctx) for x in exp.split("+")) - -def _make_handler(msg_name: str, v: dict) -> Callable[[bytes], Tuple[str, dict]]: - """ - Generate a message handler function (taking bytes) - for message type `msg_name` with specification `v` - - Check lib/lightning.json, `msg_name` could be 'init', - and `v` could be - - { type: 16, payload: { 'gflen': ..., ... }, ... } - - Returns function taking bytes - """ - def handler(data: bytes) -> Tuple[str, dict]: - ma = {} # map of field name -> field data; after parsing msg - pos = 0 - for fieldname in v["payload"]: - poslenMap = v["payload"][fieldname] - if "feature" in poslenMap and pos == len(data): - continue - #assert pos == _eval_exp_with_ctx(poslenMap["position"], ma) # this assert is expensive... - length = poslenMap["length"] - length = _eval_exp_with_ctx(length, ma) - ma[fieldname] = data[pos:pos+length] - pos += length - # BOLT-01: "MUST ignore any additional data within a message beyond the length that it expects for that type." - assert pos <= len(data), (msg_name, pos, len(data)) - return msg_name, ma - return handler +import csv +import io +from typing import Callable, Tuple, Any, Dict, List, Sequence, Union + + +class MalformedMsg(Exception): + pass + + +class UnknownMsgFieldType(MalformedMsg): + pass + + +class UnexpectedEndOfStream(MalformedMsg): + pass + + +def _assert_can_read_at_least_n_bytes(fd: io.BytesIO, n: int) -> None: + cur_pos = fd.tell() + end_pos = fd.seek(0, io.SEEK_END) + fd.seek(cur_pos) + if end_pos - cur_pos < n: + raise UnexpectedEndOfStream(f"cur_pos={cur_pos}. end_pos={end_pos}. wants to read: {n}") + + +# TODO return int when it makes sense +def _read_field(*, fd: io.BytesIO, field_type: str, count: int) -> bytes: + if not fd: raise Exception() + assert isinstance(count, int) and count >= 0, f"{count!r} must be non-neg int" + if count == 0: + return b"" + type_len = None + if field_type == 'byte': + type_len = 1 + elif field_type == 'u16': + type_len = 2 + elif field_type == 'u32': + type_len = 4 + elif field_type == 'u64': + type_len = 8 + # TODO tu16/tu32/tu64 + elif field_type == 'chain_hash': + type_len = 32 + elif field_type == 'channel_id': + type_len = 32 + elif field_type == 'sha256': + type_len = 32 + elif field_type == 'signature': + type_len = 64 + elif field_type == 'point': + type_len = 33 + elif field_type == 'short_channel_id': + type_len = 8 + if type_len is None: + raise UnknownMsgFieldType(f"unexpected field type: {field_type!r}") + total_len = count * type_len + _assert_can_read_at_least_n_bytes(fd, total_len) + return fd.read(total_len) + + +def _write_field(*, fd: io.BytesIO, field_type: str, count: int, + value: Union[bytes, int]) -> None: + if not fd: raise Exception() + assert isinstance(count, int) and count >= 0, f"{count!r} must be non-neg int" + if count == 0: + return + type_len = None + if field_type == 'byte': + type_len = 1 + elif field_type == 'u16': + type_len = 2 + elif field_type == 'u32': + type_len = 4 + elif field_type == 'u64': + type_len = 8 + # TODO tu16/tu32/tu64 + elif field_type == 'chain_hash': + type_len = 32 + elif field_type == 'channel_id': + type_len = 32 + elif field_type == 'sha256': + type_len = 32 + elif field_type == 'signature': + type_len = 64 + elif field_type == 'point': + type_len = 33 + elif field_type == 'short_channel_id': + type_len = 8 + if type_len is None: + raise UnknownMsgFieldType(f"unexpected fundamental type: {field_type!r}") + total_len = count * type_len + if isinstance(value, int) and (count == 1 or field_type == 'byte'): + value = int.to_bytes(value, length=total_len, byteorder="big", signed=False) + if not isinstance(value, (bytes, bytearray)): + raise Exception(f"can only write bytes into fd. got: {value!r}") + if total_len != len(value): + raise Exception(f"unexpected field size. expected: {total_len}, got {len(value)}") + nbytes_written = fd.write(value) + if nbytes_written != len(value): + raise Exception(f"tried to write {len(value)} bytes, but only wrote {nbytes_written}!?") + class LNSerializer: def __init__(self): - message_types = {} - path = os.path.join(os.path.dirname(__file__), 'lightning.json') - with open(path) as f: - structured = json.loads(f.read(), object_pairs_hook=OrderedDict) - - for msg_name in structured: - v = structured[msg_name] - # these message types are skipped since their types collide - # (for example with pong, which also uses type=19) - # we don't need them yet - if msg_name in ["final_incorrect_cltv_expiry", "final_incorrect_htlc_amount"]: - continue - if len(v["payload"]) == 0: - continue - try: - num = int(v["type"]) - except ValueError: - #print("skipping", k) - continue - byts = num.to_bytes(2, 'big') - assert byts not in message_types, (byts, message_types[byts].__name__, msg_name) - names = [x.__name__ for x in message_types.values()] - assert msg_name + "_handler" not in names, (msg_name, names) - message_types[byts] = _make_handler(msg_name, v) - message_types[byts].__name__ = msg_name + "_handler" - - assert message_types[b"\x00\x10"].__name__ == "init_handler" - self.structured = structured - self.message_types = message_types - - def encode_msg(self, msg_type : str, **kwargs) -> bytes: + self.msg_scheme_from_type = {} # type: Dict[bytes, List[Sequence[str]]] + self.msg_type_from_name = {} # type: Dict[str, bytes] + path = os.path.join(os.path.dirname(__file__), "lnwire", "peer_wire.csv") + with open(path, newline='') as f: + csvreader = csv.reader(f) + for row in csvreader: + #print(f">>> {row!r}") + if row[0] == "msgtype": + msg_type_name = row[1] + msg_type_int = int(row[2]) + msg_type_bytes = msg_type_int.to_bytes(2, 'big') + assert msg_type_bytes not in self.msg_scheme_from_type, f"type collision? for {msg_type_name}" + assert msg_type_name not in self.msg_type_from_name, f"type collision? for {msg_type_name}" + row[2] = msg_type_int + self.msg_scheme_from_type[msg_type_bytes] = [tuple(row)] + self.msg_type_from_name[msg_type_name] = msg_type_bytes + elif row[0] == "msgdata": + assert msg_type_name == row[1] + self.msg_scheme_from_type[msg_type_bytes].append(tuple(row)) + else: + pass # TODO + + def encode_msg(self, msg_type: str, **kwargs) -> bytes: """ Encode kwargs into a Lightning message (bytes) of the type given in the msg_type string """ - typ = self.structured[msg_type] - data = int(typ["type"]).to_bytes(2, 'big') - lengths = {} - for k in typ["payload"]: - poslenMap = typ["payload"][k] - if k not in kwargs and "feature" in poslenMap: - continue - param = kwargs.get(k, 0) - leng = _eval_exp_with_ctx(poslenMap["length"], lengths) - try: - clone = dict(lengths) - clone.update(kwargs) - leng = _eval_exp_with_ctx(poslenMap["length"], clone) - except KeyError: - pass - try: - if not isinstance(param, bytes): - assert isinstance(param, int), "field {} is neither bytes or int".format(k) - param = param.to_bytes(leng, 'big') - except ValueError: - raise Exception("{} does not fit in {} bytes".format(k, leng)) - lengths[k] = len(param) - if lengths[k] != leng: - raise Exception("field {} is {} bytes long, should be {} bytes long".format(k, lengths[k], leng)) - data += param - return data - - def decode_msg(self, data : bytes) -> Tuple[str, dict]: + #print(f">>> encode_msg. msg_type={msg_type}, payload={kwargs!r}") + msg_type_bytes = self.msg_type_from_name[msg_type] + scheme = self.msg_scheme_from_type[msg_type_bytes] + with io.BytesIO() as fd: + fd.write(msg_type_bytes) + for row in scheme: + if row[0] == "msgtype": + pass + elif row[0] == "msgdata": + field_name = row[2] + field_type = row[3] + field_count_str = row[4] + #print(f">>> encode_msg. msgdata. field_name={field_name!r}. field_type={field_type!r}. field_count_str={field_count_str!r}") + if field_count_str == "": + field_count = 1 + else: + try: + field_count = int(field_count_str) + except ValueError: + field_count = kwargs[field_count_str] + if isinstance(field_count, (bytes, bytearray)): + field_count = int.from_bytes(field_count, byteorder="big") + assert isinstance(field_count, int) + try: + field_value = kwargs[field_name] + except KeyError: + if len(row) > 5: + break # optional feature field not present + else: + field_value = 0 # default mandatory fields to zero + #print(f">>> encode_msg. writing field: {field_name}. value={field_value!r}. field_type={field_type!r}. count={field_count!r}") + try: + _write_field(fd=fd, + field_type=field_type, + count=field_count, + value=field_value) + #print(f">>> encode_msg. so far: {fd.getvalue().hex()}") + except UnknownMsgFieldType as e: + pass # TODO + else: + pass # TODO + return fd.getvalue() + + def decode_msg(self, data: bytes) -> Tuple[str, dict]: """ Decode Lightning message by reading the first two bytes to determine message type. Returns message type string and parsed message contents dict """ - typ = data[:2] - k, parsed = self.message_types[typ](data[2:]) - return k, parsed + #print(f"decode_msg >>> {data.hex()}") + assert len(data) >= 2 + msg_type_bytes = data[:2] + msg_type_int = int.from_bytes(msg_type_bytes, byteorder="big", signed=False) + scheme = self.msg_scheme_from_type[msg_type_bytes] + assert scheme[0][2] == msg_type_int + msg_type_name = scheme[0][1] + parsed = {} + with io.BytesIO(data[2:]) as fd: + for row in scheme: + #print(f"row: {row!r}") + if row[0] == "msgtype": + pass + elif row[0] == "msgdata": + field_name = row[2] + field_type = row[3] + field_count_str = row[4] + if field_count_str == "": + field_count = 1 + else: + try: + field_count = int(field_count_str) + except ValueError: + field_count = int.from_bytes(parsed[field_count_str], byteorder="big") + #print(f">> count={field_count}. parsed={parsed}") + try: + parsed[field_name] = _read_field(fd=fd, + field_type=field_type, + count=field_count) + except UnknownMsgFieldType as e: + pass # TODO + except UnexpectedEndOfStream as e: + if len(row) > 5: + break # optional feature field not present + else: + raise + else: + pass # TODO + return msg_type_name, parsed + _inst = LNSerializer() encode_msg = _inst.encode_msg diff --git a/electrum/lnpeer.py b/electrum/lnpeer.py index 91853884d0f9..2b0aa87c1752 100644 --- a/electrum/lnpeer.py +++ b/electrum/lnpeer.py @@ -131,7 +131,7 @@ def is_initialized(self) -> bool: async def initialize(self): if isinstance(self.transport, LNTransport): await self.transport.handshake() - self.send_message("init", gflen=0, lflen=2, localfeatures=self.localfeatures) + self.send_message("init", gflen=0, flen=2, features=self.localfeatures) self._sent_init = True self.maybe_set_initialized() @@ -201,7 +201,7 @@ def on_init(self, payload): return # if they required some even flag we don't have, they will close themselves # but if we require an even flag they don't have, we close - their_localfeatures = int.from_bytes(payload['localfeatures'], byteorder="big") + their_localfeatures = int.from_bytes(payload['features'], byteorder="big") # TODO feature bit unification try: self.localfeatures = ln_compare_features(self.localfeatures, their_localfeatures) except IncompatibleLightningFeatures as e: @@ -760,16 +760,16 @@ async def reestablish_channel(self, chan: Channel): self.send_message( "channel_reestablish", channel_id=chan_id, - next_local_commitment_number=next_local_ctn, - next_remote_revocation_number=oldest_unrevoked_remote_ctn, + next_commitment_number=next_local_ctn, + next_revocation_number=oldest_unrevoked_remote_ctn, your_last_per_commitment_secret=last_rev_secret, my_current_per_commitment_point=latest_point) self.logger.info(f'channel_reestablish ({chan.get_id_for_log()}): sent channel_reestablish with ' f'(next_local_ctn={next_local_ctn}, ' f'oldest_unrevoked_remote_ctn={oldest_unrevoked_remote_ctn})') msg = await self.wait_for_message('channel_reestablish', chan_id) - their_next_local_ctn = int.from_bytes(msg["next_local_commitment_number"], 'big') - their_oldest_unrevoked_remote_ctn = int.from_bytes(msg["next_remote_revocation_number"], 'big') + their_next_local_ctn = int.from_bytes(msg["next_commitment_number"], 'big') + their_oldest_unrevoked_remote_ctn = int.from_bytes(msg["next_revocation_number"], 'big') their_local_pcp = msg.get("my_current_per_commitment_point") their_claim_of_our_last_per_commitment_secret = msg.get("your_last_per_commitment_secret") self.logger.info(f'channel_reestablish ({chan.get_id_for_log()}): received channel_reestablish with ' @@ -818,7 +818,7 @@ async def reestablish_channel(self, chan: Channel): if oldest_unrevoked_local_ctn != their_oldest_unrevoked_remote_ctn: if oldest_unrevoked_local_ctn - 1 == their_oldest_unrevoked_remote_ctn: # A node: - # if next_remote_revocation_number is equal to the commitment number of the last revoke_and_ack + # if next_revocation_number is equal to the commitment number of the last revoke_and_ack # the receiving node sent, AND the receiving node hasn't already received a closing_signed: # MUST re-send the revoke_and_ack. last_secret, last_point = chan.get_secret_and_point(LOCAL, oldest_unrevoked_local_ctn - 1) diff --git a/electrum/lnwire/README.md b/electrum/lnwire/README.md new file mode 100644 index 000000000000..72fd48f37d96 --- /dev/null +++ b/electrum/lnwire/README.md @@ -0,0 +1,5 @@ +These files are generated from the BOLT repository: +``` +$ python3 tools/extract-formats.py 01-*.md 02-*.md 07-*.md > peer_wire.csv +$ python3 tools/extract-formats.py 04-*.md > onion_wire.csv +``` diff --git a/electrum/lnwire/onion_wire.csv b/electrum/lnwire/onion_wire.csv new file mode 100644 index 000000000000..7d0258239d05 --- /dev/null +++ b/electrum/lnwire/onion_wire.csv @@ -0,0 +1,53 @@ +tlvtype,tlv_payload,amt_to_forward,2 +tlvdata,tlv_payload,amt_to_forward,amt_to_forward,tu64, +tlvtype,tlv_payload,outgoing_cltv_value,4 +tlvdata,tlv_payload,outgoing_cltv_value,outgoing_cltv_value,tu32, +tlvtype,tlv_payload,short_channel_id,6 +tlvdata,tlv_payload,short_channel_id,short_channel_id,short_channel_id, +tlvtype,tlv_payload,payment_data,8 +tlvdata,tlv_payload,payment_data,payment_secret,byte,32 +tlvdata,tlv_payload,payment_data,total_msat,tu64, +msgtype,invalid_realm,PERM|1 +msgtype,temporary_node_failure,NODE|2 +msgtype,permanent_node_failure,PERM|NODE|2 +msgtype,required_node_feature_missing,PERM|NODE|3 +msgtype,invalid_onion_version,BADONION|PERM|4 +msgdata,invalid_onion_version,sha256_of_onion,sha256, +msgtype,invalid_onion_hmac,BADONION|PERM|5 +msgdata,invalid_onion_hmac,sha256_of_onion,sha256, +msgtype,invalid_onion_key,BADONION|PERM|6 +msgdata,invalid_onion_key,sha256_of_onion,sha256, +msgtype,temporary_channel_failure,UPDATE|7 +msgdata,temporary_channel_failure,len,u16, +msgdata,temporary_channel_failure,channel_update,byte,len +msgtype,permanent_channel_failure,PERM|8 +msgtype,required_channel_feature_missing,PERM|9 +msgtype,unknown_next_peer,PERM|10 +msgtype,amount_below_minimum,UPDATE|11 +msgdata,amount_below_minimum,htlc_msat,u64, +msgdata,amount_below_minimum,len,u16, +msgdata,amount_below_minimum,channel_update,byte,len +msgtype,fee_insufficient,UPDATE|12 +msgdata,fee_insufficient,htlc_msat,u64, +msgdata,fee_insufficient,len,u16, +msgdata,fee_insufficient,channel_update,byte,len +msgtype,incorrect_cltv_expiry,UPDATE|13 +msgdata,incorrect_cltv_expiry,cltv_expiry,u32, +msgdata,incorrect_cltv_expiry,len,u16, +msgdata,incorrect_cltv_expiry,channel_update,byte,len +msgtype,expiry_too_soon,UPDATE|14 +msgdata,expiry_too_soon,len,u16, +msgdata,expiry_too_soon,channel_update,byte,len +msgtype,incorrect_or_unknown_payment_details,PERM|15 +msgdata,incorrect_or_unknown_payment_details,htlc_msat,u64, +msgdata,incorrect_or_unknown_payment_details,height,u32, +msgtype,final_incorrect_cltv_expiry,18 +msgdata,final_incorrect_cltv_expiry,cltv_expiry,u32, +msgtype,final_incorrect_htlc_amount,19 +msgdata,final_incorrect_htlc_amount,incoming_htlc_amt,u64, +msgtype,channel_disabled,UPDATE|20 +msgtype,expiry_too_far,21 +msgtype,invalid_onion_payload,PERM|22 +msgdata,invalid_onion_payload,type,varint, +msgdata,invalid_onion_payload,offset,u16, +msgtype,mpp_timeout,23 diff --git a/electrum/lnwire/peer_wire.csv b/electrum/lnwire/peer_wire.csv new file mode 100644 index 000000000000..a128e9c71804 --- /dev/null +++ b/electrum/lnwire/peer_wire.csv @@ -0,0 +1,210 @@ +msgtype,init,16 +msgdata,init,gflen,u16, +msgdata,init,globalfeatures,byte,gflen +msgdata,init,flen,u16, +msgdata,init,features,byte,flen +msgdata,init,tlvs,init_tlvs, +tlvtype,init_tlvs,networks,1 +tlvdata,init_tlvs,networks,chains,chain_hash,... +msgtype,error,17 +msgdata,error,channel_id,channel_id, +msgdata,error,len,u16, +msgdata,error,data,byte,len +msgtype,ping,18 +msgdata,ping,num_pong_bytes,u16, +msgdata,ping,byteslen,u16, +msgdata,ping,ignored,byte,byteslen +msgtype,pong,19 +msgdata,pong,byteslen,u16, +msgdata,pong,ignored,byte,byteslen +tlvtype,n1,tlv1,1 +tlvdata,n1,tlv1,amount_msat,tu64, +tlvtype,n1,tlv2,2 +tlvdata,n1,tlv2,scid,short_channel_id, +tlvtype,n1,tlv3,3 +tlvdata,n1,tlv3,node_id,point, +tlvdata,n1,tlv3,amount_msat_1,u64, +tlvdata,n1,tlv3,amount_msat_2,u64, +tlvtype,n1,tlv4,254 +tlvdata,n1,tlv4,cltv_delta,u16, +tlvtype,n2,tlv1,0 +tlvdata,n2,tlv1,amount_msat,tu64, +tlvtype,n2,tlv2,11 +tlvdata,n2,tlv2,cltv_expiry,tu32, +msgtype,open_channel,32 +msgdata,open_channel,chain_hash,chain_hash, +msgdata,open_channel,temporary_channel_id,byte,32 +msgdata,open_channel,funding_satoshis,u64, +msgdata,open_channel,push_msat,u64, +msgdata,open_channel,dust_limit_satoshis,u64, +msgdata,open_channel,max_htlc_value_in_flight_msat,u64, +msgdata,open_channel,channel_reserve_satoshis,u64, +msgdata,open_channel,htlc_minimum_msat,u64, +msgdata,open_channel,feerate_per_kw,u32, +msgdata,open_channel,to_self_delay,u16, +msgdata,open_channel,max_accepted_htlcs,u16, +msgdata,open_channel,funding_pubkey,point, +msgdata,open_channel,revocation_basepoint,point, +msgdata,open_channel,payment_basepoint,point, +msgdata,open_channel,delayed_payment_basepoint,point, +msgdata,open_channel,htlc_basepoint,point, +msgdata,open_channel,first_per_commitment_point,point, +msgdata,open_channel,channel_flags,byte, +msgdata,open_channel,shutdown_len,u16,,option_upfront_shutdown_script +msgdata,open_channel,shutdown_scriptpubkey,byte,shutdown_len,option_upfront_shutdown_script +msgtype,accept_channel,33 +msgdata,accept_channel,temporary_channel_id,byte,32 +msgdata,accept_channel,dust_limit_satoshis,u64, +msgdata,accept_channel,max_htlc_value_in_flight_msat,u64, +msgdata,accept_channel,channel_reserve_satoshis,u64, +msgdata,accept_channel,htlc_minimum_msat,u64, +msgdata,accept_channel,minimum_depth,u32, +msgdata,accept_channel,to_self_delay,u16, +msgdata,accept_channel,max_accepted_htlcs,u16, +msgdata,accept_channel,funding_pubkey,point, +msgdata,accept_channel,revocation_basepoint,point, +msgdata,accept_channel,payment_basepoint,point, +msgdata,accept_channel,delayed_payment_basepoint,point, +msgdata,accept_channel,htlc_basepoint,point, +msgdata,accept_channel,first_per_commitment_point,point, +msgdata,accept_channel,shutdown_len,u16,,option_upfront_shutdown_script +msgdata,accept_channel,shutdown_scriptpubkey,byte,shutdown_len,option_upfront_shutdown_script +msgtype,funding_created,34 +msgdata,funding_created,temporary_channel_id,byte,32 +msgdata,funding_created,funding_txid,sha256, +msgdata,funding_created,funding_output_index,u16, +msgdata,funding_created,signature,signature, +msgtype,funding_signed,35 +msgdata,funding_signed,channel_id,channel_id, +msgdata,funding_signed,signature,signature, +msgtype,funding_locked,36 +msgdata,funding_locked,channel_id,channel_id, +msgdata,funding_locked,next_per_commitment_point,point, +msgtype,shutdown,38 +msgdata,shutdown,channel_id,channel_id, +msgdata,shutdown,len,u16, +msgdata,shutdown,scriptpubkey,byte,len +msgtype,closing_signed,39 +msgdata,closing_signed,channel_id,channel_id, +msgdata,closing_signed,fee_satoshis,u64, +msgdata,closing_signed,signature,signature, +msgtype,update_add_htlc,128 +msgdata,update_add_htlc,channel_id,channel_id, +msgdata,update_add_htlc,id,u64, +msgdata,update_add_htlc,amount_msat,u64, +msgdata,update_add_htlc,payment_hash,sha256, +msgdata,update_add_htlc,cltv_expiry,u32, +msgdata,update_add_htlc,onion_routing_packet,byte,1366 +msgtype,update_fulfill_htlc,130 +msgdata,update_fulfill_htlc,channel_id,channel_id, +msgdata,update_fulfill_htlc,id,u64, +msgdata,update_fulfill_htlc,payment_preimage,byte,32 +msgtype,update_fail_htlc,131 +msgdata,update_fail_htlc,channel_id,channel_id, +msgdata,update_fail_htlc,id,u64, +msgdata,update_fail_htlc,len,u16, +msgdata,update_fail_htlc,reason,byte,len +msgtype,update_fail_malformed_htlc,135 +msgdata,update_fail_malformed_htlc,channel_id,channel_id, +msgdata,update_fail_malformed_htlc,id,u64, +msgdata,update_fail_malformed_htlc,sha256_of_onion,sha256, +msgdata,update_fail_malformed_htlc,failure_code,u16, +msgtype,commitment_signed,132 +msgdata,commitment_signed,channel_id,channel_id, +msgdata,commitment_signed,signature,signature, +msgdata,commitment_signed,num_htlcs,u16, +msgdata,commitment_signed,htlc_signature,signature,num_htlcs +msgtype,revoke_and_ack,133 +msgdata,revoke_and_ack,channel_id,channel_id, +msgdata,revoke_and_ack,per_commitment_secret,byte,32 +msgdata,revoke_and_ack,next_per_commitment_point,point, +msgtype,update_fee,134 +msgdata,update_fee,channel_id,channel_id, +msgdata,update_fee,feerate_per_kw,u32, +msgtype,channel_reestablish,136 +msgdata,channel_reestablish,channel_id,channel_id, +msgdata,channel_reestablish,next_commitment_number,u64, +msgdata,channel_reestablish,next_revocation_number,u64, +msgdata,channel_reestablish,your_last_per_commitment_secret,byte,32,option_data_loss_protect,option_static_remotekey +msgdata,channel_reestablish,my_current_per_commitment_point,point,,option_data_loss_protect,option_static_remotekey +msgtype,announcement_signatures,259 +msgdata,announcement_signatures,channel_id,channel_id, +msgdata,announcement_signatures,short_channel_id,short_channel_id, +msgdata,announcement_signatures,node_signature,signature, +msgdata,announcement_signatures,bitcoin_signature,signature, +msgtype,channel_announcement,256 +msgdata,channel_announcement,node_signature_1,signature, +msgdata,channel_announcement,node_signature_2,signature, +msgdata,channel_announcement,bitcoin_signature_1,signature, +msgdata,channel_announcement,bitcoin_signature_2,signature, +msgdata,channel_announcement,len,u16, +msgdata,channel_announcement,features,byte,len +msgdata,channel_announcement,chain_hash,chain_hash, +msgdata,channel_announcement,short_channel_id,short_channel_id, +msgdata,channel_announcement,node_id_1,point, +msgdata,channel_announcement,node_id_2,point, +msgdata,channel_announcement,bitcoin_key_1,point, +msgdata,channel_announcement,bitcoin_key_2,point, +msgtype,node_announcement,257 +msgdata,node_announcement,signature,signature, +msgdata,node_announcement,flen,u16, +msgdata,node_announcement,features,byte,flen +msgdata,node_announcement,timestamp,u32, +msgdata,node_announcement,node_id,point, +msgdata,node_announcement,rgb_color,byte,3 +msgdata,node_announcement,alias,byte,32 +msgdata,node_announcement,addrlen,u16, +msgdata,node_announcement,addresses,byte,addrlen +msgtype,channel_update,258 +msgdata,channel_update,signature,signature, +msgdata,channel_update,chain_hash,chain_hash, +msgdata,channel_update,short_channel_id,short_channel_id, +msgdata,channel_update,timestamp,u32, +msgdata,channel_update,message_flags,byte, +msgdata,channel_update,channel_flags,byte, +msgdata,channel_update,cltv_expiry_delta,u16, +msgdata,channel_update,htlc_minimum_msat,u64, +msgdata,channel_update,fee_base_msat,u32, +msgdata,channel_update,fee_proportional_millionths,u32, +msgdata,channel_update,htlc_maximum_msat,u64,,option_channel_htlc_max +msgtype,query_short_channel_ids,261,gossip_queries +msgdata,query_short_channel_ids,chain_hash,chain_hash, +msgdata,query_short_channel_ids,len,u16, +msgdata,query_short_channel_ids,encoded_short_ids,byte,len +msgdata,query_short_channel_ids,tlvs,query_short_channel_ids_tlvs, +tlvtype,query_short_channel_ids_tlvs,query_flags,1 +tlvdata,query_short_channel_ids_tlvs,query_flags,encoding_type,u8, +tlvdata,query_short_channel_ids_tlvs,query_flags,encoded_query_flags,byte,... +msgtype,reply_short_channel_ids_end,262,gossip_queries +msgdata,reply_short_channel_ids_end,chain_hash,chain_hash, +msgdata,reply_short_channel_ids_end,complete,byte, +msgtype,query_channel_range,263,gossip_queries +msgdata,query_channel_range,chain_hash,chain_hash, +msgdata,query_channel_range,first_blocknum,u32, +msgdata,query_channel_range,number_of_blocks,u32, +msgdata,query_channel_range,tlvs,query_channel_range_tlvs, +tlvtype,query_channel_range_tlvs,query_option,1 +tlvdata,query_channel_range_tlvs,query_option,query_option_flags,varint, +msgtype,reply_channel_range,264,gossip_queries +msgdata,reply_channel_range,chain_hash,chain_hash, +msgdata,reply_channel_range,first_blocknum,u32, +msgdata,reply_channel_range,number_of_blocks,u32, +msgdata,reply_channel_range,complete,byte, +msgdata,reply_channel_range,len,u16, +msgdata,reply_channel_range,encoded_short_ids,byte,len +msgdata,reply_channel_range,tlvs,reply_channel_range_tlvs, +tlvtype,reply_channel_range_tlvs,timestamps_tlv,1 +tlvdata,reply_channel_range_tlvs,timestamps_tlv,encoding_type,u8, +tlvdata,reply_channel_range_tlvs,timestamps_tlv,encoded_timestamps,byte,... +tlvtype,reply_channel_range_tlvs,checksums_tlv,3 +tlvdata,reply_channel_range_tlvs,checksums_tlv,checksums,channel_update_checksums,... +subtype,channel_update_timestamps +subtypedata,channel_update_timestamps,timestamp_node_id_1,u32, +subtypedata,channel_update_timestamps,timestamp_node_id_2,u32, +subtype,channel_update_checksums +subtypedata,channel_update_checksums,checksum_node_id_1,u32, +subtypedata,channel_update_checksums,checksum_node_id_2,u32, +msgtype,gossip_timestamp_filter,265,gossip_queries +msgdata,gossip_timestamp_filter,chain_hash,chain_hash, +msgdata,gossip_timestamp_filter,first_timestamp,u32, +msgdata,gossip_timestamp_filter,timestamp_range,u32, From 3a73f6ee5cc3cca995ab1164c8083843a0bf0046 Mon Sep 17 00:00:00 2001 From: SomberNight Date: Thu, 12 Mar 2020 04:08:13 +0100 Subject: [PATCH 02/17] lnmsg.decode_msg: dict values for numbers are int, instead of BE bytes Will be useful for TLVs where it makes sense to do the conversion in lnmsg, as it might be more complicated than just int.from_bytes(). --- electrum/channel_db.py | 20 ++++++------ electrum/lnchannel.py | 12 +++---- electrum/lnmsg.py | 17 +++++++--- electrum/lnpeer.py | 57 ++++++++++++++++----------------- electrum/tests/test_lnrouter.py | 37 +++++++++++---------- 5 files changed, 75 insertions(+), 68 deletions(-) diff --git a/electrum/channel_db.py b/electrum/channel_db.py index 54fc45460cb5..aef25effedc0 100644 --- a/electrum/channel_db.py +++ b/electrum/channel_db.py @@ -102,14 +102,14 @@ class Policy(NamedTuple): def from_msg(payload: dict) -> 'Policy': return Policy( key = payload['short_channel_id'] + payload['start_node'], - cltv_expiry_delta = int.from_bytes(payload['cltv_expiry_delta'], "big"), - htlc_minimum_msat = int.from_bytes(payload['htlc_minimum_msat'], "big"), - htlc_maximum_msat = int.from_bytes(payload['htlc_maximum_msat'], "big") if 'htlc_maximum_msat' in payload else None, - fee_base_msat = int.from_bytes(payload['fee_base_msat'], "big"), - fee_proportional_millionths = int.from_bytes(payload['fee_proportional_millionths'], "big"), + cltv_expiry_delta = payload['cltv_expiry_delta'], + htlc_minimum_msat = payload['htlc_minimum_msat'], + htlc_maximum_msat = payload.get('htlc_maximum_msat', None), + fee_base_msat = payload['fee_base_msat'], + fee_proportional_millionths = payload['fee_proportional_millionths'], message_flags = int.from_bytes(payload['message_flags'], "big"), channel_flags = int.from_bytes(payload['channel_flags'], "big"), - timestamp = int.from_bytes(payload['timestamp'], "big") + timestamp = payload['timestamp'], ) @staticmethod @@ -154,7 +154,7 @@ def from_msg(payload) -> Tuple['NodeInfo', Sequence['LNPeerAddr']]: alias = alias.decode('utf8') except: alias = '' - timestamp = int.from_bytes(payload['timestamp'], "big") + timestamp = payload['timestamp'] node_info = NodeInfo(node_id=node_id, features=features, timestamp=timestamp, alias=alias) return node_info, peer_addrs @@ -393,7 +393,7 @@ def add_channel_updates(self, payloads, max_age=None, verify=True) -> Categorize now = int(time.time()) for payload in payloads: short_channel_id = ShortChannelID(payload['short_channel_id']) - timestamp = int.from_bytes(payload['timestamp'], "big") + timestamp = payload['timestamp'] if max_age and now - timestamp > max_age: expired.append(payload) continue @@ -408,7 +408,7 @@ def add_channel_updates(self, payloads, max_age=None, verify=True) -> Categorize known.append(payload) # compare updates to existing database entries for payload in known: - timestamp = int.from_bytes(payload['timestamp'], "big") + timestamp = payload['timestamp'] start_node = payload['start_node'] short_channel_id = ShortChannelID(payload['short_channel_id']) key = (start_node, short_channel_id) @@ -673,7 +673,7 @@ def get_policy_for_node(self, short_channel_id: bytes, node_id: bytes, *, return now = int(time.time()) remote_update_decoded = decode_msg(remote_update_raw)[1] - remote_update_decoded['timestamp'] = now.to_bytes(4, byteorder="big") + remote_update_decoded['timestamp'] = now remote_update_decoded['start_node'] = node_id return Policy.from_msg(remote_update_decoded) elif node_id == chan.get_local_pubkey(): # outgoing direction (from us) diff --git a/electrum/lnchannel.py b/electrum/lnchannel.py index bd52175b72ad..4fecf6a03fc8 100644 --- a/electrum/lnchannel.py +++ b/electrum/lnchannel.py @@ -218,13 +218,13 @@ def get_outgoing_gossip_channel_update(self) -> bytes: short_channel_id=self.short_channel_id, channel_flags=channel_flags, message_flags=b'\x01', - cltv_expiry_delta=lnutil.NBLOCK_OUR_CLTV_EXPIRY_DELTA.to_bytes(2, byteorder="big"), - htlc_minimum_msat=self.config[REMOTE].htlc_minimum_msat.to_bytes(8, byteorder="big"), - htlc_maximum_msat=htlc_maximum_msat.to_bytes(8, byteorder="big"), - fee_base_msat=lnutil.OUR_FEE_BASE_MSAT.to_bytes(4, byteorder="big"), - fee_proportional_millionths=lnutil.OUR_FEE_PROPORTIONAL_MILLIONTHS.to_bytes(4, byteorder="big"), + cltv_expiry_delta=lnutil.NBLOCK_OUR_CLTV_EXPIRY_DELTA, + htlc_minimum_msat=self.config[REMOTE].htlc_minimum_msat, + htlc_maximum_msat=htlc_maximum_msat, + fee_base_msat=lnutil.OUR_FEE_BASE_MSAT, + fee_proportional_millionths=lnutil.OUR_FEE_PROPORTIONAL_MILLIONTHS, chain_hash=constants.net.rev_genesis_bytes(), - timestamp=now.to_bytes(4, byteorder="big"), + timestamp=now, ) sighash = sha256d(chan_upd[2 + 64:]) sig = ecc.ECPrivkey(self.lnworker.node_keypair.privkey).sign(sighash, ecc.sig_string_from_r_and_s) diff --git a/electrum/lnmsg.py b/electrum/lnmsg.py index 2ef48ae97b98..4e362efaec81 100644 --- a/electrum/lnmsg.py +++ b/electrum/lnmsg.py @@ -1,7 +1,7 @@ import os import csv import io -from typing import Callable, Tuple, Any, Dict, List, Sequence, Union +from typing import Callable, Tuple, Any, Dict, List, Sequence, Union, Optional class MalformedMsg(Exception): @@ -24,8 +24,7 @@ def _assert_can_read_at_least_n_bytes(fd: io.BytesIO, n: int) -> None: raise UnexpectedEndOfStream(f"cur_pos={cur_pos}. end_pos={end_pos}. wants to read: {n}") -# TODO return int when it makes sense -def _read_field(*, fd: io.BytesIO, field_type: str, count: int) -> bytes: +def _read_field(*, fd: io.BytesIO, field_type: str, count: int) -> Union[bytes, int]: if not fd: raise Exception() assert isinstance(count, int) and count >= 0, f"{count!r} must be non-neg int" if count == 0: @@ -35,10 +34,19 @@ def _read_field(*, fd: io.BytesIO, field_type: str, count: int) -> bytes: type_len = 1 elif field_type == 'u16': type_len = 2 + assert count == 1, count + _assert_can_read_at_least_n_bytes(fd, type_len) + return int.from_bytes(fd.read(type_len), byteorder="big", signed=False) elif field_type == 'u32': type_len = 4 + assert count == 1, count + _assert_can_read_at_least_n_bytes(fd, type_len) + return int.from_bytes(fd.read(type_len), byteorder="big", signed=False) elif field_type == 'u64': type_len = 8 + assert count == 1, count + _assert_can_read_at_least_n_bytes(fd, type_len) + return int.from_bytes(fd.read(type_len), byteorder="big", signed=False) # TODO tu16/tu32/tu64 elif field_type == 'chain_hash': type_len = 32 @@ -203,7 +211,8 @@ def decode_msg(self, data: bytes) -> Tuple[str, dict]: try: field_count = int(field_count_str) except ValueError: - field_count = int.from_bytes(parsed[field_count_str], byteorder="big") + field_count = parsed[field_count_str] + assert isinstance(field_count, int) #print(f">> count={field_count}. parsed={parsed}") try: parsed[field_name] = _read_field(fd=fd, diff --git a/electrum/lnpeer.py b/electrum/lnpeer.py index 2b0aa87c1752..684cbe8596ba 100644 --- a/electrum/lnpeer.py +++ b/electrum/lnpeer.py @@ -180,7 +180,7 @@ def on_error(self, payload): self.ordered_message_queues[chan_id].put_nowait((None, {'error':payload['data']})) def on_ping(self, payload): - l = int.from_bytes(payload['num_pong_bytes'], 'big') + l = payload['num_pong_bytes'] self.send_message('pong', byteslen=l) def on_pong(self, payload): @@ -417,8 +417,8 @@ def decode_short_ids(self, encoded): return ids def on_reply_channel_range(self, payload): - first = int.from_bytes(payload['first_blocknum'], 'big') - num = int.from_bytes(payload['number_of_blocks'], 'big') + first = payload['first_blocknum'] + num = payload['number_of_blocks'] complete = bool(int.from_bytes(payload['complete'], 'big')) encoded = payload['encoded_short_ids'] ids = self.decode_short_ids(encoded) @@ -541,27 +541,27 @@ async def channel_establishment_flow(self, password: Optional[str], funding_tx: ) payload = await self.wait_for_message('accept_channel', temp_channel_id) remote_per_commitment_point = payload['first_per_commitment_point'] - funding_txn_minimum_depth = int.from_bytes(payload['minimum_depth'], 'big') + funding_txn_minimum_depth = payload['minimum_depth'] if funding_txn_minimum_depth <= 0: raise Exception(f"minimum depth too low, {funding_txn_minimum_depth}") if funding_txn_minimum_depth > 30: raise Exception(f"minimum depth too high, {funding_txn_minimum_depth}") - remote_dust_limit_sat = int.from_bytes(payload['dust_limit_satoshis'], byteorder='big') + remote_dust_limit_sat = payload['dust_limit_satoshis'] remote_reserve_sat = self.validate_remote_reserve(payload["channel_reserve_satoshis"], remote_dust_limit_sat, funding_sat) if remote_dust_limit_sat > remote_reserve_sat: raise Exception(f"Remote Lightning peer reports dust_limit_sat > reserve_sat which is a BOLT-02 protocol violation.") - htlc_min = int.from_bytes(payload['htlc_minimum_msat'], 'big') + htlc_min = payload['htlc_minimum_msat'] if htlc_min > MAXIMUM_HTLC_MINIMUM_MSAT_ACCEPTED: raise Exception(f"Remote Lightning peer reports htlc_minimum_msat={htlc_min} mSAT," + f" which is above Electrums required maximum limit of that parameter ({MAXIMUM_HTLC_MINIMUM_MSAT_ACCEPTED} mSAT).") - remote_max = int.from_bytes(payload['max_htlc_value_in_flight_msat'], 'big') + remote_max = payload['max_htlc_value_in_flight_msat'] if remote_max < MINIMUM_MAX_HTLC_VALUE_IN_FLIGHT_ACCEPTED: raise Exception(f"Remote Lightning peer reports max_htlc_value_in_flight_msat at only {remote_max} mSAT" + f" which is below Electrums required minimum ({MINIMUM_MAX_HTLC_VALUE_IN_FLIGHT_ACCEPTED} mSAT).") - max_accepted_htlcs = int.from_bytes(payload["max_accepted_htlcs"], 'big') + max_accepted_htlcs = payload["max_accepted_htlcs"] if max_accepted_htlcs > 483: raise Exception("Remote Lightning peer reports max_accepted_htlcs > 483, which is a BOLT-02 protocol violation.") - remote_to_self_delay = int.from_bytes(payload['to_self_delay'], byteorder='big') + remote_to_self_delay = payload['to_self_delay'] if remote_to_self_delay > MAXIMUM_REMOTE_TO_SELF_DELAY_ACCEPTED: raise Exception(f"Remote Lightning peer reports to_self_delay={remote_to_self_delay}," + f" which is above Electrums required maximum ({MAXIMUM_REMOTE_TO_SELF_DELAY_ACCEPTED})") @@ -647,9 +647,9 @@ async def on_open_channel(self, payload): # payload['channel_flags'] if payload['chain_hash'] != constants.net.rev_genesis_bytes(): raise Exception('wrong chain_hash') - funding_sat = int.from_bytes(payload['funding_satoshis'], 'big') - push_msat = int.from_bytes(payload['push_msat'], 'big') - feerate = int.from_bytes(payload['feerate_per_kw'], 'big') + funding_sat = payload['funding_satoshis'] + push_msat = payload['push_msat'] + feerate = payload['feerate_per_kw'] temp_chan_id = payload['temporary_channel_id'] local_config = self.make_local_config(funding_sat, push_msat, REMOTE) # for the first commitment transaction @@ -674,11 +674,11 @@ async def on_open_channel(self, payload): first_per_commitment_point=per_commitment_point_first, ) funding_created = await self.wait_for_message('funding_created', temp_chan_id) - funding_idx = int.from_bytes(funding_created['funding_output_index'], 'big') + funding_idx = funding_created['funding_output_index'] funding_txid = bh2u(funding_created['funding_txid'][::-1]) channel_id, funding_txid_bytes = channel_id_from_funding_tx(funding_txid, funding_idx) remote_balance_sat = funding_sat * 1000 - push_msat - remote_dust_limit_sat = int.from_bytes(payload['dust_limit_satoshis'], byteorder='big') # TODO validate + remote_dust_limit_sat = payload['dust_limit_satoshis'] # TODO validate remote_reserve_sat = self.validate_remote_reserve(payload['channel_reserve_satoshis'], remote_dust_limit_sat, funding_sat) remote_config = RemoteConfig( payment_basepoint=OnlyPubkeyKeypair(payload['payment_basepoint']), @@ -686,13 +686,13 @@ async def on_open_channel(self, payload): htlc_basepoint=OnlyPubkeyKeypair(payload['htlc_basepoint']), delayed_basepoint=OnlyPubkeyKeypair(payload['delayed_payment_basepoint']), revocation_basepoint=OnlyPubkeyKeypair(payload['revocation_basepoint']), - to_self_delay=int.from_bytes(payload['to_self_delay'], 'big'), + to_self_delay=payload['to_self_delay'], dust_limit_sat=remote_dust_limit_sat, - max_htlc_value_in_flight_msat=int.from_bytes(payload['max_htlc_value_in_flight_msat'], 'big'), # TODO validate - max_accepted_htlcs=int.from_bytes(payload['max_accepted_htlcs'], 'big'), # TODO validate + max_htlc_value_in_flight_msat=payload['max_htlc_value_in_flight_msat'], # TODO validate + max_accepted_htlcs=payload['max_accepted_htlcs'], # TODO validate initial_msat=remote_balance_sat, reserve_sat = remote_reserve_sat, - htlc_minimum_msat=int.from_bytes(payload['htlc_minimum_msat'], 'big'), # TODO validate + htlc_minimum_msat=payload['htlc_minimum_msat'], # TODO validate next_per_commitment_point=payload['first_per_commitment_point'], current_per_commitment_point=None, ) @@ -718,8 +718,7 @@ async def on_open_channel(self, payload): chan.set_state(channel_states.OPENING) self.lnworker.add_new_channel(chan) - def validate_remote_reserve(self, payload_field: bytes, dust_limit: int, funding_sat: int) -> int: - remote_reserve_sat = int.from_bytes(payload_field, 'big') + def validate_remote_reserve(self, remote_reserve_sat: int, dust_limit: int, funding_sat: int) -> int: if remote_reserve_sat < dust_limit: raise Exception('protocol violation: reserve < dust_limit') if remote_reserve_sat > funding_sat/100: @@ -768,8 +767,8 @@ async def reestablish_channel(self, chan: Channel): f'(next_local_ctn={next_local_ctn}, ' f'oldest_unrevoked_remote_ctn={oldest_unrevoked_remote_ctn})') msg = await self.wait_for_message('channel_reestablish', chan_id) - their_next_local_ctn = int.from_bytes(msg["next_commitment_number"], 'big') - their_oldest_unrevoked_remote_ctn = int.from_bytes(msg["next_revocation_number"], 'big') + their_next_local_ctn = msg["next_commitment_number"] + their_oldest_unrevoked_remote_ctn = msg["next_revocation_number"] their_local_pcp = msg.get("my_current_per_commitment_point") their_claim_of_our_last_per_commitment_secret = msg.get("your_last_per_commitment_secret") self.logger.info(f'channel_reestablish ({chan.get_id_for_log()}): received channel_reestablish with ' @@ -1005,7 +1004,7 @@ def send_announcement_signatures(self, chan: Channel): return msg_hash, node_signature, bitcoin_signature def on_update_fail_htlc(self, chan: Channel, payload): - htlc_id = int.from_bytes(payload["id"], "big") + htlc_id = payload["id"] reason = payload["reason"] self.logger.info(f"on_update_fail_htlc. chan {chan.short_channel_id}. htlc_id {htlc_id}") chan.receive_fail_htlc(htlc_id, error_bytes=reason) # TODO handle exc and maybe fail channel (e.g. bad htlc_id) @@ -1083,7 +1082,7 @@ def on_commitment_signed(self, chan: Channel, payload): def on_update_fulfill_htlc(self, chan: Channel, payload): preimage = payload["payment_preimage"] payment_hash = sha256(preimage) - htlc_id = int.from_bytes(payload["id"], "big") + htlc_id = payload["id"] self.logger.info(f"on_update_fulfill_htlc. chan {chan.short_channel_id}. htlc_id {htlc_id}") chan.receive_htlc_settle(preimage, htlc_id) # TODO handle exc and maybe fail channel (e.g. bad htlc_id) self.lnworker.save_preimage(payment_hash, preimage) @@ -1103,10 +1102,10 @@ def on_update_fail_malformed_htlc(self, chan: Channel, payload): def on_update_add_htlc(self, chan: Channel, payload): payment_hash = payload["payment_hash"] - htlc_id = int.from_bytes(payload["id"], 'big') + htlc_id = payload["id"] self.logger.info(f"on_update_add_htlc. chan {chan.short_channel_id}. htlc_id {htlc_id}") - cltv_expiry = int.from_bytes(payload["cltv_expiry"], 'big') - amount_msat_htlc = int.from_bytes(payload["amount_msat"], 'big') + cltv_expiry = payload["cltv_expiry"] + amount_msat_htlc = payload["amount_msat"] onion_packet = payload["onion_routing_packet"] if chan.get_state() != channel_states.OPEN: raise RemoteMisbehaving(f"received update_add_htlc while chan.get_state() != OPEN. state was {chan.get_state()}") @@ -1258,7 +1257,7 @@ def on_revoke_and_ack(self, chan: Channel, payload): self.maybe_send_commitment(chan) def on_update_fee(self, chan: Channel, payload): - feerate = int.from_bytes(payload["feerate_per_kw"], "big") + feerate = payload["feerate_per_kw"] chan.update_fee(feerate, False) async def maybe_update_fee(self, chan: Channel): @@ -1378,7 +1377,7 @@ def verify_signature(tx, sig): while True: # FIXME: the remote SHOULD send closing_signed, but some don't. cs_payload = await self.wait_for_message('closing_signed', chan.channel_id) - their_fee = int.from_bytes(cs_payload['fee_satoshis'], 'big') + their_fee = cs_payload['fee_satoshis'] if their_fee > max_fee: raise Exception(f'the proposed fee exceeds the base fee of the latest commitment transaction {is_local, their_fee, max_fee}') their_sig = cs_payload['signature'] diff --git a/electrum/tests/test_lnrouter.py b/electrum/tests/test_lnrouter.py index db93b925a186..59b6996fcd23 100644 --- a/electrum/tests/test_lnrouter.py +++ b/electrum/tests/test_lnrouter.py @@ -57,46 +57,45 @@ class fake_network: 'bitcoin_key_1': b'\x02bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb', 'bitcoin_key_2': b'\x02cccccccccccccccccccccccccccccccc', 'short_channel_id': bfh('0000000000000001'), 'chain_hash': BitcoinTestnet.rev_genesis_bytes(), - 'len': b'\x00\x00', 'features': b''}, trusted=True) + 'len': 0, 'features': b''}, trusted=True) self.assertEqual(cdb.num_channels, 1) cdb.add_channel_announcement({'node_id_1': b'\x02bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb', 'node_id_2': b'\x02eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee', 'bitcoin_key_1': b'\x02bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb', 'bitcoin_key_2': b'\x02eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee', 'short_channel_id': bfh('0000000000000002'), 'chain_hash': BitcoinTestnet.rev_genesis_bytes(), - 'len': b'\x00\x00', 'features': b''}, trusted=True) + 'len': 0, 'features': b''}, trusted=True) cdb.add_channel_announcement({'node_id_1': b'\x02aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa', 'node_id_2': b'\x02bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb', 'bitcoin_key_1': b'\x02aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa', 'bitcoin_key_2': b'\x02bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb', 'short_channel_id': bfh('0000000000000003'), 'chain_hash': BitcoinTestnet.rev_genesis_bytes(), - 'len': b'\x00\x00', 'features': b''}, trusted=True) + 'len': 0, 'features': b''}, trusted=True) cdb.add_channel_announcement({'node_id_1': b'\x02cccccccccccccccccccccccccccccccc', 'node_id_2': b'\x02dddddddddddddddddddddddddddddddd', 'bitcoin_key_1': b'\x02cccccccccccccccccccccccccccccccc', 'bitcoin_key_2': b'\x02dddddddddddddddddddddddddddddddd', 'short_channel_id': bfh('0000000000000004'), 'chain_hash': BitcoinTestnet.rev_genesis_bytes(), - 'len': b'\x00\x00', 'features': b''}, trusted=True) + 'len': 0, 'features': b''}, trusted=True) cdb.add_channel_announcement({'node_id_1': b'\x02dddddddddddddddddddddddddddddddd', 'node_id_2': b'\x02eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee', 'bitcoin_key_1': b'\x02dddddddddddddddddddddddddddddddd', 'bitcoin_key_2': b'\x02eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee', 'short_channel_id': bfh('0000000000000005'), 'chain_hash': BitcoinTestnet.rev_genesis_bytes(), - 'len': b'\x00\x00', 'features': b''}, trusted=True) + 'len': 0, 'features': b''}, trusted=True) cdb.add_channel_announcement({'node_id_1': b'\x02aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa', 'node_id_2': b'\x02dddddddddddddddddddddddddddddddd', 'bitcoin_key_1': b'\x02aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa', 'bitcoin_key_2': b'\x02dddddddddddddddddddddddddddddddd', 'short_channel_id': bfh('0000000000000006'), 'chain_hash': BitcoinTestnet.rev_genesis_bytes(), - 'len': b'\x00\x00', 'features': b''}, trusted=True) - o = lambda i: i.to_bytes(8, "big") - cdb.add_channel_update({'short_channel_id': bfh('0000000000000001'), 'message_flags': b'\x00', 'channel_flags': b'\x00', 'cltv_expiry_delta': o(10), 'htlc_minimum_msat': o(250), 'fee_base_msat': o(100), 'fee_proportional_millionths': o(150), 'chain_hash': BitcoinTestnet.rev_genesis_bytes(), 'timestamp': b'\x00\x00\x00\x00'}) - cdb.add_channel_update({'short_channel_id': bfh('0000000000000001'), 'message_flags': b'\x00', 'channel_flags': b'\x01', 'cltv_expiry_delta': o(10), 'htlc_minimum_msat': o(250), 'fee_base_msat': o(100), 'fee_proportional_millionths': o(150), 'chain_hash': BitcoinTestnet.rev_genesis_bytes(), 'timestamp': b'\x00\x00\x00\x00'}) - cdb.add_channel_update({'short_channel_id': bfh('0000000000000002'), 'message_flags': b'\x00', 'channel_flags': b'\x00', 'cltv_expiry_delta': o(99), 'htlc_minimum_msat': o(250), 'fee_base_msat': o(100), 'fee_proportional_millionths': o(150), 'chain_hash': BitcoinTestnet.rev_genesis_bytes(), 'timestamp': b'\x00\x00\x00\x00'}) - cdb.add_channel_update({'short_channel_id': bfh('0000000000000002'), 'message_flags': b'\x00', 'channel_flags': b'\x01', 'cltv_expiry_delta': o(10), 'htlc_minimum_msat': o(250), 'fee_base_msat': o(100), 'fee_proportional_millionths': o(150), 'chain_hash': BitcoinTestnet.rev_genesis_bytes(), 'timestamp': b'\x00\x00\x00\x00'}) - cdb.add_channel_update({'short_channel_id': bfh('0000000000000003'), 'message_flags': b'\x00', 'channel_flags': b'\x01', 'cltv_expiry_delta': o(10), 'htlc_minimum_msat': o(250), 'fee_base_msat': o(100), 'fee_proportional_millionths': o(150), 'chain_hash': BitcoinTestnet.rev_genesis_bytes(), 'timestamp': b'\x00\x00\x00\x00'}) - cdb.add_channel_update({'short_channel_id': bfh('0000000000000003'), 'message_flags': b'\x00', 'channel_flags': b'\x00', 'cltv_expiry_delta': o(10), 'htlc_minimum_msat': o(250), 'fee_base_msat': o(100), 'fee_proportional_millionths': o(150), 'chain_hash': BitcoinTestnet.rev_genesis_bytes(), 'timestamp': b'\x00\x00\x00\x00'}) - cdb.add_channel_update({'short_channel_id': bfh('0000000000000004'), 'message_flags': b'\x00', 'channel_flags': b'\x01', 'cltv_expiry_delta': o(10), 'htlc_minimum_msat': o(250), 'fee_base_msat': o(100), 'fee_proportional_millionths': o(150), 'chain_hash': BitcoinTestnet.rev_genesis_bytes(), 'timestamp': b'\x00\x00\x00\x00'}) - cdb.add_channel_update({'short_channel_id': bfh('0000000000000004'), 'message_flags': b'\x00', 'channel_flags': b'\x00', 'cltv_expiry_delta': o(10), 'htlc_minimum_msat': o(250), 'fee_base_msat': o(100), 'fee_proportional_millionths': o(150), 'chain_hash': BitcoinTestnet.rev_genesis_bytes(), 'timestamp': b'\x00\x00\x00\x00'}) - cdb.add_channel_update({'short_channel_id': bfh('0000000000000005'), 'message_flags': b'\x00', 'channel_flags': b'\x01', 'cltv_expiry_delta': o(10), 'htlc_minimum_msat': o(250), 'fee_base_msat': o(100), 'fee_proportional_millionths': o(150), 'chain_hash': BitcoinTestnet.rev_genesis_bytes(), 'timestamp': b'\x00\x00\x00\x00'}) - cdb.add_channel_update({'short_channel_id': bfh('0000000000000005'), 'message_flags': b'\x00', 'channel_flags': b'\x00', 'cltv_expiry_delta': o(10), 'htlc_minimum_msat': o(250), 'fee_base_msat': o(100), 'fee_proportional_millionths': o(999), 'chain_hash': BitcoinTestnet.rev_genesis_bytes(), 'timestamp': b'\x00\x00\x00\x00'}) - cdb.add_channel_update({'short_channel_id': bfh('0000000000000006'), 'message_flags': b'\x00', 'channel_flags': b'\x00', 'cltv_expiry_delta': o(10), 'htlc_minimum_msat': o(250), 'fee_base_msat': o(100), 'fee_proportional_millionths': o(99999999), 'chain_hash': BitcoinTestnet.rev_genesis_bytes(), 'timestamp': b'\x00\x00\x00\x00'}) - cdb.add_channel_update({'short_channel_id': bfh('0000000000000006'), 'message_flags': b'\x00', 'channel_flags': b'\x01', 'cltv_expiry_delta': o(10), 'htlc_minimum_msat': o(250), 'fee_base_msat': o(100), 'fee_proportional_millionths': o(150), 'chain_hash': BitcoinTestnet.rev_genesis_bytes(), 'timestamp': b'\x00\x00\x00\x00'}) + 'len': 0, 'features': b''}, trusted=True) + cdb.add_channel_update({'short_channel_id': bfh('0000000000000001'), 'message_flags': b'\x00', 'channel_flags': b'\x00', 'cltv_expiry_delta': 10, 'htlc_minimum_msat': 250, 'fee_base_msat': 100, 'fee_proportional_millionths': 150, 'chain_hash': BitcoinTestnet.rev_genesis_bytes(), 'timestamp': 0}) + cdb.add_channel_update({'short_channel_id': bfh('0000000000000001'), 'message_flags': b'\x00', 'channel_flags': b'\x01', 'cltv_expiry_delta': 10, 'htlc_minimum_msat': 250, 'fee_base_msat': 100, 'fee_proportional_millionths': 150, 'chain_hash': BitcoinTestnet.rev_genesis_bytes(), 'timestamp': 0}) + cdb.add_channel_update({'short_channel_id': bfh('0000000000000002'), 'message_flags': b'\x00', 'channel_flags': b'\x00', 'cltv_expiry_delta': 99, 'htlc_minimum_msat': 250, 'fee_base_msat': 100, 'fee_proportional_millionths': 150, 'chain_hash': BitcoinTestnet.rev_genesis_bytes(), 'timestamp': 0}) + cdb.add_channel_update({'short_channel_id': bfh('0000000000000002'), 'message_flags': b'\x00', 'channel_flags': b'\x01', 'cltv_expiry_delta': 10, 'htlc_minimum_msat': 250, 'fee_base_msat': 100, 'fee_proportional_millionths': 150, 'chain_hash': BitcoinTestnet.rev_genesis_bytes(), 'timestamp': 0}) + cdb.add_channel_update({'short_channel_id': bfh('0000000000000003'), 'message_flags': b'\x00', 'channel_flags': b'\x01', 'cltv_expiry_delta': 10, 'htlc_minimum_msat': 250, 'fee_base_msat': 100, 'fee_proportional_millionths': 150, 'chain_hash': BitcoinTestnet.rev_genesis_bytes(), 'timestamp': 0}) + cdb.add_channel_update({'short_channel_id': bfh('0000000000000003'), 'message_flags': b'\x00', 'channel_flags': b'\x00', 'cltv_expiry_delta': 10, 'htlc_minimum_msat': 250, 'fee_base_msat': 100, 'fee_proportional_millionths': 150, 'chain_hash': BitcoinTestnet.rev_genesis_bytes(), 'timestamp': 0}) + cdb.add_channel_update({'short_channel_id': bfh('0000000000000004'), 'message_flags': b'\x00', 'channel_flags': b'\x01', 'cltv_expiry_delta': 10, 'htlc_minimum_msat': 250, 'fee_base_msat': 100, 'fee_proportional_millionths': 150, 'chain_hash': BitcoinTestnet.rev_genesis_bytes(), 'timestamp': 0}) + cdb.add_channel_update({'short_channel_id': bfh('0000000000000004'), 'message_flags': b'\x00', 'channel_flags': b'\x00', 'cltv_expiry_delta': 10, 'htlc_minimum_msat': 250, 'fee_base_msat': 100, 'fee_proportional_millionths': 150, 'chain_hash': BitcoinTestnet.rev_genesis_bytes(), 'timestamp': 0}) + cdb.add_channel_update({'short_channel_id': bfh('0000000000000005'), 'message_flags': b'\x00', 'channel_flags': b'\x01', 'cltv_expiry_delta': 10, 'htlc_minimum_msat': 250, 'fee_base_msat': 100, 'fee_proportional_millionths': 150, 'chain_hash': BitcoinTestnet.rev_genesis_bytes(), 'timestamp': 0}) + cdb.add_channel_update({'short_channel_id': bfh('0000000000000005'), 'message_flags': b'\x00', 'channel_flags': b'\x00', 'cltv_expiry_delta': 10, 'htlc_minimum_msat': 250, 'fee_base_msat': 100, 'fee_proportional_millionths': 999, 'chain_hash': BitcoinTestnet.rev_genesis_bytes(), 'timestamp': 0}) + cdb.add_channel_update({'short_channel_id': bfh('0000000000000006'), 'message_flags': b'\x00', 'channel_flags': b'\x00', 'cltv_expiry_delta': 10, 'htlc_minimum_msat': 250, 'fee_base_msat': 100, 'fee_proportional_millionths': 99999999, 'chain_hash': BitcoinTestnet.rev_genesis_bytes(), 'timestamp': 0}) + cdb.add_channel_update({'short_channel_id': bfh('0000000000000006'), 'message_flags': b'\x00', 'channel_flags': b'\x01', 'cltv_expiry_delta': 10, 'htlc_minimum_msat': 250, 'fee_base_msat': 100, 'fee_proportional_millionths': 150, 'chain_hash': BitcoinTestnet.rev_genesis_bytes(), 'timestamp': 0}) path = path_finder.find_path_for_payment(b'\x02aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa', b'\x02eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee', 100000) self.assertEqual([(b'\x02bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb', b'\x00\x00\x00\x00\x00\x00\x00\x03'), (b'\x02eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee', b'\x00\x00\x00\x00\x00\x00\x00\x02'), From 69497522630b95a632875cf87bbeec7d1d7b3242 Mon Sep 17 00:00:00 2001 From: SomberNight Date: Fri, 13 Mar 2020 21:20:31 +0100 Subject: [PATCH 03/17] lnmsg: initial TLV implementation --- electrum/lnmsg.py | 272 ++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 239 insertions(+), 33 deletions(-) diff --git a/electrum/lnmsg.py b/electrum/lnmsg.py index 4e362efaec81..49afb7319b80 100644 --- a/electrum/lnmsg.py +++ b/electrum/lnmsg.py @@ -2,6 +2,7 @@ import csv import io from typing import Callable, Tuple, Any, Dict, List, Sequence, Union, Optional +from collections import OrderedDict class MalformedMsg(Exception): @@ -16,12 +17,56 @@ class UnexpectedEndOfStream(MalformedMsg): pass -def _assert_can_read_at_least_n_bytes(fd: io.BytesIO, n: int) -> None: +class FieldEncodingNotMinimal(MalformedMsg): + pass + + +class UnknownMandatoryTLVRecordType(MalformedMsg): + pass + + +def _num_remaining_bytes_to_read(fd: io.BytesIO) -> int: cur_pos = fd.tell() end_pos = fd.seek(0, io.SEEK_END) fd.seek(cur_pos) - if end_pos - cur_pos < n: - raise UnexpectedEndOfStream(f"cur_pos={cur_pos}. end_pos={end_pos}. wants to read: {n}") + return end_pos - cur_pos + + +def _assert_can_read_at_least_n_bytes(fd: io.BytesIO, n: int) -> None: + nremaining = _num_remaining_bytes_to_read(fd) + if nremaining < n: + raise UnexpectedEndOfStream(f"wants to read {n} bytes but only {nremaining} bytes left") + + +def bigsize_from_int(i: int) -> bytes: + assert i >= 0, i + if i < 0xfd: + return int.to_bytes(i, length=1, byteorder="big", signed=False) + elif i < 0x1_0000: + return b"\xfd" + int.to_bytes(i, length=2, byteorder="big", signed=False) + elif i < 0x1_0000_0000: + return b"\xfe" + int.to_bytes(i, length=4, byteorder="big", signed=False) + else: + return b"\xff" + int.to_bytes(i, length=8, byteorder="big", signed=False) + + +def read_int_from_bigsize(fd: io.BytesIO) -> Optional[int]: + try: + first = fd.read(1)[0] + except IndexError: + return None # end of file + if first < 0xfd: + return first + elif first == 0xfd: + _assert_can_read_at_least_n_bytes(fd, 2) + return int.from_bytes(fd.read(2), byteorder="big", signed=False) + elif first == 0xfe: + _assert_can_read_at_least_n_bytes(fd, 4) + return int.from_bytes(fd.read(4), byteorder="big", signed=False) + elif first == 0xff: + _assert_can_read_at_least_n_bytes(fd, 8) + return int.from_bytes(fd.read(8), byteorder="big", signed=False) + raise Exception() def _read_field(*, fd: io.BytesIO, field_type: str, count: int) -> Union[bytes, int]: @@ -32,22 +77,36 @@ def _read_field(*, fd: io.BytesIO, field_type: str, count: int) -> Union[bytes, type_len = None if field_type == 'byte': type_len = 1 - elif field_type == 'u16': - type_len = 2 + elif field_type in ('u16', 'u32', 'u64'): + if field_type == 'u16': + type_len = 2 + elif field_type == 'u32': + type_len = 4 + else: + assert field_type == 'u64' + type_len = 8 assert count == 1, count _assert_can_read_at_least_n_bytes(fd, type_len) return int.from_bytes(fd.read(type_len), byteorder="big", signed=False) - elif field_type == 'u32': - type_len = 4 + elif field_type in ('tu16', 'tu32', 'tu64'): + if field_type == 'tu16': + type_len = 2 + elif field_type == 'tu32': + type_len = 4 + else: + assert field_type == 'tu64' + type_len = 8 assert count == 1, count - _assert_can_read_at_least_n_bytes(fd, type_len) - return int.from_bytes(fd.read(type_len), byteorder="big", signed=False) - elif field_type == 'u64': - type_len = 8 + raw = fd.read(type_len) + if len(raw) > 0 and raw[0] == 0x00: + raise FieldEncodingNotMinimal() + return int.from_bytes(raw, byteorder="big", signed=False) + elif field_type == 'varint': assert count == 1, count - _assert_can_read_at_least_n_bytes(fd, type_len) - return int.from_bytes(fd.read(type_len), byteorder="big", signed=False) - # TODO tu16/tu32/tu64 + val = read_int_from_bigsize(fd) + if val is None: + raise UnexpectedEndOfStream() + return val elif field_type == 'chain_hash': type_len = 32 elif field_type == 'channel_id': @@ -82,7 +141,35 @@ def _write_field(*, fd: io.BytesIO, field_type: str, count: int, type_len = 4 elif field_type == 'u64': type_len = 8 - # TODO tu16/tu32/tu64 + elif field_type in ('tu16', 'tu32', 'tu64'): + if field_type == 'tu16': + type_len = 2 + elif field_type == 'tu32': + type_len = 4 + else: + assert field_type == 'tu64' + type_len = 8 + assert count == 1, count + if isinstance(value, int): + value = int.to_bytes(value, length=type_len, byteorder="big", signed=False) + if not isinstance(value, (bytes, bytearray)): + raise Exception(f"can only write bytes into fd. got: {value!r}") + while len(value) > 0 and value[0] == 0x00: + value = value[1:] + nbytes_written = fd.write(value) + if nbytes_written != len(value): + raise Exception(f"tried to write {len(value)} bytes, but only wrote {nbytes_written}!?") + return + elif field_type == 'varint': + assert count == 1, count + if isinstance(value, int): + value = bigsize_from_int(value) + if not isinstance(value, (bytes, bytearray)): + raise Exception(f"can only write bytes into fd. got: {value!r}") + nbytes_written = fd.write(value) + if nbytes_written != len(value): + raise Exception(f"tried to write {len(value)} bytes, but only wrote {nbytes_written}!?") + return elif field_type == 'chain_hash': type_len = 32 elif field_type == 'channel_id': @@ -109,16 +196,55 @@ def _write_field(*, fd: io.BytesIO, field_type: str, count: int, raise Exception(f"tried to write {len(value)} bytes, but only wrote {nbytes_written}!?") +def _read_tlv_record(*, fd: io.BytesIO) -> Tuple[int, bytes]: + if not fd: raise Exception() + tlv_type = _read_field(fd=fd, field_type="varint", count=1) + tlv_len = _read_field(fd=fd, field_type="varint", count=1) + tlv_val = _read_field(fd=fd, field_type="byte", count=tlv_len) + return tlv_type, tlv_val + + +def _write_tlv_record(*, fd: io.BytesIO, tlv_type: int, tlv_val: bytes) -> None: + if not fd: raise Exception() + tlv_len = len(tlv_val) + _write_field(fd=fd, field_type="varint", count=1, value=tlv_type) + _write_field(fd=fd, field_type="varint", count=1, value=tlv_len) + _write_field(fd=fd, field_type="byte", count=tlv_len, value=tlv_val) + + +def _resolve_field_count(field_count_str: str, *, vars_dict: dict) -> int: + if field_count_str == "": + field_count = 1 + elif field_count_str == "...": + raise NotImplementedError() # TODO... + else: + try: + field_count = int(field_count_str) + except ValueError: + field_count = vars_dict[field_count_str] + if isinstance(field_count, (bytes, bytearray)): + field_count = int.from_bytes(field_count, byteorder="big") + assert isinstance(field_count, int) + return field_count + + class LNSerializer: def __init__(self): + # TODO msg_type could be 'int' everywhere... self.msg_scheme_from_type = {} # type: Dict[bytes, List[Sequence[str]]] self.msg_type_from_name = {} # type: Dict[str, bytes] + + self.in_tlv_stream_get_tlv_record_scheme_from_type = {} # type: Dict[str, Dict[int, List[Sequence[str]]]] + self.in_tlv_stream_get_record_type_from_name = {} # type: Dict[str, Dict[str, int]] + self.in_tlv_stream_get_record_name_from_type = {} # type: Dict[str, Dict[int, str]] + path = os.path.join(os.path.dirname(__file__), "lnwire", "peer_wire.csv") with open(path, newline='') as f: csvreader = csv.reader(f) for row in csvreader: #print(f">>> {row!r}") if row[0] == "msgtype": + # msgtype,,[,