Misc enhancements

tests/events/test-spec.md

@@ -18,11 +18,12 @@ SEQUENCE_STEP := INDENT4* NUMBER `.` SPACE+ EVENT_OR_SERIES OPTION_SPEC*
 EVENT_OR_SERIES := EVENT | `One of:` | `Any order:`
 EVENT := INPUT_EVENT | OUTPUT_EVENT | `nothing`
 
-META_LINE := COMMENT | SPACE* | VARSET | INCLUDE
+META_LINE := COMMENT | SPACE* | VARSET | INCLUDE | SKIPIF
 
 COMMENT := `#` [SPACE|STRING]*
 VARSET := IDENTIFIER`=`[SPACE|STRING]* OPTION_SPEC*
 INCLUDE := `include` SPACE STRING OPTION_SPEC*
+SKIPIF := `skipif` SPACE STRING OPTION_SPEC*
 
 Comment and blank lines are ignored.
 
@@ -31,6 +32,9 @@ a `$` prefix.  There's currently no scope to variables.
 
 Include lines pull in other files, which is helpful for complex tests.
 
+Skipif lines cause immediate passing if the OPTION_SPECs match (and
+print out the string); helpful for tests which require specific features.
+
 Other lines are indented by multiples of 4 spaces; a line not indented
 by a multiple of 4 is be joined to the previous line (this allows
 nicer formatting for long lines).
@@ -99,7 +103,7 @@ compulsory (`even`).
 
 ## Input Events
 
-INPUT_EVENT := CONNECT | RECV | BLOCK | DISCONNECT | OPENCMD
+INPUT_EVENT := CONNECT | RECV | BLOCK | DISCONNECT | FUNDCHANCMD | INVOICECMD | ADDHTLCCMD
 
 CONNECT := `connect:` SPACE+ CONNECT_OPTS
 CONNECT_OPTS := `privkey=` HEX64
@@ -118,6 +122,8 @@ FUNDCHANCMD := `fundchannel:` [CONNSPEC] SPACE+ `amount=`NUMBER SPACE+ `utxo=`HE
 
 INVOICECMD := `invoice:` SPACE+ `amount=`NUMBER SPACE+ `preimage=`HEX64
 
+ADDHTLCCMD := `addhtlc:` [CONNSPEC] SPACE+ `amount=`NUMBER SPACE+ `preimage=`HEX64
+
 CONNSPEC := SPACE+ `conn=`HEX64
 
 Input events are:
@@ -137,6 +143,7 @@ Input events are:
 * `disconnect`: a connection closed by a peer.
 * `fundchannel`: tell the implementation to initiate the opening of a channel of the given `amount` of satoshis with the specific peer identified by `conn` (default, last `connect`).  The funding comes from a single `utxo`, as specified by txid and output number.
 * `invoice`: tell the implementation to accept a payment of `amount` msatoshis, with payment_preimage `preimage`.
+* `addhtlc`: tell the implementation to add (and commit) a simple htlc directly to the test peer, with CLTV 5 past the current block.
 
 ## Output Events
 

tools/test-events-clightning.py

@@ -6,9 +6,11 @@
 
 import bitcoin
 import bitcoin.rpc
+import hashlib
 import importlib
 import lightning
 import os
+import secp256k1
 import shutil
 import struct
 import subprocess
@@ -146,6 +148,7 @@ def start(self):
                                       '--dev-bitcoind-poll=1',
                                       '--dev-fast-gossip',
                                       '--dev-gossip-time=1565587763',
+                                      '--dev-no-htlc-timeout',
                                       '--bind-addr=127.0.0.1:{}'.format(self.lightning_port),
                                       '--network=regtest',
                                       '--bitcoin-rpcuser=rpcuser',
@@ -339,6 +342,22 @@ def invoice(self, amount, preimage, line):
                          description='invoice from {}'.format(line),
                          preimage=preimage)
 
+
+    def addhtlc(self, conn, amount, preimage, line):
+        # Here's the completely undocumented way to use python secp!
+        pubkey = secp256k1.PrivateKey(bytes.fromhex(conn.connkey)).pubkey.serialize().hex()
+        payhash = hashlib.sha256(bytes.fromhex(preimage)).hexdigest()
+        routestep = {
+            'msatoshi': amount,
+            'id': pubkey,
+            # We internally add one.
+            'delay': 4,
+            # We actually ignore this.
+            'channel': '1x1x1'
+        }
+        self.rpc.sendpay([routestep], payhash)
+
+
     def _readmsg(self, conn):
         rawl = conn.proc.stdout.read(2)
         length = struct.unpack('>H', rawl)[0]

tools/test-events.py

@@ -239,6 +239,11 @@ def invoice(self, amount, preimage, line):
             print("[INVOICE for {} with PREIMAGE {} {}]"
                   .format(amount, preimage, line))
 
+    def addhtlc(self, conn, amount, preimage, line):
+        if self.verbose:
+            print("[ADDHTLC TO {} for {} with PREIMAGE {} {}]"
+                  .format(conn, amount, preimage, line))
+
     def expect_send(self, conn, line):
         if self.verbose:
             print("[EXPECT-SEND {}]".format(line))
@@ -268,8 +273,8 @@ def __init__(self, message, name, typename, count, options):
         self.options = options
         self.islenvar = False
         # This contains all the integer types: otherwise it's a hexstring,
-        self.isinteger = (typename in name2structfmt and
-            not (typename == 'byte' and count))
+        self.isinteger = (typename in name2structfmt
+                          and not (typename == 'byte' and count))
 
         # This is set for static-sized array.
         self.arraylen = None
@@ -864,6 +869,7 @@ def action(self, runner, line):
         runner.recv(which_connection(line, runner, self.connkey),
                     self.b, line)
 
+
 def compare_results(msgname, f, v, exp):
     """ f -> field; v -> value; exp -> expected value """
 
@@ -879,24 +885,25 @@ def compare_results(msgname, f, v, exp):
                 .format(f.name))
 
     # Do signature verification, if necessary
-    if (f.typename == 'signature' and isinstance(exp,tuple)) or (f.typename == 'signature'
-            and (f.arrayvar or f.arraylen) and isinstance(exp,list)):
-            if f.arrayvar or f.arraylen:
-                for (e, val) in list(map(lambda x,y: (x,y), exp, v)):
-                    if isinstance(e, tuple):
-                        if not Sigs.verify_sig(e[0], e[1], val):
-                            return "Invalid signature ({}) for privkey {}, hash {}".format(
-                                    val.hex(), e[0], e[1])
-                    elif e != val:
-                        return ("Expected {}.{}(type:{}) {} but got {}"
-                                .format(msgname,
-                                        f.name, f.typename, e.hex(), val.hex()))
-            else:
-                # v should be a valid signature, a byte-array r||s
-                if not Sigs.verify_sig(exp[0], exp[1], v):
-                    return "Invalid signature ({}) for privkey {}, hash {}".format(v.hex(), exp[0], exp[1])
-            # Successfully matched all sigs!!
-            return None
+    if ((f.typename == 'signature' and isinstance(exp, tuple))
+        or (f.typename == 'signature'
+            and (f.arrayvar or f.arraylen) and isinstance(exp, list))):
+        if f.arrayvar or f.arraylen:
+            for (e, val) in list(map(lambda x, y: (x, y), exp, v)):
+                if isinstance(e, tuple):
+                    if not Sigs.verify_sig(e[0], e[1], val):
+                        return "Invalid signature ({}) for privkey {}, hash {}".format(
+                            val.hex(), e[0], e[1])
+                elif e != val:
+                    return ("Expected {}.{}(type:{}) {} but got {}"
+                            .format(msgname,
+                                    f.name, f.typename, e.hex(), val.hex()))
+        else:
+            # v should be a valid signature, a byte-array r||s
+            if not Sigs.verify_sig(exp[0], exp[1], v):
+                return "Invalid signature ({}) for privkey {}, hash {}".format(v.hex(), exp[0], exp[1])
+        # Successfully matched all sigs!!
+        return None
 
     if isinstance(exp, tuple):
         # Out-of-range bitmaps are considered 0 (eg. feature tests)
@@ -933,7 +940,7 @@ def compare_results(msgname, f, v, exp):
             valstr = str(v)
             expectstr = str(exp)
         # Same as above note about a range of length
-        elif isinstance(exp,str) and exp.startswith('*'):
+        elif isinstance(exp, str) and exp.startswith('*'):
             if check_range(exp[1:], len(v.hex()) // 2):
                 return None
             expectstr = "result of bytelen {}".format(exp[1:])
@@ -957,6 +964,7 @@ def check_range(exp, val_len):
 
     return int(len_range[0]) <= val_len and int(len_range[1]) >= val_len
 
+
 def message_match(expectmsg, expectfields, b):
     """Internal helper to see if b matches expectmsg & expectfields.
 
@@ -1147,6 +1155,19 @@ def action(self, runner, line):
         runner.invoice(self.amount, self.preimage, line)
 
 
+class AddHtlcEvent(object):
+    def __init__(self, line, parts):
+        d = parse_params(line, parts, ['amount', 'preimage'], ['conn'])
+        self.connkey = optional_connection(line, d)
+        self.preimage = d['preimage']
+        check_hex(line, self.preimage, 64)
+        self.amount = int(d['amount'])
+
+    def action(self, runner, line):
+        runner.addhtlc(which_connection(line, runner, self.connkey),
+                       self.amount, self.preimage, line)
+
+
 class ExpectErrorEvent(object):
     def __init__(self, line, parts):
         d = parse_params(line, parts, [], ['conn'])
@@ -1185,6 +1206,8 @@ def __init__(self, args, desc, line):
             self.actor = FundChannelEvent(line, parts[1:])
         elif parts[0] == 'invoice:':
             self.actor = InvoiceEvent(line, parts[1:])
+        elif parts[0] == 'addhtlc:':
+            self.actor = AddHtlcEvent(line, parts[1:])
         elif parts[0] == 'expect-error:':
             self.actor = ExpectErrorEvent(line, parts[1:])
         elif parts[0] == 'nothing':
@@ -1524,31 +1547,31 @@ def line_minus_comments(verbose, line, linenum):
 
 
 def filter_out(args, line, filename, linenum):
-        """Trim options: we discard the line if it doesn't qualify."""
-        while True:
-            m = re.search("(?P<invert>!?)"
-                          "(?P<optname>opt[A-Za-z_]*)"
-                          r"(?P<oddoreven>(/(odd|even))?)\s*$", line)
-            if m is None:
-                return line
-
-            if m.group('oddoreven') != '':
-                present = m.group('optname') + m.group('oddoreven') in args.option
-            else:
-                present = (m.group('optname') + '/odd' in args.option
-                           or m.group('optname') + '/even' in args.option)
+    """Trim options: we discard the line if it doesn't qualify."""
+    while True:
+        m = re.search("(?P<invert>!?)"
+                      "(?P<optname>opt[A-Za-z_]*)"
+                      r"(?P<oddoreven>(/(odd|even))?)\s*$", line)
+        if m is None:
+            return line
+
+        if m.group('oddoreven') != '':
+            present = m.group('optname') + m.group('oddoreven') in args.option
+        else:
+            present = (m.group('optname') + '/odd' in args.option
+                       or m.group('optname') + '/even' in args.option)
 
-            # If option was specified as --option, invert must be set.
-            wanted = m.group('invert') != '!'
-            if present != wanted:
-                if args.verbose:
-                    print("# Removing line {}: requires {}{}{}"
-                          .format(Line(filename, linenum, linenum, 0, line),
-                                  m.group('invert'),
-                                  m.group('optname'),
-                                  m.group('oddoreven')))
-                return ''
-            line = line[:m.start()]
+        # If option was specified as --option, invert must be set.
+        wanted = m.group('invert') != '!'
+        if present != wanted:
+            if args.verbose:
+                print("# Removing line {}: requires {}{}{}"
+                      .format(Line(filename, linenum, linenum, 0, line),
+                              m.group('invert'),
+                              m.group('optname'),
+                              m.group('oddoreven')))
+            return ''
+        line = line[:m.start()]
 
 
 def indentation(s):
@@ -1567,6 +1590,13 @@ def indentation(s):
     return s[consumed:], level
 
 
+class SkipParsingException(Exception):
+    """Exception raised when skipif is found during parsing"""
+    def __init__(self, reason):
+        super().__init__()
+        self.reason = reason
+
+
 def parse_file(args, f, filename, variables):
     """Get non-comment lines, as [(linenums,indentlevel,line)], grab vars"""
     content = []
@@ -1631,7 +1661,7 @@ def parse_file(args, f, filename, variables):
                                      indentlevel, lines[i]),
                                 "Re-setting var {}".format(parts[0]))
             variables[parts[0]] = parts[2]
-        # Similarly, do include directives immediately.
+        # Similarly, do include/skipif directives immediately.
         elif line.startswith('include '):
             # Filenames are assumed to be relative.
             subfilename = path.join(path.dirname(filename), line[8:])
@@ -1642,6 +1672,8 @@ def parse_file(args, f, filename, variables):
             for l in sublines:
                 l.indentlevel += indentlevel
             content += sublines
+        elif line.startswith('skipif '):
+            raise SkipParsingException(line[7:])
         else:
             content.append(Line(filename, line_start, line_end, indentlevel,
                                 line))
@@ -1662,7 +1694,13 @@ def main(args, runner):
         else:
             f = open(filename)
 
-        lines, _ = parse_file(args, f, filename, {})
+        try:
+            lines, _ = parse_file(args, f, filename, {})
+        except SkipParsingException as e:
+            print("{}: skipped: {}".format(filename, e.reason))
+            f.close()
+            continue
+
         f.close()
 
         graph = nx.DiGraph()