From ea702c4a03111f20408a349bab76e1e071bf7c01 Mon Sep 17 00:00:00 2001
From: Rusty Russell <rusty@rustcorp.com.au>
Date: Fri, 26 Jul 2019 19:09:33 +0930
Subject: [PATCH 1/2] tools/extract-formats.py: allow '*' as well as '1.'/'2.'

The format for TLV types looked pretty, but @ZmnSCPxj points out that
successive ordered lists in markdown get merged into one megalist.

If we allow ordered or unordered lists, we're a bit more futureproof
against formatting changes.

Signed-off-by: Rusty Russell <rusty@rustcorp.com.au>
---
 tools/extract-formats.py | 19 ++++++++++++-------
 1 file changed, 12 insertions(+), 7 deletions(-)

diff --git a/tools/extract-formats.py b/tools/extract-formats.py
index 6ca2f2aba..64e1406a2 100755
--- a/tools/extract-formats.py
+++ b/tools/extract-formats.py
@@ -22,14 +22,19 @@
 import re
 import fileinput
 
+# We allow either ordered or unordered lists.
 typeline = re.compile(
-    '1\. type: (?P<value>[-0-9A-Za-z_|]+) \(`(?P<name>[A-Za-z0-9_]+)`\)( \(`?(?P<option>[^)`]*)`\))?')
+    '(1\.|\*) type: (?P<value>[-0-9A-Za-z_|]+) \(`(?P<name>[A-Za-z0-9_]+)`\)( \(`?(?P<option>[^)`]*)`\))?')
 tlvline = re.compile(
-    '1\. tlvs: `(?P<name>[A-Za-z0-9_]+)`( \(`?(?P<option>[^)`]*)`\))?')
+    '(1\.|\*) tlvs: `(?P<name>[A-Za-z0-9_]+)`( \(`?(?P<option>[^)`]*)`\))?')
 subtypeline = re.compile(
-    '1\. subtype: `(?P<name>[A-Za-z0-9_]+)`( \(`?(?P<option>[^)`]*)`\))?')
+    '(1\.|\*) subtype: `(?P<name>[A-Za-z0-9_]+)`( \(`?(?P<option>[^)`]*)`\))?')
 dataline = re.compile(
-    '\s+\* \[`(?P<typefield>[-_a-zA-Z0-9*+]+)`:`(?P<name>[_a-z0-9]+)`\]( \(`?(?P<option>[^)`]*)`?\))?')
+    '\s+([0-9]+\.|\*) \[`(?P<typefield>[-_a-zA-Z0-9*+]+)`:`(?P<name>[_a-z0-9]+)`\]( \(`?(?P<option>[^)`]*)`?\))?')
+datastartline = re.compile(
+    '(2\.|\*) data:')
+tlvtypesline = re.compile(
+    '(2\.|\*) types:')
 
 # Generator to give us one line at a time.
 def next_line(args, lines):
@@ -96,7 +101,7 @@ def parse_type(genline, output, name, value, option, in_tlv=None):
     print_csv(output, '{},{},{}'.format(type_prefix, name, value), option)
 
     # Expect a data: line before values, if any
-    if line.lstrip() != '2. data:':
+    if not datastartline.fullmatch(line.lstrip()):
         return _, line
 
     while True:
@@ -131,7 +136,7 @@ def parse_tlv(genline, output, name, option):
     i, line = next(genline)
 
     # Expect a types: line after tlvs.
-    if line != '2. types:':
+    if not tlvtypesline.fullmatch(line):
         raise ValueError('{}: Expected "2. types:" line'.format(i))
 
     _, line = next(genline)
@@ -158,7 +163,7 @@ def parse_subtype(genline, output, name, option):
     i, line = next(genline)
 
     # Expect a data: line after subtype.
-    if line != '2. data:':
+    if not datastartline.fullmatch(line):
         raise ValueError('{}: Expected "2. data:" line'.format(i))
 
     print_csv(output, 'subtype,{}'.format(name), option)

From d08b26f12aaf2f0b94a56dacad14221bb1ebeabc Mon Sep 17 00:00:00 2001
From: Rusty Russell <rusty@rustcorp.com.au>
Date: Tue, 30 Jul 2019 17:01:30 +0930
Subject: [PATCH 2/2] tools/extract-formats.py: allow '.' in length fields.

In practice, using '...*type' is the clearest and simplest way to specify
the common case of "the rest of the TLV is an array of 'type'", rather
than some arbitrary expression with a made-up length field.

Signed-off-by: Rusty Russell <rusty@rustcorp.com.au>
---
 tools/extract-formats.py | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/tools/extract-formats.py b/tools/extract-formats.py
index 64e1406a2..b505683d5 100755
--- a/tools/extract-formats.py
+++ b/tools/extract-formats.py
@@ -30,7 +30,7 @@
 subtypeline = re.compile(
     '(1\.|\*) subtype: `(?P<name>[A-Za-z0-9_]+)`( \(`?(?P<option>[^)`]*)`\))?')
 dataline = re.compile(
-    '\s+([0-9]+\.|\*) \[`(?P<typefield>[-_a-zA-Z0-9*+]+)`:`(?P<name>[_a-z0-9]+)`\]( \(`?(?P<option>[^)`]*)`?\))?')
+    '\s+([0-9]+\.|\*) \[`(?P<typefield>[-._a-zA-Z0-9*+]+)`:`(?P<name>[_a-z0-9]+)`\]( \(`?(?P<option>[^)`]*)`?\))?')
 datastartline = re.compile(
     '(2\.|\*) data:')
 tlvtypesline = re.compile(
@@ -80,14 +80,14 @@ def print_csv(output, fmt, option):
 #     * [`chain_hash`:`chain_hash`]
 #     * [`u16`:`len`]
 #     * [`len*byte`:`encoded_short_ids`]
-#     * [`tlvs`:`query_short_channel_ids_tlvs`]
+#     * [`query_short_channel_ids_tlvs`:`tlvs`]
 #
 # output:
 #   msgtype,query_short_channel_ids,261,gossip_queries
 #   msgdata,query_short_channel_ids,chain_hash,chain_hash,
 #   msgdata,query_short_channel_ids,len,u16,
 #   msgdata,query_short_channel_ids,encoded_short_ids,byte,len
-#   msgdata,query_short_channel_ids,query_short_channel_ids_tlvs,tlvs,
+#   msgdata,query_short_channel_ids,tlvs,query_short_channel_ids_tlvs,
 def parse_type(genline, output, name, value, option, in_tlv=None):
     _, line = next(genline)
 
@@ -126,12 +126,12 @@ def parse_type(genline, output, name, value, option, in_tlv=None):
 #    1. type: 1 (`query_flags`)
 #    2. data:
 #      * [`byte`:`encoding_type`]
-#      * [`tlv_len-1*byte`:`encoded_query_flags`]
+#      * [`...*byte`:`encoded_query_flags`]
 #
 # output:
 #  tlvtype,query_short_channel_ids_tlvs,query_flags,1
 #  tlvdata,query_short_channel_ids_tlvs,query_flags,encoding_type,byte,
-#  tlvdata,query_short_channel_ids_tlvs,query_flags,encoded_query_flags,byte,tlv_len-1
+#  tlvdata,query_short_channel_ids_tlvs,query_flags,encoded_query_flags,byte,...
 def parse_tlv(genline, output, name, option):
     i, line = next(genline)