From 243174228559fc66268f3e8d6f5a684f37d77757 Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Thu, 31 May 2018 12:01:27 +0930 Subject: [PATCH 01/46] gossipd: don't publish private updates after channel_announce. We generate new ones anyway; removing this code changes fixes coming up which now only need to change one place. Signed-off-by: Rusty Russell --- gossipd/routing.c | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/gossipd/routing.c b/gossipd/routing.c index 23dcb1fe4e23..f86344c74c53 100644 --- a/gossipd/routing.c +++ b/gossipd/routing.c @@ -638,13 +638,10 @@ bool routing_add_channel_announcement(struct routing_state *rstate, /* Now we can broadcast channel announce */ insert_broadcast(rstate->broadcasts, chan->channel_announce); - /* If we had private updates for channels, we can broadcast them too. */ - for (size_t i = 0; i < ARRAY_SIZE(chan->half); i++) { - if (!is_halfchan_defined(&chan->half[i])) - continue; - insert_broadcast(rstate->broadcasts, - chan->half[i].channel_update); - } + /* Clear any private updates. */ + for (size_t i = 0; i < ARRAY_SIZE(chan->half); i++) + chan->half[i].channel_update + = tal_free(chan->half[i].channel_update); return true; } From 498ad913185d337e1162c213395c3320287f8421 Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Thu, 31 May 2018 12:02:27 +0930 Subject: [PATCH 02/46] tests: new helper to get pid of a particular subdaemon. Signed-off-by: Rusty Russell --- tests/test_lightningd.py | 7 ++----- tests/utils.py | 5 +++++ 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/tests/test_lightningd.py b/tests/test_lightningd.py index e9376be8aaf1..21bc85187c66 100644 --- a/tests/test_lightningd.py +++ b/tests/test_lightningd.py @@ -4250,16 +4250,13 @@ def test_io_logging(self): # Fundchannel manually so we get channeld pid. self.give_funds(l1, 10**6 + 1000000) l1.rpc.fundchannel(l2.info['id'], 10**6)['tx'] - # Get pid of channeld, eg 'lightning_channeld(...): pid 13933, msgfd 12' - pidline = l1.daemon.wait_for_log(r'lightning_channeld.*: pid [0-9]*,') - pid1 = re.search(r'pid ([0-9]*),', pidline).group(1) + pid1 = l1.subd_pid('channeld') l1.daemon.wait_for_log('sendrawtx exit 0') l1.bitcoin.generate_block(1) l1.daemon.wait_for_log(' to CHANNELD_NORMAL') - pidline = l2.daemon.wait_for_log(r'lightning_channeld.*: pid [0-9]*,') - pid2 = re.search(r'pid ([0-9]*),', pidline).group(1) + pid2 = l2.subd_pid('channeld') l2.daemon.wait_for_log(' to CHANNELD_NORMAL') # Send it sigusr1: should turn on logging. diff --git a/tests/utils.py b/tests/utils.py index f6ef25c6a3fc..850667f3c858 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -470,6 +470,11 @@ def fund_channel(self, l2, amount): decoded2 = self.bitcoin.rpc.decoderawtransaction(tx) raise ValueError("Can't find {} payment in {} (1={} 2={})".format(amount, tx, decoded, decoded2)) + def subd_pid(self, subd): + """Get the process id of the given subdaemon, eg channeld or gossipd""" + pidline = self.daemon.is_in_log('lightning_{}.*: pid [0-9]*,'.format(subd)) + return re.search(r'pid ([0-9]*),', pidline).group(1) + def is_channel_active(self, chanid): channels = self.rpc.listchannels()['channels'] active = [(c['short_channel_id'], c['flags']) for c in channels if c['active']] From 601a8555437b2adbc9c3011e98f7de5c7c53592d Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Thu, 31 May 2018 12:03:27 +0930 Subject: [PATCH 03/46] tests/test_lightningd.py: test new delayed gossip channel_announce behaviour. The gossip-query spec enhancements say not to forward an channel_announcement until you have receive a channel_update. This test fails for now. Signed-off-by: Rusty Russell --- tests/test_lightningd.py | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/tests/test_lightningd.py b/tests/test_lightningd.py index 21bc85187c66..8b55b1287a60 100644 --- a/tests/test_lightningd.py +++ b/tests/test_lightningd.py @@ -2642,6 +2642,41 @@ def test_routing_gossip_reconnect(self): for n in [l1, l2, l3]: wait_for(lambda: len(n.rpc.listchannels()['channels']) == 4) + @unittest.expectedFailure + @unittest.skipIf(not DEVELOPER, "needs DEVELOPER=1") + def test_gossip_no_empty_announcements(self): + # Need full IO logging so we can see gossip + l1 = self.node_factory.get_node(options={'log-level': 'io'}) + l2 = self.node_factory.get_node(options={'log-level': 'io', + 'dev-no-reconnect': None}) + # l3 sends CHANNEL_ANNOUNCEMENT to l2, but not CHANNEL_UDPATE. + l3 = self.node_factory.get_node(disconnect=['+WIRE_CHANNEL_ANNOUNCEMENT']) + l4 = self.node_factory.get_node() + + # Turn on IO logging for gossipds + subprocess.run(['kill', '-USR1', l1.subd_pid('gossipd')]) + subprocess.run(['kill', '-USR1', l2.subd_pid('gossipd')]) + + l1.rpc.connect(l2.info['id'], 'localhost', l2.port) + l2.rpc.connect(l3.info['id'], 'localhost', l3.port) + l3.rpc.connect(l4.info['id'], 'localhost', l4.port) + + # Make an announced-but-not-updated channel. + self.fund_channel(l3, l4, 10**5) + bitcoind.generate_block(5) + sync_blockheight([l3, l4]) + # 0x0100 = channel_announcement, which goes to l2 before l3 dies. + l2.daemon.wait_for_log('\[IN\] 0100') + + # But it never goes to l1, as there's no channel_update. + time.sleep(2) + assert not l1.daemon.is_in_log('\[IN\] 0100') + assert len(l1.rpc.listchannels()['channels']) == 0 + + # If we reconnect, gossip will now flow. + l3.rpc.connect(l2.info['id'], 'localhost', l2.port) + wait_for(lambda: len(l1.rpc.listchannels()['channels']) == 2) + def test_second_channel(self): l1 = self.node_factory.get_node() l2 = self.node_factory.get_node() From c2189229cad83ffe6f31d5b6fecb4b7510688eb9 Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Thu, 31 May 2018 12:04:27 +0930 Subject: [PATCH 04/46] gossipd: only broadcast channel_announcement once we have a channel_update. Signed-off-by: Rusty Russell --- gossipd/routing.c | 18 ++++++++++++++---- tests/test_lightningd.py | 1 - 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/gossipd/routing.c b/gossipd/routing.c index f86344c74c53..74392bad29ce 100644 --- a/gossipd/routing.c +++ b/gossipd/routing.c @@ -635,10 +635,8 @@ bool routing_add_channel_announcement(struct routing_state *rstate, /* Channel is now public. */ chan->channel_announce = tal_dup_arr(chan, u8, msg, tal_len(msg), 0); - /* Now we can broadcast channel announce */ - insert_broadcast(rstate->broadcasts, chan->channel_announce); - - /* Clear any private updates. */ + /* Clear any private updates: new updates will trigger broadcast of + * this channel_announce. */ for (size_t i = 0; i < ARRAY_SIZE(chan->half); i++) chan->half[i].channel_update = tal_free(chan->half[i].channel_update); @@ -934,6 +932,7 @@ bool routing_add_channel_update(struct routing_state *rstate, struct bitcoin_blkid chain_hash; struct chan *chan; u8 direction; + bool have_broadcast_announce; if (!fromwire_channel_update(update, &signature, &chain_hash, &short_channel_id, ×tamp, &flags, @@ -944,6 +943,10 @@ bool routing_add_channel_update(struct routing_state *rstate, if (!chan) return false; + /* We broadcast announce once we have one update */ + have_broadcast_announce = is_halfchan_defined(&chan->half[0]) + || is_halfchan_defined(&chan->half[1]); + direction = flags & 0x1; set_connection_values(chan, direction, fee_base_msat, fee_proportional_millionths, expiry, @@ -959,6 +962,13 @@ bool routing_add_channel_update(struct routing_state *rstate, if (!chan->channel_announce) return true; + /* BOLT #7: + * - MUST consider whether to send the `channel_announcement` after + * receiving the first corresponding `channel_update`. + */ + if (!have_broadcast_announce) + insert_broadcast(rstate->broadcasts, chan->channel_announce); + insert_broadcast(rstate->broadcasts, chan->half[direction].channel_update); return true; diff --git a/tests/test_lightningd.py b/tests/test_lightningd.py index 8b55b1287a60..1717b93a6889 100644 --- a/tests/test_lightningd.py +++ b/tests/test_lightningd.py @@ -2642,7 +2642,6 @@ def test_routing_gossip_reconnect(self): for n in [l1, l2, l3]: wait_for(lambda: len(n.rpc.listchannels()['channels']) == 4) - @unittest.expectedFailure @unittest.skipIf(not DEVELOPER, "needs DEVELOPER=1") def test_gossip_no_empty_announcements(self): # Need full IO logging so we can see gossip From c2cc3823db7b79d097b583b422580a1b7774ff69 Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Mon, 4 Jun 2018 13:45:25 +0930 Subject: [PATCH 05/46] gossipd: announce own node only after channel announcement actually broadcast. handle_pending_cannouncement might not actually add the announcment, as it could be waiting for a channel_update. We need to wait for the actual announcement before considering announcing our node. Signed-off-by: Rusty Russell --- gossipd/gossip.c | 24 ++++++++++++++++++++++-- gossipd/routing.c | 35 +++++++++++++++++++++++------------ gossipd/routing.h | 9 ++++----- 3 files changed, 49 insertions(+), 19 deletions(-) diff --git a/gossipd/gossip.c b/gossipd/gossip.c index df4f694922ea..7abc1be00c48 100644 --- a/gossipd/gossip.c +++ b/gossipd/gossip.c @@ -623,6 +623,17 @@ static void send_node_announcement(struct daemon *daemon) tal_hex(tmpctx, err)); } +/* Should we announce our own node? */ +static void consider_own_node_announce(struct daemon *daemon) +{ + if (!daemon->rstate->local_channel_announced) + return; + + /* FIXME: We may not need to retransmit here, if previous still valid. */ + send_node_announcement(daemon); + daemon->rstate->local_channel_announced = false; +} + /** * Handle an incoming gossip message * @@ -661,6 +672,8 @@ static u8 *handle_gossip_msg(struct daemon *daemon, const u8 *msg, err = handle_channel_update(rstate, msg, source); if (err) return err; + /* In case we just announced a new local channel. */ + consider_own_node_announce(daemon); break; } @@ -1042,6 +1055,9 @@ static void handle_local_channel_update(struct peer *peer, const u8 *msg) /* We always tell peer, even if it's not public yet */ if (!is_chan_public(chan)) queue_peer_msg(peer, take(cupdate)); + + /* That channel_update might trigger our first channel_announcement */ + consider_own_node_announce(peer->daemon); } /** @@ -2005,6 +2021,10 @@ static struct io_plan *gossip_activate(struct daemon_conn *master, else binding = NULL; + /* Now we know our addresses, re-announce ourselves if we have a + * channel, in case options have changed. */ + consider_own_node_announce(daemon); + /* OK, we're ready! */ daemon_conn_send(&daemon->master, take(towire_gossipctl_activate_reply(NULL, @@ -2562,8 +2582,8 @@ static struct io_plan *handle_txout_reply(struct io_conn *conn, if (!fromwire_gossip_get_txout_reply(msg, msg, &scid, &satoshis, &outscript)) master_badmsg(WIRE_GOSSIP_GET_TXOUT_REPLY, msg); - if (handle_pending_cannouncement(daemon->rstate, &scid, satoshis, outscript)) - send_node_announcement(daemon); + handle_pending_cannouncement(daemon->rstate, &scid, satoshis, outscript); + consider_own_node_announce(daemon); return daemon_conn_read_next(conn, &daemon->master); } diff --git a/gossipd/routing.c b/gossipd/routing.c index 74392bad29ce..1b844869ae8a 100644 --- a/gossipd/routing.c +++ b/gossipd/routing.c @@ -98,6 +98,7 @@ struct routing_state *new_routing_state(const tal_t *ctx, rstate->prune_timeout = prune_timeout; rstate->store = gossip_store_new(rstate); rstate->dev_allow_localhost = dev_allow_localhost; + rstate->local_channel_announced = false; list_head_init(&rstate->pending_cannouncement); uintmap_init(&rstate->chanmap); @@ -604,6 +605,20 @@ static void destroy_pending_cannouncement(struct pending_cannouncement *pending, list_del_from(&rstate->pending_cannouncement, &pending->list); } +static bool is_local_channel(const struct routing_state *rstate, + const struct chan *chan) +{ + return pubkey_eq(&chan->nodes[0]->id, &rstate->local_id) + || pubkey_eq(&chan->nodes[1]->id, &rstate->local_id); +} + +static void add_channel_announce_to_broadcast(struct routing_state *rstate, + struct chan *chan) +{ + insert_broadcast(rstate->broadcasts, chan->channel_announce); + rstate->local_channel_announced |= is_local_channel(rstate, chan); +} + bool routing_add_channel_announcement(struct routing_state *rstate, const u8 *msg TAKES, u64 satoshis) { @@ -797,18 +812,17 @@ static void process_pending_channel_update(struct routing_state *rstate, } } -bool handle_pending_cannouncement(struct routing_state *rstate, +void handle_pending_cannouncement(struct routing_state *rstate, const struct short_channel_id *scid, const u64 satoshis, const u8 *outscript) { - bool local; const u8 *s; struct pending_cannouncement *pending; pending = find_pending_cannouncement(rstate, scid); if (!pending) - return false; + return; /* BOLT #7: * @@ -819,7 +833,7 @@ bool handle_pending_cannouncement(struct routing_state *rstate, type_to_string(pending, struct short_channel_id, scid)); tal_free(pending); - return false; + return; } /* BOLT #7: @@ -841,7 +855,7 @@ bool handle_pending_cannouncement(struct routing_state *rstate, scid), tal_hex(tmpctx, s), tal_hex(tmpctx, outscript)); tal_free(pending); - return false; + return; } if (!routing_add_channel_announcement(rstate, pending->announce, satoshis)) @@ -849,9 +863,6 @@ bool handle_pending_cannouncement(struct routing_state *rstate, "Could not add channel_announcement"); gossip_store_add_channel_announcement(rstate->store, pending->announce, satoshis); - local = pubkey_eq(&pending->node_id_1, &rstate->local_id) || - pubkey_eq(&pending->node_id_2, &rstate->local_id); - /* Did we have an update waiting? If so, apply now. */ process_pending_channel_update(rstate, scid, pending->updates[0]); process_pending_channel_update(rstate, scid, pending->updates[1]); @@ -860,7 +871,6 @@ bool handle_pending_cannouncement(struct routing_state *rstate, process_pending_node_announcement(rstate, &pending->node_id_2); tal_free(pending); - return local; } static void update_pending(struct pending_cannouncement *pending, @@ -967,7 +977,7 @@ bool routing_add_channel_update(struct routing_state *rstate, * receiving the first corresponding `channel_update`. */ if (!have_broadcast_announce) - insert_broadcast(rstate->broadcasts, chan->channel_announce); + add_channel_announce_to_broadcast(rstate, chan); insert_broadcast(rstate->broadcasts, chan->half[direction].channel_update); @@ -1077,11 +1087,12 @@ u8 *handle_channel_update(struct routing_state *rstate, const u8 *update, return err; } - status_trace("Received channel_update for channel %s(%d) now %s", + status_trace("Received channel_update for channel %s(%d) now %s (from %s)", type_to_string(tmpctx, struct short_channel_id, &short_channel_id), flags & 0x01, - flags & ROUTING_FLAGS_DISABLED ? "DISABLED" : "ACTIVE"); + flags & ROUTING_FLAGS_DISABLED ? "DISABLED" : "ACTIVE", + source); if (!routing_add_channel_update(rstate, serialized)) status_failed(STATUS_FAIL_INTERNAL_ERROR, diff --git a/gossipd/routing.h b/gossipd/routing.h index 39b90d54fc42..b557b0339b36 100644 --- a/gossipd/routing.h +++ b/gossipd/routing.h @@ -175,6 +175,9 @@ struct routing_state { /* A map of channels indexed by short_channel_ids */ UINTMAP(struct chan *) chanmap; + + /* Has one of our own channels been announced? */ + bool local_channel_announced; }; static inline struct chan * @@ -217,12 +220,8 @@ u8 *handle_channel_announcement(struct routing_state *rstate, /** * handle_pending_cannouncement -- handle channel_announce once we've * completed short_channel_id lookup. - * - * Returns true if the channel was new and is local. This means that - * if we haven't sent a node_announcement just yet, now would be a - * good time. */ -bool handle_pending_cannouncement(struct routing_state *rstate, +void handle_pending_cannouncement(struct routing_state *rstate, const struct short_channel_id *scid, const u64 satoshis, const u8 *txscript); From 803e4f8895241e9bb457eec7f162fa225f7cec6f Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Mon, 4 Jun 2018 13:46:25 +0930 Subject: [PATCH 06/46] gossipd: announce nodes after channel announcement. In general, we need to only publish node announcements after publishing channel announcements, though we can accept node announcements as soon as we see channel announcements. So we keep a flag for those node_announcement which haven't been broadcast yet. Signed-off-by: Rusty Russell --- gossipd/routing.c | 39 ++++++++++++++++++++++++++++++--------- gossipd/routing.h | 1 + 2 files changed, 31 insertions(+), 9 deletions(-) diff --git a/gossipd/routing.c b/gossipd/routing.c index 1b844869ae8a..168dcf7465d6 100644 --- a/gossipd/routing.c +++ b/gossipd/routing.c @@ -150,6 +150,7 @@ static struct node *new_node(struct routing_state *rstate, n->chans = tal_arr(n, struct chan *, 0); n->alias = NULL; n->node_announcement = NULL; + n->node_announcement_public = false; n->last_timestamp = -1; n->addresses = tal_arr(n, struct wireaddr, 0); node_map_add(rstate->nodes, n); @@ -617,6 +618,18 @@ static void add_channel_announce_to_broadcast(struct routing_state *rstate, { insert_broadcast(rstate->broadcasts, chan->channel_announce); rstate->local_channel_announced |= is_local_channel(rstate, chan); + + /* If we've been waiting for this, now we can announce node */ + for (size_t i = 0; i < ARRAY_SIZE(chan->nodes); i++) { + struct node *node = chan->nodes[i]; + if (!node->node_announcement) + continue; + if (!node->node_announcement_public) { + node->node_announcement_public = true; + insert_broadcast(rstate->broadcasts, + node->node_announcement); + } + } } bool routing_add_channel_announcement(struct routing_state *rstate, @@ -1140,6 +1153,14 @@ static struct wireaddr *read_addresses(const tal_t *ctx, const u8 *ser) return wireaddrs; } +static bool node_has_public_channels(struct node *node) +{ + for (size_t i = 0; i < tal_count(node->chans); i++) + if (is_chan_public(node->chans[i])) + return true; + return false; +} + bool routing_add_node_announcement(struct routing_state *rstate, const u8 *msg TAKES) { struct node *node; @@ -1175,16 +1196,16 @@ bool routing_add_node_announcement(struct routing_state *rstate, const u8 *msg T tal_free(node->node_announcement); node->node_announcement = tal_dup_arr(node, u8, msg, tal_len(msg), 0); - insert_broadcast(rstate->broadcasts, node->node_announcement); - return true; -} -static bool node_has_public_channels(struct node *node) -{ - for (size_t i = 0; i < tal_count(node->chans); i++) - if (is_chan_public(node->chans[i])) - return true; - return false; + /* FIXME: + * When a channel is closed, the node announce may now be out of + * order. It's not vital, but would be nice to fix. + */ + /* We might be waiting for channel_announce to be released. */ + node->node_announcement_public = node_has_public_channels(node); + if (node->node_announcement_public) + insert_broadcast(rstate->broadcasts, node->node_announcement); + return true; } u8 *handle_node_announcement(struct routing_state *rstate, const u8 *node_ann) diff --git a/gossipd/routing.h b/gossipd/routing.h index b557b0339b36..46be560ab69a 100644 --- a/gossipd/routing.h +++ b/gossipd/routing.h @@ -101,6 +101,7 @@ struct node { /* Cached `node_announcement` we might forward to new peers (or NULL). */ const u8 *node_announcement; + bool node_announcement_public; }; const secp256k1_pubkey *node_map_keyof_node(const struct node *n); From 3e07971582bd96c15a954b7eddd72dbe845fb589 Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Mon, 4 Jun 2018 13:47:25 +0930 Subject: [PATCH 07/46] features: define LOCAL_GOSSIP_QUERIES feature. From BOLT #9 proposed update. Signed-off-by: Rusty Russell --- common/features.c | 3 ++- common/features.h | 2 ++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/common/features.c b/common/features.c index 81b42eabc138..3a2c3806b1b5 100644 --- a/common/features.c +++ b/common/features.c @@ -4,7 +4,8 @@ #include static const u32 local_features[] = { - LOCAL_INITIAL_ROUTING_SYNC + LOCAL_INITIAL_ROUTING_SYNC, + LOCAL_GOSSIP_QUERIES }; static const u32 global_features[] = { diff --git a/common/features.h b/common/features.h index 82a9fbdd3d1b..2723939006fe 100644 --- a/common/features.h +++ b/common/features.h @@ -25,9 +25,11 @@ bool feature_offered(const u8 *features, size_t f); * | 0/1 | `option-data-loss-protect` |... * | 3 | `initial_routing_sync` |... * | 4/5 | `option_upfront_shutdown_script` |... + * | 6/7 | `gossip_queries` |... */ #define LOCAL_DATA_LOSS_PROTECT 0 #define LOCAL_INITIAL_ROUTING_SYNC 2 #define LOCAL_UPFRONT_SHUTDOWN_SCRIPT 4 +#define LOCAL_GOSSIP_QUERIES 6 #endif /* LIGHTNING_COMMON_FEATURES_H */ From 6c6da45f531a389a452eeb0f5d4eb9fd0ce6094c Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Mon, 4 Jun 2018 13:48:25 +0930 Subject: [PATCH 08/46] wire: Update to lastest BOLT draft. This includes the gossip query messages. Signed-off-by: Rusty Russell --- channeld/channel.c | 5 +++++ gossipd/gossip.c | 8 ++++++++ wire/gen_peer_wire_csv | 22 ++++++++++++++++++++++ wire/peer_wire.c | 10 ++++++++++ 4 files changed, 45 insertions(+) diff --git a/channeld/channel.c b/channeld/channel.c index b5632d9bfe26..74b360e9a4ba 100644 --- a/channeld/channel.c +++ b/channeld/channel.c @@ -1591,6 +1591,11 @@ static void peer_in(struct peer *peer, const u8 *msg) case WIRE_CHANNEL_ANNOUNCEMENT: case WIRE_CHANNEL_UPDATE: case WIRE_NODE_ANNOUNCEMENT: + case WIRE_QUERY_SHORT_CHANNEL_IDS: + case WIRE_QUERY_CHANNEL_RANGE: + case WIRE_REPLY_CHANNEL_RANGE: + case WIRE_GOSSIP_TIMESTAMP_FILTER: + case WIRE_REPLY_SHORT_CHANNEL_IDS_END: case WIRE_PING: case WIRE_ERROR: abort(); diff --git a/gossipd/gossip.c b/gossipd/gossip.c index 7abc1be00c48..62950fb27efe 100644 --- a/gossipd/gossip.c +++ b/gossipd/gossip.c @@ -810,6 +810,14 @@ static struct io_plan *peer_msgin(struct io_conn *conn, /* This will wait. */ return peer_next_in(conn, peer); + + case WIRE_QUERY_SHORT_CHANNEL_IDS: + case WIRE_REPLY_SHORT_CHANNEL_IDS_END: + case WIRE_QUERY_CHANNEL_RANGE: + case WIRE_REPLY_CHANNEL_RANGE: + case WIRE_GOSSIP_TIMESTAMP_FILTER: + /* FIXME: Implement */ + return peer_next_in(conn, peer); } /* BOLT #1: diff --git a/wire/gen_peer_wire_csv b/wire/gen_peer_wire_csv index 4ac2c04ae974..5c6e81450a2e 100644 --- a/wire/gen_peer_wire_csv +++ b/wire/gen_peer_wire_csv @@ -146,3 +146,25 @@ channel_update,110,cltv_expiry_delta,2 channel_update,112,htlc_minimum_msat,8 channel_update,120,fee_base_msat,4 channel_update,124,fee_proportional_millionths,4 +query_short_channel_ids,261 +query_short_channel_ids,0,chain_hash,32 +query_short_channel_ids,32,len,2 +query_short_channel_ids,34,encoded_short_ids,len +reply_short_channel_ids_end,262 +reply_short_channel_ids_end,0,chain_hash,32 +reply_short_channel_ids_end,32,complete,1 +query_channel_range,263 +query_channel_range,0,chain_hash,32 +query_channel_range,32,first_blocknum,4 +query_channel_range,36,number_of_blocks,4 +reply_channel_range,264 +reply_channel_range,0,chain_hash,32 +reply_channel_range,32,first_blocknum,4 +reply_channel_range,36,number_of_blocks,4 +reply_channel_range,40,complete,1 +reply_channel_range,41,len,2 +reply_channel_range,43,encoded_short_ids,len +gossip_timestamp_filter,265 +gossip_timestamp_filter,0,chain_hash,32 +gossip_timestamp_filter,32,first_timestamp,4 +gossip_timestamp_filter,36,timestamp_range,4 diff --git a/wire/peer_wire.c b/wire/peer_wire.c index de8218c035f9..777b85dc685c 100644 --- a/wire/peer_wire.c +++ b/wire/peer_wire.c @@ -26,6 +26,11 @@ static bool unknown_type(enum wire_type t) case WIRE_CHANNEL_UPDATE: case WIRE_PING: case WIRE_PONG: + case WIRE_QUERY_SHORT_CHANNEL_IDS: + case WIRE_REPLY_SHORT_CHANNEL_IDS_END: + case WIRE_QUERY_CHANNEL_RANGE: + case WIRE_REPLY_CHANNEL_RANGE: + case WIRE_GOSSIP_TIMESTAMP_FILTER: return false; } return true; @@ -37,6 +42,11 @@ bool is_msg_for_gossipd(const u8 *cursor) case WIRE_CHANNEL_ANNOUNCEMENT: case WIRE_NODE_ANNOUNCEMENT: case WIRE_CHANNEL_UPDATE: + case WIRE_QUERY_SHORT_CHANNEL_IDS: + case WIRE_REPLY_SHORT_CHANNEL_IDS_END: + case WIRE_QUERY_CHANNEL_RANGE: + case WIRE_REPLY_CHANNEL_RANGE: + case WIRE_GOSSIP_TIMESTAMP_FILTER: return true; case WIRE_INIT: case WIRE_ERROR: From 5864415d31b6f6e906e65444f55dd6058448916c Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Mon, 4 Jun 2018 13:49:25 +0930 Subject: [PATCH 09/46] gossipd: infrastructure to handle short_channel_id replies. We use the same system as for gossip: we trickle out replies when we're otherwise idle. This is minimal infrastructure: we don't actually process the query_short_channel_ids message yet, nor do we append node announcements. Signed-off-by: Rusty Russell --- gossipd/gossip.c | 62 +++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 61 insertions(+), 1 deletion(-) diff --git a/gossipd/gossip.c b/gossipd/gossip.c index 62950fb27efe..3eeef30faf10 100644 --- a/gossipd/gossip.c +++ b/gossipd/gossip.c @@ -216,6 +216,10 @@ struct peer { /* High water mark for the staggered broadcast */ u64 broadcast_index; + /* Are there outstanding queries on short_channel_ids? */ + const struct short_channel_id *scid_queries; + size_t scid_query_idx; + /* Is it time to continue the staggered broadcast? */ bool gossip_sync; @@ -335,6 +339,8 @@ static struct peer *new_peer(const tal_t *ctx, peer->daemon = daemon; peer->local = new_local_peer_state(peer, cs); peer->remote = NULL; + peer->scid_queries = NULL; + peer->scid_query_idx = 0; return peer; } @@ -857,10 +863,58 @@ static void wake_pkt_out(struct peer *peer) /* Mutual recursion. */ static struct io_plan *peer_pkt_out(struct io_conn *conn, struct peer *peer); +static bool create_next_scid_reply(struct peer *peer) +{ + struct routing_state *rstate = peer->daemon->rstate; + size_t i, num; + bool sent = false; + + /* BOLT #7: + * + * - SHOULD respond to each known `short_channel_id` with a + * `channel_announce` and the latest `channel_update`s for + * each end + * + * - SHOULD NOT wait for the next outgoing announcement flush + * to send these. + */ + num = tal_count(peer->scid_queries); + for (i = peer->scid_query_idx; !sent && i < num; i++) { + struct chan *chan; + + chan = get_channel(rstate, &peer->scid_queries[i]); + if (!chan || is_chan_public(chan)) + continue; + + queue_peer_msg(peer, chan->channel_announce); + if (chan->half[0].channel_update) + queue_peer_msg(peer, chan->half[0].channel_update); + if (chan->half[1].channel_update) + queue_peer_msg(peer, chan->half[0].channel_update); + sent = true; + } + peer->scid_query_idx = i; + + /* All finished? */ + if (peer->scid_queries && peer->scid_query_idx == num) { + /* FIXME: Send node_announcements now */ + u8 *end = towire_reply_short_channel_ids_end(peer, + &rstate->chain_hash, + true); + queue_peer_msg(peer, take(end)); + peer->scid_queries = tal_free(peer->scid_queries); + } + + return sent; +} + static struct io_plan *peer_pkt_out(struct io_conn *conn, struct peer *peer) { /* First priority is queued packets, if any */ - const u8 *out = msg_dequeue(&peer->local->peer_out); + const u8 *out; + +again: + out = msg_dequeue(&peer->local->peer_out); if (out) { if (is_all_channel_error(out)) return peer_write_message(conn, &peer->local->pcs, @@ -874,6 +928,8 @@ static struct io_plan *peer_pkt_out(struct io_conn *conn, struct peer *peer) if (peer->local->return_to_master) { if (!peer_in_started(conn, &peer->local->pcs)) return ready_for_master(conn, peer); + } else if (create_next_scid_reply(peer)) { + goto again; } else if (peer->gossip_sync) { /* If we're supposed to be sending gossip, do so now. */ const u8 *next; @@ -1159,6 +1215,10 @@ static bool nonlocal_dump_gossip(struct io_conn *conn, struct daemon_conn *dc) /* Make sure we are not connected directly */ assert(!peer->local); + /* Do we have scid query replies to send? */ + if (create_next_scid_reply(peer)) + return true; + /* Nothing to do if we're not gossiping */ if (!peer->gossip_sync) return false; From 32c39c297973d47186fd12224918e2fd8a860127 Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Mon, 4 Jun 2018 13:50:25 +0930 Subject: [PATCH 10/46] gossipd: send node announcements after short_channel_id replies. We use the same system as for gossip: we trickle out replies when we're otherwise idle. As we trickle out replies to query_short_channel_ids, we remember the pubkeys of nodes we mention. At the end, we sort and uniquify, and then send any node_announcements we have for those. Signed-off-by: Rusty Russell --- gossipd/gossip.c | 96 +++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 94 insertions(+), 2 deletions(-) diff --git a/gossipd/gossip.c b/gossipd/gossip.c index 3eeef30faf10..d157b122e4b9 100644 --- a/gossipd/gossip.c +++ b/gossipd/gossip.c @@ -1,3 +1,4 @@ +#include #include #include #include @@ -220,6 +221,10 @@ struct peer { const struct short_channel_id *scid_queries; size_t scid_query_idx; + /* Are there outstanding node_announcements from scid_queries? */ + struct pubkey *scid_query_nodes; + size_t scid_query_nodes_idx; + /* Is it time to continue the staggered broadcast? */ bool gossip_sync; @@ -341,6 +346,8 @@ static struct peer *new_peer(const tal_t *ctx, peer->remote = NULL; peer->scid_queries = NULL; peer->scid_query_idx = 0; + peer->scid_query_nodes = NULL; + peer->scid_query_nodes_idx = 0; return peer; } @@ -863,6 +870,51 @@ static void wake_pkt_out(struct peer *peer) /* Mutual recursion. */ static struct io_plan *peer_pkt_out(struct io_conn *conn, struct peer *peer); +/* We keep a simple array of node ids while we're sending channel info */ +static void append_query_node(struct peer *peer, const struct pubkey *id) +{ + size_t n; + n = tal_count(peer->scid_query_nodes); + tal_resize(&peer->scid_query_nodes, n+1); + peer->scid_query_nodes[n] = *id; +} + +/* Arbitrary ordering function of pubkeys. + * + * Note that we could use memcmp() here: even if they had somehow different + * bitwise representations for the same key, we copied them all from struct + * node which should make them unique. Even if not (say, a node vanished + * and reappeared) we'd just end up sending two node_announcement for the + * same node. + */ +static int pubkey_order(const struct pubkey *k1, const struct pubkey *k2, + void *unused UNUSED) +{ + return pubkey_cmp(k1, k2); +} + +static void uniquify_node_ids(struct pubkey **ids) +{ + size_t dst, src; + + /* BOLT #7: + * + * - MUST follow with any `node_announcement`s for each + * `channel_announcement` + * + * - SHOULD avoid sending duplicate `node_announcements` in + * response to a single `query_short_channel_ids`. + */ + asort(*ids, tal_count(*ids), pubkey_order, NULL); + + for (dst = 0, src = 0; src < tal_count(*ids); src++) { + if (dst && pubkey_eq(&(*ids)[dst-1], &(*ids)[src])) + continue; + (*ids)[dst++] = (*ids)[src]; + } + tal_resize(ids, dst); +} + static bool create_next_scid_reply(struct peer *peer) { struct routing_state *rstate = peer->daemon->rstate; @@ -891,18 +943,58 @@ static bool create_next_scid_reply(struct peer *peer) queue_peer_msg(peer, chan->half[0].channel_update); if (chan->half[1].channel_update) queue_peer_msg(peer, chan->half[0].channel_update); + + /* Record node ids for later transmission of node_announcement */ + append_query_node(peer, &chan->nodes[0]->id); + append_query_node(peer, &chan->nodes[1]->id); sent = true; } + + /* Just finished channels? Remove duplicate nodes. */ + if (peer->scid_query_idx != num && i == num) + uniquify_node_ids(&peer->scid_query_nodes); peer->scid_query_idx = i; + /* BOLT #7: + * + * - MUST follow with any `node_announcement`s for each + * `channel_announcement` + * - SHOULD avoid sending duplicate `node_announcements` in response + * to a single `query_short_channel_ids`. + */ + num = tal_count(peer->scid_query_nodes); + for (i = peer->scid_query_nodes_idx; !sent && i < num; i++) { + const struct node *n; + + n = get_node(rstate, &peer->scid_query_nodes[i]); + if (!n || !n->node_announcement || !n->node_announcement_public) + continue; + + queue_peer_msg(peer, n->node_announcement); + sent = true; + } + peer->scid_query_nodes_idx = i; + /* All finished? */ - if (peer->scid_queries && peer->scid_query_idx == num) { - /* FIXME: Send node_announcements now */ + if (peer->scid_queries && peer->scid_query_nodes_idx == num) { + /* BOLT #7: + * + * - MUST follow these responses with + * `reply_short_channel_ids_end`. + * - if does not maintain up-to-date channel information for + * `chain_hash`: + * - MUST set `complete` to 0. + * - otherwise: + * - SHOULD set `complete` to 1. + */ u8 *end = towire_reply_short_channel_ids_end(peer, &rstate->chain_hash, true); queue_peer_msg(peer, take(end)); peer->scid_queries = tal_free(peer->scid_queries); + peer->scid_query_idx = 0; + peer->scid_query_nodes = tal_free(peer->scid_query_nodes); + peer->scid_query_nodes_idx = 0; } return sent; From 7ee5da858c176f6fb3fda5acbc191d32a6cda2c5 Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Mon, 4 Jun 2018 13:51:25 +0930 Subject: [PATCH 11/46] gossipd: handle query_short_channel_ids message. This doesn't handle zlib yet. Signed-off-by: Rusty Russell --- gossipd/gossip.c | 107 ++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 105 insertions(+), 2 deletions(-) diff --git a/gossipd/gossip.c b/gossipd/gossip.c index d157b122e4b9..6ab24a7efdc2 100644 --- a/gossipd/gossip.c +++ b/gossipd/gossip.c @@ -694,6 +694,105 @@ static u8 *handle_gossip_msg(struct daemon *daemon, const u8 *msg, return NULL; } +/* BOLT #7: + * + * the first byte indicates the encoding, the rest contains the data. + * + * Encoding types: + * * `0`: uncompressed array of `short_channel_id` types, in ascending order. + */ +static struct short_channel_id *decode_short_ids(const tal_t *ctx, + const u8 *encoded) +{ + struct short_channel_id *scids; + size_t max = tal_len(encoded), n; + u8 type; + + /* BOLT #7: + * + * The receiver: + * - if the first byte of `encoded_short_ids` is not zero: + * - MAY fail the connection + * - if `encoded_short_ids` does not decode into a whole number of + * `short_channel_id`: + * - MAY fail the connection + */ + type = fromwire_u8(&encoded, &max); + if (type != 0) + return NULL; + + n = 0; + scids = tal_arr(ctx, struct short_channel_id, n); + while (max) { + tal_resize(&scids, n+1); + fromwire_short_channel_id(&encoded, &max, &scids[n++]); + } + + /* encoded is set to NULL if we ran over */ + if (!encoded) + return tal_free(scids); + return scids; +} + +static void handle_query_short_channel_ids(struct peer *peer, u8 *msg) +{ + struct routing_state *rstate =peer->daemon->rstate; + struct bitcoin_blkid chain; + u8 *encoded; + struct short_channel_id *scids; + + if (!fromwire_query_short_channel_ids(tmpctx, msg, &chain, &encoded)) { + peer_error(peer, "Bad query_short_channel_ids %s", + tal_hex(tmpctx, msg)); + return; + } + + if (!structeq(&rstate->chain_hash, &chain)) { + status_trace("%s sent query_short_channel_ids chainhash %s", + type_to_string(tmpctx, struct pubkey, &peer->id), + type_to_string(tmpctx, struct bitcoin_blkid, &chain)); + return; + } + + /* BOLT #7: + * + * - if it has not sent `reply_short_channel_ids_end` to a + * previously received `query_short_channel_ids` from this + * sender: + * - MAY fail the connection. + */ + if (peer->scid_queries || peer->scid_query_nodes) { + peer_error(peer, "Bad second query_short_channel_ids"); + return; + } + + scids = decode_short_ids(tmpctx, encoded); + if (!scids) { + peer_error(peer, "Bad query_short_channel_ids encoding %s", + tal_hex(tmpctx, encoded)); + return; + } + + /* BOLT #7: + * + * - SHOULD respond to each known `short_channel_id` with a + * `channel_announce` and the latest `channel_update`s for each end + * - SHOULD NOT wait for the next outgoing announcement flush to send + * these. + */ + peer->scid_queries = tal_steal(peer, scids); + peer->scid_query_idx = 0; + peer->scid_query_nodes = tal_arr(peer, struct pubkey, 0); + + /* Wake writer. */ + if (peer->local) + /* Notify the peer-write loop */ + msg_wake(&peer->local->peer_out); + else + /* Notify the daemon_conn-write loop */ + msg_wake(&peer->remote->out); +} + static void handle_ping(struct peer *peer, u8 *ping) { u8 *pong; @@ -800,6 +899,10 @@ static struct io_plan *peer_msgin(struct io_conn *conn, handle_pong(peer, msg); return peer_next_in(conn, peer); + case WIRE_QUERY_SHORT_CHANNEL_IDS: + handle_query_short_channel_ids(peer, msg); + return peer_next_in(conn, peer); + case WIRE_OPEN_CHANNEL: case WIRE_CHANNEL_REESTABLISH: case WIRE_ACCEPT_CHANNEL: @@ -824,7 +927,6 @@ static struct io_plan *peer_msgin(struct io_conn *conn, /* This will wait. */ return peer_next_in(conn, peer); - case WIRE_QUERY_SHORT_CHANNEL_IDS: case WIRE_REPLY_SHORT_CHANNEL_IDS_END: case WIRE_QUERY_CHANNEL_RANGE: case WIRE_REPLY_CHANNEL_RANGE: @@ -1232,7 +1334,8 @@ static struct io_plan *owner_msg_in(struct io_conn *conn, err = handle_gossip_msg(peer->daemon, dc->msg_in, "subdaemon"); if (err) queue_peer_msg(peer, take(err)); - + } else if (type == WIRE_QUERY_SHORT_CHANNEL_IDS) { + handle_query_short_channel_ids(peer, dc->msg_in); } else if (type == WIRE_GOSSIP_GET_UPDATE) { handle_get_update(peer, dc->msg_in); } else if (type == WIRE_GOSSIP_LOCAL_ADD_CHANNEL) { From 4d8b29089b047d2014776ce8a4f03114c916b4a1 Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Mon, 4 Jun 2018 13:52:25 +0930 Subject: [PATCH 12/46] gossipd: wire up infrastructure to generate query_short_channel_ids msg. Signed-off-by: Rusty Russell --- gossipd/gossip.c | 143 ++++++++++++++++++++++++++++++++++-- gossipd/gossip_wire.csv | 11 +++ lightningd/gossip_control.c | 2 + 3 files changed, 151 insertions(+), 5 deletions(-) diff --git a/gossipd/gossip.c b/gossipd/gossip.c index 6ab24a7efdc2..31b9e5f9420f 100644 --- a/gossipd/gossip.c +++ b/gossipd/gossip.c @@ -228,6 +228,9 @@ struct peer { /* Is it time to continue the staggered broadcast? */ bool gossip_sync; + /* How many query responses are we expecting? */ + size_t num_scid_queries_outstanding; + /* Only one of these is set: */ struct local_peer_state *local; struct daemon_conn *remote; @@ -348,6 +351,7 @@ static struct peer *new_peer(const tal_t *ctx, peer->scid_query_idx = 0; peer->scid_query_nodes = NULL; peer->scid_query_nodes_idx = 0; + peer->num_scid_queries_outstanding = 0; return peer; } @@ -403,6 +407,30 @@ static void reached_peer(struct peer *peer, struct io_conn *conn) tal_free(r); } +static u8 *encode_short_channel_ids_start(const tal_t *ctx) +{ + /* BOLT #7: + * + * Encoding types: + * * `0`: uncompressed array of `short_channel_id` types, in ascending + * order. + */ + u8 *encoded = tal_arr(tmpctx, u8, 0); + towire_u8(&encoded, 0); + return encoded; +} + +static void encode_add_short_channel_id(u8 **encoded, + const struct short_channel_id *scid) +{ + towire_short_channel_id(encoded, scid); +} + +static bool encode_short_channel_ids_end(u8 **encoded, size_t max_bytes) +{ + return tal_len(*encoded) <= max_bytes; +} + static void queue_peer_msg(struct peer *peer, const u8 *msg TAKES) { if (peer->local) { @@ -820,6 +848,34 @@ static void handle_pong(struct peer *peer, const u8 *pong) tal_len(pong)))); } +static void handle_reply_short_channel_ids_end(struct peer *peer, u8 *msg) +{ + struct bitcoin_blkid chain; + u8 complete; + + if (!fromwire_reply_short_channel_ids_end(msg, &chain, &complete)) { + peer_error(peer, "Bad reply_short_channel_ids_end %s", + tal_hex(tmpctx, msg)); + return; + } + + if (!structeq(&peer->daemon->rstate->chain_hash, &chain)) { + peer_error(peer, "reply_short_channel_ids_end for bad chain: %s", + tal_hex(tmpctx, msg)); + return; + } + + if (peer->num_scid_queries_outstanding == 0) { + peer_error(peer, "unexpected reply_short_channel_ids_end: %s", + tal_hex(tmpctx, msg)); + return; + } + + peer->num_scid_queries_outstanding--; + msg = towire_gossip_scids_reply(msg, true, complete); + daemon_conn_send(&peer->daemon->master, take(msg)); +} + /* If master asks us to release peer, we attach this destructor in case it * dies while we're waiting for it to finish IO */ static void fail_release(struct peer *peer) @@ -903,6 +959,10 @@ static struct io_plan *peer_msgin(struct io_conn *conn, handle_query_short_channel_ids(peer, msg); return peer_next_in(conn, peer); + case WIRE_REPLY_SHORT_CHANNEL_IDS_END: + handle_reply_short_channel_ids_end(peer, msg); + return peer_next_in(conn, peer); + case WIRE_OPEN_CHANNEL: case WIRE_CHANNEL_REESTABLISH: case WIRE_ACCEPT_CHANNEL: @@ -927,7 +987,6 @@ static struct io_plan *peer_msgin(struct io_conn *conn, /* This will wait. */ return peer_next_in(conn, peer); - case WIRE_REPLY_SHORT_CHANNEL_IDS_END: case WIRE_QUERY_CHANNEL_RANGE: case WIRE_REPLY_CHANNEL_RANGE: case WIRE_GOSSIP_TIMESTAMP_FILTER: @@ -1037,7 +1096,7 @@ static bool create_next_scid_reply(struct peer *peer) struct chan *chan; chan = get_channel(rstate, &peer->scid_queries[i]); - if (!chan || is_chan_public(chan)) + if (!chan || !is_chan_public(chan)) continue; queue_peer_msg(peer, chan->channel_announce); @@ -1336,6 +1395,8 @@ static struct io_plan *owner_msg_in(struct io_conn *conn, queue_peer_msg(peer, take(err)); } else if (type == WIRE_QUERY_SHORT_CHANNEL_IDS) { handle_query_short_channel_ids(peer, dc->msg_in); + } else if (type == WIRE_REPLY_SHORT_CHANNEL_IDS_END) { + handle_reply_short_channel_ids_end(peer, dc->msg_in); } else if (type == WIRE_GOSSIP_GET_UPDATE) { handle_get_update(peer, dc->msg_in); } else if (type == WIRE_GOSSIP_LOCAL_ADD_CHANNEL) { @@ -1751,6 +1812,7 @@ static struct io_plan *getnodes(struct io_conn *conn, struct daemon *daemon, return daemon_conn_read_next(conn, &daemon->master); } +#if DEVELOPER static struct io_plan *ping_req(struct io_conn *conn, struct daemon *daemon, const u8 *msg) { @@ -1793,6 +1855,67 @@ static struct io_plan *ping_req(struct io_conn *conn, struct daemon *daemon, return daemon_conn_read_next(conn, &daemon->master); } +static struct io_plan *query_scids_req(struct io_conn *conn, + struct daemon *daemon, + const u8 *msg) +{ + struct pubkey id; + struct short_channel_id *scids; + struct peer *peer; + u8 *encoded; + /* BOLT #7: + * + * 1. type: 261 (`query_short_channel_ids`) (`gossip_queries`) + * 2. data: + * * [`32`:`chain_hash`] + * * [`2`:`len`] + * * [`len`:`encoded_short_ids`] + */ + const size_t reply_overhead = 32 + 2; + const size_t max_encoded_bytes = 65535 - 2 - reply_overhead; + + if (!fromwire_gossip_query_scids(msg, msg, &id, &scids)) + master_badmsg(WIRE_GOSSIP_QUERY_SCIDS, msg); + + peer = find_peer(daemon, &id); + if (!peer) { + status_broken("query_scids: unknown peer %s", + type_to_string(tmpctx, struct pubkey, &id)); + goto fail; + } + + if (!feature_offered(peer->lfeatures, LOCAL_GOSSIP_QUERIES)) { + status_broken("query_scids: no gossip_query support in peer %s", + type_to_string(tmpctx, struct pubkey, &id)); + goto fail; + } + + encoded = encode_short_channel_ids_start(tmpctx); + for (size_t i = 0; i < tal_count(scids); i++) + encode_add_short_channel_id(&encoded, &scids[i]); + + if (!encode_short_channel_ids_end(&encoded, max_encoded_bytes)) { + status_broken("query_short_channel_ids: %zu is too many", + tal_count(scids)); + goto fail; + } + + msg = towire_query_short_channel_ids(NULL, &daemon->rstate->chain_hash, + encoded); + queue_peer_msg(peer, take(msg)); + peer->num_scid_queries_outstanding++; + + status_trace("sending query for %zu scids", tal_count(scids)); +out: + return daemon_conn_read_next(conn, &daemon->master); + +fail: + daemon_conn_send(&daemon->master, + take(towire_gossip_scids_reply(NULL, false, false))); + goto out; +} +#endif /* DEVELOPER */ + static int make_listen_fd(int domain, void *addr, socklen_t len, bool mayfail) { int fd = socket(domain, SOCK_STREAM, 0); @@ -2966,9 +3089,6 @@ static struct io_plan *recv_req(struct io_conn *conn, struct daemon_conn *master case WIRE_GOSSIP_GETCHANNELS_REQUEST: return getchannels_req(conn, daemon, daemon->master.msg_in); - case WIRE_GOSSIP_PING: - return ping_req(conn, daemon, daemon->master.msg_in); - case WIRE_GOSSIP_RESOLVE_CHANNEL_REQUEST: return resolve_channel_req(conn, daemon, daemon->master.msg_in); @@ -3008,6 +3128,18 @@ static struct io_plan *recv_req(struct io_conn *conn, struct daemon_conn *master case WIRE_GOSSIP_LOCAL_CHANNEL_CLOSE: return handle_local_channel_close(conn, daemon, master->msg_in); +#if DEVELOPER + case WIRE_GOSSIP_PING: + return ping_req(conn, daemon, daemon->master.msg_in); + + case WIRE_GOSSIP_QUERY_SCIDS: + return query_scids_req(conn, daemon, daemon->master.msg_in); +#else + case WIRE_GOSSIP_PING: + case WIRE_GOSSIP_QUERY_SCIDS: + break; +#endif /* !DEVELOPER */ + /* We send these, we don't receive them */ case WIRE_GOSSIPCTL_ACTIVATE_REPLY: case WIRE_GOSSIPCTL_RELEASE_PEER_REPLY: @@ -3017,6 +3149,7 @@ static struct io_plan *recv_req(struct io_conn *conn, struct daemon_conn *master case WIRE_GOSSIP_GETCHANNELS_REPLY: case WIRE_GOSSIP_GETPEERS_REPLY: case WIRE_GOSSIP_PING_REPLY: + case WIRE_GOSSIP_SCIDS_REPLY: case WIRE_GOSSIP_RESOLVE_CHANNEL_REPLY: case WIRE_GOSSIP_PEER_CONNECTED: case WIRE_GOSSIPCTL_CONNECT_TO_PEER_RESULT: diff --git a/gossipd/gossip_wire.csv b/gossipd/gossip_wire.csv index 53acd72dfec1..c5805067b18e 100644 --- a/gossipd/gossip_wire.csv +++ b/gossipd/gossip_wire.csv @@ -155,6 +155,17 @@ gossip_ping_reply,,sent,bool # 0 == no pong expected gossip_ping_reply,,totlen,u16 +# Test of query_short_channel_ids. Master->gossipd +gossip_query_scids,3031 +gossip_query_scids,,id,struct pubkey +gossip_query_scids,,num_ids,u16 +gossip_query_scids,,ids,num_ids*struct short_channel_id + +# Gossipd -> master +gossip_scids_reply,3131 +gossip_scids_reply,,ok,bool +gossip_scids_reply,,complete,bool + # Given a short_channel_id, return the endpoints gossip_resolve_channel_request,3009 gossip_resolve_channel_request,,channel_id,struct short_channel_id diff --git a/lightningd/gossip_control.c b/lightningd/gossip_control.c index 39e553bd4d44..1cecba96fbc8 100644 --- a/lightningd/gossip_control.c +++ b/lightningd/gossip_control.c @@ -136,6 +136,7 @@ static unsigned gossip_msg(struct subd *gossip, const u8 *msg, const int *fds) case WIRE_GOSSIP_OUTPOINT_SPENT: case WIRE_GOSSIP_ROUTING_FAILURE: case WIRE_GOSSIP_MARK_CHANNEL_UNROUTABLE: + case WIRE_GOSSIP_QUERY_SCIDS: case WIRE_GOSSIPCTL_PEER_DISCONNECT: case WIRE_GOSSIPCTL_PEER_IMPORTANT: case WIRE_GOSSIPCTL_PEER_DISCONNECTED: @@ -147,6 +148,7 @@ static unsigned gossip_msg(struct subd *gossip, const u8 *msg, const int *fds) case WIRE_GOSSIP_GETCHANNELS_REPLY: case WIRE_GOSSIP_GETPEERS_REPLY: case WIRE_GOSSIP_PING_REPLY: + case WIRE_GOSSIP_SCIDS_REPLY: case WIRE_GOSSIP_RESOLVE_CHANNEL_REPLY: case WIRE_GOSSIPCTL_RELEASE_PEER_REPLY: case WIRE_GOSSIPCTL_RELEASE_PEER_REPLYFAIL: From c633cbe2ee82f725710baf865d3940aabd60306c Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Mon, 4 Jun 2018 13:53:25 +0930 Subject: [PATCH 13/46] tests: add dev-query-scids And write the test for it. Signed-off-by: Rusty Russell --- contrib/pylightning/lightning/lightning.py | 10 +++ lightningd/gossip_control.c | 85 ++++++++++++++++++++++ tests/test_lightningd.py | 59 +++++++++++++++ 3 files changed, 154 insertions(+) diff --git a/contrib/pylightning/lightning/lightning.py b/contrib/pylightning/lightning/lightning.py index f8d87b7edd57..049bc6944192 100644 --- a/contrib/pylightning/lightning/lightning.py +++ b/contrib/pylightning/lightning/lightning.py @@ -249,6 +249,16 @@ def dev_crash(self): """ return self.call("dev-crash") + def dev_query_scids(self, id, scids): + """ + Ask peer for a particular set of scids + """ + payload = { + "id": id, + "scids": scids + } + return self.call("dev-query-scids", payload) + def getinfo(self): """ Show information about this node diff --git a/lightningd/gossip_control.c b/lightningd/gossip_control.c index 1cecba96fbc8..aa68a86298ef 100644 --- a/lightningd/gossip_control.c +++ b/lightningd/gossip_control.c @@ -545,3 +545,88 @@ static const struct json_command listchannels_command = { "Show channel {short_channel_id} (or all known channels, if no {short_channel_id})" }; AUTODATA(json_command, &listchannels_command); + +#if DEVELOPER +static void json_scids_reply(struct subd *gossip UNUSED, const u8 *reply, + const int *fds UNUSED, struct command *cmd) +{ + bool ok, complete; + struct json_result *response = new_json_result(cmd); + + if (!fromwire_gossip_scids_reply(reply, &ok, &complete)) { + command_fail(cmd, LIGHTNINGD, + "Gossip gave bad gossip_scids_reply"); + return; + } + + if (!ok) { + command_fail(cmd, LIGHTNINGD, + "Gossip refused to query peer"); + return; + } + + json_object_start(response, NULL); + json_add_bool(response, "complete", complete); + json_object_end(response); + command_success(cmd, response); +} + +static void json_dev_query_scids(struct command *cmd, + const char *buffer, const jsmntok_t *params) +{ + u8 *msg; + jsmntok_t *idtok, *scidstok; + const jsmntok_t *t, *end; + struct pubkey id; + struct short_channel_id *scids; + size_t i; + + if (!json_get_params(cmd, buffer, params, + "id", &idtok, + "scids", &scidstok, + NULL)) { + return; + } + + if (!json_tok_pubkey(buffer, idtok, &id)) { + command_fail(cmd, JSONRPC2_INVALID_PARAMS, + "'%.*s' is not a valid id", + idtok->end - idtok->start, + buffer + idtok->start); + return; + } + + if (scidstok->type != JSMN_ARRAY) { + command_fail(cmd, JSONRPC2_INVALID_PARAMS, + "'%.*s' is not an array", + scidstok->end - scidstok->start, + buffer + scidstok->start); + return; + } + + scids = tal_arr(cmd, struct short_channel_id, scidstok->size); + end = json_next(scidstok); + for (i = 0, t = scidstok + 1; t < end; t = json_next(t), i++) { + if (!json_tok_short_channel_id(buffer, t, &scids[i])) { + command_fail(cmd, JSONRPC2_INVALID_PARAMS, + "scid %zu '%.*s' is not an scid", + i, t->end - t->start, + buffer + t->start); + return; + } + } + + /* Tell gossipd, since this is a gossip query. */ + msg = towire_gossip_query_scids(cmd, &id, scids); + subd_req(cmd->ld->gossip, cmd->ld->gossip, + take(msg), -1, 0, json_scids_reply, cmd); + command_still_pending(cmd); +} + +static const struct json_command dev_query_scids_command = { + "dev-query-scids", + json_dev_query_scids, + "Query {peerid} for [scids]" +}; +AUTODATA(json_command, &dev_query_scids_command); +#endif /* DEVELOPER */ diff --git a/tests/test_lightningd.py b/tests/test_lightningd.py index 1717b93a6889..4cc80459e850 100644 --- a/tests/test_lightningd.py +++ b/tests/test_lightningd.py @@ -2622,6 +2622,65 @@ def test_ping(self): l1.daemon.wait_for_log('Got pong 1000 bytes \({}\.\.\.\)' .format(l2.info['version'])) + @unittest.skipIf(not DEVELOPER, "needs DEVELOPER=1") + def test_query_short_channel_id(self): + l1 = self.node_factory.get_node(options={'log-level': 'io'}) + l2 = self.node_factory.get_node() + l3 = self.node_factory.get_node() + l1.rpc.connect(l2.info['id'], 'localhost', l2.port) + l2.rpc.connect(l3.info['id'], 'localhost', l3.port) + + # Need full IO logging so we can see gossip (from gossipd and channeld) + subprocess.run(['kill', '-USR1', l1.subd_pid('gossipd')]) + + # Empty result tests. + reply = l1.rpc.dev_query_scids(l2.info['id'], ['1:1:1', '2:2:2']) + # 0x0105 = query_short_channel_ids + l1.daemon.wait_for_log('\[OUT\] 0105.*0000000100000100010000020000020002') + assert reply['complete'] + + # Make channels public. + scid12 = self.fund_channel(l1, l2, 10**5) + scid23 = self.fund_channel(l2, l3, 10**5) + bitcoind.generate_block(5) + sync_blockheight([l1, l2, l3]) + + # It will know about everything. + l1.daemon.wait_for_log('Received node_announcement for node {}'.format(l3.info['id'])) + subprocess.run(['kill', '-USR1', l1.subd_pid('channeld')]) + + # This query should get channel announcements, channel updates, and node announcements. + reply = l1.rpc.dev_query_scids(l2.info['id'], [scid23]) + # 0x0105 = query_short_channel_ids + l1.daemon.wait_for_log('\[OUT\] 0105') + assert reply['complete'] + + # 0x0100 = channel_announcement + l1.daemon.wait_for_log('\[IN\] 0100') + # 0x0102 = channel_update + l1.daemon.wait_for_log('\[IN\] 0102') + l1.daemon.wait_for_log('\[IN\] 0102') + # 0x0101 = node_announcement + l1.daemon.wait_for_log('\[IN\] 0101') + l1.daemon.wait_for_log('\[IN\] 0101') + + reply = l1.rpc.dev_query_scids(l2.info['id'], [scid12, scid23]) + assert reply['complete'] + # Technically, this order could be different, but this matches code. + # 0x0100 = channel_announcement + l1.daemon.wait_for_log('\[IN\] 0100') + # 0x0102 = channel_update + l1.daemon.wait_for_log('\[IN\] 0102') + l1.daemon.wait_for_log('\[IN\] 0102') + # 0x0100 = channel_announcement + l1.daemon.wait_for_log('\[IN\] 0100') + # 0x0102 = channel_update + l1.daemon.wait_for_log('\[IN\] 0102') + l1.daemon.wait_for_log('\[IN\] 0102') + # 0x0101 = node_announcement + l1.daemon.wait_for_log('\[IN\] 0101') + l1.daemon.wait_for_log('\[IN\] 0101') + @unittest.skipIf(not DEVELOPER, "needs DEVELOPER=1") def test_routing_gossip_reconnect(self): # Connect two peers, reconnect and then see if we resume the From 7a32637b5fdc3e41d4385878a2fdaef93b29ad5c Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Mon, 4 Jun 2018 13:54:25 +0930 Subject: [PATCH 14/46] gossipd: add timestamp to each broadcast message. This lets us filter by timestamp. Signed-off-by: Rusty Russell --- gossipd/broadcast.c | 12 ++++++++++-- gossipd/broadcast.h | 3 ++- gossipd/routing.c | 18 ++++++++++++------ gossipd/test/run-bench-find_route.c | 3 ++- gossipd/test/run-find_route-specific.c | 3 ++- gossipd/test/run-find_route.c | 3 ++- 6 files changed, 30 insertions(+), 12 deletions(-) diff --git a/gossipd/broadcast.c b/gossipd/broadcast.c index 92b083e99bb2..196772c4f077 100644 --- a/gossipd/broadcast.c +++ b/gossipd/broadcast.c @@ -5,6 +5,9 @@ struct queued_message { /* Broadcast index. */ u64 index; + /* Timestamp, for filtering. */ + u32 timestamp; + /* Serialized payload */ const u8 *payload; }; @@ -27,20 +30,25 @@ static void destroy_queued_message(struct queued_message *msg, static struct queued_message *new_queued_message(const tal_t *ctx, struct broadcast_state *bstate, const u8 *payload, + u32 timestamp, u64 index) { struct queued_message *msg = tal(ctx, struct queued_message); + assert(payload); msg->payload = payload; msg->index = index; + msg->timestamp = timestamp; uintmap_add(&bstate->broadcasts, index, msg); tal_add_destructor2(msg, destroy_queued_message, bstate); return msg; } -void insert_broadcast(struct broadcast_state *bstate, const u8 *payload) +void insert_broadcast(struct broadcast_state *bstate, + const u8 *payload, u32 timestamp) { /* Free payload, free index. */ - new_queued_message(payload, bstate, payload, bstate->next_index++); + new_queued_message(payload, bstate, payload, timestamp, + bstate->next_index++); } const u8 *next_broadcast(struct broadcast_state *bstate, u64 *last_index) diff --git a/gossipd/broadcast.h b/gossipd/broadcast.h index b88f8bb97a3f..e728b681fd38 100644 --- a/gossipd/broadcast.h +++ b/gossipd/broadcast.h @@ -17,7 +17,8 @@ struct broadcast_state { struct broadcast_state *new_broadcast_state(tal_t *ctx); /* Append a queued message for broadcast. Freeing the msg will remove it. */ -void insert_broadcast(struct broadcast_state *bstate, const u8 *msg); +void insert_broadcast(struct broadcast_state *bstate, const u8 *msg, + u32 timestamp); /* Return the broadcast with index >= *last_index, and update *last_index. * There's no broadcast with index 0. */ diff --git a/gossipd/routing.c b/gossipd/routing.c index 168dcf7465d6..0926736f9b61 100644 --- a/gossipd/routing.c +++ b/gossipd/routing.c @@ -614,9 +614,10 @@ static bool is_local_channel(const struct routing_state *rstate, } static void add_channel_announce_to_broadcast(struct routing_state *rstate, - struct chan *chan) + struct chan *chan, + u32 timestamp) { - insert_broadcast(rstate->broadcasts, chan->channel_announce); + insert_broadcast(rstate->broadcasts, chan->channel_announce, timestamp); rstate->local_channel_announced |= is_local_channel(rstate, chan); /* If we've been waiting for this, now we can announce node */ @@ -627,7 +628,8 @@ static void add_channel_announce_to_broadcast(struct routing_state *rstate, if (!node->node_announcement_public) { node->node_announcement_public = true; insert_broadcast(rstate->broadcasts, - node->node_announcement); + node->node_announcement, + node->last_timestamp); } } } @@ -986,14 +988,17 @@ bool routing_add_channel_update(struct routing_state *rstate, return true; /* BOLT #7: + * - MUST consider the `timestamp` of the `channel_announcement` to be + * the `timestamp` of a corresponding `channel_update`. * - MUST consider whether to send the `channel_announcement` after * receiving the first corresponding `channel_update`. */ if (!have_broadcast_announce) - add_channel_announce_to_broadcast(rstate, chan); + add_channel_announce_to_broadcast(rstate, chan, timestamp); insert_broadcast(rstate->broadcasts, - chan->half[direction].channel_update); + chan->half[direction].channel_update, + timestamp); return true; } @@ -1204,7 +1209,8 @@ bool routing_add_node_announcement(struct routing_state *rstate, const u8 *msg T /* We might be waiting for channel_announce to be released. */ node->node_announcement_public = node_has_public_channels(node); if (node->node_announcement_public) - insert_broadcast(rstate->broadcasts, node->node_announcement); + insert_broadcast(rstate->broadcasts, node->node_announcement, + timestamp); return true; } diff --git a/gossipd/test/run-bench-find_route.c b/gossipd/test/run-bench-find_route.c index d98d9b01cd3f..28c66af2f828 100644 --- a/gossipd/test/run-bench-find_route.c +++ b/gossipd/test/run-bench-find_route.c @@ -93,7 +93,8 @@ u8 fromwire_u8(const u8 **cursor UNNEEDED, size_t *max UNNEEDED) bool fromwire_wireaddr(const u8 **cursor UNNEEDED, size_t *max UNNEEDED, struct wireaddr *addr UNNEEDED) { fprintf(stderr, "fromwire_wireaddr called!\n"); abort(); } /* Generated stub for insert_broadcast */ -void insert_broadcast(struct broadcast_state *bstate UNNEEDED, const u8 *msg UNNEEDED) +void insert_broadcast(struct broadcast_state *bstate UNNEEDED, const u8 *msg UNNEEDED, + u32 timestamp UNNEEDED) { fprintf(stderr, "insert_broadcast called!\n"); abort(); } /* Generated stub for onion_type_name */ const char *onion_type_name(int e UNNEEDED) diff --git a/gossipd/test/run-find_route-specific.c b/gossipd/test/run-find_route-specific.c index 776684aaf555..ab97544b99fa 100644 --- a/gossipd/test/run-find_route-specific.c +++ b/gossipd/test/run-find_route-specific.c @@ -57,7 +57,8 @@ u8 fromwire_u8(const u8 **cursor UNNEEDED, size_t *max UNNEEDED) bool fromwire_wireaddr(const u8 **cursor UNNEEDED, size_t *max UNNEEDED, struct wireaddr *addr UNNEEDED) { fprintf(stderr, "fromwire_wireaddr called!\n"); abort(); } /* Generated stub for insert_broadcast */ -void insert_broadcast(struct broadcast_state *bstate UNNEEDED, const u8 *msg UNNEEDED) +void insert_broadcast(struct broadcast_state *bstate UNNEEDED, const u8 *msg UNNEEDED, + u32 timestamp UNNEEDED) { fprintf(stderr, "insert_broadcast called!\n"); abort(); } /* Generated stub for onion_type_name */ const char *onion_type_name(int e UNNEEDED) diff --git a/gossipd/test/run-find_route.c b/gossipd/test/run-find_route.c index 45e488a7fc33..f34e44b083b3 100644 --- a/gossipd/test/run-find_route.c +++ b/gossipd/test/run-find_route.c @@ -55,7 +55,8 @@ u8 fromwire_u8(const u8 **cursor UNNEEDED, size_t *max UNNEEDED) bool fromwire_wireaddr(const u8 **cursor UNNEEDED, size_t *max UNNEEDED, struct wireaddr *addr UNNEEDED) { fprintf(stderr, "fromwire_wireaddr called!\n"); abort(); } /* Generated stub for insert_broadcast */ -void insert_broadcast(struct broadcast_state *bstate UNNEEDED, const u8 *msg UNNEEDED) +void insert_broadcast(struct broadcast_state *bstate UNNEEDED, const u8 *msg UNNEEDED, + u32 timestamp UNNEEDED) { fprintf(stderr, "insert_broadcast called!\n"); abort(); } /* Generated stub for onion_type_name */ const char *onion_type_name(int e UNNEEDED) From 97bb6c5a28822842beeb11d2492529ca33d9f9cd Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Mon, 4 Jun 2018 13:55:25 +0930 Subject: [PATCH 15/46] gossipd: ensure incoming timestamps are reasonable. This is kind of orthogonal to the other changes, but makes sense: if we would instantly or never prune the message, don't accept it. Signed-off-by: Rusty Russell --- gossipd/routing.c | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/gossipd/routing.c b/gossipd/routing.c index 0926736f9b61..83b53cf5aba0 100644 --- a/gossipd/routing.c +++ b/gossipd/routing.c @@ -1071,6 +1071,28 @@ u8 *handle_channel_update(struct routing_state *rstate, const u8 *update, } } + /* BOLT #7: + * + * - if the `timestamp` is unreasonably far in the future: + * - MAY discard the `channel_announcement`. + */ + if (timestamp > time_now().ts.tv_sec + rstate->prune_timeout) { + status_debug("Received channel_update for %s with far time %u", + type_to_string(tmpctx, struct short_channel_id, + &short_channel_id), + timestamp); + return NULL; + } + + /* Note: we can consider old timestamps a case of "instant prune" too */ + if (timestamp < time_now().ts.tv_sec - rstate->prune_timeout) { + status_debug("Received channel_update for %s with old time %u", + type_to_string(tmpctx, struct short_channel_id, + &short_channel_id), + timestamp); + return NULL; + } + c = &chan->half[direction]; if (is_halfchan_defined(c) && timestamp <= c->last_timestamp) { From 531c82b6adc76bd86fde8747e5237dc082883e44 Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Mon, 4 Jun 2018 13:56:25 +0930 Subject: [PATCH 16/46] gossipd: handle gossip_timestamp_filter message. And initialize filter (to "never") when we negotiated LOCAL_GOSSIP_QUERIES, and send initial filter message. Signed-off-by: Rusty Russell --- gossipd/broadcast.c | 12 ++-- gossipd/broadcast.h | 7 +- gossipd/gossip.c | 159 +++++++++++++++++++++++++++++++++----------- 3 files changed, 132 insertions(+), 46 deletions(-) diff --git a/gossipd/broadcast.c b/gossipd/broadcast.c index 196772c4f077..9695ee87a89a 100644 --- a/gossipd/broadcast.c +++ b/gossipd/broadcast.c @@ -51,12 +51,16 @@ void insert_broadcast(struct broadcast_state *bstate, bstate->next_index++); } -const u8 *next_broadcast(struct broadcast_state *bstate, u64 *last_index) +const u8 *next_broadcast(struct broadcast_state *bstate, + u32 timestamp_min, u32 timestamp_max, + u64 *last_index) { struct queued_message *m; - m = uintmap_after(&bstate->broadcasts, last_index); - if (m) - return m->payload; + while ((m = uintmap_after(&bstate->broadcasts, last_index)) != NULL) { + if (m->timestamp >= timestamp_min + && m->timestamp <= timestamp_max) + return m->payload; + } return NULL; } diff --git a/gossipd/broadcast.h b/gossipd/broadcast.h index e728b681fd38..6de0fec38e6d 100644 --- a/gossipd/broadcast.h +++ b/gossipd/broadcast.h @@ -20,7 +20,10 @@ struct broadcast_state *new_broadcast_state(tal_t *ctx); void insert_broadcast(struct broadcast_state *bstate, const u8 *msg, u32 timestamp); -/* Return the broadcast with index >= *last_index, and update *last_index. +/* Return the broadcast with index >= *last_index, timestamp >= min and <= max + * and update *last_index. * There's no broadcast with index 0. */ -const u8 *next_broadcast(struct broadcast_state *bstate, u64 *last_index); +const u8 *next_broadcast(struct broadcast_state *bstate, + u32 timestamp_min, u32 timestamp_max, + u64 *last_index); #endif /* LIGHTNING_GOSSIPD_BROADCAST_H */ diff --git a/gossipd/gossip.c b/gossipd/gossip.c index 31b9e5f9420f..57cc0dc802db 100644 --- a/gossipd/gossip.c +++ b/gossipd/gossip.c @@ -217,6 +217,9 @@ struct peer { /* High water mark for the staggered broadcast */ u64 broadcast_index; + /* Timestamp range to filter gossip by */ + u32 gossip_timestamp_min, gossip_timestamp_max; + /* Are there outstanding queries on short_channel_ids? */ const struct short_channel_id *scid_queries; size_t scid_query_idx; @@ -352,6 +355,8 @@ static struct peer *new_peer(const tal_t *ctx, peer->scid_query_nodes = NULL; peer->scid_query_nodes_idx = 0; peer->num_scid_queries_outstanding = 0; + peer->gossip_timestamp_min = 0; + peer->gossip_timestamp_max = UINT32_MAX; return peer; } @@ -493,6 +498,25 @@ static struct io_plan *retry_peer_connected(struct io_conn *conn, return peer_connected(conn, peer); } +static void setup_gossip_range(struct peer *peer) +{ + bool gossip_queries; + u8 *msg; + + gossip_queries = feature_offered(peer->lfeatures, LOCAL_GOSSIP_QUERIES) + && feature_offered(peer->daemon->localfeatures, + LOCAL_GOSSIP_QUERIES); + + if (!gossip_queries) + return; + + /* Tell it to start gossip! (And give us everything!) */ + msg = towire_gossip_timestamp_filter(peer, + &peer->daemon->rstate->chain_hash, + 0, UINT32_MAX); + queue_peer_msg(peer, take(msg)); +} + static struct io_plan *peer_connected(struct io_conn *conn, struct peer *peer) { struct peer *old_peer; @@ -523,16 +547,34 @@ static struct io_plan *peer_connected(struct io_conn *conn, struct peer *peer) /* BOLT #7: * - * Upon receiving an `init` message with the `initial_routing_sync` - * flag set the node sends `channel_announcement`s, `channel_update`s - * and `node_announcement`s for all known channels and nodes as if - * they were just received. + * - if the `gossip_queries` feature is negotiated: + * - MUST NOT relay any gossip messages unless explicitly requested. */ - if (feature_offered(peer->lfeatures, LOCAL_INITIAL_ROUTING_SYNC)) - peer->broadcast_index = 0; - else - peer->broadcast_index - = peer->daemon->rstate->broadcasts->next_index; + if (feature_offered(peer->lfeatures, LOCAL_GOSSIP_QUERIES) + && feature_offered(peer->daemon->localfeatures, LOCAL_GOSSIP_QUERIES)) { + peer->broadcast_index = UINT64_MAX; + /* Nothing in this range */ + peer->gossip_timestamp_min = UINT32_MAX; + peer->gossip_timestamp_max = 0; + } else { + /* BOLT #7: + * + * - upon receiving an `init` message with the + * `initial_routing_sync` flag set to 1: + * - SHOULD send `channel_announcement`s, `channel_update`s + * and `node_announcement`s for all known channels and + * nodes, as if they were just received. + * - if the `initial_routing_sync` flag is set to 0, OR if the + * initial sync was completed: + * - SHOULD resume normal operation, as specified in the + * following [Rebroadcasting](#rebroadcasting) section. + */ + if (feature_offered(peer->lfeatures, LOCAL_INITIAL_ROUTING_SYNC)) + peer->broadcast_index = 0; + else + peer->broadcast_index + = peer->daemon->rstate->broadcasts->next_index; + } /* This is a full peer now; we keep it around until master says * it's dead. */ @@ -548,6 +590,7 @@ static struct io_plan *peer_connected(struct io_conn *conn, struct peer *peer) /* Start the gossip flowing. */ wake_pkt_out(peer); + setup_gossip_range(peer); return io_close_taken_fd(conn); } @@ -821,6 +864,40 @@ static void handle_query_short_channel_ids(struct peer *peer, u8 *msg) msg_wake(&peer->remote->out); } +static void handle_gossip_timestamp_filter(struct peer *peer, u8 *msg) +{ + struct bitcoin_blkid chain_hash; + u32 first_timestamp, timestamp_range; + + if (!fromwire_gossip_timestamp_filter(msg, &chain_hash, + &first_timestamp, + ×tamp_range)) { + peer_error(peer, "Bad gossip_timestamp_filter %s", + tal_hex(tmpctx, msg)); + return; + } + + if (!structeq(&peer->daemon->rstate->chain_hash, &chain_hash)) { + status_trace("%s sent gossip_timestamp_filter chainhash %s", + type_to_string(tmpctx, struct pubkey, &peer->id), + type_to_string(tmpctx, struct bitcoin_blkid, + &chain_hash)); + return; + } + + /* First time, start gossip sync immediately. */ + if (peer->gossip_timestamp_min > peer->gossip_timestamp_max) + wake_pkt_out(peer); + + /* FIXME: We don't index by timestamp, so this forces a brute + * search! */ + peer->gossip_timestamp_min = first_timestamp; + peer->gossip_timestamp_max = first_timestamp + timestamp_range - 1; + if (peer->gossip_timestamp_max < peer->gossip_timestamp_min) + peer->gossip_timestamp_max = UINT32_MAX; + peer->broadcast_index = 0; +} + static void handle_ping(struct peer *peer, u8 *ping) { u8 *pong; @@ -963,6 +1040,10 @@ static struct io_plan *peer_msgin(struct io_conn *conn, handle_reply_short_channel_ids_end(peer, msg); return peer_next_in(conn, peer); + case WIRE_GOSSIP_TIMESTAMP_FILTER: + handle_gossip_timestamp_filter(peer, msg); + return peer_next_in(conn, peer); + case WIRE_OPEN_CHANNEL: case WIRE_CHANNEL_REESTABLISH: case WIRE_ACCEPT_CHANNEL: @@ -989,7 +1070,6 @@ static struct io_plan *peer_msgin(struct io_conn *conn, case WIRE_QUERY_CHANNEL_RANGE: case WIRE_REPLY_CHANNEL_RANGE: - case WIRE_GOSSIP_TIMESTAMP_FILTER: /* FIXME: Implement */ return peer_next_in(conn, peer); } @@ -1161,6 +1241,29 @@ static bool create_next_scid_reply(struct peer *peer) return sent; } +/* If we're supposed to be sending gossip, do so now. */ +static bool maybe_queue_gossip(struct peer *peer) +{ + const u8 *next; + + if (!peer->gossip_sync) + return false; + + next = next_broadcast(peer->daemon->rstate->broadcasts, + peer->gossip_timestamp_min, + peer->gossip_timestamp_max, + &peer->broadcast_index); + + if (next) { + queue_peer_msg(peer, next); + return true; + } + + /* Gossip is drained. Wait for next timer. */ + peer->gossip_sync = false; + return false; +} + static struct io_plan *peer_pkt_out(struct io_conn *conn, struct peer *peer) { /* First priority is queued packets, if any */ @@ -1183,20 +1286,8 @@ static struct io_plan *peer_pkt_out(struct io_conn *conn, struct peer *peer) return ready_for_master(conn, peer); } else if (create_next_scid_reply(peer)) { goto again; - } else if (peer->gossip_sync) { - /* If we're supposed to be sending gossip, do so now. */ - const u8 *next; - - next = next_broadcast(peer->daemon->rstate->broadcasts, - &peer->broadcast_index); - - if (next) - return peer_write_message(conn, &peer->local->pcs, - next, - peer_pkt_out); - - /* Gossip is drained. Wait for next timer. */ - peer->gossip_sync = false; + } else if (maybe_queue_gossip(peer)) { + goto again; } return msg_queue_wait(conn, &peer->local->peer_out, peer_pkt_out, peer); @@ -1397,6 +1488,8 @@ static struct io_plan *owner_msg_in(struct io_conn *conn, handle_query_short_channel_ids(peer, dc->msg_in); } else if (type == WIRE_REPLY_SHORT_CHANNEL_IDS_END) { handle_reply_short_channel_ids_end(peer, dc->msg_in); + } else if (type == WIRE_GOSSIP_TIMESTAMP_FILTER) { + handle_gossip_timestamp_filter(peer, dc->msg_in); } else if (type == WIRE_GOSSIP_GET_UPDATE) { handle_get_update(peer, dc->msg_in); } else if (type == WIRE_GOSSIP_LOCAL_ADD_CHANNEL) { @@ -1465,7 +1558,6 @@ static bool send_peer_with_fds(struct peer *peer, const u8 *msg) */ static bool nonlocal_dump_gossip(struct io_conn *conn, struct daemon_conn *dc) { - const u8 *next; struct peer *peer = dc->ctx; /* Make sure we are not connected directly */ @@ -1475,21 +1567,8 @@ static bool nonlocal_dump_gossip(struct io_conn *conn, struct daemon_conn *dc) if (create_next_scid_reply(peer)) return true; - /* Nothing to do if we're not gossiping */ - if (!peer->gossip_sync) - return false; - - next = next_broadcast(peer->daemon->rstate->broadcasts, - &peer->broadcast_index); - - if (!next) { - peer->gossip_sync = false; - return false; - } else { - u8 *msg = towire_gossip_send_gossip(NULL, next); - daemon_conn_send(peer->remote, take(msg)); - return true; - } + /* Otherwise queue any gossip we want to send */ + return maybe_queue_gossip(peer); } static struct io_plan *new_peer_got_fd(struct io_conn *conn, struct peer *peer) From db6a6442cb08d6f6918594cfc08188c8ea6c1117 Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Mon, 4 Jun 2018 13:57:25 +0930 Subject: [PATCH 17/46] gossipd: single-thread the gossip timer. We have a function called 'wake_pkt_out' which is really 'start gossiping', so rename it to 'wake_gossip_out'. In addition, it's fired both on a timer, and in response to our first gossip_timestamp_filter, which leads to very confusing (though, technically, not incorrect) behavior. Keep a single timer at all times, which now doubles as the flag to indicating we're syncing right now. Set it once we're done syncing gossip. Technically this means we got from once-every-60-seconds to quiet-for-60-seconds-between-gossip, but that's OK. Signed-off-by: Rusty Russell --- gossipd/gossip.c | 52 ++++++++++++++++++++++-------------------------- 1 file changed, 24 insertions(+), 28 deletions(-) diff --git a/gossipd/gossip.c b/gossipd/gossip.c index 57cc0dc802db..ca88db4e3c73 100644 --- a/gossipd/gossip.c +++ b/gossipd/gossip.c @@ -228,8 +228,8 @@ struct peer { struct pubkey *scid_query_nodes; size_t scid_query_nodes_idx; - /* Is it time to continue the staggered broadcast? */ - bool gossip_sync; + /* If this is NULL, we're syncing gossip now. */ + struct oneshot *gossip_timer; /* How many query responses are we expecting? */ size_t num_scid_queries_outstanding; @@ -252,7 +252,6 @@ struct addrhint { static struct io_plan *peer_start_gossip(struct io_conn *conn, struct peer *peer); static bool send_peer_with_fds(struct peer *peer, const u8 *msg); -static void wake_pkt_out(struct peer *peer); static void retry_important(struct important_peerid *imp); static void destroy_peer(struct peer *peer) @@ -349,6 +348,7 @@ static struct peer *new_peer(const tal_t *ctx, peer->addr = *addr; peer->daemon = daemon; peer->local = new_local_peer_state(peer, cs); + peer->gossip_timer = NULL; peer->remote = NULL; peer->scid_queries = NULL; peer->scid_query_idx = 0; @@ -448,6 +448,19 @@ static void queue_peer_msg(struct peer *peer, const u8 *msg TAKES) } } +static void wake_gossip_out(struct peer *peer) +{ + /* If we were waiting, we're not any more */ + peer->gossip_timer = tal_free(peer->gossip_timer); + + if (peer->local) + /* Notify the peer-write loop */ + msg_wake(&peer->local->peer_out); + else if (peer->remote) + /* Notify the daemon_conn-write loop */ + msg_wake(&peer->remote->out); +} + static void peer_error(struct peer *peer, const char *fmt, ...) { va_list ap; @@ -588,7 +601,7 @@ static struct io_plan *peer_connected(struct io_conn *conn, struct peer *peer) return io_close(conn); /* Start the gossip flowing. */ - wake_pkt_out(peer); + wake_gossip_out(peer); setup_gossip_range(peer); return io_close_taken_fd(conn); @@ -887,7 +900,7 @@ static void handle_gossip_timestamp_filter(struct peer *peer, u8 *msg) /* First time, start gossip sync immediately. */ if (peer->gossip_timestamp_min > peer->gossip_timestamp_max) - wake_pkt_out(peer); + wake_gossip_out(peer); /* FIXME: We don't index by timestamp, so this forces a brute * search! */ @@ -1088,26 +1101,6 @@ static struct io_plan *peer_msgin(struct io_conn *conn, return peer_next_in(conn, peer); } -/* Wake up the outgoing direction of the connection and write any - * queued messages. Needed since the `io_wake` method signature does - * not allow us to specify it as the callback for `new_reltimer`, but - * it allows us to set an additional flag for the routing dump.. - */ -static void wake_pkt_out(struct peer *peer) -{ - peer->gossip_sync = true; - new_reltimer(&peer->daemon->timers, peer, - time_from_msec(peer->daemon->broadcast_interval), - wake_pkt_out, peer); - - if (peer->local) - /* Notify the peer-write loop */ - msg_wake(&peer->local->peer_out); - else if (peer->remote) - /* Notify the daemon_conn-write loop */ - msg_wake(&peer->remote->out); -} - /* Mutual recursion. */ static struct io_plan *peer_pkt_out(struct io_conn *conn, struct peer *peer); @@ -1246,7 +1239,7 @@ static bool maybe_queue_gossip(struct peer *peer) { const u8 *next; - if (!peer->gossip_sync) + if (peer->gossip_timer) return false; next = next_broadcast(peer->daemon->rstate->broadcasts, @@ -1260,7 +1253,10 @@ static bool maybe_queue_gossip(struct peer *peer) } /* Gossip is drained. Wait for next timer. */ - peer->gossip_sync = false; + peer->gossip_timer + = new_reltimer(&peer->daemon->timers, peer, + time_from_msec(peer->daemon->broadcast_interval), + wake_gossip_out, peer); return false; } @@ -1296,7 +1292,7 @@ static struct io_plan *peer_pkt_out(struct io_conn *conn, struct peer *peer) /* Now we're a fully-fledged peer. */ static struct io_plan *peer_start_gossip(struct io_conn *conn, struct peer *peer) { - wake_pkt_out(peer); + wake_gossip_out(peer); return io_duplex(conn, peer_next_in(conn, peer), peer_pkt_out(conn, peer)); From c34b49c356b9429389f6efa0bab50d1edaa37c37 Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Mon, 4 Jun 2018 13:58:02 +0930 Subject: [PATCH 18/46] gossipd: add dev-send-timestamp-filter command for testing timestamp filtering. Since we currently only (ab)use it to send everything, we need a way to generate boutique queries for testing. Signed-off-by: Rusty Russell --- gossipd/gossip.c | 35 +++++++++++++++ gossipd/gossip_wire.csv | 6 +++ lightningd/gossip_control.c | 48 ++++++++++++++++++++ tests/test_lightningd.py | 87 +++++++++++++++++++++++++++++++++++++ 4 files changed, 176 insertions(+) diff --git a/gossipd/gossip.c b/gossipd/gossip.c index ca88db4e3c73..6b61d4c39f2f 100644 --- a/gossipd/gossip.c +++ b/gossipd/gossip.c @@ -1989,6 +1989,37 @@ static struct io_plan *query_scids_req(struct io_conn *conn, take(towire_gossip_scids_reply(NULL, false, false))); goto out; } + +static struct io_plan *send_timestamp_filter(struct io_conn *conn, + struct daemon *daemon, + const u8 *msg) +{ + struct pubkey id; + u32 first, range; + struct peer *peer; + + if (!fromwire_gossip_send_timestamp_filter(msg, &id, &first, &range)) + master_badmsg(WIRE_GOSSIP_SEND_TIMESTAMP_FILTER, msg); + + peer = find_peer(daemon, &id); + if (!peer) { + status_broken("send_timestamp_filter: unknown peer %s", + type_to_string(tmpctx, struct pubkey, &id)); + goto out; + } + + if (!feature_offered(peer->lfeatures, LOCAL_GOSSIP_QUERIES)) { + status_broken("send_timestamp_filter: no gossip_query support in peer %s", + type_to_string(tmpctx, struct pubkey, &id)); + goto out; + } + + msg = towire_gossip_timestamp_filter(NULL, &daemon->rstate->chain_hash, + first, range); + queue_peer_msg(peer, take(msg)); +out: + return daemon_conn_read_next(conn, &daemon->master); +} #endif /* DEVELOPER */ static int make_listen_fd(int domain, void *addr, socklen_t len, bool mayfail) @@ -3209,9 +3240,13 @@ static struct io_plan *recv_req(struct io_conn *conn, struct daemon_conn *master case WIRE_GOSSIP_QUERY_SCIDS: return query_scids_req(conn, daemon, daemon->master.msg_in); + + case WIRE_GOSSIP_SEND_TIMESTAMP_FILTER: + return send_timestamp_filter(conn, daemon, daemon->master.msg_in); #else case WIRE_GOSSIP_PING: case WIRE_GOSSIP_QUERY_SCIDS: + case WIRE_GOSSIP_SEND_TIMESTAMP_FILTER: break; #endif /* !DEVELOPER */ diff --git a/gossipd/gossip_wire.csv b/gossipd/gossip_wire.csv index c5805067b18e..d062f144a288 100644 --- a/gossipd/gossip_wire.csv +++ b/gossipd/gossip_wire.csv @@ -166,6 +166,12 @@ gossip_scids_reply,3131 gossip_scids_reply,,ok,bool gossip_scids_reply,,complete,bool +# Test gossip timestamp filtering. +gossip_send_timestamp_filter,3028 +gossip_send_timestamp_filter,,id,struct pubkey +gossip_send_timestamp_filter,,first_timestamp,u32 +gossip_send_timestamp_filter,,timestamp_range,u32 + # Given a short_channel_id, return the endpoints gossip_resolve_channel_request,3009 gossip_resolve_channel_request,,channel_id,struct short_channel_id diff --git a/lightningd/gossip_control.c b/lightningd/gossip_control.c index aa68a86298ef..c1b8dd886b34 100644 --- a/lightningd/gossip_control.c +++ b/lightningd/gossip_control.c @@ -137,6 +137,7 @@ static unsigned gossip_msg(struct subd *gossip, const u8 *msg, const int *fds) case WIRE_GOSSIP_ROUTING_FAILURE: case WIRE_GOSSIP_MARK_CHANNEL_UNROUTABLE: case WIRE_GOSSIP_QUERY_SCIDS: + case WIRE_GOSSIP_SEND_TIMESTAMP_FILTER: case WIRE_GOSSIPCTL_PEER_DISCONNECT: case WIRE_GOSSIPCTL_PEER_IMPORTANT: case WIRE_GOSSIPCTL_PEER_DISCONNECTED: @@ -629,4 +630,51 @@ static const struct json_command dev_query_scids_command = { "Query {peerid} for [scids]" }; AUTODATA(json_command, &dev_query_scids_command); + +static void json_dev_send_timestamp_filter(struct command *cmd, + const char *buffer, + const jsmntok_t *params) +{ + u8 *msg; + jsmntok_t *idtok, *firsttok, *rangetok; + struct pubkey id; + u32 first, range; + + if (!json_get_params(cmd, buffer, params, + "id", &idtok, + "first", &firsttok, + "range", &rangetok, + NULL)) { + return; + } + + if (!json_tok_pubkey(buffer, idtok, &id)) { + command_fail(cmd, JSONRPC2_INVALID_PARAMS, + "'%.*s' is not a valid id", + idtok->end - idtok->start, + buffer + idtok->start); + return; + } + + if (!json_tok_number(buffer, firsttok, &first) + || !json_tok_number(buffer, rangetok, &range)) { + command_fail(cmd, JSONRPC2_INVALID_PARAMS, + "bad first or range numbers"); + return; + } + + log_debug(cmd->ld->log, "Setting timestamp range %u+%u", first, range); + /* Tell gossipd, since this is a gossip query. */ + msg = towire_gossip_send_timestamp_filter(NULL, &id, first, range); + subd_send_msg(cmd->ld->gossip, take(msg)); + + command_success(cmd, null_response(cmd)); +} + +static const struct json_command dev_send_timestamp_filter = { + "dev-send-timestamp-filter", + json_dev_send_timestamp_filter, + "Send {peerid} the timestamp filter {first} {range}" +}; +AUTODATA(json_command, &dev_send_timestamp_filter); #endif /* DEVELOPER */ diff --git a/tests/test_lightningd.py b/tests/test_lightningd.py index 4cc80459e850..8caf535b46c4 100644 --- a/tests/test_lightningd.py +++ b/tests/test_lightningd.py @@ -2681,6 +2681,93 @@ def test_query_short_channel_id(self): l1.daemon.wait_for_log('\[IN\] 0101') l1.daemon.wait_for_log('\[IN\] 0101') + @unittest.skipIf(not DEVELOPER, "needs DEVELOPER=1") + def test_gossip_timestamp_filter(self): + # Need full IO logging so we can see gossip (from gossipd and channeld) + l1 = self.node_factory.get_node(options={'log-level': 'io'}) + l2 = self.node_factory.get_node() + l3 = self.node_factory.get_node() + + # Full IO logging for gossipds + subprocess.run(['kill', '-USR1', l1.subd_pid('gossipd')]) + subprocess.run(['kill', '-USR1', l2.subd_pid('gossipd')]) + + l1.rpc.connect(l2.info['id'], 'localhost', l2.port) + l2.rpc.connect(l3.info['id'], 'localhost', l3.port) + + before_anything = int(time.time() - 1.0) + + # Make a public channel. + chan12 = self.fund_channel(l1, l2, 10**5) + bitcoind.generate_block(5) + sync_blockheight([l1, l2]) + + self.wait_for_routes(l3, [chan12]) + after_12 = int(time.time()) + # Full IO logging for l1's channeld + subprocess.run(['kill', '-USR1', l1.subd_pid('channeld')]) + + # Make another one, different timestamp. + chan23 = self.fund_channel(l2, l3, 10**5) + bitcoind.generate_block(5) + sync_blockheight([l2, l3]) + + self.wait_for_routes(l1, [chan23]) + after_23 = int(time.time()) + + # Make sure l1 has received all the gossip. + wait_for(lambda: ['alias' in node for node in l1.rpc.listnodes()['nodes']] == [True, True, True]) + + # l1 sets broad timestamp, will receive info about both channels again. + l1.rpc.dev_send_timestamp_filter(id=l2.info['id'], + first=0, + range=0xFFFFFFFF) + before_sendfilter = l1.daemon.logsearch_start + + # 0x0100 = channel_announcement + # 0x0102 = channel_update + # 0x0101 = node_announcement + l1.daemon.wait_for_log('\[IN\] 0100') + # The order of node_announcements relative to others is undefined. + l1.daemon.wait_for_logs(['\[IN\] 0102', + '\[IN\] 0102', + '\[IN\] 0100', + '\[IN\] 0102', + '\[IN\] 0102', + '\[IN\] 0101', + '\[IN\] 0101', + '\[IN\] 0101']) + + # Now timestamp which doesn't overlap (gives nothing). + before_sendfilter = l1.daemon.logsearch_start + l1.rpc.dev_send_timestamp_filter(id=l2.info['id'], + first=0, + range=before_anything) + time.sleep(1) + assert not l1.daemon.is_in_log('\[IN\] 0100', before_sendfilter) + + # Now choose range which will only give first update. + l1.rpc.dev_send_timestamp_filter(id=l2.info['id'], + first=before_anything, + range=after_12 - before_anything + 1) + # 0x0100 = channel_announcement + l1.daemon.wait_for_log('\[IN\] 0100') + # 0x0102 = channel_update + # (Node announcement may have any timestamp) + l1.daemon.wait_for_log('\[IN\] 0102') + l1.daemon.wait_for_log('\[IN\] 0102') + + # Now choose range which will only give second update. + l1.rpc.dev_send_timestamp_filter(id=l2.info['id'], + first=after_12, + range=after_23 - after_12 + 1) + # 0x0100 = channel_announcement + l1.daemon.wait_for_log('\[IN\] 0100') + # 0x0102 = channel_update + # (Node announcement may have any timestamp) + l1.daemon.wait_for_log('\[IN\] 0102') + l1.daemon.wait_for_log('\[IN\] 0102') + @unittest.skipIf(not DEVELOPER, "needs DEVELOPER=1") def test_routing_gossip_reconnect(self): # Connect two peers, reconnect and then see if we resume the From 0dda5d4e1c6ec47a3b84fa92dd92ec99c6c2d977 Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Mon, 4 Jun 2018 13:58:02 +0930 Subject: [PATCH 19/46] gossipd: handle query_channel_range We send them all the short_channel_ids we have in a given range. Signed-off-by: Rusty Russell --- gossipd/gossip.c | 120 ++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 119 insertions(+), 1 deletion(-) diff --git a/gossipd/gossip.c b/gossipd/gossip.c index 6b61d4c39f2f..653b12c39043 100644 --- a/gossipd/gossip.c +++ b/gossipd/gossip.c @@ -911,6 +911,121 @@ static void handle_gossip_timestamp_filter(struct peer *peer, u8 *msg) peer->broadcast_index = 0; } +static void reply_channel_range(struct peer *peer, + u32 first_blocknum, u32 number_of_blocks, + const u8 *encoded) +{ + /* BOLT #7: + * + * - For each `reply_channel_range`: + * - MUST set with `chain_hash` equal to that of `query_channel_range`, + * - MUST encode a `short_channel_id` for every open channel it + * knows in blocks `first_blocknum` to `first_blocknum` plus + * `number_of_blocks` minus one. + * - MUST limit `number_of_blocks` to the maximum number of blocks + * whose results could fit in `encoded_short_ids` + * - if does not maintain up-to-date channel information for + * `chain_hash`: + * - MUST set `complete` to 0. + * - otherwise: + * - SHOULD set `complete` to 1. + */ + u8 *msg = towire_reply_channel_range(NULL, + &peer->daemon->rstate->chain_hash, + first_blocknum, + number_of_blocks, + 1, encoded); + queue_peer_msg(peer, take(msg)); +} + +static void queue_channel_ranges(struct peer *peer, + u32 first_blocknum, u32 number_of_blocks) +{ + struct routing_state *rstate = peer->daemon->rstate; + u8 *encoded = encode_short_channel_ids_start(tmpctx); + struct short_channel_id scid; + + /* BOLT #7: + * + * 1. type: 264 (`reply_channel_range`) (`gossip_queries`) + * 2. data: + * * [`32`:`chain_hash`] + * * [`4`:`first_blocknum`] + * * [`4`:`number_of_blocks`] + * * [`1`:`complete`] + * * [`2`:`len`] + * * [`len`:`encoded_short_ids`] + */ + const size_t reply_overhead = 32 + 4 + 4 + 1 + 2; + const size_t max_encoded_bytes = 65535 - 2 - reply_overhead; + + /* Avoid underflow: we don't use block 0 anyway */ + if (first_blocknum == 0) + mk_short_channel_id(&scid, 1, 0, 0); + else + mk_short_channel_id(&scid, first_blocknum, 0, 0); + scid.u64--; + + while (uintmap_after(&rstate->chanmap, &scid.u64)) { + u32 blocknum = short_channel_id_blocknum(&scid); + if (blocknum >= first_blocknum + number_of_blocks) + break; + + encode_add_short_channel_id(&encoded, &scid); + } + + if (encode_short_channel_ids_end(&encoded, max_encoded_bytes)) { + reply_channel_range(peer, first_blocknum, number_of_blocks, + encoded); + return; + } + + /* It wouldn't all fit: divide in half */ + /* We assume we can always send one block! */ + if (number_of_blocks <= 1) { + /* We always assume we can send 1 blocks worth */ + status_broken("Could not fit scids for single block %u", + first_blocknum); + return; + } + status_debug("queue_channel_ranges full: splitting %u+%u and %u+%u", + first_blocknum, + number_of_blocks / 2, + first_blocknum + number_of_blocks / 2, + number_of_blocks - number_of_blocks / 2); + queue_channel_ranges(peer, first_blocknum, number_of_blocks / 2); + queue_channel_ranges(peer, first_blocknum + number_of_blocks / 2, + number_of_blocks - number_of_blocks / 2); +} + +static void handle_query_channel_range(struct peer *peer, u8 *msg) +{ + struct bitcoin_blkid chain_hash; + u32 first_blocknum, number_of_blocks; + + if (!fromwire_query_channel_range(msg, &chain_hash, + &first_blocknum, &number_of_blocks)) { + peer_error(peer, "Bad query_channel_range %s", + tal_hex(tmpctx, msg)); + return; + } + + if (!structeq(&peer->daemon->rstate->chain_hash, &chain_hash)) { + status_trace("%s sent query_channel_range chainhash %s", + type_to_string(tmpctx, struct pubkey, &peer->id), + type_to_string(tmpctx, struct bitcoin_blkid, + &chain_hash)); + return; + } + + if (first_blocknum + number_of_blocks < first_blocknum) { + peer_error(peer, "query_channel_range overflow %u+%u", + first_blocknum, number_of_blocks); + return; + } + queue_channel_ranges(peer, first_blocknum, number_of_blocks); +} + static void handle_ping(struct peer *peer, u8 *ping) { u8 *pong; @@ -1057,6 +1172,10 @@ static struct io_plan *peer_msgin(struct io_conn *conn, handle_gossip_timestamp_filter(peer, msg); return peer_next_in(conn, peer); + case WIRE_QUERY_CHANNEL_RANGE: + handle_query_channel_range(peer, msg); + return peer_next_in(conn, peer); + case WIRE_OPEN_CHANNEL: case WIRE_CHANNEL_REESTABLISH: case WIRE_ACCEPT_CHANNEL: @@ -1081,7 +1200,6 @@ static struct io_plan *peer_msgin(struct io_conn *conn, /* This will wait. */ return peer_next_in(conn, peer); - case WIRE_QUERY_CHANNEL_RANGE: case WIRE_REPLY_CHANNEL_RANGE: /* FIXME: Implement */ return peer_next_in(conn, peer); From 118f099dd8f4cb3737b60588a225f3eb59f63656 Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Mon, 4 Jun 2018 13:58:02 +0930 Subject: [PATCH 20/46] gossip: dev-query-channel-range to test query_channel_range. We keep a crappy bitmap, and finish when their replies cover everything we asked. Signed-off-by: Rusty Russell --- gossipd/gossip.c | 167 +++++++++++++++++++++++++++++++++++- gossipd/gossip_wire.csv | 14 +++ lightningd/gossip_control.c | 84 ++++++++++++++++++ tests/test_lightningd.py | 92 ++++++++++++++++++++ 4 files changed, 353 insertions(+), 4 deletions(-) diff --git a/gossipd/gossip.c b/gossipd/gossip.c index 653b12c39043..5aa661d09f27 100644 --- a/gossipd/gossip.c +++ b/gossipd/gossip.c @@ -234,6 +234,11 @@ struct peer { /* How many query responses are we expecting? */ size_t num_scid_queries_outstanding; + /* Map of outstanding channel_range requests. */ + u8 *query_channel_blocks; + u32 first_channel_range; + struct short_channel_id *query_channel_scids; + /* Only one of these is set: */ struct local_peer_state *local; struct daemon_conn *remote; @@ -355,6 +360,7 @@ static struct peer *new_peer(const tal_t *ctx, peer->scid_query_nodes = NULL; peer->scid_query_nodes_idx = 0; peer->num_scid_queries_outstanding = 0; + peer->query_channel_blocks = NULL; peer->gossip_timestamp_min = 0; peer->gossip_timestamp_max = UINT32_MAX; @@ -1081,6 +1087,97 @@ static void handle_reply_short_channel_ids_end(struct peer *peer, u8 *msg) daemon_conn_send(&peer->daemon->master, take(msg)); } +static void handle_reply_channel_range(struct peer *peer, u8 *msg) +{ + struct bitcoin_blkid chain; + u8 complete; + u32 first_blocknum, number_of_blocks; + u8 *encoded, *p; + struct short_channel_id *scids; + size_t n; + + if (!fromwire_reply_channel_range(tmpctx, msg, &chain, &first_blocknum, + &number_of_blocks, &complete, + &encoded)) { + peer_error(peer, "Bad reply_channel_range %s", + tal_hex(tmpctx, msg)); + return; + } + + if (!structeq(&peer->daemon->rstate->chain_hash, &chain)) { + peer_error(peer, "reply_channel_range for bad chain: %s", + tal_hex(tmpctx, msg)); + return; + } + + if (!peer->query_channel_blocks) { + peer_error(peer, "reply_channel_range without query: %s", + tal_hex(tmpctx, msg)); + return; + } + + if (first_blocknum + number_of_blocks < first_blocknum) { + peer_error(peer, "reply_channel_range invalid %u+%u", + first_blocknum, number_of_blocks); + return; + } + + scids = decode_short_ids(tmpctx, encoded); + if (!scids) { + peer_error(peer, "Bad reply_channel_range encoding %s", + tal_hex(tmpctx, encoded)); + return; + } + + n = first_blocknum - peer->first_channel_range; + if (first_blocknum < peer->first_channel_range + || n + number_of_blocks > tal_count(peer->query_channel_blocks)) { + peer_error(peer, "reply_channel_range invalid %u+%u for query %u+%u", + first_blocknum, number_of_blocks, + peer->first_channel_range, + tal_count(peer->query_channel_blocks)); + return; + } + + p = memchr(peer->query_channel_blocks + n, 1, number_of_blocks); + if (p) { + peer_error(peer, "reply_channel_range %u+%u already have block %zu", + first_blocknum, number_of_blocks, + peer->first_channel_range + (p - peer->query_channel_blocks)); + return; + } + + /* Mark these blocks received */ + memset(peer->query_channel_blocks + n, 1, number_of_blocks); + + /* Add scids */ + n = tal_count(peer->query_channel_scids); + tal_resize(&peer->query_channel_scids, n + tal_count(scids)); + memcpy(peer->query_channel_scids + n, scids, tal_len(scids)); + + status_debug("peer %s reply_channel_range %u+%u (of %u+%zu) %zu scids", + type_to_string(tmpctx, struct pubkey, &peer->id), + first_blocknum, number_of_blocks, + peer->first_channel_range, + tal_count(peer->query_channel_blocks), + tal_count(scids)); + + /* Still more to go? */ + if (memchr(peer->query_channel_blocks, 0, + tal_count(peer->query_channel_blocks))) + return; + + /* All done, send reply */ + msg = towire_gossip_query_channel_range_reply(NULL, + first_blocknum, + number_of_blocks, + complete, + peer->query_channel_scids); + daemon_conn_send(&peer->daemon->master, take(msg)); + peer->query_channel_scids = tal_free(peer->query_channel_scids); + peer->query_channel_blocks = tal_free(peer->query_channel_blocks); +} + /* If master asks us to release peer, we attach this destructor in case it * dies while we're waiting for it to finish IO */ static void fail_release(struct peer *peer) @@ -1176,6 +1273,10 @@ static struct io_plan *peer_msgin(struct io_conn *conn, handle_query_channel_range(peer, msg); return peer_next_in(conn, peer); + case WIRE_REPLY_CHANNEL_RANGE: + handle_reply_channel_range(peer, msg); + return peer_next_in(conn, peer); + case WIRE_OPEN_CHANNEL: case WIRE_CHANNEL_REESTABLISH: case WIRE_ACCEPT_CHANNEL: @@ -1199,10 +1300,6 @@ static struct io_plan *peer_msgin(struct io_conn *conn, /* This will wait. */ return peer_next_in(conn, peer); - - case WIRE_REPLY_CHANNEL_RANGE: - /* FIXME: Implement */ - return peer_next_in(conn, peer); } /* BOLT #1: @@ -1612,6 +1709,10 @@ static struct io_plan *owner_msg_in(struct io_conn *conn, handle_local_add_channel(peer->daemon->rstate, dc->msg_in); } else if (type == WIRE_GOSSIP_LOCAL_CHANNEL_UPDATE) { handle_local_channel_update(peer, dc->msg_in); + } else if (type == WIRE_QUERY_CHANNEL_RANGE) { + handle_query_channel_range(peer, dc->msg_in); + } else if (type == WIRE_REPLY_CHANNEL_RANGE) { + handle_reply_channel_range(peer, dc->msg_in); } else { status_broken("peer %s: send us unknown msg of type %s", type_to_string(tmpctx, struct pubkey, &peer->id), @@ -2138,6 +2239,58 @@ static struct io_plan *send_timestamp_filter(struct io_conn *conn, out: return daemon_conn_read_next(conn, &daemon->master); } + +static struct io_plan *query_channel_range(struct io_conn *conn, + struct daemon *daemon, + const u8 *msg) +{ + struct pubkey id; + u32 first_blocknum, number_of_blocks; + struct peer *peer; + + if (!fromwire_gossip_query_channel_range(msg, &id, &first_blocknum, + &number_of_blocks)) + master_badmsg(WIRE_GOSSIP_QUERY_SCIDS, msg); + + peer = find_peer(daemon, &id); + if (!peer) { + status_broken("query_channel_range: unknown peer %s", + type_to_string(tmpctx, struct pubkey, &id)); + goto fail; + } + + if (!feature_offered(peer->lfeatures, LOCAL_GOSSIP_QUERIES)) { + status_broken("query_channel_range: no gossip_query support in peer %s", + type_to_string(tmpctx, struct pubkey, &id)); + goto fail; + } + + if (peer->query_channel_blocks) { + status_broken("query_channel_range: previous query active"); + goto fail; + } + + status_debug("sending query_channel_range for blocks %u+%u", + first_blocknum, number_of_blocks); + msg = towire_query_channel_range(NULL, &daemon->rstate->chain_hash, + first_blocknum, number_of_blocks); + queue_peer_msg(peer, take(msg)); + peer->first_channel_range = first_blocknum; + /* This uses 8 times as much as it needs to, but it's only for dev */ + peer->query_channel_blocks = tal_arrz(peer, u8, number_of_blocks); + peer->query_channel_scids = tal_arr(peer, struct short_channel_id, 0); + +out: + return daemon_conn_read_next(conn, &daemon->master); + +fail: + daemon_conn_send(&daemon->master, + take(towire_gossip_query_channel_range_reply(NULL, + 0, 0, + false, + NULL))); + goto out; +} #endif /* DEVELOPER */ static int make_listen_fd(int domain, void *addr, socklen_t len, bool mayfail) @@ -3361,10 +3514,15 @@ static struct io_plan *recv_req(struct io_conn *conn, struct daemon_conn *master case WIRE_GOSSIP_SEND_TIMESTAMP_FILTER: return send_timestamp_filter(conn, daemon, daemon->master.msg_in); + + case WIRE_GOSSIP_QUERY_CHANNEL_RANGE: + return query_channel_range(conn, daemon, daemon->master.msg_in); + #else case WIRE_GOSSIP_PING: case WIRE_GOSSIP_QUERY_SCIDS: case WIRE_GOSSIP_SEND_TIMESTAMP_FILTER: + case WIRE_GOSSIP_QUERY_CHANNEL_RANGE: break; #endif /* !DEVELOPER */ @@ -3378,6 +3536,7 @@ static struct io_plan *recv_req(struct io_conn *conn, struct daemon_conn *master case WIRE_GOSSIP_GETPEERS_REPLY: case WIRE_GOSSIP_PING_REPLY: case WIRE_GOSSIP_SCIDS_REPLY: + case WIRE_GOSSIP_QUERY_CHANNEL_RANGE_REPLY: case WIRE_GOSSIP_RESOLVE_CHANNEL_REPLY: case WIRE_GOSSIP_PEER_CONNECTED: case WIRE_GOSSIPCTL_CONNECT_TO_PEER_RESULT: diff --git a/gossipd/gossip_wire.csv b/gossipd/gossip_wire.csv index d062f144a288..2c41258318a4 100644 --- a/gossipd/gossip_wire.csv +++ b/gossipd/gossip_wire.csv @@ -172,6 +172,20 @@ gossip_send_timestamp_filter,,id,struct pubkey gossip_send_timestamp_filter,,first_timestamp,u32 gossip_send_timestamp_filter,,timestamp_range,u32 +# Test of query_channel_range. Master->gossipd +gossip_query_channel_range,3029 +gossip_query_channel_range,,id,struct pubkey +gossip_query_channel_range,,first_blocknum,u32 +gossip_query_channel_range,,number_of_blocks,u32 + +# Gossipd -> master +gossip_query_channel_range_reply,3129 +gossip_query_channel_range_reply,,final_first_block,u32 +gossip_query_channel_range_reply,,final_num_blocks,u32 +gossip_query_channel_range_reply,,final_complete,bool +gossip_query_channel_range_reply,,num,u16 +gossip_query_channel_range_reply,,scids,num*struct short_channel_id + # Given a short_channel_id, return the endpoints gossip_resolve_channel_request,3009 gossip_resolve_channel_request,,channel_id,struct short_channel_id diff --git a/lightningd/gossip_control.c b/lightningd/gossip_control.c index c1b8dd886b34..60c2c6890e6a 100644 --- a/lightningd/gossip_control.c +++ b/lightningd/gossip_control.c @@ -137,6 +137,7 @@ static unsigned gossip_msg(struct subd *gossip, const u8 *msg, const int *fds) case WIRE_GOSSIP_ROUTING_FAILURE: case WIRE_GOSSIP_MARK_CHANNEL_UNROUTABLE: case WIRE_GOSSIP_QUERY_SCIDS: + case WIRE_GOSSIP_QUERY_CHANNEL_RANGE: case WIRE_GOSSIP_SEND_TIMESTAMP_FILTER: case WIRE_GOSSIPCTL_PEER_DISCONNECT: case WIRE_GOSSIPCTL_PEER_IMPORTANT: @@ -150,6 +151,7 @@ static unsigned gossip_msg(struct subd *gossip, const u8 *msg, const int *fds) case WIRE_GOSSIP_GETPEERS_REPLY: case WIRE_GOSSIP_PING_REPLY: case WIRE_GOSSIP_SCIDS_REPLY: + case WIRE_GOSSIP_QUERY_CHANNEL_RANGE_REPLY: case WIRE_GOSSIP_RESOLVE_CHANNEL_REPLY: case WIRE_GOSSIPCTL_RELEASE_PEER_REPLY: case WIRE_GOSSIPCTL_RELEASE_PEER_REPLYFAIL: @@ -677,4 +679,86 @@ static const struct json_command dev_send_timestamp_filter = { "Send {peerid} the timestamp filter {first} {range}" }; AUTODATA(json_command, &dev_send_timestamp_filter); + +static void json_channel_range_reply(struct subd *gossip UNUSED, const u8 *reply, + const int *fds UNUSED, struct command *cmd) +{ + struct json_result *response = new_json_result(cmd); + u32 final_first_block, final_num_blocks; + bool final_complete; + struct short_channel_id *scids; + + if (!fromwire_gossip_query_channel_range_reply(tmpctx, reply, + &final_first_block, + &final_num_blocks, + &final_complete, + &scids)) { + command_fail(cmd, LIGHTNINGD, + "Gossip gave bad gossip_query_channel_range_reply"); + return; + } + + if (final_num_blocks == 0 && final_num_blocks == 0 && !final_complete) { + command_fail(cmd, LIGHTNINGD, + "Gossip refused to query peer"); + return; + } + + json_object_start(response, NULL); + json_add_num(response, "final_first_block", final_first_block); + json_add_num(response, "final_num_blocks", final_num_blocks); + json_add_bool(response, "final_complete", final_complete); + json_array_start(response, "short_channel_ids"); + for (size_t i = 0; i < tal_count(scids); i++) + json_add_short_channel_id(response, NULL, &scids[i]); + json_array_end(response); + json_object_end(response); + command_success(cmd, response); +} + +static void json_dev_query_channel_range(struct command *cmd, + const char *buffer, + const jsmntok_t *params) +{ + u8 *msg; + jsmntok_t *idtok, *firsttok, *numtok; + struct pubkey id; + u32 first, num; + + if (!json_get_params(cmd, buffer, params, + "id", &idtok, + "first", &firsttok, + "num", &numtok, + NULL)) { + return; + } + + if (!json_tok_pubkey(buffer, idtok, &id)) { + command_fail(cmd, JSONRPC2_INVALID_PARAMS, + "'%.*s' is not a valid id", + idtok->end - idtok->start, + buffer + idtok->start); + return; + } + + if (!json_tok_number(buffer, firsttok, &first) + || !json_tok_number(buffer, numtok, &num)) { + command_fail(cmd, JSONRPC2_INVALID_PARAMS, + "first and num must be numbers"); + return; + } + + /* Tell gossipd, since this is a gossip query. */ + msg = towire_gossip_query_channel_range(cmd, &id, first, num); + subd_req(cmd->ld->gossip, cmd->ld->gossip, + take(msg), -1, 0, json_channel_range_reply, cmd); + command_still_pending(cmd); +} + +static const struct json_command dev_query_channel_range_command = { + "dev-query-channel-range", + json_dev_query_channel_range, + "Query {peerid} for short_channel_ids for {first} block + {num} blocks" +}; +AUTODATA(json_command, &dev_query_channel_range_command); #endif /* DEVELOPER */ diff --git a/tests/test_lightningd.py b/tests/test_lightningd.py index 8caf535b46c4..a948bfb7d49f 100644 --- a/tests/test_lightningd.py +++ b/tests/test_lightningd.py @@ -2622,6 +2622,98 @@ def test_ping(self): l1.daemon.wait_for_log('Got pong 1000 bytes \({}\.\.\.\)' .format(l2.info['version'])) + @unittest.skipIf(not DEVELOPER, "needs DEVELOPER=1") + def test_gossip_query_channel_range(self): + l1 = self.node_factory.get_node() + l2 = self.node_factory.get_node() + l3 = self.node_factory.get_node() + + l1.rpc.connect(l2.info['id'], 'localhost', l2.port) + l2.rpc.connect(l3.info['id'], 'localhost', l3.port) + + # Make public channels. + scid12 = self.fund_channel(l1, l2, 10**5) + block12 = int(scid12.split(':')[0]) + scid23 = self.fund_channel(l2, l3, 10**5) + block23 = int(scid23.split(':')[0]) + bitcoind.generate_block(5) + sync_blockheight([l2, l3]) + + # Make sure l2 has received all the gossip. + l2.daemon.wait_for_logs(['Received node_announcement for node ' + l1.info['id'], + 'Received node_announcement for node ' + l3.info['id']]) + + # l1 asks for all channels, gets both. + ret = l1.rpc.dev_query_channel_range(id=l2.info['id'], + first=0, + num=1000000) + + assert ret['final_first_block'] == 0 + assert ret['final_num_blocks'] == 1000000 + assert ret['final_complete'] + assert len(ret['short_channel_ids']) == 2 + assert ret['short_channel_ids'][0] == scid12 + assert ret['short_channel_ids'][1] == scid23 + + # Does not include scid12 + ret = l1.rpc.dev_query_channel_range(id=l2.info['id'], + first=0, + num=block12) + assert ret['final_first_block'] == 0 + assert ret['final_num_blocks'] == block12 + assert ret['final_complete'] + assert len(ret['short_channel_ids']) == 0 + + # Does include scid12 + ret = l1.rpc.dev_query_channel_range(id=l2.info['id'], + first=0, + num=block12 + 1) + assert ret['final_first_block'] == 0 + assert ret['final_num_blocks'] == block12 + 1 + assert ret['final_complete'] + assert len(ret['short_channel_ids']) == 1 + assert ret['short_channel_ids'][0] == scid12 + + # Doesn't include scid23 + ret = l1.rpc.dev_query_channel_range(id=l2.info['id'], + first=0, + num=block23) + assert ret['final_first_block'] == 0 + assert ret['final_num_blocks'] == block23 + assert ret['final_complete'] + assert len(ret['short_channel_ids']) == 1 + assert ret['short_channel_ids'][0] == scid12 + + # Does include scid23 + ret = l1.rpc.dev_query_channel_range(id=l2.info['id'], + first=block12, + num=block23 - block12 + 1) + assert ret['final_first_block'] == block12 + assert ret['final_num_blocks'] == block23 - block12 + 1 + assert ret['final_complete'] + assert len(ret['short_channel_ids']) == 2 + assert ret['short_channel_ids'][0] == scid12 + assert ret['short_channel_ids'][1] == scid23 + + # Only includes scid23 + ret = l1.rpc.dev_query_channel_range(id=l2.info['id'], + first=block23, + num=1) + assert ret['final_first_block'] == block23 + assert ret['final_num_blocks'] == 1 + assert ret['final_complete'] + assert len(ret['short_channel_ids']) == 1 + assert ret['short_channel_ids'][0] == scid23 + + # Past both + ret = l1.rpc.dev_query_channel_range(id=l2.info['id'], + first=block23 + 1, + num=1000000) + assert ret['final_first_block'] == block23 + 1 + assert ret['final_num_blocks'] == 1000000 + assert ret['final_complete'] + assert len(ret['short_channel_ids']) == 0 + @unittest.skipIf(not DEVELOPER, "needs DEVELOPER=1") def test_query_short_channel_id(self): l1 = self.node_factory.get_node(options={'log-level': 'io'}) From 9e51e196c14906fea850cef5a4d36360cd568fed Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Mon, 4 Jun 2018 13:58:02 +0930 Subject: [PATCH 21/46] gossipd: dev-set-max-scids-encode-size to artificially force "full" replies. We cap each reply at a single one, which forces the code into our recursion logic. Signed-off-by: Rusty Russell --- gossipd/gossip.c | 24 +++++++++++++++++++++++- gossipd/gossip_wire.csv | 4 ++++ lightningd/gossip_control.c | 34 ++++++++++++++++++++++++++++++++++ tests/test_lightningd.py | 15 +++++++++++++++ 4 files changed, 76 insertions(+), 1 deletion(-) diff --git a/gossipd/gossip.c b/gossipd/gossip.c index 5aa661d09f27..dfeee3d43c2b 100644 --- a/gossipd/gossip.c +++ b/gossipd/gossip.c @@ -101,6 +101,10 @@ HTABLE_DEFINE_TYPE(struct important_peerid, important_peerid_eq, important_peerid_map); +#if DEVELOPER +static u32 max_scids_encode_bytes = -1U; +#endif + struct daemon { /* Who am I? */ struct pubkey id; @@ -439,6 +443,10 @@ static void encode_add_short_channel_id(u8 **encoded, static bool encode_short_channel_ids_end(u8 **encoded, size_t max_bytes) { +#if DEVELOPER + if (tal_len(*encoded) > max_scids_encode_bytes) + return false; +#endif return tal_len(*encoded) <= max_bytes; } @@ -2291,6 +2299,17 @@ static struct io_plan *query_channel_range(struct io_conn *conn, NULL))); goto out; } + +static struct io_plan *dev_set_max_scids_encode_size(struct io_conn *conn, + struct daemon *daemon, + const u8 *msg) +{ + if (!fromwire_gossip_dev_set_max_scids_encode_size(msg, + &max_scids_encode_bytes)) + master_badmsg(WIRE_GOSSIP_DEV_SET_MAX_SCIDS_ENCODE_SIZE, msg); + + return daemon_conn_read_next(conn, &daemon->master); +} #endif /* DEVELOPER */ static int make_listen_fd(int domain, void *addr, socklen_t len, bool mayfail) @@ -3518,11 +3537,15 @@ static struct io_plan *recv_req(struct io_conn *conn, struct daemon_conn *master case WIRE_GOSSIP_QUERY_CHANNEL_RANGE: return query_channel_range(conn, daemon, daemon->master.msg_in); + case WIRE_GOSSIP_DEV_SET_MAX_SCIDS_ENCODE_SIZE: + return dev_set_max_scids_encode_size(conn, daemon, + daemon->master.msg_in); #else case WIRE_GOSSIP_PING: case WIRE_GOSSIP_QUERY_SCIDS: case WIRE_GOSSIP_SEND_TIMESTAMP_FILTER: case WIRE_GOSSIP_QUERY_CHANNEL_RANGE: + case WIRE_GOSSIP_DEV_SET_MAX_SCIDS_ENCODE_SIZE: break; #endif /* !DEVELOPER */ @@ -3581,7 +3604,6 @@ int main(int argc, char *argv[]) timers_init(&daemon->timers, time_mono()); daemon->broadcast_interval = 30000; daemon->last_announce_timestamp = 0; - /* stdin == control */ daemon_conn_init(daemon, &daemon->master, STDIN_FILENO, recv_req, master_gone); diff --git a/gossipd/gossip_wire.csv b/gossipd/gossip_wire.csv index 2c41258318a4..7d2461632f36 100644 --- a/gossipd/gossip_wire.csv +++ b/gossipd/gossip_wire.csv @@ -186,6 +186,10 @@ gossip_query_channel_range_reply,,final_complete,bool gossip_query_channel_range_reply,,num,u16 gossip_query_channel_range_reply,,scids,num*struct short_channel_id +# Set artificial maximum reply_channel_range size. Master->gossipd +gossip_dev_set_max_scids_encode_size,3030 +gossip_dev_set_max_scids_encode_size,,max,u32 + # Given a short_channel_id, return the endpoints gossip_resolve_channel_request,3009 gossip_resolve_channel_request,,channel_id,struct short_channel_id diff --git a/lightningd/gossip_control.c b/lightningd/gossip_control.c index 60c2c6890e6a..051f45d8e799 100644 --- a/lightningd/gossip_control.c +++ b/lightningd/gossip_control.c @@ -139,6 +139,7 @@ static unsigned gossip_msg(struct subd *gossip, const u8 *msg, const int *fds) case WIRE_GOSSIP_QUERY_SCIDS: case WIRE_GOSSIP_QUERY_CHANNEL_RANGE: case WIRE_GOSSIP_SEND_TIMESTAMP_FILTER: + case WIRE_GOSSIP_DEV_SET_MAX_SCIDS_ENCODE_SIZE: case WIRE_GOSSIPCTL_PEER_DISCONNECT: case WIRE_GOSSIPCTL_PEER_IMPORTANT: case WIRE_GOSSIPCTL_PEER_DISCONNECTED: @@ -761,4 +762,37 @@ static const struct json_command dev_query_channel_range_command = { "Query {peerid} for short_channel_ids for {first} block + {num} blocks" }; AUTODATA(json_command, &dev_query_channel_range_command); + +static void json_dev_set_max_scids_encode_size(struct command *cmd, + const char *buffer, + const jsmntok_t *params) +{ + u8 *msg; + jsmntok_t *maxtok; + u32 max; + + if (!json_get_params(cmd, buffer, params, + "max", &maxtok, + NULL)) { + return; + } + + if (!json_tok_number(buffer, maxtok, &max)) { + command_fail(cmd, JSONRPC2_INVALID_PARAMS, + "max must be a number"); + return; + } + + msg = towire_gossip_dev_set_max_scids_encode_size(NULL, max); + subd_send_msg(cmd->ld->gossip, take(msg)); + + command_success(cmd, null_response(cmd)); +} + +static const struct json_command dev_set_max_scids_encode_size = { + "dev-set-max-scids-encode-size", + json_dev_set_max_scids_encode_size, + "Set {max} bytes of short_channel_ids per reply_channel_range" +}; +AUTODATA(json_command, &dev_set_max_scids_encode_size); #endif /* DEVELOPER */ diff --git a/tests/test_lightningd.py b/tests/test_lightningd.py index a948bfb7d49f..8d2bd7b38f3d 100644 --- a/tests/test_lightningd.py +++ b/tests/test_lightningd.py @@ -2714,6 +2714,21 @@ def test_gossip_query_channel_range(self): assert ret['final_complete'] assert len(ret['short_channel_ids']) == 0 + # Make l2 split reply into two. + l2.rpc.dev_set_max_scids_encode_size(max=9) + ret = l1.rpc.dev_query_channel_range(id=l2.info['id'], + first=0, + num=1000000) + + # It should definitely have split + assert ret['final_first_block'] != 0 or ret['final_num_blocks'] != 1000000 + assert ret['final_complete'] + assert len(ret['short_channel_ids']) == 2 + assert ret['short_channel_ids'][0] == scid12 + assert ret['short_channel_ids'][1] == scid23 + + l2.daemon.wait_for_log('queue_channel_ranges full: splitting') + @unittest.skipIf(not DEVELOPER, "needs DEVELOPER=1") def test_query_short_channel_id(self): l1 = self.node_factory.get_node(options={'log-level': 'io'}) From 083a2cee0744e93c9f43ab553466b0c6fe01867f Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Mon, 4 Jun 2018 13:58:02 +0930 Subject: [PATCH 22/46] zlib: add as a requirement. Signed-off-by: Rusty Russell --- Makefile | 2 +- README.md | 2 +- doc/INSTALL.md | 3 ++- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index eaea8fe58b65..436e1b78fb6c 100644 --- a/Makefile +++ b/Makefile @@ -175,7 +175,7 @@ CFLAGS = $(CPPFLAGS) $(CWARNFLAGS) $(CDEBUGFLAGS) -I $(CCANDIR) $(EXTERNAL_INCLU CONFIGURATOR_CC := $(CC) LDFLAGS = $(PIE_LDFLAGS) -LDLIBS = -L/usr/local/lib -lm -lgmp -lsqlite3 $(COVFLAGS) +LDLIBS = -L/usr/local/lib -lm -lgmp -lsqlite3 -lz $(COVFLAGS) default: all-programs all-test-programs diff --git a/README.md b/README.md index 19513768c5fe..6f8066d23d13 100644 --- a/README.md +++ b/README.md @@ -58,7 +58,7 @@ For the impatient here's the gist of it for Ubuntu and Debian: sudo apt-get update sudo apt-get install -y \ autoconf automake build-essential git libtool libgmp-dev \ - libsqlite3-dev python python3 net-tools + libsqlite3-dev python python3 net-tools zlib1g-dev git clone https://github.com/ElementsProject/lightning.git cd lightning make diff --git a/doc/INSTALL.md b/doc/INSTALL.md index 4cf18f95a6dc..1a426bd73e7b 100644 --- a/doc/INSTALL.md +++ b/doc/INSTALL.md @@ -15,6 +15,7 @@ Library Requirements You will need several development libraries: * libsqlite3: for database support. * libgmp: for secp256k1 +* zlib: for compression routines. For actually doing development and running the tests, you will also need: * pip3: to install python-bitcoinlib @@ -34,7 +35,7 @@ Get dependencies: sudo apt-get update sudo apt-get install -y \ autoconf automake build-essential git libtool libgmp-dev \ - libsqlite3-dev python python3 net-tools + libsqlite3-dev python python3 net-tools zlib1g-dev If you don't have Bitcoin installed locally you'll need to install that as well: From f52245d4427b99159e5efca227bd3a016539a538 Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Mon, 4 Jun 2018 13:58:02 +0930 Subject: [PATCH 23/46] gossipd: support and use zlib encoding in short_channel_id encoding. We still use uncompressed if zlib turns out to be larger. Signed-off-by: Rusty Russell --- contrib/Dockerfile.builder | 3 +- contrib/Dockerfile.builder.fedora | 3 +- contrib/Dockerfile.builder.i386 | 3 +- gossipd/gossip.c | 126 +++++++++++++++++++++++------- 4 files changed, 104 insertions(+), 31 deletions(-) diff --git a/contrib/Dockerfile.builder b/contrib/Dockerfile.builder index ba1f901a9f49..f3821e6e7d32 100644 --- a/contrib/Dockerfile.builder +++ b/contrib/Dockerfile.builder @@ -32,7 +32,8 @@ RUN apt-get -qq update && \ python3-setuptools \ python-pkg-resources \ shellcheck \ - wget && \ + wget \ + zlib1g-dev && \ rm -rf /var/lib/apt/lists/* ENV LANGUAGE=en_US.UTF-8 diff --git a/contrib/Dockerfile.builder.fedora b/contrib/Dockerfile.builder.fedora index 91ab2c71ccf4..517e4da345e4 100644 --- a/contrib/Dockerfile.builder.fedora +++ b/contrib/Dockerfile.builder.fedora @@ -17,7 +17,8 @@ RUN dnf update -y && \ python3-setuptools \ net-tools \ valgrind \ - wget && \ + wget \ + zlib-devel && \ dnf clean all RUN wget https://bitcoin.org/bin/bitcoin-core-$BITCOIN_VERSION/bitcoin-$BITCOIN_VERSION-x86_64-linux-gnu.tar.gz -O bitcoin.tar.gz && \ diff --git a/contrib/Dockerfile.builder.i386 b/contrib/Dockerfile.builder.i386 index b425b47b8eee..c1421c113440 100644 --- a/contrib/Dockerfile.builder.i386 +++ b/contrib/Dockerfile.builder.i386 @@ -32,7 +32,8 @@ RUN apt-get -qq update && \ python3-setuptools \ python-pkg-resources \ shellcheck \ - wget && \ + wget \ + zlib1g-dev && \ rm -rf /var/lib/apt/lists/* ENV LANGUAGE=en_US.UTF-8 diff --git a/gossipd/gossip.c b/gossipd/gossip.c index dfeee3d43c2b..44072c9dcb7d 100644 --- a/gossipd/gossip.c +++ b/gossipd/gossip.c @@ -56,6 +56,7 @@ #include #include #include +#include #define GOSSIP_MAX_REACH_ATTEMPTS 10 @@ -64,6 +65,18 @@ #define INITIAL_WAIT_SECONDS 1 #define MAX_WAIT_SECONDS 300 +/* BOLT #7: + * + * Encoding types: + * * `0`: uncompressed array of `short_channel_id` types, in ascending order. + * * `1`: array of `short_channel_id` types, in ascending order, compressed with + * zlib[1](#reference-1) + */ +enum scid_encode_types { + SHORTIDS_UNCOMPRESSED = 0, + SHORTIDS_ZLIB = 1 +}; + /* We put everything in this struct (redundantly) to pass it to timer cb */ struct important_peerid { struct daemon *daemon; @@ -424,14 +437,8 @@ static void reached_peer(struct peer *peer, struct io_conn *conn) static u8 *encode_short_channel_ids_start(const tal_t *ctx) { - /* BOLT #7: - * - * Encoding types: - * * `0`: uncompressed array of `short_channel_id` types, in ascending - * order. - */ u8 *encoded = tal_arr(tmpctx, u8, 0); - towire_u8(&encoded, 0); + towire_u8(&encoded, SHORTIDS_ZLIB); return encoded; } @@ -441,8 +448,48 @@ static void encode_add_short_channel_id(u8 **encoded, towire_short_channel_id(encoded, scid); } +static u8 *zencode_scids(const tal_t *ctx, const u8 *scids, size_t len) +{ + u8 *z; + int err; + unsigned long compressed_len = len; + + /* Prefer to fail if zlib makes it larger */ + z = tal_arr(ctx, u8, len); + err = compress2(z, &compressed_len, scids, len, Z_BEST_COMPRESSION); + if (err == Z_OK) { + status_trace("short_ids compressed %zu into %lu", + len, compressed_len); + tal_resize(&z, compressed_len); + return z; + } + status_trace("short_ids compress %zu returned %i:" + " not compresssing", len, err); + return NULL; +} + static bool encode_short_channel_ids_end(u8 **encoded, size_t max_bytes) { + u8 *z; + + switch ((enum scid_encode_types)(*encoded)[0]) { + case SHORTIDS_ZLIB: + z = zencode_scids(tmpctx, *encoded + 1, tal_len(*encoded) - 1); + if (z) { + tal_resize(encoded, 1 + tal_len(z)); + memcpy((*encoded) + 1, z, tal_len(z)); + goto check_length; + } + (*encoded)[0] = SHORTIDS_UNCOMPRESSED; + /* Fall thru */ + case SHORTIDS_UNCOMPRESSED: + goto check_length; + } + + status_failed(STATUS_FAIL_INTERNAL_ERROR, + "Unknown short_ids encoding %u", (*encoded)[0]); + +check_length: #if DEVELOPER if (tal_len(*encoded) > max_scids_encode_bytes) return false; @@ -792,44 +839,67 @@ static u8 *handle_gossip_msg(struct daemon *daemon, const u8 *msg, return NULL; } -/* BOLT #7: - * - * the first byte indicates the encoding, the rest contains the data. - * - * Encoding types: - * * `0`: uncompressed array of `short_channel_id` types, in ascending order. - */ +static u8 *unzlib(const tal_t *ctx, const u8 *encoded, size_t len) +{ + /* http://www.zlib.net/zlib_tech.html gives 1032:1 as worst-case, + * which is 67632120 bytes for us. But they're not encoding zeroes, + * and each scid must be unique. So 1MB is far more reasonable. */ + unsigned long unclen = 1024*1024; + int zerr; + u8 *unc = tal_arr(ctx, u8, unclen); + + zerr = uncompress(unc, &unclen, encoded, len); + if (zerr != Z_OK) { + status_trace("unzlib: error %i", zerr); + return tal_free(unc); + } + + /* Truncate and return. */ + tal_resize(&unc, unclen); + return unc; +} + static struct short_channel_id *decode_short_ids(const tal_t *ctx, const u8 *encoded) { struct short_channel_id *scids; size_t max = tal_len(encoded), n; - u8 type; + enum scid_encode_types type; /* BOLT #7: * * The receiver: - * - if the first byte of `encoded_short_ids` is not zero: + * - if the first byte of `encoded_short_ids` is not a known encoding + * type: * - MAY fail the connection * - if `encoded_short_ids` does not decode into a whole number of * `short_channel_id`: * - MAY fail the connection */ type = fromwire_u8(&encoded, &max); - if (type != 0) - return NULL; + switch (type) { + case SHORTIDS_ZLIB: + encoded = unzlib(tmpctx, encoded, max); + if (!encoded) + return NULL; + status_trace("Uncompressed %zu into %zu bytes (%s)", + max, tal_len(encoded), tal_hex(tmpctx, encoded)); + max = tal_len(encoded); + /* fall thru */ + case SHORTIDS_UNCOMPRESSED: + n = 0; + scids = tal_arr(ctx, struct short_channel_id, n); + while (max) { + tal_resize(&scids, n+1); + fromwire_short_channel_id(&encoded, &max, &scids[n++]); + } - n = 0; - scids = tal_arr(ctx, struct short_channel_id, n); - while (max) { - tal_resize(&scids, n+1); - fromwire_short_channel_id(&encoded, &max, &scids[n++]); + /* encoded is set to NULL if we ran over */ + if (!encoded) + return tal_free(scids); + return scids; } - - /* encoded is set to NULL if we ran over */ - if (!encoded) - return tal_free(scids); - return scids; + return NULL; } static void handle_query_short_channel_ids(struct peer *peer, u8 *msg) From e9483f12b6488ec3c2c70d3aff26aa9f87b34428 Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Mon, 4 Jun 2018 13:58:02 +0930 Subject: [PATCH 24/46] test_lightningd.py: actually make sure we do zlib encoding. Previous replies weren't large enough; add another channel and then use IO tracing to make sure the reply is zlib encoded. Signed-off-by: Rusty Russell --- tests/test_lightningd.py | 35 +++++++++++++++++++++++++++++++++-- 1 file changed, 33 insertions(+), 2 deletions(-) diff --git a/tests/test_lightningd.py b/tests/test_lightningd.py index 8d2bd7b38f3d..47474e33f375 100644 --- a/tests/test_lightningd.py +++ b/tests/test_lightningd.py @@ -2624,12 +2624,14 @@ def test_ping(self): @unittest.skipIf(not DEVELOPER, "needs DEVELOPER=1") def test_gossip_query_channel_range(self): - l1 = self.node_factory.get_node() + l1 = self.node_factory.get_node(options={'log-level': 'io'}) l2 = self.node_factory.get_node() l3 = self.node_factory.get_node() + l4 = self.node_factory.get_node() l1.rpc.connect(l2.info['id'], 'localhost', l2.port) l2.rpc.connect(l3.info['id'], 'localhost', l3.port) + l3.rpc.connect(l4.info['id'], 'localhost', l4.port) # Make public channels. scid12 = self.fund_channel(l1, l2, 10**5) @@ -2726,9 +2728,38 @@ def test_gossip_query_channel_range(self): assert len(ret['short_channel_ids']) == 2 assert ret['short_channel_ids'][0] == scid12 assert ret['short_channel_ids'][1] == scid23 - l2.daemon.wait_for_log('queue_channel_ranges full: splitting') + # This should actually be large enough for zlib to kick in! + self.fund_channel(l3, l4, 10**5) + bitcoind.generate_block(5) + l2.daemon.wait_for_log('Received node_announcement for node ' + l4.info['id']) + + # Turn on IO logging in l1 channeld. + subprocess.run(['kill', '-USR1', l1.subd_pid('channeld')]) + + # Restore infinite encode size. + l2.rpc.dev_set_max_scids_encode_size(max=(2**32 - 1)) + ret = l1.rpc.dev_query_channel_range(id=l2.info['id'], + first=0, + num=65535) + l1.daemon.wait_for_log( + # WIRE_REPLY_CHANNEL_RANGE + '\[IN\] 0108' + + # chain_hash + '................................................................' + + # first_blocknum + '00000000' + + # number_of_blocks + '0000ffff' + + # complete + '01' + + # length + '....' + + # encoding + '01' + ) + @unittest.skipIf(not DEVELOPER, "needs DEVELOPER=1") def test_query_short_channel_id(self): l1 = self.node_factory.get_node(options={'log-level': 'io'}) From 5ec454c7b2f9d5b90400c3eba7dcfcbec80f82f9 Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Mon, 4 Jun 2018 13:58:02 +0930 Subject: [PATCH 25/46] gossipd: don't queue node_announce unless we've queued channel_announce. We *accept* a node_announce if we have a channel_announce, but we can't queue it until we queue the channel_announce, which we only do once we have recieved a channel_update. Signed-off-by: Rusty Russell --- gossipd/routing.c | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/gossipd/routing.c b/gossipd/routing.c index 83b53cf5aba0..03ddd362bd3f 100644 --- a/gossipd/routing.c +++ b/gossipd/routing.c @@ -1180,6 +1180,7 @@ static struct wireaddr *read_addresses(const tal_t *ctx, const u8 *ser) return wireaddrs; } +/* We've received a channel_announce for a channel attached to this node */ static bool node_has_public_channels(struct node *node) { for (size_t i = 0; i < tal_count(node->chans); i++) @@ -1188,6 +1189,20 @@ static bool node_has_public_channels(struct node *node) return false; } +/* We can *send* a channel_announce for a channel attached to this node: + * we only send once we have a channel_update. */ +static bool node_has_broadcastable_channels(struct node *node) +{ + for (size_t i = 0; i < tal_count(node->chans); i++) { + if (!is_chan_public(node->chans[i])) + continue; + if (is_halfchan_defined(&node->chans[i]->half[0]) + || is_halfchan_defined(&node->chans[i]->half[1])) + return true; + } + return false; +} + bool routing_add_node_announcement(struct routing_state *rstate, const u8 *msg TAKES) { struct node *node; @@ -1229,7 +1244,7 @@ bool routing_add_node_announcement(struct routing_state *rstate, const u8 *msg T * order. It's not vital, but would be nice to fix. */ /* We might be waiting for channel_announce to be released. */ - node->node_announcement_public = node_has_public_channels(node); + node->node_announcement_public = node_has_broadcastable_channels(node); if (node->node_announcement_public) insert_broadcast(rstate->broadcasts, node->node_announcement, timestamp); From 383b309a00328574df6c79e02b81e8ad09b368ad Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Mon, 4 Jun 2018 13:58:52 +0930 Subject: [PATCH 26/46] utils: make subd_pid return the *last* pid, in case we restarted daemon. Signed-off-by: Rusty Russell --- tests/utils.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/tests/utils.py b/tests/utils.py index 850667f3c858..3bafa0df3193 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -472,8 +472,13 @@ def fund_channel(self, l2, amount): def subd_pid(self, subd): """Get the process id of the given subdaemon, eg channeld or gossipd""" - pidline = self.daemon.is_in_log('lightning_{}.*: pid [0-9]*,'.format(subd)) - return re.search(r'pid ([0-9]*),', pidline).group(1) + ex = re.compile(r'lightning_{}.*: pid ([0-9]*),'.format(subd)) + # Make sure we get latest one if it's restarted! + for l in reversed(self.daemon.logs): + group = ex.search(l) + if group: + return group.group(1) + raise ValueError("No daemon {} found".format(subd)) def is_channel_active(self, chanid): channels = self.rpc.listchannels()['channels'] From 035d6067e47430474a918ed730054b2e85082e4c Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Mon, 4 Jun 2018 14:08:39 +0930 Subject: [PATCH 27/46] Rename consider_own_node_announce to maybe_send_own_node_announce. Suggested-by: @cdecker Signed-off-by: Rusty Russell --- gossipd/gossip.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/gossipd/gossip.c b/gossipd/gossip.c index 44072c9dcb7d..566e3dc10ffd 100644 --- a/gossipd/gossip.c +++ b/gossipd/gossip.c @@ -782,7 +782,7 @@ static void send_node_announcement(struct daemon *daemon) } /* Should we announce our own node? */ -static void consider_own_node_announce(struct daemon *daemon) +static void maybe_send_own_node_announce(struct daemon *daemon) { if (!daemon->rstate->local_channel_announced) return; @@ -831,7 +831,7 @@ static u8 *handle_gossip_msg(struct daemon *daemon, const u8 *msg, if (err) return err; /* In case we just announced a new local channel. */ - consider_own_node_announce(daemon); + maybe_send_own_node_announce(daemon); break; } @@ -1754,7 +1754,7 @@ static void handle_local_channel_update(struct peer *peer, const u8 *msg) queue_peer_msg(peer, take(cupdate)); /* That channel_update might trigger our first channel_announcement */ - consider_own_node_announce(peer->daemon); + maybe_send_own_node_announce(peer->daemon); } /** @@ -2875,7 +2875,7 @@ static struct io_plan *gossip_activate(struct daemon_conn *master, /* Now we know our addresses, re-announce ourselves if we have a * channel, in case options have changed. */ - consider_own_node_announce(daemon); + maybe_send_own_node_announce(daemon); /* OK, we're ready! */ daemon_conn_send(&daemon->master, @@ -3435,7 +3435,7 @@ static struct io_plan *handle_txout_reply(struct io_conn *conn, master_badmsg(WIRE_GOSSIP_GET_TXOUT_REPLY, msg); handle_pending_cannouncement(daemon->rstate, &scid, satoshis, outscript); - consider_own_node_announce(daemon); + maybe_send_own_node_announce(daemon); return daemon_conn_read_next(conn, &daemon->master); } From e4457e59ed34d38e405f2eb880dba1f0c9f84ae0 Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Mon, 4 Jun 2018 14:08:41 +0930 Subject: [PATCH 28/46] pytest: test_gossip_jsonrpc can always test aliases. The whole test is under DEVELOPER anyway, but even if it weren't it would work fine. Signed-off-by: Rusty Russell --- tests/test_lightningd.py | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/tests/test_lightningd.py b/tests/test_lightningd.py index 47474e33f375..7247ec19f5d2 100644 --- a/tests/test_lightningd.py +++ b/tests/test_lightningd.py @@ -2455,16 +2455,14 @@ def test_gossip_jsonrpc(self): assert n2['nodeid'] == l2.info['id'] # Might not have seen other node-announce yet. - # TODO(cdecker) Can't check these without DEVELOPER=1, re-enable after we get alias and color into getinfo - if DEVELOPER: - assert n1['alias'].startswith('JUNIORBEAM') - assert n1['color'] == '0266e4' - if 'alias' not in n2: - assert 'color' not in n2 - assert 'addresses' not in n2 - else: - assert n2['alias'].startswith('SILENTARTIST') - assert n2['color'] == '022d22' + assert n1['alias'].startswith('JUNIORBEAM') + assert n1['color'] == '0266e4' + if 'alias' not in n2: + assert 'color' not in n2 + assert 'addresses' not in n2 + else: + assert n2['alias'].startswith('SILENTARTIST') + assert n2['color'] == '022d22' assert [c['active'] for c in l1.rpc.listchannels()['channels']] == [True, True] assert [c['public'] for c in l1.rpc.listchannels()['channels']] == [True, True] From 1bb77132744927fab1a61b682ace2ad193e4c16e Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Mon, 4 Jun 2018 14:08:42 +0930 Subject: [PATCH 29/46] gossipd: minor cleanups. Suggested-by: @cdecker Signed-off-by: Rusty Russell --- gossipd/gossip.c | 4 ++-- lightningd/gossip_control.c | 2 ++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/gossipd/gossip.c b/gossipd/gossip.c index 566e3dc10ffd..75e7cff24629 100644 --- a/gossipd/gossip.c +++ b/gossipd/gossip.c @@ -904,7 +904,7 @@ static struct short_channel_id *decode_short_ids(const tal_t *ctx, static void handle_query_short_channel_ids(struct peer *peer, u8 *msg) { - struct routing_state *rstate =peer->daemon->rstate; + struct routing_state *rstate = peer->daemon->rstate; struct bitcoin_blkid chain; u8 *encoded; struct short_channel_id *scids; @@ -930,7 +930,7 @@ static void handle_query_short_channel_ids(struct peer *peer, u8 *msg) * - MAY fail the connection. */ if (peer->scid_queries || peer->scid_query_nodes) { - peer_error(peer, "Bad second query_short_channel_ids"); + peer_error(peer, "Bad concurrent query_short_channel_ids"); return; } diff --git a/lightningd/gossip_control.c b/lightningd/gossip_control.c index 051f45d8e799..0e20dac1bbb3 100644 --- a/lightningd/gossip_control.c +++ b/lightningd/gossip_control.c @@ -706,6 +706,8 @@ static void json_channel_range_reply(struct subd *gossip UNUSED, const u8 *reply } json_object_start(response, NULL); + /* As this is a dev interface, we don't bother saving and + * returning all the replies, just the final one. */ json_add_num(response, "final_first_block", final_first_block); json_add_num(response, "final_num_blocks", final_num_blocks); json_add_bool(response, "final_complete", final_complete); From 0d4b7eaa2c1fc69e580e36ea9da0627847375fb0 Mon Sep 17 00:00:00 2001 From: Christian Decker Date: Mon, 4 Jun 2018 14:46:05 +0200 Subject: [PATCH 30/46] topo: Have chain_topology track both min and max block heights Signed-off-by: Christian Decker --- lightningd/chaintopology.c | 23 +++++++++++++---------- lightningd/chaintopology.h | 4 ++-- 2 files changed, 15 insertions(+), 12 deletions(-) diff --git a/lightningd/chaintopology.c b/lightningd/chaintopology.c index 3b19bcde9809..767de73abb5c 100644 --- a/lightningd/chaintopology.c +++ b/lightningd/chaintopology.c @@ -422,6 +422,7 @@ static void add_tip(struct chain_topology *topo, struct block *b) filter_block_txs(topo, b); block_map_add(&topo->block_map, b); + topo->max_blockheight = b->height; } static struct block *new_block(struct chain_topology *topo, @@ -509,7 +510,7 @@ static void init_topo(struct bitcoind *bitcoind UNUSED, struct bitcoin_block *blk, struct chain_topology *topo) { - topo->root = new_block(topo, blk, topo->first_blocknum); + topo->root = new_block(topo, blk, topo->max_blockheight); block_map_add(&topo->block_map, topo->root); topo->tip = topo->prev_tip = topo->root; @@ -533,32 +534,32 @@ static void get_init_blockhash(struct bitcoind *bitcoind, u32 blockcount, /* If bitcoind's current blockheight is below the requested height, just * go back to that height. This might be a new node catching up, or * bitcoind is processing a reorg. */ - if (blockcount < topo->first_blocknum) { + if (blockcount < topo->max_blockheight) { if (bitcoind->ld->config.rescan < 0) { /* Absolute blockheight, but bitcoind's blockheight isn't there yet */ /* Protect against underflow in subtraction. * Possible in regtest mode. */ if (blockcount < 1) - topo->first_blocknum = 0; + topo->max_blockheight = 0; else - topo->first_blocknum = blockcount - 1; - } else if (topo->first_blocknum == UINT32_MAX) { + topo->max_blockheight = blockcount - 1; + } else if (topo->max_blockheight == UINT32_MAX) { /* Relative rescan, but we didn't know the blockheight */ /* Protect against underflow in subtraction. * Possible in regtest mode. */ if (blockcount < bitcoind->ld->config.rescan) - topo->first_blocknum = 0; + topo->max_blockheight = 0; else - topo->first_blocknum = blockcount - bitcoind->ld->config.rescan; + topo->max_blockheight = blockcount - bitcoind->ld->config.rescan; } } /* Rollback to the given blockheight, so we start track * correctly again */ - wallet_blocks_rollback(topo->wallet, topo->first_blocknum); + wallet_blocks_rollback(topo->wallet, topo->max_blockheight); /* Get up to speed with topology. */ - bitcoind_getblockhash(bitcoind, topo->first_blocknum, + bitcoind_getblockhash(bitcoind, topo->max_blockheight, get_init_block, topo); } @@ -749,7 +750,9 @@ void setup_topology(struct chain_topology *topo, memset(&topo->feerate, 0, sizeof(topo->feerate)); topo->timers = timers; - topo->first_blocknum = first_blocknum; + /* FIXME(cdecker) Actually load this from DB */ + topo->min_blockheight = first_blocknum; + topo->max_blockheight = first_blocknum; /* Make sure bitcoind is started, and ready */ wait_for_bitcoind(topo->bitcoind); diff --git a/lightningd/chaintopology.h b/lightningd/chaintopology.h index 26ee80afc67c..ce63bc4ffeb7 100644 --- a/lightningd/chaintopology.h +++ b/lightningd/chaintopology.h @@ -89,8 +89,8 @@ struct chain_topology { /* Where to log things. */ struct log *log; - /* How far back (in blocks) to go. */ - unsigned int first_blocknum; + /* What range of blocks do we have in our database? */ + u32 min_blockheight, max_blockheight; /* How often to poll. */ u32 poll_seconds; From 024dca0fff7f1696ad391f4cab048c8aa5334faf Mon Sep 17 00:00:00 2001 From: Christian Decker Date: Mon, 4 Jun 2018 14:47:32 +0200 Subject: [PATCH 31/46] wallet: Return both min and max block heights Signed-off-by: Christian Decker --- lightningd/lightningd.c | 4 ++-- lightningd/test/run-find_my_path.c | 6 +++--- wallet/wallet.c | 16 ++++++++-------- wallet/wallet.h | 8 +++++--- 4 files changed, 18 insertions(+), 16 deletions(-) diff --git a/lightningd/lightningd.c b/lightningd/lightningd.c index 1c9af655641a..bdffd4e25ab8 100644 --- a/lightningd/lightningd.c +++ b/lightningd/lightningd.c @@ -306,7 +306,7 @@ void notify_new_block(struct lightningd *ld, int main(int argc, char *argv[]) { struct lightningd *ld; - u32 blockheight; + u32 blockheight, first_block; setup_locale(); daemon_setup(argv[0], log_backtrace_print, log_backtrace_exit); @@ -388,7 +388,7 @@ int main(int argc, char *argv[]) /* Get the blockheight we are currently at, UINT32_MAX is used to signal * an unitialized wallet and that we should start off of bitcoind's * current height */ - blockheight = wallet_blocks_height(ld->wallet, UINT32_MAX); + wallet_blocks_heights(ld->wallet, UINT32_MAX, &first_block, &blockheight); /* If we were asked to rescan from an absolute height (--rescan < 0) * then just go there. Otherwise take compute the diff to our current diff --git a/lightningd/test/run-find_my_path.c b/lightningd/test/run-find_my_path.c index dd123f81b2c6..38c6303712a1 100644 --- a/lightningd/test/run-find_my_path.c +++ b/lightningd/test/run-find_my_path.c @@ -121,9 +121,9 @@ struct txfilter *txfilter_new(const tal_t *ctx UNNEEDED) /* Generated stub for version */ const char *version(void) { fprintf(stderr, "version called!\n"); abort(); } -/* Generated stub for wallet_blocks_height */ -u32 wallet_blocks_height(struct wallet *w UNNEEDED, u32 def UNNEEDED) -{ fprintf(stderr, "wallet_blocks_height called!\n"); abort(); } +/* Generated stub for wallet_blocks_heights */ +void wallet_blocks_heights(struct wallet *w UNNEEDED, u32 def UNNEEDED, u32 *min UNNEEDED, u32 *max UNNEEDED) +{ fprintf(stderr, "wallet_blocks_heights called!\n"); abort(); } /* Generated stub for wallet_channels_load_active */ bool wallet_channels_load_active(const tal_t *ctx UNNEEDED, struct wallet *w UNNEEDED) { fprintf(stderr, "wallet_channels_load_active called!\n"); abort(); } diff --git a/wallet/wallet.c b/wallet/wallet.c index cb32eb250629..300a84b99df4 100644 --- a/wallet/wallet.c +++ b/wallet/wallet.c @@ -769,20 +769,20 @@ void wallet_channel_stats_load(struct wallet *w, db_stmt_done(stmt); } -u32 wallet_blocks_height(struct wallet *w, u32 def) +void wallet_blocks_heights(struct wallet *w, u32 def, u32 *min, u32 *max) { - u32 blockheight; - sqlite3_stmt *stmt = db_prepare(w->db, "SELECT MAX(height) FROM blocks;"); + assert(min != NULL && max != NULL); + sqlite3_stmt *stmt = db_prepare(w->db, "SELECT MIN(height), MAX(height) FROM blocks;"); /* If we ever processed a block we'll get the latest block in the chain */ if (sqlite3_step(stmt) == SQLITE_ROW && sqlite3_column_type(stmt, 0) != SQLITE_NULL) { - blockheight = sqlite3_column_int(stmt, 0); - db_stmt_done(stmt); - return blockheight; + *min = sqlite3_column_int(stmt, 0); + *max = sqlite3_column_int(stmt, 1); } else { - db_stmt_done(stmt); - return def; + *min = def; + *max = def; } + db_stmt_done(stmt); } static void wallet_channel_config_insert(struct wallet *w, diff --git a/wallet/wallet.h b/wallet/wallet.h index 164f6eb824c3..aa359be7cccf 100644 --- a/wallet/wallet.h +++ b/wallet/wallet.h @@ -316,13 +316,15 @@ void wallet_channel_stats_load(struct wallet *w, u64 cdbid, struct channel_stats /** * Retrieve the blockheight of the last block processed by lightningd. * - * Will return either the maximal blockheight or the default value if the wallet - * was never used before. + * Will set min/max either the minimal/maximal blockheight or the default value + * if the wallet was never used before. * * @w: wallet to load from. * @def: the default value to return if we've never used the wallet before + * @min(out): height of the first block we track + * @max(out): height of the last block we added */ -u32 wallet_blocks_height(struct wallet *w, u32 def); +void wallet_blocks_heights(struct wallet *w, u32 def, u32 *min, u32 *max); /** * wallet_extract_owned_outputs - given a tx, extract all of our outputs From 2415f4872377d5cabcd6c63827bf71d3cb3ed9f8 Mon Sep 17 00:00:00 2001 From: Christian Decker Date: Mon, 4 Jun 2018 15:00:05 +0200 Subject: [PATCH 32/46] topo: Tell chain_topology about the min and max block height Signed-off-by: Christian Decker --- lightningd/chaintopology.c | 6 +++--- lightningd/chaintopology.h | 5 ++--- lightningd/lightningd.c | 22 ++++++++++------------ lightningd/test/run-find_my_path.c | 5 ++--- 4 files changed, 17 insertions(+), 21 deletions(-) diff --git a/lightningd/chaintopology.c b/lightningd/chaintopology.c index 767de73abb5c..212f67f0bb9c 100644 --- a/lightningd/chaintopology.c +++ b/lightningd/chaintopology.c @@ -745,14 +745,14 @@ struct chain_topology *new_topology(struct lightningd *ld, struct log *log) void setup_topology(struct chain_topology *topo, struct timers *timers, - u32 first_blocknum) + u32 min_blockheight, u32 max_blockheight) { memset(&topo->feerate, 0, sizeof(topo->feerate)); topo->timers = timers; /* FIXME(cdecker) Actually load this from DB */ - topo->min_blockheight = first_blocknum; - topo->max_blockheight = first_blocknum; + topo->min_blockheight = min_blockheight; + topo->max_blockheight = max_blockheight; /* Make sure bitcoind is started, and ready */ wait_for_bitcoind(topo->bitcoind); diff --git a/lightningd/chaintopology.h b/lightningd/chaintopology.h index ce63bc4ffeb7..b6021984aed4 100644 --- a/lightningd/chaintopology.h +++ b/lightningd/chaintopology.h @@ -150,9 +150,8 @@ void broadcast_tx(struct chain_topology *topo, const char *err)); struct chain_topology *new_topology(struct lightningd *ld, struct log *log); -void setup_topology(struct chain_topology *topology, - struct timers *timers, - u32 first_channel_block); +void setup_topology(struct chain_topology *topology, struct timers *timers, + u32 min_blockheight, u32 max_blockheight); void begin_topology(struct chain_topology *topo); diff --git a/lightningd/lightningd.c b/lightningd/lightningd.c index bdffd4e25ab8..d520d0f1ce36 100644 --- a/lightningd/lightningd.c +++ b/lightningd/lightningd.c @@ -306,7 +306,7 @@ void notify_new_block(struct lightningd *ld, int main(int argc, char *argv[]) { struct lightningd *ld; - u32 blockheight, first_block; + u32 min_blockheight, max_blockheight; setup_locale(); daemon_setup(argv[0], log_backtrace_print, log_backtrace_exit); @@ -388,24 +388,22 @@ int main(int argc, char *argv[]) /* Get the blockheight we are currently at, UINT32_MAX is used to signal * an unitialized wallet and that we should start off of bitcoind's * current height */ - wallet_blocks_heights(ld->wallet, UINT32_MAX, &first_block, &blockheight); + wallet_blocks_heights(ld->wallet, UINT32_MAX, &min_blockheight, &max_blockheight); /* If we were asked to rescan from an absolute height (--rescan < 0) - * then just go there. Otherwise take compute the diff to our current - * height, lowerbounded by 0. */ + * then just go there. Otherwise compute the diff to our current height, + * lowerbounded by 0. */ if (ld->config.rescan < 0) - blockheight = -ld->config.rescan; - else if (blockheight < (u32)ld->config.rescan) - blockheight = 0; - else if (blockheight != UINT32_MAX) - blockheight -= ld->config.rescan; + max_blockheight = -ld->config.rescan; + else if (max_blockheight < (u32)ld->config.rescan) + max_blockheight = 0; + else if (max_blockheight != UINT32_MAX) + max_blockheight -= ld->config.rescan; db_commit_transaction(ld->wallet->db); /* Initialize block topology (does its own transaction) */ - setup_topology(ld->topology, - &ld->timers, - blockheight); + setup_topology(ld->topology, &ld->timers, min_blockheight, max_blockheight); /* Create RPC socket (if any) */ setup_jsonrpc(ld, ld->rpc_filename); diff --git a/lightningd/test/run-find_my_path.c b/lightningd/test/run-find_my_path.c index 38c6303712a1..06208b423618 100644 --- a/lightningd/test/run-find_my_path.c +++ b/lightningd/test/run-find_my_path.c @@ -101,9 +101,8 @@ void setup_color_and_alias(struct lightningd *ld UNNEEDED) void setup_jsonrpc(struct lightningd *ld UNNEEDED, const char *rpc_filename UNNEEDED) { fprintf(stderr, "setup_jsonrpc called!\n"); abort(); } /* Generated stub for setup_topology */ -void setup_topology(struct chain_topology *topology UNNEEDED, - struct timers *timers UNNEEDED, - u32 first_channel_block UNNEEDED) +void setup_topology(struct chain_topology *topology UNNEEDED, struct timers *timers UNNEEDED, + u32 min_blockheight UNNEEDED, u32 max_blockheight UNNEEDED) { fprintf(stderr, "setup_topology called!\n"); abort(); } /* Generated stub for subd_shutdown */ void subd_shutdown(struct subd *subd UNNEEDED, unsigned int seconds UNNEEDED) From 6298ce3b03ac6f4272fff0087245db214390458a Mon Sep 17 00:00:00 2001 From: Christian Decker Date: Mon, 4 Jun 2018 15:27:57 +0200 Subject: [PATCH 33/46] gossip: Don't ask bitcoind for outpoints we should know Compares the `blocknum` in the `short_channel_id` with the range of blocks we store in the database and abort if we should have known about it. Avoids bombarding `bitcoind` with requests for channels that have already been spent or were invalid in the first place. Signed-off-by: Christian Decker --- lightningd/gossip_control.c | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/lightningd/gossip_control.c b/lightningd/gossip_control.c index 0e20dac1bbb3..cd8800096d3a 100644 --- a/lightningd/gossip_control.c +++ b/lightningd/gossip_control.c @@ -89,12 +89,15 @@ static void get_txout(struct subd *gossip, const u8 *msg) { struct short_channel_id *scid = tal(gossip, struct short_channel_id); struct outpoint *op; + u32 blockheight; + struct chain_topology *topo = gossip->ld->topology; if (!fromwire_gossip_get_txout(msg, scid)) fatal("Gossip gave bad GOSSIP_GET_TXOUT message %s", tal_hex(msg, msg)); /* FIXME: Block less than 6 deep? */ + blockheight = short_channel_id_blocknum(scid); op = wallet_outpoint_for_scid(gossip->ld->wallet, scid, scid); @@ -103,8 +106,17 @@ static void get_txout(struct subd *gossip, const u8 *msg) towire_gossip_get_txout_reply( scid, scid, op->satoshis, op->scriptpubkey)); tal_free(scid); + } else if (blockheight >= topo->min_blockheight && + blockheight <= topo->max_blockheight) { + /* We should have known about this outpoint since it is included + * in the range in the DB. The fact that we don't means that + * this is either a spent outpoint or an invalid one. Return a + * failure. */ + subd_send_msg(gossip, take(towire_gossip_get_txout_reply( + NULL, scid, 0, NULL))); + tal_free(scid); } else { - bitcoind_getoutput(gossip->ld->topology->bitcoind, + bitcoind_getoutput(topo->bitcoind, short_channel_id_blocknum(scid), short_channel_id_txnum(scid), short_channel_id_outnum(scid), From d43403257d1f06313223d1346ef8ab9b4939d461 Mon Sep 17 00:00:00 2001 From: Christian Decker Date: Mon, 4 Jun 2018 16:19:14 +0200 Subject: [PATCH 34/46] wallet: Cleanup db files if the tests succeed Signed-off-by: Christian Decker --- wallet/test/run-wallet.c | 56 ++++++++++++++++------------------------ 1 file changed, 22 insertions(+), 34 deletions(-) diff --git a/wallet/test/run-wallet.c b/wallet/test/run-wallet.c index 73e5c9929648..9205f4f8e13b 100644 --- a/wallet/test/run-wallet.c +++ b/wallet/test/run-wallet.c @@ -490,9 +490,16 @@ static void mempat(void *dst, size_t len) p[i] = n % 251; /* Prime */ } +/* Destructor for the wallet which unlinks the underlying file */ +static void cleanup_test_wallet(struct wallet *w, char *filename) +{ + unlink(filename); + tal_free(filename); +} + static struct wallet *create_test_wallet(struct lightningd *ld, const tal_t *ctx) { - char filename[] = "/tmp/ldb-XXXXXX"; + char *filename = tal_fmt(ctx, "/tmp/ldb-XXXXXX"); int fd = mkstemp(filename); struct wallet *w = tal(ctx, struct wallet); static unsigned char badseed[BIP32_ENTROPY_LEN_128]; @@ -500,6 +507,7 @@ static struct wallet *create_test_wallet(struct lightningd *ld, const tal_t *ctx close(fd); w->db = db_open(w, filename); + tal_add_destructor2(w, cleanup_test_wallet, filename); list_head_init(&w->unstored_payments); w->ld = ld; @@ -518,23 +526,14 @@ static struct wallet *create_test_wallet(struct lightningd *ld, const tal_t *ctx return w; } -static bool test_wallet_outputs(void) +static bool test_wallet_outputs(struct lightningd *ld, const tal_t *ctx) { - char filename[] = "/tmp/ldb-XXXXXX"; + struct wallet *w = create_test_wallet(ld, ctx); struct utxo u; - int fd = mkstemp(filename); - CHECK_MSG(fd != -1, "Unable to generate temp filename"); - close(fd); - - struct wallet *w = tal(NULL, struct wallet); struct pubkey pk; u64 fee_estimate, change_satoshis; const struct utxo **utxos; - - w->db = db_open(w, filename); - CHECK_MSG(w->db, "Failed opening the db"); - db_migrate(w->db, NULL); - CHECK_MSG(!wallet_err, "DB migration failed"); + CHECK(w); memset(&u, 0, sizeof(u)); u.amount = 1; @@ -596,33 +595,20 @@ static bool test_wallet_outputs(void) "could not change output state ignoring oldstate"); db_commit_transaction(w->db); - - tal_free(w); return true; } -static bool test_shachain_crud(void) +static bool test_shachain_crud(struct lightningd *ld, const tal_t *ctx) { struct wallet_shachain a, b; - char filename[] = "/tmp/ldb-XXXXXX"; - int fd = mkstemp(filename); - struct wallet *w = tal(NULL, struct wallet); + struct wallet *w = create_test_wallet(ld, ctx); struct sha256 seed, hash; uint64_t index = UINT64_MAX >> (64 - SHACHAIN_BITS); - w->db = db_open(w, filename); - CHECK_MSG(w->db, "Failed opening the db"); - db_migrate(w->db, NULL); - CHECK_MSG(!wallet_err, "DB migration failed"); - - CHECK_MSG(fd != -1, "Unable to generate temp filename"); - close(fd); memset(&seed, 'A', sizeof(seed)); - memset(&a, 0, sizeof(a)); memset(&b, 0, sizeof(b)); - w->db = db_open(w, filename); db_begin_transaction(w->db); CHECK_MSG(!wallet_err, "db_begin_transaction failed"); wallet_shachain_init(w, &a); @@ -644,7 +630,6 @@ static bool test_shachain_crud(void) db_commit_transaction(w->db); CHECK(!wallet_err); - tal_free(w); return true; } @@ -831,7 +816,6 @@ static bool test_channel_crud(struct lightningd *ld, const tal_t *ctx) db_commit_transaction(w->db); CHECK(!wallet_err); - tal_free(w); /* Normally freed by destroy_channel, but we don't call that */ tal_free(p); return true; @@ -1010,15 +994,19 @@ int main(void) htlc_in_map_init(&ld->htlcs_in); htlc_out_map_init(&ld->htlcs_out); - ok &= test_wallet_outputs(); - ok &= test_shachain_crud(); + ok &= test_wallet_outputs(ld, tmpctx); + ok &= test_shachain_crud(ld, tmpctx); ok &= test_channel_crud(ld, tmpctx); ok &= test_channel_config_crud(ld, tmpctx); ok &= test_htlc_crud(ld, tmpctx); ok &= test_payment_crud(ld, tmpctx); - take_cleanup(); - tal_free(tmpctx); + /* Do not clean up in the case of an error, we might want to debug the + * database. */ + if (ok) { + tal_free(tmpctx); + take_cleanup(); + } wally_cleanup(0); return !ok; } From 38b7a0e2d25da5ffbdf3342822301d8c86401791 Mon Sep 17 00:00:00 2001 From: Christian Decker Date: Tue, 5 Jun 2018 15:12:05 +0200 Subject: [PATCH 35/46] pytest: Remove directories of successful tests Signed-off-by: Christian Decker --- tests/fixtures.py | 3 +++ tests/test_lightningd.py | 2 ++ 2 files changed, 5 insertions(+) diff --git a/tests/fixtures.py b/tests/fixtures.py index 992506f95384..1f6c3a4a90a7 100644 --- a/tests/fixtures.py +++ b/tests/fixtures.py @@ -5,6 +5,7 @@ import os import pytest import re +import shutil import tempfile import utils @@ -99,6 +100,8 @@ def node_factory(directory, test_name, bitcoind, executor): if not ok: raise Exception("At least one lightning exited with unexpected non-zero return code") + shutil.rmtree(nf.directory) + def getValgrindErrors(node): for error_file in os.listdir(node.daemon.lightning_dir): diff --git a/tests/test_lightningd.py b/tests/test_lightningd.py index 7247ec19f5d2..4704b83c6431 100644 --- a/tests/test_lightningd.py +++ b/tests/test_lightningd.py @@ -337,6 +337,8 @@ def tearDown(self): if not ok: raise Exception("At least one lightning exited with unexpected non-zero return code") + shutil.rmtree(self.node_factory.directory) + class LightningDTests(BaseLightningDTests): def connect(self, may_reconnect=False): From de3b359782f9fd5558ed548a691bea809d5f5af1 Mon Sep 17 00:00:00 2001 From: Christian Decker Date: Tue, 5 Jun 2018 15:41:18 +0200 Subject: [PATCH 36/46] pytest: Use pytest fixtures for the test directory and clean it up The modern, pytest based, tests now clean up after themselves by removing directories of successful tests and the base directory if there was no failure. Signed-off-by: Christian Decker --- tests/fixtures.py | 24 ++++++++++++++++++------ tests/test_lightningd.py | 1 - 2 files changed, 18 insertions(+), 7 deletions(-) diff --git a/tests/fixtures.py b/tests/fixtures.py index 1f6c3a4a90a7..49814f8957a2 100644 --- a/tests/fixtures.py +++ b/tests/fixtures.py @@ -10,7 +10,6 @@ import utils -TEST_DIR = tempfile.mkdtemp(prefix='ltests-') VALGRIND = os.getenv("NO_VALGRIND", "0") == "0" DEVELOPER = os.getenv("DEVELOPER", "0") == "1" TEST_DEBUG = os.getenv("TEST_DEBUG", "0") == "1" @@ -21,17 +20,32 @@ __attempts = {} +@pytest.fixture(scope="session") +def test_base_dir(): + directory = tempfile.mkdtemp(prefix='ltests-') + print("Running tests in {}".format(directory)) + + yield directory + + if os.listdir(directory) == []: + shutil.rmtree(directory) + + @pytest.fixture -def directory(test_name): +def directory(test_base_dir, test_name): """Return a per-test specific directory. This makes a unique test-directory even if a test is rerun multiple times. """ - global TEST_DIR, __attempts + global __attempts # Auto set value if it isn't in the dict yet __attempts[test_name] = __attempts.get(test_name, 0) + 1 - yield os.path.join(TEST_DIR, "{}_{}".format(test_name, __attempts[test_name])) + directory = os.path.join(test_base_dir, "{}_{}".format(test_name, __attempts[test_name])) + + yield directory + + shutil.rmtree(directory) @pytest.fixture @@ -100,8 +114,6 @@ def node_factory(directory, test_name, bitcoind, executor): if not ok: raise Exception("At least one lightning exited with unexpected non-zero return code") - shutil.rmtree(nf.directory) - def getValgrindErrors(node): for error_file in os.listdir(node.daemon.lightning_dir): diff --git a/tests/test_lightningd.py b/tests/test_lightningd.py index 4704b83c6431..ee02932888cd 100644 --- a/tests/test_lightningd.py +++ b/tests/test_lightningd.py @@ -32,7 +32,6 @@ DEVELOPER = os.getenv("DEVELOPER", "0") == "1" TEST_DEBUG = os.getenv("TEST_DEBUG", "0") == "1" -print("Testing results are in {}".format(TEST_DIR)) if TEST_DEBUG: logging.basicConfig(level=logging.DEBUG, stream=sys.stdout) From 294c76e749996bd5d035d0bcc72dad12d63113dd Mon Sep 17 00:00:00 2001 From: Christian Decker Date: Tue, 5 Jun 2018 22:53:56 +0200 Subject: [PATCH 37/46] gitignore: Ignore .pytest_cache Signed-off-by: Christian Decker --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index ae7cc6e97414..1a127129550d 100644 --- a/.gitignore +++ b/.gitignore @@ -46,3 +46,4 @@ test/test_protocol test/test_sphinx tests/.pytest.restart gossip_store +.pytest_cache From 437fc5feb23add5e8a3437d70f68755b7d04a1e4 Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Mon, 4 Jun 2018 14:22:57 +0930 Subject: [PATCH 38/46] ccan: update to have new configurator. Signed-off-by: Rusty Russell --- Makefile | 2 +- ccan/README | 2 +- ccan/tools/configurator/configurator.1 | 216 ++++++++++ ccan/tools/configurator/configurator.c | 520 +++++++++++++++++++------ 4 files changed, 623 insertions(+), 117 deletions(-) create mode 100644 ccan/tools/configurator/configurator.1 diff --git a/Makefile b/Makefile index 436e1b78fb6c..9a8853efd0c3 100644 --- a/Makefile +++ b/Makefile @@ -336,7 +336,7 @@ update-ccan: mv ccan ccan.old DIR=$$(pwd)/ccan; cd ../ccan && ./tools/create-ccan-tree -a $$DIR `cd $$DIR.old/ccan && find * -name _info | sed s,/_info,, | sort` $(CCAN_NEW) mkdir -p ccan/tools/configurator - cp ../ccan/tools/configurator/configurator.c ccan/tools/configurator/ + cp ../ccan/tools/configurator/configurator.c ../ccan/doc/configurator.1 ccan/tools/configurator/ $(MAKE) ccan/config.h grep -v '^CCAN version:' ccan.old/README > ccan/README echo CCAN version: `git -C ../ccan describe` >> ccan/README diff --git a/ccan/README b/ccan/README index 69db7ae6452f..effdcf745209 100644 --- a/ccan/README +++ b/ccan/README @@ -1,3 +1,3 @@ CCAN imported from http://ccodearchive.net. -CCAN version: init-2423-g696c9b68 +CCAN version: init-2432-gd830ca0e diff --git a/ccan/tools/configurator/configurator.1 b/ccan/tools/configurator/configurator.1 new file mode 100644 index 000000000000..aaa92c527f9e --- /dev/null +++ b/ccan/tools/configurator/configurator.1 @@ -0,0 +1,216 @@ +'\" t +.\" Title: configurator +.\" Author: [see the "AUTHOR" section] +.\" Generator: DocBook XSL Stylesheets v1.79.1 +.\" Date: 03/01/2018 +.\" Manual: \ \& +.\" Source: \ \& +.\" Language: English +.\" +.TH "CONFIGURATOR" "1" "03/01/2018" "\ \&" "\ \&" +.\" ----------------------------------------------------------------- +.\" * Define some portability stuff +.\" ----------------------------------------------------------------- +.\" ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +.\" http://bugs.debian.org/507673 +.\" http://lists.gnu.org/archive/html/groff/2009-02/msg00013.html +.\" ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +.ie \n(.g .ds Aq \(aq +.el .ds Aq ' +.\" ----------------------------------------------------------------- +.\" * set default formatting +.\" ----------------------------------------------------------------- +.\" disable hyphenation +.nh +.\" disable justification (adjust text to left margin only) +.ad l +.\" ----------------------------------------------------------------- +.\" * MAIN CONTENT STARTS HERE * +.\" ----------------------------------------------------------------- +.SH "NAME" +configurator \- Generate a simple config\&.h or variable file +.SH "SYNOPSIS" +.sp +\fBconfigurator\fR [\fIOPTIONS\fR] [\fICC\fR] [\fICFLAGS\fR\&...] +.SH "DESCRIPTION" +.sp +\fBconfigurator\fR is a standalone C program which evaluates the C environment using code snippets\&. +.sp +The C compiler (and flags) can be provided on the command\-line, otherwise built\-in defaults are used\&. +.sp +It has a builtin set of tests, to which more can be added\&. By default it produces a C header file to standard output, but it can also produce a file containing simple "key=value" lines suitable for parsing by \fBsh\fR or \fBmake\fR\&. +.SH "OPTIONS" +.PP +\fB\-v\fR +.RS 4 +Print out every test result; specified twice, print out each test too\&. +.RE +.PP +\fB\-vv\fR +.RS 4 +Shortcut for two +\fB\-v\fR +options\&. +.RE +.PP +\fB\-\-var\-file=\fR +.RS 4 +Output results in format +\fI=\fR +to +\fI\fR, or stdout if +\fI\fR +is +\fI\-\fR\&. Default is not to output this\&. +.RE +.PP +\fB\-\-header\-file=\fR +.RS 4 +Output C\-style header to +\fI\fR +instead out stdout\&. +.RE +.PP +\fB\-\-autotools\-style\fR +.RS 4 +Produce output to stdout like autotools\*(Aq configure script\&. This usually means you want to use +\fB\-\-header\-file\fR +so that doesn\(cqt mix with stdout\&. +.RE +.PP +\fB\-O\fR +.RS 4 +Override option to set compiler output file\&. +.RE +.PP +\fB\-\-configurator\-cc=\fR +.RS 4 +This gives the real compiler command to use for tests, instead of the first commandline argument or the default\&. +.RE +.PP +\fB\-\-extra\-tests\fR +.RS 4 +Read additional tests from stdin, see +\fIEXTRA TESTS\fR +below\&. +.RE +.SH "OUTPUT" +.sp +The header output is \fI#ifndef/#define\fR idempotent\-wrapped using \fICCAN_CONFIG_H\fR, and defines \fI_GNU_SOURCE\fR\&. It also defines \fICCAN_COMPILER\fR, \fICCAN_CFLAGS\fR and \fICCAN_OUTPUT_EXE_CFLAG\fR as either the built\-in definitions or those provided on the command line\&. The remainder is \fI#define\fR of the test names followed by a \fI0\fR or \fI1\fR: note that this means you should use \fI#if\fR not \fI#ifdef\fR to test features in your C programs! +.sp +The var\-file output is simply the test names followed by \fI=1\fR or \fI=0\fR\&. +.SH "EXTRA TESTS" +.sp +Extra tests must be formatted as \fI=\fR pairs, with leading whitespace and \fI#\fR lines ignored\&. +.sp +The first three lines are always the same: +.PP +\fBvar=\fR +.RS 4 +Define the variable set by the test, e\&.g\&. +\fIvar=HAVE_FOO\fR\&. +.RE +.PP +\fBdesc=\fR +.RS 4 +The description printed out with +\fB\-\-autotools\-style\fR, e\&.g\&. +\fIfoo support\fR\&. +.RE +.PP +\fBstyle=