Merge with master

CHANGELOG.md

@@ -9,6 +9,8 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 ### Added
 
 - JSON API: `listpeers` has new field `scratch_txid`: the latest tx in channel.
+- Bitcoind: more parallelism in requests, for very slow nodes.
+- Testing: fixed logging, cleaner interception of bitcoind, minor fixes.
 
 ### Changed
 
@@ -21,6 +23,12 @@ changes.
 
 ### Fixed
 
+- Startup: more coherent complaint if daemon already running.
+- JSON RPC: `getinfo` now shows correct Tor port.
+- JSON RPC: `ping` now works even after one peer fails to respond.
+- JSON RPC: `getroute` `fuzzpercent` and `pay` `maxfeepercent` can now be > 100.
+- Protocol: fix occasional deadlock when both peers flood with gossip.
+
 ### Security
 
 ## [0.6.1] - 2018-09-11: "Principled Opposition To Segwit"

channeld/Makefile

@@ -72,6 +72,7 @@ CHANNELD_COMMON_OBJS :=				\
 	common/wire_error.o			\
 	common/wireaddr.o			\
 	gossipd/gen_gossip_wire.o		\
+	hsmd/gen_hsm_wire.o			\
 	lightningd/gossip_msg.o			\
 	wire/fromwire.o				\
 	wire/towire.o

channeld/channeld.c

@@ -45,7 +45,7 @@
 #include <fcntl.h>
 #include <gossipd/gen_gossip_wire.h>
 #include <gossipd/gossip_constants.h>
-#include <hsmd/gen_hsm_client_wire.h>
+#include <hsmd/gen_hsm_wire.h>
 #include <inttypes.h>
 #include <secp256k1.h>
 #include <stdio.h>
@@ -223,14 +223,14 @@ static const u8 *hsm_req(const tal_t *ctx, const u8 *req TAKES)
 	if (!wire_sync_write(HSM_FD, req))
 		status_failed(STATUS_FAIL_HSM_IO,
 			      "Writing %s to HSM: %s",
-			      hsm_client_wire_type_name(type),
+			      hsm_wire_type_name(type),
 			      strerror(errno));
 
 	msg = wire_sync_read(ctx, HSM_FD);
 	if (!msg)
 		status_failed(STATUS_FAIL_HSM_IO,
 			      "Reading resp to %s: %s",
-			      hsm_client_wire_type_name(type),
+			      hsm_wire_type_name(type),
 			      strerror(errno));
 
 	return msg;

closingd/Makefile

@@ -72,6 +72,7 @@ CLOSINGD_COMMON_OBJS :=				\
 	common/wire_error.o			\
 	common/wireaddr.o			\
 	gossipd/gen_gossip_wire.o		\
+	hsmd/gen_hsm_wire.o			\
 	lightningd/gossip_msg.o
 
 closingd/gen_closing_wire.h: $(WIRE_GEN) closingd/closing_wire.csv

closingd/closingd.c

@@ -15,7 +15,7 @@
 #include <common/version.h>
 #include <common/wire_error.h>
 #include <errno.h>
-#include <hsmd/gen_hsm_client_wire.h>
+#include <hsmd/gen_hsm_wire.h>
 #include <inttypes.h>
 #include <stdio.h>
 #include <unistd.h>

common/bolt11.c

@@ -11,7 +11,7 @@
 #include <common/bolt11.h>
 #include <common/utils.h>
 #include <errno.h>
-#include <hsmd/gen_hsm_client_wire.h>
+#include <hsmd/gen_hsm_wire.h>
 #include <inttypes.h>
 #include <lightningd/hsm_control.h>
 #include <lightningd/jsonrpc.h>

common/wireaddr.c

@@ -430,6 +430,16 @@ bool parse_wireaddr_internal(const char *arg, struct wireaddr_internal *addr,
 		return true;
 	}
 
+	/* 'autotor:' is a special prefix meaning talk to Tor to create
+	 * an onion address. */
+	if (strstarts(arg, "autotor:")) {
+		addr->itype = ADDR_INTERNAL_AUTOTOR;
+		return parse_wireaddr(arg + strlen("autotor:"),
+				      &addr->u.torservice, 9051,
+				      dns_ok ? NULL : &needed_dns,
+				      err_msg);
+	}
+
 	splitport = port;
 	if (!separate_address_and_port(tmpctx, arg, &ip, &splitport)) {
 		if (err_msg) {
@@ -446,16 +456,6 @@ bool parse_wireaddr_internal(const char *arg, struct wireaddr_internal *addr,
 		return true;
 	}
 
-	/* 'autotor:' is a special prefix meaning talk to Tor to create
-	 * an onion address. */
-	if (strstarts(arg, "autotor:")) {
-		addr->itype = ADDR_INTERNAL_AUTOTOR;
-		return parse_wireaddr(arg + strlen("autotor:"),
-				      &addr->u.torservice, 9051,
-				      dns_ok ? NULL : &needed_dns,
-				      err_msg);
-	}
-
 	addr->itype = ADDR_INTERNAL_WIREADDR;
 	if (parse_wireaddr(arg, &addr->u.wireaddr, port,
 			   dns_ok ? NULL : &needed_dns, err_msg))

connectd/Makefile

@@ -61,8 +61,7 @@ CONNECTD_COMMON_OBJS :=				\
 	common/wireaddr.o			\
 	common/wire_error.o			\
 	gossipd/gen_gossip_wire.o		\
-	hsmd/client.o				\
-	hsmd/gen_hsm_client_wire.o		\
+	hsmd/gen_hsm_wire.o			\
 	lightningd/gossip_msg.o			\
 	wire/gen_onion_wire.o
 

connectd/connectd.c

@@ -39,8 +39,7 @@
 #include <connectd/tor_autoservice.h>
 #include <errno.h>
 #include <gossipd/gen_gossip_wire.h>
-#include <hsmd/client.h>
-#include <hsmd/gen_hsm_client_wire.h>
+#include <hsmd/gen_hsm_wire.h>
 #include <inttypes.h>
 #include <lightningd/gossip_msg.h>
 #include <netdb.h>
@@ -1286,6 +1285,21 @@ static struct io_plan *recv_req(struct io_conn *conn, struct daemon_conn *master
 		      t, tal_hex(tmpctx, master->msg_in));
 }
 
+/* Helper for handshake.c */
+bool hsm_do_ecdh(struct secret *ss, const struct pubkey *point)
+{
+	u8 *req = towire_hsm_ecdh_req(tmpctx, point), *resp;
+
+	if (!wire_sync_write(HSM_FD, req))
+		return false;
+	resp = wire_sync_read(req, HSM_FD);
+	if (!resp)
+		return false;
+	if (!fromwire_hsm_ecdh_resp(resp, ss))
+		return false;
+	return true;
+}
+
 #ifndef TESTING
 static void master_gone(struct io_conn *unused UNUSED, struct daemon_conn *dc UNUSED)
 {
@@ -1312,7 +1326,6 @@ int main(int argc, char *argv[])
 	daemon_conn_init(daemon, &daemon->master, STDIN_FILENO, recv_req,
 			 master_gone);
 	status_setup_async(&daemon->master);
-	hsm_setup(HSM_FD);
 
 	/* When conn closes, everything is freed. */
 	tal_steal(daemon->master.conn, daemon);

connectd/handshake.c

@@ -13,7 +13,6 @@
 #include <common/wireaddr.h>
 #include <connectd/handshake.h>
 #include <errno.h>
-#include <hsmd/client.h>
 #include <secp256k1.h>
 #include <secp256k1_ecdh.h>
 #include <sodium/crypto_aead_chacha20poly1305.h>

connectd/handshake.h

@@ -51,4 +51,6 @@ struct io_plan *responder_handshake_(struct io_conn *conn,
 							   void *cbarg),
 				     void *cbarg);
 
+/* helper which is defined in connect.c */
+bool hsm_do_ecdh(struct secret *ss, const struct pubkey *point);
 #endif /* LIGHTNING_CONNECTD_HANDSHAKE_H */

doc/lightning-getroute.7

@@ -2,12 +2,12 @@
 .\"     Title: lightning-getroute
 .\"    Author: [see the "AUTHOR" section]
 .\" Generator: DocBook XSL Stylesheets v1.79.1 <http://docbook.sf.net/>
-.\"      Date: 04/26/2018
+.\"      Date: 09/17/2018
 .\"    Manual: \ \&
 .\"    Source: \ \&
 .\"  Language: English
 .\"
-.TH "LIGHTNING\-GETROUTE" "7" "04/26/2018" "\ \&" "\ \&"
+.TH "LIGHTNING\-GETROUTE" "7" "09/17/2018" "\ \&" "\ \&"
 .\" -----------------------------------------------------------------
 .\" * Define some portability stuff
 .\" -----------------------------------------------------------------
@@ -42,7 +42,7 @@ For example, if you thought there was a 1% chance that a node would fail, and it
 .sp
 If you didn\(cqt care about risk, \fIriskfactor\fR would be zero\&.
 .sp
-The \fIfuzzpercent\fR is a floating\-point number, between 0\&.0 to 100\&.0, unit of percent\&. The \fIfuzzpercent\fR is used to distort computed fees along each channel, to provide some randomization to the route generated\&. 0\&.0 means the exact fee of that channel is used, while 100\&.0 means the fee used might be from 0 to twice the actual fee\&. The default is 5\&.0, or up to 5% fee distortion\&.
+The \fIfuzzpercent\fR is a positive floating\-point number, representing a percentage of the actual fee\&. The \fIfuzzpercent\fR is used to distort computed fees along each channel, to provide some randomization to the route generated\&. 0\&.0 means the exact fee of that channel is used, while 100\&.0 means the fee used might be from 0 to twice the actual fee\&. The default is 5\&.0, or up to 5% fee distortion\&.
 .sp
 The \fIseed\fR is a string whose bytes are used to seed the RNG for the route randomization\&. If not specified, a random string is used\&.
 .SH "RISKFACTOR EFFECT ON ROUTING"

doc/lightning-getroute.7.txt

@@ -29,8 +29,7 @@ fail, and it would cost you 20% per annum if that happened,
 
 If you didn't care about risk, 'riskfactor' would be zero.
 
-The 'fuzzpercent' is a floating-point number, between 0.0 to 100.0,
-unit of percent.
+The 'fuzzpercent' is a positive floating-point number, representing a percentage of the actual fee.
 The 'fuzzpercent' is used to distort computed fees along each channel,
 to provide some randomization to the route generated.
 0.0 means the exact fee of that channel is used,

doc/lightning-listfunds.7

@@ -2,12 +2,12 @@
 .\"     Title: lightning-listfunds
 .\"    Author: [see the "AUTHOR" section]
 .\" Generator: DocBook XSL Stylesheets v1.79.1 <http://docbook.sf.net/>
-.\"      Date: 09/07/2018
+.\"      Date: 09/18/2018
 .\"    Manual: \ \&
 .\"    Source: \ \&
 .\"  Language: English
 .\"
-.TH "LIGHTNING\-LISTFUNDS" "7" "09/07/2018" "\ \&" "\ \&"
+.TH "LIGHTNING\-LISTFUNDS" "7" "09/18/2018" "\ \&" "\ \&"
 .\" -----------------------------------------------------------------
 .\" * Define some portability stuff
 .\" -----------------------------------------------------------------

gossipd/Makefile

@@ -61,8 +61,7 @@ GOSSIPD_COMMON_OBJS :=				\
 	common/wireaddr.o			\
 	common/wire_error.o			\
 	connectd/gen_connect_gossip_wire.o	\
-	hsmd/client.o				\
-	hsmd/gen_hsm_client_wire.o		\
+	hsmd/gen_hsm_wire.o			\
 	lightningd/gossip_msg.o			\
 	wire/gen_onion_wire.o
 

gossipd/gossipd.c

@@ -36,8 +36,7 @@
 #include <gossipd/broadcast.h>
 #include <gossipd/gen_gossip_wire.h>
 #include <gossipd/routing.h>
-#include <hsmd/client.h>
-#include <hsmd/gen_hsm_client_wire.h>
+#include <hsmd/gen_hsm_wire.h>
 #include <inttypes.h>
 #include <lightningd/gossip_msg.h>
 #include <netdb.h>
@@ -2258,7 +2257,6 @@ int main(int argc, char *argv[])
 	daemon_conn_init(daemon, &daemon->master, STDIN_FILENO, recv_req,
 			 master_gone);
 	status_setup_async(&daemon->master);
-	hsm_setup(HSM_FD);
 	daemon_conn_init(daemon, &daemon->connectd, CONNECTD_FD, connectd_req,
 			 NULL);
 

hsmd/Makefile

@@ -6,16 +6,9 @@ hsmd-wrongdir:
 
 default: hsmd-all
 
-# Clients use this:
-LIGHTNINGD_HSM_CLIENT_HEADERS := hsmd/client.h
-LIGHTNINGD_HSM_CLIENT_SRC := hsmd/client.c hsmd/gen_hsm_client_wire.c
-LIGHTNINGD_HSM_CLIENT_OBJS := $(LIGHTNINGD_HSM_CLIENT_SRC:.c=.o)
-
-# lightningd/hsm needs these:
-LIGHTNINGD_HSM_HEADERS := hsmd/gen_hsm_client_wire.h
-
 LIGHTNINGD_HSM_SRC := hsmd/hsmd.c	\
-	$(LIGHTNINGD_HSM_HEADERS:.h=.c)
+	hsmd/gen_hsm_wire.c
+LIGHTNINGD_HSM_HEADERS := hsmd/gen_hsm_wire.h
 LIGHTNINGD_HSM_OBJS := $(LIGHTNINGD_HSM_SRC:.c=.o)
 
 # Common source we use.
@@ -40,37 +33,28 @@ HSMD_COMMON_OBJS :=				\
 	common/withdraw_tx.o
 
 # For checking
-LIGHTNINGD_HSM_ALLSRC_NOGEN := $(filter-out hsmd/gen_%, $(LIGHTNINGD_HSM_CLIENT_SRC) $(LIGHTNINGD_HSM_SRC))
-LIGHTNINGD_HSM_ALLHEADERS_NOGEN := $(filter-out hsmd/gen_%, $(LIGHTNINGD_HSM_CLIENT_HEADERS) $(LIGHTNINGD_HSM_HEADERS))
-
-# Add to headers which any object might need.
-LIGHTNINGD_HEADERS_GEN += $(LIGHTNINGD_HSM_HEADERS) $(LIGHTNINGD_HSM_CLIENT_HEADERS)
+LIGHTNINGD_HSM_ALLSRC_NOGEN := $(filter-out hsmd/gen_%, $(LIGHTNINGD_HSM_SRC) $(LIGHTNINGD_HSM_SRC))
+LIGHTNINGD_HSM_ALLHEADERS_NOGEN := $(filter-out hsmd/gen_%, $(LIGHTNINGD_HSM_HEADERS))
 
-$(LIGHTNINGD_HSM_OBJS) $(LIGHTNINGD_HSM_CLIENT_OBJS): $(LIGHTNINGD_HEADERS)
+$(LIGHTNINGD_HSM_OBJS): $(LIGHTNINGD_HEADERS)
 
 # Make sure these depend on everything.
-ALL_OBJS += $(LIGHTNINGD_HSM_OBJS) $(LIGHTNINGD_HSM_CLIENT_OBJS)
+ALL_OBJS += $(LIGHTNINGD_HSM_OBJS)
 ALL_PROGRAMS += lightningd/lightning_hsmd
-ALL_GEN_HEADERS += $(LIGHTNINGD_HSM_HEADERS)
+ALL_GEN_HEADERS += hsmd/gen_hsm_wire.h
 
-hsmd-all: lightningd/lightning_hsmd $(LIGHTNINGD_HSM_CLIENT_OBJS)
+hsmd-all: lightningd/lightning_hsmd
 
 lightningd/lightning_hsmd: $(LIGHTNINGD_HSM_OBJS) $(LIGHTNINGD_LIB_OBJS) $(HSMD_COMMON_OBJS) $(BITCOIN_OBJS) $(WIRE_OBJS)
 
-hsmd/gen_hsm_client_wire.h: $(WIRE_GEN) hsmd/hsm_client_wire_csv
-	$(WIRE_GEN) --header $@ hsm_client_wire_type < hsmd/hsm_client_wire_csv > $@
-
-hsmd/gen_hsm_client_wire.c: $(WIRE_GEN) hsmd/hsm_client_wire_csv
-	$(WIRE_GEN) ${@:.c=.h} hsm_client_wire_type< hsmd/hsm_client_wire_csv > $@
-
 hsmd/gen_hsm_wire.h: $(WIRE_GEN) hsmd/hsm_wire.csv
 	$(WIRE_GEN) --header $@ hsm_wire_type < hsmd/hsm_wire.csv > $@
 
 hsmd/gen_hsm_wire.c: $(WIRE_GEN) hsmd/hsm_wire.csv
 	$(WIRE_GEN) ${@:.c=.h} hsm_wire_type < hsmd/hsm_wire.csv > $@
 
 check-source: $(LIGHTNINGD_HSM_ALLSRC_NOGEN:%=check-src-include-order/%) $(LIGHTNINGD_HSM_ALLHEADERS_NOGEN:%=check-hdr-include-order/%)
-check-source-bolt: $(LIGHTNINGD_HSM_SRC:%=bolt-check/%) $(LIGHTNINGD_HSM_HEADERS:%=bolt-check/%)
+check-source-bolt: $(LIGHTNINGD_HSM_SRC:%=bolt-check/%)
 
 check-whitespace: $(LIGHTNINGD_HSM_ALLSRC_NOGEN:%=check-whitespace/%) $(LIGHTNINGD_HSM_ALLHEADERS_NOGEN:%=check-whitespace/%)
 

hsmd/hsm_client_wire_csv → hsmd/hsm_wire.csv

@@ -1,6 +1,7 @@
 # Clients should not give a bad request but not the HSM's decision to crash.
 hsmstatus_client_bad_request,1000
 hsmstatus_client_bad_request,,id,struct pubkey
+hsmstatus_client_bad_request,,description,wirestring
 hsmstatus_client_bad_request,,len,u16
 hsmstatus_client_bad_request,,msg,len*u8