From baa37320dc98d2272b1c804775e1008de3f576bf Mon Sep 17 00:00:00 2001
From: Lagrang3 <eduardo.quintana@pm.me>
Date: Fri, 14 Apr 2023 14:02:15 +0100
Subject: [PATCH 01/64] add a simple MCF to get started

---
 plugins/renepay/mcf.c | 499 ++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 499 insertions(+)
 create mode 100644 plugins/renepay/mcf.c

diff --git a/plugins/renepay/mcf.c b/plugins/renepay/mcf.c
new file mode 100644
index 000000000000..68947e4b4c91
--- /dev/null
+++ b/plugins/renepay/mcf.c
@@ -0,0 +1,499 @@
+#include "config.h"
+#include <plugins/renepay/mcf.h>
+
+static const size_t PARTS_BITS = 2;
+static const size_t CHANNEL_PARTS = 1 << PARTS_BITS;
+
+// how many bits for linearization parts plus 1 bit for the direction of the
+// channel plus 1 bit for the dual representation.
+static const size_t ARC_ADDITIONAL_BITS = PARTS_BITS + 2;
+
+static const s64 INFINITE = 9e18;
+static const u32 INVALID_INDEX=0xffffffff;
+			
+/* Let's try this encoding of arcs:
+ * Each channel `c` has two possible directions identified by a bit 
+ * `half` or `!half`, and each one of them has to be
+ * decomposed into 4 liquidity parts in order to
+ * linearize the cost function, but also to solve MCF
+ * problem we need to keep track of flows in the
+ * residual network hence we need for each directed arc
+ * in the network there must be another arc in the
+ * opposite direction refered to as it's dual. In total
+ * 1+2+1 additional bits of information:
+ * 
+ * 	(chan_idx)(half)(part)(dual)
+ * 
+ * That means, for each channel we need to store the
+ * information of 16 arcs. If we implement a convex-cost
+ * solver then we can reduce that number to size(half)size(dual)=4.
+ * 
+ * In the adjacency of a `node` we are going to store
+ * the outgoing arcs. If we ever need to loop over the
+ * incoming arcs then we will define a reverse adjacency
+ * API. 
+ * Then for each outgoing channel `(c,!half)` there will
+ * be 4 parts for the actual residual capacity, hence
+ * with the dual bit set to 0:
+ * 
+ * 	(c,!half,0,0)
+ * 	(c,!half,1,0)
+ * 	(c,!half,2,0)
+ * 	(c,!half,3,0)
+ * 	
+ * and also we need to consider the dual arcs
+ * corresponding to the channel direction `(c,half)`
+ * (the dual has reverse direction):
+ * 
+ * 	(c,half,0,1)
+ * 	(c,half,1,1)
+ * 	(c,half,2,1)
+ * 	(c,half,3,1)
+ *  
+ * These are the 8 outgoing arcs relative to `node` and
+ * associated with channel `c`. The incoming arcs will
+ * be:
+ * 
+ * 	(c,half,0,0)
+ * 	(c,half,1,0)
+ * 	(c,half,2,0)
+ * 	(c,half,3,0)
+ * 
+ * 	(c,!half,0,1)
+ * 	(c,!half,1,1)
+ * 	(c,!half,2,1)
+ * 	(c,!half,3,1)
+ * 	
+ * but they will be stored as outgoing arcs on the peer
+ * node `next`.
+ * 
+ * I hope this will clarify my future self when I forget.
+ * 
+ * */
+
+#define MAX(x, y) (((x) > (y)) ? (x) : (y))
+#define MIN(x, y) (((x) < (y)) ? (x) : (y))
+
+struct pay_parameters {
+	/* The gossmap we are using */
+	struct gossmap *gossmap;
+	struct gossmap_node *source;
+	struct gossmap_node *target;
+	
+	/* Extra information we intuited about the channels */
+	struct chan_extra_map *chan_extra_map;
+	
+	/* Optional bitarray of disabled chans. */
+	// TODO: did you consider these?, hint: disable channels by ignoring
+	// them when constructing the adjacency list of nodes in
+	// `init_residual_network`.
+	const bitmap *disabled;
+	
+	u32 *arc_tail_node, *arc_head_node;
+	u32 *arc_adjacency_next_arc;
+	
+	u32 *node_adjacency_first_arc;	
+};
+
+struct residual_network {
+	/* residual capacity on arcs */
+	s64 *cap; 
+	s64 *cost;
+};
+				
+
+/* Helper function. 
+ * Given an arc idx, return the dual's idx in the residual network. */
+static inline u32 arc_dual(const struct pay_parameters * params UNUSED,
+                           const u32 arc)
+{
+	return arc ^ 1;
+}
+
+/* Helper function. 
+ * Given an arc idx, return the node from which this arc emanates in the residual network. */
+static inline u32 arc_tail(const struct pay_parameters *params,
+                           const u32 arc)
+{
+	return params->arc_tail_node[arc];
+}
+/* Helper function. 
+ * Given an arc idx, return the node that this arc is pointing to in the residual network. */
+static inline u32 arc_head(const struct pay_parameters *params,
+                           const u32 arc)
+{
+	return params->arc_head_node[arc];
+}
+		
+/* Helper function. 
+ * Given node idx `node`, return the idx of the first arc whose tail is `node`.
+ * */
+static inline u32 node_adjacency_begin(const struct pay_parameters *params,
+                                       const u32 node)
+{
+	return params->node_adjacency_first[node];
+}
+
+/* Helper function. 
+ * Given node idx `node`, return the idx of one past the last arc whose tail is `node`. */
+static inline u32 node_adjacency_end(const struct pay_parameters *params UNUSED,
+                                     const u32 node UNUSED)
+{
+	return INVALID_INDEX;
+}
+
+/* Helper function. 
+ * Given node idx `node` and `arc`, returns the idx of the next arc whose tail is `node`. */
+static inline u32 node_adjacency_next(const struct pay_parameters *params,
+                                     const u32 node UNUSED,
+				     const u32 arc)
+{
+	return params->adjacency_next[arc];
+}
+
+/* Helper function.
+ * Given an arc index, we should be able to deduce the channel id, and part that
+ * corresponds to this arc. */
+static u32 arc_to_channel_idx(const u32 arc,
+                              int *half_ptr,
+			      int *part_ptr,
+			      int *dual_ptr)
+{
+	&half_ptr = (arc >> (1+PARTS_BITS)) & 1;
+	&part_ptr = (arc >> 1) & (CHANNEL_PARTS-1);
+	&dual_ptr = arc & 1;
+	return arc>>ARC_ADDITIONAL_BITS;
+}
+
+/* Helper function.
+ * Given a channel index, we should be able to deduce the arc id. */
+static u32 channel_idx_to_arc(const u32 chan_idx,
+                              int half,
+			      int part,
+			      int dual)
+{
+	half &= 1;
+	dual &= 1;
+	part &= (CHANNEL_PARTS-1);
+	return (chan_idx << ARC_ADDITIONAL_BITS)|(half<<(1+PARTS_BITS))|(part<<1)|(dual);
+}
+
+static void init_residual_network(struct pay_parameters *params,
+                                  struct residual_network *network)
+{
+	const size_t max_num_chans = gossmap_max_chan_idx(params->gossmap);
+	const size_t max_num_arcs = max_num_chans << ARC_ADDITIONAL_BITS;
+	const size_t max_num_nodes = gossmap_max_node_idx(params->gossmap);
+	
+	network->cap = tal_arr(network,s64,max_num_arcs);
+	for(size_t i=0;i<tal_count(network->cap);++i)
+		network->cap[i]=0;
+	
+	network->cost = tal_arr(network,s64,max_num_arcs);
+	for(size_t i=0;i<tal_count(network->cost);++i)
+		network->cost[i]=INFINITE;
+	
+	params->arc_tail_node = tal_arr(params,u32,max_num_arcs);
+	for(size_t i=0;i<tal_count(params->arc_tail_node);++i)
+		params->arc_tail_node[i]=INVALID_INDEX;
+	
+	params->arc_head_node = tal_arr(params,u32,max_num_arcs);
+	for(size_t i=0;i<tal_count(params->arc_head_node);++i)
+		params->arc_head_node[i]=INVALID_INDEX;
+		
+	params->arc_adjacency_next_arc = tal_arr(params,u32,max_num_arcs);
+	for(size_t i=0;i<tal_count(params->arc_adjacency_next_arc);++i)
+		params->arc_adjacency_next_arc[i]=INVALID_INDEX;
+	
+	params->node_adjacency_first_arc = tal_arr(params,u32,max_num_nodes);
+	for(size_t i=0;i<tal_count(params->node_adjacency_first_arc);++i)
+		params->node_adjacency_first_arc[i]=INVALID_INDEX;
+	
+	struct gossmap_node *node = gossmap_first_node(params->gossmap);
+	const size_t num_nodes = gossmap_num_nodes(params->gossmap);
+	for(size_t i=0; i<num_nodes; ++i, node = gossmap_next_node(params->gossmap,node))
+	{
+		const u32 node_id = gossmap_node_idx(params->gossmap,node);
+		
+		u32 prev_arc = INVALID_INDEX;
+		
+		for(size_t j=0;j<node->num_chans;++j)
+		{
+			int half;
+			const struct gossmap_chan *c = gossmap_nth_chan(params->gossmap,
+			                                                node, j, &half);
+			const u32 chan_id = gossmap_chan_idx(params->gossmap, c);
+			
+			// TODO: question here, is the pair `(c,!half)` identified
+			// with the outgoing channel from `node` or the incoming
+			// channel?
+			
+			const struct gossmap_node *next = gossmap_nth_node(params->gossmap,
+									   c,!half);
+			const u32 next_id = gossmap_node_idx(params->gossmap,next);
+			
+			ASSERT(node_id!=next_id);
+			
+			// let's subscribe the 4 parts of the channel direction
+			// (c,!half), the dual of these guys will be subscribed
+			// when the `i` hits the `next` node.
+			for(size_t k=0;k<CHANNEL_PARTS;++k)
+			{
+				u32 arc = channel_idx_to_arc(chan_id,!half,k,0);
+				arc_tail_node[arc] = node_id;
+				arc_head_node[arc] = next_id;
+				
+				// Is this is the first arc?
+				if(prev_arc==INVALID_INDEX)
+				// yes, then set it at the head of the linked list
+				{
+					node_adjacency_first_arc[node_id]=arc;
+				} else
+				// no, then link it to the previous arc
+				{
+					arc_adjacency_next_arc[prev_arc]=arc;
+				}
+				
+				prev_arc = arc;
+				
+				// TODO: set costs and capacity
+				// use (c,!half)
+			}
+			
+			// let's subscribe the 4 parts of the channel direction
+			// (c,half) in dual representation
+			for(size_t k=0;k<CHANNEL_PARTS;++k)
+			{
+				u32 arc = channel_idx_to_arc(chan_id,half,k,1);
+				arc_tail_node[arc] = node_id;
+				arc_head_node[arc] = next_id;
+				
+				// Is this is the first arc?
+				if(prev_arc==INVALID_INDEX)
+				// yes, then set it at the head of the linked list
+				{
+					node_adjacency_first_arc[node_id]=arc;
+				} else
+				// no, then link it to the previous arc
+				{
+					arc_adjacency_next_arc[prev_arc]=arc;
+				}
+				
+				prev_arc = arc;
+				
+				// TODO: set costs and capacity
+				// use (u,half)
+			}
+		}
+	}
+}
+
+/* Finds an admissible path from source to target, traversing arcs in the
+ * residual network with capacity greater than 0. 
+ * The path is encoded into prev, which contains the idx of the arcs that are
+ * traversed. 
+ * Returns RENEPAY_ERR_OK if the path exists. */
+static int find_admissible_path(const struct pay_parameters *params,
+			        struct residual_network *network,
+                                const u32 source,
+				const u32 target,
+				u32 *prev)
+{
+	int status = RENEPAY_ERR_NOFEASIBLEFLOW;
+	
+	// TODO: memset(prev) = INVALID_INDEX
+	
+	// The graph is dense, and the farthest node is just a few hops away,
+	// hence let's BFS search.
+	queue Q; // TODO
+	
+	// TODO: Q.push(source);
+	while // TODO: Q not empty
+	{
+		// TODO: u32 cur = Q.front(); Q.pop();
+		if(cur==target)
+		{
+			status = RENEPAY_ERR_OK;
+			break;
+		}
+		
+		for(u32 arc = node_adjacency_begin(params,cur);
+		        arc!= node_adjacency_end(params,cur);
+			arc = node_adjacency_next(params,cur,arc))
+		{
+			// check if this arc is traversable
+			if(network->cap[arc] <= 0)
+				continue;
+			
+			u32 next = arc_head(params,arc);
+			
+			// if that node has been seen previously
+			if(prev[next]!=INVALID_INDEX)
+				continue;
+			
+			prev[next] = arc;
+			// TODO: Q.push(next);
+		}
+	}
+	
+	return status;
+}
+
+/* Get the max amount of flow one can send from source to target along the path
+ * encoded in `prev`. */
+static s64 get_augmenting_flow(const struct pay_parameters *params,
+			       struct residual_network *network,
+	                       const u32 source,
+			       const u32 target,
+			       const u32 *prev)
+{
+	s64 flow = INFINITE;
+	
+	u32 cur = target;
+	while(cur!=source)
+	{
+		const u32 arc = prev[cur];
+		const u32 dual = arc_dual(params,arc);
+		
+		flow = MIN(flow , network->cap[arc]);
+		
+		// we are traversing in the opposite direction to the flow,
+		// hence the next node is at the head of the `dual` arc.
+		cur = arc_destination(params,dual);
+	}
+	
+	ASSERT(flow<INFINITE && flow>0);
+	return flow;
+}
+
+/* Augment a `flow` amount along the path defined by `prev`.*/
+static void augment_flow(const struct pay_parameters *params,
+			 struct residual_network *network,
+	                 const u32 source,
+			 const u32 target,
+			 const u32 *prev,
+			 s64 flow)
+{
+	u32 cur = target;
+	
+	while(cur!=source)
+	{
+		const u32 arc = prev[cur];
+		const u32 dual = arc_dual(params,arc);
+		
+		network->cap[arc] -= flow;
+		ASSERT(network->cap[arc] >=0 );
+		network->cap[dual] += flow;
+		
+		// we are traversing in the opposite direction to the flow,
+		// hence the next node is at the head of the `dual` arc.
+		cur = arc_destination(params,dual);
+	}
+}
+
+/* Finds any flow that satisfy the capacity and balance constraints of the
+ * uncertainty network. For the balance function condition we have:
+ * 	balance(source) = - balance(target) = amount
+ * 	balance(node) = 0 , for every other node
+ * Returns an error code if no feasible flow is found.
+ * 
+ * 13/04/2023 This implementation uses a simple augmenting path approach.
+ * */
+static int find_feasible_flow(const struct pay_parameters *params,
+			      struct residual_network *network,
+			      const u32 source, 
+			      const u32 target,
+			      s64 amount)
+{
+	int status = RENEPAY_ERR_OK;
+	
+	/* path information 
+	 * prev: is the id of the arc that lead to the node. */
+	// TODO: how do we encode the arc idx?
+	u32 *prev = tal_arr(tmpctx,u32
+		            gossmap_max_node_idx(params->gossmap));
+	
+	while(amount>0)
+	{
+		// find a path from source to target
+		int err = find_admissible_path(params,network,source,target,prev);
+		if(err!=RENEPAY_ERR_OK)
+		{
+			status = RENEPAY_ERR_NOFEASIBLEFLOW;
+			break;
+		}
+		
+		// traverse the path and see how much flow we can send
+		s64 delta = get_augmenting_flow(params,network,source,target,prev);
+		
+		// commit that flow to the path
+		delta = MIN(amount,delta);
+		augment_flow(params,network,source,target,prev,delta);
+		
+		ASSERT(delta>0 && delta<=amount);
+		amount -= delta;
+	}
+	
+	tal_free(prev);	
+	
+	return status;
+}
+
+
+struct flow **optimal_payment_flow(
+                      const tal_t *ctx,
+		      struct gossmap *gossmap,
+		      const struct gossmap_node *source,
+		      const struct gossmap_node *target,
+		      struct chan_extra_map *chan_extra_map,
+		      const bitmap *disabled,
+		      struct amount_msat amount,
+		      double max_fee_ppm,
+		      double delay_feefactor,
+		      int *errorcode)
+{
+	struct pay_parameters *params = tal(tmpctx,struct pay_parameters);	
+	
+	params->gossmap = gossmap;
+	params->source = source;
+	params->target = target;
+	params->chan_extra_map = chan_extra_map;
+	params->disabled = disabled;
+	
+	// TODO: handle max_fee_ppm, delay_feefactor and basefee_penalty
+	// params->max_fee_ppm = max_fee_ppm;
+	// params->delay_feefactor = delay_feefactor;
+	// params->basefee_penalty = 
+	
+	// build the uncertainty network with linearization and residual arcs
+	struct residual_network *network = tal(tmpctx,struct residual_network);
+	init_residual_network(params,network);
+	
+	const u32 target_id = get_node_id(params,target);
+	const u32 source_id = get_node_id(params,source);
+	
+	// TODO
+	// 
+	// -> network_flow = compute any feasible flow
+	// -> if network_flow cannot be computed return an error code: 
+	// 	No feasible flow
+	// 	return NULL
+	// 
+	// -> select a default value for mu
+	// -> loop
+	// 	-> set the cost as a combination of prob. cost and fee cost
+	// 	-> network_flow = refine network_flow to minimize costs
+	// 	-> if fee ratio < max_fee_ppm break, else increase mu
+	// 	-> if mu is beyond the limits then return an error code: 
+	// 		cannot find a cheap route
+	// 		break
+	// 		
+	// -> flow = get flow paths from network_flow
+	
+	tal_free(network);
+	tal_free(params);
+	
+	// TODO
+	// return flow
+}
+

From 26a5d71f5d4f275f8906df0c63ca9c649bb9278e Mon Sep 17 00:00:00 2001
From: Lagrang3 <eduardo.quintana@pm.me>
Date: Mon, 17 Apr 2023 13:39:20 +0100
Subject: [PATCH 02/64] allocate gossmap as children of pay

---
 plugins/renepay/pay.c | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/plugins/renepay/pay.c b/plugins/renepay/pay.c
index 46b2f8788402..9ebd0859f15d 100644
--- a/plugins/renepay/pay.c
+++ b/plugins/renepay/pay.c
@@ -85,7 +85,8 @@ static const char *init(struct plugin *p,
 	list_head_init(&pay_plugin->payments);
 	chan_extra_map_init(&pay_plugin->chan_extra_map);
 
-	pay_plugin->gossmap = gossmap_load(NULL,
+	// TODO(eduardo) is it ok to use NULL or pay_plugin as `ctx`?
+	pay_plugin->gossmap = gossmap_load(pay_plugin,
 					   GOSSIP_STORE_FILENAME,
 					   &num_channel_updates_rejected);
 	if (!pay_plugin->gossmap)

From 2212096d066a1e5986481353afad6c41406f9d19 Mon Sep 17 00:00:00 2001
From: Lagrang3 <eduardo.quintana@pm.me>
Date: Mon, 17 Apr 2023 18:54:03 +0100
Subject: [PATCH 03/64] add comments on the idea to combine costs

---
 plugins/renepay/mcf.c | 181 ++++++++++++++++++++++++++++++++++++++----
 1 file changed, 164 insertions(+), 17 deletions(-)

diff --git a/plugins/renepay/mcf.c b/plugins/renepay/mcf.c
index 68947e4b4c91..0bf6f02a38aa 100644
--- a/plugins/renepay/mcf.c
+++ b/plugins/renepay/mcf.c
@@ -10,6 +10,7 @@ static const size_t ARC_ADDITIONAL_BITS = PARTS_BITS + 2;
 
 static const s64 INFINITE = 9e18;
 static const u32 INVALID_INDEX=0xffffffff;
+static const s64 COST_FACTOR=10000;
 			
 /* Let's try this encoding of arcs:
  * Each channel `c` has two possible directions identified by a bit 
@@ -89,9 +90,15 @@ struct pay_parameters {
 	// `init_residual_network`.
 	const bitmap *disabled;
 	
-	u32 *arc_tail_node, *arc_head_node;
-	u32 *arc_adjacency_next_arc;
 	
+	u32 *arc_head_node; 
+	// notice that a tail node is not needed, 
+	// because the tail of arc is the head of dual(arc)
+	
+	// probability cost associated to an arc
+	s64 *arc_prob_cost;
+	
+	u32 *arc_adjacency_next_arc;
 	u32 *node_adjacency_first_arc;	
 };
 
@@ -115,7 +122,7 @@ static inline u32 arc_dual(const struct pay_parameters * params UNUSED,
 static inline u32 arc_tail(const struct pay_parameters *params,
                            const u32 arc)
 {
-	return params->arc_tail_node[arc];
+	return params->arc_head_node[ arc_dual(params,arc) ];
 }
 /* Helper function. 
  * Given an arc idx, return the node that this arc is pointing to in the residual network. */
@@ -154,7 +161,7 @@ static inline u32 node_adjacency_next(const struct pay_parameters *params,
 /* Helper function.
  * Given an arc index, we should be able to deduce the channel id, and part that
  * corresponds to this arc. */
-static u32 arc_to_channel_idx(const u32 arc,
+static inline u32 arc_to_channel_idx(const u32 arc,
                               int *half_ptr,
 			      int *part_ptr,
 			      int *dual_ptr)
@@ -167,7 +174,7 @@ static u32 arc_to_channel_idx(const u32 arc,
 
 /* Helper function.
  * Given a channel index, we should be able to deduce the arc id. */
-static u32 channel_idx_to_arc(const u32 chan_idx,
+static inline u32 channel_idx_to_arc(const u32 chan_idx,
                               int half,
 			      int part,
 			      int dual)
@@ -178,6 +185,48 @@ static u32 channel_idx_to_arc(const u32 chan_idx,
 	return (chan_idx << ARC_ADDITIONAL_BITS)|(half<<(1+PARTS_BITS))|(part<<1)|(dual);
 }
 
+/* Split a directed channel into parts with linear cost function. */
+static int linearize_channel(
+		const struct pay_parameters *params,
+		const struct gossmap_chan *c,
+		const int dir,
+		s64 *capacity,
+		s64 *cost)
+{
+	// TODO
+}
+
+/* Get the fee cost associated to this directed channel. 
+ * Cost is expressed as PPM of the payment. */
+static s64 compute_fee_cost(
+		const struct pay_parameters *params,
+		const struct gossmap_chan *c,
+		const int dir)
+{
+	s64 pfee = c->half[dir].proportional_fee,
+	    bfee = c->half[dir].base_fee;
+	
+	// Base fee to proportional fee. We want
+	// 	cost_eff(x) >= cost_real(x)
+	// 	x * (p + b*a) >= x * p + b*1000
+	// where the effective proportional cost is `(p+b*a)`, `a` is factor we
+	// need to choose. Then
+	// 	x >= 1000/a
+	// therefore if `a`=1, our effective cost is good only after 1000 sats,
+	// otherwise the cost is underestimated. If we want to make it valid
+	// from 1 sat, then a=1000, but that means that setting a base fee of
+	// 1msat will be consider like having a proportional cost of 1000ppm,
+	// which is a lot. Some middle ground can be obtained with numbers in
+	// between, but in general `a` does not make sense above 1000.
+	
+	// In this case having a base fee of 1 is equivalent of having a
+	// proportional fee of 10 ppm.
+	return pfee + bfee * 10;
+	
+	// TODO(eduardo) How to convert the base fee properly into an effective
+	// proportional fee?
+}
+
 static void init_residual_network(struct pay_parameters *params,
                                   struct residual_network *network)
 {
@@ -193,13 +242,13 @@ static void init_residual_network(struct pay_parameters *params,
 	for(size_t i=0;i<tal_count(network->cost);++i)
 		network->cost[i]=INFINITE;
 	
-	params->arc_tail_node = tal_arr(params,u32,max_num_arcs);
-	for(size_t i=0;i<tal_count(params->arc_tail_node);++i)
-		params->arc_tail_node[i]=INVALID_INDEX;
-	
 	params->arc_head_node = tal_arr(params,u32,max_num_arcs);
 	for(size_t i=0;i<tal_count(params->arc_head_node);++i)
 		params->arc_head_node[i]=INVALID_INDEX;
+	
+	params->arc_prob_cost = tal_arr(params,s64,max_num_arcs);
+	for(size_t i=0;i<tal_count(params->arc_prob_cost);++i)
+		params->arc_prob_cost[i]=INFINITE;
 		
 	params->arc_adjacency_next_arc = tal_arr(params,u32,max_num_arcs);
 	for(size_t i=0;i<tal_count(params->arc_adjacency_next_arc);++i)
@@ -234,13 +283,20 @@ static void init_residual_network(struct pay_parameters *params,
 			
 			ASSERT(node_id!=next_id);
 			
+			// `cost` is the word normally used to denote cost per
+			// unit of flow in the context of MCF.
+			s64 prob_cost[CHANNEL_PARTS], capacity[CHANNEL_PARTS];
+			
+			// split this channel direction to obtain the arcs
+			// that are outgoing to `node`
+			linearize_channel(params,c,!half,capacity,prob_cost);
+			
 			// let's subscribe the 4 parts of the channel direction
 			// (c,!half), the dual of these guys will be subscribed
 			// when the `i` hits the `next` node.
 			for(size_t k=0;k<CHANNEL_PARTS;++k)
 			{
 				u32 arc = channel_idx_to_arc(chan_id,!half,k,0);
-				arc_tail_node[arc] = node_id;
 				arc_head_node[arc] = next_id;
 				
 				// Is this is the first arc?
@@ -256,16 +312,20 @@ static void init_residual_network(struct pay_parameters *params,
 				
 				prev_arc = arc;
 				
-				// TODO: set costs and capacity
-				// use (c,!half)
+				network->cap[arc] = capacity[k];
+				params->arc_prob_cost[arc] = prob_cost[k];
+				params->fee_cost[arc] = compute_fee_cost(params,c,!half);
 			}
 			
+			// split the opposite direction to obtain the dual arcs
+			// that are outgoing to `node`
+			linearize_channel(params,c,half,capacity,prob_cost);
+			
 			// let's subscribe the 4 parts of the channel direction
 			// (c,half) in dual representation
 			for(size_t k=0;k<CHANNEL_PARTS;++k)
 			{
 				u32 arc = channel_idx_to_arc(chan_id,half,k,1);
-				arc_tail_node[arc] = node_id;
 				arc_head_node[arc] = next_id;
 				
 				// Is this is the first arc?
@@ -281,8 +341,9 @@ static void init_residual_network(struct pay_parameters *params,
 				
 				prev_arc = arc;
 				
-				// TODO: set costs and capacity
-				// use (u,half)
+				network->cap[arc] = 0;
+				params->arc_prob_cost[arc] = -prob_cost[k];
+				params->fee_cost[arc] = -compute_fee_cost(params,c,!half);
 			}
 		}
 	}
@@ -359,7 +420,7 @@ static s64 get_augmenting_flow(const struct pay_parameters *params,
 		
 		// we are traversing in the opposite direction to the flow,
 		// hence the next node is at the head of the `dual` arc.
-		cur = arc_destination(params,dual);
+		cur = arc_head(params,dual);
 	}
 	
 	ASSERT(flow<INFINITE && flow>0);
@@ -387,7 +448,7 @@ static void augment_flow(const struct pay_parameters *params,
 		
 		// we are traversing in the opposite direction to the flow,
 		// hence the next node is at the head of the `dual` arc.
-		cur = arc_destination(params,dual);
+		cur = arc_head(params,dual);
 	}
 }
 
@@ -439,6 +500,92 @@ static int find_feasible_flow(const struct pay_parameters *params,
 	return status;
 }
 
+/* TODO(eduardo): How to combine probability cost and fee cost?
+ * This is tricky, because we say that a fee cost is high when the ratio of fee
+ * and actual payment is high, typically 1000ppm or more. 
+ * On the other hand a probability cost is high if the probability of success is
+ * very low; since cost = - log prob, we have
+ * 	prob | cost
+ * 	-----------
+ * 	0.01 | 4.6
+ * 	0.02 | 3.9
+ * 	0.05 | 3.0
+ * 	0.10 | 2.3
+ * 	0.20 | 1.6
+ * 	0.50 | 0.69
+ * 	0.80 | 0.22
+ * 	0.90 | 0.10
+ * 	0.95 | 0.05
+ * 	0.98 | 0.02
+ * 	0.99 | 0.01
+ * 	
+ * $ cost \approx 1-P $ when $ P > 0.9 $.
+ * 
+ * Interesting to see:
+ * How much do we pay for reliability?
+ * Cost_fee(most reliable solution) - Cost_fee(cheapest solution)
+ * 
+ * Interesting to see:
+ * Is the most reliable path much more reliable than the cheapest?
+ * Prob(reliable)/Prob(cheapest) = Exp(Cost_prob(cheapest)-Cost_prob(reliable))
+ * 
+ * When combining probability cost and fee cost, we need to understand that this
+ * is like comparing apples to oranges. 
+ * Probability cost is, for small values, equal to the probability of failure;
+ * and we say this cost is acceptable if c(x) = 0.01 (1% probability of failure).
+ * On the other hand fee cost is the money we pay for payment delivery and our
+ * acceptance criteria depends on how much we want to send; so for instance a
+ * fee cost is acceptable if c(x) = 100 T (we pay 100 ppm of T), 
+ * where T is the total value we want to pay.
+ * These numbers can be tuned; but the lesson is we evaluate probability cost on
+ * its absolute value and fee cost relative to T. 
+ * 
+ * Notice also that the fee cost per unit flow is an integer number,
+ * representing ppm (parts per million). On the other hand the first linear
+ * arc with non zero prob. cost on a directed channel has a cost per unit flow
+ * of around 1.3/(b-a), depending on the linearization partition (I prefer the
+ * first partition to contain 50% of the (b-a) range, i.e. up to the point where
+ * the probability of forwarding the flow reaches 50%), in essence this prob.
+ * cost per unit of flow is a real number and we want to use algorithms (for
+ * example cost scaling) that assume that cost is integer. 
+ * 
+ * How to solve both problems? 
+ * Well, we know that we must combine both prob. cost and fee cost linearly and
+ * we like the fact that fee cost is an integer. We can multiply the prob. cost
+ * times a factor such that the prob. cost becomes becomes equivalent to the fee
+ * cost, and approximate it to the nearest integer number in the process.
+ * For instance, define a new prob. cost $ c_prob(x) = - 10^4 T log(P(x))$,
+ * then when $1-P(x) \approx 1%$, i.e. the probability of failure is around 1%
+ * is perceived as if we pay 100 ppm of the total payment T. 
+ * 
+ * Is this enough to make integer prob. cost per unit flow?
+ * Let's see: 
+ * 	-> consider an arc with $(b-a) >> 10^4 T$, recall for the first
+ * 	partition $ c_prob(x) = x 1.3/(b-a) 10^4 T $, hence this arc has cost
+ * 	per unit flow 0. Which is reasonable because T is very small and can be
+ * 	sent through this arc without any issues.
+ * 	
+ * 	-> if $(b-a) \approx 10^4 T$ we obtain $c_prob(x) = x$ which is
+ * 	equivalent to say we just pay 1 ppm for flows on this arc, which is non
+ * 	zero but very small, and it is reasonable because $(b-a)$ is $10^4$
+ * 	times bigger than T.
+ * 	
+ * 	-> if $(b-a)/2 \approx T$ then we start to enter a danger zone because
+ * 	it is very likely that T cannot go through this arc with capacity $(b-a)/2$,
+ * 	non-surprisingly the monetary cost associated to this case is
+ * 	$c_prob(x) \approx 5000 x $, which is like 5000 ppm.
+ * 	
+ * 	-> for $(b-a)/2 < T$ we are paying an ever increasing price in ppm of T
+ * 	per unit flow sent through this arc. The MCF algorithm will try keep
+ * 	this fake monetary cost as low as possible.
+ * 
+ * Summary: we combine use as prob. cost the function $c_prob(x) = -10^4 T log(P(x))$,
+ * and in this scheme we can approximate to integer numbers the cost per unit
+ * flow, and 1% prob. of failure is equivalent to pay 100 ppm of the total
+ * payment T. Then we can combine the fee cost $c_fee(x)$ and $c_prob(x)$ with
+ * a factor mu ~ [0,1]. Instead of hard-coding the 10^4 factor, we use the
+ * variable COST_FACTOR.
+ * */
 
 struct flow **optimal_payment_flow(
                       const tal_t *ctx,

From e9b8fb0a25396d5dee4e99ccfc04a8b1a35baae3 Mon Sep 17 00:00:00 2001
From: Lagrang3 <eduardo.quintana@pm.me>
Date: Tue, 18 Apr 2023 08:34:55 +0100
Subject: [PATCH 04/64] add channel/arc linearization

---
 plugins/renepay/mcf.c | 71 ++++++++++++++++++++++++++++++++++++-------
 1 file changed, 60 insertions(+), 11 deletions(-)

diff --git a/plugins/renepay/mcf.c b/plugins/renepay/mcf.c
index 0bf6f02a38aa..dff551fd0879 100644
--- a/plugins/renepay/mcf.c
+++ b/plugins/renepay/mcf.c
@@ -1,8 +1,11 @@
 #include "config.h"
 #include <plugins/renepay/mcf.h>
+#include <plugins/renepay/flow.h>
+#include <math.h>
 
-static const size_t PARTS_BITS = 2;
-static const size_t CHANNEL_PARTS = 1 << PARTS_BITS;
+#define PARTS_BITS 2
+#define CHANNEL_PARTS (1 << PARTS_BITS)
+static const double CHANNEL_PIVOTS[]={0,0.5,0.8,0.95};
 
 // how many bits for linearization parts plus 1 bit for the direction of the
 // channel plus 1 bit for the dual representation.
@@ -10,7 +13,8 @@ static const size_t ARC_ADDITIONAL_BITS = PARTS_BITS + 2;
 
 static const s64 INFINITE = 9e18;
 static const u32 INVALID_INDEX=0xffffffff;
-static const s64 COST_FACTOR=10000;
+static const s64 COST_FACTOR=10;
+static const s64 MU_MAX = 128;
 			
 /* Let's try this encoding of arcs:
  * Each channel `c` has two possible directions identified by a bit 
@@ -90,6 +94,7 @@ struct pay_parameters {
 	// `init_residual_network`.
 	const bitmap *disabled;
 	
+	struct amount_msat total_payment;
 	
 	u32 *arc_head_node; 
 	// notice that a tail node is not needed, 
@@ -100,6 +105,10 @@ struct pay_parameters {
 	
 	u32 *arc_adjacency_next_arc;
 	u32 *node_adjacency_first_arc;	
+	
+	// channel linearization parameters
+	double cap_fraction[CHANNEL_PARTS], 
+	       cost_fraction[CHANNEL_PARTS];
 };
 
 struct residual_network {
@@ -186,14 +195,29 @@ static inline u32 channel_idx_to_arc(const u32 chan_idx,
 }
 
 /* Split a directed channel into parts with linear cost function. */
-static int linearize_channel(
+static void linearize_channel(
 		const struct pay_parameters *params,
 		const struct gossmap_chan *c,
 		const int dir,
 		s64 *capacity,
 		s64 *cost)
 {
-	// TODO
+	struct chan_extra_half *extra_half = get_chan_extra_half_by_chan(
+							params->gossmap,
+							params->chan_extra_map,
+							c,
+							dir);
+	
+	s64 a = extra_half->known_min.millisatoshis/1000,
+	    b = extra_half->known_max.millisatoshis/1000;
+	
+	capacity[0]=a;
+	cost[0]=0;
+	for(size_t i=1;i<CHANNEL_PARTS;++i)
+	{
+		capacity[i] = params->cap_fraction[i]*(b-a);
+		cost[i] = params->cost_fraction[i]*COST_FACTOR*params->total_payment.millisatoshis;
+	}
 }
 
 /* Get the fee cost associated to this directed channel. 
@@ -258,6 +282,8 @@ static void init_residual_network(struct pay_parameters *params,
 	for(size_t i=0;i<tal_count(params->node_adjacency_first_arc);++i)
 		params->node_adjacency_first_arc[i]=INVALID_INDEX;
 	
+	s64 capacity[CHANNEL_PARTS],prob_cost[CHANNEL_PARTS];
+	
 	struct gossmap_node *node = gossmap_first_node(params->gossmap);
 	const size_t num_nodes = gossmap_num_nodes(params->gossmap);
 	for(size_t i=0; i<num_nodes; ++i, node = gossmap_next_node(params->gossmap,node))
@@ -606,6 +632,20 @@ struct flow **optimal_payment_flow(
 	params->target = target;
 	params->chan_extra_map = chan_extra_map;
 	params->disabled = disabled;
+	params->total_payment = amount;
+	
+	params->cap_fraction[0]=0;
+	params->cost_fraction[0]=0;
+	for(size_t i =0;i<CHANNEL_PARTS;++i)
+	{
+		params->cap_fraction[i]=CHANNEL_PIVOTS[i]-CHANNEL_PIVOTS[i-1];
+		params->cost_fraction[i]=
+			log((1-CHANNEL_PIVOTS[i-1])/(1-CHANNEL_PIVOTS[i]))
+			/params->cap_fraction[i];
+	}
+	
+	// cap_fraction[i] = pivot[i]-pivot[i-1]
+	// cost_fraction[i] = log((1-pivot[i-1])/(1-pivot[i]))/(pivot[i]-pivot[i-1])
 	
 	// TODO: handle max_fee_ppm, delay_feefactor and basefee_penalty
 	// params->max_fee_ppm = max_fee_ppm;
@@ -626,14 +666,23 @@ struct flow **optimal_payment_flow(
 	// 	No feasible flow
 	// 	return NULL
 	// 
-	// -> select a default value for mu
-	// -> loop
-	// 	-> set the cost as a combination of prob. cost and fee cost
+	// range for mu: mu_left = 0, mu_right=MU_MAX
+	// -> loop a binary seach for mu
+	// 	-> mu = ( mu_left + mu_right )/ 2
+	// 	-> set c(x) = (MU_MAX-1-mu)*c_prob(x) + mu*c_fee(x)
 	// 	-> network_flow = refine network_flow to minimize costs
-	// 	-> if fee ratio < max_fee_ppm break, else increase mu
-	// 	-> if mu is beyond the limits then return an error code: 
-	// 		cannot find a cheap route
+	// 	-> if fee ratio > max_fee_ppm:
+	// 		mu_left = mu + 1
+	// 	   else if prob < min_prob:
+	// 	   	mu_right = mu
+	// 	   else:
+	// 	   	a good flow was found, break
+	// 	-> if mu_left >= mu_right:
+	// 		cannot find a cheap and reliable route
+	// 		flow not found
 	// 		break
+	// 
+	// // if flow not found fallback to c_prob or c_fee?
 	// 		
 	// -> flow = get flow paths from network_flow
 	

From 6cd316bf25a29743db96161dca0aa634a4782b41 Mon Sep 17 00:00:00 2001
From: Lagrang3 <eduardo.quintana@pm.me>
Date: Tue, 18 Apr 2023 15:23:07 +0100
Subject: [PATCH 05/64] renepay: use a queue to traverse the graph

- add ccan/lqueue
- use an lqueue to traverse the residual network graph in order to seach
for an admissible path from `source` to `target`.
---
 Makefile                    |   1 +
 ccan/ccan/lqueue/LICENSE    |   1 +
 ccan/ccan/lqueue/_info      |  57 +++++++++
 ccan/ccan/lqueue/lqueue.h   | 238 ++++++++++++++++++++++++++++++++++++
 ccan/ccan/lqueue/test/run.c |  69 +++++++++++
 plugins/renepay/mcf.c       |  78 +++++++-----
 6 files changed, 412 insertions(+), 32 deletions(-)
 create mode 120000 ccan/ccan/lqueue/LICENSE
 create mode 100644 ccan/ccan/lqueue/_info
 create mode 100644 ccan/ccan/lqueue/lqueue.h
 create mode 100644 ccan/ccan/lqueue/test/run.c

diff --git a/Makefile b/Makefile
index 011fd5f9a05e..65dcec1bbaeb 100644
--- a/Makefile
+++ b/Makefile
@@ -191,6 +191,7 @@ CCAN_HEADERS :=						\
 	$(CCANDIR)/ccan/json_out/json_out.h		\
 	$(CCANDIR)/ccan/likely/likely.h			\
 	$(CCANDIR)/ccan/list/list.h			\
+	$(CCANDIR)/ccan/lqueue/lqueue.h			\
 	$(CCANDIR)/ccan/mem/mem.h			\
 	$(CCANDIR)/ccan/membuf/membuf.h			\
 	$(CCANDIR)/ccan/noerr/noerr.h			\
diff --git a/ccan/ccan/lqueue/LICENSE b/ccan/ccan/lqueue/LICENSE
new file mode 120000
index 000000000000..2354d12945d3
--- /dev/null
+++ b/ccan/ccan/lqueue/LICENSE
@@ -0,0 +1 @@
+../../licenses/BSD-MIT
\ No newline at end of file
diff --git a/ccan/ccan/lqueue/_info b/ccan/ccan/lqueue/_info
new file mode 100644
index 000000000000..d6b6232dac39
--- /dev/null
+++ b/ccan/ccan/lqueue/_info
@@ -0,0 +1,57 @@
+#include "config.h"
+#include <stdio.h>
+#include <string.h>
+
+/**
+ * lqueue - Simple, singly-linked-list queue implementation
+ *
+ * This code provides a simple implementation of the Queue abstract
+ * data type in terms of a singly linked (circular) list.
+ *
+ * License: BSD-MIT
+ * Author: David Gibson <david@gibson.dropbear.id.au>
+ *
+ * Example:
+ *	#include <ccan/lqueue/lqueue.h>
+ *
+ *	struct arg {
+ *		const char *arg;
+ *		struct lqueue_link ql;
+ *	};
+ *
+ *	int main(int argc, char *argv[])
+ *	{
+ *		int i;
+ *		struct arg *a;
+ *		LQUEUE(struct arg, ql) argq = LQUEUE_INIT;
+ *
+ *		for (i = 0; i < argc; i++) {
+ *			a = malloc(sizeof(*a));
+ *			a->arg = argv[i];
+ *			lqueue_enqueue(&argq, a);
+ *		}
+ *
+ *		printf("Command line arguments in order:\n");
+ *
+ *		while (!lqueue_empty(&argq)) {
+ *			a = lqueue_dequeue(&argq);
+ *			printf("Argument: %s\n", a->arg);
+ *			free(a);
+ *		}
+ *
+ *		return 0;
+ *	}
+ */
+int main(int argc, char *argv[])
+{
+	/* Expect exactly one argument */
+	if (argc != 2)
+		return 1;
+
+	if (strcmp(argv[1], "depends") == 0) {
+		printf("ccan/tcon\n");
+		return 0;
+	}
+
+	return 1;
+}
diff --git a/ccan/ccan/lqueue/lqueue.h b/ccan/ccan/lqueue/lqueue.h
new file mode 100644
index 000000000000..cff51bb89103
--- /dev/null
+++ b/ccan/ccan/lqueue/lqueue.h
@@ -0,0 +1,238 @@
+/* Licensed under BSD-MIT - see LICENSE file for details */
+#ifndef CCAN_LQUEUE_H
+#define CCAN_LQUEUE_H
+
+#include <stdbool.h>
+#include <stdio.h>
+#include <assert.h>
+
+#include <ccan/tcon/tcon.h>
+
+/**
+ * struct lqueue_link - a queue link
+ * @next: next entry, or front of queue, if this is the back
+ *
+ * This is used as a link within a queue entry.
+ *
+ * Example:
+ *	struct waiter {
+ *		char *name;
+ *		struct lqueue_link ql;
+ *	};
+ */
+struct lqueue_link {
+	struct lqueue_link *next;
+};
+
+/**
+ * struct lqueue_ - a queue (internal type)
+ * @b: the back of the queue (NULL if empty)
+ */
+struct lqueue_ {
+	struct lqueue_link *back;
+};
+
+/**
+ * LQUEUE - declare a queue
+ * @type: the type of elements in the queue
+ * @link: the field containing the lqueue_link in @type
+ *
+ * The LQUEUE macro declares an lqueue.  It can be prepended by
+ * "static" to define a static lqueue.  The queue begins in undefined
+ * state, you must either initialize with LQUEUE_INIT, or call
+ * lqueue_init() before using it.
+ *
+ * See also:
+ *	lqueue_init()
+ *
+ * Example:
+ *	struct element {
+ *		int value;
+ *		struct lqueue_link link;
+ *	};
+ *	LQUEUE(struct element, link) my_queue;
+ */
+#define LQUEUE(etype, link)						\
+	TCON_WRAP(struct lqueue_,					\
+		  TCON_CONTAINER(canary, etype, link))
+
+/**
+ * LQUEUE_INIT - initializer for an empty queue
+ *
+ * The LQUEUE_INIT macro returns a suitable initializer for a queue
+ * defined with LQUEUE.
+ *
+ * Example:
+ *	struct element {
+ *		int value;
+ *		struct lqueue_link link;
+ *	};
+ *	LQUEUE(struct element, link) my_queue = LQUEUE_INIT;
+ *
+ *	assert(lqueue_empty(&my_queue));
+ */
+#define LQUEUE_INIT				\
+	TCON_WRAP_INIT({ NULL, })
+
+/**
+ * lqueue_entry - convert an lqueue_link back into the structure containing it.
+ * @q: the queue
+ * @l: the lqueue_link
+ *
+ * Example:
+ *	struct waiter {
+ *		char *name;
+ *		struct lqueue_link ql;
+ *	} w;
+ *	LQUEUE(struct waiter, ql) my_queue;
+ *	assert(lqueue_entry(&my_queue, &w.ql) == &w);
+ */
+#define lqueue_entry(q_, l_) tcon_container_of((q_), canary, (l_))
+
+/**
+ * lqueue_init_from_back - initialize a queue with a specific back element
+ * @s: the lqueue to initialize
+ * @e: pointer to the back element of the new queue
+ *
+ * USE WITH CAUTION: This is for handling unusual cases where you have
+ * a pointer to an element in a previously constructed queue but can't
+ * conveniently pass around a normal struct lqueue.  Usually you
+ * should use lqueue_init().
+ *
+ * Example:
+ *	struct element {
+ *		int value;
+ *		struct lqueue_link link;
+ *	} el;
+ *	LQUEUE(struct element, link) queue1;
+ *	LQUEUE(struct element, link) queue2;
+ *
+ *	lqueue_enqueue(&queue1, &el);
+ *
+ *	lqueue_init_from_back(&queue2, lqueue_back(&queue1));
+ */
+#define lqueue_init_from_back(q_, e_)					\
+	(lqueue_init_(tcon_unwrap(q_), tcon_member_of((q_), canary, (e_))))
+
+/**
+ * lqueue_init - initialize a queue
+ * @h: the lqueue to set to an empty queue
+ *
+ * Example:
+ *	struct element {
+ *		int value;
+ *		struct lqueue_link link;
+ *	};
+ *	LQUEUE(struct element, link) *qp = malloc(sizeof(*qp));
+ *	lqueue_init(qp);
+ */
+#define lqueue_init(q_) \
+	(lqueue_init_(tcon_unwrap(q_), NULL))
+static inline void lqueue_init_(struct lqueue_ *q, struct lqueue_link *back)
+{
+	q->back = back;
+}
+
+/**
+ * lqueue_empty - is a queue empty?
+ * @q: the queue
+ *
+ * If the queue is empty, returns true.
+ */
+#define lqueue_empty(q_) \
+	lqueue_empty_(tcon_unwrap(q_))
+static inline bool lqueue_empty_(const struct lqueue_ *q)
+{
+	return (q->back == NULL);
+}
+
+/**
+ * lqueue_front - get front entry in a queue
+ * @q: the queue
+ *
+ * If the queue is empty, returns NULL.
+ *
+ * Example:
+ *	struct element *f;
+ *
+ *	f = lqueue_front(qp);
+ *	assert(lqueue_dequeue(qp) == f);
+ */
+#define lqueue_front(q_) \
+	lqueue_entry((q_), lqueue_front_(tcon_unwrap(q_)))
+static inline struct lqueue_link *lqueue_front_(const struct lqueue_ *q)
+{
+	if (!q->back)
+		return NULL;
+	else
+		return q->back->next;
+}
+
+/**
+ * lqueue_back - get back entry in a queue
+ * @q: the queue
+ *
+ * If the queue is empty, returns NULL.
+ *
+ * Example:
+ *	struct element b;
+ *
+ *	lqueue_enqueue(qp, &b);
+ *	assert(lqueue_back(qp) == &b);
+ */
+#define lqueue_back(q_) \
+	lqueue_entry((q_), lqueue_back_(tcon_unwrap(q_)))
+static inline struct lqueue_link *lqueue_back_(const struct lqueue_ *q)
+{
+	return q->back;
+}
+
+/**
+ * lqueue_enqueue - add an entry to the back of a queue
+ * @q: the queue to add the node to
+ * @e: the item to enqueue
+ *
+ * The lqueue_link does not need to be initialized; it will be overwritten.
+ */
+#define lqueue_enqueue(q_, e_)			\
+	lqueue_enqueue_(tcon_unwrap(q_), tcon_member_of((q_), canary, (e_)))
+static inline void lqueue_enqueue_(struct lqueue_ *q, struct lqueue_link *e)
+{
+	if (lqueue_empty_(q)) {
+		/* New entry will be both front and back of queue */
+		e->next = e;
+		q->back = e;
+	} else {
+		e->next = lqueue_front_(q);
+		q->back->next = e;
+		q->back = e;
+	}
+}
+
+/**
+ * lqueue_dequeue - remove and return the entry from the front of the queue
+ * @q: the queue
+ *
+ * Note that this leaves the returned entry's link in an undefined
+ * state; it can be added to another queue, but not deleted again.
+ */
+#define lqueue_dequeue(q_) \
+	lqueue_entry((q_), lqueue_dequeue_(tcon_unwrap(q_)))
+static inline struct lqueue_link *lqueue_dequeue_(struct lqueue_ *q)
+{
+	struct lqueue_link *front;
+
+	if (lqueue_empty_(q))
+		return NULL;
+
+	front = lqueue_front_(q);
+	if (front == lqueue_back_(q)) {
+		assert(front->next == front);
+		q->back = NULL;
+	} else {
+		q->back->next = front->next;
+	}
+	return front;
+}
+
+#endif /* CCAN_LQUEUE_H */
diff --git a/ccan/ccan/lqueue/test/run.c b/ccan/ccan/lqueue/test/run.c
new file mode 100644
index 000000000000..6791d2518890
--- /dev/null
+++ b/ccan/ccan/lqueue/test/run.c
@@ -0,0 +1,69 @@
+#include "config.h"
+
+#include <ccan/lqueue/lqueue.h>
+#include <ccan/tap/tap.h>
+
+struct waiter {
+	const char *name;
+	struct lqueue_link ql;
+};
+
+int main(void)
+{
+	LQUEUE(struct waiter, ql) q = LQUEUE_INIT;
+	struct waiter a = { "Alice" };
+	struct waiter b = { "Bob" };
+	struct waiter c = { "Carol" };
+	struct waiter *waiter;
+
+	/* This is how many tests you plan to run */
+	plan_tests(25);
+
+	ok1(lqueue_empty(&q));
+	ok1(lqueue_front(&q) == NULL);
+	ok1(lqueue_back(&q) == NULL);
+
+	lqueue_enqueue(&q, &a);
+
+	ok1(!lqueue_empty(&q));
+	ok1(lqueue_front(&q) == &a);
+	ok1(lqueue_back(&q) == &a);
+
+	lqueue_enqueue(&q, &b);
+
+	ok1(!lqueue_empty(&q));
+	ok1(lqueue_front(&q) == &a);
+	ok1(lqueue_back(&q) == &b);
+
+	lqueue_enqueue(&q, &c);
+
+	ok1(!lqueue_empty(&q));
+	ok1(lqueue_front(&q) == &a);
+	ok1(lqueue_back(&q) == &c);
+
+	waiter = lqueue_dequeue(&q);
+	ok1(waiter == &a);
+
+	ok1(!lqueue_empty(&q));
+	ok1(lqueue_front(&q) == &b);
+	ok1(lqueue_back(&q) == &c);
+
+	waiter = lqueue_dequeue(&q);
+	ok1(waiter == &b);
+
+	ok1(!lqueue_empty(&q));
+	ok1(lqueue_front(&q) == &c);
+	ok1(lqueue_back(&q) == &c);
+
+	waiter = lqueue_dequeue(&q);
+	ok1(waiter == &c);
+
+	ok1(lqueue_empty(&q));
+	ok1(lqueue_front(&q) == NULL);
+	ok1(lqueue_back(&q) == NULL);
+
+	ok1(lqueue_dequeue(&q) == NULL);
+
+	/* This exits depending on whether all tests passed */
+	return exit_status();
+}
diff --git a/plugins/renepay/mcf.c b/plugins/renepay/mcf.c
index dff551fd0879..7cff54db9123 100644
--- a/plugins/renepay/mcf.c
+++ b/plugins/renepay/mcf.c
@@ -1,10 +1,14 @@
 #include "config.h"
+#include <ccan/lqueue/lqueue.h>
 #include <plugins/renepay/mcf.h>
 #include <plugins/renepay/flow.h>
 #include <math.h>
 
 #define PARTS_BITS 2
 #define CHANNEL_PARTS (1 << PARTS_BITS)
+
+// These are the probability intervals we use to decompose a channel into linear
+// cost function arcs.
 static const double CHANNEL_PIVOTS[]={0,0.5,0.8,0.95};
 
 // how many bits for linearization parts plus 1 bit for the direction of the
@@ -15,6 +19,12 @@ static const s64 INFINITE = 9e18;
 static const u32 INVALID_INDEX=0xffffffff;
 static const s64 COST_FACTOR=10;
 static const s64 MU_MAX = 128;
+
+struct queue_data
+{
+	u32 idx;
+	struct lqueue_link ql;
+};
 			
 /* Let's try this encoding of arcs:
  * Each channel `c` has two possible directions identified by a bit 
@@ -100,8 +110,8 @@ struct pay_parameters {
 	// notice that a tail node is not needed, 
 	// because the tail of arc is the head of dual(arc)
 	
-	// probability cost associated to an arc
-	s64 *arc_prob_cost;
+	// probability and fee cost associated to an arc
+	s64 *arc_prob_cost, *arc_fee_cost;
 	
 	u32 *arc_adjacency_next_arc;
 	u32 *node_adjacency_first_arc;	
@@ -220,6 +230,7 @@ static void linearize_channel(
 	}
 }
 
+	
 /* Get the fee cost associated to this directed channel. 
  * Cost is expressed as PPM of the payment. */
 static s64 compute_fee_cost(
@@ -246,9 +257,6 @@ static s64 compute_fee_cost(
 	// In this case having a base fee of 1 is equivalent of having a
 	// proportional fee of 10 ppm.
 	return pfee + bfee * 10;
-	
-	// TODO(eduardo) How to convert the base fee properly into an effective
-	// proportional fee?
 }
 
 static void init_residual_network(struct pay_parameters *params,
@@ -299,10 +307,6 @@ static void init_residual_network(struct pay_parameters *params,
 			                                                node, j, &half);
 			const u32 chan_id = gossmap_chan_idx(params->gossmap, c);
 			
-			// TODO: question here, is the pair `(c,!half)` identified
-			// with the outgoing channel from `node` or the incoming
-			// channel?
-			
 			const struct gossmap_node *next = gossmap_nth_node(params->gossmap,
 									   c,!half);
 			const u32 next_id = gossmap_node_idx(params->gossmap,next);
@@ -315,14 +319,14 @@ static void init_residual_network(struct pay_parameters *params,
 			
 			// split this channel direction to obtain the arcs
 			// that are outgoing to `node`
-			linearize_channel(params,c,!half,capacity,prob_cost);
+			linearize_channel(params,c,half,capacity,prob_cost);
 			
 			// let's subscribe the 4 parts of the channel direction
-			// (c,!half), the dual of these guys will be subscribed
+			// (c,half), the dual of these guys will be subscribed
 			// when the `i` hits the `next` node.
 			for(size_t k=0;k<CHANNEL_PARTS;++k)
 			{
-				u32 arc = channel_idx_to_arc(chan_id,!half,k,0);
+				u32 arc = channel_idx_to_arc(chan_id,half,k,0);
 				arc_head_node[arc] = next_id;
 				
 				// Is this is the first arc?
@@ -340,18 +344,18 @@ static void init_residual_network(struct pay_parameters *params,
 				
 				network->cap[arc] = capacity[k];
 				params->arc_prob_cost[arc] = prob_cost[k];
-				params->fee_cost[arc] = compute_fee_cost(params,c,!half);
+				params->arc_fee_cost[arc] = compute_fee_cost(params,c,half);
 			}
 			
 			// split the opposite direction to obtain the dual arcs
 			// that are outgoing to `node`
-			linearize_channel(params,c,half,capacity,prob_cost);
+			linearize_channel(params,c,!half,capacity,prob_cost);
 			
 			// let's subscribe the 4 parts of the channel direction
-			// (c,half) in dual representation
+			// (c,!half) in dual representation
 			for(size_t k=0;k<CHANNEL_PARTS;++k)
 			{
-				u32 arc = channel_idx_to_arc(chan_id,half,k,1);
+				u32 arc = channel_idx_to_arc(chan_id,!half,k,1);
 				arc_head_node[arc] = next_id;
 				
 				// Is this is the first arc?
@@ -369,7 +373,7 @@ static void init_residual_network(struct pay_parameters *params,
 				
 				network->cap[arc] = 0;
 				params->arc_prob_cost[arc] = -prob_cost[k];
-				params->fee_cost[arc] = -compute_fee_cost(params,c,!half);
+				params->arc_fee_cost[arc] = -compute_fee_cost(params,c,!half);
 			}
 		}
 	}
@@ -381,26 +385,34 @@ static void init_residual_network(struct pay_parameters *params,
  * traversed. 
  * Returns RENEPAY_ERR_OK if the path exists. */
 static int find_admissible_path(const struct pay_parameters *params,
-			        struct residual_network *network,
+			        const struct residual_network *network,
                                 const u32 source,
 				const u32 target,
 				u32 *prev)
 {
-	int status = RENEPAY_ERR_NOFEASIBLEFLOW;
+	int ret = RENEPAY_ERR_NOFEASIBLEFLOW;
 	
-	// TODO: memset(prev) = INVALID_INDEX
+	for(size_t i=0;i<tal_count(prev);++i)
+		prev[i]=INVALID_INDEX;
 	
 	// The graph is dense, and the farthest node is just a few hops away,
 	// hence let's BFS search.
-	queue Q; // TODO
+	LQUEUE(struct queue_data,ql) myqueue = LQUEUE_INIT;
+	struct queue_data *qdata;
 	
-	// TODO: Q.push(source);
-	while // TODO: Q not empty
+	qdata = tal(tmpctx,struct queue_data);
+	qdata->idx = source;
+	lqueue_enqueue(&myqueue,qdata);
+	
+	while(!lqueue_empty(&myqueue))
 	{
-		// TODO: u32 cur = Q.front(); Q.pop();
+		qdata = lqueue_dequeue(&myqueue);
+		u32 cur = qdata->idx;
+		tal_free(qdata);
+		
 		if(cur==target)
 		{
-			status = RENEPAY_ERR_OK;
+			ret = RENEPAY_ERR_OK;
 			break;
 		}
 		
@@ -419,11 +431,14 @@ static int find_admissible_path(const struct pay_parameters *params,
 				continue;
 			
 			prev[next] = arc;
-			// TODO: Q.push(next);
+			
+			qdata = tal(tmpctx,struct queue_data);
+			qdata->idx = next;
+			lqueue_enqueue(&myqueue,qdata);
 		}
 	}
 	
-	return status;
+	return ret;
 }
 
 /* Get the max amount of flow one can send from source to target along the path
@@ -492,11 +507,10 @@ static int find_feasible_flow(const struct pay_parameters *params,
 			      const u32 target,
 			      s64 amount)
 {
-	int status = RENEPAY_ERR_OK;
+	int ret = RENEPAY_ERR_OK;
 	
 	/* path information 
 	 * prev: is the id of the arc that lead to the node. */
-	// TODO: how do we encode the arc idx?
 	u32 *prev = tal_arr(tmpctx,u32
 		            gossmap_max_node_idx(params->gossmap));
 	
@@ -506,7 +520,7 @@ static int find_feasible_flow(const struct pay_parameters *params,
 		int err = find_admissible_path(params,network,source,target,prev);
 		if(err!=RENEPAY_ERR_OK)
 		{
-			status = RENEPAY_ERR_NOFEASIBLEFLOW;
+			ret = RENEPAY_ERR_NOFEASIBLEFLOW;
 			break;
 		}
 		
@@ -523,10 +537,10 @@ static int find_feasible_flow(const struct pay_parameters *params,
 	
 	tal_free(prev);	
 	
-	return status;
+	return ret;
 }
 
-/* TODO(eduardo): How to combine probability cost and fee cost?
+/* How to combine probability cost and fee cost?
  * This is tricky, because we say that a fee cost is high when the ratio of fee
  * and actual payment is high, typically 1000ppm or more. 
  * On the other hand a probability cost is high if the probability of success is

From 8be7a36e761a40f81a8a04d49dbe8eb5af5e5686 Mon Sep 17 00:00:00 2001
From: Lagrang3 <eduardo.quintana@pm.me>
Date: Tue, 18 Apr 2023 20:57:05 +0100
Subject: [PATCH 06/64] renepay: implement prob. and fee cost from flow

---
 plugins/renepay/mcf.c | 215 ++++++++++++++++++++++++++++++++++--------
 1 file changed, 177 insertions(+), 38 deletions(-)

diff --git a/plugins/renepay/mcf.c b/plugins/renepay/mcf.c
index 7cff54db9123..0432080de974 100644
--- a/plugins/renepay/mcf.c
+++ b/plugins/renepay/mcf.c
@@ -90,6 +90,7 @@ struct queue_data
 #define MIN(x, y) (((x) < (y)) ? (x) : (y))
 
 struct pay_parameters {
+	
 	/* The gossmap we are using */
 	struct gossmap *gossmap;
 	struct gossmap_node *source;
@@ -99,12 +100,13 @@ struct pay_parameters {
 	struct chan_extra_map *chan_extra_map;
 	
 	/* Optional bitarray of disabled chans. */
-	// TODO: did you consider these?, hint: disable channels by ignoring
+	// TODO(eduardo): did you consider these?, hint: disable channels by ignoring
 	// them when constructing the adjacency list of nodes in
 	// `init_residual_network`.
 	const bitmap *disabled;
 	
-	struct amount_msat total_payment;
+	// how much we pay
+	struct amount_msat amount;
 	
 	u32 *arc_head_node; 
 	// notice that a tail node is not needed, 
@@ -136,6 +138,17 @@ static inline u32 arc_dual(const struct pay_parameters * params UNUSED,
 	return arc ^ 1;
 }
 
+/* Helper function. 
+ * Given an arc of the network (not residual) give me the flow. */
+static inline s64 get_arc_flow(
+			const struct pay_parameters * params,
+			const struct residual_network *network,
+			const u32 arc)
+{
+	ASSERT(arc & 1 == 0); // arc is not a dual arc
+	return network->cap[ arc_dual(params,arc) ];
+}
+
 /* Helper function. 
  * Given an arc idx, return the node from which this arc emanates in the residual network. */
 static inline u32 arc_tail(const struct pay_parameters *params,
@@ -204,6 +217,17 @@ static inline u32 channel_idx_to_arc(const u32 chan_idx,
 	return (chan_idx << ARC_ADDITIONAL_BITS)|(half<<(1+PARTS_BITS))|(part<<1)|(dual);
 }
 
+/* Helper function.
+ * Evaluate Pickhardt's probability cost function. */
+static inline double pickhardt_probability_cost(
+	double lim_low, double lim_high, double flow)
+{
+	if(flow<=lim_low)
+		return 0.;
+	ASSSERT(flow<lim_high);
+	return -log(1.-(flow-lim_low)/(lim_high-lim_low));
+}
+
 /* Split a directed channel into parts with linear cost function. */
 static void linearize_channel(
 		const struct pay_parameters *params,
@@ -226,7 +250,7 @@ static void linearize_channel(
 	for(size_t i=1;i<CHANNEL_PARTS;++i)
 	{
 		capacity[i] = params->cap_fraction[i]*(b-a);
-		cost[i] = params->cost_fraction[i]*COST_FACTOR*params->total_payment.millisatoshis;
+		cost[i] = params->cost_fraction[i]*COST_FACTOR*params->amount.millisatoshis;
 	}
 }
 
@@ -281,6 +305,10 @@ static void init_residual_network(struct pay_parameters *params,
 	params->arc_prob_cost = tal_arr(params,s64,max_num_arcs);
 	for(size_t i=0;i<tal_count(params->arc_prob_cost);++i)
 		params->arc_prob_cost[i]=INFINITE;
+	
+	params->arc_fee_cost = tal_arr(params,s64,max_num_arcs);
+	for(size_t i=0;i<tal_count(params->arc_fee_cost);++i)
+		params->arc_fee_cost[i]=INFINITE;
 		
 	params->arc_adjacency_next_arc = tal_arr(params,u32,max_num_arcs);
 	for(size_t i=0;i<tal_count(params->arc_adjacency_next_arc);++i)
@@ -539,6 +567,77 @@ static int find_feasible_flow(const struct pay_parameters *params,
 	
 	return ret;
 }
+	
+/* Starting from a feasible flow (satisfies the balance and capacity
+ * constraints), find a solution that minimizes the network->cost function. 
+ * 
+ * Cycle cancelling? */
+static void optimize_mcf(const struct pay_parameters *params,
+			 struct residual_network *network)
+{
+	// TODO
+}
+	
+/* Given a flow in the residual network, compute the probability and fee cost
+ * ppm. 
+ * Instead of using the cost function, which contains many approximations we
+ * will use a direct approach to compute in O(n+m) the fees and prob. */
+static void estimate_costs(const struct pay_parameters *params,
+                           const struct residual_network *network,
+			   double *prob_ptr,
+			   double *fee_ppm_ptr)
+{
+	double fee_microsats = 0;
+	double prob_cost = 0;
+	
+	struct gossmap_node *node = gossmap_first_node(params->gossmap);
+	const size_t num_nodes = gossmap_num_nodes(params->gossmap);
+	for(size_t i=0; i<num_nodes; ++i, node = gossmap_next_node(params->gossmap,node))
+	{
+		for(size_t j=0;j<node->num_chans;++j)
+		{
+			int dir;
+			const struct gossmap_chan *c = gossmap_nth_chan(params->gossmap,
+			                                                node, j, &dir);
+			const u32 chan_idx = gossmap_chan_idx(params->gossmap, c);
+			
+			// in sats
+			s64 chan_flow=0;
+			
+			for(size_t k=0;k<CHANNEL_PARTS;++k)
+			{
+				const u32 arc = channel_idx_to_arc(chan_idx,dir,k,0);
+				chan_flow += get_arc_flow(params,network,arc);	
+			}
+			
+			fee_microsats += chan_flow * c->half[dir].proportional_fee 
+			                  + c->half[dir].base_fee*1e3; 
+			
+			struct chan_extra_half *extra_half 
+				= get_chan_extra_half_by_chan(
+						params->gossmap,
+						params->chan_extra_map
+						c,
+						dir);  
+			
+			prob_cost += pickhardt_probability_cost(
+				extra_half->known_min.millisatoshis*1e3,
+				extra_half->known_max.millisatoshis*1e3,
+				chan_flow);
+		}
+	}
+	*prob_str = exp(-prob_cost);
+	*fee_ppm_str = fee_microsats * 1e3 / params->amount.millisatoshis;
+}
+		
+/* Given a flow in the residual network, build a set of payment flows in the
+ * gossmap that corresponds to this flow. */		
+static struct flow** get_flow_paths(const struct pay_parameters *params,
+				    const struct residual_network *network,
+				    const tal_t *ctx)
+{
+	// TODO
+}
 
 /* How to combine probability cost and fee cost?
  * This is tricky, because we say that a fee cost is high when the ratio of fee
@@ -636,9 +735,11 @@ struct flow **optimal_payment_flow(
 		      const bitmap *disabled,
 		      struct amount_msat amount,
 		      double max_fee_ppm,
+		      double min_probability,
 		      double delay_feefactor,
 		      int *errorcode)
 {
+	// TODO(eduardo) on which ctx should I allocate the MCF data?
 	struct pay_parameters *params = tal(tmpctx,struct pay_parameters);	
 	
 	params->gossmap = gossmap;
@@ -646,7 +747,7 @@ struct flow **optimal_payment_flow(
 	params->target = target;
 	params->chan_extra_map = chan_extra_map;
 	params->disabled = disabled;
-	params->total_payment = amount;
+	params->amount = amount;
 	
 	params->cap_fraction[0]=0;
 	params->cost_fraction[0]=0;
@@ -658,10 +759,7 @@ struct flow **optimal_payment_flow(
 			/params->cap_fraction[i];
 	}
 	
-	// cap_fraction[i] = pivot[i]-pivot[i-1]
-	// cost_fraction[i] = log((1-pivot[i-1])/(1-pivot[i]))/(pivot[i]-pivot[i-1])
-	
-	// TODO: handle max_fee_ppm, delay_feefactor and basefee_penalty
+	// TODO(eduardo): handle max_fee_ppm, delay_feefactor and basefee_penalty
 	// params->max_fee_ppm = max_fee_ppm;
 	// params->delay_feefactor = delay_feefactor;
 	// params->basefee_penalty = 
@@ -670,40 +768,81 @@ struct flow **optimal_payment_flow(
 	struct residual_network *network = tal(tmpctx,struct residual_network);
 	init_residual_network(params,network);
 	
-	const u32 target_id = get_node_id(params,target);
-	const u32 source_id = get_node_id(params,source);
+	const u32 target_idx = gossmap_node_idx(params->gossmap,target);
+	const u32 source_idx = gossmap_node_idx(params->gossmap,source);
 	
-	// TODO
-	// 
-	// -> network_flow = compute any feasible flow
-	// -> if network_flow cannot be computed return an error code: 
-	// 	No feasible flow
-	// 	return NULL
-	// 
-	// range for mu: mu_left = 0, mu_right=MU_MAX
-	// -> loop a binary seach for mu
-	// 	-> mu = ( mu_left + mu_right )/ 2
-	// 	-> set c(x) = (MU_MAX-1-mu)*c_prob(x) + mu*c_fee(x)
-	// 	-> network_flow = refine network_flow to minimize costs
-	// 	-> if fee ratio > max_fee_ppm:
-	// 		mu_left = mu + 1
-	// 	   else if prob < min_prob:
-	// 	   	mu_right = mu
-	// 	   else:
-	// 	   	a good flow was found, break
-	// 	-> if mu_left >= mu_right:
-	// 		cannot find a cheap and reliable route
-	// 		flow not found
-	// 		break
-	// 
-	// // if flow not found fallback to c_prob or c_fee?
-	// 		
-	// -> flow = get flow paths from network_flow
+	int err = find_feasible_flow(params,network,source_idx,target_idx,
+	                             params->amount->millisatoshis/1000);
+	
+	if(err!=RENEPAY_ERR_OK)
+	{
+		// there is no flow that satisfy the constraints, we stop here
+		goto end;
+	}
+	
+	const size_t max_num_arcs = tal_count(params->arc_prob_cost);
+	ASSERT(max_num_arcs==tal_count(params->arc_prob_cost));
+	ASSERT(max_num_arcs==tal_count(params->arc_fee_cost));
+	ASSERT(max_num_arcs==tal_count(network->cap));
+	ASSERT(max_num_arcs==tal_count(network->cost));
+	
+	// binary search for a value of `mu` that fits our fee and prob.
+	// constraints.
+	bool flow_found = false;
+	s64 mu_left = 0, mu_right = MU_MAX;
+	while(mu_left<mu_right)
+	{
+		s64 mu = (mu_left + mu_right)/2;
+		
+		for(u32 arc=0;arc<max_num_arcs;++arc)
+		{
+			const s64 pcost = params->arc_prob_cost[arc],
+			          fcost = params->arc_fee_cost[arc];
+				  
+			const s64 combined = pcost==INFINITE || fcost==INFINITE ? INFINITE :
+			                     mu*fcost + (MU_MAX-1-mu)*pcost;
+				       
+			network->cost[arc]
+				= mu==0 ? pcost : 
+				          (mu==(MU_MAX-1) ? fcost : combined);
+		}
+		
+		optimize_mcf(params,network);
+		
+		double prob_success,fee_ppm;
+		estimate_costs(params,network,&prob,&fee_ppm);
+		
+		if(fee_ppm>max_fee_ppm)
+		{
+			// too expensive
+			mu_left = mu+1;
+		}else if(prob<min_probability)
+		{
+			// too unlikely
+			mu_right = mu;
+		}else
+		{
+			// a good compromise
+			flow_found=true;
+			break;
+		}
+	}
+	
+	end:
+	
+	struct flow **flow_paths = NULL;
+
+	if(flow_found)
+	{
+		flow_paths = get_flow_paths(params,network,ctx);
+	}else
+	{
+	// TODO(eduardo) fallback to c_prob or c_fee?
+	}
 	
 	tal_free(network);
 	tal_free(params);
 	
-	// TODO
-	// return flow
+	return flow_paths;
 }
 

From ff8b4c9511b2a43f50c789caac346de42303d5e0 Mon Sep 17 00:00:00 2001
From: Lagrang3 <eduardo.quintana@pm.me>
Date: Wed, 19 Apr 2023 11:52:54 +0100
Subject: [PATCH 07/64] renepay: split a flow into path flows

---
 plugins/renepay/mcf.c | 197 +++++++++++++++++++++++++++++++++++++++---
 plugins/renepay/mcf.h |  36 ++++++--
 2 files changed, 218 insertions(+), 15 deletions(-)

diff --git a/plugins/renepay/mcf.c b/plugins/renepay/mcf.c
index 0432080de974..062289924f63 100644
--- a/plugins/renepay/mcf.c
+++ b/plugins/renepay/mcf.c
@@ -137,6 +137,13 @@ static inline u32 arc_dual(const struct pay_parameters * params UNUSED,
 {
 	return arc ^ 1;
 }
+/* Helper function. */
+static inline bool arc_is_dual(
+		const struct pay_parameters *params UNUSED,
+		const u32 arc)
+{
+	return (arc & 1) == 1;
+}
 
 /* Helper function. 
  * Given an arc of the network (not residual) give me the flow. */
@@ -610,6 +617,9 @@ static void estimate_costs(const struct pay_parameters *params,
 				chan_flow += get_arc_flow(params,network,arc);	
 			}
 			
+			// TODO(eduardo): this is wrong, because the base fee is
+			// invoked for each HTLC, we may use a single payment
+			// channel in multiple payment flows.
 			fee_microsats += chan_flow * c->half[dir].proportional_fee 
 			                  + c->half[dir].base_fee*1e3; 
 			
@@ -629,14 +639,171 @@ static void estimate_costs(const struct pay_parameters *params,
 	*prob_str = exp(-prob_cost);
 	*fee_ppm_str = fee_microsats * 1e3 / params->amount.millisatoshis;
 }
-		
+
+// flow on directed channels
+struct chan_flow
+{
+	s64 half[2];
+};
+
+struct list_data
+{
+	struct flow_path *flow_path;
+	struct list_node list;
+};
+
 /* Given a flow in the residual network, build a set of payment flows in the
  * gossmap that corresponds to this flow. */		
-static struct flow** get_flow_paths(const struct pay_parameters *params,
+static struct flow_path **
+	get_flow_paths(const struct pay_parameters *params,
 				    const struct residual_network *network,
 				    const tal_t *ctx)
 {
-	// TODO
+	tal_t *this_ctx = tal(tmpctx,tal_t);
+	
+	const size_t max_num_chans = gossmap_max_chan_idx(params->gossmap);
+	struct chan_flow *chan_flow = tal_arr(this_ctx,struct chan_flow,max_num_chans);
+	
+	for(size_t i=0;i<max_num_chans;++i)
+		chan_flow[i].half[0]=chan_flow[i].half[1]=0;
+	
+	const size_t max_num_nodes = gossmap_max_node_idx(params->gossmap);
+	s64 *balance = tal_arr(this_ctx,s64,max_num_nodes);
+	struct gossmap_chan **prev_chan = tal_arr(this_ctx,struct gossmap_chan*,max_num_nodes);
+	int *prev_dir = tal_arr(this_ctx,int,max_num_nodes);
+	u32 *prev_idx = tal_arr(this_ctx,u32,max_num_nodes);
+	
+	for(size_t i=0;i<max_num_nodes;++i)
+	{
+		balance[i]=0;
+	}
+	
+	// Convert the arc based residual network flow into a flow in the
+	// directed channel network.
+	// Compute balance on the nodes.
+	for(u32 n = 0;n<max_num_nodes;++n)
+	{
+		for(u32 arc = node_adjacency_begin(params,cur);
+		        arc!= node_adjacency_end(params,cur);
+			arc = node_adjacency_next(params,cur,arc))
+		{
+			if(arc_is_dual(arc))
+				continue;
+			u32 m = arc_head(params,arc);
+			s64 flow = get_arc_flow(params,network,arc);
+			
+			balance[n] -= flow;
+			balance[m] += flow;
+			
+			u32 chan_idx;
+			int dir,part,dual;
+			chan_idx = arc_to_channel_idx(arc,dir,part,dual);
+			
+			chan_flow[chan_idx].half[dir] +=flow;
+		}
+			
+	}
+	
+	
+	size_t num_paths=0;
+	tal_t *list_ctx = tal(this_ctx,tal_t);
+	LIST_HEAD(path_list);
+	struct list_data *ld;
+	
+	// Select all nodes with negative balance and find a flow that reaches a
+	// positive balance node.
+	for(u32 node_idx=0;node_idx<max_num_nodes;++node_idx)
+	{
+		// this node has negative balance, flows leaves from here
+		while(balance[node_idx]<0)
+		{	
+			s64 delta=-balance[node_idx];
+			u32 final_idx;
+			
+			int length = 0;
+			prev_chan[node_idx]=NULL;
+			
+			u32 final_idx = node_idx;
+			
+			/* TODO(eduardo)
+			 * This is guaranteed to halt if there are no directed flow cycles. 
+			 * There souldn't be any. In fact if cost is strickly
+			 * positive, then flow cycles do not exist at all in the
+			 * MCF solution. But if cost is allowed to be zero for
+			 * some arcs, then we might have flow cyles in the final
+			 * solution. We must somehow ensure that the MCF
+			 * algorithm does not come up with spurious flow cycles. */
+			while(balance[final_idx]<=0)
+			{
+				const struct gossmap_node *cur
+					= gossmap_node_byidx(params->gossmap,final_idx);
+				
+				for(size_t i=0;i<cur->num_chans;++i)
+				{
+					int dir;
+					const struct gossmap_chan *c 
+						= gossmap_nth_chan(params->gossmap,
+						                   cur,i,&dir);
+					
+					const u32 c_idx = gossmap_chan_idx(params->gossmap,c);
+					
+					if(chan_flow[c_idx].half[dir]>0)
+					{
+						// follow the flow
+						const struct gossmap_node *next
+							= gossmap_nth_node(params->gossmap,
+									   c,!dir);
+						u32 next_idx = gossmap_node_idx(params->gossmap,next);
+						
+						delta=MIN(delta,chan_flow[c_idx].half[dir]);
+						
+						prev_dir[next_idx] = dir;
+						prev_chan[next_idx] = c;
+						prev_idx[next_idx] = final_idx;
+						length ++;
+						final_idx = next_idx;
+						break;
+					}
+				}
+			}
+			delta = MIN(delta,balance[final_idx]);
+			
+			struct flow_path *fp = tal(ctx,struct flow_path);
+			fp->path = tal_arr(fp,struct gossmap_chan*,length);
+			fp->dirs = tal_arr(fp,int,length);
+			fp->amount.satoshis = delta;
+			
+			balance[node_idx] += delta;
+			balance[final_idx]-= delta;
+			
+			// walk backwards, substract flow
+			for(u32 cur_idx = final_idx;cur_idx!=node_idx;cur_idx=prev_idx[node_idx])
+			{
+				length--;
+				fp->path[length]=prev_chan[cur_idx];
+				fp->dirs[length]=prev_dir[cur_idx];
+				
+				chan_flow[prev_chan[cur_idx]].half[prev_dir[cur_idx]]-=delta;
+			}
+			
+			// add fp to list
+			ld = tal(list_ctx,struct list_data);
+			ld->flow_path = fp;
+			list_add(&path_list,&ld->list);
+			num_paths++;
+		}
+	}
+	
+	// copy the list into the array we are going to return	
+	struct flow_path **flows = tal_arr(ctx,struct flow_path*,num_paths);
+	size_t pos=0;
+	list_for_each(&path_list,ld,list)
+	{
+		flows[pos++] = ld->flow_path;
+	}
+	
+	tal_free(this_ctx);
+	return flows;
 }
 
 /* How to combine probability cost and fee cost?
@@ -726,7 +893,13 @@ static struct flow** get_flow_paths(const struct pay_parameters *params,
  * variable COST_FACTOR.
  * */
 
-struct flow **optimal_payment_flow(
+
+/* eduardo: I think it should be clear that this module deals with linear
+ * flows, ie. base fees are not considered. Hence a flow along a path is
+ * described with a sequence of directed channels and one amount. 
+ * In the `pay_flow` module there are dedicated routes to compute the actual
+ * amount to be forward on each hop. */
+int optimal_payment_flow(
                       const tal_t *ctx,
 		      struct gossmap *gossmap,
 		      const struct gossmap_node *source,
@@ -737,11 +910,13 @@ struct flow **optimal_payment_flow(
 		      double max_fee_ppm,
 		      double min_probability,
 		      double delay_feefactor,
-		      int *errorcode)
+		      struct flow_paths **flow_paths)
 {
 	// TODO(eduardo) on which ctx should I allocate the MCF data?
 	struct pay_parameters *params = tal(tmpctx,struct pay_parameters);	
 	
+	int ret;
+	
 	params->gossmap = gossmap;
 	params->source = source;
 	params->target = target;
@@ -777,6 +952,7 @@ struct flow **optimal_payment_flow(
 	if(err!=RENEPAY_ERR_OK)
 	{
 		// there is no flow that satisfy the constraints, we stop here
+		ret = RENEPAY_ERR_NOFEASIBLEFLOW;
 		goto end;
 	}
 	
@@ -790,6 +966,7 @@ struct flow **optimal_payment_flow(
 	// constraints.
 	bool flow_found = false;
 	s64 mu_left = 0, mu_right = MU_MAX;
+	ret=RENEPAY_ERR_NOCHEAPFLOW;
 	while(mu_left<mu_right)
 	{
 		s64 mu = (mu_left + mu_right)/2;
@@ -823,18 +1000,18 @@ struct flow **optimal_payment_flow(
 		}else
 		{
 			// a good compromise
-			flow_found=true;
+			ret=RENEPAY_ERR_OK;
 			break;
 		}
 	}
 	
 	end:
 	
-	struct flow **flow_paths = NULL;
+	*flow_paths = NULL;
 
-	if(flow_found)
+	if(ret==RENEPAY_ERR_OK)
 	{
-		flow_paths = get_flow_paths(params,network,ctx);
+		*flow_paths = get_flow_paths(params,network,ctx);
 	}else
 	{
 	// TODO(eduardo) fallback to c_prob or c_fee?
@@ -843,6 +1020,6 @@ struct flow **optimal_payment_flow(
 	tal_free(network);
 	tal_free(params);
 	
-	return flow_paths;
+	return ret;
 }
 
diff --git a/plugins/renepay/mcf.h b/plugins/renepay/mcf.h
index 0a2d73c798e1..eed76ddb9789 100644
--- a/plugins/renepay/mcf.h
+++ b/plugins/renepay/mcf.h
@@ -1,11 +1,34 @@
 #ifndef LIGHTNING_PLUGINS_RENEPAY_MCF_H
 #define LIGHTNING_PLUGINS_RENEPAY_MCF_H
 #include <ccan/bitmap/bitmap.h>
+#include <common/amount.h>
+#include <common/gossmap.h>
 
 struct chan_extra_map;
 
+enum {
+	RENEPAY_ERR_OK,
+	// No feasible flow found, either there is not enough known liquidity (or capacity) 
+	// in the channels to complete the payment
+	RENEPAY_ERR_NOFEASIBLEFLOW,
+	// There is at least one feasible flow, but the the cheapest solution that we
+	// found is too expensive, we return the result anyways.
+	RENEPAY_ERR_NOCHEAPFLOW
+};
+
+/* In this module, internally a flow is non-negative integer function on the
+ * arcs that satisfies the capacity constraints; outside, a flow is seen as one
+ * amount sent through a sequence of directed channels. */
+struct flow_path
+{
+	struct gossmap_chan **path;
+	int *dirs;
+	struct amount_sat amount;
+};
+
+
 /**
- * minflow - API for min cost flow function(s).
+ * optimal_payment_flow - API for min cost flow function(s).
  * @ctx: context to allocate returned flows from
  * @gossmap: the gossip map
  * @source: the source to start from
@@ -13,8 +36,9 @@ struct chan_extra_map;
  * @chan_extra_map: hashtable of extra per-channel information
  * @disabled: NULL, or a bitmap by channel index of channels not to use.
  * @amount: the amount we want to reach @target
- * @frugality: how important fees are compared to certainty (0.1 = certain, 10 = fees)
+ * @max_fee_ppm: the maximum ppm allowed in fees
  * @delay_feefactor: how important delays are compared to fees
+ * @errorcode: error code in case of failure
  *
  * @delay_feefactor converts 1 block delay into msat, as if it were an additional
  * fee.  So if a CTLV delay on a node is 5 blocks, that's treated as if it
@@ -24,13 +48,15 @@ struct chan_extra_map;
  *
  * Return a series of subflows which deliver amount to target, or NULL.
  */
-struct flow **minflow(const tal_t *ctx,
+struct flow **optimal_payment_flow(
+                      const tal_t *ctx,
 		      struct gossmap *gossmap,
 		      const struct gossmap_node *source,
 		      const struct gossmap_node *target,
 		      struct chan_extra_map *chan_extra_map,
 		      const bitmap *disabled,
 		      struct amount_msat amount,
-		      double frugality,
-		      double delay_feefactor);
+		      double max_fee_ppm,
+		      double delay_feefactor,
+		      int *errorcode);
 #endif /* LIGHTNING_PLUGINS_RENEPAY_MCF_H */

From 5c97151227a4483eba591e2e2e9fb86375ba232d Mon Sep 17 00:00:00 2001
From: Lagrang3 <eduardo.quintana@pm.me>
Date: Fri, 21 Apr 2023 11:24:42 +0100
Subject: [PATCH 08/64] renepay: use shortest augmenting path for MCF

---
 plugins/renepay/mcf.c | 152 +++++++++++++++++++++++++++++++++++++++++-
 1 file changed, 149 insertions(+), 3 deletions(-)

diff --git a/plugins/renepay/mcf.c b/plugins/renepay/mcf.c
index 062289924f63..215a4eddbbae 100644
--- a/plugins/renepay/mcf.c
+++ b/plugins/renepay/mcf.c
@@ -127,6 +127,7 @@ struct residual_network {
 	/* residual capacity on arcs */
 	s64 *cap; 
 	s64 *cost;
+    s64 *potential;
 };
 				
 
@@ -305,6 +306,10 @@ static void init_residual_network(struct pay_parameters *params,
 	for(size_t i=0;i<tal_count(network->cost);++i)
 		network->cost[i]=INFINITE;
 	
+    network->potential = tal_arr(network,s64,max_num_nodes);
+    for(size_t i=0;i<tal_count(network->potential);++i)
+        network->potential[i]=0;
+    
 	params->arc_head_node = tal_arr(params,u32,max_num_arcs);
 	for(size_t i=0;i<tal_count(params->arc_head_node);++i)
 		params->arc_head_node[i]=INVALID_INDEX;
@@ -575,14 +580,155 @@ static int find_feasible_flow(const struct pay_parameters *params,
 	return ret;
 }
 	
+/* Similar to `find_admissible_path` but use Dijkstra to optimize the distance
+ * label. Stops when the target is hit. */
+static int  find_optimal_path(
+	const struct pay_parameters *params,
+	const struct residual_network* network,
+	const u32 source,
+	const u32 target,
+	s32 *prev,
+	s64 *distance)
+{
+	tal_t *this_ctx = tal(tmpctx,tal_t);
+	int ret = RENEPAY_ERR_NOFEASIBLEFLOW;
+	
+	char *visited = tal_array(this_ctx,char,tal_count(prev));
+	
+	for(size_t i=0;i<tal_count(prev);++i)
+	{	
+		prev[i]=INVALID_INDEX;
+		distance[i]=INFINITE;
+		visited[i]=0;
+	}
+	distance[source]=0;
+	
+	// TODO add source to queue
+	while(1)
+	{
+		// TODO: get best node cur from queue 
+		
+		if (visited[cur])
+			continue;
+		
+		visited[cur]=1;
+		
+		if(cur==target)
+		{
+			ret = RENEPAY_ERR_OK;
+			break;
+		}
+		
+		for(u32 arc = node_adjacency_begin(params,cur);
+		        arc!= node_adjacency_end(params,cur);
+			arc = node_adjacency_next(params,cur,arc))
+		{
+			// check if this arc is traversable
+			if(network->cap[arc] <= 0)
+				continue;
+			
+			u32 next = arc_head(params,arc);
+			
+			s64 cij = network->cost[arc] - network->potential[cur]
+			                             + network->potential[next];
+			// Dijkstra only works with non-negative weights
+			ASSERT(cij>=0);
+			if(distance[next]>distance[cur]+cij)
+			{
+				prev[next]=arc;
+				distance[next]=distance[cur]+cij;
+				// TODO: add next to queue
+			}
+		}
+	}
+	tal_free(this_ctx);	
+	return ret;
+}
+    
+/* Set zero flow in the residual network. */
+static void zero_flow(
+	const struct pay_parameters *params,
+	struct residual_network *network)
+{
+	for(u32 node=0;node<tal_count(network->potential);++node)
+	{
+		network->potential[node]=0;
+		for(u32 arc=node_adjacency_begin(params,node);
+			arc!=node_adjacency_end(params,node);
+			arc = node_adjacency_next(params,node,arc))
+		{
+			if(arc_is_dual(params,arc))continue;
+			
+			u32 dual = arc_dual(params,arc);
+			
+			network->cap[arc] += network->cap[dual];
+			network->cap[dual] = 0;
+		}
+	}
+}
+    
 /* Starting from a feasible flow (satisfies the balance and capacity
  * constraints), find a solution that minimizes the network->cost function. 
  * 
- * Cycle cancelling? */
-static void optimize_mcf(const struct pay_parameters *params,
+ * TODO(eduardo) The MCF must be called several times until we get a good
+ * compromise between fees and probabilities. Instead of re-computing the MCF at
+ * each step, we might use the previous flow result, which is not optimal in the
+ * current iteration but I might be not too far from the truth.
+ * It comes to mind to use cycle cancelling. */
+static int optimize_mcf(const struct pay_parameters *params,
 			 struct residual_network *network)
 {
-	// TODO
+	int ret = RENEPAY_ERR_OK;
+	
+	zero_flow(params,network);
+   	s64 amount = params->amount->millisatoshis/1000;
+	u32 source = gossmap_node_idx(params->gossmap,params->source),
+	    target = gossmap_node_idx(params->gossmap,params->target);
+	
+	tal_t *this_ctx = tal(tmpctx,tal_t);
+	
+	u32 *prev = tal_arr(this_ctx,u32,tal_count(network->potential));
+	s64 *distance = tal_arr(this_ctx,s64,tal_count(network->potential));
+	
+	while(amount>0)
+	{
+		int err = find_optimal_path(params,network,source,target,prev,distance);
+		if(err!=RENEPAY_ERR_OK)
+		{
+			// unexpected error
+			ret = RENEPAY_ERR_NOFEASIBLEFLOW;
+			break;
+		}
+		
+		// traverse the path and see how much flow we can send
+		s64 delta = get_augmenting_flow(params,network,source,target,prev);
+		
+		// commit that flow to the path
+		delta = MIN(amount,delta);
+		augment_flow(params,network,source,target,prev,delta);
+		
+		ASSERT(delta>0 && delta<=amount);
+		amount -= delta;
+		
+		// update potentials
+		for(u32 n=0;n<tal_count(network->potential);++n)
+		{
+			// see page 323 of Ahuja-Magnanti-Orlin
+			network->potential[n] -= MIN(distance[target],distance[n]);
+			
+			// Notice:
+			// if node i is permanently labeled we have
+			// 	d_i<=d_t 
+			// which implies
+			// 	MIN(d_i,d_t) = d_i
+			// if node i is temporarily labeled we have
+			// 	d_i>=d_t
+			// which implies
+			// 	MIN(d_i,d_t) = d_t
+		}
+	}
+	tal_free(this_ctx);
+	return ret;
 }
 	
 /* Given a flow in the residual network, compute the probability and fee cost

From cb6f459bb33f10194d801923bcc0b80bb1960411 Mon Sep 17 00:00:00 2001
From: Lagrang3 <eduardo.quintana@pm.me>
Date: Mon, 24 Apr 2023 16:08:53 +0100
Subject: [PATCH 09/64] renepay: search the optimal path with Dijkstra

---
 plugins/renepay/mcf.c | 25 ++++++++++++++++---------
 1 file changed, 16 insertions(+), 9 deletions(-)

diff --git a/plugins/renepay/mcf.c b/plugins/renepay/mcf.c
index 215a4eddbbae..6bb5a62a708e 100644
--- a/plugins/renepay/mcf.c
+++ b/plugins/renepay/mcf.c
@@ -2,6 +2,7 @@
 #include <ccan/lqueue/lqueue.h>
 #include <plugins/renepay/mcf.h>
 #include <plugins/renepay/flow.h>
+#include <plugins/renepay/heap.h>
 #include <math.h>
 
 #define PARTS_BITS 2
@@ -603,10 +604,14 @@ static int  find_optimal_path(
 	}
 	distance[source]=0;
 	
-	// TODO add source to queue
-	while(1)
+	struct heap *myheap = heap_new(this_ctx);	
+	heap_insert(myheap,source,0);
+	
+	while(!heap_empty(myheap))
 	{
-		// TODO: get best node cur from queue 
+		struct heap_data *top = heap_top(myheap);
+		u32 cur = top->idx;
+		heap_pop(myheap);
 		
 		if (visited[cur])
 			continue;
@@ -633,12 +638,14 @@ static int  find_optimal_path(
 			                             + network->potential[next];
 			// Dijkstra only works with non-negative weights
 			ASSERT(cij>=0);
-			if(distance[next]>distance[cur]+cij)
-			{
-				prev[next]=arc;
-				distance[next]=distance[cur]+cij;
-				// TODO: add next to queue
-			}
+			
+			if(distance[next]<=distance[cur]+cij)
+				continue;
+			
+			prev[next]=arc;
+			distance[next]=distance[cur]+cij;
+			
+			heap_insert(myheap,next,distance[next]);
 		}
 	}
 	tal_free(this_ctx);	

From c670d3b92d4e21d759ea773064fded071f1a5c5e Mon Sep 17 00:00:00 2001
From: Lagrang3 <eduardo.quintana@pm.me>
Date: Mon, 24 Apr 2023 16:14:18 +0100
Subject: [PATCH 10/64] renepay: infinity is INT64_MAX

---
 plugins/renepay/mcf.c | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/plugins/renepay/mcf.c b/plugins/renepay/mcf.c
index 6bb5a62a708e..f8b3babdff55 100644
--- a/plugins/renepay/mcf.c
+++ b/plugins/renepay/mcf.c
@@ -3,6 +3,7 @@
 #include <plugins/renepay/mcf.h>
 #include <plugins/renepay/flow.h>
 #include <plugins/renepay/heap.h>
+#include <stdint.h>
 #include <math.h>
 
 #define PARTS_BITS 2
@@ -16,7 +17,7 @@ static const double CHANNEL_PIVOTS[]={0,0.5,0.8,0.95};
 // channel plus 1 bit for the dual representation.
 static const size_t ARC_ADDITIONAL_BITS = PARTS_BITS + 2;
 
-static const s64 INFINITE = 9e18;
+static const s64 INFINITE = INT64_MAX;
 static const u32 INVALID_INDEX=0xffffffff;
 static const s64 COST_FACTOR=10;
 static const s64 MU_MAX = 128;

From 4ca1294f027514494df9d73c0f6077bc40580bb9 Mon Sep 17 00:00:00 2001
From: Lagrang3 <eduardo.quintana@pm.me>
Date: Mon, 24 Apr 2023 16:20:33 +0100
Subject: [PATCH 11/64] renepay: remove inline specifier and ASSERT

---
 plugins/renepay/mcf.c | 45 ++++++++++++++++++++++---------------------
 1 file changed, 23 insertions(+), 22 deletions(-)

diff --git a/plugins/renepay/mcf.c b/plugins/renepay/mcf.c
index f8b3babdff55..7372472a2583 100644
--- a/plugins/renepay/mcf.c
+++ b/plugins/renepay/mcf.c
@@ -5,6 +5,7 @@
 #include <plugins/renepay/heap.h>
 #include <stdint.h>
 #include <math.h>
+#include <assert.h>
 
 #define PARTS_BITS 2
 #define CHANNEL_PARTS (1 << PARTS_BITS)
@@ -135,13 +136,13 @@ struct residual_network {
 
 /* Helper function. 
  * Given an arc idx, return the dual's idx in the residual network. */
-static inline u32 arc_dual(const struct pay_parameters * params UNUSED,
+static u32 arc_dual(const struct pay_parameters * params UNUSED,
                            const u32 arc)
 {
 	return arc ^ 1;
 }
 /* Helper function. */
-static inline bool arc_is_dual(
+static bool arc_is_dual(
 		const struct pay_parameters *params UNUSED,
 		const u32 arc)
 {
@@ -150,25 +151,25 @@ static inline bool arc_is_dual(
 
 /* Helper function. 
  * Given an arc of the network (not residual) give me the flow. */
-static inline s64 get_arc_flow(
+static s64 get_arc_flow(
 			const struct pay_parameters * params,
 			const struct residual_network *network,
 			const u32 arc)
 {
-	ASSERT(arc & 1 == 0); // arc is not a dual arc
+	assert(!arc_is_dual(arc));
 	return network->cap[ arc_dual(params,arc) ];
 }
 
 /* Helper function. 
  * Given an arc idx, return the node from which this arc emanates in the residual network. */
-static inline u32 arc_tail(const struct pay_parameters *params,
+static u32 arc_tail(const struct pay_parameters *params,
                            const u32 arc)
 {
 	return params->arc_head_node[ arc_dual(params,arc) ];
 }
 /* Helper function. 
  * Given an arc idx, return the node that this arc is pointing to in the residual network. */
-static inline u32 arc_head(const struct pay_parameters *params,
+static u32 arc_head(const struct pay_parameters *params,
                            const u32 arc)
 {
 	return params->arc_head_node[arc];
@@ -177,7 +178,7 @@ static inline u32 arc_head(const struct pay_parameters *params,
 /* Helper function. 
  * Given node idx `node`, return the idx of the first arc whose tail is `node`.
  * */
-static inline u32 node_adjacency_begin(const struct pay_parameters *params,
+static u32 node_adjacency_begin(const struct pay_parameters *params,
                                        const u32 node)
 {
 	return params->node_adjacency_first[node];
@@ -185,7 +186,7 @@ static inline u32 node_adjacency_begin(const struct pay_parameters *params,
 
 /* Helper function. 
  * Given node idx `node`, return the idx of one past the last arc whose tail is `node`. */
-static inline u32 node_adjacency_end(const struct pay_parameters *params UNUSED,
+static u32 node_adjacency_end(const struct pay_parameters *params UNUSED,
                                      const u32 node UNUSED)
 {
 	return INVALID_INDEX;
@@ -193,7 +194,7 @@ static inline u32 node_adjacency_end(const struct pay_parameters *params UNUSED,
 
 /* Helper function. 
  * Given node idx `node` and `arc`, returns the idx of the next arc whose tail is `node`. */
-static inline u32 node_adjacency_next(const struct pay_parameters *params,
+static u32 node_adjacency_next(const struct pay_parameters *params,
                                      const u32 node UNUSED,
 				     const u32 arc)
 {
@@ -203,7 +204,7 @@ static inline u32 node_adjacency_next(const struct pay_parameters *params,
 /* Helper function.
  * Given an arc index, we should be able to deduce the channel id, and part that
  * corresponds to this arc. */
-static inline u32 arc_to_channel_idx(const u32 arc,
+static u32 arc_to_channel_idx(const u32 arc,
                               int *half_ptr,
 			      int *part_ptr,
 			      int *dual_ptr)
@@ -216,7 +217,7 @@ static inline u32 arc_to_channel_idx(const u32 arc,
 
 /* Helper function.
  * Given a channel index, we should be able to deduce the arc id. */
-static inline u32 channel_idx_to_arc(const u32 chan_idx,
+static u32 channel_idx_to_arc(const u32 chan_idx,
                               int half,
 			      int part,
 			      int dual)
@@ -229,7 +230,7 @@ static inline u32 channel_idx_to_arc(const u32 chan_idx,
 
 /* Helper function.
  * Evaluate Pickhardt's probability cost function. */
-static inline double pickhardt_probability_cost(
+static double pickhardt_probability_cost(
 	double lim_low, double lim_high, double flow)
 {
 	if(flow<=lim_low)
@@ -353,7 +354,7 @@ static void init_residual_network(struct pay_parameters *params,
 									   c,!half);
 			const u32 next_id = gossmap_node_idx(params->gossmap,next);
 			
-			ASSERT(node_id!=next_id);
+			assert(node_id!=next_id);
 			
 			// `cost` is the word normally used to denote cost per
 			// unit of flow in the context of MCF.
@@ -506,7 +507,7 @@ static s64 get_augmenting_flow(const struct pay_parameters *params,
 		cur = arc_head(params,dual);
 	}
 	
-	ASSERT(flow<INFINITE && flow>0);
+	assert(flow<INFINITE && flow>0);
 	return flow;
 }
 
@@ -526,7 +527,7 @@ static void augment_flow(const struct pay_parameters *params,
 		const u32 dual = arc_dual(params,arc);
 		
 		network->cap[arc] -= flow;
-		ASSERT(network->cap[arc] >=0 );
+		assert(network->cap[arc] >=0 );
 		network->cap[dual] += flow;
 		
 		// we are traversing in the opposite direction to the flow,
@@ -573,7 +574,7 @@ static int find_feasible_flow(const struct pay_parameters *params,
 		delta = MIN(amount,delta);
 		augment_flow(params,network,source,target,prev,delta);
 		
-		ASSERT(delta>0 && delta<=amount);
+		assert(delta>0 && delta<=amount);
 		amount -= delta;
 	}
 	
@@ -638,7 +639,7 @@ static int  find_optimal_path(
 			s64 cij = network->cost[arc] - network->potential[cur]
 			                             + network->potential[next];
 			// Dijkstra only works with non-negative weights
-			ASSERT(cij>=0);
+			assert(cij>=0);
 			
 			if(distance[next]<=distance[cur]+cij)
 				continue;
@@ -715,7 +716,7 @@ static int optimize_mcf(const struct pay_parameters *params,
 		delta = MIN(amount,delta);
 		augment_flow(params,network,source,target,prev,delta);
 		
-		ASSERT(delta>0 && delta<=amount);
+		assert(delta>0 && delta<=amount);
 		amount -= delta;
 		
 		// update potentials
@@ -1111,10 +1112,10 @@ int optimal_payment_flow(
 	}
 	
 	const size_t max_num_arcs = tal_count(params->arc_prob_cost);
-	ASSERT(max_num_arcs==tal_count(params->arc_prob_cost));
-	ASSERT(max_num_arcs==tal_count(params->arc_fee_cost));
-	ASSERT(max_num_arcs==tal_count(network->cap));
-	ASSERT(max_num_arcs==tal_count(network->cost));
+	assert(max_num_arcs==tal_count(params->arc_prob_cost));
+	assert(max_num_arcs==tal_count(params->arc_fee_cost));
+	assert(max_num_arcs==tal_count(network->cap));
+	assert(max_num_arcs==tal_count(network->cost));
 	
 	// binary search for a value of `mu` that fits our fee and prob.
 	// constraints.

From 24467701bda0d4e9500a615aa915b1236b314423 Mon Sep 17 00:00:00 2001
From: Lagrang3 <eduardo.quintana@pm.me>
Date: Mon, 24 Apr 2023 20:46:50 +0100
Subject: [PATCH 12/64] renepay: use tal_arrz to initialize to zero

---
 plugins/renepay/mcf.c | 23 +++++------------------
 1 file changed, 5 insertions(+), 18 deletions(-)

diff --git a/plugins/renepay/mcf.c b/plugins/renepay/mcf.c
index 7372472a2583..5bfc8f09d191 100644
--- a/plugins/renepay/mcf.c
+++ b/plugins/renepay/mcf.c
@@ -301,17 +301,13 @@ static void init_residual_network(struct pay_parameters *params,
 	const size_t max_num_arcs = max_num_chans << ARC_ADDITIONAL_BITS;
 	const size_t max_num_nodes = gossmap_max_node_idx(params->gossmap);
 	
-	network->cap = tal_arr(network,s64,max_num_arcs);
-	for(size_t i=0;i<tal_count(network->cap);++i)
-		network->cap[i]=0;
+	network->cap = tal_arrz(network,s64,max_num_arcs);
 	
 	network->cost = tal_arr(network,s64,max_num_arcs);
 	for(size_t i=0;i<tal_count(network->cost);++i)
 		network->cost[i]=INFINITE;
 	
-    network->potential = tal_arr(network,s64,max_num_nodes);
-    for(size_t i=0;i<tal_count(network->potential);++i)
-        network->potential[i]=0;
+	network->potential = tal_arrz(network,s64,max_num_nodes);
     
 	params->arc_head_node = tal_arr(params,u32,max_num_arcs);
 	for(size_t i=0;i<tal_count(params->arc_head_node);++i)
@@ -596,13 +592,12 @@ static int  find_optimal_path(
 	tal_t *this_ctx = tal(tmpctx,tal_t);
 	int ret = RENEPAY_ERR_NOFEASIBLEFLOW;
 	
-	char *visited = tal_array(this_ctx,char,tal_count(prev));
+	char *visited = tal_arrz(this_ctx,char,tal_count(prev));
 	
 	for(size_t i=0;i<tal_count(prev);++i)
 	{	
 		prev[i]=INVALID_INDEX;
 		distance[i]=INFINITE;
-		visited[i]=0;
 	}
 	distance[source]=0;
 	
@@ -817,22 +812,14 @@ static struct flow_path **
 	tal_t *this_ctx = tal(tmpctx,tal_t);
 	
 	const size_t max_num_chans = gossmap_max_chan_idx(params->gossmap);
-	struct chan_flow *chan_flow = tal_arr(this_ctx,struct chan_flow,max_num_chans);
-	
-	for(size_t i=0;i<max_num_chans;++i)
-		chan_flow[i].half[0]=chan_flow[i].half[1]=0;
+	struct chan_flow *chan_flow = tal_arrz(this_ctx,struct chan_flow,max_num_chans);
 	
 	const size_t max_num_nodes = gossmap_max_node_idx(params->gossmap);
-	s64 *balance = tal_arr(this_ctx,s64,max_num_nodes);
+	s64 *balance = tal_arrz(this_ctx,s64,max_num_nodes);
 	struct gossmap_chan **prev_chan = tal_arr(this_ctx,struct gossmap_chan*,max_num_nodes);
 	int *prev_dir = tal_arr(this_ctx,int,max_num_nodes);
 	u32 *prev_idx = tal_arr(this_ctx,u32,max_num_nodes);
 	
-	for(size_t i=0;i<max_num_nodes;++i)
-	{
-		balance[i]=0;
-	}
-	
 	// Convert the arc based residual network flow into a flow in the
 	// directed channel network.
 	// Compute balance on the nodes.

From 349641cc61762c888e6221dabe50ddb0793338e9 Mon Sep 17 00:00:00 2001
From: Lagrang3 <eduardo.quintana@pm.me>
Date: Mon, 24 Apr 2023 21:13:57 +0100
Subject: [PATCH 13/64] renepay: fix a couple of minor details

---
 plugins/renepay/mcf.c | 28 +++++++++++++++-------------
 1 file changed, 15 insertions(+), 13 deletions(-)

diff --git a/plugins/renepay/mcf.c b/plugins/renepay/mcf.c
index 5bfc8f09d191..377b881807e0 100644
--- a/plugins/renepay/mcf.c
+++ b/plugins/renepay/mcf.c
@@ -331,9 +331,9 @@ static void init_residual_network(struct pay_parameters *params,
 	
 	s64 capacity[CHANNEL_PARTS],prob_cost[CHANNEL_PARTS];
 	
-	struct gossmap_node *node = gossmap_first_node(params->gossmap);
-	const size_t num_nodes = gossmap_num_nodes(params->gossmap);
-	for(size_t i=0; i<num_nodes; ++i, node = gossmap_next_node(params->gossmap,node))
+	for(struct gossmap_node *node = gossmap_first_node(params->gossmap);
+	    node;
+	    node=gossmap_next_node(params->gossmap,node))
 	{
 		const u32 node_id = gossmap_node_idx(params->gossmap,node);
 		
@@ -350,7 +350,8 @@ static void init_residual_network(struct pay_parameters *params,
 									   c,!half);
 			const u32 next_id = gossmap_node_idx(params->gossmap,next);
 			
-			assert(node_id!=next_id);
+			if(node_id==next_id)
+				continue;
 			
 			// `cost` is the word normally used to denote cost per
 			// unit of flow in the context of MCF.
@@ -483,7 +484,7 @@ static int find_admissible_path(const struct pay_parameters *params,
 /* Get the max amount of flow one can send from source to target along the path
  * encoded in `prev`. */
 static s64 get_augmenting_flow(const struct pay_parameters *params,
-			       struct residual_network *network,
+			       const struct residual_network *network,
 	                       const u32 source,
 			       const u32 target,
 			       const u32 *prev)
@@ -747,9 +748,9 @@ static void estimate_costs(const struct pay_parameters *params,
 	double fee_microsats = 0;
 	double prob_cost = 0;
 	
-	struct gossmap_node *node = gossmap_first_node(params->gossmap);
-	const size_t num_nodes = gossmap_num_nodes(params->gossmap);
-	for(size_t i=0; i<num_nodes; ++i, node = gossmap_next_node(params->gossmap,node))
+	for(struct gossmap_node *node = gossmap_first_node(params->gossmap);
+	    node;
+	    node=gossmap_next_node(params->gossmap,node))
 	{
 		for(size_t j=0;j<node->num_chans;++j)
 		{
@@ -798,16 +799,17 @@ struct chan_flow
 
 struct list_data
 {
-	struct flow_path *flow_path;
 	struct list_node list;
+	struct flow_path *flow_path;
 };
 
 /* Given a flow in the residual network, build a set of payment flows in the
  * gossmap that corresponds to this flow. */		
 static struct flow_path **
-	get_flow_paths(const struct pay_parameters *params,
-				    const struct residual_network *network,
-				    const tal_t *ctx)
+	get_flow_paths(
+		const tal_t *ctx,
+		const struct pay_parameters *params,
+		const struct residual_network *network)
 {
 	tal_t *this_ctx = tal(tmpctx,tal_t);
 	
@@ -1153,7 +1155,7 @@ int optimal_payment_flow(
 
 	if(ret==RENEPAY_ERR_OK)
 	{
-		*flow_paths = get_flow_paths(params,network,ctx);
+		*flow_paths = get_flow_paths(ctx,params,network);
 	}else
 	{
 	// TODO(eduardo) fallback to c_prob or c_fee?

From 686a90c76299915e2874869d7f9df31d54ff9176 Mon Sep 17 00:00:00 2001
From: Lagrang3 <eduardo.quintana@pm.me>
Date: Mon, 24 Apr 2023 22:22:19 +0100
Subject: [PATCH 14/64] renepay: encode arcs with bitfields

---
 plugins/renepay/mcf.c | 179 ++++++++++++++++++++----------------------
 1 file changed, 86 insertions(+), 93 deletions(-)

diff --git a/plugins/renepay/mcf.c b/plugins/renepay/mcf.c
index 377b881807e0..a86c7e286abc 100644
--- a/plugins/renepay/mcf.c
+++ b/plugins/renepay/mcf.c
@@ -89,6 +89,17 @@ struct queue_data
  * 
  * */
 
+typedef union
+{
+	struct{
+		u32 dual: 1;
+		u32 part: PARTS_BITS;
+		u32 chandir: 1;
+		u32 chanidx: (32-1-PARTS_BITS-1);
+	};
+	u32 idx;
+} arc_t;
+
 #define MAX(x, y) (((x) > (y)) ? (x) : (y))
 #define MIN(x, y) (((x) < (y)) ? (x) : (y))
 
@@ -118,8 +129,8 @@ struct pay_parameters {
 	// probability and fee cost associated to an arc
 	s64 *arc_prob_cost, *arc_fee_cost;
 	
-	u32 *arc_adjacency_next_arc;
-	u32 *node_adjacency_first_arc;	
+	arc_t *arc_adjacency_next_arc;
+	arc_t *node_adjacency_first_arc;	
 	
 	// channel linearization parameters
 	double cap_fraction[CHANNEL_PARTS], 
@@ -130,56 +141,53 @@ struct residual_network {
 	/* residual capacity on arcs */
 	s64 *cap; 
 	s64 *cost;
-    s64 *potential;
+	s64 *potential;
 };
 				
 
 /* Helper function. 
  * Given an arc idx, return the dual's idx in the residual network. */
-static u32 arc_dual(const struct pay_parameters * params UNUSED,
-                           const u32 arc)
+static arc_t arc_dual(const arc_t arc)
 {
-	return arc ^ 1;
+	arc.dual ^= 1;
+	return arc;
 }
 /* Helper function. */
-static bool arc_is_dual(
-		const struct pay_parameters *params UNUSED,
-		const u32 arc)
+static bool arc_is_dual(const arc_t arc)
 {
-	return (arc & 1) == 1;
+	return arc.dual == 1;
 }
 
 /* Helper function. 
  * Given an arc of the network (not residual) give me the flow. */
 static s64 get_arc_flow(
-			const struct pay_parameters * params,
 			const struct residual_network *network,
-			const u32 arc)
+			const arc_t arc)
 {
 	assert(!arc_is_dual(arc));
-	return network->cap[ arc_dual(params,arc) ];
+	return network->cap[ arc_dual(arc).idx ];
 }
 
 /* Helper function. 
  * Given an arc idx, return the node from which this arc emanates in the residual network. */
 static u32 arc_tail(const struct pay_parameters *params,
-                           const u32 arc)
+                    const arc_t arc)
 {
-	return params->arc_head_node[ arc_dual(params,arc) ];
+	return params->arc_head_node[ arc_dual(arc).idx ];
 }
 /* Helper function. 
  * Given an arc idx, return the node that this arc is pointing to in the residual network. */
 static u32 arc_head(const struct pay_parameters *params,
-                           const u32 arc)
+                    const arc_t arc)
 {
-	return params->arc_head_node[arc];
+	return params->arc_head_node[arc.idx];
 }
 		
 /* Helper function. 
  * Given node idx `node`, return the idx of the first arc whose tail is `node`.
  * */
 static u32 node_adjacency_begin(const struct pay_parameters *params,
-                                       const u32 node)
+                                const u32 node)
 {
 	return params->node_adjacency_first[node];
 }
@@ -187,7 +195,7 @@ static u32 node_adjacency_begin(const struct pay_parameters *params,
 /* Helper function. 
  * Given node idx `node`, return the idx of one past the last arc whose tail is `node`. */
 static u32 node_adjacency_end(const struct pay_parameters *params UNUSED,
-                                     const u32 node UNUSED)
+                              const u32 node UNUSED)
 {
 	return INVALID_INDEX;
 }
@@ -195,37 +203,26 @@ static u32 node_adjacency_end(const struct pay_parameters *params UNUSED,
 /* Helper function. 
  * Given node idx `node` and `arc`, returns the idx of the next arc whose tail is `node`. */
 static u32 node_adjacency_next(const struct pay_parameters *params,
-                                     const u32 node UNUSED,
-				     const u32 arc)
-{
-	return params->adjacency_next[arc];
-}
-
-/* Helper function.
- * Given an arc index, we should be able to deduce the channel id, and part that
- * corresponds to this arc. */
-static u32 arc_to_channel_idx(const u32 arc,
-                              int *half_ptr,
-			      int *part_ptr,
-			      int *dual_ptr)
+                               const u32 node UNUSED,
+			       const arc_t arc)
 {
-	&half_ptr = (arc >> (1+PARTS_BITS)) & 1;
-	&part_ptr = (arc >> 1) & (CHANNEL_PARTS-1);
-	&dual_ptr = arc & 1;
-	return arc>>ARC_ADDITIONAL_BITS;
+	return params->adjacency_next[arc.idx];
 }
 
 /* Helper function.
  * Given a channel index, we should be able to deduce the arc id. */
-static u32 channel_idx_to_arc(const u32 chan_idx,
-                              int half,
-			      int part,
-			      int dual)
+static arc_t channel_idx_to_arc(
+		const u32 chan_idx,
+                int half,
+		int part,
+		int dual)
 {
-	half &= 1;
-	dual &= 1;
-	part &= (CHANNEL_PARTS-1);
-	return (chan_idx << ARC_ADDITIONAL_BITS)|(half<<(1+PARTS_BITS))|(part<<1)|(dual);
+	arc_t arc;
+	arc.dual=dual;
+	arc.part=part;
+	arc.chandir=half;
+	arc.chanidx = chan_idx;
+	return arc;
 }
 
 /* Helper function.
@@ -321,13 +318,13 @@ static void init_residual_network(struct pay_parameters *params,
 	for(size_t i=0;i<tal_count(params->arc_fee_cost);++i)
 		params->arc_fee_cost[i]=INFINITE;
 		
-	params->arc_adjacency_next_arc = tal_arr(params,u32,max_num_arcs);
+	params->arc_adjacency_next_arc = tal_arr(params,arc_t,max_num_arcs);
 	for(size_t i=0;i<tal_count(params->arc_adjacency_next_arc);++i)
-		params->arc_adjacency_next_arc[i]=INVALID_INDEX;
+		params->arc_adjacency_next_arc[i].idx=INVALID_INDEX;
 	
-	params->node_adjacency_first_arc = tal_arr(params,u32,max_num_nodes);
+	params->node_adjacency_first_arc = tal_arr(params,arc_t,max_num_nodes);
 	for(size_t i=0;i<tal_count(params->node_adjacency_first_arc);++i)
-		params->node_adjacency_first_arc[i]=INVALID_INDEX;
+		params->node_adjacency_first_arc[i].idx=INVALID_INDEX;
 	
 	s64 capacity[CHANNEL_PARTS],prob_cost[CHANNEL_PARTS];
 	
@@ -337,7 +334,7 @@ static void init_residual_network(struct pay_parameters *params,
 	{
 		const u32 node_id = gossmap_node_idx(params->gossmap,node);
 		
-		u32 prev_arc = INVALID_INDEX;
+		arc_t prev_arc = (arc_t){.idx = INVALID_INDEX};
 		
 		for(size_t j=0;j<node->num_chans;++j)
 		{
@@ -366,11 +363,11 @@ static void init_residual_network(struct pay_parameters *params,
 			// when the `i` hits the `next` node.
 			for(size_t k=0;k<CHANNEL_PARTS;++k)
 			{
-				u32 arc = channel_idx_to_arc(chan_id,half,k,0);
-				arc_head_node[arc] = next_id;
+				arc_t arc = channel_idx_to_arc(chan_id,half,k,0);
+				arc_head_node[arc.idx] = next_id;
 				
 				// Is this is the first arc?
-				if(prev_arc==INVALID_INDEX)
+				if(prev_arc.idx==INVALID_INDEX)
 				// yes, then set it at the head of the linked list
 				{
 					node_adjacency_first_arc[node_id]=arc;
@@ -382,9 +379,9 @@ static void init_residual_network(struct pay_parameters *params,
 				
 				prev_arc = arc;
 				
-				network->cap[arc] = capacity[k];
-				params->arc_prob_cost[arc] = prob_cost[k];
-				params->arc_fee_cost[arc] = compute_fee_cost(params,c,half);
+				network->cap[arc.idx] = capacity[k];
+				params->arc_prob_cost[arc.idx] = prob_cost[k];
+				params->arc_fee_cost[arc.idx] = compute_fee_cost(params,c,half);
 			}
 			
 			// split the opposite direction to obtain the dual arcs
@@ -395,8 +392,8 @@ static void init_residual_network(struct pay_parameters *params,
 			// (c,!half) in dual representation
 			for(size_t k=0;k<CHANNEL_PARTS;++k)
 			{
-				u32 arc = channel_idx_to_arc(chan_id,!half,k,1);
-				arc_head_node[arc] = next_id;
+				arc_t arc = channel_idx_to_arc(chan_id,!half,k,1);
+				arc_head_node[arc.idx] = next_id;
 				
 				// Is this is the first arc?
 				if(prev_arc==INVALID_INDEX)
@@ -406,14 +403,14 @@ static void init_residual_network(struct pay_parameters *params,
 				} else
 				// no, then link it to the previous arc
 				{
-					arc_adjacency_next_arc[prev_arc]=arc;
+					arc_adjacency_next_arc[prev_arc.idx]=arc;
 				}
 				
 				prev_arc = arc;
 				
-				network->cap[arc] = 0;
-				params->arc_prob_cost[arc] = -prob_cost[k];
-				params->arc_fee_cost[arc] = -compute_fee_cost(params,c,!half);
+				network->cap[arc.idx] = 0;
+				params->arc_prob_cost[arc.idx] = -prob_cost[k];
+				params->arc_fee_cost[arc.idx] = -compute_fee_cost(params,c,!half);
 			}
 		}
 	}
@@ -428,12 +425,12 @@ static int find_admissible_path(const struct pay_parameters *params,
 			        const struct residual_network *network,
                                 const u32 source,
 				const u32 target,
-				u32 *prev)
+				arc_t *prev)
 {
 	int ret = RENEPAY_ERR_NOFEASIBLEFLOW;
 	
 	for(size_t i=0;i<tal_count(prev);++i)
-		prev[i]=INVALID_INDEX;
+		prev[i].idx=INVALID_INDEX;
 	
 	// The graph is dense, and the farthest node is just a few hops away,
 	// hence let's BFS search.
@@ -456,12 +453,12 @@ static int find_admissible_path(const struct pay_parameters *params,
 			break;
 		}
 		
-		for(u32 arc = node_adjacency_begin(params,cur);
+		for(arc_t arc = node_adjacency_begin(params,cur);
 		        arc!= node_adjacency_end(params,cur);
 			arc = node_adjacency_next(params,cur,arc))
 		{
 			// check if this arc is traversable
-			if(network->cap[arc] <= 0)
+			if(network->cap[arc.idx] <= 0)
 				continue;
 			
 			u32 next = arc_head(params,arc);
@@ -487,17 +484,17 @@ static s64 get_augmenting_flow(const struct pay_parameters *params,
 			       const struct residual_network *network,
 	                       const u32 source,
 			       const u32 target,
-			       const u32 *prev)
+			       const arc_t *prev)
 {
 	s64 flow = INFINITE;
 	
 	u32 cur = target;
 	while(cur!=source)
 	{
-		const u32 arc = prev[cur];
-		const u32 dual = arc_dual(params,arc);
+		const arc_t arc = prev[cur];
+		const arc_t dual = arc_dual(params,arc);
 		
-		flow = MIN(flow , network->cap[arc]);
+		flow = MIN(flow , network->cap[arc.idx]);
 		
 		// we are traversing in the opposite direction to the flow,
 		// hence the next node is at the head of the `dual` arc.
@@ -513,19 +510,19 @@ static void augment_flow(const struct pay_parameters *params,
 			 struct residual_network *network,
 	                 const u32 source,
 			 const u32 target,
-			 const u32 *prev,
+			 const arc_t *prev,
 			 s64 flow)
 {
 	u32 cur = target;
 	
 	while(cur!=source)
 	{
-		const u32 arc = prev[cur];
-		const u32 dual = arc_dual(params,arc);
+		const arc_t arc = prev[cur];
+		const arc_t dual = arc_dual(params,arc);
 		
-		network->cap[arc] -= flow;
-		assert(network->cap[arc] >=0 );
-		network->cap[dual] += flow;
+		network->cap[arc.idx] -= flow;
+		assert(network->cap[arc.idx] >=0 );
+		network->cap[dual.idx] += flow;
 		
 		// we are traversing in the opposite direction to the flow,
 		// hence the next node is at the head of the `dual` arc.
@@ -551,7 +548,7 @@ static int find_feasible_flow(const struct pay_parameters *params,
 	
 	/* path information 
 	 * prev: is the id of the arc that lead to the node. */
-	u32 *prev = tal_arr(tmpctx,u32
+	arc_t *prev = tal_arr(tmpctx,arc_t
 		            gossmap_max_node_idx(params->gossmap));
 	
 	while(amount>0)
@@ -587,7 +584,7 @@ static int  find_optimal_path(
 	const struct residual_network* network,
 	const u32 source,
 	const u32 target,
-	s32 *prev,
+	arc_t *prev,
 	s64 *distance)
 {
 	tal_t *this_ctx = tal(tmpctx,tal_t);
@@ -597,7 +594,7 @@ static int  find_optimal_path(
 	
 	for(size_t i=0;i<tal_count(prev);++i)
 	{	
-		prev[i]=INVALID_INDEX;
+		prev[i].idx=INVALID_INDEX;
 		distance[i]=INFINITE;
 	}
 	distance[source]=0;
@@ -622,18 +619,18 @@ static int  find_optimal_path(
 			break;
 		}
 		
-		for(u32 arc = node_adjacency_begin(params,cur);
+		for(arc_t arc = node_adjacency_begin(params,cur);
 		        arc!= node_adjacency_end(params,cur);
 			arc = node_adjacency_next(params,cur,arc))
 		{
 			// check if this arc is traversable
-			if(network->cap[arc] <= 0)
+			if(network->cap[arc.idx] <= 0)
 				continue;
 			
 			u32 next = arc_head(params,arc);
 			
-			s64 cij = network->cost[arc] - network->potential[cur]
-			                             + network->potential[next];
+			s64 cij = network->cost[arc.idx] - network->potential[cur]
+			                                 + network->potential[next];
 			// Dijkstra only works with non-negative weights
 			assert(cij>=0);
 			
@@ -658,16 +655,16 @@ static void zero_flow(
 	for(u32 node=0;node<tal_count(network->potential);++node)
 	{
 		network->potential[node]=0;
-		for(u32 arc=node_adjacency_begin(params,node);
+		for(arc_t arc=node_adjacency_begin(params,node);
 			arc!=node_adjacency_end(params,node);
 			arc = node_adjacency_next(params,node,arc))
 		{
 			if(arc_is_dual(params,arc))continue;
 			
-			u32 dual = arc_dual(params,arc);
+			arc_t dual = arc_dual(params,arc);
 			
-			network->cap[arc] += network->cap[dual];
-			network->cap[dual] = 0;
+			network->cap[arc.idx] += network->cap[dual.idx];
+			network->cap[dual.idx] = 0;
 		}
 	}
 }
@@ -692,7 +689,7 @@ static int optimize_mcf(const struct pay_parameters *params,
 	
 	tal_t *this_ctx = tal(tmpctx,tal_t);
 	
-	u32 *prev = tal_arr(this_ctx,u32,tal_count(network->potential));
+	arc_t *prev = tal_arr(this_ctx,arc_t,tal_count(network->potential));
 	s64 *distance = tal_arr(this_ctx,s64,tal_count(network->potential));
 	
 	while(amount>0)
@@ -764,7 +761,7 @@ static void estimate_costs(const struct pay_parameters *params,
 			
 			for(size_t k=0;k<CHANNEL_PARTS;++k)
 			{
-				const u32 arc = channel_idx_to_arc(chan_idx,dir,k,0);
+				const arc_t arc = channel_idx_to_arc(chan_idx,dir,k,0);
 				chan_flow += get_arc_flow(params,network,arc);	
 			}
 			
@@ -839,11 +836,7 @@ static struct flow_path **
 			balance[n] -= flow;
 			balance[m] += flow;
 			
-			u32 chan_idx;
-			int dir,part,dual;
-			chan_idx = arc_to_channel_idx(arc,dir,part,dual);
-			
-			chan_flow[chan_idx].half[dir] +=flow;
+			chan_flow[arc.chanidx].half[arc.chandir] +=flow;
 		}
 			
 	}

From 71b86333a79ebf1552ce562c847e95332220156d Mon Sep 17 00:00:00 2001
From: Lagrang3 <eduardo.quintana@pm.me>
Date: Thu, 27 Apr 2023 15:25:38 +0100
Subject: [PATCH 15/64] renepay: fix some compiler errors

---
 plugins/renepay/mcf.c | 89 +++++++++++++++++++++----------------------
 plugins/renepay/mcf.h | 14 +++----
 2 files changed, 50 insertions(+), 53 deletions(-)

diff --git a/plugins/renepay/mcf.c b/plugins/renepay/mcf.c
index a86c7e286abc..06173d80bf75 100644
--- a/plugins/renepay/mcf.c
+++ b/plugins/renepay/mcf.c
@@ -1,8 +1,8 @@
 #include "config.h"
 #include <ccan/lqueue/lqueue.h>
+#include <ccan/tal/tal.h>
 #include <plugins/renepay/mcf.h>
 #include <plugins/renepay/flow.h>
-#include <plugins/renepay/heap.h>
 #include <stdint.h>
 #include <math.h>
 #include <assert.h>
@@ -129,7 +129,7 @@ struct pay_parameters {
 	// probability and fee cost associated to an arc
 	s64 *arc_prob_cost, *arc_fee_cost;
 	
-	arc_t *arc_adjacency_next_arc;
+	arc_t *node_adjacency_next_arc;
 	arc_t *node_adjacency_first_arc;	
 	
 	// channel linearization parameters
@@ -147,7 +147,7 @@ struct residual_network {
 
 /* Helper function. 
  * Given an arc idx, return the dual's idx in the residual network. */
-static arc_t arc_dual(const arc_t arc)
+static arc_t arc_dual(arc_t arc)
 {
 	arc.dual ^= 1;
 	return arc;
@@ -186,27 +186,29 @@ static u32 arc_head(const struct pay_parameters *params,
 /* Helper function. 
  * Given node idx `node`, return the idx of the first arc whose tail is `node`.
  * */
-static u32 node_adjacency_begin(const struct pay_parameters *params,
+static arc_t node_adjacency_begin(const struct pay_parameters *params,
                                 const u32 node)
 {
-	return params->node_adjacency_first[node];
+	return params->node_adjacency_first_arc[node];
 }
 
 /* Helper function. 
- * Given node idx `node`, return the idx of one past the last arc whose tail is `node`. */
-static u32 node_adjacency_end(const struct pay_parameters *params UNUSED,
-                              const u32 node UNUSED)
+ * Is this the end of the adjacency list. */
+static bool node_adjacency_end(
+	const struct pay_parameters *params UNUSED,
+	const u32 node UNUSED,
+	const arc_t arc)
 {
-	return INVALID_INDEX;
+	return arc.idx == INVALID_INDEX;
 }
 
 /* Helper function. 
  * Given node idx `node` and `arc`, returns the idx of the next arc whose tail is `node`. */
-static u32 node_adjacency_next(const struct pay_parameters *params,
+static arc_t node_adjacency_next(const struct pay_parameters *params,
                                const u32 node UNUSED,
 			       const arc_t arc)
 {
-	return params->adjacency_next[arc.idx];
+	return params->node_adjacency_next_arc[arc.idx];
 }
 
 /* Helper function.
@@ -232,7 +234,7 @@ static double pickhardt_probability_cost(
 {
 	if(flow<=lim_low)
 		return 0.;
-	ASSSERT(flow<lim_high);
+	assert(flow<lim_high);
 	return -log(1.-(flow-lim_low)/(lim_high-lim_low));
 }
 
@@ -318,16 +320,14 @@ static void init_residual_network(struct pay_parameters *params,
 	for(size_t i=0;i<tal_count(params->arc_fee_cost);++i)
 		params->arc_fee_cost[i]=INFINITE;
 		
-	params->arc_adjacency_next_arc = tal_arr(params,arc_t,max_num_arcs);
-	for(size_t i=0;i<tal_count(params->arc_adjacency_next_arc);++i)
-		params->arc_adjacency_next_arc[i].idx=INVALID_INDEX;
+	params->node_adjacency_next_arc = tal_arr(params,arc_t,max_num_arcs);
+	for(size_t i=0;i<tal_count(params->node_adjacency_next_arc);++i)
+		params->node_adjacency_next_arc[i].idx=INVALID_INDEX;
 	
 	params->node_adjacency_first_arc = tal_arr(params,arc_t,max_num_nodes);
 	for(size_t i=0;i<tal_count(params->node_adjacency_first_arc);++i)
 		params->node_adjacency_first_arc[i].idx=INVALID_INDEX;
 	
-	s64 capacity[CHANNEL_PARTS],prob_cost[CHANNEL_PARTS];
-	
 	for(struct gossmap_node *node = gossmap_first_node(params->gossmap);
 	    node;
 	    node=gossmap_next_node(params->gossmap,node))
@@ -364,17 +364,17 @@ static void init_residual_network(struct pay_parameters *params,
 			for(size_t k=0;k<CHANNEL_PARTS;++k)
 			{
 				arc_t arc = channel_idx_to_arc(chan_id,half,k,0);
-				arc_head_node[arc.idx] = next_id;
+				params->arc_head_node[arc.idx] = next_id;
 				
 				// Is this is the first arc?
 				if(prev_arc.idx==INVALID_INDEX)
 				// yes, then set it at the head of the linked list
 				{
-					node_adjacency_first_arc[node_id]=arc;
+					params->node_adjacency_first_arc[node_id]=arc;
 				} else
 				// no, then link it to the previous arc
 				{
-					arc_adjacency_next_arc[prev_arc]=arc;
+					params->node_adjacency_next_arc[prev_arc.idx]=arc;
 				}
 				
 				prev_arc = arc;
@@ -393,17 +393,17 @@ static void init_residual_network(struct pay_parameters *params,
 			for(size_t k=0;k<CHANNEL_PARTS;++k)
 			{
 				arc_t arc = channel_idx_to_arc(chan_id,!half,k,1);
-				arc_head_node[arc.idx] = next_id;
+				params->arc_head_node[arc.idx] = next_id;
 				
 				// Is this is the first arc?
-				if(prev_arc==INVALID_INDEX)
+				if(prev_arc.idx==INVALID_INDEX)
 				// yes, then set it at the head of the linked list
 				{
-					node_adjacency_first_arc[node_id]=arc;
+					params->node_adjacency_first_arc[node_id]=arc;
 				} else
 				// no, then link it to the previous arc
 				{
-					arc_adjacency_next_arc[prev_arc.idx]=arc;
+					params->node_adjacency_next_arc[prev_arc.idx]=arc;
 				}
 				
 				prev_arc = arc;
@@ -454,7 +454,7 @@ static int find_admissible_path(const struct pay_parameters *params,
 		}
 		
 		for(arc_t arc = node_adjacency_begin(params,cur);
-		        arc!= node_adjacency_end(params,cur);
+		        !node_adjacency_end(params,cur,arc);
 			arc = node_adjacency_next(params,cur,arc))
 		{
 			// check if this arc is traversable
@@ -464,7 +464,7 @@ static int find_admissible_path(const struct pay_parameters *params,
 			u32 next = arc_head(params,arc);
 			
 			// if that node has been seen previously
-			if(prev[next]!=INVALID_INDEX)
+			if(prev[next].idx!=INVALID_INDEX)
 				continue;
 			
 			prev[next] = arc;
@@ -492,7 +492,7 @@ static s64 get_augmenting_flow(const struct pay_parameters *params,
 	while(cur!=source)
 	{
 		const arc_t arc = prev[cur];
-		const arc_t dual = arc_dual(params,arc);
+		const arc_t dual = arc_dual(arc);
 		
 		flow = MIN(flow , network->cap[arc.idx]);
 		
@@ -518,7 +518,7 @@ static void augment_flow(const struct pay_parameters *params,
 	while(cur!=source)
 	{
 		const arc_t arc = prev[cur];
-		const arc_t dual = arc_dual(params,arc);
+		const arc_t dual = arc_dual(arc);
 		
 		network->cap[arc.idx] -= flow;
 		assert(network->cap[arc.idx] >=0 );
@@ -548,7 +548,7 @@ static int find_feasible_flow(const struct pay_parameters *params,
 	
 	/* path information 
 	 * prev: is the id of the arc that lead to the node. */
-	arc_t *prev = tal_arr(tmpctx,arc_t
+	arc_t *prev = tal_arr(tmpctx,arc_t,
 		            gossmap_max_node_idx(params->gossmap));
 	
 	while(amount>0)
@@ -620,7 +620,7 @@ static int  find_optimal_path(
 		}
 		
 		for(arc_t arc = node_adjacency_begin(params,cur);
-		        arc!= node_adjacency_end(params,cur);
+		        !node_adjacency_end(params,cur,arc);
 			arc = node_adjacency_next(params,cur,arc))
 		{
 			// check if this arc is traversable
@@ -656,12 +656,12 @@ static void zero_flow(
 	{
 		network->potential[node]=0;
 		for(arc_t arc=node_adjacency_begin(params,node);
-			arc!=node_adjacency_end(params,node);
-			arc = node_adjacency_next(params,node,arc))
+			  !node_adjacency_end(params,node,arc);
+			  arc = node_adjacency_next(params,node,arc))
 		{
-			if(arc_is_dual(params,arc))continue;
+			if(arc_is_dual(arc))continue;
 			
-			arc_t dual = arc_dual(params,arc);
+			arc_t dual = arc_dual(arc);
 			
 			network->cap[arc.idx] += network->cap[dual.idx];
 			network->cap[dual.idx] = 0;
@@ -683,7 +683,7 @@ static int optimize_mcf(const struct pay_parameters *params,
 	int ret = RENEPAY_ERR_OK;
 	
 	zero_flow(params,network);
-   	s64 amount = params->amount->millisatoshis/1000;
+   	s64 amount = params->amount.millisatoshis/1000;
 	u32 source = gossmap_node_idx(params->gossmap,params->source),
 	    target = gossmap_node_idx(params->gossmap,params->target);
 	
@@ -762,7 +762,7 @@ static void estimate_costs(const struct pay_parameters *params,
 			for(size_t k=0;k<CHANNEL_PARTS;++k)
 			{
 				const arc_t arc = channel_idx_to_arc(chan_idx,dir,k,0);
-				chan_flow += get_arc_flow(params,network,arc);	
+				chan_flow += get_arc_flow(network,arc);	
 			}
 			
 			// TODO(eduardo): this is wrong, because the base fee is
@@ -825,13 +825,13 @@ static struct flow_path **
 	for(u32 n = 0;n<max_num_nodes;++n)
 	{
 		for(u32 arc = node_adjacency_begin(params,cur);
-		        arc!= node_adjacency_end(params,cur);
+		        !node_adjacency_end(params,cur,arc);
 			arc = node_adjacency_next(params,cur,arc))
 		{
 			if(arc_is_dual(arc))
 				continue;
 			u32 m = arc_head(params,arc);
-			s64 flow = get_arc_flow(params,network,arc);
+			s64 flow = get_arc_flow(network,arc);
 			
 			balance[n] -= flow;
 			balance[m] += flow;
@@ -1036,7 +1036,7 @@ static struct flow_path **
  * described with a sequence of directed channels and one amount. 
  * In the `pay_flow` module there are dedicated routes to compute the actual
  * amount to be forward on each hop. */
-int optimal_payment_flow(
+struct flow_path* minflow(
                       const tal_t *ctx,
 		      struct gossmap *gossmap,
 		      const struct gossmap_node *source,
@@ -1044,10 +1044,9 @@ int optimal_payment_flow(
 		      struct chan_extra_map *chan_extra_map,
 		      const bitmap *disabled,
 		      struct amount_msat amount,
-		      double max_fee_ppm,
+		      struct amount_msat max_fee,
 		      double min_probability,
-		      double delay_feefactor,
-		      struct flow_paths **flow_paths)
+		      double delay_feefactor UNUSED)
 {
 	// TODO(eduardo) on which ctx should I allocate the MCF data?
 	struct pay_parameters *params = tal(tmpctx,struct pay_parameters);	
@@ -1144,11 +1143,11 @@ int optimal_payment_flow(
 	
 	end:
 	
-	*flow_paths = NULL;
+	struct flow_path *flow_paths = NULL;
 
 	if(ret==RENEPAY_ERR_OK)
 	{
-		*flow_paths = get_flow_paths(ctx,params,network);
+		flow_paths = get_flow_paths(ctx,params,network);
 	}else
 	{
 	// TODO(eduardo) fallback to c_prob or c_fee?
@@ -1157,6 +1156,6 @@ int optimal_payment_flow(
 	tal_free(network);
 	tal_free(params);
 	
-	return ret;
+	return flow_paths;
 }
 
diff --git a/plugins/renepay/mcf.h b/plugins/renepay/mcf.h
index eed76ddb9789..930cbf225b7e 100644
--- a/plugins/renepay/mcf.h
+++ b/plugins/renepay/mcf.h
@@ -36,19 +36,17 @@ struct flow_path
  * @chan_extra_map: hashtable of extra per-channel information
  * @disabled: NULL, or a bitmap by channel index of channels not to use.
  * @amount: the amount we want to reach @target
- * @max_fee_ppm: the maximum ppm allowed in fees
+ * @max_fee: the maximum allowed in fees
+ * @min_probability: minimum probability accepted
  * @delay_feefactor: how important delays are compared to fees
- * @errorcode: error code in case of failure
  *
  * @delay_feefactor converts 1 block delay into msat, as if it were an additional
  * fee.  So if a CTLV delay on a node is 5 blocks, that's treated as if it
  * were a fee of 5 * @delay_feefactor.
  *
- * @mu converts fees to add it to the uncertainty term.
- *
  * Return a series of subflows which deliver amount to target, or NULL.
  */
-struct flow **optimal_payment_flow(
+struct flow_path* minflow(
                       const tal_t *ctx,
 		      struct gossmap *gossmap,
 		      const struct gossmap_node *source,
@@ -56,7 +54,7 @@ struct flow **optimal_payment_flow(
 		      struct chan_extra_map *chan_extra_map,
 		      const bitmap *disabled,
 		      struct amount_msat amount,
-		      double max_fee_ppm,
-		      double delay_feefactor,
-		      int *errorcode);
+		      struct amount_msat max_fee,
+		      double min_probability,
+		      double delay_feefactor);
 #endif /* LIGHTNING_PLUGINS_RENEPAY_MCF_H */

From 89099e5f6b872a1fb24ae9861f610ff65aff712a Mon Sep 17 00:00:00 2001
From: Lagrang3 <eduardo.quintana@pm.me>
Date: Sat, 29 Apr 2023 11:06:51 +0100
Subject: [PATCH 16/64] renepay: add a heap without global variables

---
 plugins/renepay/heap.h | 195 +++++++++++++++++++++++++++++++++++++++++
 plugins/renepay/mcf.c  |  15 ++--
 2 files changed, 204 insertions(+), 6 deletions(-)
 create mode 100644 plugins/renepay/heap.h

diff --git a/plugins/renepay/heap.h b/plugins/renepay/heap.h
new file mode 100644
index 000000000000..49d81fda49d0
--- /dev/null
+++ b/plugins/renepay/heap.h
@@ -0,0 +1,195 @@
+#ifndef LIGHTNING_PLUGINS_RENEPAY_HEAP_H
+#define LIGHTNING_PLUGINS_RENEPAY_HEAP_H
+
+#include <gheap.h>
+
+
+/* A functionality missing in gheap that can be used to update elements. 
+ * Input: item
+ * Output: the position of the smallest element p, such is greater equal item.
+ * Formally:
+ * 	Let X={x in heap: !(x<item) }, all the elements that are greater or 
+ * 	equal item,
+ * 	then p in X, and for every x in X: !(x<p), p is the smallest.*/
+size_t gheap_upper_bound(const struct gheap_ctx *ctx,
+	const void *base, size_t heap_size, void *item);
+
+struct heap_data
+{
+	u32 idx;
+	s64 distance;
+};
+
+struct heap
+{
+	size_t size;
+	size_t max_size;
+	struct heap_data *data;
+	struct gheap_ctx gheap_ctx;
+};
+struct heap* heap_new(const tal_t *ctx, const size_t max_capacity);
+void heap_insert(struct heap* heap, u32 idx, s64 distance);
+void heap_update(struct heap* heap, u32 idx, s64 old_distance,s64 new_distance);
+bool heap_empty(const struct heap* heap);
+void heap_pop(struct heap* heap);
+struct heap_data * heap_top(const struct heap * heap);
+
+
+//------------------------------ 
+#include <ccan/tal/tal.h>
+#include <stdint.h>
+
+
+static int less_comparer(const void *const ctx UNUSED,
+		  const void *const a,
+		  const void *const b)
+{
+	s64 da = ((struct heap_data*)a)->distance,
+	    db = ((struct heap_data*)b)->distance;
+	u32 ia = ((struct heap_data*)a)->idx,
+	    ib = ((struct heap_data*)b)->idx;
+	return da==db ? ia > ib : da > db;
+}
+
+static void item_mover(void *const dst, const void *const src)
+{
+	*(struct heap_data*)dst = *(struct heap_data*)src;
+}
+
+struct heap* heap_new(const tal_t *ctx, const size_t max_capacity)
+{
+	struct heap* heap = tal(ctx,struct heap);
+	heap->size=0;
+	heap->data = tal_arr(heap,struct heap_data,max_capacity);
+	heap->max_size = max_capacity;
+	
+	heap->gheap_ctx.fanout=2;
+	heap->gheap_ctx.page_chunks=1;
+	heap->gheap_ctx.item_size= sizeof(struct heap_data);
+	heap->gheap_ctx.less_comparer=less_comparer;
+	heap->gheap_ctx.less_comparer_ctx=heap;
+	heap->gheap_ctx.item_mover=item_mover;
+	
+	return heap;
+}
+
+
+void heap_insert(struct heap* heap, u32 idx, s64 distance)
+{
+	heap->data[heap->size].idx=idx;
+	heap->data[heap->size].distance=distance;
+	heap->size++;
+	
+	assert(heap->size<=heap->max_size);
+	
+	gheap_restore_heap_after_item_increase(&heap->gheap_ctx,
+					       heap->data,
+					       heap->size,
+					       heap->size-1);
+}
+bool heap_empty(const struct heap* heap)
+{
+	return heap->size==0;
+}
+struct heap_data * heap_top(const struct heap * heap)
+{
+	return &heap->data[0];
+}
+void heap_pop(struct heap* heap)
+{
+	if(heap->size>0)
+		gheap_pop_heap(&heap->gheap_ctx,heap->data,heap->size--);
+}
+
+/* Input: item
+ * Output: the smallest x such that !(x<item) */
+size_t gheap_upper_bound(const struct gheap_ctx *ctx,
+	const void *base, size_t heap_size, void *item)
+{
+	const size_t fanout = ctx->fanout;	
+	const size_t item_size = ctx->item_size;
+	const void*const less_comparer_ctx = ctx->less_comparer_ctx;
+	const gheap_less_comparer_t less_comparer = ctx->less_comparer;
+	
+	if(less_comparer(less_comparer_ctx,base,item))
+	{
+		// root<item, so x<=root<item is true for every node
+		return heap_size;
+	}
+	
+	size_t last=0;
+	// the root is an upper bound, now let's go down
+	while(1)
+	{
+		// last is an upper bound, seach for a smaller one
+		size_t first_child = gheap_get_child_index(ctx,last);
+		size_t best_child = last; 
+		
+		for(size_t i=0;i<fanout;++i)
+		{
+			size_t child = i+first_child;
+			if(child>=heap_size)
+				break;
+			if(!less_comparer(less_comparer_ctx,
+					  ((char*)base) + child*item_size,
+					  item))
+			{
+				// satisfies the condition,
+				// is it the smallest one?
+				if(!less_comparer(less_comparer_ctx,
+				                  ((char*)base) + best_child*item_size,
+						  ((char*)base) + child*item_size))
+				{
+					// child <= best_child, so child is a
+					// better upper bound
+					best_child = child;
+				}
+			}
+		}
+		
+		if(best_child==last)
+		{
+			// no change, we stop
+			break;
+		}
+		last = best_child;
+	}
+	return last;
+}
+void heap_update(struct heap* heap, u32 idx, s64 old_distance, s64 new_distance)
+{
+	const gheap_less_comparer_t less_comparer = heap->gheap_ctx.less_comparer;
+	const void *const less_comparer_ctx = heap->gheap_ctx.less_comparer_ctx;
+	
+	struct heap_data old_item = (struct heap_data){.idx=idx, .distance=old_distance};
+	
+	size_t pos = gheap_upper_bound(&heap->gheap_ctx,heap->data,heap->size,&old_item);
+	if(pos>=heap->size || heap->data[pos].idx!=idx)
+	{
+		heap_insert(heap,idx,new_distance);
+	}
+	else
+	{
+		struct heap_data new_item = (struct heap_data){.idx=idx, .distance=new_distance};
+			
+		if(less_comparer(less_comparer_ctx,&new_item,&heap->data[pos]))
+		{
+			heap->data[pos].distance = new_distance;
+			gheap_restore_heap_after_item_decrease(
+					&heap->gheap_ctx,
+					heap->data,
+					heap->size,
+					pos);
+		}else
+		{
+			heap->data[pos].distance = new_distance;
+			gheap_restore_heap_after_item_increase(
+					&heap->gheap_ctx,
+					heap->data,
+					heap->size,
+					pos);
+		}
+	}
+}
+
+#endif /* LIGHTNING_PLUGINS_RENEPAY_HEAP_H */
diff --git a/plugins/renepay/mcf.c b/plugins/renepay/mcf.c
index 06173d80bf75..802e63680c85 100644
--- a/plugins/renepay/mcf.c
+++ b/plugins/renepay/mcf.c
@@ -3,6 +3,7 @@
 #include <ccan/tal/tal.h>
 #include <plugins/renepay/mcf.h>
 #include <plugins/renepay/flow.h>
+#include <plugins/renepay/heap.h>
 #include <stdint.h>
 #include <math.h>
 #include <assert.h>
@@ -104,7 +105,6 @@ typedef union
 #define MIN(x, y) (((x) < (y)) ? (x) : (y))
 
 struct pay_parameters {
-	
 	/* The gossmap we are using */
 	struct gossmap *gossmap;
 	struct gossmap_node *source;
@@ -143,7 +143,7 @@ struct residual_network {
 	s64 *cost;
 	s64 *potential;
 };
-				
+
 
 /* Helper function. 
  * Given an arc idx, return the dual's idx in the residual network. */
@@ -299,7 +299,9 @@ static void init_residual_network(struct pay_parameters *params,
 	const size_t max_num_chans = gossmap_max_chan_idx(params->gossmap);
 	const size_t max_num_arcs = max_num_chans << ARC_ADDITIONAL_BITS;
 	const size_t max_num_nodes = gossmap_max_node_idx(params->gossmap);
-	
+
+	network->dijkstra = tal_arrz(network,dijkstra,max_num_nodes);
+
 	network->cap = tal_arrz(network,s64,max_num_arcs);
 	
 	network->cost = tal_arr(network,s64,max_num_arcs);
@@ -599,7 +601,7 @@ static int  find_optimal_path(
 	}
 	distance[source]=0;
 	
-	struct heap *myheap = heap_new(this_ctx);	
+	struct heap *myheap = heap_new(this_ctx,tal_count(distance));	
 	heap_insert(myheap,source,0);
 	
 	while(!heap_empty(myheap))
@@ -637,10 +639,11 @@ static int  find_optimal_path(
 			if(distance[next]<=distance[cur]+cij)
 				continue;
 			
+			
+			heap_update(myheap,next,distance[next],distance[cur]+cij);
+			
 			prev[next]=arc;
 			distance[next]=distance[cur]+cij;
-			
-			heap_insert(myheap,next,distance[next]);
 		}
 	}
 	tal_free(this_ctx);	

From a9058be0c9d2d8f1858b2c5c993161ad2512c144 Mon Sep 17 00:00:00 2001
From: Lagrang3 <eduardo.quintana@pm.me>
Date: Sat, 29 Apr 2023 16:11:35 +0100
Subject: [PATCH 17/64] renepay: use a global dijkstra heap

---
 plugins/renepay/mcf.c | 148 +++++++++++++++++++++++++++++++++++-------
 1 file changed, 125 insertions(+), 23 deletions(-)

diff --git a/plugins/renepay/mcf.c b/plugins/renepay/mcf.c
index 802e63680c85..ecfe827a3151 100644
--- a/plugins/renepay/mcf.c
+++ b/plugins/renepay/mcf.c
@@ -144,6 +144,107 @@ struct residual_network {
 	s64 *potential;
 };
 
+/* In the heap we keep node idx, but in this structure we keep the distance
+ * value associated to every node, and their position in the heap as a pointer
+ * so that we can update the nodes inside the heap when the distance label is
+ * changed. */
+struct dijkstra {
+	// 
+	s64 *distance;
+	u32 *base;
+	u32 **heapptr;
+	size_t heapsize;
+	struct gheap_ctx gheap_ctx;
+};
+/* Required a global dijkstra for gheap. */
+static struct dijkstra *global_dijkstra;
+
+static int dijkstra_less_comparer(
+	const void *const ctx UNUSED,
+	const void *const a,
+	const void *const b)
+{
+	return global_dijkstra->distance[*(u32*)a]
+		> global_dijkstra->distance[*(u32*)b];
+}
+static void dijkstra_item_mover(void *const dst, const void *const src)
+{
+	u32 src_idx = *(u32*)src;
+	*(u32*)dst = src_idx;
+	global_dijkstra->heapptr[src_idx] = dst;
+}
+static void dijkstra_malloc(const tal_t ctx, const size_t max_num_nodes)
+{
+	global_dijkstra = tal(ctx,struct dijkstra);
+	global_dijkstra->distance = tal_arr(global_dijkstra,s64,max_num_nodes);
+	global_dijkstra->base = tal_arr(global_dijkstra,u32,max_num_nodes);
+	global_dijkstra->heapptr = tal_arr(global_dijkstra,u32*,max_num_nodes);
+	
+	global_dijkstra->heapsize=0;
+	
+	global_dijkstra->gheap_ctx.fanout=2;
+	global_dijkstra->gheap_ctx.page_chunks=1024;
+	global_dijkstra->gheap_ctx.less_comparer=dijkstra_less_comparer;
+	global_dijkstra->gheap_ctx.less_comparer_ctx=NULL;
+	global_dijkstra->gheap_ctx.item_mover=dijkstra_item_mover;
+}
+static void dijkstra_init()
+{
+	const size_t max_num_nodes = tal_count(global_dijkstra->distance);
+	global_dijkstra->heapsize=0;
+	for(size_t i=0;i<max_num_nodes;++i)
+	{
+		global_dijkstra->distance[i]=INFINITE;
+		global_dijkstra->heapptr[i] = NULL;
+	}
+}
+static void dijkstra_append(u32 node_idx, s64 distance)
+{
+	const size_t pos = global_dijkstra->heapsize;
+	
+	global_dijkstra->base[pos]=node_idx;
+	global_dijkstra->distance[node_idx]=distance;
+	global_dijkstra->heapptr[node_idx] = &(global_dijkstra->base[pos]);
+	global_dijkstra->heapsize++;
+}
+static void dijkstra_update(u32 node_idx, s64 distance)
+{
+	if(!global_dijkstra->heapptr[node_idx])
+	{
+		// not in the heap
+		dijkstra_append(node_idx,distance);
+	}
+	
+	gheap_restore_heap_after_item_increase(
+		&global_dijkstra->gheap_ctx
+		global_dijkstra->base,
+		global_dijkstra->heapsize,
+		global_dijkstra->heapptr[node_idx]
+			- global_dijkstra->base);
+}
+static u32 dijkstra_top()
+{
+	return global_dijkstra->base[0];
+}
+static bool dijkstra_empty()
+{
+	return global_dijkstra->heapsize==0;
+}
+static void dijkstra_pop()
+{
+	if(global_dijkstra->heapsize==0)
+		return;
+		
+	const u32 top = dijkstra_top();
+	assert(global_dijkstra->heapptr[top]==global_dijkstra->base);
+	
+	gheap_pop_heap(
+		&global_dijkstra->gheap_ctx
+		global_dijkstra->base,
+		global_dijkstra->heapsize--);
+		
+	global_dijkstra->heapptr[top]=NULL;
+}
 
 /* Helper function. 
  * Given an arc idx, return the dual's idx in the residual network. */
@@ -586,8 +687,7 @@ static int  find_optimal_path(
 	const struct residual_network* network,
 	const u32 source,
 	const u32 target,
-	arc_t *prev,
-	s64 *distance)
+	arc_t *prev)
 {
 	tal_t *this_ctx = tal(tmpctx,tal_t);
 	int ret = RENEPAY_ERR_NOFEASIBLEFLOW;
@@ -597,18 +697,17 @@ static int  find_optimal_path(
 	for(size_t i=0;i<tal_count(prev);++i)
 	{	
 		prev[i].idx=INVALID_INDEX;
-		distance[i]=INFINITE;
 	}
-	distance[source]=0;
 	
-	struct heap *myheap = heap_new(this_ctx,tal_count(distance));	
-	heap_insert(myheap,source,0);
+	const s64 *const distance=global_dijkstra->distance;
+	
+	dijkstra_init();
+	dijkstra_append(source,0);
 	
-	while(!heap_empty(myheap))
+	while(!dijkstra_empty())
 	{
-		struct heap_data *top = heap_top(myheap);
-		u32 cur = top->idx;
-		heap_pop(myheap);
+		u32 cur = dijkstra_top();
+		dijkstra_pop();
 		
 		if (visited[cur])
 			continue;
@@ -633,17 +732,15 @@ static int  find_optimal_path(
 			
 			s64 cij = network->cost[arc.idx] - network->potential[cur]
 			                                 + network->potential[next];
+			
 			// Dijkstra only works with non-negative weights
 			assert(cij>=0);
 			
 			if(distance[next]<=distance[cur]+cij)
 				continue;
 			
-			
-			heap_update(myheap,next,distance[next],distance[cur]+cij);
-			
+			dijkstra_update(next,distance[cur]+cij);
 			prev[next]=arc;
-			distance[next]=distance[cur]+cij;
 		}
 	}
 	tal_free(this_ctx);	
@@ -693,11 +790,14 @@ static int optimize_mcf(const struct pay_parameters *params,
 	tal_t *this_ctx = tal(tmpctx,tal_t);
 	
 	arc_t *prev = tal_arr(this_ctx,arc_t,tal_count(network->potential));
-	s64 *distance = tal_arr(this_ctx,s64,tal_count(network->potential));
+	
+	
+	// s64 *distance = tal_arr(this_ctx,s64,tal_count(network->potential));
+	const s64 *const distance = global_dijkstra->distance;
 	
 	while(amount>0)
 	{
-		int err = find_optimal_path(params,network,source,target,prev,distance);
+		int err = find_optimal_path(params,network,source,target,prev);
 		if(err!=RENEPAY_ERR_OK)
 		{
 			// unexpected error
@@ -1040,7 +1140,7 @@ static struct flow_path **
  * In the `pay_flow` module there are dedicated routes to compute the actual
  * amount to be forward on each hop. */
 struct flow_path* minflow(
-                      const tal_t *ctx,
+                      const tal_t *caller_ctx,
 		      struct gossmap *gossmap,
 		      const struct gossmap_node *source,
 		      const struct gossmap_node *target,
@@ -1052,7 +1152,9 @@ struct flow_path* minflow(
 		      double delay_feefactor UNUSED)
 {
 	// TODO(eduardo) on which ctx should I allocate the MCF data?
-	struct pay_parameters *params = tal(tmpctx,struct pay_parameters);	
+	tal_t *this_ctx = tal(tmpctx,tal_t);
+	
+	struct pay_parameters *params = tal(this_ctx,struct pay_parameters);	
 	
 	int ret;
 	
@@ -1079,9 +1181,11 @@ struct flow_path* minflow(
 	// params->basefee_penalty = 
 	
 	// build the uncertainty network with linearization and residual arcs
-	struct residual_network *network = tal(tmpctx,struct residual_network);
+	struct residual_network *network = tal(this_ctx,struct residual_network);
 	init_residual_network(params,network);
 	
+	dijkstra_malloc(this_ctx);
+	
 	const u32 target_idx = gossmap_node_idx(params->gossmap,target);
 	const u32 source_idx = gossmap_node_idx(params->gossmap,source);
 	
@@ -1150,15 +1254,13 @@ struct flow_path* minflow(
 
 	if(ret==RENEPAY_ERR_OK)
 	{
-		flow_paths = get_flow_paths(ctx,params,network);
+		flow_paths = get_flow_paths(caller_ctx,params,network);
 	}else
 	{
 	// TODO(eduardo) fallback to c_prob or c_fee?
 	}
 	
-	tal_free(network);
-	tal_free(params);
-	
+	tal_free(this_ctx);
 	return flow_paths;
 }
 

From 69bc4b5e33fceab3eaa12c678da20f32e9f50603 Mon Sep 17 00:00:00 2001
From: Lagrang3 <eduardo.quintana@pm.me>
Date: Sat, 29 Apr 2023 16:16:00 +0100
Subject: [PATCH 18/64] reneplay: bugfix

---
 plugins/renepay/mcf.c | 1 +
 1 file changed, 1 insertion(+)

diff --git a/plugins/renepay/mcf.c b/plugins/renepay/mcf.c
index ecfe827a3151..bc2964aa4f5e 100644
--- a/plugins/renepay/mcf.c
+++ b/plugins/renepay/mcf.c
@@ -184,6 +184,7 @@ static void dijkstra_malloc(const tal_t ctx, const size_t max_num_nodes)
 	
 	global_dijkstra->gheap_ctx.fanout=2;
 	global_dijkstra->gheap_ctx.page_chunks=1024;
+	global_dijkstra->item_size=sizeof(global_dijkstra->base[0]);
 	global_dijkstra->gheap_ctx.less_comparer=dijkstra_less_comparer;
 	global_dijkstra->gheap_ctx.less_comparer_ctx=NULL;
 	global_dijkstra->gheap_ctx.item_mover=dijkstra_item_mover;

From 328c74eb370a71304110f2f2e8f339a966126ada Mon Sep 17 00:00:00 2001
From: Lagrang3 <eduardo.quintana@pm.me>
Date: Thu, 4 May 2023 14:54:53 +0100
Subject: [PATCH 19/64] renepay: fix small details

---
 plugins/renepay/flow.h     |  1 -
 plugins/renepay/mcf.c      | 39 ++++++++++++++++++++------------------
 plugins/renepay/mcf.h      | 11 +----------
 plugins/renepay/pay_flow.c | 11 ++++++++---
 4 files changed, 30 insertions(+), 32 deletions(-)

diff --git a/plugins/renepay/flow.h b/plugins/renepay/flow.h
index d5bbaf8e8152..514a8ecd29ff 100644
--- a/plugins/renepay/flow.h
+++ b/plugins/renepay/flow.h
@@ -65,7 +65,6 @@ struct chan_extra_half *new_chan_extra_half(struct chan_extra_map *chan_extra_ma
 
 /* An actual partial flow. */
 struct flow {
-	/* The series of channels to traverse. */
 	const struct gossmap_chan **path;
 	/* The directions to traverse. */
 	int *dirs;
diff --git a/plugins/renepay/mcf.c b/plugins/renepay/mcf.c
index bc2964aa4f5e..f42038507da3 100644
--- a/plugins/renepay/mcf.c
+++ b/plugins/renepay/mcf.c
@@ -173,7 +173,7 @@ static void dijkstra_item_mover(void *const dst, const void *const src)
 	*(u32*)dst = src_idx;
 	global_dijkstra->heapptr[src_idx] = dst;
 }
-static void dijkstra_malloc(const tal_t ctx, const size_t max_num_nodes)
+static void dijkstra_malloc(const tal_t *ctx, const size_t max_num_nodes)
 {
 	global_dijkstra = tal(ctx,struct dijkstra);
 	global_dijkstra->distance = tal_arr(global_dijkstra,s64,max_num_nodes);
@@ -184,12 +184,12 @@ static void dijkstra_malloc(const tal_t ctx, const size_t max_num_nodes)
 	
 	global_dijkstra->gheap_ctx.fanout=2;
 	global_dijkstra->gheap_ctx.page_chunks=1024;
-	global_dijkstra->item_size=sizeof(global_dijkstra->base[0]);
+	global_dijkstra->gheap_ctx.item_size=sizeof(global_dijkstra->base[0]);
 	global_dijkstra->gheap_ctx.less_comparer=dijkstra_less_comparer;
 	global_dijkstra->gheap_ctx.less_comparer_ctx=NULL;
 	global_dijkstra->gheap_ctx.item_mover=dijkstra_item_mover;
 }
-static void dijkstra_init()
+static void dijkstra_init(void)
 {
 	const size_t max_num_nodes = tal_count(global_dijkstra->distance);
 	global_dijkstra->heapsize=0;
@@ -217,21 +217,21 @@ static void dijkstra_update(u32 node_idx, s64 distance)
 	}
 	
 	gheap_restore_heap_after_item_increase(
-		&global_dijkstra->gheap_ctx
+		&global_dijkstra->gheap_ctx,
 		global_dijkstra->base,
 		global_dijkstra->heapsize,
 		global_dijkstra->heapptr[node_idx]
 			- global_dijkstra->base);
 }
-static u32 dijkstra_top()
+static u32 dijkstra_top(void)
 {
 	return global_dijkstra->base[0];
 }
-static bool dijkstra_empty()
+static bool dijkstra_empty(void)
 {
 	return global_dijkstra->heapsize==0;
 }
-static void dijkstra_pop()
+static void dijkstra_pop(void)
 {
 	if(global_dijkstra->heapsize==0)
 		return;
@@ -240,7 +240,7 @@ static void dijkstra_pop()
 	assert(global_dijkstra->heapptr[top]==global_dijkstra->base);
 	
 	gheap_pop_heap(
-		&global_dijkstra->gheap_ctx
+		&global_dijkstra->gheap_ctx,
 		global_dijkstra->base,
 		global_dijkstra->heapsize--);
 		
@@ -402,8 +402,6 @@ static void init_residual_network(struct pay_parameters *params,
 	const size_t max_num_arcs = max_num_chans << ARC_ADDITIONAL_BITS;
 	const size_t max_num_nodes = gossmap_max_node_idx(params->gossmap);
 
-	network->dijkstra = tal_arrz(network,dijkstra,max_num_nodes);
-
 	network->cap = tal_arrz(network,s64,max_num_arcs);
 	
 	network->cost = tal_arr(network,s64,max_num_arcs);
@@ -878,7 +876,7 @@ static void estimate_costs(const struct pay_parameters *params,
 			struct chan_extra_half *extra_half 
 				= get_chan_extra_half_by_chan(
 						params->gossmap,
-						params->chan_extra_map
+						params->chan_extra_map,
 						c,
 						dir);  
 			
@@ -901,12 +899,12 @@ struct chan_flow
 struct list_data
 {
 	struct list_node list;
-	struct flow_path *flow_path;
+	struct flow *flow_path;
 };
 
 /* Given a flow in the residual network, build a set of payment flows in the
  * gossmap that corresponds to this flow. */		
-static struct flow_path **
+static struct flow **
 	get_flow_paths(
 		const tal_t *ctx,
 		const struct pay_parameters *params,
@@ -1009,10 +1007,9 @@ static struct flow_path **
 			}
 			delta = MIN(delta,balance[final_idx]);
 			
-			struct flow_path *fp = tal(ctx,struct flow_path);
+			struct flow *fp = tal(ctx,struct flow);
 			fp->path = tal_arr(fp,struct gossmap_chan*,length);
 			fp->dirs = tal_arr(fp,int,length);
-			fp->amount.satoshis = delta;
 			
 			balance[node_idx] += delta;
 			balance[final_idx]-= delta;
@@ -1027,6 +1024,12 @@ static struct flow_path **
 				chan_flow[prev_chan[cur_idx]].half[prev_dir[cur_idx]]-=delta;
 			}
 			
+			flow_complete(
+				fp,
+				params->gossmap,
+				params->chan_extra_map,
+				struct amount_msat(delta));
+			
 			// add fp to list
 			ld = tal(list_ctx,struct list_data);
 			ld->flow_path = fp;
@@ -1036,7 +1039,7 @@ static struct flow_path **
 	}
 	
 	// copy the list into the array we are going to return	
-	struct flow_path **flows = tal_arr(ctx,struct flow_path*,num_paths);
+	struct flow **flows = tal_arr(ctx,struct flow*,num_paths);
 	size_t pos=0;
 	list_for_each(&path_list,ld,list)
 	{
@@ -1140,7 +1143,7 @@ static struct flow_path **
  * described with a sequence of directed channels and one amount. 
  * In the `pay_flow` module there are dedicated routes to compute the actual
  * amount to be forward on each hop. */
-struct flow_path* minflow(
+struct flow** minflow(
                       const tal_t *caller_ctx,
 		      struct gossmap *gossmap,
 		      const struct gossmap_node *source,
@@ -1251,7 +1254,7 @@ struct flow_path* minflow(
 	
 	end:
 	
-	struct flow_path *flow_paths = NULL;
+	struct flow **flow_paths = NULL;
 
 	if(ret==RENEPAY_ERR_OK)
 	{
diff --git a/plugins/renepay/mcf.h b/plugins/renepay/mcf.h
index 930cbf225b7e..4eb5146527fa 100644
--- a/plugins/renepay/mcf.h
+++ b/plugins/renepay/mcf.h
@@ -16,15 +16,6 @@ enum {
 	RENEPAY_ERR_NOCHEAPFLOW
 };
 
-/* In this module, internally a flow is non-negative integer function on the
- * arcs that satisfies the capacity constraints; outside, a flow is seen as one
- * amount sent through a sequence of directed channels. */
-struct flow_path
-{
-	struct gossmap_chan **path;
-	int *dirs;
-	struct amount_sat amount;
-};
 
 
 /**
@@ -46,7 +37,7 @@ struct flow_path
  *
  * Return a series of subflows which deliver amount to target, or NULL.
  */
-struct flow_path* minflow(
+struct flow** minflow(
                       const tal_t *ctx,
 		      struct gossmap *gossmap,
 		      const struct gossmap_node *source,
diff --git a/plugins/renepay/pay_flow.c b/plugins/renepay/pay_flow.c
index 3bdf34fdee91..4e569e98ca32 100644
--- a/plugins/renepay/pay_flow.c
+++ b/plugins/renepay/pay_flow.c
@@ -261,6 +261,7 @@ static bitmap *make_disabled_bitmap(const tal_t *ctx,
 
 static double flows_probability(struct flow **flows)
 {
+	// TODO
 	double prob = 1.0;
 
 	for (size_t i = 0; i < tal_count(flows); i++)
@@ -271,6 +272,7 @@ static double flows_probability(struct flow **flows)
 
 static struct amount_msat flows_fee(struct flow **flows)
 {
+	// TODO
 	struct amount_msat fee = AMOUNT_MSAT(0);
 
 	for (size_t i = 0; i < tal_count(flows); i++) {
@@ -398,13 +400,16 @@ struct pay_flow **get_payflows(struct payment *p,
 		u64 delay;
 		bool too_unlikely, too_expensive, too_delayed;
 		const u32 *final_cltvs;
-
+		
+		// TODO(eduardo): set a maximum admissible fee (1%)?
+		struct amount_msat max_fee 
+			= (struct amount_msat){.millisatoshis = amount.millisatoshis/100};
+		
 		/* Note!  This actually puts flows in chan_extra_map, so
 		 * flows must be removed if not used! */
 		flows = minflow(tmpctx, pay_plugin->gossmap, src, dst,
 				&pay_plugin->chan_extra_map, disabled,
-				amount,
-				frugality,
+				amount, max_fee ,/* min prob = */ 0.01 ,
 				p->delay_feefactor);
 		if (!flows) {
 			paynote(p, "Failed to find any paths for %s",

From 72b0823481a904d44d41734da233d04c7151954f Mon Sep 17 00:00:00 2001
From: Lagrang3 <eduardo.quintana@pm.me>
Date: Fri, 5 May 2023 06:13:02 +0100
Subject: [PATCH 20/64] renepay: added conditional probability on arc

---
 plugins/renepay/flow.c | 182 +++++++++++++++++++++++++----------------
 plugins/renepay/flow.h |   2 +-
 2 files changed, 113 insertions(+), 71 deletions(-)

diff --git a/plugins/renepay/flow.c b/plugins/renepay/flow.c
index 5db859db3679..35915bca0870 100644
--- a/plugins/renepay/flow.c
+++ b/plugins/renepay/flow.c
@@ -19,29 +19,69 @@ const struct half_chan *flow_edge(const struct flow *flow, size_t idx)
 }
 
 /* Assuming a uniform distribution, what is the chance this f gets through?
+ * Here we compute the conditional probability of success for a flow f, given
+ * the knowledge that the liquidity is in the range [a,b) and some amount
+ * x is already committed on another part of the payment.
+ * 
+ * The probability equation for x=0 is:
+ * 
+ * 	prob(f) = 
+ * 	
+ * 	for f<a:	1.
+ * 	for f>=a:	(b-f)/(b-a)
+ * 
+ * When x>0 the prob. of success for passing x and f is:
+ * 
+ * 	prob(f and x) = prob(x) * prob(f|x)
+ * 
+ * and it can be shown to be equal to
+ * 
+ * 	prob(f and x) = prob(f+x)
+ * 
+ * The purpose of this function is to obtain prob(f|x), i.e. the probability of
+ * getting f through provided that we already succeeded in getting x.
+ * This conditional probability comes with three cases:
+ * 
+ * 	prob(f|x) = 
+ * 	
+ * 	for x<a and f<a-x: 	1.
+ * 	for x<a and f>=a-x:	(b-x-f)/(b-a)
+ * 	for x>=a:		(b-x-f)/(b-x)
  *
- * This is the "(c_e + 1 − f_e) / (c_e + 1)" in the paper.
+ * This is the same as the probability of success of f when the bounds are
+ * shifted by x amount, the new bounds be [MAX(0,a-x),b-x).
  */
 static double edge_probability(struct amount_msat min, struct amount_msat max,
+			       struct amount_msat in_flight,
 			       struct amount_msat f)
 {
-	struct amount_msat range_plus_one, numerator;
-
-	/* If f is <= known minimum, probability is 1. */
-	if (!amount_msat_sub(&f, f, min))
-		return 1.0;
-
-	/* +1 because the number of elements in the range is min-max + 1 */
-	if (!amount_msat_sub(&range_plus_one, max, min))
+	struct amount_msat B; // =  max +1 - in_flight
+	
+	// one past the last known value, makes computations simpler
+	if(!amount_msat_add(&B,B,AMOUNT_MSAT(1)))
 		abort();
-	if (!amount_msat_add(&range_plus_one, range_plus_one, AMOUNT_MSAT(1)))
+		
+	// in_flight cannot be greater than max
+	if(!amount_msat_sub(&B,B,in_flight))
 		abort();
-
-	/* If f > capacity, probability is 0 */
-	if (!amount_msat_sub(&numerator, range_plus_one, f))
-		return 0.0;
-
-	return amount_msat_ratio(numerator, range_plus_one);
+		
+	struct amount_msat A; // = MAX(0,min-in_flight);
+	
+	if(!amount_msat_sub(&A,A,in_flight))
+		A = AMOUNT_MSAT(0);
+	
+	struct amount_msat denominator; // = B-A
+	
+	// B cannot be smaller than or equal A
+	if(!amount_msat_sub(&denominator,B,A) || amount_msat_less_eq(B,A))
+		abort();
+	
+	struct amount_msat numerator; // MAX(0,B-f)
+	
+	if(!amount_msat_sub(&numerator,B,f))
+		numerator = AMOUNT_MSAT(0);
+	
+	return amount_msat_less_eq(f,A) ? 1.0 : amount_msat_ratio(numerator,denominator);
 }
 
 bool flow_path_eq(const struct gossmap_chan **path1,
@@ -152,57 +192,57 @@ void flow_add(struct flow *flow,
 /* From the paper:
  * −log((c_e + 1 − f_e) / (c_e + 1)) + μ f_e fee(e)
  */
-double flow_edge_cost(const struct gossmap *gossmap,
-		      const struct gossmap_chan *c, int dir,
-		      const struct amount_msat known_min,
-		      const struct amount_msat known_max,
-		      struct amount_msat prev_flow,
-		      struct amount_msat f,
-		      double mu,
-		      double basefee_penalty,
-		      double delay_riskfactor)
-{
-	double prob, effective_feerate;
-	double certainty_term, feerate_term;
-
-#ifdef SUPERVERBOSE_ENABLED
-	struct short_channel_id scid
-		= gossmap_chan_scid(gossmap, c);
-	SUPERVERBOSE("flow_edge_cost %s/%i, cap %"PRIu64"-%"PRIu64", prev_flow=%"PRIu64", f=%"PRIu64", mu=%f, basefee_penalty=%f, delay_riskfactor=%f: ",
-		     type_to_string(tmpctx, struct short_channel_id, &scid),
-		     dir,
-		     known_min.millisatoshis, known_max.millisatoshis,
-		     prev_flow.millisatoshis, f.millisatoshis,
-		     mu, basefee_penalty, delay_riskfactor);
-#endif
-
-	/* Probability depends on any previous flows, too! */
-	if (!amount_msat_add(&prev_flow, prev_flow, f))
-		abort();
-	prob = edge_probability(known_min, known_max, prev_flow);
-	if (prob == 0) {
-		SUPERVERBOSE(" INFINITE\n");
-		return FLOW_INF_COST;
-	}
-
-	certainty_term = -log(prob);
-
-	/* This is in parts-per-million */
-	effective_feerate = (c->half[dir].proportional_fee
-			     + c->half[dir].base_fee * basefee_penalty)
-		/ 1000000.0;
-
-	/* Feerate term includes delay factor */
-	feerate_term = (mu
-			* (f.millisatoshis /* Raw: costfn */
-			   * effective_feerate
-			   + c->half[dir].delay * delay_riskfactor));
-
-	SUPERVERBOSE(" %f + %f = %f\n",
-		     certainty_term, feerate_term,
-		     certainty_term + feerate_term);
-	return certainty_term + feerate_term;
-}
+// double flow_edge_cost(const struct gossmap *gossmap,
+// 		      const struct gossmap_chan *c, int dir,
+// 		      const struct amount_msat known_min,
+// 		      const struct amount_msat known_max,
+// 		      struct amount_msat prev_flow,
+// 		      struct amount_msat f,
+// 		      double mu,
+// 		      double basefee_penalty,
+// 		      double delay_riskfactor)
+// {
+// 	double prob, effective_feerate;
+// 	double certainty_term, feerate_term;
+// 
+// #ifdef SUPERVERBOSE_ENABLED
+// 	struct short_channel_id scid
+// 		= gossmap_chan_scid(gossmap, c);
+// 	SUPERVERBOSE("flow_edge_cost %s/%i, cap %"PRIu64"-%"PRIu64", prev_flow=%"PRIu64", f=%"PRIu64", mu=%f, basefee_penalty=%f, delay_riskfactor=%f: ",
+// 		     type_to_string(tmpctx, struct short_channel_id, &scid),
+// 		     dir,
+// 		     known_min.millisatoshis, known_max.millisatoshis,
+// 		     prev_flow.millisatoshis, f.millisatoshis,
+// 		     mu, basefee_penalty, delay_riskfactor);
+// #endif
+// 
+// 	/* Probability depends on any previous flows, too! */
+// 	if (!amount_msat_add(&prev_flow, prev_flow, f))
+// 		abort();
+// 	prob = edge_probability(known_min, known_max, prev_flow);
+// 	if (prob == 0) {
+// 		SUPERVERBOSE(" INFINITE\n");
+// 		return FLOW_INF_COST;
+// 	}
+// 
+// 	certainty_term = -log(prob);
+// 
+// 	/* This is in parts-per-million */
+// 	effective_feerate = (c->half[dir].proportional_fee
+// 			     + c->half[dir].base_fee * basefee_penalty)
+// 		/ 1000000.0;
+// 
+// 	/* Feerate term includes delay factor */
+// 	feerate_term = (mu
+// 			* (f.millisatoshis /* Raw: costfn */
+// 			   * effective_feerate
+// 			   + c->half[dir].delay * delay_riskfactor));
+// 
+// 	SUPERVERBOSE(" %f + %f = %f\n",
+// 		     certainty_term, feerate_term,
+// 		     certainty_term + feerate_term);
+// 	return certainty_term + feerate_term;
+// }
 
 /* Helper function to fill in amounts and success_prob for flow */
 void flow_complete(struct flow *flow,
@@ -234,12 +274,14 @@ void flow_complete(struct flow *flow,
 		}
 
 		flow->amounts[i] = delivered;
+		flow->success_prob
+			*= edge_probability(h->known_min, h->known_max,
+					    h->htlc_total,
+					    delivered);
+		
 		if (!amount_msat_add(&h->htlc_total, h->htlc_total, delivered))
 			abort();
 		h->num_htlcs++;
-		flow->success_prob
-			*= edge_probability(h->known_min, h->known_max,
-					    h->htlc_total);
 		if (!amount_msat_add_fee(&delivered,
 					 flow_edge(flow, i)->base_fee,
 					 flow_edge(flow, i)->proportional_fee))
diff --git a/plugins/renepay/flow.h b/plugins/renepay/flow.h
index 514a8ecd29ff..b76f1b786a8c 100644
--- a/plugins/renepay/flow.h
+++ b/plugins/renepay/flow.h
@@ -65,7 +65,7 @@ struct chan_extra_half *new_chan_extra_half(struct chan_extra_map *chan_extra_ma
 
 /* An actual partial flow. */
 struct flow {
-	const struct gossmap_chan **path;
+	struct gossmap_chan const **path;
 	/* The directions to traverse. */
 	int *dirs;
 	/* Amounts for this flow (fees mean this shrinks across path). */

From f4217bbe0f8c8f93c1ade560a54afd0870c6bd49 Mon Sep 17 00:00:00 2001
From: Lagrang3 <eduardo.quintana@pm.me>
Date: Mon, 8 May 2023 14:10:02 +0100
Subject: [PATCH 21/64] renepay: refactoring the code and make it compile

---
 plugins/renepay/Makefile   |    4 +-
 plugins/renepay/dijkstra.c |  114 ++++
 plugins/renepay/dijkstra.h |   38 ++
 plugins/renepay/flow.c     |   58 ++
 plugins/renepay/flow.h     |   11 +
 plugins/renepay/mcf.c      | 1222 +++++++++++++++++++-----------------
 plugins/renepay/mcf.h      |   45 +-
 plugins/renepay/pay_flow.c |   93 +--
 8 files changed, 920 insertions(+), 665 deletions(-)
 create mode 100644 plugins/renepay/dijkstra.c
 create mode 100644 plugins/renepay/dijkstra.h

diff --git a/plugins/renepay/Makefile b/plugins/renepay/Makefile
index a3e35956c891..273ecfc18b87 100644
--- a/plugins/renepay/Makefile
+++ b/plugins/renepay/Makefile
@@ -1,5 +1,5 @@
-PLUGIN_RENEPAY_SRC := plugins/renepay/pay.c plugins/renepay/pay_flow.c plugins/renepay/flow.c plugins/renepay/not_mcf.c
-PLUGIN_RENEPAY_HDRS := plugins/renepay/pay.h plugins/renepay/pay_flow.h plugins/renepay/flow.h plugins/renepay/mcf.h
+PLUGIN_RENEPAY_SRC := plugins/renepay/pay.c plugins/renepay/pay_flow.c plugins/renepay/flow.c plugins/renepay/mcf.c plugins/renepay/dijkstra.c
+PLUGIN_RENEPAY_HDRS := plugins/renepay/pay.h plugins/renepay/pay_flow.h plugins/renepay/flow.h plugins/renepay/mcf.h plugins/renepay/heap.h plugins/renepay/dijkstra.h
 PLUGIN_RENEPAY_OBJS := $(PLUGIN_RENEPAY_SRC:.c=.o)
 
 # Make sure these depend on everything.
diff --git a/plugins/renepay/dijkstra.c b/plugins/renepay/dijkstra.c
new file mode 100644
index 000000000000..56d93061b5fe
--- /dev/null
+++ b/plugins/renepay/dijkstra.c
@@ -0,0 +1,114 @@
+#include <plugins/renepay/dijkstra.h>
+
+static const s64 INFINITE = INT64_MAX;
+
+/* Required a global dijkstra for gheap. */
+static struct dijkstra *global_dijkstra;
+
+/* The heap comparer for Dijkstra search. Since the top element must be the one
+ * with the smallest distance, we user the operator >, rather than <. */
+static int dijkstra_less_comparer(
+	const void *const ctx UNUSED,
+	const void *const a,
+	const void *const b)
+{
+	return global_dijkstra->distance[*(u32*)a]
+		> global_dijkstra->distance[*(u32*)b];
+}
+
+/* The heap move operator for Dijkstra search. */
+static void dijkstra_item_mover(void *const dst, const void *const src)
+{
+	u32 src_idx = *(u32*)src;
+	*(u32*)dst = src_idx;
+	
+	// we keep track of the pointer position of each element in the heap,
+	// for easy update.
+	global_dijkstra->heapptr[src_idx] = dst;
+}
+
+/* Allocation of resources for the heap. */
+void dijkstra_malloc(const tal_t *ctx, const size_t max_num_nodes)
+{
+	global_dijkstra = tal(ctx,struct dijkstra);
+	global_dijkstra->distance = tal_arr(global_dijkstra,s64,max_num_nodes);
+	global_dijkstra->base = tal_arr(global_dijkstra,u32,max_num_nodes);
+	global_dijkstra->heapptr = tal_arr(global_dijkstra,u32*,max_num_nodes);
+	
+	global_dijkstra->heapsize=0;
+	
+	global_dijkstra->gheap_ctx.fanout=2;
+	global_dijkstra->gheap_ctx.page_chunks=1024;
+	global_dijkstra->gheap_ctx.item_size=sizeof(global_dijkstra->base[0]);
+	global_dijkstra->gheap_ctx.less_comparer=dijkstra_less_comparer;
+	global_dijkstra->gheap_ctx.less_comparer_ctx=NULL;
+	global_dijkstra->gheap_ctx.item_mover=dijkstra_item_mover;
+}
+
+/* Initialization of the heap for a new Dijkstra search. */
+void dijkstra_init(void)
+{
+	const size_t max_num_nodes = tal_count(global_dijkstra->distance);
+	global_dijkstra->heapsize=0;
+	for(size_t i=0;i<max_num_nodes;++i)
+	{
+		global_dijkstra->distance[i]=INFINITE;
+		global_dijkstra->heapptr[i] = NULL;
+	}
+}
+
+void dijkstra_append(u32 node_idx, s64 distance)
+{
+	const size_t pos = global_dijkstra->heapsize;
+	
+	global_dijkstra->base[pos]=node_idx;
+	global_dijkstra->distance[node_idx]=distance;
+	global_dijkstra->heapptr[node_idx] = &(global_dijkstra->base[pos]);
+	global_dijkstra->heapsize++;
+}
+void dijkstra_update(u32 node_idx, s64 distance)
+{
+	if(!global_dijkstra->heapptr[node_idx])
+	{
+		// not in the heap
+		dijkstra_append(node_idx,distance);
+	}
+	assert(node_idx < tal_count(global_dijkstra->distance));
+	assert(global_dijkstra->distance[node_idx] >= distance);
+	
+	global_dijkstra->distance[node_idx] = distance;
+	
+	gheap_restore_heap_after_item_increase(
+		&global_dijkstra->gheap_ctx,
+		global_dijkstra->base,
+		global_dijkstra->heapsize,
+		global_dijkstra->heapptr[node_idx]
+			- global_dijkstra->base);
+}
+u32 dijkstra_top(void)
+{
+	return global_dijkstra->base[0];
+}
+bool dijkstra_empty(void)
+{
+	return global_dijkstra->heapsize==0;
+}
+void dijkstra_pop(void)
+{
+	if(global_dijkstra->heapsize==0)
+		return;
+		
+	const u32 top = dijkstra_top();
+	assert(global_dijkstra->heapptr[top]==global_dijkstra->base);
+	
+	gheap_pop_heap(
+		&global_dijkstra->gheap_ctx,
+		global_dijkstra->base,
+		global_dijkstra->heapsize--);
+		
+	global_dijkstra->heapptr[top]=NULL;
+}
+const s64* dijkstra_distance_data(void)
+{
+	return global_dijkstra->distance;
+}
diff --git a/plugins/renepay/dijkstra.h b/plugins/renepay/dijkstra.h
new file mode 100644
index 000000000000..ae00655f5062
--- /dev/null
+++ b/plugins/renepay/dijkstra.h
@@ -0,0 +1,38 @@
+#ifndef LIGHTNING_PLUGINS_RENEPAY_DIJKSTRA_H
+#define LIGHTNING_PLUGINS_RENEPAY_DIJKSTRA_H
+
+// TODO(eduardo): unit test this
+
+
+#include <ccan/short_types/short_types.h>
+#include <ccan/tal/tal.h>
+#include <gheap.h>
+
+/* In the heap we keep node idx, but in this structure we keep the distance
+ * value associated to every node, and their position in the heap as a pointer
+ * so that we can update the nodes inside the heap when the distance label is
+ * changed. */
+struct dijkstra {
+	// 
+	s64 *distance;
+	u32 *base;
+	u32 **heapptr;
+	size_t heapsize;
+	struct gheap_ctx gheap_ctx;
+};
+
+/* Allocation of resources for the heap. */
+void dijkstra_malloc(const tal_t *ctx, const size_t max_num_nodes);
+
+/* Initialization of the heap for a new Dijkstra search. */
+void dijkstra_init(void);
+
+void dijkstra_append(u32 node_idx, s64 distance);
+void dijkstra_update(u32 node_idx, s64 distance);
+u32 dijkstra_top(void);
+bool dijkstra_empty(void);
+void dijkstra_pop(void);
+
+const s64* dijkstra_distance_data(void);
+
+#endif // LIGHTNING_PLUGINS_RENEPAY_DIJKSTRA_H
diff --git a/plugins/renepay/flow.c b/plugins/renepay/flow.c
index 35915bca0870..5cabc7970370 100644
--- a/plugins/renepay/flow.c
+++ b/plugins/renepay/flow.c
@@ -18,6 +18,7 @@ const struct half_chan *flow_edge(const struct flow *flow, size_t idx)
 	return &flow->path[idx]->half[flow->dirs[idx]];
 }
 
+// TODO(eduardo): check this
 /* Assuming a uniform distribution, what is the chance this f gets through?
  * Here we compute the conditional probability of success for a flow f, given
  * the knowledge that the liquidity is in the range [a,b) and some amount
@@ -244,6 +245,7 @@ void flow_add(struct flow *flow,
 // 	return certainty_term + feerate_term;
 // }
 
+// TODO(eduardo): check this
 /* Helper function to fill in amounts and success_prob for flow */
 void flow_complete(struct flow *flow,
 		   const struct gossmap *gossmap,
@@ -386,6 +388,62 @@ double derive_mu(const struct gossmap *gossmap,
 		/ (median_fee.millisatoshis + 1); /* Raw: derive_mu */
 }
 
+/* Get the fee cost associated to this directed channel. 
+ * Cost is expressed as PPM of the payment.
+ * 
+ * Choose and integer `c_fee` to linearize the following fee function
+ * 
+ *  	fee_msat = base_msat + floor(millionths*x_msat / 10^6)
+ *  	
+ * into
+ * 
+ *  	fee_microsat = c_fee * x_sat
+ *  
+ *  use `base_fee_penalty` to weight the base fee and `delay_feefactor` to
+ *  weight the CTLV delay.
+ *  */
+s64 linear_fee_cost(
+		const struct gossmap_chan *c,
+		const int dir,
+		double base_fee_penalty,
+		double delay_feefactor)
+{
+	s64 pfee = c->half[dir].proportional_fee,
+	    bfee = c->half[dir].base_fee,
+	    delay = c->half[dir].delay;
+	
+	return pfee + (bfee + delay_feefactor*delay) * base_fee_penalty;
+}
+
+double flows_probability(struct flow **flows)
+{
+	double prob = 1.0;
+
+	for (size_t i = 0; i < tal_count(flows); i++)
+		prob *= flows[i]->success_prob;
+
+	return prob;
+}
+
+struct amount_msat flows_fee(struct flow **flows)
+{
+	struct amount_msat fee = AMOUNT_MSAT(0);
+
+	for (size_t i = 0; i < tal_count(flows); i++) {
+		struct amount_msat this_fee;
+		size_t n = tal_count(flows[i]->amounts);
+
+		if (!amount_msat_sub(&this_fee,
+				     flows[i]->amounts[0],
+				     flows[i]->amounts[n-1]))
+			abort();
+		if(!amount_msat_add(&fee, this_fee,fee))
+			abort();
+	}
+	return fee;
+}
+
+
 #ifndef SUPERVERBOSE_ENABLED
 #undef SUPERVERBOSE
 #endif
diff --git a/plugins/renepay/flow.h b/plugins/renepay/flow.h
index b76f1b786a8c..17897c61a0e7 100644
--- a/plugins/renepay/flow.h
+++ b/plugins/renepay/flow.h
@@ -104,6 +104,7 @@ double flow_edge_cost(const struct gossmap *gossmap,
 		      double basefee_penalty,
 		      double delay_riskfactor);
 
+// TODO(eduardo): check this
 /* Function to fill in amounts and success_prob for flow, and add to
  * chan_extra_map */
 void flow_complete(struct flow *flow,
@@ -116,6 +117,9 @@ void remove_completed_flow(const struct gossmap *gossmap,
 			   struct chan_extra_map *chan_extra_map,
 			   struct flow *flow);
 
+double flows_probability(struct flow **flows);
+struct amount_msat flows_fee(struct flow **flows);
+
 /*
  * mu (μ) is used as follows in the cost function:
  *
@@ -157,4 +161,11 @@ double derive_mu(const struct gossmap *gossmap,
 		 struct amount_msat amount,
 		 double frugality);
 
+// TODO(eduardo):
+s64 linear_fee_cost(
+		const struct gossmap_chan *c,
+		const int dir,
+		double base_fee_penalty,
+		double delay_feefactor);
+
 #endif /* LIGHTNING_PLUGINS_RENEPAY_FLOW_H */
diff --git a/plugins/renepay/mcf.c b/plugins/renepay/mcf.c
index f42038507da3..be6288e15da8 100644
--- a/plugins/renepay/mcf.c
+++ b/plugins/renepay/mcf.c
@@ -1,13 +1,186 @@
 #include "config.h"
+#include <ccan/list/list.h>
 #include <ccan/lqueue/lqueue.h>
 #include <ccan/tal/tal.h>
+#include <ccan/bitmap/bitmap.h>
 #include <plugins/renepay/mcf.h>
 #include <plugins/renepay/flow.h>
-#include <plugins/renepay/heap.h>
+#include <plugins/renepay/dijkstra.h>
 #include <stdint.h>
 #include <math.h>
 #include <assert.h>
 
+/* # Optimal payments
+ * 
+ * In this module we reduce the routing optimization problem to a linear
+ * cost optimization problem and find a solution using MCF algorithms.
+ * The optimization of the routing itself doesn't need a precise numerical
+ * solution, since we can be happy near optimal results; e.g. paying 100 msat or
+ * 101 msat for fees doesn't make any difference if we wish to deliver 1M sats.
+ * On the other hand, we are now also considering Pickhard's 
+ * [1] model to improve payment reliability,
+ * hence our optimization moves to a 2D space: either we like to maximize the
+ * probability of success of a payment or minimize the routing fees, or
+ * alternatively we construct a function of the two that gives a good compromise.
+ * 
+ * Therefore from now own, the definition of optimal is a matter of choice.
+ * To simplify the API of this module, we think the best way to state the
+ * problem is: 
+ * 
+ * 	Find a routing solution that pays the least of fees while keeping
+ * 	the probability of success above a certain value `min_probability`.
+ * 
+ *
+ * # Fee Cost
+ *
+ * Routing fees is non-linear function of the payment flow x, that's true even 
+ * without the base fee:
+ * 
+ * 	fee_msat = base_msat + floor(millionths*x_msat / 10^6)
+ *
+ * We approximate this fee into a linear function by computing a slope `c_fee` such
+ * that:
+ * 
+ * 	fee_microsat = c_fee * x_sat
+ * 	
+ * Function `linear_fee_cost` computes `c_fee` based on the base and
+ * proportional fees of a channel. 
+ * The final product if microsat because if only
+ * the proportional fee was considered we can have c_fee = millionths.
+ * Moving to costs based in msats means we have to either truncate payments
+ * below 1ksats or estimate as 0 cost for channels with less than 1000ppm.
+ * 
+ * TODO(eduardo): shall we build a linear cost function in msats?
+ * 
+ * # Probability cost
+ * 
+ * The probability of success P of the payment is the product of the prob. of
+ * success of forwarding parts of the payment over all routing channels. This
+ * problem is separable if we log it, and since we would like to increase P,
+ * then we can seek to minimize -log(P), and that's our prob. cost function [1].
+ * 
+ * 	- log P = sum_{i} - log P_i
+ * 
+ * The probability of success `P_i` of sending some flow `x` on a channel with
+ * liquidity l in the range a<=l<b is
+ * 
+ * 	P_{a,b}(x) = (b-x)/(b-a); for x > a
+ * 		   = 1.         ; for x <= a
+ * 
+ * Notice that unlike the similar formula in [1], the one we propose does not
+ * contain the quantization shot noise for counting states. The formula remains
+ * valid independently of the liquidity units (sats or msats).
+ * 
+ * The cost associated to probability P is then -k log P, where k is some
+ * constant. For k=1 we get the following table:
+ *
+ * 	prob | cost
+ * 	-----------
+ * 	0.01 | 4.6
+ * 	0.02 | 3.9
+ * 	0.05 | 3.0
+ * 	0.10 | 2.3
+ * 	0.20 | 1.6
+ * 	0.50 | 0.69
+ * 	0.80 | 0.22
+ * 	0.90 | 0.10
+ * 	0.95 | 0.05
+ * 	0.98 | 0.02
+ * 	0.99 | 0.01
+ * 	
+ * Clearly -log P(x) is non-linear; we try to linearize it piecewise:
+ * split the channel into 4 arcs representing 4 liquidity regions:
+ * 
+ * 	arc_0 -> [0, a)
+ * 	arc_1 -> [a, a+(b-a)*f1)
+ * 	arc_2 -> [a+(b-a)*f1, a+(b-a)*f2)
+ * 	arc_3 -> [a+(b-a)*f2, a+(b-a)*f3)
+ * 
+ * where f1 = 0.5, f2 = 0.8, f3 = 0.95;
+ * We fill arc_0's capacity with complete certainty P=1, then if more flow is
+ * needed we start filling the capacity in arc_1 until the total probability 
+ * of success reaches P=0.5, then arc_2 until P=1-0.8=0.2, and finally arc_3 until 
+ * P=1-0.95=0.05. We don't go further than 5% prob. of success per channel.
+ 
+ * TODO(eduardo): this channel linearization is hard coded into
+ * `CHANNEL_PIVOTS`, maybe we can parametrize this to take values from the config file.
+ * 
+ * With this choice, the slope of the linear cost function becomes:
+ * 
+ * 	m_0 = 0
+ * 	m_1 = 1.38 k /(b-a)
+ * 	m_2 = 3.05 k /(b-a)
+ * 	m_3 = 9.24 k /(b-a)
+ * 
+ * Notice that one of the assumptions in [2] for the MCF problem is that flows
+ * and the slope of the costs functions are integer numbers. The only way we
+ * have at hand to make it so, is to choose a universal value of `k` that scales
+ * up the slopes so that floor(m_i) is not zero for every arc.
+ *
+ * # Combine fee and prob. costs
+ * 
+ * We attempt to solve the original problem of finding the solution that
+ * pays the least fees while keeping the prob. of success above a certain value,
+ * by constructing a cost function which is a linear combination of fee and
+ * prob. costs.
+ * TODO(eduardo): investigate how this procedure is justified,
+ * possibly with the use of Lagrange optimization theory.
+ * 
+ * At first, prob. and fee costs live in different dimensions, they cannot be
+ * summed, it's like comparing apples and oranges.
+ * However we propose to scale the prob. cost by a global factor k that
+ * translates into the monetization of prob. cost.
+ * 
+ * k/1000, for instance, becomes the equivalent monetary cost
+ * of increasing the probability of success by 0.1% for P~100%.
+ * 
+ * The input parameter `prob_cost_factor` in the function `minflow` is defined
+ * as the PPM from the delivery amount `T` we are *willing to pay* to increase the
+ * prob. of success by 0.1%:
+ * 
+ * 	k_microsat = floor(1000*prob_cost_factor * T_sat)
+ * 	
+ * Is this enough to make integer prob. cost per unit flow?
+ * For `prob_cost_factor=10`; i.e. we pay 10ppm for increasing the prob. by
+ * 0.1%, we get that 
+ * 
+ * 	-> any arc with (b-a) > 10000 T, will have zero prob. cost, which is
+ * 	reasonable because even if all the flow passes through that arc, we get
+ * 	a 1.3 T/(b-a) ~ 0.01% prob. of failure at most.
+ * 	
+ * 	-> if (b-a) ~ 10000 T, then the arc will have unit cost, or just that we
+ * 	pay 1 microsat for every sat we send through this arc.
+ * 	
+ * 	-> it would be desirable to have a high proportional fee when (b-a)~T, 
+ * 	because prob. of failure start to become very high.
+ * 	In this case we get to pay 10000 microsats for every sat.
+ * 
+ * Once `k` is fixed then we can combine the linear prob. and fee costs, both
+ * are in monetary units.
+ * 
+ * Note: with costs in microsats, because slopes represent ppm and flows are in
+ * sats, then our integer bounds with 64 bits are such that we can move as many
+ * as 10'000 BTC without overflow:
+ * 
+ * 	10^6 (max ppm) * 10^8 (sats per BTC) * 10^4 = 10^18
+ *
+ * # References
+ * 
+ * [1] Pickhardt and Richter, https://arxiv.org/abs/2107.05322
+ * [2] R.K. Ahuja, T.L. Magnanti, and J.B. Orlin. Network Flows: 
+ * Theory, Algorithms, and Applications. Prentice Hall, 1993.
+ * 
+ * 	
+ * TODO(eduardo) it would be interesting to see:
+ * how much do we pay for reliability?
+ * Cost_fee(most reliable solution) - Cost_fee(cheapest solution)
+ * 
+ * TODO(eduardo): it would be interesting to see:
+ * how likely is the most reliable path with respect to the cheapest?
+ * Prob(reliable)/Prob(cheapest) = Exp(Cost_prob(cheapest)-Cost_prob(reliable))
+ * 
+ * */
+
 #define PARTS_BITS 2
 #define CHANNEL_PARTS (1 << PARTS_BITS)
 
@@ -21,15 +194,8 @@ static const size_t ARC_ADDITIONAL_BITS = PARTS_BITS + 2;
 
 static const s64 INFINITE = INT64_MAX;
 static const u32 INVALID_INDEX=0xffffffff;
-static const s64 COST_FACTOR=10;
 static const s64 MU_MAX = 128;
 
-struct queue_data
-{
-	u32 idx;
-	struct lqueue_link ql;
-};
-			
 /* Let's try this encoding of arcs:
  * Each channel `c` has two possible directions identified by a bit 
  * `half` or `!half`, and each one of them has to be
@@ -89,7 +255,6 @@ struct queue_data
  * I hope this will clarify my future self when I forget.
  * 
  * */
-
 typedef union
 {
 	struct{
@@ -107,146 +272,63 @@ typedef union
 struct pay_parameters {
 	/* The gossmap we are using */
 	struct gossmap *gossmap;
-	struct gossmap_node *source;
-	struct gossmap_node *target;
+	struct gossmap_node const*source;
+	struct gossmap_node const*target;
 	
 	/* Extra information we intuited about the channels */
 	struct chan_extra_map *chan_extra_map;
 	
-	/* Optional bitarray of disabled chans. */
-	// TODO(eduardo): did you consider these?, hint: disable channels by ignoring
-	// them when constructing the adjacency list of nodes in
-	// `init_residual_network`.
+	/* Optional bitarray of disabled channels. */
 	const bitmap *disabled;
 	
 	// how much we pay
 	struct amount_msat amount;
 	
+	// channel linearization parameters
+	double cap_fraction[CHANNEL_PARTS], 
+	       cost_fraction[CHANNEL_PARTS];
+	
+	struct amount_msat max_fee;
+	double min_probability;
+	double delay_feefactor;
+	double base_fee_penalty;
+	u32 prob_cost_factor;
+};
+
+/* Representation of the linear MCF network. 
+ * This contains the topology of the extended network (after linearization and
+ * addition of arc duality).
+ * This contains also the arc probability and linear fee cost, as well as
+ * capacity; these quantities remain constant during MCF execution. */
+struct linear_network
+{
 	u32 *arc_head_node; 
 	// notice that a tail node is not needed, 
 	// because the tail of arc is the head of dual(arc)
 	
-	// probability and fee cost associated to an arc
-	s64 *arc_prob_cost, *arc_fee_cost;
-	
 	arc_t *node_adjacency_next_arc;
 	arc_t *node_adjacency_first_arc;	
 	
-	// channel linearization parameters
-	double cap_fraction[CHANNEL_PARTS], 
-	       cost_fraction[CHANNEL_PARTS];
+	// probability and fee cost associated to an arc
+	s64 *arc_prob_cost, *arc_fee_cost;
+	s64 *capacity;
+	
+	size_t max_num_arcs,max_num_nodes;
 };
 
+/* This is the structure that keeps track of the network properties while we
+ * seek for a solution. */
 struct residual_network {
 	/* residual capacity on arcs */
 	s64 *cap; 
+	
+	/* some combination of prob. cost and fee cost on arcs */
 	s64 *cost;
+	
+	/* potential function on nodes */
 	s64 *potential;
 };
 
-/* In the heap we keep node idx, but in this structure we keep the distance
- * value associated to every node, and their position in the heap as a pointer
- * so that we can update the nodes inside the heap when the distance label is
- * changed. */
-struct dijkstra {
-	// 
-	s64 *distance;
-	u32 *base;
-	u32 **heapptr;
-	size_t heapsize;
-	struct gheap_ctx gheap_ctx;
-};
-/* Required a global dijkstra for gheap. */
-static struct dijkstra *global_dijkstra;
-
-static int dijkstra_less_comparer(
-	const void *const ctx UNUSED,
-	const void *const a,
-	const void *const b)
-{
-	return global_dijkstra->distance[*(u32*)a]
-		> global_dijkstra->distance[*(u32*)b];
-}
-static void dijkstra_item_mover(void *const dst, const void *const src)
-{
-	u32 src_idx = *(u32*)src;
-	*(u32*)dst = src_idx;
-	global_dijkstra->heapptr[src_idx] = dst;
-}
-static void dijkstra_malloc(const tal_t *ctx, const size_t max_num_nodes)
-{
-	global_dijkstra = tal(ctx,struct dijkstra);
-	global_dijkstra->distance = tal_arr(global_dijkstra,s64,max_num_nodes);
-	global_dijkstra->base = tal_arr(global_dijkstra,u32,max_num_nodes);
-	global_dijkstra->heapptr = tal_arr(global_dijkstra,u32*,max_num_nodes);
-	
-	global_dijkstra->heapsize=0;
-	
-	global_dijkstra->gheap_ctx.fanout=2;
-	global_dijkstra->gheap_ctx.page_chunks=1024;
-	global_dijkstra->gheap_ctx.item_size=sizeof(global_dijkstra->base[0]);
-	global_dijkstra->gheap_ctx.less_comparer=dijkstra_less_comparer;
-	global_dijkstra->gheap_ctx.less_comparer_ctx=NULL;
-	global_dijkstra->gheap_ctx.item_mover=dijkstra_item_mover;
-}
-static void dijkstra_init(void)
-{
-	const size_t max_num_nodes = tal_count(global_dijkstra->distance);
-	global_dijkstra->heapsize=0;
-	for(size_t i=0;i<max_num_nodes;++i)
-	{
-		global_dijkstra->distance[i]=INFINITE;
-		global_dijkstra->heapptr[i] = NULL;
-	}
-}
-static void dijkstra_append(u32 node_idx, s64 distance)
-{
-	const size_t pos = global_dijkstra->heapsize;
-	
-	global_dijkstra->base[pos]=node_idx;
-	global_dijkstra->distance[node_idx]=distance;
-	global_dijkstra->heapptr[node_idx] = &(global_dijkstra->base[pos]);
-	global_dijkstra->heapsize++;
-}
-static void dijkstra_update(u32 node_idx, s64 distance)
-{
-	if(!global_dijkstra->heapptr[node_idx])
-	{
-		// not in the heap
-		dijkstra_append(node_idx,distance);
-	}
-	
-	gheap_restore_heap_after_item_increase(
-		&global_dijkstra->gheap_ctx,
-		global_dijkstra->base,
-		global_dijkstra->heapsize,
-		global_dijkstra->heapptr[node_idx]
-			- global_dijkstra->base);
-}
-static u32 dijkstra_top(void)
-{
-	return global_dijkstra->base[0];
-}
-static bool dijkstra_empty(void)
-{
-	return global_dijkstra->heapsize==0;
-}
-static void dijkstra_pop(void)
-{
-	if(global_dijkstra->heapsize==0)
-		return;
-		
-	const u32 top = dijkstra_top();
-	assert(global_dijkstra->heapptr[top]==global_dijkstra->base);
-	
-	gheap_pop_heap(
-		&global_dijkstra->gheap_ctx,
-		global_dijkstra->base,
-		global_dijkstra->heapsize--);
-		
-	global_dijkstra->heapptr[top]=NULL;
-}
-
 /* Helper function. 
  * Given an arc idx, return the dual's idx in the residual network. */
 static arc_t arc_dual(arc_t arc)
@@ -263,54 +345,57 @@ static bool arc_is_dual(const arc_t arc)
 /* Helper function. 
  * Given an arc of the network (not residual) give me the flow. */
 static s64 get_arc_flow(
-			const struct residual_network *network,
-			const arc_t arc)
+		const struct residual_network *network,
+		const arc_t arc)
 {
 	assert(!arc_is_dual(arc));
+	assert(arc_dual(arc).idx < tal_count(network->cap));
 	return network->cap[ arc_dual(arc).idx ];
 }
 
 /* Helper function. 
  * Given an arc idx, return the node from which this arc emanates in the residual network. */
-static u32 arc_tail(const struct pay_parameters *params,
+static u32 arc_tail(const struct linear_network *linear_network,
                     const arc_t arc)
 {
-	return params->arc_head_node[ arc_dual(arc).idx ];
+	assert(arc_dual(arc).idx < tal_count(linear_network->arc_head_node));
+	return linear_network->arc_head_node[ arc_dual(arc).idx ];
 }
 /* Helper function. 
  * Given an arc idx, return the node that this arc is pointing to in the residual network. */
-static u32 arc_head(const struct pay_parameters *params,
+static u32 arc_head(const struct linear_network *linear_network,
                     const arc_t arc)
 {
-	return params->arc_head_node[arc.idx];
+	assert(arc.idx < tal_count(linear_network->arc_head_node));
+	return linear_network->arc_head_node[arc.idx];
 }
 		
 /* Helper function. 
  * Given node idx `node`, return the idx of the first arc whose tail is `node`.
  * */
-static arc_t node_adjacency_begin(const struct pay_parameters *params,
-                                const u32 node)
+static arc_t node_adjacency_begin(
+		const struct linear_network * linear_network,
+		const u32 node)
 {
-	return params->node_adjacency_first_arc[node];
+	assert(node < tal_count(linear_network->node_adjacency_first_arc));
+	return linear_network->node_adjacency_first_arc[node];
 }
 
 /* Helper function. 
  * Is this the end of the adjacency list. */
-static bool node_adjacency_end(
-	const struct pay_parameters *params UNUSED,
-	const u32 node UNUSED,
-	const arc_t arc)
+static bool node_adjacency_end(const arc_t arc)
 {
 	return arc.idx == INVALID_INDEX;
 }
 
 /* Helper function. 
  * Given node idx `node` and `arc`, returns the idx of the next arc whose tail is `node`. */
-static arc_t node_adjacency_next(const struct pay_parameters *params,
-                               const u32 node UNUSED,
-			       const arc_t arc)
+static arc_t node_adjacency_next(
+		const struct linear_network *linear_network,
+		const arc_t arc)
 {
-	return params->node_adjacency_next_arc[arc.idx];
+	assert(arc.idx < tal_count(linear_network->node_adjacency_next_arc));
+	return linear_network->node_adjacency_next_arc[arc.idx];
 }
 
 /* Helper function.
@@ -329,17 +414,7 @@ static arc_t channel_idx_to_arc(
 	return arc;
 }
 
-/* Helper function.
- * Evaluate Pickhardt's probability cost function. */
-static double pickhardt_probability_cost(
-	double lim_low, double lim_high, double flow)
-{
-	if(flow<=lim_low)
-		return 0.;
-	assert(flow<lim_high);
-	return -log(1.-(flow-lim_low)/(lim_high-lim_low));
-}
-
+// TODO(eduardo): unit test this
 /* Split a directed channel into parts with linear cost function. */
 static void linearize_channel(
 		const struct pay_parameters *params,
@@ -355,80 +430,106 @@ static void linearize_channel(
 							dir);
 	
 	s64 a = extra_half->known_min.millisatoshis/1000,
-	    b = extra_half->known_max.millisatoshis/1000;
+	    b = 1 + extra_half->known_max.millisatoshis/1000;
 	
 	capacity[0]=a;
 	cost[0]=0;
 	for(size_t i=1;i<CHANNEL_PARTS;++i)
 	{
 		capacity[i] = params->cap_fraction[i]*(b-a);
-		cost[i] = params->cost_fraction[i]*COST_FACTOR*params->amount.millisatoshis;
+		
+		cost[i] = params->cost_fraction[i]
+		          *params->amount.millisatoshis
+		          *params->prob_cost_factor;
 	}
 }
 
+static void alloc_residual_netork(
+		const struct linear_network * linear_network,
+		struct residual_network* residual_network)
+{
+	const size_t max_num_arcs = linear_network->max_num_arcs;
+	const size_t max_num_nodes = linear_network->max_num_nodes;
 	
-/* Get the fee cost associated to this directed channel. 
- * Cost is expressed as PPM of the payment. */
-static s64 compute_fee_cost(
-		const struct pay_parameters *params,
-		const struct gossmap_chan *c,
-		const int dir)
+	residual_network->cap = tal_arrz(residual_network,s64,max_num_arcs);
+	residual_network->cost = tal_arrz(residual_network,s64,max_num_arcs);	
+	residual_network->potential = tal_arrz(residual_network,s64,max_num_nodes);	
+}
+static void init_residual_netork(
+		const struct linear_network * linear_network,
+		struct residual_network* residual_network)
 {
-	s64 pfee = c->half[dir].proportional_fee,
-	    bfee = c->half[dir].base_fee;
-	
-	// Base fee to proportional fee. We want
-	// 	cost_eff(x) >= cost_real(x)
-	// 	x * (p + b*a) >= x * p + b*1000
-	// where the effective proportional cost is `(p+b*a)`, `a` is factor we
-	// need to choose. Then
-	// 	x >= 1000/a
-	// therefore if `a`=1, our effective cost is good only after 1000 sats,
-	// otherwise the cost is underestimated. If we want to make it valid
-	// from 1 sat, then a=1000, but that means that setting a base fee of
-	// 1msat will be consider like having a proportional cost of 1000ppm,
-	// which is a lot. Some middle ground can be obtained with numbers in
-	// between, but in general `a` does not make sense above 1000.
-	
-	// In this case having a base fee of 1 is equivalent of having a
-	// proportional fee of 10 ppm.
-	return pfee + bfee * 10;
+	const size_t max_num_arcs = linear_network->max_num_arcs;
+	const size_t max_num_nodes = linear_network->max_num_nodes;
+	for(u32 idx=0;idx<max_num_arcs;++idx)
+	{
+		arc_t arc = (arc_t){.idx=idx};
+		
+		if(arc_is_dual(arc))
+			continue;
+		
+		arc_t dual = arc_dual(arc); 
+		residual_network->cap[arc.idx]=linear_network->capacity[arc.idx];
+		residual_network->cap[dual.idx]=0;
+		
+		residual_network->cost[arc.idx]=residual_network->cost[dual.idx]=0;
+	}
+	for(u32 i=0;i<max_num_nodes;++i)
+	{
+		residual_network->potential[i]=0;
+	}
 }
 
-static void init_residual_network(struct pay_parameters *params,
-                                  struct residual_network *network)
+static void combine_cost_function(
+		const struct linear_network* linear_network,
+		struct residual_network *residual_network,
+		s64 mu)
+{
+	for(u32 arc_idx=0;arc_idx<linear_network->max_num_arcs;++arc_idx)
+	{
+		const s64 pcost = linear_network->arc_prob_cost[arc_idx],
+		          fcost = linear_network->arc_fee_cost[arc_idx];
+			  
+		const s64 combined = pcost==INFINITE || fcost==INFINITE ? INFINITE :
+		                     mu*fcost + (MU_MAX-1-mu)*pcost;
+			       
+		residual_network->cost[arc_idx]
+			= mu==0 ? pcost : 
+			          (mu==(MU_MAX-1) ? fcost : combined);
+	}
+}
+
+static void init_linear_network(
+		const struct pay_parameters *params,
+		struct linear_network *linear_network)
 {
 	const size_t max_num_chans = gossmap_max_chan_idx(params->gossmap);
 	const size_t max_num_arcs = max_num_chans << ARC_ADDITIONAL_BITS;
 	const size_t max_num_nodes = gossmap_max_node_idx(params->gossmap);
-
-	network->cap = tal_arrz(network,s64,max_num_arcs);
 	
-	network->cost = tal_arr(network,s64,max_num_arcs);
-	for(size_t i=0;i<tal_count(network->cost);++i)
-		network->cost[i]=INFINITE;
+	linear_network->max_num_arcs = max_num_arcs;
+	linear_network->max_num_nodes = max_num_nodes;
 	
-	network->potential = tal_arrz(network,s64,max_num_nodes);
-    
-	params->arc_head_node = tal_arr(params,u32,max_num_arcs);
-	for(size_t i=0;i<tal_count(params->arc_head_node);++i)
-		params->arc_head_node[i]=INVALID_INDEX;
+	linear_network->arc_head_node = tal_arr(linear_network,u32,max_num_arcs);
+	for(size_t i=0;i<tal_count(linear_network->arc_head_node);++i)
+		linear_network->arc_head_node[i]=INVALID_INDEX;
 	
-	params->arc_prob_cost = tal_arr(params,s64,max_num_arcs);
-	for(size_t i=0;i<tal_count(params->arc_prob_cost);++i)
-		params->arc_prob_cost[i]=INFINITE;
+	linear_network->node_adjacency_next_arc = tal_arr(linear_network,arc_t,max_num_arcs);
+	for(size_t i=0;i<tal_count(linear_network->node_adjacency_next_arc);++i)
+		linear_network->node_adjacency_next_arc[i].idx=INVALID_INDEX;
 	
-	params->arc_fee_cost = tal_arr(params,s64,max_num_arcs);
-	for(size_t i=0;i<tal_count(params->arc_fee_cost);++i)
-		params->arc_fee_cost[i]=INFINITE;
-		
-	params->node_adjacency_next_arc = tal_arr(params,arc_t,max_num_arcs);
-	for(size_t i=0;i<tal_count(params->node_adjacency_next_arc);++i)
-		params->node_adjacency_next_arc[i].idx=INVALID_INDEX;
+	linear_network->node_adjacency_first_arc = tal_arr(linear_network,arc_t,max_num_nodes);
+	for(size_t i=0;i<tal_count(linear_network->node_adjacency_first_arc);++i)
+		linear_network->node_adjacency_first_arc[i].idx=INVALID_INDEX;
+	
+	linear_network->arc_prob_cost = tal_arr(linear_network,s64,max_num_arcs);
+	for(size_t i=0;i<tal_count(linear_network->arc_prob_cost);++i)
+		linear_network->arc_prob_cost[i]=INFINITE;
 	
-	params->node_adjacency_first_arc = tal_arr(params,arc_t,max_num_nodes);
-	for(size_t i=0;i<tal_count(params->node_adjacency_first_arc);++i)
-		params->node_adjacency_first_arc[i].idx=INVALID_INDEX;
+	linear_network->arc_fee_cost = tal_arr(linear_network,s64,max_num_arcs);
+	for(size_t i=0;i<tal_count(linear_network->arc_fee_cost);++i)
+		linear_network->arc_fee_cost[i]=INFINITE;
+		
 	
 	for(struct gossmap_node *node = gossmap_first_node(params->gossmap);
 	    node;
@@ -443,10 +544,20 @@ static void init_residual_network(struct pay_parameters *params,
 			int half;
 			const struct gossmap_chan *c = gossmap_nth_chan(params->gossmap,
 			                                                node, j, &half);
+			
+			// TODO(eduardo): do we check if the channel is public?
+			if (!gossmap_chan_set(c,half))
+				continue;
+				
 			const u32 chan_id = gossmap_chan_idx(params->gossmap, c);
 			
+			if (params->disabled && bitmap_test_bit(params->disabled,chan_id))
+				continue;
+				
+			
 			const struct gossmap_node *next = gossmap_nth_node(params->gossmap,
 									   c,!half);
+									   
 			const u32 next_id = gossmap_node_idx(params->gossmap,next);
 			
 			if(node_id==next_id)
@@ -466,24 +577,28 @@ static void init_residual_network(struct pay_parameters *params,
 			for(size_t k=0;k<CHANNEL_PARTS;++k)
 			{
 				arc_t arc = channel_idx_to_arc(chan_id,half,k,0);
-				params->arc_head_node[arc.idx] = next_id;
+				linear_network->arc_head_node[arc.idx] = next_id;
 				
 				// Is this is the first arc?
 				if(prev_arc.idx==INVALID_INDEX)
 				// yes, then set it at the head of the linked list
 				{
-					params->node_adjacency_first_arc[node_id]=arc;
+					linear_network->node_adjacency_first_arc[node_id]=arc;
 				} else
 				// no, then link it to the previous arc
 				{
-					params->node_adjacency_next_arc[prev_arc.idx]=arc;
+					linear_network->node_adjacency_next_arc[prev_arc.idx]=arc;
 				}
 				
 				prev_arc = arc;
 				
-				network->cap[arc.idx] = capacity[k];
-				params->arc_prob_cost[arc.idx] = prob_cost[k];
-				params->arc_fee_cost[arc.idx] = compute_fee_cost(params,c,half);
+				linear_network->capacity[arc.idx] = capacity[k];
+				linear_network->arc_prob_cost[arc.idx] = prob_cost[k];
+				
+				linear_network->arc_fee_cost[arc.idx] 
+					= linear_fee_cost(c,half,
+							   params->base_fee_penalty,
+							   params->delay_feefactor);
 			}
 			
 			// split the opposite direction to obtain the dual arcs
@@ -495,42 +610,55 @@ static void init_residual_network(struct pay_parameters *params,
 			for(size_t k=0;k<CHANNEL_PARTS;++k)
 			{
 				arc_t arc = channel_idx_to_arc(chan_id,!half,k,1);
-				params->arc_head_node[arc.idx] = next_id;
+				linear_network->arc_head_node[arc.idx] = next_id;
 				
 				// Is this is the first arc?
 				if(prev_arc.idx==INVALID_INDEX)
 				// yes, then set it at the head of the linked list
 				{
-					params->node_adjacency_first_arc[node_id]=arc;
+					linear_network->node_adjacency_first_arc[node_id]=arc;
 				} else
 				// no, then link it to the previous arc
 				{
-					params->node_adjacency_next_arc[prev_arc.idx]=arc;
+					linear_network->node_adjacency_next_arc[prev_arc.idx]=arc;
 				}
 				
 				prev_arc = arc;
 				
-				network->cap[arc.idx] = 0;
-				params->arc_prob_cost[arc.idx] = -prob_cost[k];
-				params->arc_fee_cost[arc.idx] = -compute_fee_cost(params,c,!half);
+				linear_network->capacity[arc.idx] = 0;
+				linear_network->arc_prob_cost[arc.idx] = -prob_cost[k];
+				linear_network->arc_fee_cost[arc.idx] 
+					= -linear_fee_cost(c,!half,
+							    params->base_fee_penalty,
+					                    params->delay_feefactor);
 			}
 		}
 	}
 }
 
+/* Simple queue to traverse the network. */
+struct queue_data
+{
+	u32 idx;
+	struct lqueue_link ql;
+};
+
+// TODO(eduardo): unit test this
 /* Finds an admissible path from source to target, traversing arcs in the
  * residual network with capacity greater than 0. 
  * The path is encoded into prev, which contains the idx of the arcs that are
  * traversed. 
  * Returns RENEPAY_ERR_OK if the path exists. */
-static int find_admissible_path(const struct pay_parameters *params,
-			        const struct residual_network *network,
-                                const u32 source,
-				const u32 target,
-				arc_t *prev)
+static int find_admissible_path(
+		const struct linear_network *linear_network,
+		const struct residual_network *residual_network,
+                const u32 source,
+		const u32 target,
+		arc_t *prev)
 {
-	int ret = RENEPAY_ERR_NOFEASIBLEFLOW;
+	tal_t *this_ctx = tal(tmpctx,tal_t);
 	
+	int ret = RENEPAY_ERR_NOFEASIBLEFLOW;
 	for(size_t i=0;i<tal_count(prev);++i)
 		prev[i].idx=INVALID_INDEX;
 	
@@ -539,7 +667,7 @@ static int find_admissible_path(const struct pay_parameters *params,
 	LQUEUE(struct queue_data,ql) myqueue = LQUEUE_INIT;
 	struct queue_data *qdata;
 	
-	qdata = tal(tmpctx,struct queue_data);
+	qdata = tal(this_ctx,struct queue_data);
 	qdata->idx = source;
 	lqueue_enqueue(&myqueue,qdata);
 	
@@ -555,15 +683,17 @@ static int find_admissible_path(const struct pay_parameters *params,
 			break;
 		}
 		
-		for(arc_t arc = node_adjacency_begin(params,cur);
-		        !node_adjacency_end(params,cur,arc);
-			arc = node_adjacency_next(params,cur,arc))
+		for(arc_t arc = node_adjacency_begin(linear_network,cur);
+		        !node_adjacency_end(arc);
+			arc = node_adjacency_next(linear_network,arc))
 		{
 			// check if this arc is traversable
-			if(network->cap[arc.idx] <= 0)
+			if(residual_network->cap[arc.idx] <= 0)
 				continue;
 			
-			u32 next = arc_head(params,arc);
+			u32 next = arc_head(linear_network,arc);
+			
+			assert(next < tal_count(prev));
 			
 			// if that node has been seen previously
 			if(prev[next].idx!=INVALID_INDEX)
@@ -576,31 +706,31 @@ static int find_admissible_path(const struct pay_parameters *params,
 			lqueue_enqueue(&myqueue,qdata);
 		}
 	}
-	
+	tal_free(this_ctx);	
 	return ret;
 }
 
 /* Get the max amount of flow one can send from source to target along the path
  * encoded in `prev`. */
-static s64 get_augmenting_flow(const struct pay_parameters *params,
-			       const struct residual_network *network,
-	                       const u32 source,
-			       const u32 target,
-			       const arc_t *prev)
+static s64 get_augmenting_flow(
+		const struct linear_network* linear_network,
+		const struct residual_network *residual_network,
+	        const u32 source,
+		const u32 target,
+		const arc_t *prev)
 {
 	s64 flow = INFINITE;
 	
 	u32 cur = target;
 	while(cur!=source)
 	{
+		assert(cur<tal_count(prev));
 		const arc_t arc = prev[cur];
-		const arc_t dual = arc_dual(arc);
-		
-		flow = MIN(flow , network->cap[arc.idx]);
+		flow = MIN(flow , residual_network->cap[arc.idx]);
 		
 		// we are traversing in the opposite direction to the flow,
-		// hence the next node is at the head of the `dual` arc.
-		cur = arc_head(params,dual);
+		// hence the next node is at the tail of the arc.
+		cur = arc_tail(linear_network,arc);
 	}
 	
 	assert(flow<INFINITE && flow>0);
@@ -608,30 +738,38 @@ static s64 get_augmenting_flow(const struct pay_parameters *params,
 }
 
 /* Augment a `flow` amount along the path defined by `prev`.*/
-static void augment_flow(const struct pay_parameters *params,
-			 struct residual_network *network,
-	                 const u32 source,
-			 const u32 target,
-			 const arc_t *prev,
-			 s64 flow)
+static void augment_flow(
+		const struct linear_network *linear_network,
+		struct residual_network *residual_network,
+	        const u32 source,
+		const u32 target,
+		const arc_t *prev,
+		s64 flow)
 {
 	u32 cur = target;
 	
 	while(cur!=source)
 	{
+		assert(cur < tal_count(prev));
 		const arc_t arc = prev[cur];
 		const arc_t dual = arc_dual(arc);
 		
-		network->cap[arc.idx] -= flow;
-		assert(network->cap[arc.idx] >=0 );
-		network->cap[dual.idx] += flow;
+		assert(arc.idx < tal_count(residual_network->cap));
+		assert(dual.idx < tal_count(residual_network->cap));
+		
+		residual_network->cap[arc.idx] -= flow;
+		residual_network->cap[dual.idx] += flow;
+		
+		assert(residual_network->cap[arc.idx] >=0 );
 		
 		// we are traversing in the opposite direction to the flow,
-		// hence the next node is at the head of the `dual` arc.
-		cur = arc_head(params,dual);
+		// hence the next node is at the tail of the arc.
+		cur = arc_tail(linear_network,arc);
 	}
 }
 
+
+// TODO(eduardo): unit test this
 /* Finds any flow that satisfy the capacity and balance constraints of the
  * uncertainty network. For the balance function condition we have:
  * 	balance(source) = - balance(target) = amount
@@ -640,23 +778,28 @@ static void augment_flow(const struct pay_parameters *params,
  * 
  * 13/04/2023 This implementation uses a simple augmenting path approach.
  * */
-static int find_feasible_flow(const struct pay_parameters *params,
-			      struct residual_network *network,
-			      const u32 source, 
-			      const u32 target,
-			      s64 amount)
+static int find_feasible_flow(
+		const struct linear_network *linear_network,
+		struct residual_network *residual_network,
+		const u32 source, 
+		const u32 target,
+		s64 amount)
 {
+	assert(amount>=0);
+	
+	tal_t *this_ctx = tal(tmpctx,tal_t);
 	int ret = RENEPAY_ERR_OK;
 	
 	/* path information 
 	 * prev: is the id of the arc that lead to the node. */
-	arc_t *prev = tal_arr(tmpctx,arc_t,
-		            gossmap_max_node_idx(params->gossmap));
+	arc_t *prev = tal_arr(this_ctx,arc_t,linear_network->max_num_nodes);
 	
 	while(amount>0)
 	{
 		// find a path from source to target
-		int err = find_admissible_path(params,network,source,target,prev);
+		int err = find_admissible_path(
+					linear_network,
+					residual_network,source,target,prev);
 		if(err!=RENEPAY_ERR_OK)
 		{
 			ret = RENEPAY_ERR_NOFEASIBLEFLOW;
@@ -664,41 +807,42 @@ static int find_feasible_flow(const struct pay_parameters *params,
 		}
 		
 		// traverse the path and see how much flow we can send
-		s64 delta = get_augmenting_flow(params,network,source,target,prev);
+		s64 delta = get_augmenting_flow(linear_network,
+						residual_network,
+						source,target,prev);
 		
 		// commit that flow to the path
 		delta = MIN(amount,delta);
-		augment_flow(params,network,source,target,prev,delta);
+		augment_flow(linear_network,residual_network,source,target,prev,delta);
 		
 		assert(delta>0 && delta<=amount);
 		amount -= delta;
 	}
 	
-	tal_free(prev);	
-	
+	tal_free(this_ctx);	
 	return ret;
 }
-	
+
+// TODO(eduardo): unit test this
 /* Similar to `find_admissible_path` but use Dijkstra to optimize the distance
  * label. Stops when the target is hit. */
 static int  find_optimal_path(
-	const struct pay_parameters *params,
-	const struct residual_network* network,
-	const u32 source,
-	const u32 target,
-	arc_t *prev)
+		const struct linear_network *linear_network,
+		const struct residual_network* residual_network,
+		const u32 source,
+		const u32 target,
+		arc_t *prev)
 {
 	tal_t *this_ctx = tal(tmpctx,tal_t);
 	int ret = RENEPAY_ERR_NOFEASIBLEFLOW;
 	
-	char *visited = tal_arrz(this_ctx,char,tal_count(prev));
-	
+	bitmap *visited = tal_arrz(this_ctx, bitmap,
+					BITMAP_NWORDS(linear_network->max_num_nodes));
+
 	for(size_t i=0;i<tal_count(prev);++i)
-	{	
 		prev[i].idx=INVALID_INDEX;
-	}
 	
-	const s64 *const distance=global_dijkstra->distance;
+	s64 const * const distance=dijkstra_distance_data();
 	
 	dijkstra_init();
 	dijkstra_append(source,0);
@@ -708,10 +852,10 @@ static int  find_optimal_path(
 		u32 cur = dijkstra_top();
 		dijkstra_pop();
 		
-		if (visited[cur])
+		if(bitmap_test_bit(visited,cur))
 			continue;
 		
-		visited[cur]=1;
+		bitmap_set_bit(visited,cur);
 		
 		if(cur==target)
 		{
@@ -719,18 +863,19 @@ static int  find_optimal_path(
 			break;
 		}
 		
-		for(arc_t arc = node_adjacency_begin(params,cur);
-		        !node_adjacency_end(params,cur,arc);
-			arc = node_adjacency_next(params,cur,arc))
+		for(arc_t arc = node_adjacency_begin(linear_network,cur);
+		        !node_adjacency_end(arc);
+			arc = node_adjacency_next(linear_network,arc))
 		{
 			// check if this arc is traversable
-			if(network->cap[arc.idx] <= 0)
+			if(residual_network->cap[arc.idx] <= 0)
 				continue;
 			
-			u32 next = arc_head(params,arc);
+			u32 next = arc_head(linear_network,arc);
 			
-			s64 cij = network->cost[arc.idx] - network->potential[cur]
-			                                 + network->potential[next];
+			s64 cij = residual_network->cost[arc.idx] 
+					- residual_network->potential[cur] 
+					+ residual_network->potential[next];
 			
 			// Dijkstra only works with non-negative weights
 			assert(cij>=0);
@@ -748,26 +893,27 @@ static int  find_optimal_path(
     
 /* Set zero flow in the residual network. */
 static void zero_flow(
-	const struct pay_parameters *params,
-	struct residual_network *network)
+		const struct linear_network *linear_network,
+		struct residual_network *residual_network)
 {
-	for(u32 node=0;node<tal_count(network->potential);++node)
+	for(u32 node=0;node<linear_network->max_num_nodes;++node)
 	{
-		network->potential[node]=0;
-		for(arc_t arc=node_adjacency_begin(params,node);
-			  !node_adjacency_end(params,node,arc);
-			  arc = node_adjacency_next(params,node,arc))
+		residual_network->potential[node]=0;
+		for(arc_t arc=node_adjacency_begin(linear_network,node);
+			  !node_adjacency_end(arc);
+			  arc = node_adjacency_next(linear_network,arc))
 		{
 			if(arc_is_dual(arc))continue;
 			
 			arc_t dual = arc_dual(arc);
 			
-			network->cap[arc.idx] += network->cap[dual.idx];
-			network->cap[dual.idx] = 0;
+			residual_network->cap[arc.idx] = linear_network->capacity[arc.idx];
+			residual_network->cap[dual.idx] = 0;
 		}
 	}
 }
-    
+
+// TODO(eduardo): unit test this
 /* Starting from a feasible flow (satisfies the balance and capacity
  * constraints), find a solution that minimizes the network->cost function. 
  * 
@@ -776,27 +922,28 @@ static void zero_flow(
  * each step, we might use the previous flow result, which is not optimal in the
  * current iteration but I might be not too far from the truth.
  * It comes to mind to use cycle cancelling. */
-static int optimize_mcf(const struct pay_parameters *params,
-			 struct residual_network *network)
+static int optimize_mcf(
+		const struct linear_network *linear_network,
+		struct residual_network *residual_network,
+		const u32 source,
+		const u32 target,
+		const s64 amount)
 {
-	int ret = RENEPAY_ERR_OK;
-	
-	zero_flow(params,network);
-   	s64 amount = params->amount.millisatoshis/1000;
-	u32 source = gossmap_node_idx(params->gossmap,params->source),
-	    target = gossmap_node_idx(params->gossmap,params->target);
-	
+	assert(amount>=0);
 	tal_t *this_ctx = tal(tmpctx,tal_t);
 	
-	arc_t *prev = tal_arr(this_ctx,arc_t,tal_count(network->potential));
+	int ret = RENEPAY_ERR_OK;
 	
+	zero_flow(linear_network,residual_network);
+	arc_t *prev = tal_arr(this_ctx,arc_t,linear_network->max_num_nodes);
 	
-	// s64 *distance = tal_arr(this_ctx,s64,tal_count(network->potential));
-	const s64 *const distance = global_dijkstra->distance;
+	s64 const*const distance = dijkstra_distance_data();
 	
-	while(amount>0)
+	s64 remaining_amount = amount;
+	
+	while(remaining_amount>0)
 	{
-		int err = find_optimal_path(params,network,source,target,prev);
+		int err = find_optimal_path(linear_network,residual_network,source,target,prev);
 		if(err!=RENEPAY_ERR_OK)
 		{
 			// unexpected error
@@ -805,20 +952,20 @@ static int optimize_mcf(const struct pay_parameters *params,
 		}
 		
 		// traverse the path and see how much flow we can send
-		s64 delta = get_augmenting_flow(params,network,source,target,prev);
+		s64 delta = get_augmenting_flow(linear_network,residual_network,source,target,prev);
 		
 		// commit that flow to the path
-		delta = MIN(amount,delta);
-		augment_flow(params,network,source,target,prev,delta);
+		delta = MIN(remaining_amount,delta);
+		augment_flow(linear_network,residual_network,source,target,prev,delta);
 		
-		assert(delta>0 && delta<=amount);
-		amount -= delta;
+		assert(delta>0 && delta<=remaining_amount);
+		remaining_amount -= delta;
 		
 		// update potentials
-		for(u32 n=0;n<tal_count(network->potential);++n)
+		for(u32 n=0;n<linear_network->max_num_nodes;++n)
 		{
 			// see page 323 of Ahuja-Magnanti-Orlin
-			network->potential[n] -= MIN(distance[target],distance[n]);
+			residual_network->potential[n] -= MIN(distance[target],distance[n]);
 			
 			// Notice:
 			// if node i is permanently labeled we have
@@ -834,90 +981,109 @@ static int optimize_mcf(const struct pay_parameters *params,
 	tal_free(this_ctx);
 	return ret;
 }
-	
-/* Given a flow in the residual network, compute the probability and fee cost
- * ppm. 
- * Instead of using the cost function, which contains many approximations we
- * will use a direct approach to compute in O(n+m) the fees and prob. */
-static void estimate_costs(const struct pay_parameters *params,
-                           const struct residual_network *network,
-			   double *prob_ptr,
-			   double *fee_ppm_ptr)
+
+// flow on directed channels
+struct chan_flow
 {
-	double fee_microsats = 0;
-	double prob_cost = 0;
-	
-	for(struct gossmap_node *node = gossmap_first_node(params->gossmap);
-	    node;
-	    node=gossmap_next_node(params->gossmap,node))
+	s64 half[2];
+};
+
+/* Search in the network a path of positive flow until we reach a node with
+ * positive balance. */
+static u32 find_positive_balance(
+		const struct gossmap *gossmap,
+		const struct chan_flow *chan_flow,
+		const u32 start_idx,
+		const s64 *balance,
+		
+		struct gossmap_chan const** prev_chan,
+		int *prev_dir,
+		u32 *prev_idx)
+{
+	u32 final_idx = start_idx;
+	
+	/* TODO(eduardo)
+	 * This is guaranteed to halt if there are no directed flow cycles. 
+	 * There souldn't be any. In fact if cost is strickly
+	 * positive, then flow cycles do not exist at all in the
+	 * MCF solution. But if cost is allowed to be zero for
+	 * some arcs, then we might have flow cyles in the final
+	 * solution. We must somehow ensure that the MCF
+	 * algorithm does not come up with spurious flow cycles. */
+	while(balance[final_idx]<=0)
 	{
-		for(size_t j=0;j<node->num_chans;++j)
+		u32 updated_idx=INVALID_INDEX;
+		struct gossmap_node *cur
+			= gossmap_node_byidx(gossmap,final_idx);
+		
+		for(size_t i=0;i<cur->num_chans;++i)
 		{
 			int dir;
-			const struct gossmap_chan *c = gossmap_nth_chan(params->gossmap,
-			                                                node, j, &dir);
-			const u32 chan_idx = gossmap_chan_idx(params->gossmap, c);
+			struct gossmap_chan const *c 
+				= gossmap_nth_chan(gossmap,
+				                   cur,i,&dir);
 			
-			// in sats
-			s64 chan_flow=0;
+			// TODO(eduardo): do we check if the channel is public?
+			if (!gossmap_chan_set(c,dir))
+				continue;
 			
-			for(size_t k=0;k<CHANNEL_PARTS;++k)
+			const u32 c_idx = gossmap_chan_idx(gossmap,c);
+			
+			// follow the flow
+			if(chan_flow[c_idx].half[dir]>0)
 			{
-				const arc_t arc = channel_idx_to_arc(chan_idx,dir,k,0);
-				chan_flow += get_arc_flow(network,arc);	
+				const struct gossmap_node *next
+					= gossmap_nth_node(gossmap,c,!dir);
+				u32 next_idx = gossmap_node_idx(gossmap,next);
+				
+				
+				prev_dir[next_idx] = dir;
+				prev_chan[next_idx] = c;
+				prev_idx[next_idx] = final_idx;
+				
+				updated_idx = next_idx;
+				break;
 			}
-			
-			// TODO(eduardo): this is wrong, because the base fee is
-			// invoked for each HTLC, we may use a single payment
-			// channel in multiple payment flows.
-			fee_microsats += chan_flow * c->half[dir].proportional_fee 
-			                  + c->half[dir].base_fee*1e3; 
-			
-			struct chan_extra_half *extra_half 
-				= get_chan_extra_half_by_chan(
-						params->gossmap,
-						params->chan_extra_map,
-						c,
-						dir);  
-			
-			prob_cost += pickhardt_probability_cost(
-				extra_half->known_min.millisatoshis*1e3,
-				extra_half->known_max.millisatoshis*1e3,
-				chan_flow);
 		}
+		
+		assert(updated_idx!=INVALID_INDEX);
+		assert(updated_idx!=final_idx);
+		final_idx = updated_idx;
 	}
-	*prob_str = exp(-prob_cost);
-	*fee_ppm_str = fee_microsats * 1e3 / params->amount.millisatoshis;
+	return final_idx;
 }
 
-// flow on directed channels
-struct chan_flow
-{
-	s64 half[2];
-};
-
 struct list_data
 {
 	struct list_node list;
 	struct flow *flow_path;
 };
 
+// TODO(eduardo): check this
 /* Given a flow in the residual network, build a set of payment flows in the
  * gossmap that corresponds to this flow. */		
 static struct flow **
 	get_flow_paths(
 		const tal_t *ctx,
-		const struct pay_parameters *params,
-		const struct residual_network *network)
+		const struct gossmap *gossmap,
+		
+		// chan_extra_map cannot be const because we use it to keep
+		// track of htlcs and in_flight sats.
+		struct chan_extra_map *chan_extra_map,
+		const struct linear_network *linear_network,
+		const struct residual_network *residual_network)
 {
 	tal_t *this_ctx = tal(tmpctx,tal_t);
 	
-	const size_t max_num_chans = gossmap_max_chan_idx(params->gossmap);
+	const size_t max_num_chans = gossmap_max_chan_idx(gossmap);
 	struct chan_flow *chan_flow = tal_arrz(this_ctx,struct chan_flow,max_num_chans);
 	
-	const size_t max_num_nodes = gossmap_max_node_idx(params->gossmap);
+	const size_t max_num_nodes = gossmap_max_node_idx(gossmap);
 	s64 *balance = tal_arrz(this_ctx,s64,max_num_nodes);
-	struct gossmap_chan **prev_chan = tal_arr(this_ctx,struct gossmap_chan*,max_num_nodes);
+	
+	struct gossmap_chan const **prev_chan 
+		= tal_arr(this_ctx,struct gossmap_chan const*,max_num_nodes);
+	
 	int *prev_dir = tal_arr(this_ctx,int,max_num_nodes);
 	u32 *prev_idx = tal_arr(this_ctx,u32,max_num_nodes);
 	
@@ -926,14 +1092,14 @@ static struct flow **
 	// Compute balance on the nodes.
 	for(u32 n = 0;n<max_num_nodes;++n)
 	{
-		for(u32 arc = node_adjacency_begin(params,cur);
-		        !node_adjacency_end(params,cur,arc);
-			arc = node_adjacency_next(params,cur,arc))
+		for(arc_t arc = node_adjacency_begin(linear_network,n);
+		        !node_adjacency_end(arc);
+			arc = node_adjacency_next(linear_network,arc))
 		{
 			if(arc_is_dual(arc))
 				continue;
-			u32 m = arc_head(params,arc);
-			s64 flow = get_arc_flow(network,arc);
+			u32 m = arc_head(linear_network,arc);
+			s64 flow = get_arc_flow(residual_network,arc);
 			
 			balance[n] -= flow;
 			balance[m] += flow;
@@ -956,59 +1122,34 @@ static struct flow **
 		// this node has negative balance, flows leaves from here
 		while(balance[node_idx]<0)
 		{	
-			s64 delta=-balance[node_idx];
-			u32 final_idx;
 			
-			int length = 0;
 			prev_chan[node_idx]=NULL;
+			u32 final_idx = find_positive_balance(gossmap,chan_flow,node_idx,balance,
+							prev_chan,prev_dir,prev_idx);
 			
-			u32 final_idx = node_idx;
+			s64 delta=-balance[node_idx];
+			int length = 0;
+			delta = MIN(delta,balance[final_idx]);
 			
-			/* TODO(eduardo)
-			 * This is guaranteed to halt if there are no directed flow cycles. 
-			 * There souldn't be any. In fact if cost is strickly
-			 * positive, then flow cycles do not exist at all in the
-			 * MCF solution. But if cost is allowed to be zero for
-			 * some arcs, then we might have flow cyles in the final
-			 * solution. We must somehow ensure that the MCF
-			 * algorithm does not come up with spurious flow cycles. */
-			while(balance[final_idx]<=0)
+			// walk backwards, get me the length and the max flow we
+			// can send.
+			for(u32 cur_idx = final_idx;cur_idx!=node_idx;cur_idx=prev_idx[node_idx])
 			{
-				const struct gossmap_node *cur
-					= gossmap_node_byidx(params->gossmap,final_idx);
+				const int dir = prev_dir[cur_idx];
+				struct gossmap_chan const * const c = prev_chan[cur_idx];
+				const u32 c_idx = gossmap_chan_idx(gossmap,c);
 				
-				for(size_t i=0;i<cur->num_chans;++i)
-				{
-					int dir;
-					const struct gossmap_chan *c 
-						= gossmap_nth_chan(params->gossmap,
-						                   cur,i,&dir);
-					
-					const u32 c_idx = gossmap_chan_idx(params->gossmap,c);
-					
-					if(chan_flow[c_idx].half[dir]>0)
-					{
-						// follow the flow
-						const struct gossmap_node *next
-							= gossmap_nth_node(params->gossmap,
-									   c,!dir);
-						u32 next_idx = gossmap_node_idx(params->gossmap,next);
-						
-						delta=MIN(delta,chan_flow[c_idx].half[dir]);
-						
-						prev_dir[next_idx] = dir;
-						prev_chan[next_idx] = c;
-						prev_idx[next_idx] = final_idx;
-						length ++;
-						final_idx = next_idx;
-						break;
-					}
-				}
+				delta=MIN(delta,chan_flow[c_idx].half[dir]);
+				length++;
+				
+				// TODO(eduardo) does htlc_max has any relevance
+				// here?
+				delta = MIN(delta,chan_flow[c_idx].half[dir]);
 			}
-			delta = MIN(delta,balance[final_idx]);
 			
-			struct flow *fp = tal(ctx,struct flow);
-			fp->path = tal_arr(fp,struct gossmap_chan*,length);
+			
+			struct flow *fp = tal(list_ctx,struct flow);
+			fp->path = tal_arr(fp,struct gossmap_chan const*,length);
 			fp->dirs = tal_arr(fp,int,length);
 			
 			balance[node_idx] += delta;
@@ -1017,18 +1158,22 @@ static struct flow **
 			// walk backwards, substract flow
 			for(u32 cur_idx = final_idx;cur_idx!=node_idx;cur_idx=prev_idx[node_idx])
 			{
+				const int dir = prev_dir[cur_idx];
+				struct gossmap_chan const * const c = prev_chan[cur_idx];
+				const u32 c_idx = gossmap_chan_idx(gossmap,c);
+				
 				length--;
-				fp->path[length]=prev_chan[cur_idx];
-				fp->dirs[length]=prev_dir[cur_idx];
+				fp->path[length]=c;
+				fp->dirs[length]=dir;
+				// notice: fp->path and fp->dirs have the path
+				// in the correct order.
 				
-				chan_flow[prev_chan[cur_idx]].half[prev_dir[cur_idx]]-=delta;
+				chan_flow[c_idx].half[prev_dir[cur_idx]]-=delta;
 			}
 			
-			flow_complete(
-				fp,
-				params->gossmap,
-				params->chan_extra_map,
-				struct amount_msat(delta));
+			// complete the flow path by adding real fees and
+			// probabilities.
+			flow_complete(fp,gossmap,chan_extra_map,amount_msat(delta*1000));
 			
 			// add fp to list
 			ld = tal(list_ctx,struct list_data);
@@ -1043,132 +1188,51 @@ static struct flow **
 	size_t pos=0;
 	list_for_each(&path_list,ld,list)
 	{
-		flows[pos++] = ld->flow_path;
+		flows[pos++] = tal_steal(flows,ld->flow_path);
 	}
 	
 	tal_free(this_ctx);
 	return flows;
 }
 
-/* How to combine probability cost and fee cost?
- * This is tricky, because we say that a fee cost is high when the ratio of fee
- * and actual payment is high, typically 1000ppm or more. 
- * On the other hand a probability cost is high if the probability of success is
- * very low; since cost = - log prob, we have
- * 	prob | cost
- * 	-----------
- * 	0.01 | 4.6
- * 	0.02 | 3.9
- * 	0.05 | 3.0
- * 	0.10 | 2.3
- * 	0.20 | 1.6
- * 	0.50 | 0.69
- * 	0.80 | 0.22
- * 	0.90 | 0.10
- * 	0.95 | 0.05
- * 	0.98 | 0.02
- * 	0.99 | 0.01
- * 	
- * $ cost \approx 1-P $ when $ P > 0.9 $.
- * 
- * Interesting to see:
- * How much do we pay for reliability?
- * Cost_fee(most reliable solution) - Cost_fee(cheapest solution)
- * 
- * Interesting to see:
- * Is the most reliable path much more reliable than the cheapest?
- * Prob(reliable)/Prob(cheapest) = Exp(Cost_prob(cheapest)-Cost_prob(reliable))
- * 
- * When combining probability cost and fee cost, we need to understand that this
- * is like comparing apples to oranges. 
- * Probability cost is, for small values, equal to the probability of failure;
- * and we say this cost is acceptable if c(x) = 0.01 (1% probability of failure).
- * On the other hand fee cost is the money we pay for payment delivery and our
- * acceptance criteria depends on how much we want to send; so for instance a
- * fee cost is acceptable if c(x) = 100 T (we pay 100 ppm of T), 
- * where T is the total value we want to pay.
- * These numbers can be tuned; but the lesson is we evaluate probability cost on
- * its absolute value and fee cost relative to T. 
- * 
- * Notice also that the fee cost per unit flow is an integer number,
- * representing ppm (parts per million). On the other hand the first linear
- * arc with non zero prob. cost on a directed channel has a cost per unit flow
- * of around 1.3/(b-a), depending on the linearization partition (I prefer the
- * first partition to contain 50% of the (b-a) range, i.e. up to the point where
- * the probability of forwarding the flow reaches 50%), in essence this prob.
- * cost per unit of flow is a real number and we want to use algorithms (for
- * example cost scaling) that assume that cost is integer. 
- * 
- * How to solve both problems? 
- * Well, we know that we must combine both prob. cost and fee cost linearly and
- * we like the fact that fee cost is an integer. We can multiply the prob. cost
- * times a factor such that the prob. cost becomes becomes equivalent to the fee
- * cost, and approximate it to the nearest integer number in the process.
- * For instance, define a new prob. cost $ c_prob(x) = - 10^4 T log(P(x))$,
- * then when $1-P(x) \approx 1%$, i.e. the probability of failure is around 1%
- * is perceived as if we pay 100 ppm of the total payment T. 
- * 
- * Is this enough to make integer prob. cost per unit flow?
- * Let's see: 
- * 	-> consider an arc with $(b-a) >> 10^4 T$, recall for the first
- * 	partition $ c_prob(x) = x 1.3/(b-a) 10^4 T $, hence this arc has cost
- * 	per unit flow 0. Which is reasonable because T is very small and can be
- * 	sent through this arc without any issues.
- * 	
- * 	-> if $(b-a) \approx 10^4 T$ we obtain $c_prob(x) = x$ which is
- * 	equivalent to say we just pay 1 ppm for flows on this arc, which is non
- * 	zero but very small, and it is reasonable because $(b-a)$ is $10^4$
- * 	times bigger than T.
- * 	
- * 	-> if $(b-a)/2 \approx T$ then we start to enter a danger zone because
- * 	it is very likely that T cannot go through this arc with capacity $(b-a)/2$,
- * 	non-surprisingly the monetary cost associated to this case is
- * 	$c_prob(x) \approx 5000 x $, which is like 5000 ppm.
- * 	
- * 	-> for $(b-a)/2 < T$ we are paying an ever increasing price in ppm of T
- * 	per unit flow sent through this arc. The MCF algorithm will try keep
- * 	this fake monetary cost as low as possible.
- * 
- * Summary: we combine use as prob. cost the function $c_prob(x) = -10^4 T log(P(x))$,
- * and in this scheme we can approximate to integer numbers the cost per unit
- * flow, and 1% prob. of failure is equivalent to pay 100 ppm of the total
- * payment T. Then we can combine the fee cost $c_fee(x)$ and $c_prob(x)$ with
- * a factor mu ~ [0,1]. Instead of hard-coding the 10^4 factor, we use the
- * variable COST_FACTOR.
- * */
 
 
+// TODO(eduardo): choose some default values for the minflow parameters
 /* eduardo: I think it should be clear that this module deals with linear
  * flows, ie. base fees are not considered. Hence a flow along a path is
  * described with a sequence of directed channels and one amount. 
  * In the `pay_flow` module there are dedicated routes to compute the actual
  * amount to be forward on each hop. */
 struct flow** minflow(
-                      const tal_t *caller_ctx,
-		      struct gossmap *gossmap,
-		      const struct gossmap_node *source,
-		      const struct gossmap_node *target,
-		      struct chan_extra_map *chan_extra_map,
-		      const bitmap *disabled,
-		      struct amount_msat amount,
-		      struct amount_msat max_fee,
-		      double min_probability,
-		      double delay_feefactor UNUSED)
+		const tal_t *ctx,
+		struct gossmap *gossmap,
+		const struct gossmap_node *source,
+		const struct gossmap_node *target,
+		struct chan_extra_map *chan_extra_map,
+		const bitmap *disabled,
+		struct amount_msat amount,
+		struct amount_msat max_fee,
+		double min_probability,
+		double delay_feefactor,
+		double base_fee_penalty,
+		u32 prob_cost_factor )
 {
-	// TODO(eduardo) on which ctx should I allocate the MCF data?
 	tal_t *this_ctx = tal(tmpctx,tal_t);
 	
 	struct pay_parameters *params = tal(this_ctx,struct pay_parameters);	
 	
-	int ret;
-	
 	params->gossmap = gossmap;
 	params->source = source;
 	params->target = target;
 	params->chan_extra_map = chan_extra_map;
+	
 	params->disabled = disabled;
+	assert(!disabled
+	       || tal_bytelen(disabled) == bitmap_sizeof(gossmap_max_chan_idx(gossmap)));
+	       
 	params->amount = amount;
 	
+	// template the channel partition into linear arcs
 	params->cap_fraction[0]=0;
 	params->cost_fraction[0]=0;
 	for(size_t i =0;i<CHANNEL_PARTS;++i)
@@ -1179,92 +1243,98 @@ struct flow** minflow(
 			/params->cap_fraction[i];
 	}
 	
-	// TODO(eduardo): handle max_fee_ppm, delay_feefactor and basefee_penalty
-	// params->max_fee_ppm = max_fee_ppm;
-	// params->delay_feefactor = delay_feefactor;
-	// params->basefee_penalty = 
+	params->max_fee = max_fee;
+	params->min_probability = min_probability;
+	params->delay_feefactor = delay_feefactor;
+	params->base_fee_penalty = base_fee_penalty;
+	params->prob_cost_factor = prob_cost_factor;
 	
 	// build the uncertainty network with linearization and residual arcs
-	struct residual_network *network = tal(this_ctx,struct residual_network);
-	init_residual_network(params,network);
+	struct linear_network *linear_network= tal(this_ctx,struct linear_network);
+	init_linear_network(params,linear_network);
 	
-	dijkstra_malloc(this_ctx);
+	struct residual_network *residual_network = tal(this_ctx,struct residual_network);
+	alloc_residual_netork(linear_network,residual_network);
+	
+	dijkstra_malloc(this_ctx,gossmap_max_node_idx(params->gossmap));
 	
 	const u32 target_idx = gossmap_node_idx(params->gossmap,target);
 	const u32 source_idx = gossmap_node_idx(params->gossmap,source);
 	
-	int err = find_feasible_flow(params,network,source_idx,target_idx,
-	                             params->amount->millisatoshis/1000);
+	init_residual_netork(linear_network,residual_network);
+	
 	
+	struct amount_msat best_fee;
+	// double best_prob_success;
+	struct flow **best_flow_paths = NULL;
+	
+	int err = find_feasible_flow(linear_network,residual_network,source_idx,target_idx,
+	                             params->amount.millisatoshis/1000);
 	if(err!=RENEPAY_ERR_OK)
 	{
 		// there is no flow that satisfy the constraints, we stop here
-		ret = RENEPAY_ERR_NOFEASIBLEFLOW;
-		goto end;
+		goto fail;
 	}
 	
-	const size_t max_num_arcs = tal_count(params->arc_prob_cost);
-	assert(max_num_arcs==tal_count(params->arc_prob_cost));
-	assert(max_num_arcs==tal_count(params->arc_fee_cost));
-	assert(max_num_arcs==tal_count(network->cap));
-	assert(max_num_arcs==tal_count(network->cost));
+	// first flow found
+	best_flow_paths = get_flow_paths(ctx,params->gossmap,params->chan_extra_map,
+	                            linear_network,residual_network);
+	// best_prob_success = flows_probability(best_flow_paths);
+	best_fee = flows_fee(best_flow_paths);
+	
 	
 	// binary search for a value of `mu` that fits our fee and prob.
 	// constraints.
-	bool flow_found = false;
+	// mu=0 corresponds to only probabilities
+	// mu=MU_MAX-1 corresponds to only fee
 	s64 mu_left = 0, mu_right = MU_MAX;
-	ret=RENEPAY_ERR_NOCHEAPFLOW;
 	while(mu_left<mu_right)
 	{
+		
 		s64 mu = (mu_left + mu_right)/2;
 		
-		for(u32 arc=0;arc<max_num_arcs;++arc)
-		{
-			const s64 pcost = params->arc_prob_cost[arc],
-			          fcost = params->arc_fee_cost[arc];
-				  
-			const s64 combined = pcost==INFINITE || fcost==INFINITE ? INFINITE :
-			                     mu*fcost + (MU_MAX-1-mu)*pcost;
-				       
-			network->cost[arc]
-				= mu==0 ? pcost : 
-				          (mu==(MU_MAX-1) ? fcost : combined);
-		}
+		combine_cost_function(linear_network,residual_network,mu);
 		
-		optimize_mcf(params,network);
+		optimize_mcf(linear_network,residual_network,
+				source_idx,target_idx,params->amount.millisatoshis/1000);
 		
-		double prob_success,fee_ppm;
-		estimate_costs(params,network,&prob,&fee_ppm);
+		struct flow **flow_paths;
+		flow_paths = get_flow_paths(this_ctx,params->gossmap,params->chan_extra_map,
+		                            linear_network,residual_network);
 		
-		if(fee_ppm>max_fee_ppm)
+		double prob_success = flows_probability(flow_paths);
+		struct amount_msat fee = flows_fee(flow_paths);
+		
+		if(amount_msat_greater(fee,params->max_fee))
 		{
 			// too expensive
 			mu_left = mu+1;
-		}else if(prob<min_probability)
+		}else if(prob_success < params->min_probability)
 		{
 			// too unlikely
 			mu_right = mu;
 		}else
 		{
-			// a good compromise
-			ret=RENEPAY_ERR_OK;
-			break;
+			// with mu constraints are satisfied, now let's optimize
+			// the fees
+			mu_left = mu+1;
+			
+			if(!best_flow_paths || amount_msat_less(fee,best_fee))
+			{
+				best_flow_paths = tal_steal(ctx,flow_paths);
+				best_fee = fee;
+				// best_prob_success=prob_success;
+				flow_paths = NULL;
+			}
 		}
+		
+		if(flow_paths)
+			tal_free(flow_paths);
 	}
 	
-	end:
-	
-	struct flow **flow_paths = NULL;
-
-	if(ret==RENEPAY_ERR_OK)
-	{
-		flow_paths = get_flow_paths(caller_ctx,params,network);
-	}else
-	{
-	// TODO(eduardo) fallback to c_prob or c_fee?
-	}
+	fail:
 	
 	tal_free(this_ctx);
-	return flow_paths;
+	return best_flow_paths;
 }
 
diff --git a/plugins/renepay/mcf.h b/plugins/renepay/mcf.h
index 4eb5146527fa..009165646744 100644
--- a/plugins/renepay/mcf.h
+++ b/plugins/renepay/mcf.h
@@ -27,25 +27,44 @@ enum {
  * @chan_extra_map: hashtable of extra per-channel information
  * @disabled: NULL, or a bitmap by channel index of channels not to use.
  * @amount: the amount we want to reach @target
- * @max_fee: the maximum allowed in fees
+ * 
+ * @max_fee: the maximum allowed in fees 
+ * 
  * @min_probability: minimum probability accepted
- * @delay_feefactor: how important delays are compared to fees
- *
+ * 
  * @delay_feefactor converts 1 block delay into msat, as if it were an additional
  * fee.  So if a CTLV delay on a node is 5 blocks, that's treated as if it
  * were a fee of 5 * @delay_feefactor.
+ * 
+ * @base_fee_penalty: factor to compute additional proportional cost from each
+ * unit of base fee. So #base_fee_penalty will be added to the effective
+ * proportional fee for each msat of base fee.
+ * 	
+ * 	effective_ppm = proportional_fee + base_fee_msat * base_fee_penalty
+ *
+ * @prob_cost_factor: factor used to monetize the probability cost. It is
+ * defined as the number of ppm (parts per million of the total payment) we
+ * are willing to pay to improve the probability of success by 0.1%.
+ * 
+ * 	k_microsat = floor(1000*prob_cost_factor * payment_sat)
+ * 
+ * this k is used to compute a prob. cost in units of microsats
+ * 
+ * 	cost(payment) = - k_microsat * log Prob(payment)
  *
  * Return a series of subflows which deliver amount to target, or NULL.
  */
 struct flow** minflow(
-                      const tal_t *ctx,
-		      struct gossmap *gossmap,
-		      const struct gossmap_node *source,
-		      const struct gossmap_node *target,
-		      struct chan_extra_map *chan_extra_map,
-		      const bitmap *disabled,
-		      struct amount_msat amount,
-		      struct amount_msat max_fee,
-		      double min_probability,
-		      double delay_feefactor);
+		const tal_t *ctx,
+		struct gossmap *gossmap,
+		const struct gossmap_node *source,
+		const struct gossmap_node *target,
+		struct chan_extra_map *chan_extra_map,
+		const bitmap *disabled,
+		struct amount_msat amount,
+		struct amount_msat max_fee,
+		double min_probability,
+		double delay_feefactor,
+		double base_fee_penalty,
+		u32 prob_cost_factor);
 #endif /* LIGHTNING_PLUGINS_RENEPAY_MCF_H */
diff --git a/plugins/renepay/pay_flow.c b/plugins/renepay/pay_flow.c
index 4e569e98ca32..adf7395a57c8 100644
--- a/plugins/renepay/pay_flow.c
+++ b/plugins/renepay/pay_flow.c
@@ -259,34 +259,6 @@ static bitmap *make_disabled_bitmap(const tal_t *ctx,
 	return disabled;
 }
 
-static double flows_probability(struct flow **flows)
-{
-	// TODO
-	double prob = 1.0;
-
-	for (size_t i = 0; i < tal_count(flows); i++)
-		prob *= flows[i]->success_prob;
-
-	return prob;
-}
-
-static struct amount_msat flows_fee(struct flow **flows)
-{
-	// TODO
-	struct amount_msat fee = AMOUNT_MSAT(0);
-
-	for (size_t i = 0; i < tal_count(flows); i++) {
-		struct amount_msat this_fee;
-		size_t n = tal_count(flows[i]->amounts);
-
-		if (!amount_msat_sub(&this_fee,
-				     flows[i]->amounts[0],
-				     flows[i]->amounts[n-1]))
-			abort();
-		amount_msat_accumulate(&fee, this_fee);
-	}
-	return fee;
-}
 
 static u64 flows_worst_delay(struct flow **flows)
 {
@@ -363,6 +335,7 @@ static void remove_flows(const struct gossmap *gossmap,
 		remove_completed_flow(gossmap, chan_extra_map, flows[i]);
 }
 
+// TODO(eduardo): check this
 /* Get some payment flows to get this amount to destination, or NULL. */
 struct pay_flow **get_payflows(struct payment *p,
 			       struct amount_msat amount,
@@ -370,8 +343,8 @@ struct pay_flow **get_payflows(struct payment *p,
 			       bool unlikely_ok,
 			       bool is_entire_payment)
 {
-	double frugality = 1.0;
-	bool was_too_expensive = false;
+	// double frugality = 1.0;
+	// bool was_too_expensive = false;
 	bitmap *disabled;
 	struct pay_flow **pay_flows;
 	const struct gossmap_node *src, *dst;
@@ -401,7 +374,6 @@ struct pay_flow **get_payflows(struct payment *p,
 		bool too_unlikely, too_expensive, too_delayed;
 		const u32 *final_cltvs;
 		
-		// TODO(eduardo): set a maximum admissible fee (1%)?
 		struct amount_msat max_fee 
 			= (struct amount_msat){.millisatoshis = amount.millisatoshis/100};
 		
@@ -409,8 +381,12 @@ struct pay_flow **get_payflows(struct payment *p,
 		 * flows must be removed if not used! */
 		flows = minflow(tmpctx, pay_plugin->gossmap, src, dst,
 				&pay_plugin->chan_extra_map, disabled,
-				amount, max_fee ,/* min prob = */ 0.01 ,
-				p->delay_feefactor);
+				amount, 
+				max_fee ,
+				/* min probability = */ 0.1 ,
+				p->delay_feefactor,
+				/* base_fee_penalty = */ 1,
+				/* prob_cost_factor = */ 10);
 		if (!flows) {
 			paynote(p, "Failed to find any paths for %s",
 				type_to_string(tmpctx,
@@ -426,14 +402,20 @@ struct pay_flow **get_payflows(struct payment *p,
 
 		too_unlikely = (prob < 0.01);
 		if (too_unlikely)
+		{
 			paynote(p, "Flows too unlikely, P() = %f%%", prob * 100);
-
+			
+			if(!unlikely_ok)
+				goto fail_path;
+		}
 		too_expensive = amount_msat_greater(fee, feebudget);
 		if (too_expensive)
+		{
 			paynote(p, "Flows too expensive, fee = %s (max %s)",
 				type_to_string(tmpctx, struct amount_msat, &fee),
 				type_to_string(tmpctx, struct amount_msat, &feebudget));
-
+			goto fail_path;
+		}
 		too_delayed = (delay > p->maxdelay);
 		if (too_delayed) {
 			paynote(p, "Flows too delayed, delay = %"PRIu64" (max %u)",
@@ -447,48 +429,11 @@ struct pay_flow **get_payflows(struct payment *p,
 			p->delay_feefactor *= 2;
 			paynote(p, "Doubling delay_feefactor to %f",
 				p->delay_feefactor);
-		}
-
-		/* too expensive vs too unlikely is a tradeoff... */
-		if (too_expensive) {
-			if (too_unlikely && !unlikely_ok) {
-				paynote(p, "Giving up!");
-				goto fail_path;
-			}
-
-			/* Try increasing frugality? */
-			if (frugality >= 32) {
-				paynote(p, "Still too expensive, giving up!");
-				goto fail_path;
-			}
-			frugality *= 2;
-			was_too_expensive = true;
-			paynote(p, "... retry with frugality increased to %f",
-				frugality);
-			goto retry;
-		} else if (too_unlikely) {
-			/* Are we bouncing between "too expensive" and
-			 * "too unlikely"? */
-			if (was_too_expensive) {
-				paynote(p, "Bouncing between expensive and unlikely");
-				if (!unlikely_ok)
-					goto fail_path;
-				paynote(p, "... picking unlikely");
-				goto seems_ok;
-			}
-
-			/* Try decreasing frugality? */
-			if (frugality < 0.01) {
-				paynote(p, "Still too unlikely, giving up!");
-				goto fail_path;
-			}
-			frugality /= 2;
-			paynote(p, "... retry with frugality reduced to %f",
-				frugality);
+			
 			goto retry;
 		}
 
-	seems_ok:
+	// seems_ok:
 		/* Now we check for min/max htlc violations, and
 		 * excessive htlc counts.  It would be more efficient
 		 * to do this inside minflow(), but the diagnostics here

From 86aefc4e4569e615e09b3ffeba68edd9df14fd4f Mon Sep 17 00:00:00 2001
From: Lagrang3 <eduardo.quintana@pm.me>
Date: Wed, 10 May 2023 11:19:27 +0100
Subject: [PATCH 22/64] renepay: removing seg-fault bugs

---
 plugins/renepay/flow.c                     | 230 ++++++++++++++++-----
 plugins/renepay/flow.h                     |  18 +-
 plugins/renepay/mcf.c                      | 182 +++++++++-------
 plugins/renepay/pay_flow.c                 |   2 +-
 plugins/renepay/test/Makefile              |   3 +-
 plugins/renepay/test/run-map.c             | 217 +++++++++++++++++++
 plugins/renepay/test/run-not_mcf-diamond.c |  40 ++--
 plugins/renepay/test/run-not_mcf-gossmap.c |  14 +-
 plugins/renepay/test/run-not_mcf.c         |  25 ++-
 9 files changed, 568 insertions(+), 163 deletions(-)
 create mode 100644 plugins/renepay/test/run-map.c

diff --git a/plugins/renepay/flow.c b/plugins/renepay/flow.c
index 5cabc7970370..e2ac7c359c6e 100644
--- a/plugins/renepay/flow.c
+++ b/plugins/renepay/flow.c
@@ -3,6 +3,7 @@
 #include <ccan/asort/asort.h>
 #include <common/gossmap.h>
 #include <math.h>
+#include <stdio.h>
 #include <plugins/renepay/flow.h>
 
 #ifndef SUPERVERBOSE
@@ -56,17 +57,25 @@ static double edge_probability(struct amount_msat min, struct amount_msat max,
 			       struct amount_msat in_flight,
 			       struct amount_msat f)
 {
-	struct amount_msat B; // =  max +1 - in_flight
+	printf("%s: with min=%ld, max=%ld, in_flight=%ld, flow=%ld\n",
+		__PRETTY_FUNCTION__,min.millisatoshis,max.millisatoshis,
+		in_flight.millisatoshis,f.millisatoshis);
+	
+	struct amount_msat B=max; // =  max +1 - in_flight
 	
 	// one past the last known value, makes computations simpler
 	if(!amount_msat_add(&B,B,AMOUNT_MSAT(1)))
+	{
+		printf("%s: aborting, lineno=%d\n",__PRETTY_FUNCTION__,__LINE__);
 		abort();
-		
+	}	
 	// in_flight cannot be greater than max
 	if(!amount_msat_sub(&B,B,in_flight))
+	{
+		printf("%s: aborting, lineno=%d\n",__PRETTY_FUNCTION__,__LINE__);
 		abort();
-		
-	struct amount_msat A; // = MAX(0,min-in_flight);
+	}	
+	struct amount_msat A=min; // = MAX(0,min-in_flight);
 	
 	if(!amount_msat_sub(&A,A,in_flight))
 		A = AMOUNT_MSAT(0);
@@ -75,8 +84,10 @@ static double edge_probability(struct amount_msat min, struct amount_msat max,
 	
 	// B cannot be smaller than or equal A
 	if(!amount_msat_sub(&denominator,B,A) || amount_msat_less_eq(B,A))
+	{
+		printf("%s: aborting, lineno=%d\n",__PRETTY_FUNCTION__,__LINE__);
 		abort();
-	
+	}
 	struct amount_msat numerator; // MAX(0,B-f)
 	
 	if(!amount_msat_sub(&numerator,B,f))
@@ -101,11 +112,11 @@ bool flow_path_eq(const struct gossmap_chan **path1,
 	return true;
 }
 
-static void destroy_chan_extra(struct chan_extra *ce,
-			       struct chan_extra_map *chan_extra_map)
-{
-	chan_extra_map_del(chan_extra_map, ce);
-}
+// static void destroy_chan_extra(struct chan_extra *ce,
+// 			       struct chan_extra_map *chan_extra_map)
+// {
+// 	chan_extra_map_del(chan_extra_map, ce);
+// }
 
 /* Returns either NULL, or an entry from the hash */
 struct chan_extra_half *
@@ -133,6 +144,34 @@ get_chan_extra_half_by_chan(const struct gossmap *gossmap,
 					   dir);
 }
 
+/* Helper to get the chan_extra_half. If it doesn't exist create a new one. */
+struct chan_extra_half *
+get_chan_extra_half_by_chan_verify(
+		const struct gossmap *gossmap,
+		struct chan_extra_map *chan_extra_map,
+		const struct gossmap_chan *chan,
+		int dir)
+{
+	
+	const struct short_channel_id scid = gossmap_chan_scid(gossmap,chan);
+	struct chan_extra_half *h = get_chan_extra_half_by_scid(
+					chan_extra_map,scid,dir);
+	if (!h) {
+		struct amount_sat cap;
+		struct amount_msat cap_msat;
+
+		if (!gossmap_chan_get_capacity(gossmap,chan, &cap))
+			cap = AMOUNT_SAT(0);
+		if (!amount_sat_to_msat(&cap_msat, cap))
+		{
+			abort();
+		}
+		h = new_chan_extra_half(chan_extra_map,
+					scid,dir,cap_msat);
+	}
+	return h;
+}
+
 struct chan_extra_half *new_chan_extra_half(struct chan_extra_map *chan_extra_map,
 					    const struct short_channel_id scid,
 					    int dir,
@@ -149,7 +188,12 @@ struct chan_extra_half *new_chan_extra_half(struct chan_extra_map *chan_extra_ma
 	}
 	/* Remove self from map when done */
 	chan_extra_map_add(chan_extra_map, ce);
-	tal_add_destructor2(ce, destroy_chan_extra, chan_extra_map);
+	
+	// TODO(eduardo):
+	// Is this desctructor really necessary? the chan_extra will deallocated
+	// when the chan_extra_map is freed. Anyways valgrind complains that the
+	// hash table is removing the element with a freed pointer.
+	// tal_add_destructor2(ce, destroy_chan_extra, chan_extra_map);
 	return &ce->half[dir];
 }
 
@@ -163,9 +207,15 @@ void remove_completed_flow(const struct gossmap *gossmap,
 							       flow->path[i],
 							       flow->dirs[i]);
 		if (!amount_msat_sub(&h->htlc_total, h->htlc_total, flow->amounts[i]))
+		{
+			printf("%s: aborting\n",__PRETTY_FUNCTION__);
 			abort();
+		}
 		if (h->num_htlcs == 0)
+		{
+			printf("%s: aborting\n",__PRETTY_FUNCTION__);
 			abort();
+		}
 		h->num_htlcs--;
 	}
 }
@@ -181,8 +231,10 @@ void flow_add(struct flow *flow,
 	/* Add in new amount */
 	if (!amount_msat_add(&delivered,
 			     flow->amounts[tal_count(flow->amounts)-1], additional))
+	{
+		printf("%s: aborting\n",__PRETTY_FUNCTION__);
 		abort();
-
+	}
 	/* Remove original from current_flows */
 	remove_completed_flow(gossmap, chan_extra_map, flow);
 
@@ -245,8 +297,15 @@ void flow_add(struct flow *flow,
 // 	return certainty_term + feerate_term;
 // }
 
-// TODO(eduardo): check this
-/* Helper function to fill in amounts and success_prob for flow */
+/* Helper function to fill in amounts and success_prob for flow 
+ * 
+ * IMPORTANT: here we do not commit flows to chan_extra, flows are commited
+ * after we send those htlc.
+ * 
+ * IMPORTANT: flow->success_prob is misleading, because that's the prob. of
+ * success provided that there are no other flows in the current MPP flow set.
+ * 
+ * */
 void flow_complete(struct flow *flow,
 		   const struct gossmap *gossmap,
 		   struct chan_extra_map *chan_extra_map,
@@ -255,42 +314,119 @@ void flow_complete(struct flow *flow,
 	flow->success_prob = 1.0;
 	flow->amounts = tal_arr(flow, struct amount_msat, tal_count(flow->path));
 	for (int i = tal_count(flow->path) - 1; i >= 0; i--) {
-		struct chan_extra_half *h;
-
-		h = get_chan_extra_half_by_chan(gossmap, chan_extra_map,
-						flow->path[i], flow->dirs[i]);
-		if (!h) {
-			struct amount_sat cap;
-			struct amount_msat cap_msat;
-
-			if (!gossmap_chan_get_capacity(gossmap,
-						       flow->path[i], &cap))
-				cap = AMOUNT_SAT(0);
-			if (!amount_sat_to_msat(&cap_msat, cap))
-				abort();
-			h = new_chan_extra_half(chan_extra_map,
-						gossmap_chan_scid(gossmap,
-								  flow->path[i]),
-						flow->dirs[i],
-						cap_msat);
-		}
-
+		const struct chan_extra_half *h
+			= get_chan_extra_half_by_chan_verify(gossmap,
+							chan_extra_map,
+							flow->path[i],
+							flow->dirs[i]);
+		
 		flow->amounts[i] = delivered;
 		flow->success_prob
 			*= edge_probability(h->known_min, h->known_max,
 					    h->htlc_total,
 					    delivered);
 		
-		if (!amount_msat_add(&h->htlc_total, h->htlc_total, delivered))
-			abort();
-		h->num_htlcs++;
+		// if (!amount_msat_add(&h->htlc_total, h->htlc_total, delivered))
+		// {
+		// 	printf("%s: aborting\n",__PRETTY_FUNCTION__);
+		// 	abort();
+		// }
+		// h->num_htlcs++;
 		if (!amount_msat_add_fee(&delivered,
 					 flow_edge(flow, i)->base_fee,
 					 flow_edge(flow, i)->proportional_fee))
+		{
+			printf("%s: aborting\n",__PRETTY_FUNCTION__);
 			abort();
+		}
 	}
 }
 
+/* Compute the prob. of success of a set of concurrent set of flows. 
+ * 
+ * IMPORTANT: this is not simply the multiplication of the prob. of success of
+ * all of them, because they're not independent events. A flow that passes
+ * through a channel c changes that channel's liquidity and then if another flow
+ * passes through that same channel the previous liquidity change must be taken
+ * into account.
+ * 
+ * 	P(A and B) != P(A) * P(B),
+ * 	
+ * but	
+ * 	
+ * 	P(A and B) = P(A) * P(B | A) 
+ * 
+ * also due to the linear form of P() we have
+ * 
+ * 	P(A and B) = P(A + B)
+ * 	*/
+struct chan_inflight_flow
+{
+	struct amount_msat half[2];
+};
+
+// TODO(eduardo): here chan_extra_map should be const
+// TODO(eduardo): here flows should be const
+double flow_set_probability(
+		struct flow ** flows,
+		struct gossmap const*const gossmap,
+		struct chan_extra_map * chan_extra_map)
+{
+	tal_t *this_ctx = tal(tmpctx,tal_t);
+	double prob = 1.0;
+	
+	// TODO(eduardo): should it be better to use a map instead of an array
+	// here?
+	const size_t max_num_chans= gossmap_max_chan_idx(gossmap);
+	struct chan_inflight_flow *in_flight 
+		= tal_arr(this_ctx,struct chan_inflight_flow,max_num_chans);
+	
+	for(size_t i=0;i<max_num_chans;++i)
+	{
+		in_flight[i].half[0]=in_flight[i].half[1]=AMOUNT_MSAT(0);
+	}
+	
+	for(size_t i=0;i<tal_count(flows);++i)
+	{
+		const struct flow* f = flows[i];
+		for(size_t j=0;j<tal_count(f->path);++j)
+		{
+			const struct chan_extra_half *h
+				= get_chan_extra_half_by_chan(
+						gossmap,
+						chan_extra_map,
+						f->path[j],
+						f->dirs[j]);
+			assert(h);
+			
+			const u32 c_idx = gossmap_chan_idx(gossmap,f->path[j]);
+			const int c_dir = f->dirs[j];
+			
+			const struct amount_msat deliver = f->amounts[j];
+			
+			struct amount_msat prev_flow;
+			if(!amount_msat_add(&prev_flow,h->htlc_total,in_flight[c_idx].half[c_dir]))
+			{
+				printf("%s: aborting\n",__PRETTY_FUNCTION__);
+				abort();
+			}
+			
+			prob *= edge_probability(h->known_min,h->known_max,
+						 prev_flow,deliver);
+			
+			if(!amount_msat_add(&in_flight[c_idx].half[c_dir],
+					in_flight[c_idx].half[c_dir],
+					deliver))
+			{
+				printf("%s: aborting\n",__PRETTY_FUNCTION__);
+				abort();
+			}
+		}
+	}
+	tal_free(this_ctx);
+	return prob;
+}
+
 static int cmp_amount_msat(const struct amount_msat *a,
 			   const struct amount_msat *b,
 			   void *unused)
@@ -362,8 +498,10 @@ static void get_medians(const struct gossmap *gossmap,
 	if (!num_caps)
 		*median_capacity = amount;
 	else if (!amount_sat_to_msat(median_capacity, caps[num_caps / 2]))
+	{
+		printf("%s: aborting\n",__PRETTY_FUNCTION__);
 		abort();
-
+	}
 	asort(fees, num_fees, cmp_amount_msat, NULL);
 	if (!num_caps)
 		*median_fee = AMOUNT_MSAT(0);
@@ -415,16 +553,6 @@ s64 linear_fee_cost(
 	return pfee + (bfee + delay_feefactor*delay) * base_fee_penalty;
 }
 
-double flows_probability(struct flow **flows)
-{
-	double prob = 1.0;
-
-	for (size_t i = 0; i < tal_count(flows); i++)
-		prob *= flows[i]->success_prob;
-
-	return prob;
-}
-
 struct amount_msat flows_fee(struct flow **flows)
 {
 	struct amount_msat fee = AMOUNT_MSAT(0);
@@ -436,9 +564,15 @@ struct amount_msat flows_fee(struct flow **flows)
 		if (!amount_msat_sub(&this_fee,
 				     flows[i]->amounts[0],
 				     flows[i]->amounts[n-1]))
+		{
+			printf("%s: aborting\n",__PRETTY_FUNCTION__);
 			abort();
+		}
 		if(!amount_msat_add(&fee, this_fee,fee))
+		{
+			printf("%s: aborting\n",__PRETTY_FUNCTION__);
 			abort();
+		}
 	}
 	return fee;
 }
diff --git a/plugins/renepay/flow.h b/plugins/renepay/flow.h
index 17897c61a0e7..f198a025b353 100644
--- a/plugins/renepay/flow.h
+++ b/plugins/renepay/flow.h
@@ -56,6 +56,13 @@ struct chan_extra_half *get_chan_extra_half_by_chan(const struct gossmap *gossma
 						    struct chan_extra_map *chan_extra_map,
 						    const struct gossmap_chan *chan,
 						    int dir);
+/* Helper to get the chan_extra_half. If it doesn't exist create a new one. */
+struct chan_extra_half *
+get_chan_extra_half_by_chan_verify(
+		const struct gossmap *gossmap,
+		struct chan_extra_map *chan_extra_map,
+		const struct gossmap_chan *chan,
+		int dir);
 
 /* tal_free() this removes it from chan_extra_map */
 struct chan_extra_half *new_chan_extra_half(struct chan_extra_map *chan_extra_map,
@@ -104,20 +111,23 @@ double flow_edge_cost(const struct gossmap *gossmap,
 		      double basefee_penalty,
 		      double delay_riskfactor);
 
-// TODO(eduardo): check this
-/* Function to fill in amounts and success_prob for flow, and add to
- * chan_extra_map */
+/* Function to fill in amounts and success_prob for flow. */
 void flow_complete(struct flow *flow,
 		   const struct gossmap *gossmap,
 		   struct chan_extra_map *chan_extra_map,
 		   struct amount_msat delivered);
 
+/* Compute the prob. of success of a set of concurrent set of flows. */ 
+double flow_set_probability(
+		struct flow ** flows,
+		struct gossmap const*const gossmap,
+		struct chan_extra_map * chan_extra_map);
+
 /* Once flow is completed, this can remove it from the extra_map */
 void remove_completed_flow(const struct gossmap *gossmap,
 			   struct chan_extra_map *chan_extra_map,
 			   struct flow *flow);
 
-double flows_probability(struct flow **flows);
 struct amount_msat flows_fee(struct flow **flows);
 
 /*
diff --git a/plugins/renepay/mcf.c b/plugins/renepay/mcf.c
index be6288e15da8..9c4ad8186a88 100644
--- a/plugins/renepay/mcf.c
+++ b/plugins/renepay/mcf.c
@@ -217,37 +217,37 @@ static const s64 MU_MAX = 128;
  * the outgoing arcs. If we ever need to loop over the
  * incoming arcs then we will define a reverse adjacency
  * API. 
- * Then for each outgoing channel `(c,!half)` there will
+ * Then for each outgoing channel `(c,half)` there will
  * be 4 parts for the actual residual capacity, hence
  * with the dual bit set to 0:
  * 
- * 	(c,!half,0,0)
- * 	(c,!half,1,0)
- * 	(c,!half,2,0)
- * 	(c,!half,3,0)
+ * 	(c,half,0,0)
+ * 	(c,half,1,0)
+ * 	(c,half,2,0)
+ * 	(c,half,3,0)
  * 	
  * and also we need to consider the dual arcs
- * corresponding to the channel direction `(c,half)`
+ * corresponding to the channel direction `(c,!half)`
  * (the dual has reverse direction):
  * 
- * 	(c,half,0,1)
- * 	(c,half,1,1)
- * 	(c,half,2,1)
- * 	(c,half,3,1)
+ * 	(c,!half,0,1)
+ * 	(c,!half,1,1)
+ * 	(c,!half,2,1)
+ * 	(c,!half,3,1)
  *  
  * These are the 8 outgoing arcs relative to `node` and
  * associated with channel `c`. The incoming arcs will
  * be:
  * 
- * 	(c,half,0,0)
- * 	(c,half,1,0)
- * 	(c,half,2,0)
- * 	(c,half,3,0)
+ * 	(c,!half,0,0)
+ * 	(c,!half,1,0)
+ * 	(c,!half,2,0)
+ * 	(c,!half,3,0)
  * 
- * 	(c,!half,0,1)
- * 	(c,!half,1,1)
- * 	(c,!half,2,1)
- * 	(c,!half,3,1)
+ * 	(c,half,0,1)
+ * 	(c,half,1,1)
+ * 	(c,half,2,1)
+ * 	(c,half,3,1)
  * 	
  * but they will be stored as outgoing arcs on the peer
  * node `next`.
@@ -302,7 +302,7 @@ struct pay_parameters {
  * capacity; these quantities remain constant during MCF execution. */
 struct linear_network
 {
-	u32 *arc_head_node; 
+	u32 *arc_tail_node; 
 	// notice that a tail node is not needed, 
 	// because the tail of arc is the head of dual(arc)
 	
@@ -358,16 +358,17 @@ static s64 get_arc_flow(
 static u32 arc_tail(const struct linear_network *linear_network,
                     const arc_t arc)
 {
-	assert(arc_dual(arc).idx < tal_count(linear_network->arc_head_node));
-	return linear_network->arc_head_node[ arc_dual(arc).idx ];
+	assert(arc.idx < tal_count(linear_network->arc_tail_node));
+	return linear_network->arc_tail_node[ arc.idx ];
 }
 /* Helper function. 
  * Given an arc idx, return the node that this arc is pointing to in the residual network. */
 static u32 arc_head(const struct linear_network *linear_network,
                     const arc_t arc)
 {
-	assert(arc.idx < tal_count(linear_network->arc_head_node));
-	return linear_network->arc_head_node[arc.idx];
+	const arc_t dual = arc_dual(arc);
+	assert(dual.idx < tal_count(linear_network->arc_tail_node));
+	return linear_network->arc_tail_node[dual.idx];
 }
 		
 /* Helper function. 
@@ -407,6 +408,7 @@ static arc_t channel_idx_to_arc(
 		int dual)
 {
 	arc_t arc;
+	// arc.idx=0; // shouldn't be necessary, but valgrind complains of uninitialized field idx
 	arc.dual=dual;
 	arc.part=part;
 	arc.chandir=half;
@@ -499,6 +501,25 @@ static void combine_cost_function(
 	}
 }
 
+static void linear_network_add_adjacenct_arc(
+		struct linear_network *linear_network,
+		const u32 node_idx,
+		const arc_t arc)
+{
+	assert(arc.idx < tal_count(linear_network->arc_tail_node));
+	linear_network->arc_tail_node[arc.idx] = node_idx;
+	
+	assert(node_idx < tal_count(linear_network->node_adjacency_first_arc));
+	const arc_t first_arc = linear_network->node_adjacency_first_arc[node_idx];
+
+	assert(arc.idx < tal_count(linear_network->node_adjacency_next_arc));
+	linear_network->node_adjacency_next_arc[arc.idx]=first_arc;
+	
+	assert(node_idx < tal_count(linear_network->node_adjacency_first_arc));
+	linear_network->node_adjacency_first_arc[node_idx]=arc;
+}
+				
+
 static void init_linear_network(
 		const struct pay_parameters *params,
 		struct linear_network *linear_network)
@@ -510,9 +531,9 @@ static void init_linear_network(
 	linear_network->max_num_arcs = max_num_arcs;
 	linear_network->max_num_nodes = max_num_nodes;
 	
-	linear_network->arc_head_node = tal_arr(linear_network,u32,max_num_arcs);
-	for(size_t i=0;i<tal_count(linear_network->arc_head_node);++i)
-		linear_network->arc_head_node[i]=INVALID_INDEX;
+	linear_network->arc_tail_node = tal_arr(linear_network,u32,max_num_arcs);
+	for(size_t i=0;i<tal_count(linear_network->arc_tail_node);++i)
+		linear_network->arc_tail_node[i]=INVALID_INDEX;
 	
 	linear_network->node_adjacency_next_arc = tal_arr(linear_network,arc_t,max_num_arcs);
 	for(size_t i=0;i<tal_count(linear_network->node_adjacency_next_arc);++i)
@@ -529,7 +550,8 @@ static void init_linear_network(
 	linear_network->arc_fee_cost = tal_arr(linear_network,s64,max_num_arcs);
 	for(size_t i=0;i<tal_count(linear_network->arc_fee_cost);++i)
 		linear_network->arc_fee_cost[i]=INFINITE;
-		
+	
+	linear_network->capacity = tal_arrz(linear_network,s64,max_num_arcs);
 	
 	for(struct gossmap_node *node = gossmap_first_node(params->gossmap);
 	    node;
@@ -537,10 +559,10 @@ static void init_linear_network(
 	{
 		const u32 node_id = gossmap_node_idx(params->gossmap,node);
 		
-		arc_t prev_arc = (arc_t){.idx = INVALID_INDEX};
-		
 		for(size_t j=0;j<node->num_chans;++j)
 		{
+			
+			
 			int half;
 			const struct gossmap_chan *c = gossmap_nth_chan(params->gossmap,
 			                                                node, j, &half);
@@ -571,66 +593,33 @@ static void init_linear_network(
 			// that are outgoing to `node`
 			linearize_channel(params,c,half,capacity,prob_cost);
 			
+			const s64 fee_cost = linear_fee_cost(c,half,
+						params->base_fee_penalty,
+						params->delay_feefactor);
+			
 			// let's subscribe the 4 parts of the channel direction
 			// (c,half), the dual of these guys will be subscribed
 			// when the `i` hits the `next` node.
 			for(size_t k=0;k<CHANNEL_PARTS;++k)
 			{
 				arc_t arc = channel_idx_to_arc(chan_id,half,k,0);
-				linear_network->arc_head_node[arc.idx] = next_id;
 				
-				// Is this is the first arc?
-				if(prev_arc.idx==INVALID_INDEX)
-				// yes, then set it at the head of the linked list
-				{
-					linear_network->node_adjacency_first_arc[node_id]=arc;
-				} else
-				// no, then link it to the previous arc
-				{
-					linear_network->node_adjacency_next_arc[prev_arc.idx]=arc;
-				}
-				
-				prev_arc = arc;
+				linear_network_add_adjacenct_arc(linear_network,node_id,arc);
 				
 				linear_network->capacity[arc.idx] = capacity[k];
 				linear_network->arc_prob_cost[arc.idx] = prob_cost[k];
 				
-				linear_network->arc_fee_cost[arc.idx] 
-					= linear_fee_cost(c,half,
-							   params->base_fee_penalty,
-							   params->delay_feefactor);
-			}
-			
-			// split the opposite direction to obtain the dual arcs
-			// that are outgoing to `node`
-			linearize_channel(params,c,!half,capacity,prob_cost);
-			
-			// let's subscribe the 4 parts of the channel direction
-			// (c,!half) in dual representation
-			for(size_t k=0;k<CHANNEL_PARTS;++k)
-			{
-				arc_t arc = channel_idx_to_arc(chan_id,!half,k,1);
-				linear_network->arc_head_node[arc.idx] = next_id;
+				linear_network->arc_fee_cost[arc.idx] = fee_cost;
+				
+				// + the respective dual
+				arc_t dual = arc_dual(arc);
 				
-				// Is this is the first arc?
-				if(prev_arc.idx==INVALID_INDEX)
-				// yes, then set it at the head of the linked list
-				{
-					linear_network->node_adjacency_first_arc[node_id]=arc;
-				} else
-				// no, then link it to the previous arc
-				{
-					linear_network->node_adjacency_next_arc[prev_arc.idx]=arc;
-				}
+				linear_network_add_adjacenct_arc(linear_network,next_id,dual);
 				
-				prev_arc = arc;
+				linear_network->capacity[dual.idx] = 0;
+				linear_network->arc_prob_cost[dual.idx] = -prob_cost[k];
 				
-				linear_network->capacity[arc.idx] = 0;
-				linear_network->arc_prob_cost[arc.idx] = -prob_cost[k];
-				linear_network->arc_fee_cost[arc.idx] 
-					= -linear_fee_cost(c,!half,
-							    params->base_fee_penalty,
-					                    params->delay_feefactor);
+				linear_network->arc_fee_cost[dual.idx] = -fee_cost;
 			}
 		}
 	}
@@ -675,6 +664,7 @@ static int find_admissible_path(
 	{
 		qdata = lqueue_dequeue(&myqueue);
 		u32 cur = qdata->idx;
+	
 		tal_free(qdata);
 		
 		if(cur==target)
@@ -800,6 +790,7 @@ static int find_feasible_flow(
 		int err = find_admissible_path(
 					linear_network,
 					residual_network,source,target,prev);
+		
 		if(err!=RENEPAY_ERR_OK)
 		{
 			ret = RENEPAY_ERR_NOFEASIBLEFLOW;
@@ -1012,6 +1003,7 @@ static u32 find_positive_balance(
 	 * algorithm does not come up with spurious flow cycles. */
 	while(balance[final_idx]<=0)
 	{
+		printf("%s: node = %d\n",__PRETTY_FUNCTION__,final_idx);
 		u32 updated_idx=INVALID_INDEX;
 		struct gossmap_node *cur
 			= gossmap_node_byidx(gossmap,final_idx);
@@ -1048,6 +1040,9 @@ static u32 find_positive_balance(
 		
 		assert(updated_idx!=INVALID_INDEX);
 		assert(updated_idx!=final_idx);
+		printf("%s: balance[%d] = %ld\n",__PRETTY_FUNCTION__,
+			updated_idx,balance[updated_idx]);
+		
 		final_idx = updated_idx;
 	}
 	return final_idx;
@@ -1073,6 +1068,7 @@ static struct flow **
 		const struct linear_network *linear_network,
 		const struct residual_network *residual_network)
 {
+	printf("%s: starting\n",__PRETTY_FUNCTION__);
 	tal_t *this_ctx = tal(tmpctx,tal_t);
 	
 	const size_t max_num_chans = gossmap_max_chan_idx(gossmap);
@@ -1119,22 +1115,29 @@ static struct flow **
 	// positive balance node.
 	for(u32 node_idx=0;node_idx<max_num_nodes;++node_idx)
 	{
+		// for(size_t i=0;i<tal_count(prev_idx);++i)
+		// {
+		// 	prev_idx[i]=INVALID_INDEX;
+		// }
 		// this node has negative balance, flows leaves from here
 		while(balance[node_idx]<0)
 		{	
-			
 			prev_chan[node_idx]=NULL;
 			u32 final_idx = find_positive_balance(gossmap,chan_flow,node_idx,balance,
 							prev_chan,prev_dir,prev_idx);
-			
+		
 			s64 delta=-balance[node_idx];
 			int length = 0;
 			delta = MIN(delta,balance[final_idx]);
 			
 			// walk backwards, get me the length and the max flow we
 			// can send.
-			for(u32 cur_idx = final_idx;cur_idx!=node_idx;cur_idx=prev_idx[node_idx])
+			for(u32 cur_idx = final_idx;
+			    cur_idx!=node_idx;
+			    cur_idx=prev_idx[cur_idx])
 			{
+				assert(cur_idx!=INVALID_INDEX);
+				
 				const int dir = prev_dir[cur_idx];
 				struct gossmap_chan const * const c = prev_chan[cur_idx];
 				const u32 c_idx = gossmap_chan_idx(gossmap,c);
@@ -1156,8 +1159,12 @@ static struct flow **
 			balance[final_idx]-= delta;
 			
 			// walk backwards, substract flow
-			for(u32 cur_idx = final_idx;cur_idx!=node_idx;cur_idx=prev_idx[node_idx])
+			for(u32 cur_idx = final_idx;
+			    cur_idx!=node_idx;
+			    cur_idx=prev_idx[cur_idx])
 			{
+				assert(cur_idx!=INVALID_INDEX);
+				
 				const int dir = prev_dir[cur_idx];
 				struct gossmap_chan const * const c = prev_chan[cur_idx];
 				const u32 c_idx = gossmap_chan_idx(gossmap,c);
@@ -1192,6 +1199,7 @@ static struct flow **
 	}
 	
 	tal_free(this_ctx);
+	printf("%s: done\n",__PRETTY_FUNCTION__);
 	return flows;
 }
 
@@ -1217,6 +1225,7 @@ struct flow** minflow(
 		double base_fee_penalty,
 		u32 prob_cost_factor )
 {
+	printf("%s: starting\n",__PRETTY_FUNCTION__);
 	tal_t *this_ctx = tal(tmpctx,tal_t);
 	
 	struct pay_parameters *params = tal(this_ctx,struct pay_parameters);	
@@ -1263,18 +1272,23 @@ struct flow** minflow(
 	
 	init_residual_netork(linear_network,residual_network);
 	
+	printf("%s: done with allocation and initialization\n",__PRETTY_FUNCTION__);
 	
 	struct amount_msat best_fee;
 	// double best_prob_success;
 	struct flow **best_flow_paths = NULL;
 	
+	printf("%s: searching for a feasible flow\n",__PRETTY_FUNCTION__);
 	int err = find_feasible_flow(linear_network,residual_network,source_idx,target_idx,
 	                             params->amount.millisatoshis/1000);
+	
 	if(err!=RENEPAY_ERR_OK)
 	{
 		// there is no flow that satisfy the constraints, we stop here
+		printf("%s: feasible flow not found\n",__PRETTY_FUNCTION__);
 		goto fail;
 	}
+	printf("%s: found a feasible flow\n",__PRETTY_FUNCTION__);
 	
 	// first flow found
 	best_flow_paths = get_flow_paths(ctx,params->gossmap,params->chan_extra_map,
@@ -1292,6 +1306,7 @@ struct flow** minflow(
 	{
 		
 		s64 mu = (mu_left + mu_right)/2;
+		printf("%s: mu=%ld\n",__PRETTY_FUNCTION__,mu);
 		
 		combine_cost_function(linear_network,residual_network,mu);
 		
@@ -1302,7 +1317,10 @@ struct flow** minflow(
 		flow_paths = get_flow_paths(this_ctx,params->gossmap,params->chan_extra_map,
 		                            linear_network,residual_network);
 		
-		double prob_success = flows_probability(flow_paths);
+		double prob_success = flow_set_probability(
+						flow_paths,
+						params->gossmap,
+						params->chan_extra_map);
 		struct amount_msat fee = flows_fee(flow_paths);
 		
 		if(amount_msat_greater(fee,params->max_fee))
@@ -1332,8 +1350,12 @@ struct flow** minflow(
 			tal_free(flow_paths);
 	}
 	
+	
+	
 	fail:
 	
+	printf("%s: finished\n",__PRETTY_FUNCTION__);
+	
 	tal_free(this_ctx);
 	return best_flow_paths;
 }
diff --git a/plugins/renepay/pay_flow.c b/plugins/renepay/pay_flow.c
index adf7395a57c8..e09721f000b5 100644
--- a/plugins/renepay/pay_flow.c
+++ b/plugins/renepay/pay_flow.c
@@ -396,7 +396,7 @@ struct pay_flow **get_payflows(struct payment *p,
 		}
 
 		/* Are we unhappy? */
-		prob = flows_probability(flows);
+		prob = flow_set_probability(flows,pay_plugin->gossmap,&pay_plugin->chan_extra_map);
 		fee = flows_fee(flows);
 		delay = flows_worst_delay(flows) + p->final_cltv;
 
diff --git a/plugins/renepay/test/Makefile b/plugins/renepay/test/Makefile
index 79c7916bd475..a83aa1386cb8 100644
--- a/plugins/renepay/test/Makefile
+++ b/plugins/renepay/test/Makefile
@@ -18,7 +18,8 @@ PLUGIN_RENEPAY_TEST_COMMON_OBJS :=		\
 	common/dijkstra.o			\
 	common/setup.o				\
 	common/type_to_string.o			\
-	common/utils.o
+	common/utils.o				\
+	plugins/renepay/dijkstra.o
 
 $(PLUGIN_RENEPAY_TEST_PROGRAMS): $(CCAN_OBJS) $(BITCOIN_OBJS) $(WIRE_OBJS) $(PLUGIN_RENEPAY_TEST_COMMON_OBJS)
 
diff --git a/plugins/renepay/test/run-map.c b/plugins/renepay/test/run-map.c
new file mode 100644
index 000000000000..732418f95d6e
--- /dev/null
+++ b/plugins/renepay/test/run-map.c
@@ -0,0 +1,217 @@
+#include "config.h"
+#include <ccan/array_size/array_size.h>
+#include <ccan/read_write_all/read_write_all.h>
+#include <common/bigsize.h>
+#include <common/channel_id.h>
+#include <common/gossip_store.h>
+#include <common/node_id.h>
+#include <common/setup.h>
+#include <common/type_to_string.h>
+#include <common/wireaddr.h>
+#include <stdio.h>
+#include <assert.h>
+
+#include <bitcoin/short_channel_id.h>
+#include <ccan/htable/htable_type.h>
+
+/* not_mcf sets NDEBUG, so assert() is useless */
+#define ASSERT(x) do { if (!(x)) abort(); } while(0)
+
+/* AUTOGENERATED MOCKS START */
+/* Generated stub for fromwire_bigsize */
+bigsize_t fromwire_bigsize(const u8 **cursor UNNEEDED, size_t *max UNNEEDED)
+{ fprintf(stderr, "fromwire_bigsize called!\n"); abort(); }
+/* Generated stub for fromwire_channel_id */
+bool fromwire_channel_id(const u8 **cursor UNNEEDED, size_t *max UNNEEDED,
+			 struct channel_id *channel_id UNNEEDED)
+{ fprintf(stderr, "fromwire_channel_id called!\n"); abort(); }
+/* Generated stub for fromwire_wireaddr */
+bool fromwire_wireaddr(const u8 **cursor UNNEEDED, size_t *max UNNEEDED, struct wireaddr *addr UNNEEDED)
+{ fprintf(stderr, "fromwire_wireaddr called!\n"); abort(); }
+/* Generated stub for towire_bigsize */
+void towire_bigsize(u8 **pptr UNNEEDED, const bigsize_t val UNNEEDED)
+{ fprintf(stderr, "towire_bigsize called!\n"); abort(); }
+/* Generated stub for towire_channel_id */
+void towire_channel_id(u8 **pptr UNNEEDED, const struct channel_id *channel_id UNNEEDED)
+{ fprintf(stderr, "towire_channel_id called!\n"); abort(); }
+/* Generated stub for towire_wireaddr */
+void towire_wireaddr(u8 **pptr UNNEEDED, const struct wireaddr *addr UNNEEDED)
+{ fprintf(stderr, "towire_wireaddr called!\n"); abort(); }
+/* AUTOGENERATED MOCKS END */
+
+struct chan_extra {
+	struct short_channel_id scid;
+	int value;
+};
+
+static inline const struct short_channel_id
+chan_extra_scid(const struct chan_extra *cd)
+{
+	return cd->scid;
+}
+
+static inline size_t hash_scid(const struct short_channel_id scid)
+{
+	/* scids cost money to generate, so simple hash works here */
+	return (scid.u64 >> 32)
+		^ (scid.u64 >> 16)
+		^ scid.u64;
+}
+
+static inline bool chan_extra_eq_scid(const struct chan_extra *cd,
+				      const struct short_channel_id scid)
+{
+	return short_channel_id_eq(&scid, &cd->scid);
+}
+
+HTABLE_DEFINE_TYPE(struct chan_extra,
+		   chan_extra_scid, hash_scid, chan_extra_eq_scid,
+		   chan_extra_map);
+
+
+static void destroy_chan_extra(struct chan_extra *ce,
+			       struct chan_extra_map *chan_extra_map)
+{
+	printf("calling %s with value = %d\n",__PRETTY_FUNCTION__,ce->value);
+	chan_extra_map_del(chan_extra_map, ce);
+}
+static inline void new_chan_extra_value(
+	const tal_t *ctx,
+	struct chan_extra_map *chan_extra_map,
+	const struct short_channel_id scid,
+	int value)
+{
+	// struct chan_extra *ce = tal(chan_extra_map, struct chan_extra);
+	struct chan_extra *ce = tal(ctx, struct chan_extra);
+	
+	ce->scid = scid;
+	ce->value=value;
+	
+	/* Remove self from map when done */
+	chan_extra_map_add(chan_extra_map, ce);
+	tal_add_destructor2(ce, destroy_chan_extra, chan_extra_map);
+}
+
+static struct chan_extra*
+get_chan_extra_value(struct chan_extra_map *chan_extra_map,
+			    const struct short_channel_id scid)
+{
+	struct chan_extra *ce;
+
+	ce = chan_extra_map_get(chan_extra_map, scid);
+	if (!ce)
+		return NULL;
+	return ce;
+}
+
+static void valgrind_ok1(void)
+{
+	struct short_channel_id scid1,scid2;
+	
+	tal_t *this_ctx = tal(tmpctx,tal_t);
+	
+	struct chan_extra_map *chan_extra_map
+		= tal(this_ctx, struct chan_extra_map);
+	
+	chan_extra_map_init(chan_extra_map);
+	
+	assert(short_channel_id_from_str("1x1x0", 7, &scid1));
+	assert(short_channel_id_from_str("2x1x0", 7, &scid2));
+	
+	{
+		tal_t *local_ctx = tal(this_ctx,tal_t);
+		
+		/* in this case: elements are allocated in a local context, and freed at
+		 * the end, i.e. before chan_extra_map is freed. */
+		new_chan_extra_value(local_ctx,chan_extra_map,scid1,11);
+		new_chan_extra_value(local_ctx,chan_extra_map,scid2,21);
+		
+		struct chan_extra *x1 = get_chan_extra_value(chan_extra_map,scid1);
+		struct chan_extra *x2 = get_chan_extra_value(chan_extra_map,scid2);
+		
+		assert(x1->value==11);
+		assert(x2->value==21);
+		
+		tal_free(local_ctx);
+	}
+	
+	tal_free(this_ctx);
+
+}
+static void valgrind_ok2(void)
+{
+	struct short_channel_id scid1,scid2;
+	
+	tal_t *this_ctx = tal(tmpctx,tal_t);
+	
+	struct chan_extra_map *chan_extra_map
+		= tal(tmpctx, struct chan_extra_map);
+	
+	chan_extra_map_init(chan_extra_map);
+	
+	assert(short_channel_id_from_str("1x2x0", 7, &scid1));
+	assert(short_channel_id_from_str("2x2x0", 7, &scid2));
+	
+	
+	/* in this case: elements are allocated with chan_extra_map as parent.
+	 * the end of this function. i.e. before chan_extra_map is freed. */
+	new_chan_extra_value(chan_extra_map,chan_extra_map,scid1,12);
+	new_chan_extra_value(chan_extra_map,chan_extra_map,scid2,22);
+	
+	struct chan_extra *x1 = get_chan_extra_value(chan_extra_map,scid1);
+	struct chan_extra *x2 = get_chan_extra_value(chan_extra_map,scid2);
+	
+	assert(x1->value==12);
+	assert(x2->value==22);
+	
+	/* elements are freed before chan_extra_map is freed */
+	tal_free(x1);
+	tal_free(x2);
+	
+	tal_free(this_ctx);
+
+}
+static void valgrind_fail3(void)
+{
+	struct short_channel_id scid1,scid2;
+	
+	tal_t *this_ctx = tal(tmpctx,tal_t);
+	
+	struct chan_extra_map *chan_extra_map
+		= tal(tmpctx, struct chan_extra_map);
+	
+	chan_extra_map_init(chan_extra_map);
+	
+	assert(short_channel_id_from_str("1x3x0", 7, &scid1));
+	assert(short_channel_id_from_str("2x3x0", 7, &scid2));
+	
+	
+	/* in this case: elements are allocated with chan_extra_map as parent.
+	 * the end of this function. i.e. before chan_extra_map is freed. */
+	new_chan_extra_value(chan_extra_map,chan_extra_map,scid1,13);
+	new_chan_extra_value(chan_extra_map,chan_extra_map,scid2,23);
+	
+	struct chan_extra *x1 = get_chan_extra_value(chan_extra_map,scid1);
+	struct chan_extra *x2 = get_chan_extra_value(chan_extra_map,scid2);
+	
+	assert(x1->value==13);
+	assert(x2->value==23);
+	
+	/* Valgrind complains. It seems that the hash table is trying to remove
+	 * the element at a moment when its memory has already been released. 
+	 * If the following two lines are not commented the error disapears. */
+	// tal_free(x1);
+	// tal_free(x2);
+	
+	tal_free(this_ctx);
+}
+
+int main(int argc, char *argv[])
+{
+	common_setup(argv[0]);
+	valgrind_ok1();
+	valgrind_ok2();
+	valgrind_fail3();
+	common_shutdown();
+}
+
diff --git a/plugins/renepay/test/run-not_mcf-diamond.c b/plugins/renepay/test/run-not_mcf-diamond.c
index 50ac603b33ce..311f27353533 100644
--- a/plugins/renepay/test/run-not_mcf-diamond.c
+++ b/plugins/renepay/test/run-not_mcf-diamond.c
@@ -1,6 +1,6 @@
 #include "config.h"
 #include "../flow.c"
-#include "../not_mcf.c"
+#include "../mcf.c"
 #include <ccan/array_size/array_size.h>
 #include <ccan/read_write_all/read_write_all.h>
 #include <common/bigsize.h>
@@ -12,6 +12,9 @@
 #include <common/wireaddr.h>
 #include <stdio.h>
 
+/* not_mcf sets NDEBUG, so assert() is useless */
+#define ASSERT(x) do { if (!(x)) abort(); } while(0)
+
 /* AUTOGENERATED MOCKS START */
 /* Generated stub for fromwire_bigsize */
 bigsize_t fromwire_bigsize(const u8 **cursor UNNEEDED, size_t *max UNNEEDED)
@@ -40,9 +43,11 @@ static u8 empty_map[] = {
 
 static void print_flows(const char *desc,
 			const struct gossmap *gossmap,
+			struct chan_extra_map* chan_extra_map,
 			struct flow **flows)
 {
-	printf("%s: %zu subflows\n", desc, tal_count(flows));
+	double tot_prob = flow_set_probability(flows,gossmap,chan_extra_map);
+	printf("%s: %zu subflows, prob %.2lf\n", desc, tal_count(flows),tot_prob);
 	for (size_t i = 0; i < tal_count(flows); i++) {
 		struct amount_msat fee, delivered;
 		printf("   ");
@@ -69,7 +74,6 @@ int main(int argc, char *argv[])
 	char *gossfile;
 	struct gossmap *gossmap;
 	struct node_id l1, l2, l3, l4;
-	struct flow **flows;
 	struct short_channel_id scid12, scid13, scid24, scid34;
 	struct gossmap_localmods *mods;
 	struct chan_extra_map *chan_extra_map;
@@ -98,28 +102,28 @@ int main(int argc, char *argv[])
 	ASSERT(gossmap_local_addchan(mods, &l1, &l2, &scid12, NULL));
 	ASSERT(gossmap_local_updatechan(mods, &scid12,
 					AMOUNT_MSAT(0),
-					AMOUNT_MSAT(1000000000),
+					AMOUNT_MSAT(10000000000),
 					0, 1000, 5,
 					true,
 					0));
 	ASSERT(gossmap_local_addchan(mods, &l2, &l4, &scid24, NULL));
 	ASSERT(gossmap_local_updatechan(mods, &scid24,
 					AMOUNT_MSAT(0),
-					AMOUNT_MSAT(1000000000),
+					AMOUNT_MSAT(10000000000),
 					0, 1000, 5,
 					true,
 					0));
 	ASSERT(gossmap_local_addchan(mods, &l1, &l3, &scid13, NULL));
 	ASSERT(gossmap_local_updatechan(mods, &scid13,
 					AMOUNT_MSAT(0),
-					AMOUNT_MSAT(500000000),
+					AMOUNT_MSAT(5000000000),
 					0, 500, 5,
 					true,
 					0));
 	ASSERT(gossmap_local_addchan(mods, &l3, &l4, &scid34, NULL));
 	ASSERT(gossmap_local_updatechan(mods, &scid34,
 					AMOUNT_MSAT(0),
-					AMOUNT_MSAT(500000000),
+					AMOUNT_MSAT(5000000000),
 					0, 500, 5,
 					true,
 					0));
@@ -130,27 +134,31 @@ int main(int argc, char *argv[])
 	/* The local chans have no "capacity", so set them manually. */
 	new_chan_extra_half(chan_extra_map,
 			    scid12, 0,
-			    AMOUNT_MSAT(1000000000));
+			    AMOUNT_MSAT(10000000000));
 	new_chan_extra_half(chan_extra_map,
 			    scid24, 0,
-			    AMOUNT_MSAT(1000000000));
+			    AMOUNT_MSAT(10000000000));
 	new_chan_extra_half(chan_extra_map,
 			    scid13, 0,
-			    AMOUNT_MSAT(500000000));
+			    AMOUNT_MSAT(5000000000));
 	new_chan_extra_half(chan_extra_map,
 			    scid34, 0,
-			    AMOUNT_MSAT(500000000));
+			    AMOUNT_MSAT(5000000000));
 
+	struct flow **flows;
 	flows = minflow(tmpctx, gossmap,
 			gossmap_find_node(gossmap, &l1),
 			gossmap_find_node(gossmap, &l4),
 			chan_extra_map, NULL,
 			/* Half the capacity */
-			AMOUNT_MSAT(500000000),
-			0.2,
-			1);
-
-	print_flows("Simple minflow", gossmap, flows);
+			AMOUNT_MSAT(5000000000), // 5M sats
+			 /* max_fee = */ AMOUNT_MSAT(1000000), // 1k sats
+			 /* min probability = */ 0.1, // 10%
+			 /* delay fee factor = */ 1,
+			 /* base fee penalty */ 1,
+			 /* prob cost factor = */ 10);
+	
+	print_flows("Simple minflow", gossmap,chan_extra_map, flows);
 	ASSERT(tal_count(flows) == 2);
 
 	common_shutdown();
diff --git a/plugins/renepay/test/run-not_mcf-gossmap.c b/plugins/renepay/test/run-not_mcf-gossmap.c
index e69f947b99b2..307faf444bdf 100644
--- a/plugins/renepay/test/run-not_mcf-gossmap.c
+++ b/plugins/renepay/test/run-not_mcf-gossmap.c
@@ -16,7 +16,8 @@
 
 //static bool print_enable = true;
 //#define SUPERVERBOSE(...) do { if (print_enable) printf(__VA_ARGS__); } while(0)
-  #include "../not_mcf.c"
+
+  #include "../mcf.c"
   #include "../flow.c"
 
 /* AUTOGENERATED MOCKS START */
@@ -95,8 +96,8 @@ int main(int argc, char *argv[])
 
 	opt_parse(&argc, argv, opt_log_stderr_exit);
 
-	if (argc != 6)
-		errx(1, "Usage: %s <gossmap> <srcnode> <dstnode> <amount> <frugality>", argv[0]);
+	if (argc != 5)
+		errx(1, "Usage: %s <gossmap> <srcnode> <dstnode> <amount>", argv[0]);
 
 	gossmap = gossmap_load(tmpctx, argv[1], NULL);
 	assert(gossmap);
@@ -120,8 +121,11 @@ int main(int argc, char *argv[])
 	flows = minflow(tmpctx, gossmap, src, dst,
 			chan_extra_map, NULL,
 			amount,
-			atof(argv[5]),
-			1);
+			 /* max_fee = */ AMOUNT_MSAT(1000000), // 1k sats
+			 /* min probability = */ 0.1, // 10%
+			 /* delay fee factor = */ 1,
+			 /* base fee penalty */ 1,
+			 /* prob cost factor = */ 10);
 
 	print_flows("Flows", gossmap, flows);
 	common_shutdown();
diff --git a/plugins/renepay/test/run-not_mcf.c b/plugins/renepay/test/run-not_mcf.c
index 4764dac2035b..c23ff7c45d9d 100644
--- a/plugins/renepay/test/run-not_mcf.c
+++ b/plugins/renepay/test/run-not_mcf.c
@@ -1,6 +1,6 @@
 #include "config.h"
 #include "../flow.c"
-#include "../not_mcf.c"
+#include "../mcf.c"
 #include <ccan/array_size/array_size.h>
 #include <ccan/read_write_all/read_write_all.h>
 #include <common/bigsize.h>
@@ -309,8 +309,11 @@ int main(int argc, char *argv[])
 			chan_extra_map, NULL,
 			/* Half the capacity */
 			AMOUNT_MSAT(500000000),
-			1,
-			1);
+			 /* max_fee = */ AMOUNT_MSAT(1000000), // 1k sats
+			 /* min probability = */ 0.1, // 10%
+			 /* delay fee factor = */ 1,
+			 /* base fee penalty */ 1,
+			 /* prob cost factor = */ 10);
 	print_flows("Flow via single path l1->l2->l3", gossmap, flows);
 
 	/* Should go 1->2->3 */
@@ -387,8 +390,11 @@ int main(int argc, char *argv[])
 			chan_extra_map, NULL,
 			/* This will go first via 1-2-3, then 1->3. */
 			AMOUNT_MSAT(500000000),
-			0.1,
-			1);
+			 /* max_fee = */ AMOUNT_MSAT(1000000), // 1k sats
+			 /* min probability = */ 0.1, // 10%
+			 /* delay fee factor = */ 1,
+			 /* base fee penalty */ 1,
+			 /* prob cost factor = */ 10);
 
 	print_flows("Flow via two paths, low mu", gossmap, flows);
 
@@ -433,9 +439,12 @@ int main(int argc, char *argv[])
 			 gossmap_find_node(gossmap, &l3),
 			 chan_extra_map, NULL,
 			 /* This will go 400000000 via 1->3, rest via 1-2-3. */
-			 AMOUNT_MSAT(500000000),
-			 10,
-			 1);
+			 /* amount = */ AMOUNT_MSAT(500000000), //500k sats
+			 /* max_fee = */ AMOUNT_MSAT(1000000), // 1k sats
+			 /* min probability = */ 0.1, // 10%
+			 /* delay fee factor = */ 1,
+			 /* base fee penalty */ 1,
+			 /* prob cost factor = */ 10);
 	print_flows("Flow via two paths, high mu", gossmap, flows2);
 	ASSERT(tal_count(flows2) == 2);
 	ASSERT(tal_count(flows2[0]->path) == 1);

From a9e98305c58ecb69b53189e1304b7f9603a12a81 Mon Sep 17 00:00:00 2001
From: Lagrang3 <eduardo.quintana@pm.me>
Date: Thu, 11 May 2023 16:46:15 +0100
Subject: [PATCH 23/64] renepay: done some unitests and bugfixes

---
 plugins/renepay/flow.c                        |  52 +++-
 plugins/renepay/flow.h                        |  15 ++
 plugins/renepay/mcf.c                         | 159 ++++++++++--
 ...un-not_mcf-diamond.c => run-mcf-diamond.c} |  87 ++++---
 .../renepay/test/{run-not_mcf.c => run-mcf.c} | 239 +++++++++---------
 5 files changed, 373 insertions(+), 179 deletions(-)
 rename plugins/renepay/test/{run-not_mcf-diamond.c => run-mcf-diamond.c} (69%)
 rename plugins/renepay/test/{run-not_mcf.c => run-mcf.c} (82%)

diff --git a/plugins/renepay/flow.c b/plugins/renepay/flow.c
index e2ac7c359c6e..3b3c2d3ecf5c 100644
--- a/plugins/renepay/flow.c
+++ b/plugins/renepay/flow.c
@@ -5,6 +5,7 @@
 #include <math.h>
 #include <stdio.h>
 #include <plugins/renepay/flow.h>
+#include <common/type_to_string.h>
 
 #ifndef SUPERVERBOSE
 #define SUPERVERBOSE(...)
@@ -57,9 +58,9 @@ static double edge_probability(struct amount_msat min, struct amount_msat max,
 			       struct amount_msat in_flight,
 			       struct amount_msat f)
 {
-	printf("%s: with min=%ld, max=%ld, in_flight=%ld, flow=%ld\n",
-		__PRETTY_FUNCTION__,min.millisatoshis,max.millisatoshis,
-		in_flight.millisatoshis,f.millisatoshis);
+	//printf("%s: with min=%ld, max=%ld, in_flight=%ld, flow=%ld\n",
+	//	__PRETTY_FUNCTION__,min.millisatoshis,max.millisatoshis,
+	//	in_flight.millisatoshis,f.millisatoshis);
 	
 	struct amount_msat B=max; // =  max +1 - in_flight
 	
@@ -164,6 +165,7 @@ get_chan_extra_half_by_chan_verify(
 			cap = AMOUNT_SAT(0);
 		if (!amount_sat_to_msat(&cap_msat, cap))
 		{
+			printf("%s: aborting\n",__PRETTY_FUNCTION__);
 			abort();
 		}
 		h = new_chan_extra_half(chan_extra_map,
@@ -219,6 +221,45 @@ void remove_completed_flow(const struct gossmap *gossmap,
 		h->num_htlcs--;
 	}
 }
+void remove_completed_flow_set(
+		const struct gossmap *gossmap,
+		struct chan_extra_map *chan_extra_map,
+		struct flow **flows)
+{
+	for(size_t i=0;i<tal_count(flows);++i)
+	{
+		remove_completed_flow(gossmap,chan_extra_map,flows[i]);
+	}
+}
+
+void commit_flow(
+		const struct gossmap *gossmap,
+		struct chan_extra_map *chan_extra_map,
+		struct flow *flow)
+{
+	for (size_t i = 0; i < tal_count(flow->path); i++) {
+		struct chan_extra_half *h = get_chan_extra_half_by_chan(gossmap,
+							       chan_extra_map,
+							       flow->path[i],
+							       flow->dirs[i]);
+		if (!amount_msat_add(&h->htlc_total, h->htlc_total, flow->amounts[i]))
+		{
+			printf("%s: aborting\n",__PRETTY_FUNCTION__);
+			abort();
+		}
+		h->num_htlcs++;
+	}
+}
+void commit_flow_set(
+		const struct gossmap *gossmap,
+		struct chan_extra_map *chan_extra_map,
+		struct flow **flows)
+{
+	for(size_t i=0;i<tal_count(flows);++i)
+	{
+		commit_flow(gossmap,chan_extra_map,flows[i]);
+	}
+}
 
 /* Add this to the flow. */
 void flow_add(struct flow *flow,
@@ -311,6 +352,7 @@ void flow_complete(struct flow *flow,
 		   struct chan_extra_map *chan_extra_map,
 		   struct amount_msat delivered)
 {
+	// printf("%s:\n",__PRETTY_FUNCTION__);
 	flow->success_prob = 1.0;
 	flow->amounts = tal_arr(flow, struct amount_msat, tal_count(flow->path));
 	for (int i = tal_count(flow->path) - 1; i >= 0; i--) {
@@ -325,6 +367,9 @@ void flow_complete(struct flow *flow,
 			*= edge_probability(h->known_min, h->known_max,
 					    h->htlc_total,
 					    delivered);
+					    
+		// printf("(%s, ",type_to_string(tmpctx,struct amount_msat,&delivered));
+		// printf("%.2f)--",flow->success_prob);
 		
 		// if (!amount_msat_add(&h->htlc_total, h->htlc_total, delivered))
 		// {
@@ -340,6 +385,7 @@ void flow_complete(struct flow *flow,
 			abort();
 		}
 	}
+	// printf("\n%s: finished\n",__PRETTY_FUNCTION__);
 }
 
 /* Compute the prob. of success of a set of concurrent set of flows. 
diff --git a/plugins/renepay/flow.h b/plugins/renepay/flow.h
index f198a025b353..6f097f5a095d 100644
--- a/plugins/renepay/flow.h
+++ b/plugins/renepay/flow.h
@@ -127,6 +127,9 @@ double flow_set_probability(
 void remove_completed_flow(const struct gossmap *gossmap,
 			   struct chan_extra_map *chan_extra_map,
 			   struct flow *flow);
+void remove_completed_flow_set(const struct gossmap *gossmap,
+			   struct chan_extra_map *chan_extra_map,
+			   struct flow **flows);
 
 struct amount_msat flows_fee(struct flow **flows);
 
@@ -178,4 +181,16 @@ s64 linear_fee_cost(
 		double base_fee_penalty,
 		double delay_feefactor);
 
+/* Take the flows and commit them to the chan_extra's . */
+void commit_flow(
+		const struct gossmap *gossmap,
+		struct chan_extra_map *chan_extra_map,
+		struct flow *flow);
+
+/* Take the flows and commit them to the chan_extra's . */
+void commit_flow_set(
+		const struct gossmap *gossmap,
+		struct chan_extra_map *chan_extra_map,
+		struct flow **flows);
+
 #endif /* LIGHTNING_PLUGINS_RENEPAY_FLOW_H */
diff --git a/plugins/renepay/mcf.c b/plugins/renepay/mcf.c
index 9c4ad8186a88..49fb3c6d867f 100644
--- a/plugins/renepay/mcf.c
+++ b/plugins/renepay/mcf.c
@@ -6,6 +6,7 @@
 #include <plugins/renepay/mcf.h>
 #include <plugins/renepay/flow.h>
 #include <plugins/renepay/dijkstra.h>
+#include <common/type_to_string.h>
 #include <stdint.h>
 #include <math.h>
 #include <assert.h>
@@ -425,7 +426,7 @@ static void linearize_channel(
 		s64 *capacity,
 		s64 *cost)
 {
-	struct chan_extra_half *extra_half = get_chan_extra_half_by_chan(
+	struct chan_extra_half *extra_half = get_chan_extra_half_by_chan_verify(
 							params->gossmap,
 							params->chan_extra_map,
 							c,
@@ -442,7 +443,12 @@ static void linearize_channel(
 		
 		cost[i] = params->cost_fraction[i]
 		          *params->amount.millisatoshis
-		          *params->prob_cost_factor;
+		          *params->prob_cost_factor*1.0/(b-a);
+		
+		// printf("channel part: %ld, cost_fraction: %lf, cost: %ld\n",
+		// 	i,params->cost_fraction[i],cost[i]);
+		// printf("prob_cost_factor: %d, amount_msat: %ld\n",
+		// 	params->prob_cost_factor,params->amount.millisatoshis);
 	}
 }
 
@@ -487,8 +493,13 @@ static void combine_cost_function(
 		struct residual_network *residual_network,
 		s64 mu)
 {
+	// printf("Report on arcs cost\n");
 	for(u32 arc_idx=0;arc_idx<linear_network->max_num_arcs;++arc_idx)
 	{
+		arc_t arc = (arc_t){.idx=arc_idx};
+		if(arc_tail(linear_network,arc)==INVALID_INDEX)
+			continue;
+		
 		const s64 pcost = linear_network->arc_prob_cost[arc_idx],
 		          fcost = linear_network->arc_fee_cost[arc_idx];
 			  
@@ -498,7 +509,17 @@ static void combine_cost_function(
 		residual_network->cost[arc_idx]
 			= mu==0 ? pcost : 
 			          (mu==(MU_MAX-1) ? fcost : combined);
+		
+		// printf("arc_idx: %d, comb. cost: %ld, res. cap: %ld\n",arc_idx,
+		// 	residual_network->cost[arc_idx],
+		// 	residual_network->cap[arc_idx]);
+		// printf("cap: %ld, prob_cost: %ld, fee_cost: %ld, mu: %ld\n\n",
+		// 	linear_network->capacity[arc_idx],
+		// 	linear_network->arc_prob_cost[arc_idx],
+		// 	linear_network->arc_fee_cost[arc_idx],
+		// 	mu);
 	}
+	// printf("\n\n");
 }
 
 static void linear_network_add_adjacenct_arc(
@@ -602,6 +623,8 @@ static void init_linear_network(
 			// when the `i` hits the `next` node.
 			for(size_t k=0;k<CHANNEL_PARTS;++k)
 			{
+				// if(capacity[k]==0)continue;
+				
 				arc_t arc = channel_idx_to_arc(chan_id,half,k,0);
 				
 				linear_network_add_adjacenct_arc(linear_network,node_id,arc);
@@ -1003,7 +1026,7 @@ static u32 find_positive_balance(
 	 * algorithm does not come up with spurious flow cycles. */
 	while(balance[final_idx]<=0)
 	{
-		printf("%s: node = %d\n",__PRETTY_FUNCTION__,final_idx);
+		// printf("%s: node = %d\n",__PRETTY_FUNCTION__,final_idx);
 		u32 updated_idx=INVALID_INDEX;
 		struct gossmap_node *cur
 			= gossmap_node_byidx(gossmap,final_idx);
@@ -1040,8 +1063,8 @@ static u32 find_positive_balance(
 		
 		assert(updated_idx!=INVALID_INDEX);
 		assert(updated_idx!=final_idx);
-		printf("%s: balance[%d] = %ld\n",__PRETTY_FUNCTION__,
-			updated_idx,balance[updated_idx]);
+		// printf("%s: balance[%d] = %ld\n",__PRETTY_FUNCTION__,
+		//	updated_idx,balance[updated_idx]);
 		
 		final_idx = updated_idx;
 	}
@@ -1068,7 +1091,7 @@ static struct flow **
 		const struct linear_network *linear_network,
 		const struct residual_network *residual_network)
 {
-	printf("%s: starting\n",__PRETTY_FUNCTION__);
+	// printf("%s: starting\n",__PRETTY_FUNCTION__);
 	tal_t *this_ctx = tal(tmpctx,tal_t);
 	
 	const size_t max_num_chans = gossmap_max_chan_idx(gossmap);
@@ -1199,10 +1222,87 @@ static struct flow **
 	}
 	
 	tal_free(this_ctx);
-	printf("%s: done\n",__PRETTY_FUNCTION__);
+	// printf("%s: done\n",__PRETTY_FUNCTION__);
 	return flows;
 }
 
+/* Given the constraints on max fee and min prob.,
+ * is the flow A better than B? */
+static bool is_better(
+		struct amount_msat max_fee,
+		double min_probability,
+		
+		struct amount_msat A_fee,
+		double A_prob,
+		
+		struct amount_msat B_fee,
+		double B_prob)
+{
+	bool A_fee_pass = amount_msat_less_eq(A_fee,max_fee);
+	bool B_fee_pass = amount_msat_less_eq(B_fee,max_fee);
+	bool A_prob_pass = A_prob >= min_probability;
+	bool B_prob_pass = B_prob >= min_probability;
+	
+	// all bounds are met
+	if(A_fee_pass && B_fee_pass && A_prob_pass && B_prob_pass)
+	{
+		// prefer lower fees
+		return amount_msat_less_eq(A_fee,B_fee);
+	}
+	
+	// prefer the solution that satisfies both bounds
+	if((!A_fee_pass || !A_prob_pass) && (B_fee_pass && B_prob_pass))
+	{
+		return false;
+	}
+	// prefer the solution that satisfies both bounds
+	if((A_fee_pass && A_prob_pass) && (!B_fee_pass || !B_prob_pass))
+	{
+		return true;
+	}
+	
+	// no solution satisfies both bounds
+	
+	// bound on fee is met
+	if(A_fee_pass && B_fee_pass)
+	{
+		// pick the highest prob.
+		return A_prob > B_prob;
+	}
+	
+	// bound on prob. is met
+	if(A_prob_pass && B_prob_pass)
+	{
+		// pick the lowest fee
+		return amount_msat_less_eq(A_fee,B_fee);
+	}
+	
+	// prefer the solution that satisfies the bound on fees
+	if(A_fee_pass)
+	{
+		return true;
+	}
+	if(B_fee_pass)
+	{
+		return false;
+	}
+	
+	// none of them satisfy the fee bound
+	
+	// prefer the solution that satisfies the bound on prob.
+	if(A_prob_pass)
+	{
+		return true;
+	}
+	if(B_prob_pass)
+	{
+		return true;
+	}
+	
+	// no bound whatsoever is satisfied
+	// go for fees
+	return amount_msat_less_eq(A_fee,B_fee);
+}
 
 
 // TODO(eduardo): choose some default values for the minflow parameters
@@ -1210,7 +1310,11 @@ static struct flow **
  * flows, ie. base fees are not considered. Hence a flow along a path is
  * described with a sequence of directed channels and one amount. 
  * In the `pay_flow` module there are dedicated routes to compute the actual
- * amount to be forward on each hop. */
+ * amount to be forward on each hop. 
+ * 
+ * TODO(eduardo): notice that we don't pay fees to forward payments with local
+ * channels and we can tell with absolute certainty the liquidity on them. 
+ * Check that local channels have fee costs = 0 and bounds with certainty (min=max). */
 struct flow** minflow(
 		const tal_t *ctx,
 		struct gossmap *gossmap,
@@ -1250,6 +1354,9 @@ struct flow** minflow(
 		params->cost_fraction[i]=
 			log((1-CHANNEL_PIVOTS[i-1])/(1-CHANNEL_PIVOTS[i]))
 			/params->cap_fraction[i];
+		
+		// printf("channel part: %ld, fraction: %lf, cost_fraction: %lf\n",
+		//	i,params->cap_fraction[i],params->cost_fraction[i]);
 	}
 	
 	params->max_fee = max_fee;
@@ -1275,7 +1382,7 @@ struct flow** minflow(
 	printf("%s: done with allocation and initialization\n",__PRETTY_FUNCTION__);
 	
 	struct amount_msat best_fee;
-	// double best_prob_success;
+	double best_prob_success;
 	struct flow **best_flow_paths = NULL;
 	
 	printf("%s: searching for a feasible flow\n",__PRETTY_FUNCTION__);
@@ -1286,17 +1393,18 @@ struct flow** minflow(
 	{
 		// there is no flow that satisfy the constraints, we stop here
 		printf("%s: feasible flow not found\n",__PRETTY_FUNCTION__);
-		goto fail;
+		goto finish;
 	}
 	printf("%s: found a feasible flow\n",__PRETTY_FUNCTION__);
 	
 	// first flow found
 	best_flow_paths = get_flow_paths(ctx,params->gossmap,params->chan_extra_map,
 	                            linear_network,residual_network);
-	// best_prob_success = flows_probability(best_flow_paths);
+	best_prob_success = flow_set_probability(best_flow_paths,
+						params->gossmap,
+						params->chan_extra_map);
 	best_fee = flows_fee(best_flow_paths);
 	
-	
 	// binary search for a value of `mu` that fits our fee and prob.
 	// constraints.
 	// mu=0 corresponds to only probabilities
@@ -1323,27 +1431,36 @@ struct flow** minflow(
 						params->chan_extra_map);
 		struct amount_msat fee = flows_fee(flow_paths);
 		
+		printf("prob %.2f, fee %s\n",prob_success,
+				type_to_string(this_ctx,struct amount_msat,&fee));
+		
+		// is this better than the previous one?
+		if(!best_flow_paths || 
+			is_better(params->max_fee,params->min_probability,
+				  fee,prob_success,
+			          best_fee, best_prob_success))
+		{
+			best_flow_paths = tal_steal(ctx,flow_paths);
+			best_fee = fee;
+			best_prob_success=prob_success;
+			flow_paths = NULL;
+		}
+		
 		if(amount_msat_greater(fee,params->max_fee))
 		{
 			// too expensive
 			mu_left = mu+1;
+			printf("%s: too expensive\n",__PRETTY_FUNCTION__);
 		}else if(prob_success < params->min_probability)
 		{
 			// too unlikely
 			mu_right = mu;
+			printf("%s: too unlikely\n",__PRETTY_FUNCTION__);
 		}else
 		{
 			// with mu constraints are satisfied, now let's optimize
 			// the fees
 			mu_left = mu+1;
-			
-			if(!best_flow_paths || amount_msat_less(fee,best_fee))
-			{
-				best_flow_paths = tal_steal(ctx,flow_paths);
-				best_fee = fee;
-				// best_prob_success=prob_success;
-				flow_paths = NULL;
-			}
 		}
 		
 		if(flow_paths)
@@ -1352,7 +1469,7 @@ struct flow** minflow(
 	
 	
 	
-	fail:
+	finish:
 	
 	printf("%s: finished\n",__PRETTY_FUNCTION__);
 	
diff --git a/plugins/renepay/test/run-not_mcf-diamond.c b/plugins/renepay/test/run-mcf-diamond.c
similarity index 69%
rename from plugins/renepay/test/run-not_mcf-diamond.c
rename to plugins/renepay/test/run-mcf-diamond.c
index 311f27353533..2e48b3a2540d 100644
--- a/plugins/renepay/test/run-not_mcf-diamond.c
+++ b/plugins/renepay/test/run-mcf-diamond.c
@@ -60,7 +60,11 @@ static void print_flows(const char *desc,
 		}
 		delivered = flows[i]->amounts[tal_count(flows[i]->amounts)-1];
 		if (!amount_msat_sub(&fee, flows[i]->amounts[0], delivered))
+		{
+			fprintf(stderr,"%s: flow[i]->amount[0]<delivered\n",
+				__PRETTY_FUNCTION__);
 			abort();
+		}
 		printf(" prob %.2f, %s delivered with fee %s\n",
 		       flows[i]->success_prob,
 		       type_to_string(tmpctx, struct amount_msat, &delivered),
@@ -81,50 +85,52 @@ int main(int argc, char *argv[])
 	common_setup(argv[0]);
 
 	fd = tmpdir_mkstemp(tmpctx, "run-not_mcf-diamond.XXXXXX", &gossfile);
-	ASSERT(write_all(fd, empty_map, sizeof(empty_map)));
+	assert(write_all(fd, empty_map, sizeof(empty_map)));
 
 	gossmap = gossmap_load(tmpctx, gossfile, NULL);
-	ASSERT(gossmap);
+	assert(gossmap);
 
 	/* These are in ascending order, for easy direction setting */
-	ASSERT(node_id_from_hexstr("022d223620a359a47ff7f7ac447c85c46c923da53389221a0054c11c1e3ca31d59", 66, &l1));
-	ASSERT(node_id_from_hexstr("0266e4598d1d3c415f572a8488830b60f7e744ed9235eb0b1ba93283b315c03518", 66, &l2));
-	ASSERT(node_id_from_hexstr("035d2b1192dfba134e10e540875d366ebc8bc353d5aa766b80c090b39c3a5d885d", 66, &l3));
-	ASSERT(node_id_from_hexstr("0382ce59ebf18be7d84677c2e35f23294b9992ceca95491fcf8a56c6cb2d9de199", 66, &l4));
-	ASSERT(short_channel_id_from_str("1x2x0", 7, &scid12));
-	ASSERT(short_channel_id_from_str("1x3x0", 7, &scid13));
-	ASSERT(short_channel_id_from_str("2x4x0", 7, &scid24));
-	ASSERT(short_channel_id_from_str("3x4x0", 7, &scid34));
+	assert(node_id_from_hexstr("022d223620a359a47ff7f7ac447c85c46c923da53389221a0054c11c1e3ca31d59", 66, &l1));
+	assert(node_id_from_hexstr("0266e4598d1d3c415f572a8488830b60f7e744ed9235eb0b1ba93283b315c03518", 66, &l2));
+	assert(node_id_from_hexstr("035d2b1192dfba134e10e540875d366ebc8bc353d5aa766b80c090b39c3a5d885d", 66, &l3));
+	assert(node_id_from_hexstr("0382ce59ebf18be7d84677c2e35f23294b9992ceca95491fcf8a56c6cb2d9de199", 66, &l4));
+	assert(short_channel_id_from_str("1x2x0", 7, &scid12));
+	assert(short_channel_id_from_str("1x3x0", 7, &scid13));
+	assert(short_channel_id_from_str("2x4x0", 7, &scid24));
+	assert(short_channel_id_from_str("3x4x0", 7, &scid34));
 
 	mods = gossmap_localmods_new(tmpctx);
 
-	/* 1->2->4 has capacity 10M sat, 1->3->4 has capacity 5M sat (lower fee!) */
-	ASSERT(gossmap_local_addchan(mods, &l1, &l2, &scid12, NULL));
-	ASSERT(gossmap_local_updatechan(mods, &scid12,
-					AMOUNT_MSAT(0),
-					AMOUNT_MSAT(10000000000),
-					0, 1000, 5,
-					true,
-					0));
-	ASSERT(gossmap_local_addchan(mods, &l2, &l4, &scid24, NULL));
-	ASSERT(gossmap_local_updatechan(mods, &scid24,
+	/* 1->2->4 has capacity 10k sat, 1->3->4 has capacity 5k sat (lower fee!) */
+	assert(gossmap_local_addchan(mods, &l1, &l2, &scid12, NULL));
+	assert(gossmap_local_updatechan(mods, &scid12,
+					/*htlc_min=*/ AMOUNT_MSAT(0),
+					/*htlc_max=*/ AMOUNT_MSAT(10000000),
+					/*base_fee=*/ 0, 
+					/*ppm_fee =*/ 1001, 
+					/* delay  =*/ 5,
+					/* enabled=*/ true,
+					/* dir    =*/ 0));
+	assert(gossmap_local_addchan(mods, &l2, &l4, &scid24, NULL));
+	assert(gossmap_local_updatechan(mods, &scid24,
 					AMOUNT_MSAT(0),
-					AMOUNT_MSAT(10000000000),
-					0, 1000, 5,
+					AMOUNT_MSAT(10000000),
+					0, 1002, 5,
 					true,
 					0));
-	ASSERT(gossmap_local_addchan(mods, &l1, &l3, &scid13, NULL));
-	ASSERT(gossmap_local_updatechan(mods, &scid13,
+	assert(gossmap_local_addchan(mods, &l1, &l3, &scid13, NULL));
+	assert(gossmap_local_updatechan(mods, &scid13,
 					AMOUNT_MSAT(0),
-					AMOUNT_MSAT(5000000000),
-					0, 500, 5,
+					AMOUNT_MSAT(5000000),
+					0, 503, 5,
 					true,
 					0));
-	ASSERT(gossmap_local_addchan(mods, &l3, &l4, &scid34, NULL));
-	ASSERT(gossmap_local_updatechan(mods, &scid34,
+	assert(gossmap_local_addchan(mods, &l3, &l4, &scid34, NULL));
+	assert(gossmap_local_updatechan(mods, &scid34,
 					AMOUNT_MSAT(0),
-					AMOUNT_MSAT(5000000000),
-					0, 500, 5,
+					AMOUNT_MSAT(5000000),
+					0, 504, 5,
 					true,
 					0));
 
@@ -134,16 +140,16 @@ int main(int argc, char *argv[])
 	/* The local chans have no "capacity", so set them manually. */
 	new_chan_extra_half(chan_extra_map,
 			    scid12, 0,
-			    AMOUNT_MSAT(10000000000));
+			    AMOUNT_MSAT(10000000));
 	new_chan_extra_half(chan_extra_map,
 			    scid24, 0,
-			    AMOUNT_MSAT(10000000000));
+			    AMOUNT_MSAT(10000000));
 	new_chan_extra_half(chan_extra_map,
 			    scid13, 0,
-			    AMOUNT_MSAT(5000000000));
+			    AMOUNT_MSAT(5000000));
 	new_chan_extra_half(chan_extra_map,
 			    scid34, 0,
-			    AMOUNT_MSAT(5000000000));
+			    AMOUNT_MSAT(5000000));
 
 	struct flow **flows;
 	flows = minflow(tmpctx, gossmap,
@@ -151,15 +157,14 @@ int main(int argc, char *argv[])
 			gossmap_find_node(gossmap, &l4),
 			chan_extra_map, NULL,
 			/* Half the capacity */
-			AMOUNT_MSAT(5000000000), // 5M sats
-			 /* max_fee = */ AMOUNT_MSAT(1000000), // 1k sats
-			 /* min probability = */ 0.1, // 10%
-			 /* delay fee factor = */ 1,
-			 /* base fee penalty */ 1,
-			 /* prob cost factor = */ 10);
+			AMOUNT_MSAT(1000000), // 1000 sats
+			 /* max_fee = */ AMOUNT_MSAT(10000), // 10 sats
+			 /* min probability = */ 0.8, // 80%
+			 /* delay fee factor = */ 0,
+			 /* base fee penalty */ 0,
+			 /* prob cost factor = */ 1);
 	
 	print_flows("Simple minflow", gossmap,chan_extra_map, flows);
-	ASSERT(tal_count(flows) == 2);
 
 	common_shutdown();
 }
diff --git a/plugins/renepay/test/run-not_mcf.c b/plugins/renepay/test/run-mcf.c
similarity index 82%
rename from plugins/renepay/test/run-not_mcf.c
rename to plugins/renepay/test/run-mcf.c
index c23ff7c45d9d..f16dddd9ff02 100644
--- a/plugins/renepay/test/run-not_mcf.c
+++ b/plugins/renepay/test/run-mcf.c
@@ -278,28 +278,25 @@ int main(int argc, char *argv[])
 	char *gossfile;
 	struct gossmap *gossmap;
 	struct node_id l1, l2, l3;
-	struct flow **flows, **flows2;
-	struct short_channel_id scid12, scid23, scid13;
-	struct gossmap_localmods *mods;
+	struct flow **flows;
+	struct short_channel_id scid12, scid23;
 	struct chan_extra_map *chan_extra_map;
-	struct chan_extra *ce;
-	struct gossmap_chan *local_chan;
 
 	common_setup(argv[0]);
 
 	fd = tmpdir_mkstemp(tmpctx, "run-not_mcf.XXXXXX", &gossfile);
-	ASSERT(write_all(fd, canned_map, sizeof(canned_map)));
+	assert(write_all(fd, canned_map, sizeof(canned_map)));
 
 	gossmap = gossmap_load(tmpctx, gossfile, NULL);
-	ASSERT(gossmap);
+	assert(gossmap);
 
 	/* There is a public channel 2<->3 (103x1x0), and private
 	 * 1<->2 (110x1x1). */
-	ASSERT(node_id_from_hexstr("0266e4598d1d3c415f572a8488830b60f7e744ed9235eb0b1ba93283b315c03518", 66, &l1));
-	ASSERT(node_id_from_hexstr("022d223620a359a47ff7f7ac447c85c46c923da53389221a0054c11c1e3ca31d59", 66, &l2));
-	ASSERT(node_id_from_hexstr("035d2b1192dfba134e10e540875d366ebc8bc353d5aa766b80c090b39c3a5d885d", 66, &l3));
-	ASSERT(short_channel_id_from_str("110x1x1", 7, &scid12));
-	ASSERT(short_channel_id_from_str("103x1x0", 7, &scid23));
+	assert(node_id_from_hexstr("0266e4598d1d3c415f572a8488830b60f7e744ed9235eb0b1ba93283b315c03518", 66, &l1));
+	assert(node_id_from_hexstr("022d223620a359a47ff7f7ac447c85c46c923da53389221a0054c11c1e3ca31d59", 66, &l2));
+	assert(node_id_from_hexstr("035d2b1192dfba134e10e540875d366ebc8bc353d5aa766b80c090b39c3a5d885d", 66, &l3));
+	assert(short_channel_id_from_str("110x1x1", 7, &scid12));
+	assert(short_channel_id_from_str("103x1x0", 7, &scid23));
 
 	chan_extra_map = tal(tmpctx, struct chan_extra_map);
 	chan_extra_map_init(chan_extra_map);
@@ -310,62 +307,68 @@ int main(int argc, char *argv[])
 			/* Half the capacity */
 			AMOUNT_MSAT(500000000),
 			 /* max_fee = */ AMOUNT_MSAT(1000000), // 1k sats
-			 /* min probability = */ 0.1, // 10%
+			 /* min probability = */ 0.1,
 			 /* delay fee factor = */ 1,
 			 /* base fee penalty */ 1,
 			 /* prob cost factor = */ 10);
+	commit_flow_set(gossmap,chan_extra_map,flows);
 	print_flows("Flow via single path l1->l2->l3", gossmap, flows);
-
+	
+	
+	
 	/* Should go 1->2->3 */
-	ASSERT(tal_count(flows) == 1);
-	ASSERT(tal_count(flows[0]->path) == 2);
-	ASSERT(tal_count(flows[0]->dirs) == 2);
-	ASSERT(tal_count(flows[0]->amounts) == 2);
-
-	ASSERT(flows[0]->path[0] == gossmap_find_chan(gossmap, &scid12));
-	ASSERT(flows[0]->path[1] == gossmap_find_chan(gossmap, &scid23));
-	ASSERT(flows[0]->dirs[0] == 1);
-	ASSERT(flows[0]->dirs[1] == 0);
-	ASSERT(amount_msat_eq(flows[0]->amounts[1], AMOUNT_MSAT(500000000)));
+	assert(tal_count(flows) == 1);
+	assert(tal_count(flows[0]->path) == 2);
+	assert(tal_count(flows[0]->dirs) == 2);
+	assert(tal_count(flows[0]->amounts) == 2);
+
+	assert(flows[0]->path[0] == gossmap_find_chan(gossmap, &scid12));
+	assert(flows[0]->path[1] == gossmap_find_chan(gossmap, &scid23));
+	assert(flows[0]->dirs[0] == 1);
+	assert(flows[0]->dirs[1] == 0);
+	assert(amount_msat_eq(flows[0]->amounts[1], AMOUNT_MSAT(500000000)));
 	/* fee_base_msat == 20, fee_proportional_millionths == 1000 */
-	ASSERT(amount_msat_eq(flows[0]->amounts[0], AMOUNT_MSAT(500000000 + 500000 + 20)));
+	assert(amount_msat_eq(flows[0]->amounts[0], AMOUNT_MSAT(500000000 + 500000 + 20)));
 
 	/* Each one has probability ~ 0.5 */
-	ASSERT(flows[0]->success_prob > 0.249);
-	ASSERT(flows[0]->success_prob <= 0.250);
+	assert(flows[0]->success_prob > 0.249);
+	assert(flows[0]->success_prob <= 0.250);
 
+	
 	/* Should have filled in some extra data! */
-	ce = chan_extra_map_get(chan_extra_map, scid12);
-	ASSERT(short_channel_id_eq(&ce->scid, &scid12));
+	struct chan_extra *ce = chan_extra_map_get(chan_extra_map, scid12);
+	assert(short_channel_id_eq(&ce->scid, &scid12));
 	/* l1->l2 dir is 1 */
-	ASSERT(ce->half[1].num_htlcs == 1);
-	ASSERT(amount_msat_eq(ce->half[1].htlc_total, AMOUNT_MSAT(500000000 + 500000 + 20)));
-	ASSERT(amount_msat_eq(ce->half[1].known_min, AMOUNT_MSAT(0)));
-	ASSERT(amount_msat_eq(ce->half[1].known_max, AMOUNT_MSAT(1000000000)));
-	ASSERT(ce->half[0].num_htlcs == 0);
-	ASSERT(amount_msat_eq(ce->half[0].htlc_total, AMOUNT_MSAT(0)));
-	ASSERT(amount_msat_eq(ce->half[0].known_min, AMOUNT_MSAT(0)));
-	ASSERT(amount_msat_eq(ce->half[0].known_max, AMOUNT_MSAT(1000000000)));
+	assert(ce->half[1].num_htlcs == 1);
+	assert(amount_msat_eq(ce->half[1].htlc_total, AMOUNT_MSAT(500000000 + 500000 + 20)));
+	assert(amount_msat_eq(ce->half[1].known_min, AMOUNT_MSAT(0)));
+	assert(amount_msat_eq(ce->half[1].known_max, AMOUNT_MSAT(1000000000)));
+	assert(ce->half[0].num_htlcs == 0);
+	assert(amount_msat_eq(ce->half[0].htlc_total, AMOUNT_MSAT(0)));
+	assert(amount_msat_eq(ce->half[0].known_min, AMOUNT_MSAT(0)));
+	assert(amount_msat_eq(ce->half[0].known_max, AMOUNT_MSAT(1000000000)));
 
 	ce = chan_extra_map_get(chan_extra_map, scid23);
-	ASSERT(short_channel_id_eq(&ce->scid, &scid23));
+	assert(short_channel_id_eq(&ce->scid, &scid23));
 	/* l2->l3 dir is 0 */
-	ASSERT(ce->half[0].num_htlcs == 1);
-	ASSERT(amount_msat_eq(ce->half[0].htlc_total, AMOUNT_MSAT(500000000)));
-	ASSERT(amount_msat_eq(ce->half[0].known_min, AMOUNT_MSAT(0)));
-	ASSERT(amount_msat_eq(ce->half[0].known_max, AMOUNT_MSAT(1000000000)));
-	ASSERT(ce->half[1].num_htlcs == 0);
-	ASSERT(amount_msat_eq(ce->half[1].htlc_total, AMOUNT_MSAT(0)));
-	ASSERT(amount_msat_eq(ce->half[1].known_min, AMOUNT_MSAT(0)));
-	ASSERT(amount_msat_eq(ce->half[1].known_max, AMOUNT_MSAT(1000000000)));
-
-	/* Now try adding a local channel scid */
-	mods = gossmap_localmods_new(tmpctx);
-	ASSERT(short_channel_id_from_str("111x1x1", 7, &scid13));
+	assert(ce->half[0].num_htlcs == 1);
+	assert(amount_msat_eq(ce->half[0].htlc_total, AMOUNT_MSAT(500000000)));
+	assert(amount_msat_eq(ce->half[0].known_min, AMOUNT_MSAT(0)));
+	assert(amount_msat_eq(ce->half[0].known_max, AMOUNT_MSAT(1000000000)));
+	assert(ce->half[1].num_htlcs == 0);
+	assert(amount_msat_eq(ce->half[1].htlc_total, AMOUNT_MSAT(0)));
+	assert(amount_msat_eq(ce->half[1].known_min, AMOUNT_MSAT(0)));
+	assert(amount_msat_eq(ce->half[1].known_max, AMOUNT_MSAT(1000000000)));
+
+	// /* Now try adding a local channel scid */
+	
+	struct short_channel_id scid13;
+	struct gossmap_localmods *mods = gossmap_localmods_new(tmpctx);
+	assert(short_channel_id_from_str("111x1x1", 7, &scid13));
 
 	/* 400,000sat channel from 1->3, basefee 0, ppm 1000, delay 5 */
-	ASSERT(gossmap_local_addchan(mods, &l1, &l3, &scid13, NULL));
-	ASSERT(gossmap_local_updatechan(mods, &scid13,
+	assert(gossmap_local_addchan(mods, &l1, &l3, &scid13, NULL));
+	assert(gossmap_local_updatechan(mods, &scid13,
 					AMOUNT_MSAT(0),
 					AMOUNT_MSAT(400000000),
 					0, 1000, 5,
@@ -374,67 +377,75 @@ int main(int argc, char *argv[])
 
 	/* Apply changes, check they work. */
 	gossmap_apply_localmods(gossmap, mods);
-	local_chan = gossmap_find_chan(gossmap, &scid13);
-	ASSERT(local_chan);
+	struct gossmap_chan *local_chan = gossmap_find_chan(gossmap, &scid13);
+	assert(local_chan);
 
 	/* Clear that */
-	remove_completed_flow(gossmap, chan_extra_map, flows[0]);
+	remove_completed_flow_set(gossmap, chan_extra_map, flows);
 
 	/* The local chans have no "capacity", so set it manually. */
 	new_chan_extra_half(chan_extra_map, scid13, 0,
 			    AMOUNT_MSAT(400000000));
 
-	flows = minflow(tmpctx, gossmap,
-			gossmap_find_node(gossmap, &l1),
-			gossmap_find_node(gossmap, &l3),
-			chan_extra_map, NULL,
-			/* This will go first via 1-2-3, then 1->3. */
-			AMOUNT_MSAT(500000000),
-			 /* max_fee = */ AMOUNT_MSAT(1000000), // 1k sats
-			 /* min probability = */ 0.1, // 10%
-			 /* delay fee factor = */ 1,
-			 /* base fee penalty */ 1,
-			 /* prob cost factor = */ 10);
-
-	print_flows("Flow via two paths, low mu", gossmap, flows);
-
-	ASSERT(tal_count(flows) == 2);
-	ASSERT(tal_count(flows[0]->path) == 2);
-	ASSERT(tal_count(flows[0]->dirs) == 2);
-	ASSERT(tal_count(flows[0]->amounts) == 2);
-
-	ASSERT(flows[0]->path[0] == gossmap_find_chan(gossmap, &scid12));
-	ASSERT(flows[0]->path[1] == gossmap_find_chan(gossmap, &scid23));
-	ASSERT(flows[0]->dirs[0] == 1);
-	ASSERT(flows[0]->dirs[1] == 0);
-
-	/* First one has probability ~ 50% */
-	ASSERT(flows[0]->success_prob < 0.55);
-	ASSERT(flows[0]->success_prob > 0.45);
-
-	ASSERT(tal_count(flows[1]->path) == 1);
-	ASSERT(tal_count(flows[1]->dirs) == 1);
-	ASSERT(tal_count(flows[1]->amounts) == 1);
-
-	/* We will try cheaper path first, but not to fill it! */
-	ASSERT(flows[1]->path[0] == gossmap_find_chan(gossmap, &scid13));
-	ASSERT(flows[1]->dirs[0] == 0);
-	ASSERT(amount_msat_less(flows[1]->amounts[0], AMOUNT_MSAT(400000000)));
-
-	/* Second one has probability ~ 50% */
-	ASSERT(flows[1]->success_prob < 0.55);
-	ASSERT(flows[1]->success_prob > 0.45);
-
-	/* Delivered amount must be the total! */
-	ASSERT(flows[0]->amounts[1].millisatoshis
-	       + flows[1]->amounts[0].millisatoshis == 500000000);
-
-	/* Clear them. */
-	remove_completed_flow(gossmap, chan_extra_map, flows[0]);
-	remove_completed_flow(gossmap, chan_extra_map, flows[1]);
+	// flows = minflow(tmpctx, gossmap,
+	// 		gossmap_find_node(gossmap, &l1),
+	// 		gossmap_find_node(gossmap, &l3),
+	// 		chan_extra_map, NULL,
+	// 		/* This will go first via 1-2-3, then 1->3. */
+	// 		AMOUNT_MSAT(500000000),
+	// 		 /* max_fee = */ AMOUNT_MSAT(1000000), // 1k sats
+	// 		 /* min probability = */ 0.4,
+	// 		 /* delay fee factor = */ 1,
+	// 		 /* base fee penalty */ 1,
+	// 		 /* prob cost factor = */ 10);
+
+	// print_flows("Flow via two paths, low mu", gossmap, flows);
+
+	// assert(tal_count(flows) == 2);
+	// 
+	// if(tal_count(flows[0]->path)<tal_count(flows[1]->path))
+	// {
+	// 	struct flow* tmp = flows[0];
+	// 	flows[0] = flows[1];
+	// 	flows[1]=tmp;
+	// }
+	// 
+	// assert(tal_count(flows[0]->path) == 2);
+	// assert(tal_count(flows[0]->dirs) == 2);
+	// assert(tal_count(flows[0]->amounts) == 2);
+
+	// assert(flows[0]->path[0] == gossmap_find_chan(gossmap, &scid12));
+	// assert(flows[0]->path[1] == gossmap_find_chan(gossmap, &scid23));
+	// assert(flows[0]->dirs[0] == 1);
+	// assert(flows[0]->dirs[1] == 0);
+
+	// /* First one has probability ~ 50% */
+	// assert(flows[0]->success_prob < 0.55);
+	// assert(flows[0]->success_prob > 0.45);
+
+	// assert(tal_count(flows[1]->path) == 1);
+	// assert(tal_count(flows[1]->dirs) == 1);
+	// assert(tal_count(flows[1]->amounts) == 1);
+
+	// /* We will try cheaper path first, but not to fill it! */
+	// assert(flows[1]->path[0] == gossmap_find_chan(gossmap, &scid13));
+	// assert(flows[1]->dirs[0] == 0);
+	// assert(amount_msat_less(flows[1]->amounts[0], AMOUNT_MSAT(400000000)));
+
+	// /* Second one has probability ~ 50% */
+	// assert(flows[1]->success_prob < 0.55);
+	// assert(flows[1]->success_prob > 0.45);
+
+	// /* Delivered amount must be the total! */
+	// assert(flows[0]->amounts[1].millisatoshis
+	//        + flows[1]->amounts[0].millisatoshis == 500000000);
+
+	// /* Clear them. */
+	// remove_completed_flow(gossmap, chan_extra_map, flows[0]);
+	// remove_completed_flow(gossmap, chan_extra_map, flows[1]);
 
 	/* Higher mu values mean we pay more for certainty! */
-	flows2 = minflow(tmpctx, gossmap,
+	struct flow **flows2 = minflow(tmpctx, gossmap,
 			 gossmap_find_node(gossmap, &l1),
 			 gossmap_find_node(gossmap, &l3),
 			 chan_extra_map, NULL,
@@ -446,21 +457,21 @@ int main(int argc, char *argv[])
 			 /* base fee penalty */ 1,
 			 /* prob cost factor = */ 10);
 	print_flows("Flow via two paths, high mu", gossmap, flows2);
-	ASSERT(tal_count(flows2) == 2);
-	ASSERT(tal_count(flows2[0]->path) == 1);
-	ASSERT(tal_count(flows2[1]->path) == 2);
+	assert(tal_count(flows2) == 2);
+	assert(tal_count(flows2[0]->path) == 1);
+	assert(tal_count(flows2[1]->path) == 2);
 
-	/* Sends more via 1->3, since it's more expensive (but lower prob) */
-	ASSERT(amount_msat_greater(flows2[0]->amounts[0], flows[1]->amounts[0]));
-	ASSERT(flows2[0]->success_prob < flows[1]->success_prob);
+	// /* Sends more via 1->3, since it's more expensive (but lower prob) */
+	assert(amount_msat_greater(flows2[0]->amounts[0], flows2[1]->amounts[0]));
+	assert(flows2[0]->success_prob < flows2[1]->success_prob);
 
 	/* Delivered amount must be the total! */
-	ASSERT(flows2[0]->amounts[0].millisatoshis
+	assert(flows2[0]->amounts[0].millisatoshis
 	       + flows2[1]->amounts[1].millisatoshis == 500000000);
 
-	/* But in total it's more expensive! */
-	ASSERT(flows2[0]->amounts[0].millisatoshis + flows2[1]->amounts[0].millisatoshis
-	       > flows[0]->amounts[0].millisatoshis - flows[1]->amounts[0].millisatoshis);
+	// /* But in total it's more expensive! */
+	assert(flows2[0]->amounts[0].millisatoshis + flows2[1]->amounts[0].millisatoshis
+	       > flows2[0]->amounts[0].millisatoshis - flows2[1]->amounts[0].millisatoshis);
 
 
 	common_shutdown();

From ea93f9e766a94a9e1c8b0ecc4b4e4f9ed8d7b11d Mon Sep 17 00:00:00 2001
From: Lagrang3 <eduardo.quintana@pm.me>
Date: Fri, 12 May 2023 09:26:21 +0100
Subject: [PATCH 24/64] renepay: unit test and bugfix dijkstra heap

---
 plugins/renepay/dijkstra.c          |  80 ++++++++++++++---
 plugins/renepay/dijkstra.h          |  21 +++--
 plugins/renepay/mcf.c               |   2 +-
 plugins/renepay/test/run-dijkstra.c | 131 ++++++++++++++++++++++++++++
 4 files changed, 216 insertions(+), 18 deletions(-)
 create mode 100644 plugins/renepay/test/run-dijkstra.c

diff --git a/plugins/renepay/dijkstra.c b/plugins/renepay/dijkstra.c
index 56d93061b5fe..2f1b9574f63d 100644
--- a/plugins/renepay/dijkstra.c
+++ b/plugins/renepay/dijkstra.c
@@ -27,13 +27,31 @@ static void dijkstra_item_mover(void *const dst, const void *const src)
 	global_dijkstra->heapptr[src_idx] = dst;
 }
 
+static void dijkstra_destroy(struct dijkstra *ptr UNUSED)
+{
+	global_dijkstra=NULL;
+}
+
+void dijkstra_free(void)
+{
+	if(global_dijkstra)
+	{
+		tal_free(global_dijkstra);
+		assert(global_dijkstra==NULL);
+	}
+}
+
 /* Allocation of resources for the heap. */
 void dijkstra_malloc(const tal_t *ctx, const size_t max_num_nodes)
 {
+	dijkstra_free();
+	
 	global_dijkstra = tal(ctx,struct dijkstra);
+	tal_add_destructor(global_dijkstra,dijkstra_destroy);		
+	
 	global_dijkstra->distance = tal_arr(global_dijkstra,s64,max_num_nodes);
 	global_dijkstra->base = tal_arr(global_dijkstra,u32,max_num_nodes);
-	global_dijkstra->heapptr = tal_arr(global_dijkstra,u32*,max_num_nodes);
+	global_dijkstra->heapptr = tal_arrz(global_dijkstra,u32*,max_num_nodes);
 	
 	global_dijkstra->heapsize=0;
 	
@@ -45,7 +63,7 @@ void dijkstra_malloc(const tal_t *ctx, const size_t max_num_nodes)
 	global_dijkstra->gheap_ctx.item_mover=dijkstra_item_mover;
 }
 
-/* Initialization of the heap for a new Dijkstra search. */
+
 void dijkstra_init(void)
 {
 	const size_t max_num_nodes = tal_count(global_dijkstra->distance);
@@ -56,9 +74,21 @@ void dijkstra_init(void)
 		global_dijkstra->heapptr[i] = NULL;
 	}
 }
+size_t dijkstra_size(void)
+{
+	return global_dijkstra->heapsize;
+}
 
-void dijkstra_append(u32 node_idx, s64 distance)
+size_t dijkstra_maxsize(void)
 {
+	return tal_count(global_dijkstra->distance);
+}
+
+static void dijkstra_append(u32 node_idx, s64 distance)
+{
+	assert(dijkstra_size() < dijkstra_maxsize());
+	assert(node_idx < dijkstra_maxsize());
+	
 	const size_t pos = global_dijkstra->heapsize;
 	
 	global_dijkstra->base[pos]=node_idx;
@@ -68,22 +98,48 @@ void dijkstra_append(u32 node_idx, s64 distance)
 }
 void dijkstra_update(u32 node_idx, s64 distance)
 {
+	assert(node_idx < dijkstra_maxsize());
+	
 	if(!global_dijkstra->heapptr[node_idx])
 	{
 		// not in the heap
 		dijkstra_append(node_idx,distance);
+		gheap_restore_heap_after_item_increase(
+			&global_dijkstra->gheap_ctx,
+			global_dijkstra->base,
+			global_dijkstra->heapsize,
+			global_dijkstra->heapptr[node_idx]
+				- global_dijkstra->base);
+		return;
 	}
-	assert(node_idx < tal_count(global_dijkstra->distance));
-	assert(global_dijkstra->distance[node_idx] >= distance);
 	
-	global_dijkstra->distance[node_idx] = distance;
+	if(global_dijkstra->distance[node_idx] > distance)
+	{
+		// distance decrease
+		global_dijkstra->distance[node_idx] = distance;
+		
+		gheap_restore_heap_after_item_increase(
+			&global_dijkstra->gheap_ctx,
+			global_dijkstra->base,
+			global_dijkstra->heapsize,
+			global_dijkstra->heapptr[node_idx]
+				- global_dijkstra->base);
+	}else
+	{
+		// distance increase
+		global_dijkstra->distance[node_idx] = distance;
+		
+		gheap_restore_heap_after_item_decrease(
+			&global_dijkstra->gheap_ctx,
+			global_dijkstra->base,
+			global_dijkstra->heapsize,
+			global_dijkstra->heapptr[node_idx]
+				- global_dijkstra->base);
 	
-	gheap_restore_heap_after_item_increase(
-		&global_dijkstra->gheap_ctx,
-		global_dijkstra->base,
-		global_dijkstra->heapsize,
-		global_dijkstra->heapptr[node_idx]
-			- global_dijkstra->base);
+	}
+  	// assert(gheap_is_heap(&global_dijkstra->gheap_ctx, 
+	//                      global_dijkstra->base, 
+	// 		     dijkstra_size()));
 }
 u32 dijkstra_top(void)
 {
diff --git a/plugins/renepay/dijkstra.h b/plugins/renepay/dijkstra.h
index ae00655f5062..1f351f1d3be8 100644
--- a/plugins/renepay/dijkstra.h
+++ b/plugins/renepay/dijkstra.h
@@ -1,9 +1,6 @@
 #ifndef LIGHTNING_PLUGINS_RENEPAY_DIJKSTRA_H
 #define LIGHTNING_PLUGINS_RENEPAY_DIJKSTRA_H
 
-// TODO(eduardo): unit test this
-
-
 #include <ccan/short_types/short_types.h>
 #include <ccan/tal/tal.h>
 #include <gheap.h>
@@ -11,7 +8,10 @@
 /* In the heap we keep node idx, but in this structure we keep the distance
  * value associated to every node, and their position in the heap as a pointer
  * so that we can update the nodes inside the heap when the distance label is
- * changed. */
+ * changed. 
+ * 
+ * Therefore this is no longer a multipurpose heap, the node_idx must be an
+ * index between 0 and less than max_num_nodes. */
 struct dijkstra {
 	// 
 	s64 *distance;
@@ -24,15 +24,26 @@ struct dijkstra {
 /* Allocation of resources for the heap. */
 void dijkstra_malloc(const tal_t *ctx, const size_t max_num_nodes);
 
+/* Free resources */
+void dijkstra_free(void);
+
 /* Initialization of the heap for a new Dijkstra search. */
 void dijkstra_init(void);
 
-void dijkstra_append(u32 node_idx, s64 distance);
+/* Inserts a new element in the heap. If node_idx was already in the heap then
+ * its distance value is updated. */
 void dijkstra_update(u32 node_idx, s64 distance);
+
 u32 dijkstra_top(void);
 bool dijkstra_empty(void);
 void dijkstra_pop(void);
 
 const s64* dijkstra_distance_data(void);
 
+/* Number of elements on the heap. */
+size_t dijkstra_size(void);
+
+/* Maximum number of elements the heap can host */
+size_t dijkstra_maxsize(void);
+
 #endif // LIGHTNING_PLUGINS_RENEPAY_DIJKSTRA_H
diff --git a/plugins/renepay/mcf.c b/plugins/renepay/mcf.c
index 49fb3c6d867f..19fae7413110 100644
--- a/plugins/renepay/mcf.c
+++ b/plugins/renepay/mcf.c
@@ -859,7 +859,7 @@ static int  find_optimal_path(
 	s64 const * const distance=dijkstra_distance_data();
 	
 	dijkstra_init();
-	dijkstra_append(source,0);
+	dijkstra_update(source,0);
 	
 	while(!dijkstra_empty())
 	{
diff --git a/plugins/renepay/test/run-dijkstra.c b/plugins/renepay/test/run-dijkstra.c
new file mode 100644
index 000000000000..b97c2af686cd
--- /dev/null
+++ b/plugins/renepay/test/run-dijkstra.c
@@ -0,0 +1,131 @@
+#include "config.h"
+#include <stdio.h>
+#include <assert.h>
+#include <common/wireaddr.h>
+#include <common/bigsize.h>
+#include <common/channel_id.h>
+#include <common/setup.h>
+#include <common/utils.h>
+
+#include <plugins/renepay/dijkstra.h>
+
+// #include <ccan/array_size/array_size.h>
+// #include <ccan/read_write_all/read_write_all.h>
+// #include <common/gossip_store.h>
+// #include <common/node_id.h>
+// #include <common/type_to_string.h>
+
+// #include <bitcoin/short_channel_id.h>
+// #include <ccan/htable/htable_type.h>
+
+/* AUTOGENERATED MOCKS START */
+/* Generated stub for fromwire_bigsize */
+bigsize_t fromwire_bigsize(const u8 **cursor UNNEEDED, size_t *max UNNEEDED)
+{ fprintf(stderr, "fromwire_bigsize called!\n"); abort(); }
+/* Generated stub for fromwire_channel_id */
+bool fromwire_channel_id(const u8 **cursor UNNEEDED, size_t *max UNNEEDED,
+			 struct channel_id *channel_id UNNEEDED)
+{ fprintf(stderr, "fromwire_channel_id called!\n"); abort(); }
+/* Generated stub for fromwire_wireaddr */
+bool fromwire_wireaddr(const u8 **cursor UNNEEDED, size_t *max UNNEEDED, struct wireaddr *addr UNNEEDED)
+{ fprintf(stderr, "fromwire_wireaddr called!\n"); abort(); }
+/* Generated stub for towire_bigsize */
+void towire_bigsize(u8 **pptr UNNEEDED, const bigsize_t val UNNEEDED)
+{ fprintf(stderr, "towire_bigsize called!\n"); abort(); }
+/* Generated stub for towire_channel_id */
+void towire_channel_id(u8 **pptr UNNEEDED, const struct channel_id *channel_id UNNEEDED)
+{ fprintf(stderr, "towire_channel_id called!\n"); abort(); }
+/* Generated stub for towire_wireaddr */
+void towire_wireaddr(u8 **pptr UNNEEDED, const struct wireaddr *addr UNNEEDED)
+{ fprintf(stderr, "towire_wireaddr called!\n"); abort(); }
+/* AUTOGENERATED MOCKS END */
+
+static void insertion_in_increasing_distance(const tal_t *ctx)
+{
+	dijkstra_malloc(ctx,10);
+	
+	for(int i=0;i<dijkstra_maxsize();++i)
+	{
+		dijkstra_update(i,10+i);
+		assert(dijkstra_size()==(i+1));
+	}
+	
+	dijkstra_update(3,3);
+	assert(dijkstra_top()==3);
+	
+	dijkstra_update(3,15);
+	assert(dijkstra_top()==0);
+	
+	dijkstra_update(3,-1);
+	assert(dijkstra_top()==3);
+	
+	dijkstra_pop();
+	assert(dijkstra_size()==9);
+	assert(dijkstra_top()==0);
+	
+	// Insert again
+	dijkstra_update(3,3+10);
+
+	u32 top=0;
+	while(!dijkstra_empty())
+	{
+		assert(top==dijkstra_top());
+		top++;
+		dijkstra_pop();
+	}
+}
+static void insertion_in_decreasing_distance(const tal_t *ctx)
+{
+	dijkstra_malloc(ctx,10);
+	
+	for(int i=0;i<dijkstra_maxsize();++i)
+	{
+		dijkstra_update(i,10-i);
+		assert(dijkstra_size()==(i+1));
+	}
+	
+	dijkstra_update(3,-3);
+	assert(dijkstra_top()==3);
+	
+	dijkstra_update(3,15);
+	assert(dijkstra_top()==9);
+	
+	dijkstra_update(3,-1);
+	assert(dijkstra_top()==3);
+	
+	dijkstra_pop();
+	assert(dijkstra_size()==9);
+	assert(dijkstra_top()==9);
+	
+	// Insert again
+	dijkstra_update(3,10-3);
+
+	u32 top=9;
+	while(!dijkstra_empty())
+	{
+		assert(top==dijkstra_top());
+		top--;
+		dijkstra_pop();
+	}
+}
+
+int main(int argc, char *argv[])
+{
+	common_setup(argv[0]);
+	
+	insertion_in_increasing_distance(NULL);
+	insertion_in_decreasing_distance(tmpctx);
+	
+	// test dijkstra_free
+	dijkstra_free();
+	// we can call it twice, no problem
+	dijkstra_free();
+	
+	// does tal_free() cleansup correctly?
+	const tal_t *this_ctx = tal(NULL,tal_t);
+	insertion_in_increasing_distance(this_ctx);
+	tal_free(this_ctx);
+	insertion_in_decreasing_distance(tmpctx);
+	
+	common_shutdown();
+}

From 8562901bf243007f7608d20d32f8ea91ac17860b Mon Sep 17 00:00:00 2001
From: Lagrang3 <eduardo.quintana@pm.me>
Date: Fri, 12 May 2023 10:06:14 +0100
Subject: [PATCH 25/64] renepay: testing order of destructor calls

---
 plugins/renepay/flow.c         |  2 +-
 plugins/renepay/flow.h         |  1 -
 plugins/renepay/test/run-map.c | 36 ++++++++++++++++++++++++++++++----
 3 files changed, 33 insertions(+), 6 deletions(-)

diff --git a/plugins/renepay/flow.c b/plugins/renepay/flow.c
index 3b3c2d3ecf5c..5bb154a139f6 100644
--- a/plugins/renepay/flow.c
+++ b/plugins/renepay/flow.c
@@ -596,7 +596,7 @@ s64 linear_fee_cost(
 	    bfee = c->half[dir].base_fee,
 	    delay = c->half[dir].delay;
 	
-	return pfee + (bfee + delay_feefactor*delay) * base_fee_penalty;
+	return pfee + bfee* base_fee_penalty+ delay*delay_feefactor;
 }
 
 struct amount_msat flows_fee(struct flow **flows)
diff --git a/plugins/renepay/flow.h b/plugins/renepay/flow.h
index 6f097f5a095d..ca44ff687c4f 100644
--- a/plugins/renepay/flow.h
+++ b/plugins/renepay/flow.h
@@ -174,7 +174,6 @@ double derive_mu(const struct gossmap *gossmap,
 		 struct amount_msat amount,
 		 double frugality);
 
-// TODO(eduardo):
 s64 linear_fee_cost(
 		const struct gossmap_chan *c,
 		const int dir,
diff --git a/plugins/renepay/test/run-map.c b/plugins/renepay/test/run-map.c
index 732418f95d6e..2d70d529ab01 100644
--- a/plugins/renepay/test/run-map.c
+++ b/plugins/renepay/test/run-map.c
@@ -145,7 +145,7 @@ static void valgrind_ok2(void)
 	tal_t *this_ctx = tal(tmpctx,tal_t);
 	
 	struct chan_extra_map *chan_extra_map
-		= tal(tmpctx, struct chan_extra_map);
+		= tal(this_ctx, struct chan_extra_map);
 	
 	chan_extra_map_init(chan_extra_map);
 	
@@ -178,7 +178,7 @@ static void valgrind_fail3(void)
 	tal_t *this_ctx = tal(tmpctx,tal_t);
 	
 	struct chan_extra_map *chan_extra_map
-		= tal(tmpctx, struct chan_extra_map);
+		= tal(this_ctx, struct chan_extra_map);
 	
 	chan_extra_map_init(chan_extra_map);
 	
@@ -200,10 +200,36 @@ static void valgrind_fail3(void)
 	/* Valgrind complains. It seems that the hash table is trying to remove
 	 * the element at a moment when its memory has already been released. 
 	 * If the following two lines are not commented the error disapears. */
-	// tal_free(x1);
-	// tal_free(x2);
+	tal_free(x1);
+	tal_free(x2);
+	
+	tal_free(this_ctx);
+}
+
+static void destroy_int(int* x)
+{
+	fprintf(stderr,"Destroying %d\n",*x);
+}
+
+/* This clearly shows that the parent is destroyed before the children. 
+ * That's the reason why the chan_extra destructor fails. */
+static void test_order(void)
+{
+	const tal_t *this_ctx = tal(tmpctx,tal_t);
+	
+	int *parent = tal(this_ctx,int);
+	tal_add_destructor(parent,destroy_int);
+	*parent = 1;
+	
+	int *child = tal(parent,int);
+	tal_add_destructor(child,destroy_int);
+	*child=2;
 	
 	tal_free(this_ctx);
+	
+	// prints:
+	// Destroying 1
+	// Destroying 2
 }
 
 int main(int argc, char *argv[])
@@ -212,6 +238,8 @@ int main(int argc, char *argv[])
 	valgrind_ok1();
 	valgrind_ok2();
 	valgrind_fail3();
+	
+	test_order();
 	common_shutdown();
 }
 

From 72702a4fc26ca915d8d0130a52b40ca47ae4225d Mon Sep 17 00:00:00 2001
From: Lagrang3 <eduardo.quintana@pm.me>
Date: Fri, 12 May 2023 15:11:59 +0100
Subject: [PATCH 26/64] renepay: unittest edge_probability

---
 plugins/renepay/flow.c              |  66 +++++++-----
 plugins/renepay/flow.h              |   1 +
 plugins/renepay/test/run-testflow.c | 155 ++++++++++++++++++++++++++++
 3 files changed, 196 insertions(+), 26 deletions(-)
 create mode 100644 plugins/renepay/test/run-testflow.c

diff --git a/plugins/renepay/flow.c b/plugins/renepay/flow.c
index 5bb154a139f6..51ebed6266e4 100644
--- a/plugins/renepay/flow.c
+++ b/plugins/renepay/flow.c
@@ -20,7 +20,6 @@ const struct half_chan *flow_edge(const struct flow *flow, size_t idx)
 	return &flow->path[idx]->half[flow->dirs[idx]];
 }
 
-// TODO(eduardo): check this
 /* Assuming a uniform distribution, what is the chance this f gets through?
  * Here we compute the conditional probability of success for a flow f, given
  * the knowledge that the liquidity is in the range [a,b) and some amount
@@ -31,7 +30,8 @@ const struct half_chan *flow_edge(const struct flow *flow, size_t idx)
  * 	prob(f) = 
  * 	
  * 	for f<a:	1.
- * 	for f>=a:	(b-f)/(b-a)
+ * 	for b>=f>=a:	(b-f)/(b-a)
+ * 	for b<f:	0.
  * 
  * When x>0 the prob. of success for passing x and f is:
  * 
@@ -43,13 +43,14 @@ const struct half_chan *flow_edge(const struct flow *flow, size_t idx)
  * 
  * The purpose of this function is to obtain prob(f|x), i.e. the probability of
  * getting f through provided that we already succeeded in getting x.
- * This conditional probability comes with three cases:
+ * This conditional probability comes with 4 cases:
  * 
  * 	prob(f|x) = 
  * 	
  * 	for x<a and f<a-x: 	1.
  * 	for x<a and f>=a-x:	(b-x-f)/(b-a)
  * 	for x>=a:		(b-x-f)/(b-x)
+ * 	for f>b-x:		0.
  *
  * This is the same as the probability of success of f when the bounds are
  * shifted by x amount, the new bounds be [MAX(0,a-x),b-x).
@@ -58,22 +59,32 @@ static double edge_probability(struct amount_msat min, struct amount_msat max,
 			       struct amount_msat in_flight,
 			       struct amount_msat f)
 {
-	//printf("%s: with min=%ld, max=%ld, in_flight=%ld, flow=%ld\n",
-	//	__PRETTY_FUNCTION__,min.millisatoshis,max.millisatoshis,
-	//	in_flight.millisatoshis,f.millisatoshis);
+	assert(amount_msat_less_eq(min,max));
+	assert(amount_msat_less_eq(in_flight,max));
 	
+	const tal_t *this_ctx = tal(tmpctx,tal_t);
+	
+	// SUPERVERBOSE("%s: with min=%ld, max=%ld, in_flight=%ld, flow=%ld\n",
+	// 	__PRETTY_FUNCTION__,min.millisatoshis,max.millisatoshis,
+	// 	in_flight.millisatoshis,f.millisatoshis);
+	
+	const struct amount_msat one = AMOUNT_MSAT(1);
 	struct amount_msat B=max; // =  max +1 - in_flight
 	
 	// one past the last known value, makes computations simpler
-	if(!amount_msat_add(&B,B,AMOUNT_MSAT(1)))
+	if(!amount_msat_add(&B,B,one))
 	{
-		printf("%s: aborting, lineno=%d\n",__PRETTY_FUNCTION__,__LINE__);
+		SUPERVERBOSE("%s: aborting, cannot add %s + %s\n",__PRETTY_FUNCTION__,
+			   type_to_string(this_ctx, struct amount_msat, &B),
+			   type_to_string(this_ctx, struct amount_msat, &one));
 		abort();
 	}	
 	// in_flight cannot be greater than max
 	if(!amount_msat_sub(&B,B,in_flight))
 	{
-		printf("%s: aborting, lineno=%d\n",__PRETTY_FUNCTION__,__LINE__);
+		SUPERVERBOSE("%s: aborting, cannot substract %s - %s\n",__PRETTY_FUNCTION__,
+			   type_to_string(this_ctx, struct amount_msat, &B),
+			   type_to_string(this_ctx, struct amount_msat, &in_flight));
 		abort();
 	}	
 	struct amount_msat A=min; // = MAX(0,min-in_flight);
@@ -86,7 +97,9 @@ static double edge_probability(struct amount_msat min, struct amount_msat max,
 	// B cannot be smaller than or equal A
 	if(!amount_msat_sub(&denominator,B,A) || amount_msat_less_eq(B,A))
 	{
-		printf("%s: aborting, lineno=%d\n",__PRETTY_FUNCTION__,__LINE__);
+		SUPERVERBOSE("%s: aborting, cannot substract %s - %s\n",__PRETTY_FUNCTION__,
+			   type_to_string(this_ctx, struct amount_msat, &B),
+			   type_to_string(this_ctx, struct amount_msat, &A));
 		abort();
 	}
 	struct amount_msat numerator; // MAX(0,B-f)
@@ -94,6 +107,7 @@ static double edge_probability(struct amount_msat min, struct amount_msat max,
 	if(!amount_msat_sub(&numerator,B,f))
 		numerator = AMOUNT_MSAT(0);
 	
+	tal_free(this_ctx);
 	return amount_msat_less_eq(f,A) ? 1.0 : amount_msat_ratio(numerator,denominator);
 }
 
@@ -165,7 +179,7 @@ get_chan_extra_half_by_chan_verify(
 			cap = AMOUNT_SAT(0);
 		if (!amount_sat_to_msat(&cap_msat, cap))
 		{
-			printf("%s: aborting\n",__PRETTY_FUNCTION__);
+			SUPERVERBOSE("%s: aborting\n",__PRETTY_FUNCTION__);
 			abort();
 		}
 		h = new_chan_extra_half(chan_extra_map,
@@ -210,12 +224,12 @@ void remove_completed_flow(const struct gossmap *gossmap,
 							       flow->dirs[i]);
 		if (!amount_msat_sub(&h->htlc_total, h->htlc_total, flow->amounts[i]))
 		{
-			printf("%s: aborting\n",__PRETTY_FUNCTION__);
+			SUPERVERBOSE("%s: aborting\n",__PRETTY_FUNCTION__);
 			abort();
 		}
 		if (h->num_htlcs == 0)
 		{
-			printf("%s: aborting\n",__PRETTY_FUNCTION__);
+			SUPERVERBOSE("%s: aborting\n",__PRETTY_FUNCTION__);
 			abort();
 		}
 		h->num_htlcs--;
@@ -244,7 +258,7 @@ void commit_flow(
 							       flow->dirs[i]);
 		if (!amount_msat_add(&h->htlc_total, h->htlc_total, flow->amounts[i]))
 		{
-			printf("%s: aborting\n",__PRETTY_FUNCTION__);
+			SUPERVERBOSE("%s: aborting\n",__PRETTY_FUNCTION__);
 			abort();
 		}
 		h->num_htlcs++;
@@ -273,7 +287,7 @@ void flow_add(struct flow *flow,
 	if (!amount_msat_add(&delivered,
 			     flow->amounts[tal_count(flow->amounts)-1], additional))
 	{
-		printf("%s: aborting\n",__PRETTY_FUNCTION__);
+		SUPERVERBOSE("%s: aborting\n",__PRETTY_FUNCTION__);
 		abort();
 	}
 	/* Remove original from current_flows */
@@ -352,7 +366,7 @@ void flow_complete(struct flow *flow,
 		   struct chan_extra_map *chan_extra_map,
 		   struct amount_msat delivered)
 {
-	// printf("%s:\n",__PRETTY_FUNCTION__);
+	// SUPERVERBOSE("%s:\n",__PRETTY_FUNCTION__);
 	flow->success_prob = 1.0;
 	flow->amounts = tal_arr(flow, struct amount_msat, tal_count(flow->path));
 	for (int i = tal_count(flow->path) - 1; i >= 0; i--) {
@@ -368,12 +382,12 @@ void flow_complete(struct flow *flow,
 					    h->htlc_total,
 					    delivered);
 					    
-		// printf("(%s, ",type_to_string(tmpctx,struct amount_msat,&delivered));
-		// printf("%.2f)--",flow->success_prob);
+		// SUPERVERBOSE("(%s, ",type_to_string(tmpctx,struct amount_msat,&delivered));
+		// SUPERVERBOSE("%.2f)--",flow->success_prob);
 		
 		// if (!amount_msat_add(&h->htlc_total, h->htlc_total, delivered))
 		// {
-		// 	printf("%s: aborting\n",__PRETTY_FUNCTION__);
+		// 	SUPERVERBOSE("%s: aborting\n",__PRETTY_FUNCTION__);
 		// 	abort();
 		// }
 		// h->num_htlcs++;
@@ -381,11 +395,11 @@ void flow_complete(struct flow *flow,
 					 flow_edge(flow, i)->base_fee,
 					 flow_edge(flow, i)->proportional_fee))
 		{
-			printf("%s: aborting\n",__PRETTY_FUNCTION__);
+			SUPERVERBOSE("%s: aborting\n",__PRETTY_FUNCTION__);
 			abort();
 		}
 	}
-	// printf("\n%s: finished\n",__PRETTY_FUNCTION__);
+	// SUPERVERBOSE("\n%s: finished\n",__PRETTY_FUNCTION__);
 }
 
 /* Compute the prob. of success of a set of concurrent set of flows. 
@@ -453,7 +467,7 @@ double flow_set_probability(
 			struct amount_msat prev_flow;
 			if(!amount_msat_add(&prev_flow,h->htlc_total,in_flight[c_idx].half[c_dir]))
 			{
-				printf("%s: aborting\n",__PRETTY_FUNCTION__);
+				SUPERVERBOSE("%s: aborting\n",__PRETTY_FUNCTION__);
 				abort();
 			}
 			
@@ -464,7 +478,7 @@ double flow_set_probability(
 					in_flight[c_idx].half[c_dir],
 					deliver))
 			{
-				printf("%s: aborting\n",__PRETTY_FUNCTION__);
+				SUPERVERBOSE("%s: aborting\n",__PRETTY_FUNCTION__);
 				abort();
 			}
 		}
@@ -545,7 +559,7 @@ static void get_medians(const struct gossmap *gossmap,
 		*median_capacity = amount;
 	else if (!amount_sat_to_msat(median_capacity, caps[num_caps / 2]))
 	{
-		printf("%s: aborting\n",__PRETTY_FUNCTION__);
+		SUPERVERBOSE("%s: aborting\n",__PRETTY_FUNCTION__);
 		abort();
 	}
 	asort(fees, num_fees, cmp_amount_msat, NULL);
@@ -611,12 +625,12 @@ struct amount_msat flows_fee(struct flow **flows)
 				     flows[i]->amounts[0],
 				     flows[i]->amounts[n-1]))
 		{
-			printf("%s: aborting\n",__PRETTY_FUNCTION__);
+			SUPERVERBOSE("%s: aborting\n",__PRETTY_FUNCTION__);
 			abort();
 		}
 		if(!amount_msat_add(&fee, this_fee,fee))
 		{
-			printf("%s: aborting\n",__PRETTY_FUNCTION__);
+			SUPERVERBOSE("%s: aborting\n",__PRETTY_FUNCTION__);
 			abort();
 		}
 	}
diff --git a/plugins/renepay/flow.h b/plugins/renepay/flow.h
index ca44ff687c4f..8f35f0b3aaae 100644
--- a/plugins/renepay/flow.h
+++ b/plugins/renepay/flow.h
@@ -4,6 +4,7 @@
 #include <bitcoin/short_channel_id.h>
 #include <ccan/htable/htable_type.h>
 #include <common/amount.h>
+#include <common/gossmap.h>
 
 /* Any implementation needs to keep some data on channels which are
  * in-use (or about which we have extra information).  We use a hash
diff --git a/plugins/renepay/test/run-testflow.c b/plugins/renepay/test/run-testflow.c
new file mode 100644
index 000000000000..457d3ee5dd90
--- /dev/null
+++ b/plugins/renepay/test/run-testflow.c
@@ -0,0 +1,155 @@
+#include "config.h"
+#include <stdio.h>
+#include <assert.h>
+#include <common/wireaddr.h>
+#include <common/bigsize.h>
+#include <common/channel_id.h>
+#include <common/setup.h>
+#include <common/utils.h>
+
+static bool print_enable = true;
+#define SUPERVERBOSE(...) do { if (print_enable) printf(__VA_ARGS__); } while(0)
+
+#include <plugins/renepay/flow.h>
+#include "../flow.c"
+
+/* AUTOGENERATED MOCKS START */
+/* Generated stub for fromwire_bigsize */
+bigsize_t fromwire_bigsize(const u8 **cursor UNNEEDED, size_t *max UNNEEDED)
+{ fprintf(stderr, "fromwire_bigsize called!\n"); abort(); }
+/* Generated stub for fromwire_channel_id */
+bool fromwire_channel_id(const u8 **cursor UNNEEDED, size_t *max UNNEEDED,
+			 struct channel_id *channel_id UNNEEDED)
+{ fprintf(stderr, "fromwire_channel_id called!\n"); abort(); }
+/* Generated stub for fromwire_wireaddr */
+bool fromwire_wireaddr(const u8 **cursor UNNEEDED, size_t *max UNNEEDED, struct wireaddr *addr UNNEEDED)
+{ fprintf(stderr, "fromwire_wireaddr called!\n"); abort(); }
+/* Generated stub for towire_bigsize */
+void towire_bigsize(u8 **pptr UNNEEDED, const bigsize_t val UNNEEDED)
+{ fprintf(stderr, "towire_bigsize called!\n"); abort(); }
+/* Generated stub for towire_channel_id */
+void towire_channel_id(u8 **pptr UNNEEDED, const struct channel_id *channel_id UNNEEDED)
+{ fprintf(stderr, "towire_channel_id called!\n"); abort(); }
+/* Generated stub for towire_wireaddr */
+void towire_wireaddr(u8 **pptr UNNEEDED, const struct wireaddr *addr UNNEEDED)
+{ fprintf(stderr, "towire_wireaddr called!\n"); abort(); }
+/* AUTOGENERATED MOCKS END */
+
+static void test_edge_probability(void)
+{
+	const double eps = 1e-8;
+	
+	struct amount_msat min = AMOUNT_MSAT(10); // known min
+	struct amount_msat max = AMOUNT_MSAT(19); // known max
+	
+	struct amount_msat X = AMOUNT_MSAT(0); // in flight
+	struct amount_msat f;
+	
+	for(int i=0;i<=min.millisatoshis;++i)
+	{
+		f.millisatoshis = i;
+		// prob = 1
+		assert(fabs(edge_probability(min,max,X,f)-1.0)< eps);
+	}
+	for(int i=max.millisatoshis+1;i<=100;++i)
+	{
+		f.millisatoshis = i;
+		// prob = 0
+		assert(fabs(edge_probability(min,max,X,f))< eps);
+	}
+	f.millisatoshis=11;
+	assert(fabs(edge_probability(min,max,X,f)-0.9)< eps);
+	
+	f.millisatoshis=12;
+	assert(fabs(edge_probability(min,max,X,f)-0.8)< eps);
+	
+	f.millisatoshis=13;
+	assert(fabs(edge_probability(min,max,X,f)-0.7)< eps);
+	
+	f.millisatoshis=14;
+	assert(fabs(edge_probability(min,max,X,f)-0.6)< eps);
+	
+	f.millisatoshis=15;
+	assert(fabs(edge_probability(min,max,X,f)-0.5)< eps);
+	
+	f.millisatoshis=16;
+	assert(fabs(edge_probability(min,max,X,f)-0.4)< eps);
+	
+	f.millisatoshis=17;
+	assert(fabs(edge_probability(min,max,X,f)-0.3)< eps);
+	
+	f.millisatoshis=18;
+	assert(fabs(edge_probability(min,max,X,f)-0.2)< eps);
+	
+	f.millisatoshis=19;
+	assert(fabs(edge_probability(min,max,X,f)-0.1)< eps);
+	
+	X = AMOUNT_MSAT(5);
+	
+	// X<A, f<A-X
+	for(int i=0;i<=5;++i)
+	{
+		f.millisatoshis = i;
+		// prob = 1
+		assert(fabs(edge_probability(min,max,X,f)-1.0)< eps);
+	}
+	// X<A, f>=B-X
+	for(int i=15;i<100;++i)
+	{
+		f.millisatoshis = i;
+		// prob = 0
+		assert(fabs(edge_probability(min,max,X,f))< eps);
+	}
+	// X<A, A-X<=f<=B-X
+	f.millisatoshis=6;
+	assert(fabs(edge_probability(min,max,X,f)-0.9)< eps);
+	f.millisatoshis=7;
+	assert(fabs(edge_probability(min,max,X,f)-0.8)< eps);
+	f.millisatoshis=8;
+	assert(fabs(edge_probability(min,max,X,f)-0.7)< eps);
+	f.millisatoshis=9;
+	assert(fabs(edge_probability(min,max,X,f)-0.6)< eps);
+	f.millisatoshis=10;
+	assert(fabs(edge_probability(min,max,X,f)-0.5)< eps);
+	f.millisatoshis=11;
+	assert(fabs(edge_probability(min,max,X,f)-0.4)< eps);
+	f.millisatoshis=12;
+	assert(fabs(edge_probability(min,max,X,f)-0.3)< eps);
+	f.millisatoshis=13;
+	assert(fabs(edge_probability(min,max,X,f)-0.2)< eps);
+	f.millisatoshis=14;
+	assert(fabs(edge_probability(min,max,X,f)-0.1)< eps);
+	
+	X = AMOUNT_MSAT(15);
+	
+	// f>=B-X
+	for(int i=5;i<100;++i)
+	{
+		f.millisatoshis = i;
+		assert(fabs(edge_probability(min,max,X,f))< eps);
+	}
+	
+	// X>=A, 0<=f<=B-X
+	f.millisatoshis=0;
+	assert(fabs(edge_probability(min,max,X,f)-1.0)< eps);
+	f.millisatoshis=1;
+	assert(fabs(edge_probability(min,max,X,f)-0.8)< eps);
+	f.millisatoshis=2;
+	assert(fabs(edge_probability(min,max,X,f)-0.6)< eps);
+	f.millisatoshis=3;
+	assert(fabs(edge_probability(min,max,X,f)-0.4)< eps);
+	f.millisatoshis=4;
+	assert(fabs(edge_probability(min,max,X,f)-0.2)< eps);
+	f.millisatoshis=5;
+	assert(fabs(edge_probability(min,max,X,f)-0.0)< eps);
+}
+
+int main(int argc, char *argv[])
+{
+	common_setup(argv[0]);
+	
+	test_edge_probability();
+	
+	common_shutdown();
+}
+

From c9f8035faefab63cc0edf6dc60221b0f0b50ba70 Mon Sep 17 00:00:00 2001
From: Lagrang3 <eduardo.quintana@pm.me>
Date: Fri, 12 May 2023 15:34:24 +0100
Subject: [PATCH 27/64] renepay: remove unnecessary functions

---
 plugins/renepay/flow.c     | 109 +--------
 plugins/renepay/flow.h     |  14 +-
 plugins/renepay/mcf.c      |   4 +-
 plugins/renepay/not_mcf.c  | 485 -------------------------------------
 plugins/renepay/pay_flow.c |   2 +-
 5 files changed, 7 insertions(+), 607 deletions(-)
 delete mode 100644 plugins/renepay/not_mcf.c

diff --git a/plugins/renepay/flow.c b/plugins/renepay/flow.c
index 51ebed6266e4..0dc3bb81ee0f 100644
--- a/plugins/renepay/flow.c
+++ b/plugins/renepay/flow.c
@@ -111,22 +111,6 @@ static double edge_probability(struct amount_msat min, struct amount_msat max,
 	return amount_msat_less_eq(f,A) ? 1.0 : amount_msat_ratio(numerator,denominator);
 }
 
-bool flow_path_eq(const struct gossmap_chan **path1,
-		  const int *dirs1,
-		  const struct gossmap_chan **path2,
-		  const int *dirs2)
-{
-	if (tal_count(path1) != tal_count(path2))
-		return false;
-	for (size_t i = 0; i < tal_count(path1); i++) {
-		if (path1[i] != path2[i])
-			return false;
-		if (dirs1[i] != dirs2[i])
-			return false;
-	}
-	return true;
-}
-
 // static void destroy_chan_extra(struct chan_extra *ce,
 // 			       struct chan_extra_map *chan_extra_map)
 // {
@@ -275,83 +259,8 @@ void commit_flow_set(
 	}
 }
 
-/* Add this to the flow. */
-void flow_add(struct flow *flow,
-	      const struct gossmap *gossmap,
-	      struct chan_extra_map *chan_extra_map,
-	      struct amount_msat additional)
-{
-	struct amount_msat delivered;
-
-	/* Add in new amount */
-	if (!amount_msat_add(&delivered,
-			     flow->amounts[tal_count(flow->amounts)-1], additional))
-	{
-		SUPERVERBOSE("%s: aborting\n",__PRETTY_FUNCTION__);
-		abort();
-	}
-	/* Remove original from current_flows */
-	remove_completed_flow(gossmap, chan_extra_map, flow);
-
-	/* Recalc probability and fees, adjust chan_extra_map entries */
-	flow_complete(flow, gossmap, chan_extra_map, delivered);
-}
-
-/* From the paper:
- * −log((c_e + 1 − f_e) / (c_e + 1)) + μ f_e fee(e)
- */
-// double flow_edge_cost(const struct gossmap *gossmap,
-// 		      const struct gossmap_chan *c, int dir,
-// 		      const struct amount_msat known_min,
-// 		      const struct amount_msat known_max,
-// 		      struct amount_msat prev_flow,
-// 		      struct amount_msat f,
-// 		      double mu,
-// 		      double basefee_penalty,
-// 		      double delay_riskfactor)
-// {
-// 	double prob, effective_feerate;
-// 	double certainty_term, feerate_term;
-// 
-// #ifdef SUPERVERBOSE_ENABLED
-// 	struct short_channel_id scid
-// 		= gossmap_chan_scid(gossmap, c);
-// 	SUPERVERBOSE("flow_edge_cost %s/%i, cap %"PRIu64"-%"PRIu64", prev_flow=%"PRIu64", f=%"PRIu64", mu=%f, basefee_penalty=%f, delay_riskfactor=%f: ",
-// 		     type_to_string(tmpctx, struct short_channel_id, &scid),
-// 		     dir,
-// 		     known_min.millisatoshis, known_max.millisatoshis,
-// 		     prev_flow.millisatoshis, f.millisatoshis,
-// 		     mu, basefee_penalty, delay_riskfactor);
-// #endif
-// 
-// 	/* Probability depends on any previous flows, too! */
-// 	if (!amount_msat_add(&prev_flow, prev_flow, f))
-// 		abort();
-// 	prob = edge_probability(known_min, known_max, prev_flow);
-// 	if (prob == 0) {
-// 		SUPERVERBOSE(" INFINITE\n");
-// 		return FLOW_INF_COST;
-// 	}
-// 
-// 	certainty_term = -log(prob);
-// 
-// 	/* This is in parts-per-million */
-// 	effective_feerate = (c->half[dir].proportional_fee
-// 			     + c->half[dir].base_fee * basefee_penalty)
-// 		/ 1000000.0;
-// 
-// 	/* Feerate term includes delay factor */
-// 	feerate_term = (mu
-// 			* (f.millisatoshis /* Raw: costfn */
-// 			   * effective_feerate
-// 			   + c->half[dir].delay * delay_riskfactor));
-// 
-// 	SUPERVERBOSE(" %f + %f = %f\n",
-// 		     certainty_term, feerate_term,
-// 		     certainty_term + feerate_term);
-// 	return certainty_term + feerate_term;
-// }
-
+// TODO(eduardo): unit test this with a single path flow with known fees and
+// probabilities.
 /* Helper function to fill in amounts and success_prob for flow 
  * 
  * IMPORTANT: here we do not commit flows to chan_extra, flows are commited
@@ -359,14 +268,12 @@ void flow_add(struct flow *flow,
  * 
  * IMPORTANT: flow->success_prob is misleading, because that's the prob. of
  * success provided that there are no other flows in the current MPP flow set.
- * 
  * */
 void flow_complete(struct flow *flow,
 		   const struct gossmap *gossmap,
 		   struct chan_extra_map *chan_extra_map,
 		   struct amount_msat delivered)
 {
-	// SUPERVERBOSE("%s:\n",__PRETTY_FUNCTION__);
 	flow->success_prob = 1.0;
 	flow->amounts = tal_arr(flow, struct amount_msat, tal_count(flow->path));
 	for (int i = tal_count(flow->path) - 1; i >= 0; i--) {
@@ -382,15 +289,6 @@ void flow_complete(struct flow *flow,
 					    h->htlc_total,
 					    delivered);
 					    
-		// SUPERVERBOSE("(%s, ",type_to_string(tmpctx,struct amount_msat,&delivered));
-		// SUPERVERBOSE("%.2f)--",flow->success_prob);
-		
-		// if (!amount_msat_add(&h->htlc_total, h->htlc_total, delivered))
-		// {
-		// 	SUPERVERBOSE("%s: aborting\n",__PRETTY_FUNCTION__);
-		// 	abort();
-		// }
-		// h->num_htlcs++;
 		if (!amount_msat_add_fee(&delivered,
 					 flow_edge(flow, i)->base_fee,
 					 flow_edge(flow, i)->proportional_fee))
@@ -399,7 +297,6 @@ void flow_complete(struct flow *flow,
 			abort();
 		}
 	}
-	// SUPERVERBOSE("\n%s: finished\n",__PRETTY_FUNCTION__);
 }
 
 /* Compute the prob. of success of a set of concurrent set of flows. 
@@ -613,7 +510,7 @@ s64 linear_fee_cost(
 	return pfee + bfee* base_fee_penalty+ delay*delay_feefactor;
 }
 
-struct amount_msat flows_fee(struct flow **flows)
+struct amount_msat flow_set_fee(struct flow **flows)
 {
 	struct amount_msat fee = AMOUNT_MSAT(0);
 
diff --git a/plugins/renepay/flow.h b/plugins/renepay/flow.h
index 8f35f0b3aaae..767089a735ab 100644
--- a/plugins/renepay/flow.h
+++ b/plugins/renepay/flow.h
@@ -85,18 +85,6 @@ struct flow {
 /* Helper to access the half chan at flow index idx */
 const struct half_chan *flow_edge(const struct flow *flow, size_t idx);
 
-/* Path comparison helper */
-bool flow_path_eq(const struct gossmap_chan **path1,
-		  const int *dirs1,
-		  const struct gossmap_chan **path2,
-		  const int *dirs2);
-
-/* Add this to the completed flow. */
-void flow_add(struct flow *flow,
-	      const struct gossmap *gossmap,
-	      struct chan_extra_map *chan_extra_map,
-	      struct amount_msat additional);
-
 /* A big number, meaning "don't bother" (not infinite, since you may add) */
 #define FLOW_INF_COST 100000000.0
 
@@ -132,7 +120,7 @@ void remove_completed_flow_set(const struct gossmap *gossmap,
 			   struct chan_extra_map *chan_extra_map,
 			   struct flow **flows);
 
-struct amount_msat flows_fee(struct flow **flows);
+struct amount_msat flow_set_fee(struct flow **flows);
 
 /*
  * mu (μ) is used as follows in the cost function:
diff --git a/plugins/renepay/mcf.c b/plugins/renepay/mcf.c
index 19fae7413110..878999143477 100644
--- a/plugins/renepay/mcf.c
+++ b/plugins/renepay/mcf.c
@@ -1403,7 +1403,7 @@ struct flow** minflow(
 	best_prob_success = flow_set_probability(best_flow_paths,
 						params->gossmap,
 						params->chan_extra_map);
-	best_fee = flows_fee(best_flow_paths);
+	best_fee = flow_set_fee(best_flow_paths);
 	
 	// binary search for a value of `mu` that fits our fee and prob.
 	// constraints.
@@ -1429,7 +1429,7 @@ struct flow** minflow(
 						flow_paths,
 						params->gossmap,
 						params->chan_extra_map);
-		struct amount_msat fee = flows_fee(flow_paths);
+		struct amount_msat fee = flow_set_fee(flow_paths);
 		
 		printf("prob %.2f, fee %s\n",prob_success,
 				type_to_string(this_ctx,struct amount_msat,&fee));
diff --git a/plugins/renepay/not_mcf.c b/plugins/renepay/not_mcf.c
deleted file mode 100644
index 88cea6556b7b..000000000000
--- a/plugins/renepay/not_mcf.c
+++ /dev/null
@@ -1,485 +0,0 @@
-/* Horrible approximation to Min Cost Flow.
- *
- * i.e. https://arxiv.org/abs/2107.05322
- *
- * But the best solution for Min Cost Flow is approximate, and almost
- * linear in the number of edges: https://arxiv.org/abs/2203.00671
- *
- * It is, however, O(🤯) in implementation time :) I welcome anyone
- * who wants to implement it, and they can compare themselves to this
- * gross implementation with pride.
- */
-/* Without this, gheap is *really* slow!  Comment out for debugging. */
-#define NDEBUG
-#include "config.h"
-#include <ccan/bitmap/bitmap.h>
-#include <ccan/cast/cast.h>
-#include <common/gossmap.h>
-#include <common/pseudorand.h>
-#include <gheap.h>
-#include <plugins/renepay/flow.h>
-#include <plugins/renepay/mcf.h>
-
-#ifndef SUPERVERBOSE
-#define SUPERVERBOSE(...)
-#else
-#define SUPERVERBOSE_ENABLED 1
-#endif
-
-#define ASSERT(x) do { if (!(x)) abort(); } while(0)
-
-/* Each node has this side-info. */
-struct dijkstra {
-	/* Using our cost function */
-	double cost;
-	/* I want to use an index here, except that gheap moves things onto
-	 * a temporary on the stack and that makes things complex. */
-	/* NULL means it's been visited already. */
-	const struct gossmap_node **heapptr;
-	/* We could re-evaluate to determine this, but keeps it simple */
-	struct gossmap_chan *best_chan;
-};
-
-struct pay_parameters {
-	/* The gossmap we are using */
-	struct gossmap *gossmap;
-	/* Our array of dijkstra per-node info */
-	struct dijkstra *dij;
-	/* How much do we care about fees vs certainty? */
-	double mu;
-	/* How much do we care about locktimes? */
-	double delay_feefactor;
-	/* Penalty factor for having a basefee (adds to ppm). */
-	double basefee_penalty;
-	/* Optional bitarray of disabled chans. */
-	const bitmap *disabled;
-
-	/* The working heap */
-	struct gheap_ctx gheap_ctx;
-	const struct gossmap_node **heap;
-	size_t heapsize;
-
-	/* Extra information we intuited about the channels */
-	struct chan_extra_map *chan_extra_map;
-};
-
-/* Required global for gheap less_comparer */
-static struct pay_parameters *global_params;
-
-static struct dijkstra *get_dijkstra(const struct pay_parameters *params,
-				     const struct gossmap_node *n)
-{
-	return params->dij + gossmap_node_idx(params->gossmap, n);
-}
-
-/* We want a minheap, not a maxheap, so this is backwards! */
-static int less_comparer(const void *const ctx,
-			 const void *const a,
-			 const void *const b)
-{
-	const struct pay_parameters *params = ctx;
-	return get_dijkstra(params, *(struct gossmap_node **)a)->cost
-		> get_dijkstra(params, *(struct gossmap_node **)b)->cost;
-}
-
-static void item_mover(void *const dst, const void *const src)
-{
-	const struct pay_parameters *params = global_params;
-	struct gossmap_node *n = *((struct gossmap_node **)src);
-	get_dijkstra(params, n)->heapptr = dst;
-	*((struct gossmap_node **)dst) = n;
-}
-
-static void append_to_heap(struct pay_parameters *params,
-			   const struct gossmap_node *n,
-			   double cost)
-{
-	struct dijkstra *d = get_dijkstra(params, n);
-
-	params->heap[params->heapsize] = n;
-	d->heapptr = &params->heap[params->heapsize];
-	d->cost = cost;
-	params->heapsize++;
-}
-
-static void init_heap(struct pay_parameters *params,
-		      const struct gossmap_node *target)
-{
-	/* Initialize all heapptrs to NULL, costs to infinite */
-	for (size_t i = 0; i < tal_count(params->dij); i++) {
-		params->dij[i].cost = FLOW_INF_COST;
-		params->dij[i].heapptr = NULL;
-	}
-
-	/* First entry in heap is start, cost 0 */
-	params->heapsize = 0;
-	append_to_heap(params, target, 0);
-
-	ASSERT(gheap_is_heap(&params->gheap_ctx, params->heap, params->heapsize));
-}
-
-/* Convenient wrapper for flow_edge_cost */
-static double costfn(const struct pay_parameters *params,
-		     const struct gossmap_chan *c, int dir,
-		     struct amount_msat flow)
-{
-	struct amount_msat min, max, prev_flow;
-	struct chan_extra_half *h = get_chan_extra_half_by_chan(params->gossmap,
-							params->chan_extra_map,
-							c, dir);
-	if (h) {
-		SUPERVERBOSE("costfn found extra %p: cap %s-%s, flow %s\n",
-			     h,
-			     type_to_string(tmpctx, struct amount_msat, &h->known_min),
-			     type_to_string(tmpctx, struct amount_msat, &h->known_max),
-			     type_to_string(tmpctx, struct amount_msat, &h->htlc_total));
-		min = h->known_min;
-		max = h->known_max;
-		prev_flow = h->htlc_total;
-	} else {
-		/* We know nothing, use 0 - capacity */
-		struct amount_sat cap;
-
-		min = AMOUNT_MSAT(0);
-		if (!gossmap_chan_get_capacity(params->gossmap, c, &cap))
-			cap = AMOUNT_SAT(0);
-		if (!amount_sat_to_msat(&max, cap))
-			abort();
-		prev_flow = AMOUNT_MSAT(0);
-	}
-
-	return flow_edge_cost(params->gossmap,
-			      c, dir,
-			      min, max, prev_flow,
-			      flow,
-			      params->mu,
-			      params->basefee_penalty,
-			      params->delay_feefactor);
-}
-
-static bool run_shortest_path(struct pay_parameters *params,
-			      const struct gossmap_node *source,
-			      const struct gossmap_node *target,
-			      struct amount_msat flow)
-{
-	init_heap(params, target);
-
-	while (params->heapsize != 0) {
-		struct dijkstra *cur_d;
-		const struct gossmap_node *cur;
-
-		/* Pop off top of heap */
-		cur = params->heap[0];
-
-		/* Reached the source?  Done! */
-		if (cur == source)
-			return true;
-
-		cur_d = get_dijkstra(params, cur);
-		ASSERT(cur_d->heapptr == params->heap);
-		gheap_pop_heap(&params->gheap_ctx, params->heap, params->heapsize--);
-		cur_d->heapptr = NULL;
-
-		for (size_t i = 0; i < cur->num_chans; i++) {
-			struct gossmap_node *neighbor;
-			int which_half;
-			struct gossmap_chan *c;
-			struct dijkstra *d;
-			double cost;
-
-			c = gossmap_nth_chan(params->gossmap,
-					     cur, i, &which_half);
-			/* We're going to traverse backwards, so we need
-			 * channel_update into this node */
-			if (!gossmap_chan_set(c, !which_half))
-				continue;
-
-			neighbor = gossmap_nth_node(params->gossmap,
-						    c, !which_half);
-
-			d = get_dijkstra(params, neighbor);
-			/* Ignore if already visited. */
-			if (d->cost != FLOW_INF_COST && !d->heapptr)
-				continue;
-
-			/* They can explicitly disable some channels for this
-			 * run */
-			if (params->disabled
-			    && bitmap_test_bit(params->disabled,
-					       gossmap_chan_idx(params->gossmap,
-								c)))
-				continue;
-
-			/* Get edge cost (will be FLOW_INF_COST in
-			 * insufficient cpacity) */
-			cost = costfn(params, c, !which_half, flow);
-
-			/* If that doesn't give us a cheaper path, ignore */
-			if (cur_d->cost + cost >= d->cost)
-				continue;
-
-			/* Yay, we have a new winner! */
-			d->cost = cur_d->cost + cost;
-			d->best_chan = c;
-			if (!d->heapptr)
-				append_to_heap(params, neighbor, d->cost);
-
-			gheap_restore_heap_after_item_increase(&params->gheap_ctx,
-							       params->heap,
-							       params->heapsize,
-							       d->heapptr
-							       - params->heap);
-		}
-	}
-	return false;
-}
-
-/* Which dir for the channel is n? */
-static int gossmap_dir(const struct gossmap *gossmap,
-		       const struct gossmap_node *n,
-		       const struct gossmap_chan *c)
-{
-	/* Get the *other* peer attached to this channel */
-	if (gossmap_nth_node(gossmap, c, 0) == n)
-		return 0;
-	ASSERT(gossmap_nth_node(gossmap, c, 1) == n);
-	return 1;
-}
-
-/* We find the cheapest path between source and target for flow, ignoring
- * capacity limits. */
-static const struct gossmap_chan **
-find_cheap_flow(const tal_t *ctx,
-		struct pay_parameters *params,
-		const struct gossmap_node *source,
-		const struct gossmap_node *target,
-		struct amount_msat flow,
-		int **dirs)
-{
-	const struct gossmap_node *n;
-	const struct gossmap_chan **arr;
-
-	if (!run_shortest_path(params, source, target, flow))
-		return NULL;
-
-	arr = tal_arr(ctx, const struct gossmap_chan *, 0);
-	*dirs = tal_arr(ctx, int, 0);
-	n = source;
-	while (n != target) {
-		struct dijkstra *d = get_dijkstra(params, n);
-		int dir;
-
-		dir = gossmap_dir(params->gossmap, n, d->best_chan);
-		tal_arr_expand(&arr, d->best_chan);
-		tal_arr_expand(dirs, dir);
-
-		/* Get other end of channel */
-		n = gossmap_nth_node(params->gossmap, d->best_chan, !dir);
-	}
-	return arr;
-}
-
-static void print_path(const char *desc,
-		       const struct pay_parameters *params,
-		       const struct gossmap_chan **path,
-		       const int *dirs,
-		       struct amount_msat amt)
-{
-#ifdef SUPERVERBOSE_ENABLED
-	SUPERVERBOSE("%s: ", desc);
-
-	for (size_t i = 0; i < tal_count(path); i++) {
-		struct short_channel_id scid
-			= gossmap_chan_scid(params->gossmap, path[i]);
-		double cost;
-
-		print_enable = false;
-		cost = costfn(params, path[i], dirs[i], amt);
-		print_enable = true;
-		SUPERVERBOSE("%s%s/%i(set=%i/%i,htlc_max=%"PRIu64"/rev:%"PRIu64",fee=%u+%u,cost=%f) ",
-			     i ? "->" : "",
-			     type_to_string(tmpctx, struct short_channel_id, &scid),
-			     dirs[i],
-			     gossmap_chan_set(path[i], 0),
-			     gossmap_chan_set(path[i], 1),
-			     fp16_to_u64(path[i]->half[dirs[i]].htlc_max),
-			     fp16_to_u64(path[i]->half[!dirs[i]].htlc_max),
-			     path[i]->half[dirs[i]].base_fee,
-			     path[i]->half[dirs[i]].proportional_fee,
-			     cost);
-	}
-	SUPERVERBOSE("\n");
-#endif /* SUPERVERBOSE_ENABLED */
-}
-
-static void print_flow(const char *desc,
-		       const struct pay_parameters *params,
-		       const struct flow *flow)
-{
-	struct amount_msat fee, delivered;
-
-	delivered = flow->amounts[tal_count(flow->amounts)-1];
-	print_path(desc, params, flow->path, flow->dirs, delivered);
-	if (!amount_msat_sub(&fee, flow->amounts[0], delivered))
-		abort();
-	SUPERVERBOSE(" prob %.2f, %s delivered with fee %s\n",
-		     flow->success_prob,
-		     type_to_string(tmpctx, struct amount_msat, &delivered),
-		     type_to_string(tmpctx, struct amount_msat, &fee));
-}
-
-static double flow_cost(struct pay_parameters *params,
-			const struct gossmap_chan **path,
-			const int *dirs,
-			struct amount_msat amount)
-{
-	double cost = 0;
-
-	for (size_t i = 0; i < tal_count(path); i++)
-		cost += costfn(params, path[i], dirs[i], amount);
-	return cost;
-}
-
-static struct flow *find_similar_flow(struct pay_parameters *params,
-				      struct flow **flows, size_t num_flows,
-				      const struct gossmap_chan **path,
-				      const int *dirs,
-				      struct amount_msat amount)
-{
-	double best_cost;
-	struct flow *best;
-
-	/* First search for exact equal */
-	for (size_t i = 0; i < num_flows; i++) {
-		if (flow_path_eq(flows[i]->path, flows[i]->dirs, path, dirs))
-			return flows[i];
-	}
-
-	/* OK, see if one of the existing paths is similar cost. */
-	best_cost = flow_cost(params, path, dirs, amount) * 1.10;
-	best = NULL;
-	for (size_t i = 0; i < num_flows; i++) {
-		double cost = flow_cost(params, flows[i]->path, flows[i]->dirs, amount);
-		if (cost < best_cost) {
-			best = flows[i];
-			best_cost = cost;
-		}
-	}
-	return best;
-}
-
-struct flow **
-minflow(const tal_t *ctx,
-	struct gossmap *gossmap,
-	const struct gossmap_node *source,
-	const struct gossmap_node *target,
-	struct chan_extra_map *chan_extra_map,
-	const bitmap *disabled,
-	struct amount_msat amount,
-	double frugality,
-	double delay_feefactor)
-{
-	struct pay_parameters *params = tal(tmpctx, struct pay_parameters);
-	struct flow **flows;
-	const struct amount_msat min_flow = AMOUNT_MSAT(5000);
-	struct amount_msat step, remaining;
-	size_t num_flows;
-
-	params->gossmap = gossmap;
-	params->dij = tal_arr(params, struct dijkstra,
-			      gossmap_max_node_idx(gossmap));
-
-	params->mu = derive_mu(gossmap, amount, frugality);
-	params->delay_feefactor = delay_feefactor;
-	params->disabled = disabled;
-	ASSERT(!disabled
-	       || tal_bytelen(disabled) == bitmap_sizeof(gossmap_max_chan_idx(gossmap)));
-
-	/* If we assume we split into 5 parts, adjust ppm on basefee to
-	 * make be correct at amt / 5.  Thus 1 basefee is worth 1 ppm at
-	 * amount / 5. */
-	params->basefee_penalty
-		= 5000000.0 / amount.millisatoshis; /* Raw: penalty */
-
-	/* There doesn't seem to be much difference with fanout 2-4. */
-	params->gheap_ctx.fanout = 2;
-	/* There seems to be a slight decrease if we alter this value. */
-	params->gheap_ctx.page_chunks = 1;
-	params->gheap_ctx.item_size = sizeof(*params->heap);
-	params->gheap_ctx.less_comparer = less_comparer;
-	params->gheap_ctx.less_comparer_ctx = params;
-	params->gheap_ctx.item_mover = item_mover;
-
-	/* This is initialized (and heapsize set) in init_heap each time */
-	params->heap = tal_arr(params, const struct gossmap_node *,
-			       gossmap_num_nodes(gossmap));
-
-	params->chan_extra_map = chan_extra_map;
-	global_params = params;
-
-	/* Now gather the flows: we randomize step a little, but aim for 50. */
-	flows = tal_arr(ctx, struct flow *, 100);
-	step = amount_msat_div(amount, 50);
-	if (amount_msat_less(step, min_flow))
-		step = min_flow;
-
-	remaining = amount;
-	num_flows = 0;
-	while (amount_msat_greater(remaining, AMOUNT_MSAT(0))) {
-		struct amount_msat this_amount;
-		const struct gossmap_chan **path;
-		int *dirs;
-		struct flow *flow;
-
-		/* Randomize amount a little */
-		if (!amount_msat_scale(&this_amount, step,
-				       0.8 + 0.4 * pseudorand_double()))
-			abort();
-
-		if (amount_msat_greater(this_amount, remaining))
-			this_amount = remaining;
-
-		path = find_cheap_flow(tmpctx, params, source, target,
-				       this_amount, &dirs);
-		if (!path)
-			return tal_free(flows);
-
-		/* Maybe add to existing. */
-		flow = find_similar_flow(params, flows, num_flows, path, dirs, this_amount);
-		if (flow) {
-			struct amount_msat before, after, tot;
-
-			before = flow->amounts[tal_count(flow->amounts)-1];
-			print_flow("Duplicate flow before", params, flow);
-			flow_add(flow, params->gossmap, params->chan_extra_map,
-				 this_amount);
-			print_flow("Duplicate flow after", params, flow);
-			after = flow->amounts[tal_count(flow->amounts)-1];
-			if (!amount_msat_sub(&tot, after, before))
-				abort();
-			ASSERT(amount_msat_eq(tot, this_amount));
-		} else {
-			flow = tal(flows, struct flow);
-			flow->path = tal_steal(flow, path);
-			flow->dirs = tal_steal(flow, dirs);
-			flow_complete(flow, params->gossmap,
-				      params->chan_extra_map,
-				      this_amount);
-			print_flow("New flow", params, flow);
-			flows[num_flows++] = flow;
-			ASSERT(amount_msat_eq(flow->amounts[tal_count(flow->amounts)-1],
-					      this_amount));
-		}
-		if (!amount_msat_sub(&remaining, remaining, this_amount))
-			abort();
-	}
-
-	/* Shrink the array we return */
-	tal_resize(&flows, num_flows);
-	global_params = NULL;
-	return flows;
-}
-
-#ifndef SUPERVERBOSE_ENABLED
-#undef SUPERVERBOSE
-#endif
diff --git a/plugins/renepay/pay_flow.c b/plugins/renepay/pay_flow.c
index e09721f000b5..22fc42b5df7d 100644
--- a/plugins/renepay/pay_flow.c
+++ b/plugins/renepay/pay_flow.c
@@ -397,7 +397,7 @@ struct pay_flow **get_payflows(struct payment *p,
 
 		/* Are we unhappy? */
 		prob = flow_set_probability(flows,pay_plugin->gossmap,&pay_plugin->chan_extra_map);
-		fee = flows_fee(flows);
+		fee = flow_set_fee(flows);
 		delay = flows_worst_delay(flows) + p->final_cltv;
 
 		too_unlikely = (prob < 0.01);

From 059ee9a3efe9cc93d28333c8ba047749a706e17f Mon Sep 17 00:00:00 2001
From: Lagrang3 <eduardo.quintana@pm.me>
Date: Mon, 15 May 2023 16:01:04 +0100
Subject: [PATCH 28/64] renepay: know min and max in halves is redundant

---
 plugins/renepay/flow.h | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/plugins/renepay/flow.h b/plugins/renepay/flow.h
index 767089a735ab..9e4fc85be18f 100644
--- a/plugins/renepay/flow.h
+++ b/plugins/renepay/flow.h
@@ -9,6 +9,11 @@
 /* Any implementation needs to keep some data on channels which are
  * in-use (or about which we have extra information).  We use a hash
  * table here, since most channels are not in use. */
+// TODO(eduardo): if we know the liquidity of channel (X,dir) is [A,B]
+// then we also know that the liquidity of channel (X,!dir) is [Cap-B,Cap-A]. 
+// This means that it is redundant to store known_min and known_max for both
+// halves of the channel and it also means that once we update the knowledge of
+// (X,dir) the knowledge of (X,!dir) is updated as well.
 struct chan_extra {
 	struct short_channel_id scid;
 

From f557dc58e43576b3937b24eb0dc0fe71bb907241 Mon Sep 17 00:00:00 2001
From: Lagrang3 <eduardo.quintana@pm.me>
Date: Mon, 15 May 2023 20:16:53 +0100
Subject: [PATCH 29/64] renepay: testing flow_complete function

---
 plugins/renepay/flow.c              |   2 -
 plugins/renepay/test/run-testflow.c | 562 ++++++++++++++++++++++++++++
 2 files changed, 562 insertions(+), 2 deletions(-)

diff --git a/plugins/renepay/flow.c b/plugins/renepay/flow.c
index 0dc3bb81ee0f..49f0832b1ffe 100644
--- a/plugins/renepay/flow.c
+++ b/plugins/renepay/flow.c
@@ -259,8 +259,6 @@ void commit_flow_set(
 	}
 }
 
-// TODO(eduardo): unit test this with a single path flow with known fees and
-// probabilities.
 /* Helper function to fill in amounts and success_prob for flow 
  * 
  * IMPORTANT: here we do not commit flows to chan_extra, flows are commited
diff --git a/plugins/renepay/test/run-testflow.c b/plugins/renepay/test/run-testflow.c
index 457d3ee5dd90..eb17d77de7e7 100644
--- a/plugins/renepay/test/run-testflow.c
+++ b/plugins/renepay/test/run-testflow.c
@@ -6,6 +6,8 @@
 #include <common/channel_id.h>
 #include <common/setup.h>
 #include <common/utils.h>
+#include <common/node_id.h>
+#include <ccan/read_write_all/read_write_all.h>
 
 static bool print_enable = true;
 #define SUPERVERBOSE(...) do { if (print_enable) printf(__VA_ARGS__); } while(0)
@@ -34,6 +36,360 @@ void towire_channel_id(u8 **pptr UNNEEDED, const struct channel_id *channel_id U
 void towire_wireaddr(u8 **pptr UNNEEDED, const struct wireaddr *addr UNNEEDED)
 { fprintf(stderr, "towire_wireaddr called!\n"); abort(); }
 /* AUTOGENERATED MOCKS END */
+	
+static const u8 canned_map[] = {
+0x0c, 0x80, 0x00, 0x01, 0xbc, 0x86, 0xe4, 0xbf, 0x95, 0x00, 0x00, 0x00, 0x00, 0x10, 0x08, 0x00, 
+0x00, 0x00, 0x00, 0x00, 0x0f, 0x42, 0x40, 0x01, 0xb0, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 
+0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 
+0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 
+0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 
+0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 
+0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 
+0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 
+0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 
+0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 
+0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 
+0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 
+0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 
+0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 
+0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 
+0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 
+0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 
+0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x06, 0x22, 0x6e, 
+0x46, 0x11, 0x1a, 0x0b, 0x59, 0xca, 0xaf, 0x12, 0x60, 0x43, 0xeb, 0x5b, 0xbf, 0x28, 0xc3, 0x4f, 
+0x3a, 0x5e, 0x33, 0x2a, 0x1f, 0xc7, 0xb2, 0xb7, 0x3c, 0xf1, 0x88, 0x91, 0x0f, 0x00, 0x00, 0x71, 
+0x00, 0x00, 0x01, 0x00, 0x01, 0x02, 0x4f, 0x9d, 0xa0, 0xd7, 0x26, 0xad, 0xf0, 0xd9, 0xa4, 0xa3, 
+0xac, 0x32, 0xe3, 0x28, 0xb9, 0x3a, 0xd5, 0x27, 0xcc, 0xb9, 0xdb, 0x70, 0x77, 0xc5, 0x7a, 0x12, 
+0xc6, 0xf9, 0xfa, 0x9a, 0xdd, 0x74, 0x03, 0x7f, 0x97, 0xaf, 0x8e, 0x4f, 0xa6, 0x7c, 0x9b, 0x8d, 
+0x69, 0x70, 0x84, 0x95, 0x36, 0xf8, 0x88, 0xc3, 0x04, 0x31, 0x6d, 0x01, 0x5a, 0xf5, 0x12, 0x9e, 
+0x26, 0x8e, 0x01, 0x64, 0x96, 0x1b, 0x5e, 0x03, 0x1b, 0x84, 0xc5, 0x56, 0x7b, 0x12, 0x64, 0x40, 
+0x99, 0x5d, 0x3e, 0xd5, 0xaa, 0xba, 0x05, 0x65, 0xd7, 0x1e, 0x18, 0x34, 0x60, 0x48, 0x19, 0xff, 
+0x9c, 0x17, 0xf5, 0xe9, 0xd5, 0xdd, 0x07, 0x8f, 0x03, 0x1b, 0x84, 0xc5, 0x56, 0x7b, 0x12, 0x64, 
+0x40, 0x99, 0x5d, 0x3e, 0xd5, 0xaa, 0xba, 0x05, 0x65, 0xd7, 0x1e, 0x18, 0x34, 0x60, 0x48, 0x19, 
+0xff, 0x9c, 0x17, 0xf5, 0xe9, 0xd5, 0xdd, 0x07, 0x8f, 0x40, 0x00, 0x01, 0xb0, 0x24, 0x3a, 0xa3, 
+0x76, 0x64, 0x62, 0x19, 0xec, 0x01, 0x00, 0x66, 0x7f, 0x0f, 0xad, 0x6d, 0x9d, 0x58, 0x1b, 0x28, 
+0x8a, 0x67, 0x9d, 0xf8, 0xd1, 0x9d, 0x79, 0x4e, 0x67, 0xc8, 0x76, 0xbb, 0xdd, 0x4d, 0x8e, 0x45, 
+0x0d, 0xc9, 0x0e, 0x24, 0x76, 0xda, 0x44, 0x68, 0x7b, 0xe2, 0x14, 0xe8, 0x48, 0xfa, 0xd7, 0xc2, 
+0x35, 0xc5, 0x98, 0xd9, 0x7a, 0x6c, 0xcb, 0xb1, 0x4b, 0x19, 0xf9, 0xfa, 0xb2, 0x19, 0x3f, 0x87, 
+0xc1, 0xe9, 0x47, 0x51, 0x16, 0x64, 0x36, 0x2a, 0xeb, 0xc5, 0xaa, 0x20, 0x59, 0x4e, 0xdf, 0xae, 
+0x4e, 0x10, 0x38, 0x34, 0x8e, 0x06, 0x6e, 0x5d, 0x1b, 0x44, 0x30, 0xfb, 0x20, 0xed, 0xea, 0xde, 
+0x83, 0xcd, 0xa4, 0x8a, 0x5c, 0xad, 0x70, 0x2d, 0x8b, 0x04, 0xfb, 0xa2, 0xbd, 0x95, 0x7c, 0xdd, 
+0x66, 0xb5, 0x4e, 0xd6, 0xc6, 0x27, 0xdb, 0xa8, 0xe1, 0x26, 0x22, 0x81, 0x57, 0xe2, 0xaa, 0xe4, 
+0x82, 0xbe, 0x9e, 0x90, 0xc5, 0xc2, 0x59, 0x56, 0x9b, 0x79, 0xf3, 0xc3, 0xfe, 0x0c, 0xb3, 0x35, 
+0xeb, 0xba, 0xad, 0xf7, 0xd3, 0x24, 0x4e, 0x16, 0x15, 0x2d, 0x86, 0xd9, 0xe9, 0xd2, 0x38, 0x9b, 
+0xf9, 0xb3, 0x5f, 0x2c, 0x9b, 0xeb, 0xe0, 0x1c, 0xb3, 0xf0, 0x0f, 0xc1, 0x9d, 0x0b, 0x20, 0xa2, 
+0x19, 0xeb, 0x1a, 0x05, 0x8b, 0x8d, 0xb1, 0x22, 0x74, 0x7c, 0xa4, 0x39, 0x94, 0x6f, 0xfc, 0x34, 
+0x1b, 0xe5, 0x9f, 0x45, 0x8e, 0x12, 0x6e, 0x65, 0x73, 0x28, 0x21, 0x80, 0xfd, 0x9c, 0x0c, 0x89, 
+0x2b, 0xcb, 0x43, 0x2e, 0x7f, 0x47, 0xa1, 0xd7, 0x7e, 0xa9, 0xd7, 0x3e, 0xdd, 0xa0, 0xf8, 0x60, 
+0x9d, 0xde, 0x51, 0x3d, 0xc4, 0x21, 0x06, 0x61, 0xb3, 0x4d, 0xd8, 0x94, 0x4a, 0x3a, 0xc9, 0xb9, 
+0xc3, 0xcb, 0x09, 0xa3, 0x2f, 0x7b, 0x96, 0x53, 0x13, 0x1d, 0x6d, 0x7a, 0x28, 0xdd, 0xc8, 0x8d, 
+0xe4, 0x10, 0xad, 0x4c, 0xc6, 0xa0, 0x1b, 0x00, 0x00, 0x06, 0x22, 0x6e, 0x46, 0x11, 0x1a, 0x0b, 
+0x59, 0xca, 0xaf, 0x12, 0x60, 0x43, 0xeb, 0x5b, 0xbf, 0x28, 0xc3, 0x4f, 0x3a, 0x5e, 0x33, 0x2a, 
+0x1f, 0xc7, 0xb2, 0xb7, 0x3c, 0xf1, 0x88, 0x91, 0x0f, 0x00, 0x00, 0x71, 0x00, 0x00, 0x01, 0x00, 
+0x01, 0x02, 0x4f, 0x9d, 0xa0, 0xd7, 0x26, 0xad, 0xf0, 0xd9, 0xa4, 0xa3, 0xac, 0x32, 0xe3, 0x28, 
+0xb9, 0x3a, 0xd5, 0x27, 0xcc, 0xb9, 0xdb, 0x70, 0x77, 0xc5, 0x7a, 0x12, 0xc6, 0xf9, 0xfa, 0x9a, 
+0xdd, 0x74, 0x03, 0x7f, 0x97, 0xaf, 0x8e, 0x4f, 0xa6, 0x7c, 0x9b, 0x8d, 0x69, 0x70, 0x84, 0x95, 
+0x36, 0xf8, 0x88, 0xc3, 0x04, 0x31, 0x6d, 0x01, 0x5a, 0xf5, 0x12, 0x9e, 0x26, 0x8e, 0x01, 0x64, 
+0x96, 0x1b, 0x5e, 0x02, 0xca, 0x1a, 0xac, 0x5f, 0x7b, 0x86, 0x3a, 0x01, 0xc8, 0x69, 0x90, 0x82, 
+0xdf, 0x9a, 0x4d, 0xf8, 0x14, 0x0d, 0xd6, 0xe7, 0x10, 0x59, 0xd4, 0xec, 0x7f, 0x48, 0x13, 0xb0, 
+0x96, 0xb4, 0xa3, 0xad, 0x02, 0x21, 0x55, 0x92, 0x46, 0x1c, 0x84, 0x3d, 0x40, 0xe6, 0x01, 0x8d, 
+0x3d, 0x0c, 0xb6, 0xf4, 0xe1, 0x61, 0xe2, 0x4b, 0x59, 0x41, 0xdb, 0x3b, 0x20, 0x44, 0xbc, 0x0c, 
+0xb2, 0x0e, 0x4d, 0x3f, 0x9b, 0x00, 0x00, 0x00, 0x0a, 0x91, 0x11, 0x83, 0xf6, 0x00, 0x00, 0x00, 
+0x00, 0x10, 0x05, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0f, 0x42, 0x40, 0xc0, 0x00, 0x00, 0x8a, 0x01, 
+0x3d, 0x6f, 0x9a, 0x64, 0x62, 0x19, 0xec, 0x01, 0x02, 0x4c, 0x45, 0x7e, 0x21, 0xb8, 0xd5, 0x36, 
+0x98, 0xcd, 0x45, 0x03, 0x78, 0xa6, 0x51, 0xf1, 0xda, 0x1a, 0xb4, 0x46, 0xed, 0xfb, 0xed, 0x86, 
+0xf9, 0x31, 0x85, 0x2e, 0x3d, 0x80, 0x77, 0xf2, 0x13, 0x76, 0x91, 0x08, 0xe7, 0x52, 0x3d, 0xf4, 
+0xe5, 0x2e, 0x3b, 0x80, 0x2a, 0xbf, 0x54, 0xf8, 0x80, 0xbb, 0x77, 0x6f, 0xc6, 0xca, 0x9e, 0x3f, 
+0xe8, 0x96, 0xfa, 0x54, 0x7e, 0x94, 0x78, 0x0a, 0xec, 0x06, 0x22, 0x6e, 0x46, 0x11, 0x1a, 0x0b, 
+0x59, 0xca, 0xaf, 0x12, 0x60, 0x43, 0xeb, 0x5b, 0xbf, 0x28, 0xc3, 0x4f, 0x3a, 0x5e, 0x33, 0x2a, 
+0x1f, 0xc7, 0xb2, 0xb7, 0x3c, 0xf1, 0x88, 0x91, 0x0f, 0x00, 0x00, 0x71, 0x00, 0x00, 0x01, 0x00, 
+0x01, 0x64, 0x62, 0x19, 0xec, 0x01, 0x00, 0x00, 0x06, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 
+0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x0a, 0x00, 0x00, 0x00, 0x00, 0x3b, 0x02, 0x33, 
+0x80, 0x40, 0x00, 0x00, 0xa4, 0x07, 0xd2, 0xf1, 0x5d, 0x64, 0x62, 0x19, 0xf1, 0x01, 0x01, 0x4d, 
+0xbe, 0x8a, 0xf5, 0xd8, 0x19, 0x2b, 0x99, 0xb0, 0xa0, 0xde, 0x24, 0x36, 0x32, 0x06, 0xac, 0x40, 
+0x4c, 0x41, 0x94, 0xc1, 0xd3, 0x85, 0xb5, 0xb8, 0x76, 0xbf, 0x98, 0xa9, 0x8e, 0xdb, 0xca, 0x43, 
+0x73, 0x98, 0xa0, 0xe0, 0x11, 0xa9, 0x95, 0xf3, 0xce, 0xde, 0xe5, 0x85, 0x80, 0x63, 0x8c, 0x12, 
+0x11, 0xee, 0xee, 0xa1, 0x3e, 0xcf, 0x4e, 0xd5, 0xae, 0x8d, 0x93, 0x22, 0xce, 0xbb, 0x02, 0x00, 
+0x07, 0x88, 0xa0, 0x80, 0x2a, 0x02, 0x69, 0xa2, 0x64, 0x62, 0x19, 0xf1, 0x02, 0x4f, 0x9d, 0xa0, 
+0xd7, 0x26, 0xad, 0xf0, 0xd9, 0xa4, 0xa3, 0xac, 0x32, 0xe3, 0x28, 0xb9, 0x3a, 0xd5, 0x27, 0xcc, 
+0xb9, 0xdb, 0x70, 0x77, 0xc5, 0x7a, 0x12, 0xc6, 0xf9, 0xfa, 0x9a, 0xdd, 0x74, 0x02, 0x4f, 0x9d, 
+0x4c, 0x4f, 0x55, 0x44, 0x54, 0x52, 0x41, 0x57, 0x4c, 0x2d, 0x2e, 0x30, 0x32, 0x2d, 0x33, 0x37, 
+0x2d, 0x67, 0x63, 0x39, 0x66, 0x38, 0x30, 0x33, 0x35, 0x2d, 0x6d, 0x6f, 0x64, 0x64, 0x65, 0x64, 
+0x00, 0x00, 0x01, 0x0d, 0x02, 0x9a, 0x00, 0x32, 0x00, 0x64, 0x00, 0x00, 0x00, 0x02, 0x4c, 0x4b, 
+0x40, 0xc0, 0x00, 0x00, 0x8a, 0x06, 0x22, 0xaa, 0xb5, 0x64, 0x62, 0x19, 0xec, 0x01, 0x02, 0x2b, 
+0x9e, 0x17, 0x25, 0x0f, 0x3d, 0x8c, 0x1c, 0x07, 0x6b, 0xb8, 0x7f, 0xdc, 0xc4, 0x30, 0xf4, 0xa7, 
+0xf8, 0x8b, 0x91, 0x53, 0xd6, 0xc1, 0x9d, 0x06, 0xb9, 0x18, 0xfb, 0xf0, 0x0b, 0x9a, 0x79, 0x2a, 
+0x56, 0x12, 0x35, 0x75, 0x4e, 0xf4, 0xb8, 0xb4, 0x2e, 0x72, 0x10, 0x3c, 0x8d, 0x76, 0x69, 0x1c, 
+0x67, 0xb0, 0x7f, 0x94, 0x07, 0xee, 0xb4, 0x38, 0x11, 0x0b, 0x7f, 0x62, 0x4e, 0x2a, 0x2d, 0x06, 
+0x22, 0x6e, 0x46, 0x11, 0x1a, 0x0b, 0x59, 0xca, 0xaf, 0x12, 0x60, 0x43, 0xeb, 0x5b, 0xbf, 0x28, 
+0xc3, 0x4f, 0x3a, 0x5e, 0x33, 0x2a, 0x1f, 0xc7, 0xb2, 0xb7, 0x3c, 0xf1, 0x88, 0x91, 0x0f, 0x00, 
+0x00, 0x71, 0x00, 0x00, 0x01, 0x00, 0x01, 0x64, 0x62, 0x19, 0xec, 0x01, 0x01, 0x00, 0x06, 0x00, 
+0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x0a, 0x00, 
+0x00, 0x00, 0x00, 0x3b, 0x02, 0x33, 0x80, 0x00, 0x00, 0x00, 0xa4, 0xde, 0x6a, 0x84, 0x4d, 0x64, 
+0x62, 0x19, 0xf1, 0x01, 0x01, 0x47, 0x72, 0x62, 0xe8, 0xc7, 0x43, 0xa8, 0x2e, 0x1c, 0x97, 0x2a, 
+0x06, 0xce, 0x2f, 0xa2, 0xfa, 0x27, 0x4f, 0x28, 0x7f, 0x55, 0x32, 0x19, 0x62, 0x58, 0xc6, 0x18, 
+0x07, 0x23, 0x5f, 0x8a, 0x59, 0x00, 0x52, 0x4d, 0xc9, 0x18, 0x22, 0x9e, 0xf7, 0x87, 0xa3, 0x36, 
+0x9d, 0x01, 0x73, 0x7c, 0x5b, 0xb8, 0xb4, 0x08, 0x50, 0x0f, 0x89, 0x52, 0x3f, 0x2e, 0x44, 0xa0, 
+0xe0, 0x32, 0x3a, 0xf7, 0x20, 0x00, 0x07, 0x88, 0xa0, 0x80, 0x2a, 0x02, 0x69, 0xa2, 0x64, 0x62, 
+0x19, 0xf1, 0x03, 0x7f, 0x97, 0xaf, 0x8e, 0x4f, 0xa6, 0x7c, 0x9b, 0x8d, 0x69, 0x70, 0x84, 0x95, 
+0x36, 0xf8, 0x88, 0xc3, 0x04, 0x31, 0x6d, 0x01, 0x5a, 0xf5, 0x12, 0x9e, 0x26, 0x8e, 0x01, 0x64, 
+0x96, 0x1b, 0x5e, 0x03, 0x7f, 0x97, 0x53, 0x4c, 0x49, 0x43, 0x4b, 0x45, 0x52, 0x43, 0x48, 0x49, 
+0x50, 0x4d, 0x55, 0x4e, 0x4b, 0x2d, 0x2d, 0x67, 0x63, 0x39, 0x66, 0x38, 0x30, 0x33, 0x35, 0x2d, 
+0x6d, 0x6f, 0x64, 0x64, 0x65, 0x64, 0x00, 0x00, 0x01, 0x0d, 0x02, 0x9a, 0x00, 0x32, 0x00, 0x64, 
+0x00, 0x00, 0x00, 0x02, 0x4c, 0x4b, 0x40, 0x00, 0x00, 0x01, 0xb0, 0x31, 0xd6, 0x97, 0xf8, 0x64, 
+0x62, 0x19, 0xec, 0x01, 0x00, 0x3f, 0x22, 0x04, 0x81, 0x00, 0xfb, 0xfe, 0x52, 0x4e, 0xdf, 0x7e, 
+0xef, 0x65, 0xff, 0x41, 0xcf, 0xfc, 0x33, 0xfc, 0x27, 0xba, 0x5b, 0x5f, 0xc5, 0x40, 0xd7, 0xff, 
+0x65, 0x20, 0x37, 0x3f, 0x00, 0x0d, 0x7c, 0x9b, 0xa9, 0xf1, 0x8c, 0xc6, 0xf1, 0xf7, 0x30, 0xd8, 
+0x1a, 0x44, 0xea, 0x6a, 0xf8, 0x95, 0xde, 0xe9, 0x35, 0x5f, 0x2b, 0x09, 0xc8, 0x5e, 0xf4, 0xa4, 
+0x58, 0x5a, 0xef, 0x24, 0x14, 0x1e, 0x17, 0x5b, 0xb1, 0xa7, 0xbf, 0x69, 0xb6, 0x44, 0xbe, 0xcc, 
+0x37, 0xb3, 0x48, 0x0a, 0x83, 0x37, 0xfa, 0xdb, 0x1d, 0x2a, 0x57, 0x83, 0x50, 0x88, 0x39, 0xd7, 
+0x2d, 0xa6, 0x70, 0x19, 0x94, 0x63, 0xa3, 0x09, 0x57, 0x47, 0x80, 0x47, 0xa7, 0x9b, 0xb5, 0x20, 
+0x4a, 0x33, 0x67, 0xf7, 0x5c, 0x5d, 0x4c, 0xa3, 0xc3, 0x05, 0x81, 0x48, 0xa7, 0x5e, 0x10, 0x13, 
+0x5d, 0x64, 0x4c, 0x2e, 0x53, 0x28, 0xd1, 0x82, 0xc3, 0x7d, 0xbf, 0xb2, 0xcd, 0x36, 0xcc, 0x1e, 
+0xc6, 0xc7, 0x42, 0x65, 0x12, 0x61, 0x82, 0x5d, 0xc7, 0x3b, 0x6a, 0xaf, 0x71, 0xd4, 0xf0, 0xe9, 
+0xff, 0xdd, 0x75, 0x33, 0x96, 0x3e, 0xb7, 0x92, 0xc2, 0xcd, 0x0e, 0xda, 0xec, 0x55, 0x43, 0x20, 
+0x07, 0xe8, 0x9e, 0xff, 0x3f, 0xea, 0x2f, 0x44, 0x64, 0x43, 0xe9, 0xfd, 0x82, 0x0a, 0xd4, 0x1d, 
+0xf6, 0x14, 0x02, 0x30, 0x78, 0x34, 0x02, 0x62, 0x73, 0x90, 0x41, 0x38, 0xbe, 0xc0, 0xd2, 0xac, 
+0x59, 0xc1, 0x82, 0xd2, 0x6f, 0x4e, 0x28, 0xd9, 0x2e, 0x3c, 0x6d, 0x4b, 0xa2, 0x25, 0xc9, 0x46, 
+0x42, 0x95, 0x64, 0xb9, 0x89, 0x73, 0x30, 0xce, 0xb7, 0xca, 0x1a, 0x78, 0xac, 0xa8, 0x72, 0x71, 
+0xe8, 0x1e, 0x48, 0xe9, 0x7c, 0xe5, 0x49, 0x78, 0x16, 0x50, 0x3e, 0x26, 0x15, 0x4f, 0xaf, 0x7f, 
+0x53, 0x17, 0x14, 0xeb, 0xa6, 0x00, 0x00, 0x06, 0x22, 0x6e, 0x46, 0x11, 0x1a, 0x0b, 0x59, 0xca, 
+0xaf, 0x12, 0x60, 0x43, 0xeb, 0x5b, 0xbf, 0x28, 0xc3, 0x4f, 0x3a, 0x5e, 0x33, 0x2a, 0x1f, 0xc7, 
+0xb2, 0xb7, 0x3c, 0xf1, 0x88, 0x91, 0x0f, 0x00, 0x00, 0x71, 0x00, 0x00, 0x03, 0x00, 0x01, 0x02, 
+0x45, 0x1e, 0x9b, 0xaf, 0xf8, 0x1f, 0xaf, 0x5f, 0x41, 0x0b, 0x0b, 0xbe, 0x9e, 0x36, 0xee, 0x83, 
+0xbf, 0x8f, 0x79, 0x69, 0x8e, 0x66, 0x05, 0xa5, 0x1a, 0x97, 0x48, 0xbc, 0x73, 0xc7, 0xdc, 0x28, 
+0x03, 0x7f, 0x97, 0xaf, 0x8e, 0x4f, 0xa6, 0x7c, 0x9b, 0x8d, 0x69, 0x70, 0x84, 0x95, 0x36, 0xf8, 
+0x88, 0xc3, 0x04, 0x31, 0x6d, 0x01, 0x5a, 0xf5, 0x12, 0x9e, 0x26, 0x8e, 0x01, 0x64, 0x96, 0x1b, 
+0x5e, 0x02, 0xd0, 0xbf, 0x3d, 0xe0, 0x25, 0x7a, 0xe4, 0x02, 0x4a, 0x88, 0x2b, 0x20, 0x63, 0xb4, 
+0x68, 0x6b, 0x72, 0x27, 0x91, 0xc2, 0xe4, 0x7a, 0xd1, 0x75, 0x93, 0x5d, 0xf3, 0x3a, 0xbe, 0x99, 
+0x7b, 0x76, 0x02, 0x49, 0x4d, 0xb8, 0x75, 0x3e, 0x66, 0xc7, 0x73, 0x63, 0xec, 0xf4, 0x40, 0xa4, 
+0xcb, 0xe5, 0xe0, 0x3e, 0xc6, 0x28, 0x2b, 0xea, 0x8a, 0xd3, 0x3f, 0x66, 0x4b, 0xa3, 0x9b, 0x86, 
+0x37, 0xf7, 0x7b, 0x00, 0x00, 0x00, 0x0a, 0xea, 0x64, 0x27, 0x09, 0x00, 0x00, 0x00, 0x00, 0x10, 
+0x05, 0x00, 0x00, 0x00, 0x00, 0x00, 0x1e, 0x84, 0x80, 0x80, 0x00, 0x00, 0x8a, 0x67, 0x5f, 0xf2, 
+0xad, 0x64, 0x62, 0x19, 0xec, 0x01, 0x02, 0x28, 0x06, 0xbe, 0x81, 0x8d, 0x13, 0xe3, 0xe9, 0x45, 
+0x09, 0xdd, 0x6a, 0xbe, 0x96, 0xb5, 0x08, 0xe4, 0x87, 0xca, 0xfd, 0x72, 0xc1, 0xfd, 0xa9, 0xe8, 
+0x32, 0x68, 0x95, 0x97, 0x06, 0x47, 0x57, 0x3a, 0x38, 0x28, 0x22, 0xa1, 0x78, 0x45, 0x22, 0xd5, 
+0xac, 0x0d, 0x1d, 0x2f, 0x25, 0xf0, 0x3a, 0x11, 0x85, 0x34, 0xcc, 0xae, 0xf8, 0xdd, 0x44, 0x05, 
+0xdd, 0xe6, 0x6d, 0xfc, 0xc2, 0xa0, 0x7e, 0x06, 0x22, 0x6e, 0x46, 0x11, 0x1a, 0x0b, 0x59, 0xca, 
+0xaf, 0x12, 0x60, 0x43, 0xeb, 0x5b, 0xbf, 0x28, 0xc3, 0x4f, 0x3a, 0x5e, 0x33, 0x2a, 0x1f, 0xc7, 
+0xb2, 0xb7, 0x3c, 0xf1, 0x88, 0x91, 0x0f, 0x00, 0x00, 0x71, 0x00, 0x00, 0x03, 0x00, 0x01, 0x64, 
+0x62, 0x19, 0xec, 0x01, 0x00, 0x00, 0x06, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 
+0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x0a, 0x00, 0x00, 0x00, 0x00, 0x76, 0x04, 0x67, 0x00, 0x80, 
+0x00, 0x00, 0x8a, 0xdc, 0x8e, 0xb4, 0xa3, 0x64, 0x62, 0x19, 0xec, 0x01, 0x02, 0x27, 0x9a, 0x87, 
+0xb6, 0x8b, 0xcb, 0xc9, 0x41, 0xea, 0xc3, 0x1b, 0x18, 0xf5, 0x51, 0x2f, 0x9b, 0x71, 0xe3, 0x8d, 
+0x24, 0x8d, 0x1e, 0x53, 0xdc, 0x83, 0x6f, 0x30, 0xfe, 0x00, 0xeb, 0xbb, 0x6b, 0x35, 0xc3, 0x20, 
+0xea, 0xae, 0x27, 0xb4, 0x8a, 0xdc, 0x30, 0x9f, 0xb5, 0xee, 0xbf, 0x3c, 0x16, 0x58, 0xe1, 0xa6, 
+0xec, 0x87, 0xfd, 0xb0, 0x43, 0x8c, 0xed, 0x4d, 0x00, 0x2d, 0x85, 0x33, 0xbe, 0x06, 0x22, 0x6e, 
+0x46, 0x11, 0x1a, 0x0b, 0x59, 0xca, 0xaf, 0x12, 0x60, 0x43, 0xeb, 0x5b, 0xbf, 0x28, 0xc3, 0x4f, 
+0x3a, 0x5e, 0x33, 0x2a, 0x1f, 0xc7, 0xb2, 0xb7, 0x3c, 0xf1, 0x88, 0x91, 0x0f, 0x00, 0x00, 0x71, 
+0x00, 0x00, 0x03, 0x00, 0x01, 0x64, 0x62, 0x19, 0xec, 0x01, 0x01, 0x00, 0x06, 0x00, 0x00, 0x00, 
+0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x0a, 0x00, 0x00, 0x00, 
+0x00, 0x76, 0x04, 0x67, 0x00, 0x00, 0x00, 0x00, 0xa4, 0x1e, 0xb8, 0x7e, 0x0a, 0x64, 0x62, 0x19, 
+0xf1, 0x01, 0x01, 0x70, 0xc5, 0x12, 0xaa, 0x59, 0xee, 0xe5, 0xb5, 0x1f, 0x4c, 0x56, 0x77, 0xa1, 
+0xc5, 0x3c, 0x6b, 0x03, 0x37, 0xf9, 0x8f, 0xa9, 0x50, 0xa7, 0xe3, 0x22, 0x7b, 0x6e, 0x37, 0xd5, 
+0x46, 0x03, 0xff, 0x12, 0x91, 0x0a, 0xb8, 0x4f, 0x35, 0x63, 0xdf, 0xda, 0x03, 0xda, 0xee, 0x86, 
+0xe4, 0x43, 0xef, 0xa0, 0x8a, 0x90, 0xeb, 0xa8, 0xf3, 0x7f, 0x05, 0x84, 0x8a, 0xd8, 0xb0, 0xf8, 
+0x1b, 0x4b, 0xcf, 0x00, 0x07, 0x88, 0xa0, 0x80, 0x2a, 0x02, 0x69, 0xa2, 0x64, 0x62, 0x19, 0xf1, 
+0x02, 0x45, 0x1e, 0x9b, 0xaf, 0xf8, 0x1f, 0xaf, 0x5f, 0x41, 0x0b, 0x0b, 0xbe, 0x9e, 0x36, 0xee, 
+0x83, 0xbf, 0x8f, 0x79, 0x69, 0x8e, 0x66, 0x05, 0xa5, 0x1a, 0x97, 0x48, 0xbc, 0x73, 0xc7, 0xdc, 
+0x28, 0x02, 0x45, 0x1e, 0x4c, 0x4f, 0x55, 0x44, 0x54, 0x4f, 0x54, 0x45, 0x2d, 0x33, 0x2e, 0x30, 
+0x32, 0x2d, 0x33, 0x37, 0x2d, 0x67, 0x63, 0x39, 0x66, 0x38, 0x30, 0x33, 0x35, 0x2d, 0x6d, 0x6f, 
+0x64, 0x64, 0x65, 0x64, 0x00, 0x00, 0x01, 0x0d, 0x02, 0x9a, 0x00, 0x32, 0x00, 0x64, 0x00, 0x00, 
+0x00, 0x02, 0x4c, 0x4b, 0x40, 0x00, 0x00, 0x01, 0xb0, 0x17, 0xe0, 0xd7, 0x83, 0x64, 0x62, 0x19, 
+0xed, 0x01, 0x00, 0x48, 0xa3, 0x33, 0x5f, 0x33, 0x6c, 0x33, 0x85, 0x0f, 0xc7, 0xeb, 0x46, 0x04, 
+0x5a, 0xe7, 0x1a, 0x2d, 0xe1, 0x37, 0xb0, 0xc3, 0x8a, 0xa7, 0x6a, 0xe0, 0xa2, 0xfd, 0x1f, 0x30, 
+0x9f, 0xdc, 0x8d, 0x38, 0x05, 0xf7, 0xaf, 0x0b, 0xe6, 0xb3, 0x4d, 0x62, 0xb9, 0xa4, 0x9c, 0x53, 
+0x7d, 0x6e, 0x59, 0x5b, 0xb2, 0x2b, 0x5c, 0xda, 0x35, 0xf9, 0x90, 0x63, 0x21, 0xa8, 0xb1, 0x53, 
+0xc3, 0x35, 0x7c, 0x36, 0x76, 0x21, 0x76, 0xae, 0xa3, 0xad, 0x05, 0x53, 0xa7, 0xbd, 0x9d, 0x38, 
+0x54, 0x03, 0xc0, 0x98, 0x1d, 0x66, 0xc1, 0x04, 0x39, 0xc1, 0x88, 0xd1, 0x1f, 0x90, 0x08, 0x96, 
+0xbc, 0x59, 0x54, 0x4f, 0x5f, 0xa2, 0x70, 0xcd, 0xf0, 0xda, 0x96, 0x3c, 0x51, 0x04, 0x67, 0x5c, 
+0x1f, 0x07, 0xed, 0xf9, 0x9e, 0x98, 0xd0, 0x3b, 0x5e, 0x51, 0xa9, 0xa6, 0x82, 0xc1, 0xed, 0x35, 
+0x45, 0xa1, 0xd6, 0x36, 0x3b, 0xa1, 0xe6, 0x5d, 0x1f, 0xec, 0xe2, 0xb7, 0xf8, 0xa2, 0xe4, 0x45, 
+0xf9, 0xb6, 0xa7, 0x07, 0x18, 0xc7, 0xb5, 0x0c, 0x08, 0xd7, 0x50, 0x36, 0x98, 0x82, 0xd3, 0xc8, 
+0x40, 0xc8, 0xdc, 0x64, 0x27, 0xe2, 0x14, 0x42, 0x44, 0x0a, 0xe4, 0x1d, 0x41, 0x61, 0x57, 0x88, 
+0xfe, 0xd2, 0x51, 0x99, 0x24, 0x55, 0x1e, 0x3b, 0xaa, 0x8d, 0xa7, 0xb4, 0xc0, 0x6e, 0xf5, 0x70, 
+0x8c, 0x2a, 0xe3, 0x75, 0xcc, 0x36, 0xbf, 0xbe, 0xfc, 0x3f, 0x09, 0x83, 0x5e, 0xe4, 0x20, 0x9a, 
+0xcc, 0x11, 0x48, 0x8e, 0x2b, 0xc8, 0x8a, 0xef, 0xc0, 0x78, 0x45, 0xee, 0x1e, 0xc7, 0xce, 0x00, 
+0xfc, 0x3c, 0x0e, 0x32, 0xd2, 0x8f, 0x15, 0x8c, 0x02, 0xb3, 0x7b, 0x4c, 0xa9, 0x7a, 0x9c, 0xec, 
+0x5e, 0x6e, 0xf2, 0xd3, 0xd9, 0x15, 0x32, 0xa3, 0x74, 0x14, 0xbf, 0x1f, 0xdd, 0x2f, 0x63, 0x3c, 
+0x47, 0x04, 0x6c, 0x00, 0x00, 0x06, 0x22, 0x6e, 0x46, 0x11, 0x1a, 0x0b, 0x59, 0xca, 0xaf, 0x12, 
+0x60, 0x43, 0xeb, 0x5b, 0xbf, 0x28, 0xc3, 0x4f, 0x3a, 0x5e, 0x33, 0x2a, 0x1f, 0xc7, 0xb2, 0xb7, 
+0x3c, 0xf1, 0x88, 0x91, 0x0f, 0x00, 0x00, 0x71, 0x00, 0x00, 0x02, 0x00, 0x00, 0x02, 0x45, 0x1e, 
+0x9b, 0xaf, 0xf8, 0x1f, 0xaf, 0x5f, 0x41, 0x0b, 0x0b, 0xbe, 0x9e, 0x36, 0xee, 0x83, 0xbf, 0x8f, 
+0x79, 0x69, 0x8e, 0x66, 0x05, 0xa5, 0x1a, 0x97, 0x48, 0xbc, 0x73, 0xc7, 0xdc, 0x28, 0x02, 0xd1, 
+0xab, 0x24, 0xfe, 0x53, 0xcf, 0xca, 0xc4, 0xa4, 0x77, 0x74, 0x52, 0x35, 0xc7, 0xac, 0x3e, 0x75, 
+0x16, 0x1a, 0x5b, 0xf8, 0x42, 0x6e, 0xa8, 0xe0, 0x18, 0x2a, 0xd5, 0x8c, 0xab, 0x36, 0x7f, 0x02, 
+0x7e, 0x2a, 0xc0, 0xec, 0x93, 0xfd, 0xb3, 0xfb, 0xe3, 0x8d, 0x7a, 0x3f, 0x5e, 0xa0, 0xa6, 0x3d, 
+0xdb, 0xa9, 0x8a, 0x51, 0xb7, 0x7a, 0xf5, 0x51, 0x6f, 0xe5, 0xca, 0x10, 0x10, 0xd7, 0x95, 0x34, 
+0x02, 0x17, 0xd5, 0xb1, 0x80, 0x7d, 0x8b, 0x95, 0x7c, 0xe1, 0x0b, 0xb0, 0xaf, 0xf3, 0xc1, 0x84, 
+0x81, 0xee, 0x2f, 0xed, 0x6a, 0x7b, 0x65, 0x9c, 0xbf, 0xfd, 0x48, 0x20, 0xd0, 0x9d, 0x1a, 0xfd, 
+0xa4, 0x00, 0x00, 0x00, 0x0a, 0x91, 0x11, 0x83, 0xf6, 0x00, 0x00, 0x00, 0x00, 0x10, 0x05, 0x00, 
+0x00, 0x00, 0x00, 0x00, 0x0f, 0x42, 0x40, 0x80, 0x00, 0x00, 0x8a, 0xbd, 0x52, 0xa0, 0x78, 0x64, 
+0x62, 0x19, 0xed, 0x01, 0x02, 0x40, 0xf0, 0x06, 0x07, 0x97, 0xb8, 0x87, 0xef, 0x73, 0xdc, 0x1b, 
+0xf0, 0x20, 0x31, 0x55, 0xc9, 0xb9, 0x6f, 0xec, 0x6f, 0xad, 0x46, 0x86, 0x0a, 0xcc, 0xd9, 0x95, 
+0x61, 0x62, 0x15, 0x84, 0x70, 0x2a, 0x47, 0xd7, 0x68, 0xa9, 0xbc, 0x98, 0xb3, 0x1f, 0xc4, 0xbc, 
+0x78, 0xab, 0x5d, 0xf2, 0xf7, 0xc4, 0x97, 0x75, 0x21, 0x13, 0xcf, 0xfc, 0xd4, 0x36, 0xcd, 0xf6, 
+0xb4, 0x85, 0x7c, 0xad, 0x01, 0x06, 0x22, 0x6e, 0x46, 0x11, 0x1a, 0x0b, 0x59, 0xca, 0xaf, 0x12, 
+0x60, 0x43, 0xeb, 0x5b, 0xbf, 0x28, 0xc3, 0x4f, 0x3a, 0x5e, 0x33, 0x2a, 0x1f, 0xc7, 0xb2, 0xb7, 
+0x3c, 0xf1, 0x88, 0x91, 0x0f, 0x00, 0x00, 0x71, 0x00, 0x00, 0x02, 0x00, 0x00, 0x64, 0x62, 0x19, 
+0xed, 0x01, 0x00, 0x00, 0x06, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 
+0x01, 0x00, 0x00, 0x00, 0x0a, 0x00, 0x00, 0x00, 0x00, 0x3b, 0x02, 0x33, 0x80, 0x80, 0x00, 0x00, 
+0x8a, 0xf5, 0x5d, 0xd1, 0x12, 0x64, 0x62, 0x19, 0xed, 0x01, 0x02, 0x08, 0x97, 0x08, 0x72, 0xbe, 
+0xc8, 0x1e, 0xd0, 0xb9, 0xb8, 0x4b, 0x0f, 0x63, 0x5c, 0xeb, 0x28, 0xa5, 0xf8, 0x7a, 0x3d, 0xa1, 
+0x6a, 0xb3, 0xb4, 0x30, 0x91, 0x31, 0x57, 0xd4, 0x5b, 0x69, 0x26, 0x4d, 0xd1, 0xbb, 0xd5, 0x49, 
+0x95, 0xe9, 0x75, 0x53, 0xa4, 0xae, 0x87, 0xe9, 0x88, 0xf6, 0x86, 0x1f, 0x31, 0x8f, 0x35, 0xf9, 
+0x15, 0xcc, 0x04, 0x0a, 0x01, 0xed, 0x6e, 0x47, 0xe0, 0xea, 0x68, 0x06, 0x22, 0x6e, 0x46, 0x11, 
+0x1a, 0x0b, 0x59, 0xca, 0xaf, 0x12, 0x60, 0x43, 0xeb, 0x5b, 0xbf, 0x28, 0xc3, 0x4f, 0x3a, 0x5e, 
+0x33, 0x2a, 0x1f, 0xc7, 0xb2, 0xb7, 0x3c, 0xf1, 0x88, 0x91, 0x0f, 0x00, 0x00, 0x71, 0x00, 0x00, 
+0x02, 0x00, 0x00, 0x64, 0x62, 0x19, 0xed, 0x01, 0x01, 0x00, 0x06, 0x00, 0x00, 0x00, 0x00, 0x00, 
+0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x0a, 0x00, 0x00, 0x00, 0x00, 0x3b, 
+0x02, 0x33, 0x80, 0x00, 0x00, 0x00, 0xa4, 0x07, 0x3d, 0xb7, 0xfe, 0x64, 0x62, 0x19, 0xf2, 0x01, 
+0x01, 0x29, 0x2a, 0x41, 0x8f, 0xb7, 0x24, 0xc2, 0x82, 0xc5, 0x75, 0x0e, 0x28, 0xd9, 0x8b, 0xd4, 
+0xad, 0xa1, 0xb1, 0x9a, 0x65, 0xa8, 0x7a, 0x78, 0xc7, 0x6c, 0xc8, 0x94, 0xcb, 0xf7, 0xb1, 0xb8, 
+0x3b, 0x29, 0xce, 0xbf, 0xcc, 0x47, 0x1b, 0x5a, 0xb4, 0xec, 0xab, 0xa3, 0xbe, 0xaf, 0xd1, 0xde, 
+0xd7, 0x0e, 0x8b, 0xcc, 0xaa, 0xdb, 0x6b, 0x88, 0x51, 0xb9, 0x7a, 0x0c, 0xcd, 0x1c, 0x9c, 0x4d, 
+0x5c, 0x00, 0x07, 0x88, 0xa0, 0x80, 0x2a, 0x02, 0x69, 0xa2, 0x64, 0x62, 0x19, 0xf2, 0x02, 0xd1, 
+0xab, 0x24, 0xfe, 0x53, 0xcf, 0xca, 0xc4, 0xa4, 0x77, 0x74, 0x52, 0x35, 0xc7, 0xac, 0x3e, 0x75, 
+0x16, 0x1a, 0x5b, 0xf8, 0x42, 0x6e, 0xa8, 0xe0, 0x18, 0x2a, 0xd5, 0x8c, 0xab, 0x36, 0x7f, 0x02, 
+0xd1, 0xab, 0x43, 0x48, 0x49, 0x4c, 0x4c, 0x59, 0x46, 0x49, 0x52, 0x45, 0x2d, 0x30, 0x32, 0x2d, 
+0x33, 0x37, 0x2d, 0x67, 0x63, 0x39, 0x66, 0x38, 0x30, 0x33, 0x35, 0x2d, 0x6d, 0x6f, 0x64, 0x64, 
+0x65, 0x64, 0x00, 0x00, 0x01, 0x0d, 0x02, 0x9a, 0x00, 0x32, 0x00, 0x64, 0x00, 0x00, 0x00, 0x02, 
+0x4c, 0x4b, 0x40, 0x00, 0x00, 0x01, 0xb0, 0x5f, 0xad, 0x58, 0xa1, 0x64, 0x62, 0x19, 0xed, 0x01, 
+0x00, 0x63, 0x42, 0xba, 0xf1, 0x21, 0xe0, 0x09, 0x57, 0x0d, 0x40, 0xa4, 0xc6, 0x05, 0x78, 0x02, 
+0x8e, 0x35, 0x71, 0x66, 0x7c, 0x24, 0x51, 0x3f, 0x58, 0x3a, 0xaa, 0x14, 0x65, 0x5a, 0x2b, 0xbd, 
+0x09, 0x5b, 0xd3, 0xa8, 0x4e, 0x73, 0x3e, 0x38, 0xd3, 0x4d, 0x19, 0x9c, 0x18, 0x04, 0x60, 0x57, 
+0x32, 0xd3, 0x75, 0xf5, 0x36, 0x15, 0xc4, 0x6a, 0xf1, 0x1b, 0x3d, 0xc9, 0x07, 0x89, 0x08, 0x7a, 
+0x37, 0x2f, 0xf1, 0x89, 0x12, 0xdb, 0xf2, 0xff, 0x04, 0xbd, 0x93, 0x23, 0x00, 0x3c, 0x10, 0x05, 
+0x8a, 0x58, 0x9b, 0x96, 0xf3, 0x76, 0x94, 0x16, 0x29, 0x51, 0xc8, 0x76, 0x89, 0x61, 0xc3, 0x21, 
+0xc0, 0x0e, 0x47, 0xac, 0xa3, 0xbe, 0xc7, 0xfd, 0xa2, 0x6b, 0xe9, 0x1d, 0xe2, 0x11, 0x1c, 0x3e, 
+0xfc, 0x6d, 0x4d, 0x0b, 0x85, 0xff, 0xe9, 0x8a, 0x39, 0x3a, 0xb3, 0x0e, 0x2f, 0x28, 0x96, 0x6b, 
+0x96, 0x59, 0x4d, 0x53, 0x71, 0xd5, 0x38, 0x23, 0xe1, 0xe0, 0xad, 0x0a, 0xbf, 0x00, 0x58, 0x15, 
+0xbf, 0x53, 0x07, 0xe1, 0x13, 0x06, 0x88, 0xb3, 0xf8, 0x31, 0x06, 0x72, 0x92, 0x6f, 0xd1, 0xf0, 
+0x9b, 0x3b, 0xf2, 0x8f, 0x9c, 0xc6, 0x73, 0xf8, 0x91, 0x3e, 0x84, 0xc0, 0xed, 0xdf, 0x92, 0x43, 
+0x92, 0x5f, 0x4a, 0x6b, 0x96, 0x02, 0xaf, 0xd9, 0xd9, 0xd9, 0xf9, 0x65, 0xae, 0x08, 0xd8, 0x62, 
+0x93, 0x2b, 0xb7, 0xd3, 0x48, 0xe3, 0x02, 0x19, 0x53, 0xf9, 0x49, 0x24, 0xfa, 0x22, 0x24, 0x87, 
+0xc2, 0xd2, 0x0b, 0xc0, 0x56, 0xae, 0x09, 0x5a, 0x94, 0xc3, 0x54, 0x59, 0xb5, 0xe7, 0xbe, 0xa6, 
+0x4a, 0x47, 0xc1, 0x79, 0x80, 0xe8, 0xc2, 0xd1, 0xc5, 0xda, 0x6b, 0x25, 0x85, 0xc6, 0x02, 0x32, 
+0x8b, 0x52, 0x0e, 0x7f, 0x18, 0x1c, 0x5b, 0xf6, 0xb9, 0xaf, 0x69, 0xdc, 0xc6, 0x3d, 0x93, 0xc1, 
+0x27, 0x00, 0x00, 0x06, 0x22, 0x6e, 0x46, 0x11, 0x1a, 0x0b, 0x59, 0xca, 0xaf, 0x12, 0x60, 0x43, 
+0xeb, 0x5b, 0xbf, 0x28, 0xc3, 0x4f, 0x3a, 0x5e, 0x33, 0x2a, 0x1f, 0xc7, 0xb2, 0xb7, 0x3c, 0xf1, 
+0x88, 0x91, 0x0f, 0x00, 0x00, 0x71, 0x00, 0x00, 0x04, 0x00, 0x01, 0x02, 0xd1, 0xab, 0x24, 0xfe, 
+0x53, 0xcf, 0xca, 0xc4, 0xa4, 0x77, 0x74, 0x52, 0x35, 0xc7, 0xac, 0x3e, 0x75, 0x16, 0x1a, 0x5b, 
+0xf8, 0x42, 0x6e, 0xa8, 0xe0, 0x18, 0x2a, 0xd5, 0x8c, 0xab, 0x36, 0x7f, 0x03, 0xca, 0xec, 0x54, 
+0x08, 0x55, 0x08, 0xda, 0x06, 0xf1, 0x4f, 0xfb, 0x83, 0x61, 0x66, 0xca, 0x22, 0x04, 0x2d, 0x0f, 
+0xda, 0x69, 0x69, 0x03, 0x56, 0x23, 0x2a, 0x24, 0xb8, 0x36, 0x6c, 0x8f, 0x85, 0x03, 0xf6, 0x19, 
+0x62, 0x15, 0xf2, 0x5c, 0xfc, 0x5c, 0xae, 0x8c, 0xb6, 0x90, 0xa7, 0x81, 0xe0, 0x14, 0xb5, 0xc1, 
+0xc5, 0xda, 0xf9, 0x6d, 0x44, 0x6d, 0x1a, 0x6e, 0x24, 0x4f, 0xb6, 0x42, 0x3f, 0xdb, 0x03, 0xf9, 
+0x84, 0xe3, 0xec, 0xa9, 0x24, 0x5d, 0x1b, 0xba, 0xd2, 0xc7, 0xf3, 0x5a, 0x32, 0xaa, 0x6e, 0xdb, 
+0x21, 0xb6, 0xe8, 0xb1, 0x86, 0x5b, 0x18, 0x30, 0xe8, 0x4d, 0x23, 0xa4, 0x45, 0x23, 0x88, 0x00, 
+0x00, 0x00, 0x0a, 0x08, 0x85, 0x8a, 0xb2, 0x00, 0x00, 0x00, 0x00, 0x10, 0x05, 0x00, 0x00, 0x00, 
+0x00, 0x00, 0x2d, 0xc6, 0xc0, 0x80, 0x00, 0x00, 0x8a, 0xe9, 0x51, 0x74, 0x9b, 0x64, 0x62, 0x19, 
+0xed, 0x01, 0x02, 0x4b, 0x82, 0x87, 0x3b, 0xc9, 0x03, 0x1c, 0x6e, 0xc9, 0xbe, 0x96, 0x22, 0x97, 
+0xf7, 0xa8, 0xb0, 0xb2, 0x7c, 0x22, 0x69, 0x23, 0x2d, 0x97, 0xfb, 0x9b, 0xc2, 0xf1, 0x1e, 0x66, 
+0xfb, 0xfd, 0x80, 0x5d, 0xd7, 0xf0, 0x23, 0x31, 0x47, 0xaa, 0x54, 0x8d, 0x95, 0xbb, 0xdd, 0x33, 
+0x13, 0x32, 0x6d, 0x91, 0xc6, 0x45, 0xd5, 0x84, 0xf4, 0x76, 0x6c, 0x74, 0xf3, 0x51, 0x45, 0x24, 
+0xee, 0x5b, 0xc3, 0x06, 0x22, 0x6e, 0x46, 0x11, 0x1a, 0x0b, 0x59, 0xca, 0xaf, 0x12, 0x60, 0x43, 
+0xeb, 0x5b, 0xbf, 0x28, 0xc3, 0x4f, 0x3a, 0x5e, 0x33, 0x2a, 0x1f, 0xc7, 0xb2, 0xb7, 0x3c, 0xf1, 
+0x88, 0x91, 0x0f, 0x00, 0x00, 0x71, 0x00, 0x00, 0x04, 0x00, 0x01, 0x64, 0x62, 0x19, 0xed, 0x01, 
+0x00, 0x00, 0x06, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 
+0x00, 0x00, 0x0a, 0x00, 0x00, 0x00, 0x00, 0xb1, 0x06, 0x9a, 0x80, 0x80, 0x00, 0x00, 0x8a, 0xd2, 
+0xc8, 0xd2, 0x7c, 0x64, 0x62, 0x19, 0xed, 0x01, 0x02, 0x0f, 0x90, 0xcb, 0xb6, 0xa1, 0x44, 0x65, 
+0x10, 0x00, 0xae, 0x2f, 0x60, 0x26, 0x2f, 0x41, 0x58, 0x5b, 0xab, 0xde, 0xff, 0x7e, 0x11, 0x44, 
+0xf4, 0x2e, 0x96, 0x96, 0xfa, 0x98, 0x09, 0xee, 0xb1, 0x5d, 0x43, 0xff, 0x44, 0x7b, 0xa6, 0x03, 
+0xf6, 0x4a, 0x07, 0x38, 0x97, 0x59, 0xee, 0x5e, 0xee, 0xcb, 0xdb, 0x77, 0x69, 0xab, 0x61, 0xd8, 
+0xc3, 0x42, 0xb3, 0x1f, 0x57, 0xea, 0xf3, 0xfd, 0xe2, 0x06, 0x22, 0x6e, 0x46, 0x11, 0x1a, 0x0b, 
+0x59, 0xca, 0xaf, 0x12, 0x60, 0x43, 0xeb, 0x5b, 0xbf, 0x28, 0xc3, 0x4f, 0x3a, 0x5e, 0x33, 0x2a, 
+0x1f, 0xc7, 0xb2, 0xb7, 0x3c, 0xf1, 0x88, 0x91, 0x0f, 0x00, 0x00, 0x71, 0x00, 0x00, 0x04, 0x00, 
+0x01, 0x64, 0x62, 0x19, 0xed, 0x01, 0x01, 0x00, 0x06, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 
+0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x0a, 0x00, 0x00, 0x00, 0x00, 0xb1, 0x06, 0x9a, 
+0x80, 0x40, 0x00, 0x00, 0x8a, 0x64, 0xa7, 0x4f, 0x57, 0x64, 0x62, 0x19, 0xf7, 0x01, 0x02, 0x28, 
+0x15, 0x9f, 0xa7, 0x51, 0x3a, 0xbb, 0x33, 0xd9, 0x25, 0xaa, 0x7d, 0xe8, 0xfb, 0x3a, 0x92, 0x45, 
+0x41, 0xb3, 0x22, 0x9b, 0x12, 0x3b, 0xb0, 0x16, 0x47, 0xd6, 0xf7, 0x61, 0x44, 0x1d, 0xa7, 0x35, 
+0xfe, 0xa9, 0x7b, 0xa6, 0x42, 0x91, 0x3f, 0x5e, 0xe4, 0xca, 0x98, 0x1c, 0x0f, 0x2d, 0xed, 0x36, 
+0x0e, 0x2b, 0x2e, 0x08, 0x81, 0x2e, 0xcc, 0xc1, 0x76, 0x61, 0xf9, 0x1b, 0xd3, 0x44, 0x3e, 0x06, 
+0x22, 0x6e, 0x46, 0x11, 0x1a, 0x0b, 0x59, 0xca, 0xaf, 0x12, 0x60, 0x43, 0xeb, 0x5b, 0xbf, 0x28, 
+0xc3, 0x4f, 0x3a, 0x5e, 0x33, 0x2a, 0x1f, 0xc7, 0xb2, 0xb7, 0x3c, 0xf1, 0x88, 0x91, 0x0f, 0x00, 
+0x00, 0x71, 0x00, 0x00, 0x01, 0x00, 0x01, 0x64, 0x62, 0x19, 0xf7, 0x01, 0x00, 0x00, 0x06, 0x00, 
+0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0a, 0x00, 0x00, 0x00, 0x64, 0x00, 
+0x00, 0x00, 0x00, 0x3b, 0x02, 0x33, 0x80, 0x00, 0x00, 0x00, 0xa4, 0x30, 0x77, 0x80, 0xee, 0x64, 
+0x62, 0x19, 0xf2, 0x01, 0x01, 0x67, 0x07, 0x1d, 0x3b, 0x62, 0x2d, 0xb7, 0x1a, 0xba, 0xb8, 0x93, 
+0x56, 0xaa, 0xfa, 0xb1, 0x47, 0x4f, 0x0e, 0x02, 0x8b, 0x73, 0xd5, 0x5b, 0xce, 0xd6, 0x40, 0x55, 
+0xaf, 0xa7, 0x29, 0xd0, 0x51, 0x24, 0x5a, 0x19, 0x22, 0xc6, 0x7b, 0x6e, 0x4a, 0xae, 0x57, 0x9c, 
+0x16, 0x99, 0x46, 0x6c, 0xc3, 0x64, 0xd8, 0x20, 0x10, 0x44, 0x1e, 0xd0, 0x6b, 0x8d, 0x36, 0xdc, 
+0xae, 0x75, 0x06, 0x6e, 0xdc, 0x00, 0x07, 0x88, 0xa0, 0x80, 0x2a, 0x02, 0x69, 0xa2, 0x64, 0x62, 
+0x19, 0xf2, 0x03, 0xca, 0xec, 0x54, 0x08, 0x55, 0x08, 0xda, 0x06, 0xf1, 0x4f, 0xfb, 0x83, 0x61, 
+0x66, 0xca, 0x22, 0x04, 0x2d, 0x0f, 0xda, 0x69, 0x69, 0x03, 0x56, 0x23, 0x2a, 0x24, 0xb8, 0x36, 
+0x6c, 0x8f, 0x85, 0x03, 0xca, 0xec, 0x56, 0x49, 0x4f, 0x4c, 0x45, 0x54, 0x53, 0x45, 0x54, 0x2d, 
+0x2e, 0x30, 0x32, 0x2d, 0x33, 0x37, 0x2d, 0x67, 0x63, 0x39, 0x66, 0x38, 0x30, 0x33, 0x35, 0x2d, 
+0x6d, 0x6f, 0x64, 0x64, 0x65, 0x64, 0x00, 0x00, 0x01, 0x0d, 0x02, 0x9a, 0x00, 0x32, 0x00, 0x64, 
+0x00, 0x00, 0x00, 0x02, 0x4c, 0x4b, 0x40, 0x40, 0x00, 0x00, 0x8a, 0x07, 0xf6, 0xe8, 0x9b, 0x64, 
+0x62, 0x19, 0xf7, 0x01, 0x02, 0x62, 0xf7, 0xdc, 0xf1, 0xa6, 0x3c, 0xf0, 0xae, 0x64, 0x9c, 0x03, 
+0x62, 0x98, 0x6a, 0x18, 0x78, 0x97, 0xed, 0x8c, 0x2e, 0x3f, 0xc4, 0x1d, 0x9f, 0xa7, 0xfb, 0x58, 
+0x26, 0x48, 0x2e, 0x96, 0x9d, 0x33, 0x75, 0x60, 0x6e, 0x33, 0x95, 0xf7, 0x6e, 0x9f, 0x4f, 0xa2, 
+0xed, 0xd6, 0xa9, 0x83, 0x1b, 0x94, 0x79, 0xee, 0x4f, 0xdc, 0x20, 0xc5, 0x39, 0x74, 0x0d, 0x31, 
+0x52, 0xc7, 0x25, 0x36, 0x47, 0x06, 0x22, 0x6e, 0x46, 0x11, 0x1a, 0x0b, 0x59, 0xca, 0xaf, 0x12, 
+0x60, 0x43, 0xeb, 0x5b, 0xbf, 0x28, 0xc3, 0x4f, 0x3a, 0x5e, 0x33, 0x2a, 0x1f, 0xc7, 0xb2, 0xb7, 
+0x3c, 0xf1, 0x88, 0x91, 0x0f, 0x00, 0x00, 0x71, 0x00, 0x00, 0x01, 0x00, 0x01, 0x64, 0x62, 0x19, 
+0xf7, 0x01, 0x01, 0x00, 0x06, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 
+0x14, 0x00, 0x00, 0x00, 0xc8, 0x00, 0x00, 0x00, 0x00, 0x3b, 0x02, 0x33, 0x80, 0x00, 0x00, 0x00, 
+0x8a, 0x16, 0x2b, 0xff, 0x08, 0x64, 0x62, 0x19, 0xf7, 0x01, 0x02, 0x39, 0x36, 0x2a, 0x56, 0x61, 
+0xad, 0x48, 0x3f, 0x4e, 0x13, 0x15, 0x66, 0x43, 0x58, 0xc5, 0xc2, 0x14, 0x6e, 0xb2, 0x72, 0xfa, 
+0x73, 0xd7, 0xb5, 0x2d, 0x86, 0x14, 0xc2, 0xe8, 0xf7, 0x53, 0x8f, 0x38, 0xea, 0x35, 0x5c, 0xec, 
+0xe3, 0xc7, 0xc0, 0x46, 0x1c, 0x9f, 0x1d, 0x93, 0x94, 0x31, 0x1f, 0xf8, 0x49, 0xb1, 0x50, 0x4c, 
+0x2c, 0x2f, 0xc7, 0xe4, 0x0c, 0xaa, 0xd0, 0xa9, 0x53, 0x14, 0xca, 0x06, 0x22, 0x6e, 0x46, 0x11, 
+0x1a, 0x0b, 0x59, 0xca, 0xaf, 0x12, 0x60, 0x43, 0xeb, 0x5b, 0xbf, 0x28, 0xc3, 0x4f, 0x3a, 0x5e, 
+0x33, 0x2a, 0x1f, 0xc7, 0xb2, 0xb7, 0x3c, 0xf1, 0x88, 0x91, 0x0f, 0x00, 0x00, 0x71, 0x00, 0x00, 
+0x03, 0x00, 0x01, 0x64, 0x62, 0x19, 0xf7, 0x01, 0x01, 0x00, 0x06, 0x00, 0x00, 0x00, 0x00, 0x00, 
+0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x64, 0x00, 0x00, 0x00, 0x00, 0x76, 
+0x04, 0x67, 0x00, 0x00, 0x00, 0x00, 0x8a, 0xd5, 0x4a, 0x70, 0x0c, 0x64, 0x62, 0x19, 0xf7, 0x01, 
+0x02, 0x54, 0x16, 0x95, 0x41, 0x4f, 0x0e, 0x0f, 0xdf, 0x49, 0xb5, 0x87, 0xdc, 0x26, 0xb4, 0xef, 
+0x73, 0x3c, 0xb8, 0x19, 0x96, 0x62, 0x87, 0xfa, 0x4f, 0x02, 0x53, 0xbe, 0x12, 0x53, 0x93, 0x4b, 
+0x57, 0x3b, 0xe9, 0xb9, 0x26, 0x46, 0xda, 0x77, 0xaa, 0xdd, 0x8d, 0xf6, 0x86, 0x22, 0xf0, 0x3f, 
+0xd5, 0x56, 0xdd, 0xaa, 0xa2, 0x4e, 0x4a, 0x9a, 0x70, 0x81, 0xf8, 0xf9, 0x72, 0x7b, 0xd7, 0x90, 
+0x48, 0x06, 0x22, 0x6e, 0x46, 0x11, 0x1a, 0x0b, 0x59, 0xca, 0xaf, 0x12, 0x60, 0x43, 0xeb, 0x5b, 
+0xbf, 0x28, 0xc3, 0x4f, 0x3a, 0x5e, 0x33, 0x2a, 0x1f, 0xc7, 0xb2, 0xb7, 0x3c, 0xf1, 0x88, 0x91, 
+0x0f, 0x00, 0x00, 0x71, 0x00, 0x00, 0x03, 0x00, 0x01, 0x64, 0x62, 0x19, 0xf7, 0x01, 0x00, 0x00, 
+0x06, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 
+0xc8, 0x00, 0x00, 0x00, 0x00, 0x76, 0x04, 0x67, 0x00, 0x00, 0x00, 0x00, 0x8a, 0xc5, 0x6d, 0x8a, 
+0x5a, 0x64, 0x62, 0x19, 0xf6, 0x01, 0x02, 0x1d, 0x80, 0x09, 0x30, 0x1a, 0x4b, 0x26, 0x60, 0x6b, 
+0x9a, 0x54, 0x8d, 0x7f, 0x9b, 0x35, 0x78, 0x76, 0x7a, 0xc1, 0xe5, 0x22, 0xdc, 0x08, 0x77, 0xac, 
+0x54, 0xc7, 0xc0, 0x9b, 0x13, 0x85, 0x20, 0x2c, 0xa4, 0xa3, 0x7e, 0xc5, 0xde, 0xfd, 0x60, 0x43, 
+0xdb, 0x2e, 0xb0, 0x5b, 0xcc, 0x95, 0xc1, 0xf3, 0x02, 0x09, 0x8a, 0xe1, 0x55, 0x2a, 0x8a, 0x9a, 
+0x18, 0xe5, 0xa9, 0xee, 0xcd, 0x11, 0x27, 0x06, 0x22, 0x6e, 0x46, 0x11, 0x1a, 0x0b, 0x59, 0xca, 
+0xaf, 0x12, 0x60, 0x43, 0xeb, 0x5b, 0xbf, 0x28, 0xc3, 0x4f, 0x3a, 0x5e, 0x33, 0x2a, 0x1f, 0xc7, 
+0xb2, 0xb7, 0x3c, 0xf1, 0x88, 0x91, 0x0f, 0x00, 0x00, 0x71, 0x00, 0x00, 0x02, 0x00, 0x00, 0x64, 
+0x62, 0x19, 0xf6, 0x01, 0x00, 0x00, 0x06, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 
+0x00, 0x00, 0x0b, 0x00, 0x00, 0x00, 0x96, 0x00, 0x00, 0x00, 0x00, 0x3b, 0x02, 0x33, 0x80, 0x00, 
+0x00, 0x00, 0x8a, 0x67, 0xa5, 0x58, 0xd4, 0x64, 0x62, 0x19, 0xf6, 0x01, 0x02, 0x5a, 0xa5, 0x3e, 
+0xb8, 0x73, 0xf5, 0xdf, 0xfc, 0x72, 0x16, 0x52, 0xa1, 0x07, 0x8a, 0x2b, 0xf1, 0xc3, 0x92, 0xc5, 
+0x87, 0xa4, 0x45, 0x07, 0x1e, 0xb3, 0x7d, 0x4c, 0x1c, 0x47, 0x41, 0x2c, 0x93, 0x14, 0x46, 0x16, 
+0xba, 0xe4, 0xf9, 0xc9, 0x52, 0x4c, 0x5e, 0x6c, 0x4f, 0xc9, 0xec, 0xde, 0x83, 0x15, 0xe0, 0x8e, 
+0x39, 0xbe, 0xa9, 0x8f, 0x9d, 0xfe, 0xcf, 0xc4, 0x12, 0x32, 0xa4, 0x17, 0x2b, 0x06, 0x22, 0x6e, 
+0x46, 0x11, 0x1a, 0x0b, 0x59, 0xca, 0xaf, 0x12, 0x60, 0x43, 0xeb, 0x5b, 0xbf, 0x28, 0xc3, 0x4f, 
+0x3a, 0x5e, 0x33, 0x2a, 0x1f, 0xc7, 0xb2, 0xb7, 0x3c, 0xf1, 0x88, 0x91, 0x0f, 0x00, 0x00, 0x71, 
+0x00, 0x00, 0x02, 0x00, 0x00, 0x64, 0x62, 0x19, 0xf6, 0x01, 0x01, 0x00, 0x06, 0x00, 0x00, 0x00, 
+0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, 0x64, 0x00, 0x00, 0x00, 
+0x00, 0x3b, 0x02, 0x33, 0x80, 0x00, 0x00, 0x00, 0x8a, 0x2f, 0x71, 0xed, 0xec, 0x64, 0x62, 0x19, 
+0xf6, 0x01, 0x02, 0x75, 0x4f, 0x11, 0x1c, 0x56, 0x9f, 0x4a, 0x9d, 0x6f, 0x98, 0x96, 0x1c, 0x5a, 
+0x9f, 0x0f, 0xb9, 0x24, 0x23, 0x82, 0x7d, 0x86, 0xcf, 0xbc, 0x41, 0x14, 0x38, 0x76, 0x2e, 0x86, 
+0x47, 0x96, 0xef, 0x14, 0x91, 0x2e, 0x30, 0xe2, 0x4b, 0x1c, 0x47, 0x2d, 0x4a, 0xdc, 0xf6, 0x79, 
+0xb6, 0x11, 0x80, 0xcc, 0x51, 0xbb, 0xc4, 0x29, 0x33, 0x60, 0xc1, 0x78, 0x1e, 0x82, 0xe3, 0x40, 
+0xc0, 0xf7, 0x25, 0x06, 0x22, 0x6e, 0x46, 0x11, 0x1a, 0x0b, 0x59, 0xca, 0xaf, 0x12, 0x60, 0x43, 
+0xeb, 0x5b, 0xbf, 0x28, 0xc3, 0x4f, 0x3a, 0x5e, 0x33, 0x2a, 0x1f, 0xc7, 0xb2, 0xb7, 0x3c, 0xf1, 
+0x88, 0x91, 0x0f, 0x00, 0x00, 0x71, 0x00, 0x00, 0x04, 0x00, 0x01, 0x64, 0x62, 0x19, 0xf6, 0x01, 
+0x00, 0x00, 0x06, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x10, 0x00, 
+0x00, 0x00, 0xc8, 0x00, 0x00, 0x00, 0x00, 0xb1, 0x06, 0x9a, 0x80, 0x00, 0x00, 0x00, 0x8a, 0xc8, 
+0x56, 0xb7, 0xb1, 0x64, 0x62, 0x19, 0xf6, 0x01, 0x02, 0x3c, 0x76, 0x7a, 0x28, 0x5e, 0x65, 0x30, 
+0xac, 0x0d, 0x0f, 0x43, 0x31, 0x02, 0x56, 0xcd, 0x14, 0x51, 0x46, 0x69, 0x33, 0xa0, 0x12, 0x61, 
+0x9c, 0x34, 0xc5, 0xd8, 0x9a, 0x0c, 0x81, 0x94, 0xad, 0x5e, 0x98, 0xc4, 0xd0, 0x45, 0x3d, 0x32, 
+0x84, 0xdd, 0xd7, 0x18, 0x2b, 0xdb, 0x13, 0xa8, 0xfc, 0xb2, 0x0d, 0xd6, 0xf6, 0x8a, 0x97, 0xc7, 
+0xe9, 0x7e, 0x27, 0xb7, 0x86, 0x7a, 0x3e, 0xee, 0xfa, 0x06, 0x22, 0x6e, 0x46, 0x11, 0x1a, 0x0b, 
+0x59, 0xca, 0xaf, 0x12, 0x60, 0x43, 0xeb, 0x5b, 0xbf, 0x28, 0xc3, 0x4f, 0x3a, 0x5e, 0x33, 0x2a, 
+0x1f, 0xc7, 0xb2, 0xb7, 0x3c, 0xf1, 0x88, 0x91, 0x0f, 0x00, 0x00, 0x71, 0x00, 0x00, 0x04, 0x00, 
+0x01, 0x64, 0x62, 0x19, 0xf6, 0x01, 0x01, 0x00, 0x06, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 
+0x00, 0x00, 0x00, 0x00, 0x0f, 0x00, 0x00, 0x01, 0x2c, 0x00, 0x00, 0x00, 0x00, 0xb1, 0x06, 0x9a, 
+0x80
+	};		
 
 static void test_edge_probability(void)
 {
@@ -144,11 +500,217 @@ static void test_edge_probability(void)
 	assert(fabs(edge_probability(min,max,X,f)-0.0)< eps);
 }
 
+static void remove_file(char *fname)
+{
+	assert(!remove(fname));
+}
+
+static void test_flow_complete(void)
+{
+	const double eps = 1e-8;
+	
+	const tal_t *this_ctx = tal(tmpctx, tal_t);
+	
+	char *gossfile;
+	int fd = tmpdir_mkstemp(this_ctx,"run-testflow.XXXXXX",&gossfile);
+	tal_add_destructor(gossfile,remove_file);
+	
+	assert(write_all(fd,canned_map,sizeof(canned_map)));
+	struct gossmap *gossmap = gossmap_load(this_ctx,gossfile,NULL);
+	assert(gossmap);
+	
+	// for(struct gossmap_node *node = gossmap_first_node(gossmap);
+	//     node;
+	//     node=gossmap_next_node(gossmap,node))
+	// {
+	// 	struct node_id node_id;
+	// 	gossmap_node_get_id(gossmap,node,&node_id);
+	// 	const char *node_hex = node_id_to_hexstr(this_ctx,&node_id);
+	// 	printf("node: %s\n",node_hex);
+	// 	for(size_t i=0;i<node->num_chans;++i)
+	// 	{
+	// 		int half;
+	// 		const struct gossmap_chan *c = gossmap_nth_chan(gossmap,node,i,&half);
+	// 		if(!gossmap_chan_set(c,half))
+	// 			continue;
+	// 		
+	// 		const struct short_channel_id scid = gossmap_chan_scid(gossmap,c);
+	// 		
+	// 		const char *scid_str = short_channel_id_to_str(this_ctx,&scid);
+	// 		
+	// 		printf("    scid: %s, half: %d\n",scid_str,half);
+	// 	}
+	// }
+	
+	struct node_id l1, l2, l3, l4, l5;
+	assert(node_id_from_hexstr("024f9da0d726adf0d9a4a3ac32e328b93ad527ccb9db7077c57a12c6f9fa9add74", 66, &l1));
+	assert(node_id_from_hexstr("037f97af8e4fa67c9b8d6970849536f888c304316d015af5129e268e0164961b5e", 66, &l2));
+	assert(node_id_from_hexstr("02451e9baff81faf5f410b0bbe9e36ee83bf8f79698e6605a51a9748bc73c7dc28", 66, &l3));
+	assert(node_id_from_hexstr("02d1ab24fe53cfcac4a477745235c7ac3e75161a5bf8426ea8e0182ad58cab367f", 66, &l4));
+	assert(node_id_from_hexstr("03caec54085508da06f14ffb836166ca22042d0fda69690356232a24b8366c8f85", 66, &l5));
+	
+	struct short_channel_id scid12, scid23, scid34,scid45;
+	assert(short_channel_id_from_str("113x1x1", 7, &scid12));
+	assert(short_channel_id_from_str("113x3x1", 7, &scid23));
+	assert(short_channel_id_from_str("113x2x0", 7, &scid34));
+	assert(short_channel_id_from_str("113x4x1", 7, &scid45));
+	
+	struct chan_extra_map *chan_extra_map = tal(this_ctx, struct chan_extra_map);
+	chan_extra_map_init(chan_extra_map);
+	
+	struct chan_extra_half *h0,*h1;
+	struct gossmap_chan *c;
+	struct amount_sat cap;
+	struct amount_msat sum_min1_max0,sum_min0_max1;
+	
+	// check the bounds channel 1--2
+	c = gossmap_find_chan(gossmap,&scid12);
+	assert(gossmap_chan_get_capacity(gossmap,c,&cap));
+	
+	h0 = get_chan_extra_half_by_chan_verify(gossmap,chan_extra_map,c,0);
+	h0->known_min = AMOUNT_MSAT(0);
+	h0->known_max = AMOUNT_MSAT(500000000);
+	
+	h1 = get_chan_extra_half_by_chan_verify(gossmap,chan_extra_map,c,1);
+	h1->known_min = AMOUNT_MSAT(500000000);
+	h1->known_max = AMOUNT_MSAT(1000000000);
+	
+	assert(amount_msat_less_eq(h0->known_min,h0->known_max));
+	assert(amount_msat_less_eq_sat(h0->known_max,cap));
+	assert(amount_msat_less_eq(h1->known_min,h1->known_max));
+	assert(amount_msat_less_eq_sat(h1->known_max,cap));
+	assert(amount_msat_add(&sum_min1_max0,h1->known_min,h0->known_max));
+	assert(amount_msat_add(&sum_min0_max1,h0->known_min,h1->known_max));
+	assert(amount_msat_eq_sat(sum_min1_max0,cap));
+	assert(amount_msat_eq_sat(sum_min0_max1,cap));
+	
+	// check the bounds channel 2--3
+	c = gossmap_find_chan(gossmap,&scid23);
+	assert(gossmap_chan_get_capacity(gossmap,c,&cap));
+	
+	h1 = get_chan_extra_half_by_chan_verify(gossmap,chan_extra_map,c,1);
+	h1->known_min = AMOUNT_MSAT(0);
+	h1->known_max = AMOUNT_MSAT(2000000000);
+	
+	h0 = get_chan_extra_half_by_chan_verify(gossmap,chan_extra_map,c,0);
+	h0->known_min = AMOUNT_MSAT(0);
+	h0->known_max = AMOUNT_MSAT(2000000000);
+	
+	assert(amount_msat_less_eq(h0->known_min,h0->known_max));
+	assert(amount_msat_less_eq_sat(h0->known_max,cap));
+	assert(amount_msat_less_eq(h1->known_min,h1->known_max));
+	assert(amount_msat_less_eq_sat(h1->known_max,cap));
+	assert(amount_msat_add(&sum_min1_max0,h1->known_min,h0->known_max));
+	assert(amount_msat_add(&sum_min0_max1,h0->known_min,h1->known_max));
+	assert(amount_msat_eq_sat(sum_min1_max0,cap));
+	assert(amount_msat_eq_sat(sum_min0_max1,cap));
+	
+	// check the bounds channel 3--4
+	c = gossmap_find_chan(gossmap,&scid34);
+	assert(gossmap_chan_get_capacity(gossmap,c,&cap));
+	
+	h0 = get_chan_extra_half_by_chan_verify(gossmap,chan_extra_map,c,0);
+	h0->known_min = AMOUNT_MSAT(500000000);
+	h0->known_max = AMOUNT_MSAT(1000000000);
+	
+	h1 = get_chan_extra_half_by_chan_verify(gossmap,chan_extra_map,c,1);
+	h1->known_min = AMOUNT_MSAT(0);
+	h1->known_max = AMOUNT_MSAT(500000000);
+	
+	assert(amount_msat_less_eq(h0->known_min,h0->known_max));
+	assert(amount_msat_less_eq_sat(h0->known_max,cap));
+	assert(amount_msat_less_eq(h1->known_min,h1->known_max));
+	assert(amount_msat_less_eq_sat(h1->known_max,cap));
+	assert(amount_msat_add(&sum_min1_max0,h1->known_min,h0->known_max));
+	assert(amount_msat_add(&sum_min0_max1,h0->known_min,h1->known_max));
+	assert(amount_msat_eq_sat(sum_min1_max0,cap));
+	assert(amount_msat_eq_sat(sum_min0_max1,cap));
+	
+	// check the bounds channel 4--5
+	c = gossmap_find_chan(gossmap,&scid45);
+	assert(gossmap_chan_get_capacity(gossmap,c,&cap));
+	
+	h0 = get_chan_extra_half_by_chan_verify(gossmap,chan_extra_map,c,0);
+	h0->known_min = AMOUNT_MSAT(1000000000);
+	h0->known_max = AMOUNT_MSAT(2000000000);
+	
+	h1 = get_chan_extra_half_by_chan_verify(gossmap,chan_extra_map,c,1);
+	h1->known_min = AMOUNT_MSAT(1000000000);
+	h1->known_max = AMOUNT_MSAT(2000000000);
+	
+	assert(amount_msat_less_eq(h0->known_min,h0->known_max));
+	assert(amount_msat_less_eq_sat(h0->known_max,cap));
+	assert(amount_msat_less_eq(h1->known_min,h1->known_max));
+	assert(amount_msat_less_eq_sat(h1->known_max,cap));
+	assert(amount_msat_add(&sum_min1_max0,h1->known_min,h0->known_max));
+	assert(amount_msat_add(&sum_min0_max1,h0->known_min,h1->known_max));
+	assert(amount_msat_eq_sat(sum_min1_max0,cap));
+	assert(amount_msat_eq_sat(sum_min0_max1,cap));
+	
+	struct flow *F = tal(this_ctx,struct flow);
+	struct amount_msat deliver;
+	
+	// flow 1->2
+	F->path = tal_arr(F,struct gossmap_chan const *,1);
+	F->dirs = tal_arr(F,int,1);
+	F->path[0]=gossmap_find_chan(gossmap,&scid12);
+	F->dirs[0]=0;
+	deliver = AMOUNT_MSAT(250000000);
+	flow_complete(F,gossmap,chan_extra_map,deliver);
+	assert(amount_msat_eq(F->amounts[0],deliver));
+	assert(fabs(F->success_prob - 0.5)<eps);
+	
+	// flow 3->4->5
+	F->path = tal_arr(F,struct gossmap_chan const *,2);
+	F->dirs = tal_arr(F,int,2);
+	F->path[0]=gossmap_find_chan(gossmap,&scid34);
+	F->path[1]=gossmap_find_chan(gossmap,&scid45);
+	F->dirs[0]=0;
+	F->dirs[1]=0;
+	deliver = AMOUNT_MSAT(250000000);
+	flow_complete(F,gossmap,chan_extra_map,deliver);
+	assert(amount_msat_eq(F->amounts[0],amount_msat(250050016)));
+	assert(fabs(F->success_prob - 1.)<eps);
+	
+	// flow 2->3->4->5
+	F->path = tal_arr(F,struct gossmap_chan const *,3);
+	F->dirs = tal_arr(F,int,3);
+	F->path[0]=gossmap_find_chan(gossmap,&scid23);
+	F->path[1]=gossmap_find_chan(gossmap,&scid34);
+	F->path[2]=gossmap_find_chan(gossmap,&scid45);
+	F->dirs[0]=1;
+	F->dirs[1]=0;
+	F->dirs[2]=0;
+	deliver = AMOUNT_MSAT(250000000);
+	flow_complete(F,gossmap,chan_extra_map,deliver);
+	assert(amount_msat_eq(F->amounts[0],amount_msat(250087534)));
+	assert(fabs(F->success_prob - 1. + 250.087534/2000)<eps);
+	
+	// flow 1->2->3->4->5
+	F->path = tal_arr(F,struct gossmap_chan const *,4);
+	F->dirs = tal_arr(F,int,4);
+	F->path[0]=gossmap_find_chan(gossmap,&scid12);
+	F->path[1]=gossmap_find_chan(gossmap,&scid23);
+	F->path[2]=gossmap_find_chan(gossmap,&scid34);
+	F->path[3]=gossmap_find_chan(gossmap,&scid45);
+	F->dirs[0]=0;
+	F->dirs[1]=1;
+	F->dirs[2]=0;
+	F->dirs[3]=0;
+	deliver = AMOUNT_MSAT(250000000);
+	flow_complete(F,gossmap,chan_extra_map,deliver);
+	assert(amount_msat_eq(F->amounts[0],amount_msat(250112544)));
+	assert(fabs(F->success_prob - 0.43728117)<eps);
+	
+	tal_free(this_ctx);
+}
+
 int main(int argc, char *argv[])
 {
 	common_setup(argv[0]);
 	
 	test_edge_probability();
+	test_flow_complete();
 	
 	common_shutdown();
 }

From d927193b69c2094a450132be2250f00b61323c79 Mon Sep 17 00:00:00 2001
From: Lagrang3 <eduardo.quintana@pm.me>
Date: Wed, 17 May 2023 08:32:58 +0100
Subject: [PATCH 30/64] renepay: trying to get pay work

---
 plugins/renepay/pay.c      | 131 ++++++++++++++++++++++++-------------
 plugins/renepay/pay.h      |  50 +++++++++++++-
 plugins/renepay/pay_flow.c |  87 +++++++++++++-----------
 3 files changed, 184 insertions(+), 84 deletions(-)

diff --git a/plugins/renepay/pay.c b/plugins/renepay/pay.c
index 9ebd0859f15d..c9be00748230 100644
--- a/plugins/renepay/pay.c
+++ b/plugins/renepay/pay.c
@@ -61,7 +61,7 @@ void amount_msat_reduce_(struct amount_msat *dst,
 static void memleak_mark(struct plugin *p, struct htable *memtable)
 {
 	memleak_scan_obj(memtable, pay_plugin);
-	memleak_scan_htable(memtable, &pay_plugin->chan_extra_map.raw);
+	memleak_scan_htable(memtable, &pay_plugin->chan_extra_map->raw);
 }
 #endif
 
@@ -83,7 +83,9 @@ static const char *init(struct plugin *p,
 		 JSON_SCAN(json_to_bool, &pay_plugin->exp_offers));
 
 	list_head_init(&pay_plugin->payments);
-	chan_extra_map_init(&pay_plugin->chan_extra_map);
+	
+	pay_plugin->chan_extra_map = tal(pay_plugin,struct chan_extra_map);
+	chan_extra_map_init(pay_plugin->chan_extra_map);
 
 	// TODO(eduardo) is it ok to use NULL or pay_plugin as `ctx`?
 	pay_plugin->gossmap = gossmap_load(pay_plugin,
@@ -116,12 +118,11 @@ static void chan_update_capacity(struct payment *p,
 	/* This is assumed */
 	if (min_capacity && max_capacity)
 		assert(amount_msat_greater_eq(*max_capacity, *min_capacity));
-
-	h = get_chan_extra_half_by_scid(&pay_plugin->chan_extra_map, scid, dir);
+	
+	h = get_chan_extra_half_by_scid(pay_plugin->chan_extra_map, scid, dir);
 	if (!h)
-		h = new_chan_extra_half(&pay_plugin->chan_extra_map, scid, dir,
+		h = new_chan_extra_half(pay_plugin->chan_extra_map, scid, dir,
 					*max_capacity);
-
 	if (min_capacity && amount_msat_greater(*min_capacity, h->known_min))
 		h->known_min = *min_capacity;
 	if (max_capacity && amount_msat_less(*max_capacity, h->known_max))
@@ -205,11 +206,14 @@ static void add_routehints(struct payment *p,
 				      r[j].fee_base_msat,
 				      r[j].fee_proportional_millionths,
 				      p->maxspend);
-			dest = &r[j].pubkey;
+			// TODO(eduardo): I think should be `end`
+			end = &r[j].pubkey;
+			// dest = &r[j].pubkey;
 		}
 	}
 }
 
+// TODO(eduardo): check this
 /* listpeers gives us the certainty on local channels' capacity.  Of course,
  * this is racy and transient, but better than nothing! */
 static bool update_capacities_from_listpeers(struct plugin *plugin,
@@ -811,6 +815,7 @@ sendpay_flows(struct command *cmd,
 	return command_still_pending(cmd);
 }
 
+// TODO(eduardo): check this
 static struct command_result *try_paying(struct command *cmd,
 					 struct payment *p,
 					 bool first_time)
@@ -826,7 +831,7 @@ static struct command_result *try_paying(struct command *cmd,
 		abort();
 
 	/* Fees spent so far */
-	if (amount_msat_sub(&fees_spent, p->total_sent, p->total_delivering))
+	if (!amount_msat_sub(&fees_spent, p->total_sent, p->total_delivering))
 		abort();
 
 	/* Remaining fee budget. */
@@ -915,18 +920,23 @@ static struct command_result *json_pay(struct command *cmd,
 				       const char *buf,
 				       const jsmntok_t *params)
 {
-	struct payment *p;
-	unsigned int *retryfor;
+ 	struct payment *p;
+	
+ 	u64 invexpiry;
+ 	struct amount_msat *msat, *invmsat;
+	struct amount_msat *maxfee;
 	u64 *riskfactor_millionths;
-	struct amount_msat *invmsat, *msat, *maxfee;
-	u64 invexpiry;
-	u32 *maxdelay;
-	struct out_req *req;
+ 	u32 *maxdelay;
+	u64 *base_fee_penalty, *prob_cost_factor;
+	u64 *min_prob_success_millionths;
+	unsigned int *retryfor;
+
 #if DEVELOPER
 	bool *use_shadow;
 #endif
 
-	p = tal(cmd, struct payment);
+ 	p = tal(cmd, struct payment);
+
 	p->cmd = cmd;
 	p->paynotes = tal_arr(p, const char *, 0);
 	p->disabled = tal_arr(p, struct short_channel_id, 0);
@@ -935,20 +945,33 @@ static struct command_result *json_pay(struct command *cmd,
 	p->total_sent = AMOUNT_MSAT(0);
 	p->total_delivering = AMOUNT_MSAT(0);
 	p->rexmit_timer = NULL;
-
+ 	p->local_gossmods = gossmap_localmods_new(p);
+	
 	if (!param(cmd, buf, params,
 		   p_req("invstring", param_string, &p->invstr),
-		   p_opt("amount_msat", param_msat, &msat),
-		   p_opt("label", param_string, &p->label),
+ 		   p_opt("amount_msat", param_msat, &msat),
+ 		   p_opt("maxfee", param_msat, &maxfee),
+ 		   
+		   // MCF parameters
+		   p_opt_def("base_fee_penalty", param_millionths, &base_fee_penalty,10),
+ 		   p_opt_def("prob_cost_factor", param_millionths, &prob_cost_factor,10),
+ 		   
+		   // TODO(eduardo): probability of success as a ppm parameter
+		   // or a real number?
+		   p_opt_def("min_prob_success", param_millionths, 
+		   	&min_prob_success_millionths,100000), // default is 10%
+	
 		   p_opt_def("riskfactor", param_millionths,
 			     &riskfactor_millionths, 1.0),
-		   p_opt_def("retry_for", param_number, &retryfor, 60),
+
 		   p_opt_def("maxdelay", param_number, &maxdelay,
 			     /* We're initially called to probe usage, before init! */
 			     pay_plugin ? pay_plugin->maxdelay_default : 0),
-		   p_opt("localofferid", param_sha256, &p->local_offer_id),
-		   p_opt("maxfee", param_msat, &maxfee),
-		   p_opt("description", param_string, &p->description),
+
+ 		   p_opt_def("retry_for", param_number, &retryfor, 60),
+ 		   p_opt("localofferid", param_sha256, &p->local_offer_id),
+ 		   p_opt("description", param_string, &p->description),
+ 		   p_opt("label", param_string, &p->label),
 #if DEVELOPER
 		   p_opt_def("use_shadow", param_bool, &use_shadow, true),
 #endif
@@ -957,15 +980,29 @@ static struct command_result *json_pay(struct command *cmd,
 
 #if DEVELOPER
 	p->use_shadow = *use_shadow;
+	tal_free(use_shadow);
 #else
 	p->use_shadow = true;
 #endif
-
-	p->local_gossmods = gossmap_localmods_new(p);
-	p->delay_feefactor = *riskfactor_millionths / 1000000.0;
+ 	
+	plugin_log(pay_plugin->plugin,LOG_DBG,"Starting renepay");
+	
+	p->base_fee_penalty=*base_fee_penalty;
+	p->prob_cost_factor= *prob_cost_factor;
+	p->min_prob_success=*min_prob_success_millionths * 1e-6;
+ 	p->delay_feefactor = *riskfactor_millionths / 1000000.0;
 	p->maxdelay = *maxdelay;
+	
+	tal_free(base_fee_penalty);
+	tal_free(prob_cost_factor);
+	tal_free(min_prob_success_millionths);
+	tal_free(riskfactor_millionths);
 	tal_free(maxdelay);
 
+	p->start_time = time_now();
+	p->stop_time = timeabs_add(p->start_time, time_from_sec(*retryfor));
+	tal_free(retryfor);
+	
 	if (!bolt12_has_prefix(p->invstr)) {
 		struct bolt11 *b11;
 		char *fail;
@@ -988,7 +1025,9 @@ static struct command_result *json_pay(struct command *cmd,
 			p->payment_metadata = tal_dup_talarr(p, u8, b11->metadata);
 		else
 			p->payment_metadata = NULL;
+			
 		add_routehints(p, b11->routes, &p->dest);
+		
 		p->final_cltv = b11->min_final_cltv_expiry;
 		/* Sanity check */
 		if (feature_offered(b11->features, OPT_VAR_ONION) &&
@@ -1016,6 +1055,7 @@ static struct command_result *json_pay(struct command *cmd,
 						    "bolt11 uses description_hash, but you did not provide description parameter");
 		}
 	} else {
+		// TODO(eduardo): check this, compare with `pay`
 		const struct tlv_invoice *b12;
 		char *fail;
 		b12 = invoice_decode(tmpctx, p->invstr, strlen(p->invstr),
@@ -1078,11 +1118,20 @@ static struct command_result *json_pay(struct command *cmd,
 		else
 			invexpiry = *b12->invoice_created_at + BOLT12_DEFAULT_REL_EXPIRY;
 	}
+	
+	if (node_id_eq(&pay_plugin->my_id, &p->dest))
+		return command_fail(cmd, JSONRPC2_INVALID_PARAMS,
+				    "This payment is destined for ourselves. "
+				    "Self-payments are not supported");
 
-	if (time_now().ts.tv_sec > invexpiry)
-		return command_fail(cmd, PAY_INVOICE_EXPIRED, "Invoice expired");
-
+	/* It's now owned by the global plugin */
+	list_add_tail(&pay_plugin->payments, &p->list);
+	tal_add_destructor(p, destroy_payment);
+	tal_steal(pay_plugin, p);
+	
+	// set the payment amount
 	if (invmsat) {
+		// amount is written in the invoice
 		if (msat) {
 			return command_fail(cmd, JSONRPC2_INVALID_PARAMS,
 					    "amount_msat parameter unnecessary");
@@ -1090,22 +1139,19 @@ static struct command_result *json_pay(struct command *cmd,
 		p->amount = *invmsat;
 		tal_free(invmsat);
 	} else {
+		// amount is not written in the invoice
 		if (!msat) {
 			return command_fail(cmd, JSONRPC2_INVALID_PARAMS,
 					    "amount_msat parameter required");
 		}
 		p->amount = *msat;
+		tal_free(msat);
 	}
-
-	if (node_id_eq(&pay_plugin->my_id, &p->dest))
-		return command_fail(cmd, JSONRPC2_INVALID_PARAMS,
-				    "This payment is destined for ourselves. "
-				    "Self-payments are not supported");
-
+	
 	/* Default max fee is 5 sats, or 0.5%, whichever is *higher* */
 	if (!maxfee) {
 		struct amount_msat fee = amount_msat_div(p->amount, 200);
-		if (amount_msat_less(p->amount, AMOUNT_MSAT(5000)))
+		if (amount_msat_less(fee, AMOUNT_MSAT(5000)))
 			fee = AMOUNT_MSAT(5000);
 		maxfee = tal_dup(tmpctx, struct amount_msat, &fee);
 	}
@@ -1116,15 +1162,12 @@ static struct command_result *json_pay(struct command *cmd,
 			"Overflow when computing fee budget, fee far too high.");
 	}
 	tal_free(maxfee);
-
-	p->start_time = time_now();
-	p->stop_time = timeabs_add(p->start_time, time_from_sec(*retryfor));
-	tal_free(retryfor);
-
-	/* It's now owned by the global plugin */
-	list_add_tail(&pay_plugin->payments, &p->list);
-	tal_add_destructor(p, destroy_payment);
-	tal_steal(pay_plugin, p);
+	
+	
+	if (time_now().ts.tv_sec > invexpiry)
+		return command_fail(cmd, PAY_INVOICE_EXPIRED, "Invoice expired");
+	
+	struct out_req *req;
 
 	/* Get local capacities... */
 	req = jsonrpc_request_start(cmd->plugin, cmd, "listpeers",
diff --git a/plugins/renepay/pay.h b/plugins/renepay/pay.h
index 803c6f0154fd..8498e9ec2cd6 100644
--- a/plugins/renepay/pay.h
+++ b/plugins/renepay/pay.h
@@ -26,7 +26,7 @@ struct pay_plugin {
 	struct list_head payments;
 
 	/* Per-channel metadata: some persists between payments */
-	struct chan_extra_map chan_extra_map;
+	struct chan_extra_map *chan_extra_map;
 };
 
 /* Set in init */
@@ -34,64 +34,112 @@ extern struct pay_plugin *pay_plugin;
 
 struct payment {
 	/* Inside pay_plugin->payments list */
+	// TODO(eduardo): is it used?
 	struct list_node list;
 
 	/* The command, and our owner (needed for timer func) */
+	// TODO(eduardo): is it used?
 	struct command *cmd;
 
 	/* We promised this in pay() output */
+	// TODO(eduardo): is it used?
 	struct timeabs start_time;
 
 	/* Localmods to apply to gossip_map for our own use. */
+	// TODO(eduardo): is it used?
 	struct gossmap_localmods *local_gossmods;
 
 	/* invstring (bolt11 or bolt12) */
+	// TODO(eduardo): is it used?
 	const char *invstr;
 
 	/* How much, what, where */
+	// TODO(eduardo): is it used?
 	struct node_id dest;
+	// TODO(eduardo): is it used?
 	struct sha256 payment_hash;
+	// TODO(eduardo): is it used?
 	struct amount_msat amount;
+	
+	// TODO(eduardo): is it used?
 	u32 final_cltv;
 
 	/* Total sent, including fees. */
+	// TODO(eduardo): is it used?
 	struct amount_msat total_sent;
 	/* Total that is delivering (i.e. without fees) */
+	// TODO(eduardo): is it used?
 	struct amount_msat total_delivering;
 
 	/* payment_secret, if specified by invoice. */
+	// TODO(eduardo): is it used?
 	struct secret *payment_secret;
 	/* Payment metadata, if specified by invoice. */
+	// TODO(eduardo): is it used?
 	const u8 *payment_metadata;
 
 	/* Description and labels, if any. */
+	// TODO(eduardo): is it used?
+	// TODO(eduardo): could be NULL
 	const char *description, *label;
 
 	/* Penalty for CLTV delays */
+	// TODO(eduardo): is it used?
 	double delay_feefactor;
+	
+	/* Penalty for base fee */
+	// TODO(eduardo): is it used?
+	double base_fee_penalty;
+	
+	/* linear fee cost = 
+	 * 	millionths 
+	 * 	+ base_fee* base_fee_penalty
+	 * 	+delay*delay_feefactor;*/
 
 	/* Limits on what routes we'll accept. */
+	// TODO(eduardo): is it used?
 	struct amount_msat maxspend;
+	// TODO(eduardo): is it used?
 	unsigned int maxdelay;
+	// TODO(eduardo): is it used?
 	struct timeabs stop_time;
+	
+	/* The minimum acceptable prob. of success */
+	// TODO(eduardo): is it used?
+	double min_prob_success;
+	
+	/* linear prob. cost = 
+	 * 	- prob_cost_factor * log prob. */
+	
+	/* Conversion from prob. cost to millionths */
+	// TODO(eduardo): is it used?
+	double prob_cost_factor;
 
 	/* Channels we decided to disable for various reasons. */
+	// TODO(eduardo): is it used?
 	struct short_channel_id *disabled;
 
 	/* Chatty description of attempts. */
+	// TODO(eduardo): is it used?
 	const char **paynotes;
 
 	/* Groupid, so listpays() can group them back together */
+	// TODO(eduardo): is it used?
 	u64 groupid;
+	// TODO(eduardo): is it used?
 	u64 next_partid;
 	/* If this is paying a local offer, this is the one (sendpay ensures we
 	 * don't pay twice for single-use offers) */
+	// TODO(eduardo): is it used?
+	// TODO(eduardo): could be NULL
 	struct sha256 *local_offer_id;
 
 	/* Timers. */
+	// TODO(eduardo): is it used?
 	struct plugin_timer *rexmit_timer;
 
 	/* DEVELOPER allows disabling shadow route */
+	// TODO(eduardo): is it used?
 	bool use_shadow;
 };
 
diff --git a/plugins/renepay/pay_flow.c b/plugins/renepay/pay_flow.c
index 22fc42b5df7d..5b21ca568d98 100644
--- a/plugins/renepay/pay_flow.c
+++ b/plugins/renepay/pay_flow.c
@@ -326,14 +326,14 @@ static bool disable_htlc_violations(struct payment *p,
 	return disabled_some;
 }
 
-/* We're not using these flows after all: clean up chan_extra_map */
-static void remove_flows(const struct gossmap *gossmap,
-			 struct chan_extra_map *chan_extra_map,
-			 struct flow **flows)
-{
-	for (size_t i = 0; i < tal_count(flows); i++)
-		remove_completed_flow(gossmap, chan_extra_map, flows[i]);
-}
+// /* We're not using these flows after all: clean up chan_extra_map */
+// static void remove_flows(const struct gossmap *gossmap,
+// 			 struct chan_extra_map *chan_extra_map,
+// 			 struct flow **flows)
+// {
+// 	for (size_t i = 0; i < tal_count(flows); i++)
+// 		remove_completed_flow(gossmap, chan_extra_map, flows[i]);
+// }
 
 // TODO(eduardo): check this
 /* Get some payment flows to get this amount to destination, or NULL. */
@@ -348,12 +348,21 @@ struct pay_flow **get_payflows(struct payment *p,
 	bitmap *disabled;
 	struct pay_flow **pay_flows;
 	const struct gossmap_node *src, *dst;
-
-	if (!gossmap_refresh(pay_plugin->gossmap, NULL))
+	
+	// TODO(eduardo): is it necessary to check for return value?
+	gossmap_refresh(pay_plugin->gossmap, NULL);
+	
+	if (pay_plugin->gossmap == NULL)
 		plugin_err(pay_plugin->plugin, "Failed to refresh gossmap: %s",
 			   strerror(errno));
-
+	
+	// TODO(eduardo): remember that local channels have
+	// know_min=know_max=liquidity. Is this related to local_gossmods?
+	// 
+	// TODO(eduardo): where is p->local_gossmods set?
 	gossmap_apply_localmods(pay_plugin->gossmap, p->local_gossmods);
+	
+	// TODO(eduardo): where is p->disabled set?
 	disabled = make_disabled_bitmap(tmpctx, pay_plugin->gossmap, p->disabled);
 	src = gossmap_find_node(pay_plugin->gossmap, &pay_plugin->my_id);
 	if (!src) {
@@ -374,19 +383,16 @@ struct pay_flow **get_payflows(struct payment *p,
 		bool too_unlikely, too_expensive, too_delayed;
 		const u32 *final_cltvs;
 		
-		struct amount_msat max_fee 
-			= (struct amount_msat){.millisatoshis = amount.millisatoshis/100};
-		
 		/* Note!  This actually puts flows in chan_extra_map, so
 		 * flows must be removed if not used! */
 		flows = minflow(tmpctx, pay_plugin->gossmap, src, dst,
-				&pay_plugin->chan_extra_map, disabled,
+				pay_plugin->chan_extra_map, disabled,
 				amount, 
-				max_fee ,
-				/* min probability = */ 0.1 ,
+				feebudget,
+				p->min_prob_success ,
 				p->delay_feefactor,
-				/* base_fee_penalty = */ 1,
-				/* prob_cost_factor = */ 10);
+				p->base_fee_penalty,
+				p->prob_cost_factor);
 		if (!flows) {
 			paynote(p, "Failed to find any paths for %s",
 				type_to_string(tmpctx,
@@ -396,17 +402,15 @@ struct pay_flow **get_payflows(struct payment *p,
 		}
 
 		/* Are we unhappy? */
-		prob = flow_set_probability(flows,pay_plugin->gossmap,&pay_plugin->chan_extra_map);
+		prob = flow_set_probability(flows,pay_plugin->gossmap,pay_plugin->chan_extra_map);
 		fee = flow_set_fee(flows);
 		delay = flows_worst_delay(flows) + p->final_cltv;
 
 		too_unlikely = (prob < 0.01);
-		if (too_unlikely)
+		if (too_unlikely && !unlikely_ok)
 		{
 			paynote(p, "Flows too unlikely, P() = %f%%", prob * 100);
-			
-			if(!unlikely_ok)
-				goto fail_path;
+			goto fail_path;
 		}
 		too_expensive = amount_msat_greater(fee, feebudget);
 		if (too_expensive)
@@ -420,17 +424,19 @@ struct pay_flow **get_payflows(struct payment *p,
 		if (too_delayed) {
 			paynote(p, "Flows too delayed, delay = %"PRIu64" (max %u)",
 				delay, p->maxdelay);
-			/* FIXME: What is a sane limit? */
-			if (p->delay_feefactor > 1000) {
-				paynote(p, "Giving up!");
-				goto fail_path;
-			}
-
-			p->delay_feefactor *= 2;
-			paynote(p, "Doubling delay_feefactor to %f",
-				p->delay_feefactor);
+			// /* FIXME: What is a sane limit? */
+			// if (p->delay_feefactor > 1000) {
+			// 	paynote(p, "Giving up!");
+			// 	goto fail_path;
+			// }
+
+			// p->delay_feefactor *= 2;
+			// paynote(p, "Doubling delay_feefactor to %f",
+			// 	p->delay_feefactor);
+			// 
+			// goto retry;
 			
-			goto retry;
+			goto fail_path;
 		}
 
 	// seems_ok:
@@ -442,7 +448,14 @@ struct pay_flow **get_payflows(struct payment *p,
 		if (disable_htlc_violations(p, flows, pay_plugin->gossmap,
 					    disabled))
 			goto retry;
-
+		
+		/* Commit the flows to the chan_extra_map, 
+		 * update the htlc_total and num_htlcs. */
+		commit_flow_set(pay_plugin->gossmap,
+				pay_plugin->chan_extra_map,
+				flows);
+		
+		
 		/* This can adjust amounts and final cltv for each flow,
 		 * to make it look like it's going elsewhere */
 		final_cltvs = shadow_additions(tmpctx, pay_plugin->gossmap,
@@ -456,12 +469,8 @@ struct pay_flow **get_payflows(struct payment *p,
 		break;
 
 	retry:
-		remove_flows(pay_plugin->gossmap, &pay_plugin->chan_extra_map,
-			     flows);
 		continue;
 	fail_path:
-		remove_flows(pay_plugin->gossmap, &pay_plugin->chan_extra_map,
-			     flows);
 		goto fail;
 	}
 

From b9b62008df29ee761e84cd0cb9a734949ea3acd1 Mon Sep 17 00:00:00 2001
From: Lagrang3 <eduardo.quintana@pm.me>
Date: Wed, 17 May 2023 11:20:45 +0100
Subject: [PATCH 31/64] renepay: use listpeerchannels instead of listpeers

---
 plugins/renepay/pay.c | 162 ++++++++++++++++++++++--------------------
 1 file changed, 85 insertions(+), 77 deletions(-)

diff --git a/plugins/renepay/pay.c b/plugins/renepay/pay.c
index c9be00748230..4f16119e4bf8 100644
--- a/plugins/renepay/pay.c
+++ b/plugins/renepay/pay.c
@@ -106,13 +106,26 @@ static const char *init(struct plugin *p,
 	return NULL;
 }
 
+// TODO(eduardo): check this
+// TODO(eduardo): remember that if [a,b] is fixed on (c,dir) then [cap-b,cap-a]
+// is fixed on (c,!dir)
+// TODO(eduardo): I think this function does too many things.  I would prefer to
+// have finer grained control; like a function `chan_update_knowledge` that
+// improves the knowledge and also checks the bounds. But we also need to have
+// the possiblity to relax the knowledge in another function.
 /* We know something about this channel!  Update it! */
-static void chan_update_capacity(struct payment *p,
+static void chan_update_knowledge(struct payment *p,
 				 struct short_channel_id scid,
 				 int dir,
 				 const struct amount_msat *min_capacity,
 				 const struct amount_msat *max_capacity)
 {
+	plugin_log(pay_plugin->plugin,LOG_DBG,"Updating channel: %s [%s, %s]",
+		type_to_string(tmpctx,struct short_channel_id, &scid),
+		min_capacity ? type_to_string(tmpctx,struct amount_msat, min_capacity) : "-",
+		max_capacity ? type_to_string(tmpctx,struct amount_msat, max_capacity) : "-");
+	
+	
 	struct chan_extra_half *h;
 
 	/* This is assumed */
@@ -187,7 +200,10 @@ static void add_localchan(struct payment *p,
 				 dir);
 	/* We know (assume!) something about this channel: that it has at
 	 * sufficient capacity. */
-	chan_update_capacity(p, scid, dir, &p->maxspend, &p->maxspend);
+	plugin_log(pay_plugin->plugin,LOG_DBG,"add_localchan: %s [maxspend %s]",
+		type_to_string(tmpctx,struct short_channel_id, &scid),
+		type_to_string(tmpctx,struct amount_msat, &p->maxspend));
+	chan_update_knowledge(p, scid, dir, &p->maxspend, &p->maxspend);
 }
 
 /* Add routehints provided by bolt11 */
@@ -213,90 +229,80 @@ static void add_routehints(struct payment *p,
 	}
 }
 
-// TODO(eduardo): check this
-/* listpeers gives us the certainty on local channels' capacity.  Of course,
+/* listpeerchannels gives us the certainty on local channels' capacity.  Of course,
  * this is racy and transient, but better than nothing! */
-static bool update_capacities_from_listpeers(struct plugin *plugin,
-					     struct payment *p,
-					     const char *buf,
-					     const jsmntok_t *toks)
+static bool update_capacities_from_listpeerchannels(
+		struct plugin *plugin,
+		struct payment *p,
+		const char *buf,
+		const jsmntok_t *toks)
 {
-	const jsmntok_t *peers, *peer;
+	const jsmntok_t *channels, *channel;
 	size_t i;
 
 	if (json_get_member(buf, toks, "error"))
 		goto malformed;
 
-	peers = json_get_member(buf, toks, "peers");
-	if (!peers)
+	channels = json_get_member(buf, toks, "channels");
+	if (!channels)
 		goto malformed;
 
-	json_for_each_arr(i, peer, peers) {
-		const jsmntok_t *channel, *channels;
-		size_t j;
+	json_for_each_arr(i, channel, channels) {
+		struct short_channel_id scid;
+		const jsmntok_t *scidtok = json_get_member(buf, channel, "short_channel_id");
+		/* If channel is still opening, this won't be there. 
+		 * Also it won't be in the gossmap, so there is 
+		 * no need to mark it as disabled. */
+		if (!scidtok)
+			continue;
+		if (!json_to_short_channel_id(buf, scidtok, &scid))
+			goto malformed;
+	
 		bool connected;
-
-		channels = json_get_member(buf, peer, "channels");
-		if (!channels)
+		if(!json_to_bool(buf,
+				 json_get_member(buf,channel,"peer_connected"),
+				 &connected))
+			goto malformed;
+	
+		if (!connected) {
+			paynote(p, "local channel %s disabled:"
+				" peer disconnected",
+				type_to_string(tmpctx,
+					       struct short_channel_id,
+					       &scid));
+			tal_arr_expand(&p->disabled, scid);
 			continue;
-
-		if (!json_to_bool(buf,
-				  json_get_member(buf, peer, "connected"),
-				  &connected))
+		}
+		
+		const jsmntok_t *spendabletok, *dirtok,*statetok;
+		struct amount_msat spendable;
+		int dir;
+		
+		spendabletok = json_get_member(buf, channel, "spendable_msat");
+		dirtok = json_get_member(buf, channel, "direction");
+		statetok = json_get_member(buf, channel, "state");
+		
+		if(spendabletok==NULL || dirtok==NULL || statetok==NULL)
 			goto malformed;
-
-		json_for_each_arr(j, channel, channels) {
-			const jsmntok_t *spendabletok, *scidtok, *dirtok, *statetok;
-			struct short_channel_id scid;
-			int dir;
-			struct amount_msat spendable;
-
-			scidtok = json_get_member(buf, channel, "short_channel_id");
-			/* If channel is still opening, this won't be there */
-			if (!scidtok)
-				continue;
-
-			spendabletok = json_get_member(buf, channel, "spendable_msat");
-			dirtok = json_get_member(buf, channel, "direction");
-			/* FIXME: Example max_accepted_htlcs and htlcs */
-			statetok = json_get_member(buf, channel, "state");
-			if (spendabletok == NULL
-			    || dirtok == NULL
-			    || statetok == NULL)
-				goto malformed;
-
-			if (!json_to_short_channel_id(buf, scidtok, &scid))
-				goto malformed;
-			if (!json_to_int(buf, dirtok, &dir))
-				goto malformed;
-			if (!json_to_msat(buf, spendabletok, &spendable))
-				goto malformed;
-
-			/* Don't report opening/closing channels */
-			if (!json_tok_streq(buf, statetok, "CHANNELD_NORMAL")) {
-				tal_arr_expand(&p->disabled, scid);
-				continue;
-			}
-
-			if (!connected) {
-				paynote(p, "local channel %s disabled:"
-					" peer disconnected",
-					type_to_string(tmpctx,
-						       struct short_channel_id,
-						       &scid));
-				tal_arr_expand(&p->disabled, scid);
-				continue;
-			}
-
-			/* We know min and max capacity exactly now! */
-			chan_update_capacity(p, scid, dir, &spendable, &spendable);
+		if (!json_to_msat(buf, spendabletok, &spendable))
+			goto malformed;
+		if (!json_to_int(buf, dirtok,&dir))
+			goto malformed;
+			
+		/* Don't report opening/closing channels */
+		if (!json_tok_streq(buf, statetok, "CHANNELD_NORMAL")) {
+			tal_arr_expand(&p->disabled, scid);
+			continue;
 		}
+		
+		/* We know min and max liquidity exactly now! */
+		chan_update_knowledge(p, scid, dir, &spendable, &spendable);
 	}
 	return true;
 
 malformed:
 	plugin_log(plugin, LOG_BROKEN,
-		   "listpeers malformed: %.*s",
+		   "listpeerchannels malformed: %.*s",
 		   json_tok_full_len(toks),
 		   json_tok_full(buf, toks));
 	return false;
@@ -614,7 +620,7 @@ static struct command_result *waitsendpay_failed(struct command *cmd,
 	/* All these parts succeeded, so we know something about min
 	 * capacity! */
 	for (size_t i = 0; i < erridx; i++)
-		chan_update_capacity(p, flow->path_scids[i], flow->path_dirs[i], &flow->amounts[i],
+		chan_update_knowledge(p, flow->path_scids[i], flow->path_dirs[i], &flow->amounts[i],
 				     NULL);
 
 	switch ((enum onion_wire)onionerr) {
@@ -665,7 +671,7 @@ static struct command_result *waitsendpay_failed(struct command *cmd,
 		paynote(p, "... assuming that max capacity is %s",
 			type_to_string(tmpctx, struct amount_msat,
 				       &max_possible));
-		chan_update_capacity(p, errscid, flow->path_dirs[erridx], NULL, &max_possible);
+		chan_update_knowledge(p, errscid, flow->path_dirs[erridx], NULL, &max_possible);
 		goto done;
 	}
 
@@ -859,12 +865,15 @@ static struct command_result *try_paying(struct command *cmd,
 }
 
 static struct command_result *
-listpeers_done(struct command *cmd, const char *buf,
+listpeerchannels_done(struct command *cmd, const char *buf,
 	       const jsmntok_t *result, struct payment *p)
 {
-	if (!update_capacities_from_listpeers(cmd->plugin, p, buf, result))
+	// FILE *f = fopen("/tmp/afile","wb");
+	// fwrite(json_tok_full(buf,result),json_tok_full_len(result),1,f);
+	// fclose(f);
+	if (!update_capacities_from_listpeerchannels(cmd->plugin, p, buf, result))
 		return command_fail(cmd, LIGHTNINGD,
-				    "listpays malformed: %.*s",
+				    "listpeerchannels malformed: %.*s",
 				    json_tok_full_len(result),
 				    json_tok_full(buf, result));
 
@@ -1163,16 +1172,15 @@ static struct command_result *json_pay(struct command *cmd,
 	}
 	tal_free(maxfee);
 	
-	
 	if (time_now().ts.tv_sec > invexpiry)
 		return command_fail(cmd, PAY_INVOICE_EXPIRED, "Invoice expired");
 	
 	struct out_req *req;
 
 	/* Get local capacities... */
-	req = jsonrpc_request_start(cmd->plugin, cmd, "listpeers",
-				    listpeers_done,
-				    listpeers_done, p);
+	req = jsonrpc_request_start(cmd->plugin, cmd, "listpeerchannels",
+				    listpeerchannels_done,
+				    listpeerchannels_done, p);
 	return send_outreq(cmd->plugin, req);
 }
 

From 8bc35d8f8be859499110c83488f251503dd86aec Mon Sep 17 00:00:00 2001
From: Lagrang3 <eduardo.quintana@pm.me>
Date: Wed, 17 May 2023 11:59:54 +0100
Subject: [PATCH 32/64] renepay: routehints only after maxspend is known

---
 plugins/renepay/pay.c | 42 +++++++++++++++++++++++++++++++-----------
 1 file changed, 31 insertions(+), 11 deletions(-)

diff --git a/plugins/renepay/pay.c b/plugins/renepay/pay.c
index 4f16119e4bf8..f95536673dee 100644
--- a/plugins/renepay/pay.c
+++ b/plugins/renepay/pay.c
@@ -207,14 +207,23 @@ static void add_localchan(struct payment *p,
 }
 
 /* Add routehints provided by bolt11 */
-static void add_routehints(struct payment *p,
-			   struct route_info **routes,
-			   const struct node_id *dest)
+static void add_routehints(struct payment *p)
 {
-	for (size_t i = 0; i < tal_count(routes); i++) {
+	struct bolt11 *b11;
+	char *fail;
+
+	b11 =
+	    bolt11_decode(tmpctx, p->invstr, plugin_feature_set(p->cmd->plugin),
+			  p->description, chainparams, &fail);
+	if (b11 == NULL)
+		plugin_log(pay_plugin->plugin, LOG_BROKEN,
+				    "add_routehints: Invalid bolt11: %s", fail);
+		
+
+	for (size_t i = 0; i < tal_count(b11->routes); i++) {
 		/* Each one, presumably, leads to the destination */
-		const struct route_info *r = routes[i];
-		const struct node_id *end = dest;
+		const struct route_info *r = b11->routes[i];
+		const struct node_id *end = & p->dest;
 		for (int j = tal_count(r)-1; j >= 0; j--) {
 			add_localchan(p, &r[j].pubkey, end,
 				      r[j].cltv_expiry_delta,
@@ -222,9 +231,7 @@ static void add_routehints(struct payment *p,
 				      r[j].fee_base_msat,
 				      r[j].fee_proportional_millionths,
 				      p->maxspend);
-			// TODO(eduardo): I think should be `end`
 			end = &r[j].pubkey;
-			// dest = &r[j].pubkey;
 		}
 	}
 }
@@ -1012,6 +1019,7 @@ static struct command_result *json_pay(struct command *cmd,
 	p->stop_time = timeabs_add(p->start_time, time_from_sec(*retryfor));
 	tal_free(retryfor);
 	
+	bool invstr_is_b11=false;
 	if (!bolt12_has_prefix(p->invstr)) {
 		struct bolt11 *b11;
 		char *fail;
@@ -1022,7 +1030,9 @@ static struct command_result *json_pay(struct command *cmd,
 		if (b11 == NULL)
 			return command_fail(cmd, JSONRPC2_INVALID_PARAMS,
 					    "Invalid bolt11: %s", fail);
-
+		
+		invstr_is_b11=true;
+		
 		invmsat = b11->msat;
 		invexpiry = b11->timestamp + b11->expiry;
 
@@ -1034,8 +1044,7 @@ static struct command_result *json_pay(struct command *cmd,
 			p->payment_metadata = tal_dup_talarr(p, u8, b11->metadata);
 		else
 			p->payment_metadata = NULL;
-			
-		add_routehints(p, b11->routes, &p->dest);
+		
 		
 		p->final_cltv = b11->min_final_cltv_expiry;
 		/* Sanity check */
@@ -1175,6 +1184,10 @@ static struct command_result *json_pay(struct command *cmd,
 	if (time_now().ts.tv_sec > invexpiry)
 		return command_fail(cmd, PAY_INVOICE_EXPIRED, "Invoice expired");
 	
+	// TODO(eduardo): are there route hints for B12?
+	if(invstr_is_b11)
+		add_routehints(p);
+	
 	struct out_req *req;
 
 	/* Get local capacities... */
@@ -1182,6 +1195,13 @@ static struct command_result *json_pay(struct command *cmd,
 				    listpeerchannels_done,
 				    listpeerchannels_done, p);
 	return send_outreq(cmd->plugin, req);
+	
+	// struct json_stream *ret;
+	// ret = jsonrpc_stream_success(cmd);
+	// json_add_string(ret,"hello","world");
+	// json_add_amount_msat_only(ret,"maxspend_msat",p->maxspend);
+	// 
+	// return command_finished(cmd,ret);
 }
 
 static const struct plugin_command commands[] = {

From c475d32abdcc42de54ec26767f749582875bc74e Mon Sep 17 00:00:00 2001
From: Lagrang3 <eduardo.quintana@pm.me>
Date: Wed, 17 May 2023 12:14:16 +0100
Subject: [PATCH 33/64] renepay: remove printf loggin

---
 plugins/renepay/mcf.c | 25 ++++++++++++++-----------
 1 file changed, 14 insertions(+), 11 deletions(-)

diff --git a/plugins/renepay/mcf.c b/plugins/renepay/mcf.c
index 878999143477..1b7ce86d762e 100644
--- a/plugins/renepay/mcf.c
+++ b/plugins/renepay/mcf.c
@@ -1315,6 +1315,9 @@ static bool is_better(
  * TODO(eduardo): notice that we don't pay fees to forward payments with local
  * channels and we can tell with absolute certainty the liquidity on them. 
  * Check that local channels have fee costs = 0 and bounds with certainty (min=max). */
+
+// TODO(eduardo): we should LOG_DBG the process of finding the MCF while
+// adjusting the frugality factor.
 struct flow** minflow(
 		const tal_t *ctx,
 		struct gossmap *gossmap,
@@ -1329,7 +1332,7 @@ struct flow** minflow(
 		double base_fee_penalty,
 		u32 prob_cost_factor )
 {
-	printf("%s: starting\n",__PRETTY_FUNCTION__);
+	// printf("%s: starting\n",__PRETTY_FUNCTION__);
 	tal_t *this_ctx = tal(tmpctx,tal_t);
 	
 	struct pay_parameters *params = tal(this_ctx,struct pay_parameters);	
@@ -1379,23 +1382,23 @@ struct flow** minflow(
 	
 	init_residual_netork(linear_network,residual_network);
 	
-	printf("%s: done with allocation and initialization\n",__PRETTY_FUNCTION__);
+	// printf("%s: done with allocation and initialization\n",__PRETTY_FUNCTION__);
 	
 	struct amount_msat best_fee;
 	double best_prob_success;
 	struct flow **best_flow_paths = NULL;
 	
-	printf("%s: searching for a feasible flow\n",__PRETTY_FUNCTION__);
+	// printf("%s: searching for a feasible flow\n",__PRETTY_FUNCTION__);
 	int err = find_feasible_flow(linear_network,residual_network,source_idx,target_idx,
 	                             params->amount.millisatoshis/1000);
 	
 	if(err!=RENEPAY_ERR_OK)
 	{
 		// there is no flow that satisfy the constraints, we stop here
-		printf("%s: feasible flow not found\n",__PRETTY_FUNCTION__);
+		// printf("%s: feasible flow not found\n",__PRETTY_FUNCTION__);
 		goto finish;
 	}
-	printf("%s: found a feasible flow\n",__PRETTY_FUNCTION__);
+	// printf("%s: found a feasible flow\n",__PRETTY_FUNCTION__);
 	
 	// first flow found
 	best_flow_paths = get_flow_paths(ctx,params->gossmap,params->chan_extra_map,
@@ -1414,7 +1417,7 @@ struct flow** minflow(
 	{
 		
 		s64 mu = (mu_left + mu_right)/2;
-		printf("%s: mu=%ld\n",__PRETTY_FUNCTION__,mu);
+		// printf("%s: mu=%ld\n",__PRETTY_FUNCTION__,mu);
 		
 		combine_cost_function(linear_network,residual_network,mu);
 		
@@ -1431,8 +1434,8 @@ struct flow** minflow(
 						params->chan_extra_map);
 		struct amount_msat fee = flow_set_fee(flow_paths);
 		
-		printf("prob %.2f, fee %s\n",prob_success,
-				type_to_string(this_ctx,struct amount_msat,&fee));
+		// printf("prob %.2f, fee %s\n",prob_success,
+		//		type_to_string(this_ctx,struct amount_msat,&fee));
 		
 		// is this better than the previous one?
 		if(!best_flow_paths || 
@@ -1450,12 +1453,12 @@ struct flow** minflow(
 		{
 			// too expensive
 			mu_left = mu+1;
-			printf("%s: too expensive\n",__PRETTY_FUNCTION__);
+			// printf("%s: too expensive\n",__PRETTY_FUNCTION__);
 		}else if(prob_success < params->min_probability)
 		{
 			// too unlikely
 			mu_right = mu;
-			printf("%s: too unlikely\n",__PRETTY_FUNCTION__);
+			// printf("%s: too unlikely\n",__PRETTY_FUNCTION__);
 		}else
 		{
 			// with mu constraints are satisfied, now let's optimize
@@ -1471,7 +1474,7 @@ struct flow** minflow(
 	
 	finish:
 	
-	printf("%s: finished\n",__PRETTY_FUNCTION__);
+	// printf("%s: finished\n",__PRETTY_FUNCTION__);
 	
 	tal_free(this_ctx);
 	return best_flow_paths;

From e2609ba3ded520dd1cd9eadce46b4c8ff976c115 Mon Sep 17 00:00:00 2001
From: Lagrang3 <eduardo.quintana@pm.me>
Date: Thu, 18 May 2023 07:54:49 +0100
Subject: [PATCH 34/64] renepay: bugfix on sendpay rpc request

---
 plugins/renepay/pay.c      | 1 +
 plugins/renepay/pay_flow.c | 2 +-
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/plugins/renepay/pay.c b/plugins/renepay/pay.c
index f95536673dee..b37b42ddf4e7 100644
--- a/plugins/renepay/pay.c
+++ b/plugins/renepay/pay.c
@@ -796,6 +796,7 @@ sendpay_flows(struct command *cmd,
 						  flows[i]->amounts[j]);
 			json_add_u32(req->js, "delay",
 				     flows[i]->cltv_delays[j]);
+			json_object_end(req->js);
 		}
 		json_array_end(req->js);
 
diff --git a/plugins/renepay/pay_flow.c b/plugins/renepay/pay_flow.c
index 5b21ca568d98..38dd0c30505f 100644
--- a/plugins/renepay/pay_flow.c
+++ b/plugins/renepay/pay_flow.c
@@ -83,7 +83,7 @@ static u32 shadow_one_flow(const struct gossmap *gossmap,
 			;
 
 	/* Shouldn't happen either */
-	if (amount_msat_sub(shadow_fee, amount, f->amounts[numpath-1]))
+	if (!amount_msat_sub(shadow_fee, amount, f->amounts[numpath-1]))
 		plugin_err(pay_plugin->plugin,
 			   "Failed to calc shadow fee: %s - %s",
 			   type_to_string(tmpctx, struct amount_msat, &amount),

From 27ce759b5c993a84406850270d251988c78a0d34 Mon Sep 17 00:00:00 2001
From: Lagrang3 <eduardo.quintana@pm.me>
Date: Fri, 19 May 2023 09:18:45 +0100
Subject: [PATCH 35/64] renepay: debuging utils for pay_flows

---
 plugins/renepay/pay_flow.c | 46 ++++++++++++++++++++++++++++++++++++++
 plugins/renepay/pay_flow.h |  3 +++
 2 files changed, 49 insertions(+)

diff --git a/plugins/renepay/pay_flow.c b/plugins/renepay/pay_flow.c
index 38dd0c30505f..70701aee4a79 100644
--- a/plugins/renepay/pay_flow.c
+++ b/plugins/renepay/pay_flow.c
@@ -8,6 +8,8 @@
 #include <plugins/renepay/mcf.h>
 #include <plugins/renepay/pay.h>
 #include <plugins/renepay/pay_flow.h>
+#include <ccan/tal/str/str.h>
+#include <ccan/json_out/json_out.h>
 
 /* BOLT #7:
  *
@@ -485,3 +487,47 @@ struct pay_flow **get_payflows(struct payment *p,
 	goto out;
 }
 
+const char* fmt_payflows(const tal_t *ctx,
+			 struct pay_flow ** flows)
+{
+	struct json_out *jout = json_out_new(ctx);
+	json_out_start(jout, NULL, '{');
+	json_out_start(jout,"Pay_flows",'[');
+	
+	for(size_t i=0;i<tal_count(flows);++i)
+	{
+		struct pay_flow *f = flows[i];
+		json_out_start(jout,NULL,'{');
+		json_out_add(jout,"success_prob",false,"%.2lf",f->success_prob);
+		json_out_start(jout,"path_scids",'[');
+		for(size_t j=0;j<tal_count(f->path_scids);++j)
+		{
+			json_out_add(jout,NULL,true,"%s",
+				type_to_string(ctx,struct short_channel_id,&f->path_scids[j]));
+		}
+		json_out_end(jout,']');
+		json_out_start(jout,"path_dirs",'[');
+		for(size_t j=0;j<tal_count(f->path_dirs);++j)
+		{
+			json_out_add(jout,NULL,false,"%d",f->path_dirs[j]);
+		}
+		json_out_end(jout,']');
+		json_out_start(jout,"amounts",'[');
+		for(size_t j=0;j<tal_count(f->amounts);++j)
+		{
+			json_out_add(jout,NULL,true,"%s",
+				type_to_string(ctx,struct amount_msat,&f->amounts[j]));
+		}
+		json_out_end(jout,']');
+		json_out_end(jout,'}');
+	}
+	
+	json_out_end(jout,']');
+	json_out_end(jout, '}');
+ 	json_out_direct(jout, 1)[0] = '\n';
+ 	json_out_direct(jout, 1)[0] = '\0';
+ 	json_out_finished(jout);
+	
+	size_t len;
+	return json_out_contents(jout,&len);
+}
diff --git a/plugins/renepay/pay_flow.h b/plugins/renepay/pay_flow.h
index faabc770461f..a6cf313d71bb 100644
--- a/plugins/renepay/pay_flow.h
+++ b/plugins/renepay/pay_flow.h
@@ -28,4 +28,7 @@ struct pay_flow **get_payflows(struct payment *p,
 			       bool unlikely_ok,
 			       bool is_entire_payment);
 
+const char* fmt_payflows(const tal_t *ctx,
+			 struct pay_flow ** flows);
+
 #endif /* LIGHTNING_PLUGINS_RENEPAY_PAY_FLOW_H */

From 35bf034839591fa2deff7d39c4ade6fbea18bf2f Mon Sep 17 00:00:00 2001
From: Lagrang3 <eduardo.quintana@pm.me>
Date: Mon, 22 May 2023 14:50:50 +0100
Subject: [PATCH 36/64] renepay: add payment_secret to sendpay request

---
 plugins/renepay/pay.c      | 31 ++++++++++++++++++++++++++++++-
 plugins/renepay/pay.h      |  1 -
 plugins/renepay/pay_flow.c | 20 ++++++++++++++++++++
 3 files changed, 50 insertions(+), 2 deletions(-)

diff --git a/plugins/renepay/pay.c b/plugins/renepay/pay.c
index b37b42ddf4e7..4f2f13764940 100644
--- a/plugins/renepay/pay.c
+++ b/plugins/renepay/pay.c
@@ -17,10 +17,27 @@
 #include <plugins/renepay/pay_flow.h>
 #include <stdio.h>
 #include <wire/peer_wire.h>
+#include <ccan/json_out/json_out.h>
 
 /* Set in init */
 struct pay_plugin *pay_plugin;
 
+static void debug_payflows(struct pay_flow **flows, const char* fname)
+{
+	FILE *f = fopen(fname,"a");
+	fprintf(f,"%s\n",fmt_payflows(tmpctx,flows));
+	fclose(f);
+}
+
+static void debug_outreq(const struct out_req *req, const char*fname)
+{
+	FILE *f = fopen(fname,"a");
+	size_t len;
+	const char * str =  json_out_contents(req->js->jout,&len);
+	fprintf(f,"%s\n",str);
+	fclose(f);
+}
+
 void paynote(struct payment *p, const char *fmt, ...)
 {
 	va_list ap;
@@ -244,6 +261,7 @@ static bool update_capacities_from_listpeerchannels(
 		const char *buf,
 		const jsmntok_t *toks)
 {
+	plugin_log(pay_plugin->plugin,LOG_DBG,"calling update_capacities_from_listpeerchannels");
 	const jsmntok_t *channels, *channel;
 	size_t i;
 
@@ -773,6 +791,7 @@ static struct command_result *flow_sendpay_failed(struct command *cmd,
 	return flow_failed(cmd, flow);
 }
 
+// TODO(eduardo): check this
 static struct command_result *
 sendpay_flows(struct command *cmd,
 	      struct payment *p,
@@ -794,13 +813,17 @@ sendpay_flows(struct command *cmd,
 						  &flows[i]->path_scids[j]);
 			json_add_amount_msat_only(req->js, "amount_msat",
 						  flows[i]->amounts[j]);
+			json_add_num(req->js, "direction",
+						  flows[i]->path_dirs[j]);
 			json_add_u32(req->js, "delay",
 				     flows[i]->cltv_delays[j]);
+			json_add_string(req->js,"style","tlv");
 			json_object_end(req->js);
 		}
 		json_array_end(req->js);
 
 		json_add_sha256(req->js, "payment_hash", &p->payment_hash);
+		json_add_secret(req->js, "payment_secret", p->payment_secret);
 		json_add_amount_msat_only(req->js, "amount_msat", p->amount);
 		json_add_u64(req->js, "partid", flows[i]->partid);
 		json_add_u64(req->js, "groupid", p->groupid);
@@ -814,7 +837,9 @@ sendpay_flows(struct command *cmd,
 		json_add_string(req->js, "bolt11", p->invstr);
 		if (p->description)
 			json_add_string(req->js, "description", p->description);
-
+		
+		debug_outreq(req,"/tmp/dbg.txt");
+		
 		/* Flow now owned by request */
 		tal_steal(req, flows[i]);
 		send_outreq(cmd->plugin, req);
@@ -860,6 +885,9 @@ static struct command_result *try_paying(struct command *cmd,
 	 * than simply refuse.  Plus, models are not truth! */
 	pay_flows = get_payflows(p, remaining, feebudget, first_time,
 				 amount_msat_eq(p->total_delivering, AMOUNT_MSAT(0)));
+	
+	debug_payflows(pay_flows,"/tmp/dbg.txt");
+	
 	if (!pay_flows)
 		return command_fail(cmd, PAY_ROUTE_NOT_FOUND,
 				    "Failed to find a route for %s with budget %s",
@@ -876,6 +904,7 @@ static struct command_result *
 listpeerchannels_done(struct command *cmd, const char *buf,
 	       const jsmntok_t *result, struct payment *p)
 {
+	plugin_log(pay_plugin->plugin,LOG_DBG,"calling listpeerchannels_done");
 	// FILE *f = fopen("/tmp/afile","wb");
 	// fwrite(json_tok_full(buf,result),json_tok_full_len(result),1,f);
 	// fclose(f);
diff --git a/plugins/renepay/pay.h b/plugins/renepay/pay.h
index 8498e9ec2cd6..f197a96f30dd 100644
--- a/plugins/renepay/pay.h
+++ b/plugins/renepay/pay.h
@@ -56,7 +56,6 @@ struct payment {
 	/* How much, what, where */
 	// TODO(eduardo): is it used?
 	struct node_id dest;
-	// TODO(eduardo): is it used?
 	struct sha256 payment_hash;
 	// TODO(eduardo): is it used?
 	struct amount_msat amount;
diff --git a/plugins/renepay/pay_flow.c b/plugins/renepay/pay_flow.c
index 70701aee4a79..2186c5c75cd3 100644
--- a/plugins/renepay/pay_flow.c
+++ b/plugins/renepay/pay_flow.c
@@ -498,7 +498,9 @@ const char* fmt_payflows(const tal_t *ctx,
 	{
 		struct pay_flow *f = flows[i];
 		json_out_start(jout,NULL,'{');
+		
 		json_out_add(jout,"success_prob",false,"%.2lf",f->success_prob);
+		
 		json_out_start(jout,"path_scids",'[');
 		for(size_t j=0;j<tal_count(f->path_scids);++j)
 		{
@@ -506,12 +508,14 @@ const char* fmt_payflows(const tal_t *ctx,
 				type_to_string(ctx,struct short_channel_id,&f->path_scids[j]));
 		}
 		json_out_end(jout,']');
+		
 		json_out_start(jout,"path_dirs",'[');
 		for(size_t j=0;j<tal_count(f->path_dirs);++j)
 		{
 			json_out_add(jout,NULL,false,"%d",f->path_dirs[j]);
 		}
 		json_out_end(jout,']');
+		
 		json_out_start(jout,"amounts",'[');
 		for(size_t j=0;j<tal_count(f->amounts);++j)
 		{
@@ -519,6 +523,22 @@ const char* fmt_payflows(const tal_t *ctx,
 				type_to_string(ctx,struct amount_msat,&f->amounts[j]));
 		}
 		json_out_end(jout,']');
+		
+		json_out_start(jout,"cltv_delays",'[');
+		for(size_t j=0;j<tal_count(f->cltv_delays);++j)
+		{
+			json_out_add(jout,NULL,false,"%d",f->cltv_delays[j]);
+		}
+		json_out_end(jout,']');
+		
+		json_out_start(jout,"path_nodes",'[');
+		for(size_t j=0;j<tal_count(f->path_nodes);++j)
+		{
+			json_out_add(jout,NULL,true,"%s",
+				type_to_string(ctx,struct node_id,&f->path_nodes[j]));
+		}
+		json_out_end(jout,']');
+		
 		json_out_end(jout,'}');
 	}
 	

From 3347b03469c141c048cd99b4636b469b41bada06 Mon Sep 17 00:00:00 2001
From: Lagrang3 <eduardo.quintana@pm.me>
Date: Wed, 24 May 2023 17:10:53 +0100
Subject: [PATCH 37/64] renepay: fixed problem with MPP

---
 plugins/renepay/pay.c | 86 ++++++++++++++++++++++++++++++++++---------
 plugins/renepay/pay.h |  2 -
 2 files changed, 69 insertions(+), 19 deletions(-)

diff --git a/plugins/renepay/pay.c b/plugins/renepay/pay.c
index 4f2f13764940..cfacab555c42 100644
--- a/plugins/renepay/pay.c
+++ b/plugins/renepay/pay.c
@@ -18,6 +18,9 @@
 #include <stdio.h>
 #include <wire/peer_wire.h>
 #include <ccan/json_out/json_out.h>
+#include <pthread.h>
+
+#define MYLOG "/tmp/debug.txt"
 
 /* Set in init */
 struct pay_plugin *pay_plugin;
@@ -34,7 +37,18 @@ static void debug_outreq(const struct out_req *req, const char*fname)
 	FILE *f = fopen(fname,"a");
 	size_t len;
 	const char * str =  json_out_contents(req->js->jout,&len);
-	fprintf(f,"%s\n",str);
+	fprintf(f,"%s",str);
+	if (req->errcb)
+		fprintf(f,"}");
+	fprintf(f,"}\n");
+	fclose(f);
+}
+
+static void debug_call(const char* fun, const char* fname)
+{
+	pthread_t tid = pthread_self();
+	FILE *f = fopen(fname,"a");
+	fprintf(f,"calling function: %s (pthread_t %ld)\n",fun,tid);
 	fclose(f);
 }
 
@@ -104,7 +118,6 @@ static const char *init(struct plugin *p,
 	pay_plugin->chan_extra_map = tal(pay_plugin,struct chan_extra_map);
 	chan_extra_map_init(pay_plugin->chan_extra_map);
 
-	// TODO(eduardo) is it ok to use NULL or pay_plugin as `ctx`?
 	pay_plugin->gossmap = gossmap_load(pay_plugin,
 					   GOSSIP_STORE_FILENAME,
 					   &num_channel_updates_rejected);
@@ -217,6 +230,9 @@ static void add_localchan(struct payment *p,
 				 dir);
 	/* We know (assume!) something about this channel: that it has at
 	 * sufficient capacity. */
+	// TODO(eduardo): if this channel was already in our list then we need
+	// to update the knowledge accordingly. I think it would be better to
+	// handle this with an update_lower_bound function.
 	plugin_log(pay_plugin->plugin,LOG_DBG,"add_localchan: %s [maxspend %s]",
 		type_to_string(tmpctx,struct short_channel_id, &scid),
 		type_to_string(tmpctx,struct amount_msat, &p->maxspend));
@@ -261,7 +277,6 @@ static bool update_capacities_from_listpeerchannels(
 		const char *buf,
 		const jsmntok_t *toks)
 {
-	plugin_log(pay_plugin->plugin,LOG_DBG,"calling update_capacities_from_listpeerchannels");
 	const jsmntok_t *channels, *channel;
 	size_t i;
 
@@ -347,6 +362,8 @@ static struct command_result *try_paying(struct command *cmd,
 static struct command_result *flow_failed(struct command *cmd,
 					  const struct pay_flow *flow)
 {
+	plugin_log(pay_plugin->plugin,LOG_DBG,"calling flow_failed");
+	debug_call(__PRETTY_FUNCTION__,MYLOG);
 	struct payment *p = flow->payment;
 
 	amount_msat_reduce(&p->total_delivering, flow_delivered(flow));
@@ -365,13 +382,15 @@ static struct command_result *flow_failed(struct command *cmd,
 /* Happens when timer goes off, but also works to arm timer if nothing to do */
 static void timer_kick(struct payment *p)
 {
+	debug_call(__PRETTY_FUNCTION__,MYLOG);
 	p->rexmit_timer = tal_free(p->rexmit_timer);
 
 	/* Nothing has come back?  Re-arm timer. */
 	if (amount_msat_eq(p->total_delivering, p->amount)) {
-		p->rexmit_timer = plugin_timer(p->cmd->plugin,
-					       time_from_msec(250),
-					       timer_kick, p);
+		// TODO(eduardo): what's the purpose of this?
+		// p->rexmit_timer = plugin_timer(p->cmd->plugin,
+		// 			       time_from_msec(250),
+		// 			       timer_kick, p);
 		return;
 	}
 
@@ -383,6 +402,8 @@ static struct command_result *waitsendpay_succeeded(struct command *cmd,
 						    const jsmntok_t *result,
 						    struct pay_flow *flow)
 {
+	plugin_log(pay_plugin->plugin,LOG_DBG,"calling waitsendpay_succeeded");
+	debug_call(__PRETTY_FUNCTION__,MYLOG);
 	struct payment *p = flow->payment;
 	struct json_stream *response = jsonrpc_stream_success(cmd);
 	const jsmntok_t *preimagetok;
@@ -424,6 +445,7 @@ static struct command_result *handle_unhandleable_error(struct payment *p,
 							const struct pay_flow *flow,
 							const char *what)
 {
+	debug_call(__PRETTY_FUNCTION__,MYLOG);
 	size_t n = tal_count(flow);
 
 	/* We got a mangled reply.  We don't know who to penalize! */
@@ -466,6 +488,7 @@ static struct command_result *addgossip_done(struct command *cmd,
 					     const jsmntok_t *err,
 					     struct addgossip *adg)
 {
+	debug_call(__PRETTY_FUNCTION__,MYLOG);
 	struct payment *p = adg->flow->payment;
 
 	/* Release this: if it's the last flow we'll retry immediately */
@@ -481,6 +504,7 @@ static struct command_result *addgossip_failure(struct command *cmd,
 						struct addgossip *adg)
 
 {
+	debug_call(__PRETTY_FUNCTION__,MYLOG);
 	struct payment *p = adg->flow->payment;
 
 	paynote(p, "addgossip failed, removing channel %s (%.*s)",
@@ -491,11 +515,14 @@ static struct command_result *addgossip_failure(struct command *cmd,
 	return addgossip_done(cmd, buf, err, adg);
 }
 
+// TODO(eduardo): what is this?
 static struct command_result *submit_update(struct command *cmd,
 					    const struct pay_flow *flow,
 					    const u8 *update,
 					    struct short_channel_id errscid)
 {
+	plugin_log(pay_plugin->plugin,LOG_DBG,"calling submit_update");
+	debug_call(__PRETTY_FUNCTION__,MYLOG);
 	struct payment *p = flow->payment;
 	struct out_req *req;
 	struct addgossip *adg = tal(cmd, struct addgossip);
@@ -580,6 +607,8 @@ static struct command_result *waitsendpay_failed(struct command *cmd,
 						 const jsmntok_t *err,
 						 struct pay_flow *flow)
 {
+	plugin_log(pay_plugin->plugin,LOG_DBG,"calling waitsendpay_failed");
+	debug_call(__PRETTY_FUNCTION__,MYLOG);
 	struct payment *p = flow->payment;
 	u64 errcode;
 	struct command_result *ret;
@@ -746,6 +775,8 @@ static struct command_result *flow_sent(struct command *cmd,
 					const jsmntok_t *result,
 					struct pay_flow *flow)
 {
+	plugin_log(pay_plugin->plugin,LOG_DBG,"calling flow_sent");
+	debug_call(__PRETTY_FUNCTION__,MYLOG);
 	struct payment *p = flow->payment;
 	struct out_req *req;
 
@@ -772,6 +803,8 @@ static struct command_result *flow_sendpay_failed(struct command *cmd,
 						  const jsmntok_t *err,
 						  struct pay_flow *flow)
 {
+	plugin_log(pay_plugin->plugin,LOG_DBG,"calling flow_sendpay_failed");
+	debug_call(__PRETTY_FUNCTION__,MYLOG);
 	struct payment *p = flow->payment;
 	u64 errcode;
 	const jsmntok_t *msg = json_get_member(buf, err, "message");
@@ -791,12 +824,13 @@ static struct command_result *flow_sendpay_failed(struct command *cmd,
 	return flow_failed(cmd, flow);
 }
 
-// TODO(eduardo): check this
 static struct command_result *
 sendpay_flows(struct command *cmd,
 	      struct payment *p,
 	      struct pay_flow **flows STEALS)
 {
+	plugin_log(pay_plugin->plugin,LOG_DBG,"calling sendpay_flows");
+	debug_call(__PRETTY_FUNCTION__,MYLOG);
 	paynote(p, "Sending out batch of %zu payments", tal_count(flows));
 	for (size_t i = 0; i < tal_count(flows); i++) {
 		struct out_req *req;
@@ -824,8 +858,11 @@ sendpay_flows(struct command *cmd,
 
 		json_add_sha256(req->js, "payment_hash", &p->payment_hash);
 		json_add_secret(req->js, "payment_secret", p->payment_secret);
+		
 		json_add_amount_msat_only(req->js, "amount_msat", p->amount);
+			
 		json_add_u64(req->js, "partid", flows[i]->partid);
+		
 		json_add_u64(req->js, "groupid", p->groupid);
 		if (p->payment_metadata)
 			json_add_hex_talarr(req->js, "payment_metadata",
@@ -838,27 +875,37 @@ sendpay_flows(struct command *cmd,
 		if (p->description)
 			json_add_string(req->js, "description", p->description);
 		
-		debug_outreq(req,"/tmp/dbg.txt");
+		// TODO(eduardo): remove this line
+		debug_outreq(req,MYLOG);
 		
-		/* Flow now owned by request */
-		tal_steal(req, flows[i]);
-		send_outreq(cmd->plugin, req);
 		amount_msat_accumulate(&p->total_sent, flows[i]->amounts[0]);
 		amount_msat_accumulate(&p->total_delivering,
 				       flow_delivered(flows[i]));
+		
+		/* Flow now owned by request */
+		tal_steal(req, flows[i]);
+		
+		send_outreq(cmd->plugin, req);
 	}
-
+	
 	/* Get ready to process replies */
+	// TODO(eduardo): how is the timer mechanics?
 	timer_kick(p);
-
+	
+	// TODO(eduardo): what is this actually doing?
 	return command_still_pending(cmd);
 }
 
-// TODO(eduardo): check this
 static struct command_result *try_paying(struct command *cmd,
 					 struct payment *p,
 					 bool first_time)
 {
+	plugin_log(pay_plugin->plugin,LOG_DBG,"calling try_paying");
+	debug_call(__PRETTY_FUNCTION__,MYLOG);
+	
+	// TODO(eduardo): we should move on only if all rpc requests are done
+	// (all threads join)
+	
 	struct amount_msat feebudget, fees_spent, remaining;
 	struct pay_flow **pay_flows;
 
@@ -886,7 +933,8 @@ static struct command_result *try_paying(struct command *cmd,
 	pay_flows = get_payflows(p, remaining, feebudget, first_time,
 				 amount_msat_eq(p->total_delivering, AMOUNT_MSAT(0)));
 	
-	debug_payflows(pay_flows,"/tmp/dbg.txt");
+	// TODO(eduardo): remove this line
+	debug_payflows(pay_flows,MYLOG);
 	
 	if (!pay_flows)
 		return command_fail(cmd, PAY_ROUTE_NOT_FOUND,
@@ -904,7 +952,6 @@ static struct command_result *
 listpeerchannels_done(struct command *cmd, const char *buf,
 	       const jsmntok_t *result, struct payment *p)
 {
-	plugin_log(pay_plugin->plugin,LOG_DBG,"calling listpeerchannels_done");
 	// FILE *f = fopen("/tmp/afile","wb");
 	// fwrite(json_tok_full(buf,result),json_tok_full_len(result),1,f);
 	// fclose(f);
@@ -987,7 +1034,12 @@ static struct command_result *json_pay(struct command *cmd,
 	p->paynotes = tal_arr(p, const char *, 0);
 	p->disabled = tal_arr(p, struct short_channel_id, 0);
 	p->groupid = pseudorand_u64();
-	p->next_partid = 0;
+	
+	// TODO(eduardo): If the partid starts counting from 1 we can do
+	// parallel payments MPP. A single payment route is done with partid=0.
+	// Can we assume that all payments are MPP, even in the case of an MPP
+	// with a single route?
+	p->next_partid = 1;
 	p->total_sent = AMOUNT_MSAT(0);
 	p->total_delivering = AMOUNT_MSAT(0);
 	p->rexmit_timer = NULL;
diff --git a/plugins/renepay/pay.h b/plugins/renepay/pay.h
index f197a96f30dd..6095b8d53d63 100644
--- a/plugins/renepay/pay.h
+++ b/plugins/renepay/pay.h
@@ -64,10 +64,8 @@ struct payment {
 	u32 final_cltv;
 
 	/* Total sent, including fees. */
-	// TODO(eduardo): is it used?
 	struct amount_msat total_sent;
 	/* Total that is delivering (i.e. without fees) */
-	// TODO(eduardo): is it used?
 	struct amount_msat total_delivering;
 
 	/* payment_secret, if specified by invoice. */

From 309db4f3cf814bc8467baea4117d93c224e30a0b Mon Sep 17 00:00:00 2001
From: Lagrang3 <eduardo.quintana@pm.me>
Date: Mon, 29 May 2023 06:28:36 +0100
Subject: [PATCH 38/64] renepay: add knowledge update functions

---
 Makefile                                   |   4 +-
 plugins/renepay/flow.c                     | 489 +++++++++++++++++----
 plugins/renepay/flow.h                     | 151 ++++++-
 plugins/renepay/mcf.c                      |  10 +-
 plugins/renepay/pay.c                      | 354 ++++++++++-----
 plugins/renepay/pay.h                      |   3 +-
 plugins/renepay/pay_flow.c                 |  10 -
 plugins/renepay/test/Makefile              |   2 +-
 plugins/renepay/test/run-mcf-diamond.c     |  16 +-
 plugins/renepay/test/run-mcf.c             |   4 +-
 plugins/renepay/test/run-not_mcf-gossmap.c | 132 ------
 11 files changed, 818 insertions(+), 357 deletions(-)
 delete mode 100644 plugins/renepay/test/run-not_mcf-gossmap.c

diff --git a/Makefile b/Makefile
index 65dcec1bbaeb..f38bb0ccdd38 100644
--- a/Makefile
+++ b/Makefile
@@ -449,7 +449,9 @@ else
 PYTEST_OPTS += -x
 endif
 
-check-units:
+check-units: check-renepay
+
+check-renepay:
 
 check: check-units installcheck check-protos pytest
 
diff --git a/plugins/renepay/flow.c b/plugins/renepay/flow.c
index 49f0832b1ffe..759489e2a7d9 100644
--- a/plugins/renepay/flow.c
+++ b/plugins/renepay/flow.c
@@ -13,13 +13,400 @@
 #define SUPERVERBOSE_ENABLED 1
 #endif
 
-/* Helper to access the half chan at flow index idx */
-const struct half_chan *flow_edge(const struct flow *flow, size_t idx)
+struct chan_extra *new_chan_extra(
+		struct chan_extra_map *chan_extra_map,
+		const struct short_channel_id scid,
+		struct amount_msat capacity)
 {
-	assert(idx < tal_count(flow->path));
-	return &flow->path[idx]->half[flow->dirs[idx]];
+	struct chan_extra *ce = tal(chan_extra_map, struct chan_extra);
+
+	ce->scid = scid;
+	ce->capacity=capacity;
+	for (size_t i = 0; i <= 1; i++) {
+		ce->half[i].num_htlcs = 0;
+		ce->half[i].htlc_total = AMOUNT_MSAT(0);
+		ce->half[i].known_min = AMOUNT_MSAT(0);
+		ce->half[i].known_max = capacity;
+	}
+	chan_extra_map_add(chan_extra_map, ce);
+	
+	/* Remove self from map when done */
+	// TODO(eduardo):
+	// Is this desctructor really necessary? the chan_extra will deallocated
+	// when the chan_extra_map is freed. Anyways valgrind complains that the
+	// hash table is removing the element with a freed pointer.
+	// tal_add_destructor2(ce, destroy_chan_extra, chan_extra_map);
+	return ce;
+}
+
+bool chan_extra_check_invariants(struct chan_extra *ce)
+{
+	bool all_ok = true;
+	for(int i=0;i<2;++i)
+	{
+		all_ok &= amount_msat_less_eq(ce->half[i].known_min,
+					      ce->half[i].known_max);
+		all_ok &= amount_msat_less_eq(ce->half[i].known_max,
+					      ce->capacity);
+	}
+	struct amount_msat diff_cb,diff_ca;
+	
+	all_ok &= amount_msat_sub(&diff_cb,ce->capacity,ce->half[1].known_max);
+	all_ok &= amount_msat_sub(&diff_ca,ce->capacity,ce->half[1].known_min);
+	
+	all_ok &= amount_msat_eq(ce->half[0].known_min,diff_cb);
+	all_ok &= amount_msat_eq(ce->half[0].known_max,diff_ca);
+	return all_ok;
+}
+
+/* This helper function preserves the uncertainty network invariant after the
+ * knowledge is updated. It assumes that the (channel,!dir) knowledge is
+ * correct. */
+void chan_extra_adjust_half(struct chan_extra *ce,
+			    int dir)
+{
+	if(!amount_msat_sub(&ce->half[dir].known_max,ce->capacity,ce->half[!dir].known_min))
+		abort();
+	if(!amount_msat_sub(&ce->half[dir].known_min,ce->capacity,ce->half[!dir].known_max))
+		abort();
+}
+
+
+/* Update the knowledge that this (channel,direction) can send x msat.*/
+static void chan_extra_can_send_(
+		struct chan_extra *ce, 
+		int dir,
+		struct amount_msat x)
+{
+	if(amount_msat_greater(x,ce->capacity))
+	{
+		// It should never happen thatn x>capacity
+		abort();
+		x = ce->capacity;
+	}
+	
+	ce->half[dir].known_min = amount_msat_max(ce->half[dir].known_min,x);
+	ce->half[dir].known_max = amount_msat_max(ce->half[dir].known_max,x);
+	
+	chan_extra_adjust_half(ce,!dir);
+}
+void chan_extra_can_send(
+		struct chan_extra_map *chan_extra_map,
+		struct short_channel_id scid, 
+		int dir,
+		struct amount_msat x)
+{
+	struct chan_extra *ce = chan_extra_map_get(chan_extra_map,
+						   scid);
+	if(!ce)
+		abort();
+	chan_extra_can_send_(ce,dir,x);
+}
+/* Update the knowledge that this (channel,direction) cannot send x msat.*/
+static void chan_extra_cannot_send_(
+		struct chan_extra *ce, 
+		int dir,
+		struct amount_msat x)
+{
+	if(!amount_msat_sub(&x,x,AMOUNT_MSAT(1)))
+	{
+		// It should never happen that x==0
+		abort();
+		x = AMOUNT_MSAT(0);
+	}
+	
+	ce->half[dir].known_min = amount_msat_min(ce->half[dir].known_min,x);
+	ce->half[dir].known_max = amount_msat_min(ce->half[dir].known_max,x);
+	
+	chan_extra_adjust_half(ce,!dir);
+}
+void chan_extra_cannot_send(
+		struct chan_extra_map *chan_extra_map,
+		struct short_channel_id scid, 
+		int dir,
+		struct amount_msat x)
+{
+	struct chan_extra *ce = chan_extra_map_get(chan_extra_map,
+						   scid);
+	if(!ce)
+		abort();
+	chan_extra_cannot_send_(ce,dir,x);
+}
+/* Update the knowledge that this (channel,direction) has liquidity x.*/
+static void chan_extra_set_liquidity_(
+		struct chan_extra *ce, 
+		int dir,
+		struct amount_msat x)
+{
+	if(amount_msat_greater(x,ce->capacity))
+	{
+		// It should never happen thatn x>capacity
+		abort();
+		x = ce->capacity;
+	}
+	
+	ce->half[dir].known_min = x;
+	ce->half[dir].known_max = x;
+	
+	chan_extra_adjust_half(ce,!dir);
+}
+void chan_extra_set_liquidity(
+		struct chan_extra_map *chan_extra_map,
+		struct short_channel_id scid, 
+		int dir,
+		struct amount_msat x)
+{
+	struct chan_extra *ce = chan_extra_map_get(chan_extra_map,
+						   scid);
+	if(!ce)
+		abort();
+	chan_extra_set_liquidity_(ce,dir,x);
+}
+/* Update the knowledge that this (channel,direction) has sent x msat.*/
+static void chan_extra_sent_success_(
+		struct chan_extra *ce, 
+		int dir,
+		struct amount_msat x)
+{
+	if(amount_msat_greater(x,ce->capacity))
+	{
+		// It should never happen thatn x>capacity
+		abort();
+		x = ce->capacity;
+	}
+	
+	struct amount_msat new_a, new_b;
+	
+	if(!amount_msat_sub(&new_a,ce->half[dir].known_min,x))
+		new_a = AMOUNT_MSAT(0);
+	if(!amount_msat_sub(&new_b,ce->half[dir].known_max,x))
+		new_b = AMOUNT_MSAT(0);
+	
+	ce->half[dir].known_min = new_a;
+	ce->half[dir].known_max = new_b;
+	
+	chan_extra_adjust_half(ce,!dir);
+}
+void chan_extra_sent_success(
+		struct chan_extra_map *chan_extra_map,
+		struct short_channel_id scid, 
+		int dir,
+		struct amount_msat x)
+{
+	struct chan_extra *ce = chan_extra_map_get(chan_extra_map,
+						   scid);
+	if(!ce)
+		abort();
+	chan_extra_sent_success_(ce,dir,x);
+}
+/* Forget a bit about this (channel,direction) state. */
+static void chan_extra_relax_(
+		struct chan_extra *ce, 
+		int dir,
+		struct amount_msat down,
+		struct amount_msat up)
+{
+	struct amount_msat new_a, new_b;
+	
+	if(!amount_msat_sub(&new_a,ce->half[dir].known_min,down))
+		new_a = AMOUNT_MSAT(0);
+	if(!amount_msat_add(&new_b,ce->half[dir].known_max,up))
+		new_b = amount_msat_min(new_b,ce->capacity);
+	
+	ce->half[dir].known_min = new_a;
+	ce->half[dir].known_max = new_b;
+	
+	chan_extra_adjust_half(ce,!dir);
+}
+void chan_extra_relax(
+		struct chan_extra_map *chan_extra_map,
+		struct short_channel_id scid, 
+		int dir,
+		struct amount_msat x,
+		struct amount_msat y)
+{
+	struct chan_extra *ce = chan_extra_map_get(chan_extra_map,
+						   scid);
+	if(!ce)
+		abort();
+	chan_extra_relax_(ce,dir,x,y);
+}
+
+/* Checks the entire uncertainty network for invariant violations. */
+bool uncertainty_network_check_invariants(struct chan_extra_map *chan_extra_map)
+{
+	bool all_ok = true;
+	
+	struct chan_extra_map_iter it;
+	for(struct chan_extra *ce = chan_extra_map_first(chan_extra_map,&it);
+	    ce && all_ok;
+	    ce=chan_extra_map_next(chan_extra_map,&it))
+	{
+		all_ok &= chan_extra_check_invariants(ce);
+	}
+		
+	return all_ok;
+}
+
+/* Mirror the gossmap in the public uncertainty network.
+ * result: Every channel in gossmap must have associated data in chan_extra_map,
+ * while every channel in chan_extra_map is also registered in gossmap.
+ * */
+void uncertainty_network_update(
+		const struct gossmap *gossmap,
+		struct chan_extra_map *chan_extra_map)
+{
+	const tal_t* this_ctx = tal(tmpctx,tal_t);
+	
+	// For each chan in chan_extra_map remove if not in the gossmap
+	struct short_channel_id *del_list
+		= tal_arr(this_ctx,struct short_channel_id,0);
+		
+	struct chan_extra_map_iter it;
+	for(struct chan_extra *ce = chan_extra_map_first(chan_extra_map,&it);
+	    ce;
+	    ce=chan_extra_map_next(chan_extra_map,&it))
+	{
+		struct gossmap_chan * chan = gossmap_find_chan(gossmap,&ce->scid);
+		if(!chan)
+		{
+			// TODO(eduardo): is this efficiently implemented?
+			// otherwise i'll use a ccan list
+			tal_arr_expand(&del_list, ce->scid);
+		}
+	}
+	
+	for(size_t i=0;i<tal_count(del_list);++i)
+	{
+ 		struct chan_extra *ce = chan_extra_map_get(chan_extra_map,del_list[i]);
+		if(!ce)
+		{
+			SUPERVERBOSE("%s: aborting\n",__PRETTY_FUNCTION__);
+			abort();
+		}
+		chan_extra_map_del(chan_extra_map, ce);
+		tal_free(ce);
+	}
+	
+	// For each channel in the gossmap, create a extra data in
+	// chan_extra_map
+	for(struct gossmap_chan *chan = gossmap_first_chan(gossmap);
+	    chan;
+	    chan=gossmap_next_chan(gossmap,chan))
+	{
+		struct short_channel_id scid =
+			gossmap_chan_scid(gossmap,chan);
+		struct chan_extra *ce = chan_extra_map_get(chan_extra_map,
+							   gossmap_chan_scid(gossmap,chan));	
+		if(!ce)
+		{
+			struct amount_sat cap;
+			struct amount_msat cap_msat;
+			
+			if(!gossmap_chan_get_capacity(gossmap,chan,&cap))
+			{
+				SUPERVERBOSE("%s: aborting\n",__PRETTY_FUNCTION__);
+				abort();
+			}
+			if(!amount_sat_to_msat(&cap_msat,cap))
+			{
+				SUPERVERBOSE("%s: aborting\n",__PRETTY_FUNCTION__);
+				abort();
+			}
+			new_chan_extra(chan_extra_map,scid,cap_msat);
+		}
+	}
+	
+	
+	tal_free(this_ctx);
+}
+
+/* Returns either NULL, or an entry from the hash */
+struct chan_extra_half *
+get_chan_extra_half_by_scid(struct chan_extra_map *chan_extra_map,
+			    const struct short_channel_id scid,
+			    int dir)
+{
+	struct chan_extra *ce;
+
+	ce = chan_extra_map_get(chan_extra_map, scid);
+	if (!ce)
+		return NULL;
+	return &ce->half[dir];
+}
+/* Helper if we have a gossmap_chan */
+struct chan_extra_half *
+get_chan_extra_half_by_chan(const struct gossmap *gossmap,
+			    struct chan_extra_map *chan_extra_map,
+			    const struct gossmap_chan *chan,
+			    int dir)
+{
+	return get_chan_extra_half_by_scid(chan_extra_map,
+					   gossmap_chan_scid(gossmap, chan),
+					   dir);
 }
 
+
+// static void destroy_chan_extra(struct chan_extra *ce,
+// 			       struct chan_extra_map *chan_extra_map)
+// {
+// 	chan_extra_map_del(chan_extra_map, ce);
+// }
+/* Helper to get the chan_extra_half. If it doesn't exist create a new one. */
+struct chan_extra_half *
+get_chan_extra_half_by_chan_verify(
+		const struct gossmap *gossmap,
+		struct chan_extra_map *chan_extra_map,
+		const struct gossmap_chan *chan,
+		int dir)
+{
+	
+	const struct short_channel_id scid = gossmap_chan_scid(gossmap,chan);
+	struct chan_extra_half *h = get_chan_extra_half_by_scid(
+					chan_extra_map,scid,dir);
+	if (!h) {
+		struct amount_sat cap;
+		struct amount_msat cap_msat;
+
+		if (!gossmap_chan_get_capacity(gossmap,chan, &cap) || 
+		    !amount_sat_to_msat(&cap_msat, cap))
+		{
+			SUPERVERBOSE("%s: aborting\n",__PRETTY_FUNCTION__);
+			abort();
+		}
+		h = & new_chan_extra(chan_extra_map,scid,cap_msat)->half[dir];
+		
+	}
+	return h;
+}
+
+// // TODO(eduardo): not sure we should have this function
+// struct chan_extra_half *new_chan_extra_half(struct chan_extra_map *chan_extra_map,
+// 					    const struct short_channel_id scid,
+// 					    int dir,
+// 					    struct amount_msat capacity)
+// {
+// 	struct chan_extra *ce = tal(chan_extra_map, struct chan_extra);
+// 
+// 	ce->scid = scid;
+// 	for (size_t i = 0; i <= 1; i++) {
+// 		ce->half[i].num_htlcs = 0;
+// 		ce->half[i].htlc_total = AMOUNT_MSAT(0);
+// 		ce->half[i].known_min = AMOUNT_MSAT(0);
+// 		ce->half[i].known_max = capacity;
+// 	}
+// 	/* Remove self from map when done */
+// 	chan_extra_map_add(chan_extra_map, ce);
+// 	
+// 	// TODO(eduardo):
+// 	// Is this desctructor really necessary? the chan_extra will deallocated
+// 	// when the chan_extra_map is freed. Anyways valgrind complains that the
+// 	// hash table is removing the element with a freed pointer.
+// 	// tal_add_destructor2(ce, destroy_chan_extra, chan_extra_map);
+// 	return &ce->half[dir];
+// }
+
+
 /* Assuming a uniform distribution, what is the chance this f gets through?
  * Here we compute the conditional probability of success for a flow f, given
  * the knowledge that the liquidity is in the range [a,b) and some amount
@@ -111,91 +498,11 @@ static double edge_probability(struct amount_msat min, struct amount_msat max,
 	return amount_msat_less_eq(f,A) ? 1.0 : amount_msat_ratio(numerator,denominator);
 }
 
-// static void destroy_chan_extra(struct chan_extra *ce,
-// 			       struct chan_extra_map *chan_extra_map)
-// {
-// 	chan_extra_map_del(chan_extra_map, ce);
-// }
-
-/* Returns either NULL, or an entry from the hash */
-struct chan_extra_half *
-get_chan_extra_half_by_scid(struct chan_extra_map *chan_extra_map,
-			    const struct short_channel_id scid,
-			    int dir)
-{
-	struct chan_extra *ce;
-
-	ce = chan_extra_map_get(chan_extra_map, scid);
-	if (!ce)
-		return NULL;
-	return &ce->half[dir];
-}
 
-/* Helper if we have a gossmap_chan */
-struct chan_extra_half *
-get_chan_extra_half_by_chan(const struct gossmap *gossmap,
-			    struct chan_extra_map *chan_extra_map,
-			    const struct gossmap_chan *chan,
-			    int dir)
-{
-	return get_chan_extra_half_by_scid(chan_extra_map,
-					   gossmap_chan_scid(gossmap, chan),
-					   dir);
-}
 
-/* Helper to get the chan_extra_half. If it doesn't exist create a new one. */
-struct chan_extra_half *
-get_chan_extra_half_by_chan_verify(
-		const struct gossmap *gossmap,
-		struct chan_extra_map *chan_extra_map,
-		const struct gossmap_chan *chan,
-		int dir)
-{
-	
-	const struct short_channel_id scid = gossmap_chan_scid(gossmap,chan);
-	struct chan_extra_half *h = get_chan_extra_half_by_scid(
-					chan_extra_map,scid,dir);
-	if (!h) {
-		struct amount_sat cap;
-		struct amount_msat cap_msat;
 
-		if (!gossmap_chan_get_capacity(gossmap,chan, &cap))
-			cap = AMOUNT_SAT(0);
-		if (!amount_sat_to_msat(&cap_msat, cap))
-		{
-			SUPERVERBOSE("%s: aborting\n",__PRETTY_FUNCTION__);
-			abort();
-		}
-		h = new_chan_extra_half(chan_extra_map,
-					scid,dir,cap_msat);
-	}
-	return h;
-}
 
-struct chan_extra_half *new_chan_extra_half(struct chan_extra_map *chan_extra_map,
-					    const struct short_channel_id scid,
-					    int dir,
-					    struct amount_msat capacity)
-{
-	struct chan_extra *ce = tal(chan_extra_map, struct chan_extra);
 
-	ce->scid = scid;
-	for (size_t i = 0; i <= 1; i++) {
-		ce->half[i].num_htlcs = 0;
-		ce->half[i].htlc_total = AMOUNT_MSAT(0);
-		ce->half[i].known_min = AMOUNT_MSAT(0);
-		ce->half[i].known_max = capacity;
-	}
-	/* Remove self from map when done */
-	chan_extra_map_add(chan_extra_map, ce);
-	
-	// TODO(eduardo):
-	// Is this desctructor really necessary? the chan_extra will deallocated
-	// when the chan_extra_map is freed. Anyways valgrind complains that the
-	// hash table is removing the element with a freed pointer.
-	// tal_add_destructor2(ce, destroy_chan_extra, chan_extra_map);
-	return &ce->half[dir];
-}
 
 void remove_completed_flow(const struct gossmap *gossmap,
 			   struct chan_extra_map *chan_extra_map,
@@ -276,11 +583,17 @@ void flow_complete(struct flow *flow,
 	flow->amounts = tal_arr(flow, struct amount_msat, tal_count(flow->path));
 	for (int i = tal_count(flow->path) - 1; i >= 0; i--) {
 		const struct chan_extra_half *h
-			= get_chan_extra_half_by_chan_verify(gossmap,
+			= get_chan_extra_half_by_chan(gossmap,
 							chan_extra_map,
 							flow->path[i],
 							flow->dirs[i]);
 		
+		if(!h)
+		{
+			SUPERVERBOSE("%s: aborting\n",__PRETTY_FUNCTION__);
+			abort();
+		}
+		
 		flow->amounts[i] = delivered;
 		flow->success_prob
 			*= edge_probability(h->known_min, h->known_max,
@@ -532,6 +845,12 @@ struct amount_msat flow_set_fee(struct flow **flows)
 	return fee;
 }
 
+/* Helper to access the half chan at flow index idx */
+const struct half_chan *flow_edge(const struct flow *flow, size_t idx)
+{
+	assert(idx < tal_count(flow->path));
+	return &flow->path[idx]->half[flow->dirs[idx]];
+}
 
 #ifndef SUPERVERBOSE_ENABLED
 #undef SUPERVERBOSE
diff --git a/plugins/renepay/flow.h b/plugins/renepay/flow.h
index 9e4fc85be18f..562c6ea65ff5 100644
--- a/plugins/renepay/flow.h
+++ b/plugins/renepay/flow.h
@@ -6,6 +6,16 @@
 #include <common/amount.h>
 #include <common/gossmap.h>
 
+
+// TODO(eduardo): a hard coded constant to indicate a limit on any channel
+// capacity. Channels for which the capacity is unknown (because they are not
+// announced) use this value. It makes sense, because if we don't even know the
+// channel capacity the liquidity could be anything but it will never be greater
+// than the global number of msats.
+// It remains to be checked if this value does not lead to overflow somewhere in
+// the code.
+#define MAX_CAP (AMOUNT_MSAT(21000000*MSAT_PER_BTC))
+
 /* Any implementation needs to keep some data on channels which are
  * in-use (or about which we have extra information).  We use a hash
  * table here, since most channels are not in use. */
@@ -16,6 +26,7 @@
 // (X,dir) the knowledge of (X,!dir) is updated as well.
 struct chan_extra {
 	struct short_channel_id scid;
+	struct amount_msat capacity;
 
 	struct chan_extra_half {
 		/* How many htlcs we've directed through it */
@@ -54,15 +65,133 @@ HTABLE_DEFINE_TYPE(struct chan_extra,
 		   chan_extra_map);
 
 /* Helpers for chan_extra_map */
+/* Channel knowledge invariants:
+ * 	
+ * 	0<=a<=b<=capacity
+ * 	
+ * 	a_inv = capacity-b
+ * 	b_inv = capacity-a
+ * 
+ * where a,b are the known minimum and maximum liquidities, and a_inv and b_inv
+ * are the known minimum and maximum liquidities for the channel in the opposite
+ * direction.
+ * 
+ * Knowledge update operations can be:
+ * 
+ * 1. set liquidity (x)
+ * 	(a,b) -> (x,x)
+ * 	
+ * 	The entropy is minimum here (=0).
+ * 
+ * 2. can send (x):
+ * 	xb = min(x,capacity) 
+ * 	(a,b) -> (max(a,xb),max(b,xb))
+ * 	
+ * 	If x<=a then there is no new knowledge and the entropy remains
+ * 	the same.
+ * 	If x>a the entropy decreases.
+ * 	
+ * 
+ * 3. can't send (x):
+ * 	xb = max(0,x-1)
+ * 	(a,b) -> (min(a,xb),min(b,xb))
+ * 	
+ * 	If x>b there is no new knowledge and the entropy remains.
+ * 	If x<=b then the entropy decreases.
+ * 
+ * 4. sent success (x):
+ * 	(a,b) -> (max(0,a-x),max(0,b-x))
+ * 	
+ * 	If x<=a there is no new knowledge and the entropy remains.
+ * 	If a<x then the entropy decreases.
+ * 
+ * 5. relax (x,y):
+ * 	
+ * 	(a,b) -> (max(0,a-x),min(capacity,b+y))
+ * 	
+ * 	Entropy increases unless it is already maximum.	
+ * */
+
+/* Creates a new chan_extra and adds it to the chan_extra_map. */
+struct chan_extra *new_chan_extra(
+		struct chan_extra_map *chan_extra_map,
+		const struct short_channel_id scid,
+		struct amount_msat capacity);
+
+/* Checks for this chan_extra if the invariants are satisfied. */
+bool chan_extra_check_invariants(struct chan_extra *ce);
+
+/* Checks the entire uncertainty network for invariant violations. */
+bool uncertainty_network_check_invariants(struct chan_extra_map *chan_extra_map);
+
+/* This helper function preserves the uncertainty network invariant after the
+ * knowledge is updated. It assumes that the (channel,!dir) knowledge is
+ * correct. */
+void chan_extra_adjust_half(struct chan_extra *ce,
+			    int dir);
+
+/* Helper to find the min of two amounts */
+static inline struct amount_msat amount_msat_min(
+		struct amount_msat a,
+		struct amount_msat b)
+{
+	return a.millisatoshis < b.millisatoshis ? a : b;
+}
+/* Helper to find the max of two amounts */
+static inline struct amount_msat amount_msat_max(
+		struct amount_msat a,
+		struct amount_msat b)
+{
+	return a.millisatoshis > b.millisatoshis ? a : b;
+}
+
+/* Update the knowledge that this (channel,direction) can send x msat.*/
+void chan_extra_can_send(struct chan_extra_map *chan_extra_map,
+			 struct short_channel_id scid, 
+			 int dir,
+			 struct amount_msat x);
+
+/* Update the knowledge that this (channel,direction) cannot send x msat.*/
+void chan_extra_cannot_send(struct chan_extra_map *chan_extra_map,
+			    struct short_channel_id scid, 
+			    int dir,
+			    struct amount_msat x);
+
+/* Update the knowledge that this (channel,direction) has liquidity x.*/
+void chan_extra_set_liquidity(struct chan_extra_map *chan_extra_map,
+			      struct short_channel_id scid, 
+			      int dir,
+			      struct amount_msat x);
+
+/* Update the knowledge that this (channel,direction) has sent x msat.*/
+void chan_extra_sent_success(struct chan_extra_map *chan_extra_map,
+			     struct short_channel_id scid, 
+			     int dir,
+			     struct amount_msat x);
+			     
+/* Forget a bit about this (channel,direction) state. */
+void chan_extra_relax(struct chan_extra_map *chan_extra_map,
+		      struct short_channel_id scid, 
+		      int dir,
+		      struct amount_msat down,
+		      struct amount_msat up);
+
+/* Mirror the gossmap in the public uncertainty network.
+ * result: Every channel in gossmap must have associated data in chan_extra_map,
+ * while every channel in chan_extra_map is also registered in gossmap.
+ * */
+void uncertainty_network_update(
+		const struct gossmap *gossmap,
+		struct chan_extra_map *chan_extra_map);
+
+
+
+/* Returns either NULL, or an entry from the hash */
 struct chan_extra_half *get_chan_extra_half_by_scid(struct chan_extra_map *chan_extra_map,
 						    const struct short_channel_id scid,
 						    int dir);
-
-struct chan_extra_half *get_chan_extra_half_by_chan(const struct gossmap *gossmap,
-						    struct chan_extra_map *chan_extra_map,
-						    const struct gossmap_chan *chan,
-						    int dir);
-/* Helper to get the chan_extra_half. If it doesn't exist create a new one. */
+/* If the channel is not registered, then a new entry is created. scid must be
+ * present in the gossmap. */
 struct chan_extra_half *
 get_chan_extra_half_by_chan_verify(
 		const struct gossmap *gossmap,
@@ -70,11 +199,11 @@ get_chan_extra_half_by_chan_verify(
 		const struct gossmap_chan *chan,
 		int dir);
 
-/* tal_free() this removes it from chan_extra_map */
-struct chan_extra_half *new_chan_extra_half(struct chan_extra_map *chan_extra_map,
-					    const struct short_channel_id scid,
-					    int dir,
-					    struct amount_msat capacity);
+/* Helper if we have a gossmap_chan */
+struct chan_extra_half *get_chan_extra_half_by_chan(const struct gossmap *gossmap,
+						    struct chan_extra_map *chan_extra_map,
+						    const struct gossmap_chan *chan,
+						    int dir);
 
 /* An actual partial flow. */
 struct flow {
diff --git a/plugins/renepay/mcf.c b/plugins/renepay/mcf.c
index 1b7ce86d762e..fee3d9f6a46c 100644
--- a/plugins/renepay/mcf.c
+++ b/plugins/renepay/mcf.c
@@ -426,12 +426,14 @@ static void linearize_channel(
 		s64 *capacity,
 		s64 *cost)
 {
-	struct chan_extra_half *extra_half = get_chan_extra_half_by_chan_verify(
+	struct chan_extra_half *extra_half = get_chan_extra_half_by_chan(
 							params->gossmap,
 							params->chan_extra_map,
 							c,
 							dir);
 	
+	assert(extra_half);
+	
 	s64 a = extra_half->known_min.millisatoshis/1000,
 	    b = 1 + extra_half->known_max.millisatoshis/1000;
 	
@@ -463,7 +465,7 @@ static void alloc_residual_netork(
 	residual_network->cost = tal_arrz(residual_network,s64,max_num_arcs);	
 	residual_network->potential = tal_arrz(residual_network,s64,max_num_nodes);	
 }
-static void init_residual_netork(
+static void init_residual_network(
 		const struct linear_network * linear_network,
 		struct residual_network* residual_network)
 {
@@ -588,7 +590,7 @@ static void init_linear_network(
 			const struct gossmap_chan *c = gossmap_nth_chan(params->gossmap,
 			                                                node, j, &half);
 			
-			// TODO(eduardo): do we check if the channel is public?
+			// TODO(eduardo): in which case can this be triggered?
 			if (!gossmap_chan_set(c,half))
 				continue;
 				
@@ -1380,7 +1382,7 @@ struct flow** minflow(
 	const u32 target_idx = gossmap_node_idx(params->gossmap,target);
 	const u32 source_idx = gossmap_node_idx(params->gossmap,source);
 	
-	init_residual_netork(linear_network,residual_network);
+	init_residual_network(linear_network,residual_network);
 	
 	// printf("%s: done with allocation and initialization\n",__PRETTY_FUNCTION__);
 	
diff --git a/plugins/renepay/pay.c b/plugins/renepay/pay.c
index cfacab555c42..fd9484a988c8 100644
--- a/plugins/renepay/pay.c
+++ b/plugins/renepay/pay.c
@@ -25,6 +25,35 @@
 /* Set in init */
 struct pay_plugin *pay_plugin;
 
+
+static void renepay_cleanup(struct payment *p);
+
+// TODO(eduardo): check if knowledge is updated after payment success
+// TODO(eduardo): how do we fail a payment
+
+static void debug_knowledge(
+		struct chan_extra_map* chan_extra_map,
+		const char*fname)
+{
+	FILE *f = fopen(fname,"a");
+	fprintf(f,"Knowledge:\n");
+	struct chan_extra_map_iter it;
+	for(struct chan_extra *ch = chan_extra_map_first(chan_extra_map,&it);
+	    ch;
+	    ch=chan_extra_map_next(chan_extra_map,&it))
+	{
+		const char *scid_str = 
+			type_to_string(tmpctx,struct short_channel_id,&ch->scid);
+		for(int dir=0;dir<2;++dir)
+		{
+			fprintf(f,"%s[%d]:(%s,%s)\n",scid_str,dir,
+				type_to_string(tmpctx,struct amount_msat,&ch->half[dir].known_min),
+				type_to_string(tmpctx,struct amount_msat,&ch->half[dir].known_max));
+		}
+	}
+	fclose(f);
+}
+
 static void debug_payflows(struct pay_flow **flows, const char* fname)
 {
 	FILE *f = fopen(fname,"a");
@@ -88,6 +117,7 @@ void amount_msat_reduce_(struct amount_msat *dst,
 		   dstname, type_to_string(tmpctx, struct amount_msat, dst));
 }
 
+
 #if DEVELOPER
 static void memleak_mark(struct plugin *p, struct htable *memtable)
 {
@@ -121,6 +151,7 @@ static const char *init(struct plugin *p,
 	pay_plugin->gossmap = gossmap_load(pay_plugin,
 					   GOSSIP_STORE_FILENAME,
 					   &num_channel_updates_rejected);
+	
 	if (!pay_plugin->gossmap)
 		plugin_err(p, "Could not load gossmap %s: %s",
 			   GOSSIP_STORE_FILENAME, strerror(errno));
@@ -128,7 +159,9 @@ static const char *init(struct plugin *p,
 		plugin_log(p, LOG_DBG,
 			   "gossmap ignored %zu channel updates",
 			   num_channel_updates_rejected);
-
+			   
+	uncertainty_network_update(pay_plugin->gossmap,
+				   pay_plugin->chan_extra_map);
 #if DEVELOPER
 	plugin_set_memleak_handler(p, memleak_mark);
 #endif
@@ -136,7 +169,17 @@ static const char *init(struct plugin *p,
 	return NULL;
 }
 
-// TODO(eduardo): check this
+/* Create a entry for channel knowledge. */
+// static struct chan_extra* chan_knowledge_new(
+// 		struct gossmap* gossmap,
+// 		struct chan_extra_map *chan_extra_map,
+// 		struct short_channel_id scid)
+// {
+// 	struct chan_extra *chan = chan_extra_map_get(chan_extra_map,scid);
+// 	
+// 	return chan;
+// }
+		
 // TODO(eduardo): remember that if [a,b] is fixed on (c,dir) then [cap-b,cap-a]
 // is fixed on (c,!dir)
 // TODO(eduardo): I think this function does too many things.  I would prefer to
@@ -144,103 +187,108 @@ static const char *init(struct plugin *p,
 // improves the knowledge and also checks the bounds. But we also need to have
 // the possiblity to relax the knowledge in another function.
 /* We know something about this channel!  Update it! */
-static void chan_update_knowledge(struct payment *p,
-				 struct short_channel_id scid,
-				 int dir,
-				 const struct amount_msat *min_capacity,
-				 const struct amount_msat *max_capacity)
-{
-	plugin_log(pay_plugin->plugin,LOG_DBG,"Updating channel: %s [%s, %s]",
-		type_to_string(tmpctx,struct short_channel_id, &scid),
-		min_capacity ? type_to_string(tmpctx,struct amount_msat, min_capacity) : "-",
-		max_capacity ? type_to_string(tmpctx,struct amount_msat, max_capacity) : "-");
-	
-	
-	struct chan_extra_half *h;
-
-	/* This is assumed */
-	if (min_capacity && max_capacity)
-		assert(amount_msat_greater_eq(*max_capacity, *min_capacity));
-	
-	h = get_chan_extra_half_by_scid(pay_plugin->chan_extra_map, scid, dir);
-	if (!h)
-		h = new_chan_extra_half(pay_plugin->chan_extra_map, scid, dir,
-					*max_capacity);
-	if (min_capacity && amount_msat_greater(*min_capacity, h->known_min))
-		h->known_min = *min_capacity;
-	if (max_capacity && amount_msat_less(*max_capacity, h->known_max))
-		h->known_max = *max_capacity;
-
-	/* If we min > max, it means our previous assumptions are wrong
-	 * (i.e. things changed, or a we assumed full capacity for a routehint
-	 *  which didn't have it!) */
-	if (amount_msat_greater(h->known_min, h->known_max)) {
-		plugin_log(pay_plugin->plugin, LOG_BROKEN,
-			   "Updated %s capacity %s, now %s-%s! Resetting.",
-			   type_to_string(tmpctx, struct short_channel_id, &scid),
-			   min_capacity ? "min" : "max",
-			   type_to_string(tmpctx, struct amount_msat, &h->known_min),
-			   type_to_string(tmpctx, struct amount_msat, &h->known_max));
-
-		/* OK, assume *old* information is wrong: we can't have
-		 * just set both, since we assert() those are correct. */
-		if (min_capacity) {
-			const struct gossmap_chan *c;
-			struct amount_sat cap;
-
-			/* It might be a local channel; if we don't know better,
-			 * we reset max to infinite */
-			c = gossmap_find_chan(pay_plugin->gossmap, &scid);
-			if (!c
-			    || !gossmap_chan_get_capacity(pay_plugin->gossmap, c, &cap)
-			    || !amount_sat_to_msat(&h->known_max, cap))
-				h->known_max = p->maxspend;
-			plugin_log(pay_plugin->plugin, LOG_UNUSUAL,
-				   "... setting max to capacity (%s)",
-				   type_to_string(tmpctx, struct amount_msat,
-						  &h->known_max));
-		} else {
-			h->known_min = AMOUNT_MSAT(0);
-			plugin_log(pay_plugin->plugin, LOG_UNUSUAL,
-				   "... setting min to 0msat");
-		}
-	}
-}
-
-static void add_localchan(struct payment *p,
+// static void chan_update_knowledge(struct payment *p,
+// 				 struct short_channel_id scid,
+// 				 int dir,
+// 				 const struct amount_msat *min_capacity,
+// 				 const struct amount_msat *max_capacity)
+// {
+// 	plugin_log(pay_plugin->plugin,LOG_DBG,"Updating channel: %s [%s, %s]",
+// 		type_to_string(tmpctx,struct short_channel_id, &scid),
+// 		min_capacity ? type_to_string(tmpctx,struct amount_msat, min_capacity) : "-",
+// 		max_capacity ? type_to_string(tmpctx,struct amount_msat, max_capacity) : "-");
+// 	
+// 	
+// 	struct chan_extra_half *h;
+// 
+// 	/* This is assumed */
+// 	if (min_capacity && max_capacity)
+// 		assert(amount_msat_greater_eq(*max_capacity, *min_capacity));
+// 	
+// 	h = get_chan_extra_half_by_scid(pay_plugin->chan_extra_map, scid, dir);
+// 	if (!h)
+// 		h = new_chan_extra_half(pay_plugin->chan_extra_map, scid, dir,
+// 					*max_capacity);
+// 	if (min_capacity && amount_msat_greater(*min_capacity, h->known_min))
+// 		h->known_min = *min_capacity;
+// 	if (max_capacity && amount_msat_less(*max_capacity, h->known_max))
+// 		h->known_max = *max_capacity;
+// 
+// 	/* If we min > max, it means our previous assumptions are wrong
+// 	 * (i.e. things changed, or a we assumed full capacity for a routehint
+// 	 *  which didn't have it!) */
+// 	if (amount_msat_greater(h->known_min, h->known_max)) {
+// 		plugin_log(pay_plugin->plugin, LOG_BROKEN,
+// 			   "Updated %s capacity %s, now %s-%s! Resetting.",
+// 			   type_to_string(tmpctx, struct short_channel_id, &scid),
+// 			   min_capacity ? "min" : "max",
+// 			   type_to_string(tmpctx, struct amount_msat, &h->known_min),
+// 			   type_to_string(tmpctx, struct amount_msat, &h->known_max));
+// 
+// 		/* OK, assume *old* information is wrong: we can't have
+// 		 * just set both, since we assert() those are correct. */
+// 		if (min_capacity) {
+// 			const struct gossmap_chan *c;
+// 			struct amount_sat cap;
+// 
+// 			/* It might be a local channel; if we don't know better,
+// 			 * we reset max to infinite */
+// 			c = gossmap_find_chan(pay_plugin->gossmap, &scid);
+// 			if (!c
+// 			    || !gossmap_chan_get_capacity(pay_plugin->gossmap, c, &cap)
+// 			    || !amount_sat_to_msat(&h->known_max, cap))
+// 				h->known_max = p->maxspend;
+// 			plugin_log(pay_plugin->plugin, LOG_UNUSUAL,
+// 				   "... setting max to capacity (%s)",
+// 				   type_to_string(tmpctx, struct amount_msat,
+// 						  &h->known_max));
+// 		} else {
+// 			h->known_min = AMOUNT_MSAT(0);
+// 			plugin_log(pay_plugin->plugin, LOG_UNUSUAL,
+// 				   "... setting min to 0msat");
+// 		}
+// 	}
+// }
+
+static void add_hintchan(struct payment *p,
 			  const struct node_id *src,
 			  const struct node_id *dst,
 			  u16 cltv_expiry_delta,
 			  const struct short_channel_id scid,
 			  u32 fee_base_msat,
 			  u32 fee_proportional_millionths,
-			  struct amount_msat maxspend)
+			  struct amount_msat amount)
 {
 	int dir = node_id_cmp(src, dst) < 0 ? 0 : 1;
-	/* FIXME: features? */
-	gossmap_local_addchan(p->local_gossmods, src, dst, &scid, NULL);
-
-	gossmap_local_updatechan(p->local_gossmods,
-				 &scid,
-				 /* We assume any HTLC is allowed */
-				 AMOUNT_MSAT(0), maxspend,
-				 fee_base_msat, fee_proportional_millionths,
-				 cltv_expiry_delta,
-				 true,
-				 dir);
+	
+	struct chan_extra *ce = chan_extra_map_get(pay_plugin->chan_extra_map,
+						   scid);
+	
+	if(!ce)
+	{
+		/* this channel is not public, we don't know his capacity */
+		ce = new_chan_extra(pay_plugin->chan_extra_map,
+				    scid,
+				    MAX_CAP);
+		/* FIXME: features? */
+		gossmap_local_addchan(p->local_gossmods, src, dst, &scid, NULL);
+		gossmap_local_updatechan(p->local_gossmods,
+					 &scid,
+					 /* We assume any HTLC is allowed */
+					 AMOUNT_MSAT(0), MAX_CAP,
+					 fee_base_msat, fee_proportional_millionths,
+					 cltv_expiry_delta,
+					 true,
+					 dir);
+	}
+	
 	/* We know (assume!) something about this channel: that it has at
 	 * sufficient capacity. */
-	// TODO(eduardo): if this channel was already in our list then we need
-	// to update the knowledge accordingly. I think it would be better to
-	// handle this with an update_lower_bound function.
-	plugin_log(pay_plugin->plugin,LOG_DBG,"add_localchan: %s [maxspend %s]",
-		type_to_string(tmpctx,struct short_channel_id, &scid),
-		type_to_string(tmpctx,struct amount_msat, &p->maxspend));
-	chan_update_knowledge(p, scid, dir, &p->maxspend, &p->maxspend);
+	chan_extra_can_send(pay_plugin->chan_extra_map,scid,dir,amount);
 }
 
 /* Add routehints provided by bolt11 */
-static void add_routehints(struct payment *p)
+static void uncertainty_network_add_routehints(struct payment *p)
 {
 	struct bolt11 *b11;
 	char *fail;
@@ -258,7 +306,10 @@ static void add_routehints(struct payment *p)
 		const struct route_info *r = b11->routes[i];
 		const struct node_id *end = & p->dest;
 		for (int j = tal_count(r)-1; j >= 0; j--) {
-			add_localchan(p, &r[j].pubkey, end,
+			// TODO(eduardo): amount to send to the add_hintchan
+			// should consider fees.
+			
+			add_hintchan(p, &r[j].pubkey, end,
 				      r[j].cltv_expiry_delta,
 				      r[j].short_channel_id,
 				      r[j].fee_base_msat,
@@ -271,7 +322,7 @@ static void add_routehints(struct payment *p)
 
 /* listpeerchannels gives us the certainty on local channels' capacity.  Of course,
  * this is racy and transient, but better than nothing! */
-static bool update_capacities_from_listpeerchannels(
+static bool update_uncertainty_network_from_listpeerchannels(
 		struct plugin *plugin,
 		struct payment *p,
 		const char *buf,
@@ -314,20 +365,31 @@ static bool update_capacities_from_listpeerchannels(
 			continue;
 		}
 		
-		const jsmntok_t *spendabletok, *dirtok,*statetok;
-		struct amount_msat spendable;
+		const jsmntok_t *spendabletok, *dirtok,*statetok, *totaltok, 
+			*peeridtok;
+		struct amount_msat spendable,capacity;
 		int dir;
 		
+		const struct node_id src=pay_plugin->my_id;
+		struct node_id dst;
+		
 		spendabletok = json_get_member(buf, channel, "spendable_msat");
 		dirtok = json_get_member(buf, channel, "direction");
 		statetok = json_get_member(buf, channel, "state");
+		totaltok = json_get_member(buf, channel, "total_msat");
+		peeridtok = json_get_member(buf,channel,"peer_id");
 		
-		if(spendabletok==NULL || dirtok==NULL || statetok==NULL)
+		if(spendabletok==NULL || dirtok==NULL || statetok==NULL || 
+		   totaltok==NULL || peeridtok==NULL)
 			goto malformed;
 		if (!json_to_msat(buf, spendabletok, &spendable))
 			goto malformed;
+		if (!json_to_msat(buf, totaltok, &capacity))
+			goto malformed;
 		if (!json_to_int(buf, dirtok,&dir))
 			goto malformed;
+		if(!json_to_node_id(buf,peeridtok,&dst))
+			goto malformed;
 			
 		/* Don't report opening/closing channels */
 		if (!json_tok_streq(buf, statetok, "CHANNELD_NORMAL")) {
@@ -335,8 +397,37 @@ static bool update_capacities_from_listpeerchannels(
 			continue;
 		}
 		
+		struct chan_extra *ce = chan_extra_map_get(pay_plugin->chan_extra_map,
+							   scid);
+			  
+		if(!ce)
+		{
+			/* this channel is not public, but it belongs to us */
+			ce = new_chan_extra(pay_plugin->chan_extra_map,
+					    scid,
+					    capacity);
+			/* FIXME: features? */
+			gossmap_local_addchan(p->local_gossmods, &src, &dst, &scid, NULL);
+			gossmap_local_updatechan(p->local_gossmods,
+						 &scid,
+						 
+						 /* TODO(eduardo): does it
+						  * matter to consider HTLC
+						  * limits in our own channel? */
+						 AMOUNT_MSAT(0),capacity,
+						 
+						 /* fees = */0,0,
+						 
+						 /* TODO(eduardo): does it
+						  * matter to set this delay? */
+						 /*delay=*/0,
+						 true,
+						 dir);
+		}
+		
 		/* We know min and max liquidity exactly now! */
-		chan_update_knowledge(p, scid, dir, &spendable, &spendable);
+		chan_extra_set_liquidity(pay_plugin->chan_extra_map,
+					 scid,dir,spendable);
 	}
 	return true;
 
@@ -426,6 +517,7 @@ static struct command_result *waitsendpay_succeeded(struct command *cmd,
 				  p->total_sent);
 	json_add_string(response, "status", "complete");
 	json_add_node_id(response, "destination", &p->dest);
+	renepay_cleanup(p);
 	return command_finished(cmd, response);
 }
 
@@ -674,9 +766,12 @@ static struct command_result *waitsendpay_failed(struct command *cmd,
 	/* All these parts succeeded, so we know something about min
 	 * capacity! */
 	for (size_t i = 0; i < erridx; i++)
-		chan_update_knowledge(p, flow->path_scids[i], flow->path_dirs[i], &flow->amounts[i],
-				     NULL);
-
+	{
+		chan_extra_can_send(pay_plugin->chan_extra_map,
+				    flow->path_scids[i],
+				    flow->path_dirs[i],
+				    flow->amounts[i]);
+	}
 	switch ((enum onion_wire)onionerr) {
 	/* These definitely mean eliminate channel */
 	case WIRE_PERMANENT_CHANNEL_FAILURE:
@@ -725,7 +820,10 @@ static struct command_result *waitsendpay_failed(struct command *cmd,
 		paynote(p, "... assuming that max capacity is %s",
 			type_to_string(tmpctx, struct amount_msat,
 				       &max_possible));
-		chan_update_knowledge(p, errscid, flow->path_dirs[erridx], NULL, &max_possible);
+		chan_extra_cannot_send(pay_plugin->chan_extra_map,
+				       flow->path_scids[erridx],
+				       flow->path_dirs[erridx],
+				       flow->amounts[erridx]);
 		goto done;
 	}
 
@@ -952,18 +1050,34 @@ static struct command_result *
 listpeerchannels_done(struct command *cmd, const char *buf,
 	       const jsmntok_t *result, struct payment *p)
 {
-	// FILE *f = fopen("/tmp/afile","wb");
-	// fwrite(json_tok_full(buf,result),json_tok_full_len(result),1,f);
-	// fclose(f);
-	if (!update_capacities_from_listpeerchannels(cmd->plugin, p, buf, result))
+	if (!update_uncertainty_network_from_listpeerchannels(cmd->plugin, p, buf, result))
 		return command_fail(cmd, LIGHTNINGD,
 				    "listpeerchannels malformed: %.*s",
 				    json_tok_full_len(result),
 				    json_tok_full(buf, result));
-
+	
+	// So we have all localmods data, now we apply it. Only once per
+	// payment.
+	gossmap_apply_localmods(pay_plugin->gossmap,p->local_gossmods);
 	return try_paying(cmd, p, true);
 }
 
+/* Either the payment succeeded or failed, we need to cleanup/set the plugin
+ * into a valid state before the next payment. */
+static void renepay_cleanup(struct payment *p)
+{
+	/* Always remove our local mods (routehints) so others can use
+	 * gossmap. We do this only after the payment completes. */
+	gossmap_remove_localmods(pay_plugin->gossmap,
+				 p->local_gossmods);
+				 
+	// TODO(eduardo): can we really free the local_gossmods?
+	// I don't thinkg they'll be needed afterwards
+	tal_free(p->local_gossmods);
+	// TODO(eduardo): Am I missing other things?
+	// TODO(eduardo): When do we call this?
+}
+
 static void destroy_payment(struct payment *p)
 {
 	list_del_from(&pay_plugin->payments, &p->list);
@@ -1083,7 +1197,12 @@ static struct command_result *json_pay(struct command *cmd,
 	p->use_shadow = true;
 #endif
  	
+	
 	plugin_log(pay_plugin->plugin,LOG_DBG,"Starting renepay");
+	bool gossmap_changed = gossmap_refresh(pay_plugin->gossmap, NULL);
+	
+	// TODO(eduardo): remove this
+	debug_knowledge(pay_plugin->chan_extra_map,MYLOG);
 	
 	p->base_fee_penalty=*base_fee_penalty;
 	p->prob_cost_factor= *prob_cost_factor;
@@ -1183,7 +1302,6 @@ static struct command_result *json_pay(struct command *cmd,
 		} else
 			invmsat = NULL;
 
-		gossmap_refresh(pay_plugin->gossmap, NULL);
 		node_id_from_pubkey(&p->dest, b12->offer_node_id);
 		p->payment_hash = *b12->invoice_payment_hash;
 		if (b12->invreq_recurrence_counter && !p->label)
@@ -1266,9 +1384,39 @@ static struct command_result *json_pay(struct command *cmd,
 	if (time_now().ts.tv_sec > invexpiry)
 		return command_fail(cmd, PAY_INVOICE_EXPIRED, "Invoice expired");
 	
+	
+	/* To construct the uncertainty network we need to perform the following
+	 * steps: 
+	 * 1. check that there is a 1-to-1 map between channels in gossmap
+	 * and the uncertainty network. We call `uncertainty_network_update`
+	 * 
+	 * 2. add my local channels that could be private. 
+	 * We call `update_uncertainty_network_from_listpeerchannels`.
+	 * 
+	 * 3. add hidden/private channels listed in the routehints.
+	 * We call `uncertainty_network_add_routehints`.
+	 * 
+	 * 4. check the uncertainty network invariants.
+	 * */
+	if(gossmap_changed)
+		uncertainty_network_update(pay_plugin->gossmap,
+					   pay_plugin->chan_extra_map);
+	
+	
+	// TODO(eduardo)
+	// we should remove local_gossmods from the uncertainty network
+	// after the payment is completed?
+	// 
 	// TODO(eduardo): are there route hints for B12?
+	// Add any extra hidden channel revealed by the routehints to the uncertainty network.
 	if(invstr_is_b11)
-		add_routehints(p);
+		uncertainty_network_add_routehints(p);
+	
+	if(!uncertainty_network_check_invariants(pay_plugin->chan_extra_map))
+		plugin_log(pay_plugin->plugin, 
+			   LOG_BROKEN,
+			   "uncertainty network invariants are violated");
+		
 	
 	struct out_req *req;
 
diff --git a/plugins/renepay/pay.h b/plugins/renepay/pay.h
index 6095b8d53d63..1f7f52db91bc 100644
--- a/plugins/renepay/pay.h
+++ b/plugins/renepay/pay.h
@@ -46,7 +46,8 @@ struct payment {
 	struct timeabs start_time;
 
 	/* Localmods to apply to gossip_map for our own use. */
-	// TODO(eduardo): is it used?
+	// TODO(eduardo): shall we keep this data even after the payment
+	// ends?
 	struct gossmap_localmods *local_gossmods;
 
 	/* invstring (bolt11 or bolt12) */
diff --git a/plugins/renepay/pay_flow.c b/plugins/renepay/pay_flow.c
index 2186c5c75cd3..b772e2fd99c8 100644
--- a/plugins/renepay/pay_flow.c
+++ b/plugins/renepay/pay_flow.c
@@ -358,13 +358,6 @@ struct pay_flow **get_payflows(struct payment *p,
 		plugin_err(pay_plugin->plugin, "Failed to refresh gossmap: %s",
 			   strerror(errno));
 	
-	// TODO(eduardo): remember that local channels have
-	// know_min=know_max=liquidity. Is this related to local_gossmods?
-	// 
-	// TODO(eduardo): where is p->local_gossmods set?
-	gossmap_apply_localmods(pay_plugin->gossmap, p->local_gossmods);
-	
-	// TODO(eduardo): where is p->disabled set?
 	disabled = make_disabled_bitmap(tmpctx, pay_plugin->gossmap, p->disabled);
 	src = gossmap_find_node(pay_plugin->gossmap, &pay_plugin->my_id);
 	if (!src) {
@@ -477,9 +470,6 @@ struct pay_flow **get_payflows(struct payment *p,
 	}
 
 out:
-	/* Always remove our local mods (routehints) so others can use
-	 * gossmap */
-	gossmap_remove_localmods(pay_plugin->gossmap, p->local_gossmods);
 	return pay_flows;
 
 fail:
diff --git a/plugins/renepay/test/Makefile b/plugins/renepay/test/Makefile
index a83aa1386cb8..b012ff8c809b 100644
--- a/plugins/renepay/test/Makefile
+++ b/plugins/renepay/test/Makefile
@@ -23,4 +23,4 @@ PLUGIN_RENEPAY_TEST_COMMON_OBJS :=		\
 
 $(PLUGIN_RENEPAY_TEST_PROGRAMS): $(CCAN_OBJS) $(BITCOIN_OBJS) $(WIRE_OBJS) $(PLUGIN_RENEPAY_TEST_COMMON_OBJS)
 
-check-units: $(PLUGIN_RENEPAY_TEST_PROGRAMS:%=unittest/%)
+check-renepay: $(PLUGIN_RENEPAY_TEST_PROGRAMS:%=unittest/%)
diff --git a/plugins/renepay/test/run-mcf-diamond.c b/plugins/renepay/test/run-mcf-diamond.c
index 2e48b3a2540d..69938327f4e4 100644
--- a/plugins/renepay/test/run-mcf-diamond.c
+++ b/plugins/renepay/test/run-mcf-diamond.c
@@ -138,17 +138,17 @@ int main(int argc, char *argv[])
 	chan_extra_map = tal(tmpctx, struct chan_extra_map);
 	chan_extra_map_init(chan_extra_map);
 	/* The local chans have no "capacity", so set them manually. */
-	new_chan_extra_half(chan_extra_map,
-			    scid12, 0,
+	new_chan_extra(chan_extra_map,
+			    scid12,
 			    AMOUNT_MSAT(10000000));
-	new_chan_extra_half(chan_extra_map,
-			    scid24, 0,
+	new_chan_extra(chan_extra_map,
+			    scid24,
 			    AMOUNT_MSAT(10000000));
-	new_chan_extra_half(chan_extra_map,
-			    scid13, 0,
+	new_chan_extra(chan_extra_map,
+			    scid13,
 			    AMOUNT_MSAT(5000000));
-	new_chan_extra_half(chan_extra_map,
-			    scid34, 0,
+	new_chan_extra(chan_extra_map,
+			    scid34,
 			    AMOUNT_MSAT(5000000));
 
 	struct flow **flows;
diff --git a/plugins/renepay/test/run-mcf.c b/plugins/renepay/test/run-mcf.c
index f16dddd9ff02..29714c55b5f4 100644
--- a/plugins/renepay/test/run-mcf.c
+++ b/plugins/renepay/test/run-mcf.c
@@ -300,6 +300,8 @@ int main(int argc, char *argv[])
 
 	chan_extra_map = tal(tmpctx, struct chan_extra_map);
 	chan_extra_map_init(chan_extra_map);
+	uncertainty_network_update(gossmap,chan_extra_map);	
+	
 	flows = minflow(tmpctx, gossmap,
 			gossmap_find_node(gossmap, &l1),
 			gossmap_find_node(gossmap, &l3),
@@ -384,7 +386,7 @@ int main(int argc, char *argv[])
 	remove_completed_flow_set(gossmap, chan_extra_map, flows);
 
 	/* The local chans have no "capacity", so set it manually. */
-	new_chan_extra_half(chan_extra_map, scid13, 0,
+	new_chan_extra(chan_extra_map, scid13,
 			    AMOUNT_MSAT(400000000));
 
 	// flows = minflow(tmpctx, gossmap,
diff --git a/plugins/renepay/test/run-not_mcf-gossmap.c b/plugins/renepay/test/run-not_mcf-gossmap.c
deleted file mode 100644
index 307faf444bdf..000000000000
--- a/plugins/renepay/test/run-not_mcf-gossmap.c
+++ /dev/null
@@ -1,132 +0,0 @@
-#include "config.h"
-
-#include <ccan/array_size/array_size.h>
-#include <ccan/err/err.h>
-#include <ccan/opt/opt.h>
-#include <ccan/tal/str/str.h>
-#include <ccan/read_write_all/read_write_all.h>
-#include <common/bigsize.h>
-#include <common/channel_id.h>
-#include <common/node_id.h>
-#include <common/setup.h>
-#include <common/type_to_string.h>
-#include <common/wireaddr.h>
-#include <inttypes.h>
-#include <stdio.h>
-
-//static bool print_enable = true;
-//#define SUPERVERBOSE(...) do { if (print_enable) printf(__VA_ARGS__); } while(0)
-
-  #include "../mcf.c"
-  #include "../flow.c"
-
-/* AUTOGENERATED MOCKS START */
-/* Generated stub for fromwire_bigsize */
-bigsize_t fromwire_bigsize(const u8 **cursor UNNEEDED, size_t *max UNNEEDED)
-{ fprintf(stderr, "fromwire_bigsize called!\n"); abort(); }
-/* Generated stub for fromwire_channel_id */
-bool fromwire_channel_id(const u8 **cursor UNNEEDED, size_t *max UNNEEDED,
-			 struct channel_id *channel_id UNNEEDED)
-{ fprintf(stderr, "fromwire_channel_id called!\n"); abort(); }
-/* Generated stub for fromwire_wireaddr */
-bool fromwire_wireaddr(const u8 **cursor UNNEEDED, size_t *max UNNEEDED, struct wireaddr *addr UNNEEDED)
-{ fprintf(stderr, "fromwire_wireaddr called!\n"); abort(); }
-/* Generated stub for towire_bigsize */
-void towire_bigsize(u8 **pptr UNNEEDED, const bigsize_t val UNNEEDED)
-{ fprintf(stderr, "towire_bigsize called!\n"); abort(); }
-/* Generated stub for towire_channel_id */
-void towire_channel_id(u8 **pptr UNNEEDED, const struct channel_id *channel_id UNNEEDED)
-{ fprintf(stderr, "towire_channel_id called!\n"); abort(); }
-/* Generated stub for towire_wireaddr */
-void towire_wireaddr(u8 **pptr UNNEEDED, const struct wireaddr *addr UNNEEDED)
-{ fprintf(stderr, "towire_wireaddr called!\n"); abort(); }
-/* AUTOGENERATED MOCKS END */
-
-/* not_mcf sets NDEBUG, so assert() is useless */
-#define ASSERT(x) do { if (!(x)) abort(); } while(0)
-
-static void print_flows(const char *desc,
-			const struct gossmap *gossmap,
-			struct flow **flows)
-{
-	struct amount_msat total_fee = AMOUNT_MSAT(0),
-		total_delivered = AMOUNT_MSAT(0);
-
-	printf("%s: %zu subflows\n", desc, tal_count(flows));
-	for (size_t i = 0; i < tal_count(flows); i++) {
-		struct amount_msat fee, delivered;
-		printf("   ");
-		for (size_t j = 0; j < tal_count(flows[i]->path); j++) {
-			struct short_channel_id scid
-				= gossmap_chan_scid(gossmap,
-						    flows[i]->path[j]);
-			printf("%s%s", j ? "->" : "",
-			       type_to_string(tmpctx, struct short_channel_id, &scid));
-		}
-		delivered = flows[i]->amounts[tal_count(flows[i]->amounts)-1];
-		if (!amount_msat_sub(&fee, flows[i]->amounts[0], delivered))
-			abort();
-		printf(" prob %.2f, %s delivered with fee %s\n",
-		       flows[i]->success_prob,
-		       type_to_string(tmpctx, struct amount_msat, &delivered),
-		       type_to_string(tmpctx, struct amount_msat, &fee));
-		if (!amount_msat_add(&total_fee, total_fee, fee))
-			abort();
-		if (!amount_msat_add(&total_delivered, total_delivered, delivered))
-			abort();
-	}
-	printf("Delivered %s at fee %s\n",
-	       type_to_string(tmpctx, struct amount_msat, &total_delivered),
-	       type_to_string(tmpctx, struct amount_msat, &total_fee));
-}
-
-int main(int argc, char *argv[])
-{
-	struct gossmap *gossmap;
-	struct flow **flows;
-	struct node_id srcid, dstid;
-	struct gossmap_node *src, *dst;
-	struct amount_msat amount;
-	struct chan_extra_map *chan_extra_map;
-	bool verbose = false;
-
-	common_setup(argv[0]);
-	opt_register_noarg("-v|--verbose", opt_set_bool, &verbose,
-			   "Increase verbosity");
-
-	opt_parse(&argc, argv, opt_log_stderr_exit);
-
-	if (argc != 5)
-		errx(1, "Usage: %s <gossmap> <srcnode> <dstnode> <amount>", argv[0]);
-
-	gossmap = gossmap_load(tmpctx, argv[1], NULL);
-	assert(gossmap);
-
-	if (!node_id_from_hexstr(argv[2], strlen(argv[2]), &srcid)
-	    || !node_id_from_hexstr(argv[3], strlen(argv[3]), &dstid))
-		errx(1, "Usage: %s <srcnode> <dstnode>", argv[0]);
-
-	src = gossmap_find_node(gossmap, &srcid);
-	assert(src);
-	dst = gossmap_find_node(gossmap, &dstid);
-	assert(dst);
-	assert(src != dst);
-
-	amount = amount_msat(atol(argv[4]));
-	assert(!amount_msat_eq(amount, AMOUNT_MSAT(0)));
-
-	chan_extra_map = tal(tmpctx, struct chan_extra_map);
-	chan_extra_map_init(chan_extra_map);
-
-	flows = minflow(tmpctx, gossmap, src, dst,
-			chan_extra_map, NULL,
-			amount,
-			 /* max_fee = */ AMOUNT_MSAT(1000000), // 1k sats
-			 /* min probability = */ 0.1, // 10%
-			 /* delay fee factor = */ 1,
-			 /* base fee penalty */ 1,
-			 /* prob cost factor = */ 10);
-
-	print_flows("Flows", gossmap, flows);
-	common_shutdown();
-}

From 5f37cad757bbf760ee9f89d5d1793189f4c33562 Mon Sep 17 00:00:00 2001
From: Lagrang3 <eduardo.quintana@pm.me>
Date: Mon, 29 May 2023 13:20:00 +0100
Subject: [PATCH 39/64] renepay: fix seg. fault for waitsendpay_fail

---
 plugins/renepay/pay.c | 58 ++++++++++++++++++++++++++++++++++++-------
 1 file changed, 49 insertions(+), 9 deletions(-)

diff --git a/plugins/renepay/pay.c b/plugins/renepay/pay.c
index fd9484a988c8..0005be12c7fa 100644
--- a/plugins/renepay/pay.c
+++ b/plugins/renepay/pay.c
@@ -61,6 +61,13 @@ static void debug_payflows(struct pay_flow **flows, const char* fname)
 	fclose(f);
 }
 
+static void debug_exec_branch(int lineno, const char* fun, const char* fname)
+{
+	FILE *f = fopen(fname,"a");
+	fprintf(f,"executing line: %d (%s)\n",lineno,fun);
+	fclose(f);
+}
+
 static void debug_outreq(const struct out_req *req, const char*fname)
 {
 	FILE *f = fopen(fname,"a");
@@ -81,6 +88,26 @@ static void debug_call(const char* fun, const char* fname)
 	fclose(f);
 }
 
+static void debug_reply(const char* buf, const char*fname)
+{
+	FILE *f = fopen(fname,"a");
+	fprintf(f,"%.*s\n\n",(int)tal_count(buf),buf);
+	fclose(f);
+}
+
+static void debug(const char* fname, const char *fmt, ...)
+{
+	FILE *f = fopen(fname,"a");
+	
+	va_list args;
+	va_start(args, fmt);
+	
+	vfprintf(f,fmt,args);
+	
+	va_end(args);
+	fclose(f);
+}
+
 void paynote(struct payment *p, const char *fmt, ...)
 {
 	va_list ap;
@@ -701,6 +728,8 @@ static struct command_result *waitsendpay_failed(struct command *cmd,
 {
 	plugin_log(pay_plugin->plugin,LOG_DBG,"calling waitsendpay_failed");
 	debug_call(__PRETTY_FUNCTION__,MYLOG);
+	debug_reply(buf,MYLOG);
+	
 	struct payment *p = flow->payment;
 	u64 errcode;
 	struct command_result *ret;
@@ -715,17 +744,21 @@ static struct command_result *waitsendpay_failed(struct command *cmd,
 
 	switch (errcode) {
 	case PAY_UNPARSEABLE_ONION:
+		debug_exec_branch(__LINE__,__PRETTY_FUNCTION__,MYLOG);
 		ret = handle_unhandleable_error(p, flow, "unparsable onion reply");
 		if (ret)
 			return ret;
 		goto done;
 	case PAY_DESTINATION_PERM_FAIL:
+		debug_exec_branch(__LINE__,__PRETTY_FUNCTION__,MYLOG);
 		return command_fail(cmd, PAY_DESTINATION_PERM_FAIL,
 				    "Got an final failure from destination");
 	case PAY_TRY_OTHER_ROUTE:
+		debug_exec_branch(__LINE__,__PRETTY_FUNCTION__,MYLOG);
 		break;
 
 	default:
+		debug_exec_branch(__LINE__,__PRETTY_FUNCTION__,MYLOG);
 		plugin_err(cmd->plugin,
 			   "Unexpected errcode from waitsendpay: %.*s",
 			   json_tok_full_len(err), json_tok_full(buf, err));
@@ -754,9 +787,13 @@ static struct command_result *waitsendpay_failed(struct command *cmd,
 	failcodetok = json_get_member(buf, datatok, "failcode");
 	json_to_u32(buf, failcodetok, &onionerr);
 
-	msgtok = json_get_member(buf, datatok, "message");
+	msgtok = json_get_member(buf, err, "message");
 	rawoniontok = json_get_member(buf, datatok, "raw_message");
-
+	
+	// const char *scid_str = type_to_string(tmpctx,struct short_channel_id,&errscid);
+	// debug(MYLOG,"failcode: %d, onion error: %s, err_idx: %d, scid: %s\n",
+	// 	onionerr,onion_wire_name(onionerr),erridx,scid_str);
+	
 	paynote(p, "onion error %s from node #%u %s: %.*s",
 		onion_wire_name(onionerr),
 		erridx,
@@ -1050,6 +1087,7 @@ static struct command_result *
 listpeerchannels_done(struct command *cmd, const char *buf,
 	       const jsmntok_t *result, struct payment *p)
 {
+	debug_call(__PRETTY_FUNCTION__,MYLOG);
 	if (!update_uncertainty_network_from_listpeerchannels(cmd->plugin, p, buf, result))
 		return command_fail(cmd, LIGHTNINGD,
 				    "listpeerchannels malformed: %.*s",
@@ -1058,6 +1096,9 @@ listpeerchannels_done(struct command *cmd, const char *buf,
 	
 	// So we have all localmods data, now we apply it. Only once per
 	// payment.
+	// TODO(eduardo): check that there won't be a prob. cost associated with
+	// any gossmap local chan. The same way there aren't fees to pay for my
+	// local channels.
 	gossmap_apply_localmods(pay_plugin->gossmap,p->local_gossmods);
 	return try_paying(cmd, p, true);
 }
@@ -1066,6 +1107,7 @@ listpeerchannels_done(struct command *cmd, const char *buf,
  * into a valid state before the next payment. */
 static void renepay_cleanup(struct payment *p)
 {
+	debug_call(__PRETTY_FUNCTION__,MYLOG);
 	/* Always remove our local mods (routehints) so others can use
 	 * gossmap. We do this only after the payment completes. */
 	gossmap_remove_localmods(pay_plugin->gossmap,
@@ -1201,6 +1243,11 @@ static struct command_result *json_pay(struct command *cmd,
 	plugin_log(pay_plugin->plugin,LOG_DBG,"Starting renepay");
 	bool gossmap_changed = gossmap_refresh(pay_plugin->gossmap, NULL);
 	
+	if (pay_plugin->gossmap == NULL)
+		plugin_err(pay_plugin->plugin, "Failed to refresh gossmap: %s",
+			   strerror(errno));
+	
+	
 	// TODO(eduardo): remove this
 	debug_knowledge(pay_plugin->chan_extra_map,MYLOG);
 	
@@ -1425,13 +1472,6 @@ static struct command_result *json_pay(struct command *cmd,
 				    listpeerchannels_done,
 				    listpeerchannels_done, p);
 	return send_outreq(cmd->plugin, req);
-	
-	// struct json_stream *ret;
-	// ret = jsonrpc_stream_success(cmd);
-	// json_add_string(ret,"hello","world");
-	// json_add_amount_msat_only(ret,"maxspend_msat",p->maxspend);
-	// 
-	// return command_finished(cmd,ret);
 }
 
 static const struct plugin_command commands[] = {

From cf74699ef7c58dbe20594b3dd80d392a60db3c25 Mon Sep 17 00:00:00 2001
From: Lagrang3 <eduardo.quintana@pm.me>
Date: Mon, 29 May 2023 14:05:25 +0100
Subject: [PATCH 40/64] renepay: add debuging utils

---
 plugins/renepay/Makefile   |  6 ++-
 plugins/renepay/debug.c    | 80 ++++++++++++++++++++++++++++++++++
 plugins/renepay/debug.h    | 34 +++++++++++++++
 plugins/renepay/pay.c      | 88 ++------------------------------------
 plugins/renepay/pay_flow.c |  7 ---
 5 files changed, 122 insertions(+), 93 deletions(-)
 create mode 100644 plugins/renepay/debug.c
 create mode 100644 plugins/renepay/debug.h

diff --git a/plugins/renepay/Makefile b/plugins/renepay/Makefile
index 273ecfc18b87..b026f625dd2a 100644
--- a/plugins/renepay/Makefile
+++ b/plugins/renepay/Makefile
@@ -1,5 +1,7 @@
-PLUGIN_RENEPAY_SRC := plugins/renepay/pay.c plugins/renepay/pay_flow.c plugins/renepay/flow.c plugins/renepay/mcf.c plugins/renepay/dijkstra.c
-PLUGIN_RENEPAY_HDRS := plugins/renepay/pay.h plugins/renepay/pay_flow.h plugins/renepay/flow.h plugins/renepay/mcf.h plugins/renepay/heap.h plugins/renepay/dijkstra.h
+PLUGIN_RENEPAY_SRC := plugins/renepay/pay.c plugins/renepay/pay_flow.c plugins/renepay/flow.c plugins/renepay/mcf.c plugins/renepay/dijkstra.c \
+	plugins/renepay/debug.c
+PLUGIN_RENEPAY_HDRS := plugins/renepay/pay.h plugins/renepay/pay_flow.h plugins/renepay/flow.h plugins/renepay/mcf.h plugins/renepay/heap.h plugins/renepay/dijkstra.h \
+	plugins/renepay/debug.h
 PLUGIN_RENEPAY_OBJS := $(PLUGIN_RENEPAY_SRC:.c=.o)
 
 # Make sure these depend on everything.
diff --git a/plugins/renepay/debug.c b/plugins/renepay/debug.c
new file mode 100644
index 000000000000..f77105dbdb76
--- /dev/null
+++ b/plugins/renepay/debug.c
@@ -0,0 +1,80 @@
+#include <plugins/renepay/debug.h>
+
+
+void debug_knowledge(
+		struct chan_extra_map* chan_extra_map,
+		const char*fname)
+{
+	FILE *f = fopen(fname,"a");
+	fprintf(f,"Knowledge:\n");
+	struct chan_extra_map_iter it;
+	for(struct chan_extra *ch = chan_extra_map_first(chan_extra_map,&it);
+	    ch;
+	    ch=chan_extra_map_next(chan_extra_map,&it))
+	{
+		const char *scid_str = 
+			type_to_string(tmpctx,struct short_channel_id,&ch->scid);
+		for(int dir=0;dir<2;++dir)
+		{
+			fprintf(f,"%s[%d]:(%s,%s)\n",scid_str,dir,
+				type_to_string(tmpctx,struct amount_msat,&ch->half[dir].known_min),
+				type_to_string(tmpctx,struct amount_msat,&ch->half[dir].known_max));
+		}
+	}
+	fclose(f);
+}
+
+void debug_payflows(struct pay_flow **flows, const char* fname)
+{
+	FILE *f = fopen(fname,"a");
+	fprintf(f,"%s\n",fmt_payflows(tmpctx,flows));
+	fclose(f);
+}
+
+void debug_exec_branch(int lineno, const char* fun, const char* fname)
+{
+	FILE *f = fopen(fname,"a");
+	fprintf(f,"executing line: %d (%s)\n",lineno,fun);
+	fclose(f);
+}
+
+void debug_outreq(const struct out_req *req, const char*fname)
+{
+	FILE *f = fopen(fname,"a");
+	size_t len;
+	const char * str =  json_out_contents(req->js->jout,&len);
+	fprintf(f,"%s",str);
+	if (req->errcb)
+		fprintf(f,"}");
+	fprintf(f,"}\n");
+	fclose(f);
+}
+
+void debug_call(const char* fun, const char* fname)
+{
+	pthread_t tid = pthread_self();
+	FILE *f = fopen(fname,"a");
+	fprintf(f,"calling function: %s (pthread_t %ld)\n",fun,tid);
+	fclose(f);
+}
+
+void debug_reply(const char* buf, const char*fname)
+{
+	FILE *f = fopen(fname,"a");
+	fprintf(f,"%.*s\n\n",(int)tal_count(buf),buf);
+	fclose(f);
+}
+
+void debug(const char* fname, const char *fmt, ...)
+{
+	FILE *f = fopen(fname,"a");
+	
+	va_list args;
+	va_start(args, fmt);
+	
+	vfprintf(f,fmt,args);
+	
+	va_end(args);
+	fclose(f);
+}
+
diff --git a/plugins/renepay/debug.h b/plugins/renepay/debug.h
new file mode 100644
index 000000000000..cd4381da5254
--- /dev/null
+++ b/plugins/renepay/debug.h
@@ -0,0 +1,34 @@
+#ifndef LIGHTNING_PLUGINS_RENEPAY_DEBUG_H
+#define LIGHTNING_PLUGINS_RENEPAY_DEBUG_H
+
+#include <common/type_to_string.h>
+#include <common/json_param.h>
+#include <common/json_stream.h>
+#include <plugins/libplugin.h>
+#include <plugins/renepay/flow.h>
+#include <plugins/renepay/pay_flow.h>
+#include <stdio.h>
+#include <wire/peer_wire.h>
+#include <ccan/json_out/json_out.h>
+#include <pthread.h>
+
+#define MYLOG "/tmp/debug.txt"
+
+void debug_knowledge(
+		struct chan_extra_map* chan_extra_map,
+		const char*fname);
+		
+void debug_payflows(struct pay_flow **flows, const char* fname);
+
+void debug_exec_branch(int lineno, const char* fun, const char* fname);
+
+void debug_outreq(const struct out_req *req, const char*fname);
+
+void debug_call(const char* fun, const char* fname);
+
+void debug_reply(const char* buf, const char*fname);
+
+void debug(const char* fname, const char *fmt, ...);
+
+
+#endif
diff --git a/plugins/renepay/pay.c b/plugins/renepay/pay.c
index 0005be12c7fa..d4dd4db7e87a 100644
--- a/plugins/renepay/pay.c
+++ b/plugins/renepay/pay.c
@@ -14,13 +14,8 @@
 #include <errno.h>
 #include <plugins/libplugin.h>
 #include <plugins/renepay/pay.h>
+#include <plugins/renepay/debug.h>
 #include <plugins/renepay/pay_flow.h>
-#include <stdio.h>
-#include <wire/peer_wire.h>
-#include <ccan/json_out/json_out.h>
-#include <pthread.h>
-
-#define MYLOG "/tmp/debug.txt"
 
 /* Set in init */
 struct pay_plugin *pay_plugin;
@@ -31,83 +26,6 @@ static void renepay_cleanup(struct payment *p);
 // TODO(eduardo): check if knowledge is updated after payment success
 // TODO(eduardo): how do we fail a payment
 
-static void debug_knowledge(
-		struct chan_extra_map* chan_extra_map,
-		const char*fname)
-{
-	FILE *f = fopen(fname,"a");
-	fprintf(f,"Knowledge:\n");
-	struct chan_extra_map_iter it;
-	for(struct chan_extra *ch = chan_extra_map_first(chan_extra_map,&it);
-	    ch;
-	    ch=chan_extra_map_next(chan_extra_map,&it))
-	{
-		const char *scid_str = 
-			type_to_string(tmpctx,struct short_channel_id,&ch->scid);
-		for(int dir=0;dir<2;++dir)
-		{
-			fprintf(f,"%s[%d]:(%s,%s)\n",scid_str,dir,
-				type_to_string(tmpctx,struct amount_msat,&ch->half[dir].known_min),
-				type_to_string(tmpctx,struct amount_msat,&ch->half[dir].known_max));
-		}
-	}
-	fclose(f);
-}
-
-static void debug_payflows(struct pay_flow **flows, const char* fname)
-{
-	FILE *f = fopen(fname,"a");
-	fprintf(f,"%s\n",fmt_payflows(tmpctx,flows));
-	fclose(f);
-}
-
-static void debug_exec_branch(int lineno, const char* fun, const char* fname)
-{
-	FILE *f = fopen(fname,"a");
-	fprintf(f,"executing line: %d (%s)\n",lineno,fun);
-	fclose(f);
-}
-
-static void debug_outreq(const struct out_req *req, const char*fname)
-{
-	FILE *f = fopen(fname,"a");
-	size_t len;
-	const char * str =  json_out_contents(req->js->jout,&len);
-	fprintf(f,"%s",str);
-	if (req->errcb)
-		fprintf(f,"}");
-	fprintf(f,"}\n");
-	fclose(f);
-}
-
-static void debug_call(const char* fun, const char* fname)
-{
-	pthread_t tid = pthread_self();
-	FILE *f = fopen(fname,"a");
-	fprintf(f,"calling function: %s (pthread_t %ld)\n",fun,tid);
-	fclose(f);
-}
-
-static void debug_reply(const char* buf, const char*fname)
-{
-	FILE *f = fopen(fname,"a");
-	fprintf(f,"%.*s\n\n",(int)tal_count(buf),buf);
-	fclose(f);
-}
-
-static void debug(const char* fname, const char *fmt, ...)
-{
-	FILE *f = fopen(fname,"a");
-	
-	va_list args;
-	va_start(args, fmt);
-	
-	vfprintf(f,fmt,args);
-	
-	va_end(args);
-	fclose(f);
-}
-
 void paynote(struct payment *p, const char *fmt, ...)
 {
 	va_list ap;
@@ -1072,13 +990,15 @@ static struct command_result *try_paying(struct command *cmd,
 	debug_payflows(pay_flows,MYLOG);
 	
 	if (!pay_flows)
+	{
+		renepay_cleanup(p);
 		return command_fail(cmd, PAY_ROUTE_NOT_FOUND,
 				    "Failed to find a route for %s with budget %s",
 				    type_to_string(tmpctx, struct amount_msat,
 						   &remaining),
 				    type_to_string(tmpctx, struct amount_msat,
 						   &feebudget));
-
+	}
 	/* Now begin making payments */
 	return sendpay_flows(cmd, p, pay_flows);
 }
diff --git a/plugins/renepay/pay_flow.c b/plugins/renepay/pay_flow.c
index b772e2fd99c8..08daeb0c72a9 100644
--- a/plugins/renepay/pay_flow.c
+++ b/plugins/renepay/pay_flow.c
@@ -351,13 +351,6 @@ struct pay_flow **get_payflows(struct payment *p,
 	struct pay_flow **pay_flows;
 	const struct gossmap_node *src, *dst;
 	
-	// TODO(eduardo): is it necessary to check for return value?
-	gossmap_refresh(pay_plugin->gossmap, NULL);
-	
-	if (pay_plugin->gossmap == NULL)
-		plugin_err(pay_plugin->plugin, "Failed to refresh gossmap: %s",
-			   strerror(errno));
-	
 	disabled = make_disabled_bitmap(tmpctx, pay_plugin->gossmap, p->disabled);
 	src = gossmap_find_node(pay_plugin->gossmap, &pay_plugin->my_id);
 	if (!src) {

From 1e07e7e221c9488e6bf3c38da20f9874d18b1beb Mon Sep 17 00:00:00 2001
From: Lagrang3 <eduardo.quintana@pm.me>
Date: Mon, 29 May 2023 15:01:25 +0100
Subject: [PATCH 41/64] renepay: add destructor for payment data

---
 plugins/renepay/debug.c    |   6 ++-
 plugins/renepay/debug.h    |   2 +-
 plugins/renepay/pay.c      | 103 +++++++++++++++++++------------------
 plugins/renepay/pay.h      |  43 ++++++++++------
 plugins/renepay/pay_flow.c |   4 +-
 5 files changed, 86 insertions(+), 72 deletions(-)

diff --git a/plugins/renepay/debug.c b/plugins/renepay/debug.c
index f77105dbdb76..4db173b810a9 100644
--- a/plugins/renepay/debug.c
+++ b/plugins/renepay/debug.c
@@ -58,10 +58,12 @@ void debug_call(const char* fun, const char* fname)
 	fclose(f);
 }
 
-void debug_reply(const char* buf, const char*fname)
+void debug_reply(const char* buf,const jsmntok_t *toks, const char*fname)
 {
 	FILE *f = fopen(fname,"a");
-	fprintf(f,"%.*s\n\n",(int)tal_count(buf),buf);
+	fprintf(f,"%.*s\n\n",
+		   json_tok_full_len(toks),
+		   json_tok_full(buf, toks));
 	fclose(f);
 }
 
diff --git a/plugins/renepay/debug.h b/plugins/renepay/debug.h
index cd4381da5254..64d8d806bd7a 100644
--- a/plugins/renepay/debug.h
+++ b/plugins/renepay/debug.h
@@ -26,7 +26,7 @@ void debug_outreq(const struct out_req *req, const char*fname);
 
 void debug_call(const char* fun, const char* fname);
 
-void debug_reply(const char* buf, const char*fname);
+void debug_reply(const char* buf,const jsmntok_t *toks, const char*fname);
 
 void debug(const char* fname, const char *fmt, ...);
 
diff --git a/plugins/renepay/pay.c b/plugins/renepay/pay.c
index d4dd4db7e87a..4e16abc7a864 100644
--- a/plugins/renepay/pay.c
+++ b/plugins/renepay/pay.c
@@ -21,7 +21,7 @@
 struct pay_plugin *pay_plugin;
 
 
-static void renepay_cleanup(struct payment *p);
+static void renepay_cleanup(struct active_payment *ap);
 
 // TODO(eduardo): check if knowledge is updated after payment success
 // TODO(eduardo): how do we fail a payment
@@ -216,8 +216,9 @@ static void add_hintchan(struct payment *p,
 				    scid,
 				    MAX_CAP);
 		/* FIXME: features? */
-		gossmap_local_addchan(p->local_gossmods, src, dst, &scid, NULL);
-		gossmap_local_updatechan(p->local_gossmods,
+		gossmap_local_addchan(p->active_payment->local_gossmods, 
+				      src, dst, &scid, NULL);
+		gossmap_local_updatechan(p->active_payment->local_gossmods,
 					 &scid,
 					 /* We assume any HTLC is allowed */
 					 AMOUNT_MSAT(0), MAX_CAP,
@@ -239,7 +240,8 @@ static void uncertainty_network_add_routehints(struct payment *p)
 	char *fail;
 
 	b11 =
-	    bolt11_decode(tmpctx, p->invstr, plugin_feature_set(p->cmd->plugin),
+	    bolt11_decode(tmpctx, p->invstr, 
+	    		  plugin_feature_set(p->active_payment->cmd->plugin),
 			  p->description, chainparams, &fail);
 	if (b11 == NULL)
 		plugin_log(pay_plugin->plugin, LOG_BROKEN,
@@ -306,7 +308,7 @@ static bool update_uncertainty_network_from_listpeerchannels(
 				type_to_string(tmpctx,
 					       struct short_channel_id,
 					       &scid));
-			tal_arr_expand(&p->disabled, scid);
+			tal_arr_expand(&p->active_payment->disabled, scid);
 			continue;
 		}
 		
@@ -338,7 +340,7 @@ static bool update_uncertainty_network_from_listpeerchannels(
 			
 		/* Don't report opening/closing channels */
 		if (!json_tok_streq(buf, statetok, "CHANNELD_NORMAL")) {
-			tal_arr_expand(&p->disabled, scid);
+			tal_arr_expand(&p->active_payment->disabled, scid);
 			continue;
 		}
 		
@@ -352,8 +354,9 @@ static bool update_uncertainty_network_from_listpeerchannels(
 					    scid,
 					    capacity);
 			/* FIXME: features? */
-			gossmap_local_addchan(p->local_gossmods, &src, &dst, &scid, NULL);
-			gossmap_local_updatechan(p->local_gossmods,
+			gossmap_local_addchan(p->active_payment->local_gossmods, 
+					      &src, &dst, &scid, NULL);
+			gossmap_local_updatechan(p->active_payment->local_gossmods,
 						 &scid,
 						 
 						 /* TODO(eduardo): does it
@@ -407,7 +410,8 @@ static struct command_result *flow_failed(struct command *cmd,
 
 	/* If nothing outstanding, don't wait for timer! */
 	if (amount_msat_eq(p->total_sent, AMOUNT_MSAT(0))) {
-		p->rexmit_timer = tal_free(p->rexmit_timer);
+		p->active_payment->rexmit_timer 
+			= tal_free(p->active_payment->rexmit_timer);
 		return try_paying(cmd, p, false);
 	}
 
@@ -419,7 +423,8 @@ static struct command_result *flow_failed(struct command *cmd,
 static void timer_kick(struct payment *p)
 {
 	debug_call(__PRETTY_FUNCTION__,MYLOG);
-	p->rexmit_timer = tal_free(p->rexmit_timer);
+	p->active_payment->rexmit_timer 
+		= tal_free(p->active_payment->rexmit_timer);
 
 	/* Nothing has come back?  Re-arm timer. */
 	if (amount_msat_eq(p->total_delivering, p->amount)) {
@@ -430,7 +435,7 @@ static void timer_kick(struct payment *p)
 		return;
 	}
 
-	try_paying(p->cmd, p, false);
+	try_paying(p->active_payment->cmd, p, false);
 }
 
 static struct command_result *waitsendpay_succeeded(struct command *cmd,
@@ -462,7 +467,6 @@ static struct command_result *waitsendpay_succeeded(struct command *cmd,
 				  p->total_sent);
 	json_add_string(response, "status", "complete");
 	json_add_node_id(response, "destination", &p->dest);
-	renepay_cleanup(p);
 	return command_finished(cmd, response);
 }
 
@@ -492,9 +496,8 @@ static struct command_result *handle_unhandleable_error(struct payment *p,
 		   what, flow_path_to_str(tmpctx, flow));
 
 	if (n == 1)
-		return command_fail(p->cmd, PAY_UNPARSEABLE_ONION,
+		return command_fail(p->active_payment->cmd, PAY_UNPARSEABLE_ONION,
 				    "Got %s from the destination", what);
-
 	/* FIXME: check chan_extra_map, since we might have succeeded though
 	 * this node before? */
 
@@ -506,7 +509,7 @@ static struct command_result *handle_unhandleable_error(struct payment *p,
 		/* Assume it's not the destination */
 		n = pseudorand(n-1);
 
-	tal_arr_expand(&p->disabled, flow->path_scids[n]);
+	tal_arr_expand(&p->active_payment->disabled, flow->path_scids[n]);
 	paynote(p, "... eliminated %s",
 		type_to_string(tmpctx, struct short_channel_id,
 			       &flow->path_scids[n]));
@@ -547,7 +550,7 @@ static struct command_result *addgossip_failure(struct command *cmd,
 	paynote(p, "addgossip failed, removing channel %s (%.*s)",
 		type_to_string(tmpctx, struct short_channel_id, &adg->scid),
 		err->end - err->start, buf + err->start);
-	tal_arr_expand(&p->disabled, adg->scid);
+	tal_arr_expand(&p->active_payment->disabled, adg->scid);
 
 	return addgossip_done(cmd, buf, err, adg);
 }
@@ -569,7 +572,8 @@ static struct command_result *submit_update(struct command *cmd,
 	adg->scid = errscid;
 	adg->flow = tal_steal(adg, flow);
 	/* Disable re-xmit until this returns */
-	p->rexmit_timer = tal_free(p->rexmit_timer);
+	p->active_payment->rexmit_timer 
+		= tal_free(p->active_payment->rexmit_timer);
 
 	paynote(p, "... extracted channel_update, telling gossipd");
 	plugin_log(pay_plugin->plugin, LOG_DBG, "(update = %s)", tal_hex(tmpctx, update));
@@ -646,7 +650,7 @@ static struct command_result *waitsendpay_failed(struct command *cmd,
 {
 	plugin_log(pay_plugin->plugin,LOG_DBG,"calling waitsendpay_failed");
 	debug_call(__PRETTY_FUNCTION__,MYLOG);
-	debug_reply(buf,MYLOG);
+	debug_reply(buf,err,MYLOG);
 	
 	struct payment *p = flow->payment;
 	u64 errcode;
@@ -662,21 +666,17 @@ static struct command_result *waitsendpay_failed(struct command *cmd,
 
 	switch (errcode) {
 	case PAY_UNPARSEABLE_ONION:
-		debug_exec_branch(__LINE__,__PRETTY_FUNCTION__,MYLOG);
 		ret = handle_unhandleable_error(p, flow, "unparsable onion reply");
 		if (ret)
 			return ret;
 		goto done;
 	case PAY_DESTINATION_PERM_FAIL:
-		debug_exec_branch(__LINE__,__PRETTY_FUNCTION__,MYLOG);
 		return command_fail(cmd, PAY_DESTINATION_PERM_FAIL,
 				    "Got an final failure from destination");
 	case PAY_TRY_OTHER_ROUTE:
-		debug_exec_branch(__LINE__,__PRETTY_FUNCTION__,MYLOG);
 		break;
 
 	default:
-		debug_exec_branch(__LINE__,__PRETTY_FUNCTION__,MYLOG);
 		plugin_err(cmd->plugin,
 			   "Unexpected errcode from waitsendpay: %.*s",
 			   json_tok_full_len(err), json_tok_full(buf, err));
@@ -747,7 +747,7 @@ static struct command_result *waitsendpay_failed(struct command *cmd,
 	case WIRE_INVALID_ONION_BLINDING:
 	case WIRE_EXPIRY_TOO_FAR:
 		paynote(p, "... so we're removing scid");
-		tal_arr_expand(&p->disabled, errscid);
+		tal_arr_expand(&p->active_payment->disabled, errscid);
 		goto done;
 
 	/* These can be fixed (maybe) by applying the included channel_update */
@@ -761,7 +761,7 @@ static struct command_result *waitsendpay_failed(struct command *cmd,
 			return submit_update(cmd, flow, update, errscid);
 
 		paynote(p, "... missing an update, so we're removing scid");
-		tal_arr_expand(&p->disabled, errscid);
+		tal_arr_expand(&p->active_payment->disabled, errscid);
 		goto done;
 
 	/* Insufficient funds! */
@@ -816,7 +816,7 @@ static struct command_result *waitsendpay_failed(struct command *cmd,
 				  &flow->path_nodes[erridx]),
 		   erridx, tal_count(flow->path_nodes),
 		   onionerr);
-	tal_arr_expand(&p->disabled, errscid);
+	tal_arr_expand(&p->active_payment->disabled, errscid);
 
 done:
 	return flow_failed(cmd, flow);
@@ -873,7 +873,7 @@ static struct command_result *flow_sendpay_failed(struct command *cmd,
 	paynote(p,
 		"sendpay didn't like first hop, eliminated: %.*s",
 		msg->end - msg->start, buf + msg->start);
-	tal_arr_expand(&p->disabled, flow->path_scids[0]);
+	tal_arr_expand(&p->active_payment->disabled, flow->path_scids[0]);
 	return flow_failed(cmd, flow);
 }
 
@@ -964,7 +964,6 @@ static struct command_result *try_paying(struct command *cmd,
 
 	if (time_after(time_now(), p->stop_time))
 		return command_fail(cmd, PAY_STOPPED_RETRYING, "Timed out");
-
 	/* Total feebudget  */
 	if (!amount_msat_sub(&feebudget, p->maxspend, p->amount))
 		abort();
@@ -991,7 +990,6 @@ static struct command_result *try_paying(struct command *cmd,
 	
 	if (!pay_flows)
 	{
-		renepay_cleanup(p);
 		return command_fail(cmd, PAY_ROUTE_NOT_FOUND,
 				    "Failed to find a route for %s with budget %s",
 				    type_to_string(tmpctx, struct amount_msat,
@@ -1013,29 +1011,28 @@ listpeerchannels_done(struct command *cmd, const char *buf,
 				    "listpeerchannels malformed: %.*s",
 				    json_tok_full_len(result),
 				    json_tok_full(buf, result));
-	
 	// So we have all localmods data, now we apply it. Only once per
 	// payment.
 	// TODO(eduardo): check that there won't be a prob. cost associated with
 	// any gossmap local chan. The same way there aren't fees to pay for my
 	// local channels.
-	gossmap_apply_localmods(pay_plugin->gossmap,p->local_gossmods);
+	gossmap_apply_localmods(pay_plugin->gossmap,p->active_payment->local_gossmods);
+	p->active_payment->localmods_applied=true;
 	return try_paying(cmd, p, true);
 }
 
 /* Either the payment succeeded or failed, we need to cleanup/set the plugin
  * into a valid state before the next payment. */
-static void renepay_cleanup(struct payment *p)
+static void renepay_cleanup(struct active_payment *ap)
 {
 	debug_call(__PRETTY_FUNCTION__,MYLOG);
 	/* Always remove our local mods (routehints) so others can use
 	 * gossmap. We do this only after the payment completes. */
-	gossmap_remove_localmods(pay_plugin->gossmap,
-				 p->local_gossmods);
+	if(ap->localmods_applied)
+		gossmap_remove_localmods(pay_plugin->gossmap,
+					 ap->local_gossmods);
 				 
-	// TODO(eduardo): can we really free the local_gossmods?
-	// I don't thinkg they'll be needed afterwards
-	tal_free(p->local_gossmods);
+	tal_free(ap->local_gossmods);
 	// TODO(eduardo): Am I missing other things?
 	// TODO(eduardo): When do we call this?
 }
@@ -1105,10 +1102,9 @@ static struct command_result *json_pay(struct command *cmd,
 #endif
 
  	p = tal(cmd, struct payment);
-
-	p->cmd = cmd;
+	
+	
 	p->paynotes = tal_arr(p, const char *, 0);
-	p->disabled = tal_arr(p, struct short_channel_id, 0);
 	p->groupid = pseudorand_u64();
 	
 	// TODO(eduardo): If the partid starts counting from 1 we can do
@@ -1118,8 +1114,6 @@ static struct command_result *json_pay(struct command *cmd,
 	p->next_partid = 1;
 	p->total_sent = AMOUNT_MSAT(0);
 	p->total_delivering = AMOUNT_MSAT(0);
-	p->rexmit_timer = NULL;
- 	p->local_gossmods = gossmap_localmods_new(p);
 	
 	if (!param(cmd, buf, params,
 		   p_req("invstring", param_string, &p->invstr),
@@ -1151,6 +1145,24 @@ static struct command_result *json_pay(struct command *cmd,
 #endif
 		   NULL))
 		return command_param_failed();
+	
+	
+	tal_steal(pay_plugin,p);
+	tal_add_destructor(p, destroy_payment);
+	
+	/* Owned by cmd, because this data is destroyed after the payment
+	 * completes, while p remains to get payment status history. */
+	p->active_payment = tal(cmd,struct active_payment);
+	tal_add_destructor(p->active_payment, renepay_cleanup);
+	
+	p->active_payment->cmd = cmd;
+ 	p->active_payment->local_gossmods = gossmap_localmods_new(p->active_payment);
+	p->active_payment->localmods_applied=false;
+	p->active_payment->disabled = tal_arr(p->active_payment, 
+					      struct short_channel_id, 
+					      0);
+	p->active_payment->rexmit_timer = NULL;
+	
 
 #if DEVELOPER
 	p->use_shadow = *use_shadow;
@@ -1198,7 +1210,6 @@ static struct command_result *json_pay(struct command *cmd,
 		if (b11 == NULL)
 			return command_fail(cmd, JSONRPC2_INVALID_PARAMS,
 					    "Invalid bolt11: %s", fail);
-		
 		invstr_is_b11=true;
 		
 		invmsat = b11->msat;
@@ -1222,7 +1233,6 @@ static struct command_result *json_pay(struct command *cmd,
 			    cmd, JSONRPC2_INVALID_PARAMS,
 			    "Invalid bolt11:"
 			    " sets feature var_onion with no secret");
-
 		/* BOLT #11:
 		 * A reader:
 		 *...
@@ -1309,10 +1319,7 @@ static struct command_result *json_pay(struct command *cmd,
 				    "This payment is destined for ourselves. "
 				    "Self-payments are not supported");
 
-	/* It's now owned by the global plugin */
 	list_add_tail(&pay_plugin->payments, &p->list);
-	tal_add_destructor(p, destroy_payment);
-	tal_steal(pay_plugin, p);
 	
 	// set the payment amount
 	if (invmsat) {
@@ -1370,10 +1377,6 @@ static struct command_result *json_pay(struct command *cmd,
 					   pay_plugin->chan_extra_map);
 	
 	
-	// TODO(eduardo)
-	// we should remove local_gossmods from the uncertainty network
-	// after the payment is completed?
-	// 
 	// TODO(eduardo): are there route hints for B12?
 	// Add any extra hidden channel revealed by the routehints to the uncertainty network.
 	if(invstr_is_b11)
diff --git a/plugins/renepay/pay.h b/plugins/renepay/pay.h
index 1f7f52db91bc..70921c3a5b1d 100644
--- a/plugins/renepay/pay.h
+++ b/plugins/renepay/pay.h
@@ -32,24 +32,41 @@ struct pay_plugin {
 /* Set in init */
 extern struct pay_plugin *pay_plugin;
 
+// TODO(eduardo): does it make sense?
+/* Data only kept while the payment is being processed. */
+struct active_payment
+{
+	/* The command, and our owner (needed for timer func) */
+	// TODO(eduardo): is it used?
+	struct command *cmd;
+
+	/* Localmods to apply to gossip_map for our own use. */
+	bool localmods_applied;
+	struct gossmap_localmods *local_gossmods;
+	
+	/* Channels we decided to disable for various reasons. */
+	// TODO(eduardo): is it used?
+	struct short_channel_id *disabled;
+
+	/* Timers. */
+	// TODO(eduardo): is it used?
+	struct plugin_timer *rexmit_timer;
+};
+
 struct payment {
+	/* Data used while the payment is being processed. */
+	// TODO(eduardo): initialize this
+	// TODO(eduardo): add a destructor
+	struct active_payment *active_payment;
+
 	/* Inside pay_plugin->payments list */
 	// TODO(eduardo): is it used?
 	struct list_node list;
 
-	/* The command, and our owner (needed for timer func) */
-	// TODO(eduardo): is it used?
-	struct command *cmd;
-
 	/* We promised this in pay() output */
 	// TODO(eduardo): is it used?
 	struct timeabs start_time;
 
-	/* Localmods to apply to gossip_map for our own use. */
-	// TODO(eduardo): shall we keep this data even after the payment
-	// ends?
-	struct gossmap_localmods *local_gossmods;
-
 	/* invstring (bolt11 or bolt12) */
 	// TODO(eduardo): is it used?
 	const char *invstr;
@@ -113,10 +130,6 @@ struct payment {
 	// TODO(eduardo): is it used?
 	double prob_cost_factor;
 
-	/* Channels we decided to disable for various reasons. */
-	// TODO(eduardo): is it used?
-	struct short_channel_id *disabled;
-
 	/* Chatty description of attempts. */
 	// TODO(eduardo): is it used?
 	const char **paynotes;
@@ -132,10 +145,6 @@ struct payment {
 	// TODO(eduardo): could be NULL
 	struct sha256 *local_offer_id;
 
-	/* Timers. */
-	// TODO(eduardo): is it used?
-	struct plugin_timer *rexmit_timer;
-
 	/* DEVELOPER allows disabling shadow route */
 	// TODO(eduardo): is it used?
 	bool use_shadow;
diff --git a/plugins/renepay/pay_flow.c b/plugins/renepay/pay_flow.c
index 08daeb0c72a9..4b0427d6b5eb 100644
--- a/plugins/renepay/pay_flow.c
+++ b/plugins/renepay/pay_flow.c
@@ -301,7 +301,7 @@ static bool disable_htlc_violations_oneflow(struct payment *p,
 			reason);
 
 		/* Add this for future searches for this payment. */
-		tal_arr_expand(&p->disabled, scid);
+		tal_arr_expand(&p->active_payment->disabled, scid);
 		/* Add to existing bitmap */
 		bitmap_set_bit(disabled,
 			       gossmap_chan_idx(gossmap, flow->path[i]));
@@ -351,7 +351,7 @@ struct pay_flow **get_payflows(struct payment *p,
 	struct pay_flow **pay_flows;
 	const struct gossmap_node *src, *dst;
 	
-	disabled = make_disabled_bitmap(tmpctx, pay_plugin->gossmap, p->disabled);
+	disabled = make_disabled_bitmap(tmpctx, pay_plugin->gossmap, p->active_payment->disabled);
 	src = gossmap_find_node(pay_plugin->gossmap, &pay_plugin->my_id);
 	if (!src) {
 		paynote(p, "We don't have any channels?");

From a9fa2afc7a9b48aa2170d653544b7c8e014da1df Mon Sep 17 00:00:00 2001
From: Lagrang3 <eduardo.quintana@pm.me>
Date: Mon, 29 May 2023 16:39:38 +0100
Subject: [PATCH 42/64] renepay: remove assumption of liquidity for hints

---
 plugins/renepay/debug.c | 2 +-
 plugins/renepay/debug.h | 2 +-
 plugins/renepay/pay.c   | 8 ++++----
 3 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/plugins/renepay/debug.c b/plugins/renepay/debug.c
index 4db173b810a9..5d88c16e0f51 100644
--- a/plugins/renepay/debug.c
+++ b/plugins/renepay/debug.c
@@ -67,7 +67,7 @@ void debug_reply(const char* buf,const jsmntok_t *toks, const char*fname)
 	fclose(f);
 }
 
-void debug(const char* fname, const char *fmt, ...)
+void debug_info(const char* fname, const char *fmt, ...)
 {
 	FILE *f = fopen(fname,"a");
 	
diff --git a/plugins/renepay/debug.h b/plugins/renepay/debug.h
index 64d8d806bd7a..d25e5fc83f07 100644
--- a/plugins/renepay/debug.h
+++ b/plugins/renepay/debug.h
@@ -28,7 +28,7 @@ void debug_call(const char* fun, const char* fname);
 
 void debug_reply(const char* buf,const jsmntok_t *toks, const char*fname);
 
-void debug(const char* fname, const char *fmt, ...);
+void debug_info(const char* fname, const char *fmt, ...);
 
 
 #endif
diff --git a/plugins/renepay/pay.c b/plugins/renepay/pay.c
index 4e16abc7a864..bd60ac88d10c 100644
--- a/plugins/renepay/pay.c
+++ b/plugins/renepay/pay.c
@@ -208,7 +208,6 @@ static void add_hintchan(struct payment *p,
 	
 	struct chan_extra *ce = chan_extra_map_get(pay_plugin->chan_extra_map,
 						   scid);
-	
 	if(!ce)
 	{
 		/* this channel is not public, we don't know his capacity */
@@ -228,9 +227,10 @@ static void add_hintchan(struct payment *p,
 					 dir);
 	}
 	
-	/* We know (assume!) something about this channel: that it has at
-	 * sufficient capacity. */
-	chan_extra_can_send(pay_plugin->chan_extra_map,scid,dir,amount);
+	/* It is wrong to assume that this channel has sufficient capacity! 
+	 * Doing so leads to knowledge updates in which the known min liquidity
+	 * is greater than the channel's capacity. */
+	// chan_extra_can_send(pay_plugin->chan_extra_map,scid,dir,amount);
 }
 
 /* Add routehints provided by bolt11 */

From 0257379fa67341b24596cf1380d0a83dab16ae43 Mon Sep 17 00:00:00 2001
From: Lagrang3 <eduardo.quintana@pm.me>
Date: Mon, 29 May 2023 17:20:35 +0100
Subject: [PATCH 43/64] renepay: add/remove HTLC in the uncertainty net.

---
 plugins/renepay/flow.h     |  8 +++++++
 plugins/renepay/pay.c      | 22 +++++++++++++++++-
 plugins/renepay/pay_flow.c | 47 ++++++++++++++++++++++++++++++++++----
 plugins/renepay/pay_flow.h |  8 +++++++
 4 files changed, 79 insertions(+), 6 deletions(-)

diff --git a/plugins/renepay/flow.h b/plugins/renepay/flow.h
index 562c6ea65ff5..fbabf23dc58b 100644
--- a/plugins/renepay/flow.h
+++ b/plugins/renepay/flow.h
@@ -246,10 +246,14 @@ double flow_set_probability(
 		struct gossmap const*const gossmap,
 		struct chan_extra_map * chan_extra_map);
 
+// TODO(eduardo): we probably don't need this. Instead we should have payflow
+// input.
 /* Once flow is completed, this can remove it from the extra_map */
 void remove_completed_flow(const struct gossmap *gossmap,
 			   struct chan_extra_map *chan_extra_map,
 			   struct flow *flow);
+// TODO(eduardo): we probably don't need this. Instead we should have payflow
+// input.
 void remove_completed_flow_set(const struct gossmap *gossmap,
 			   struct chan_extra_map *chan_extra_map,
 			   struct flow **flows);
@@ -303,12 +307,16 @@ s64 linear_fee_cost(
 		double base_fee_penalty,
 		double delay_feefactor);
 
+// TODO(eduardo): we probably don't need this. Instead we should have payflow
+// input.
 /* Take the flows and commit them to the chan_extra's . */
 void commit_flow(
 		const struct gossmap *gossmap,
 		struct chan_extra_map *chan_extra_map,
 		struct flow *flow);
 
+// TODO(eduardo): we probably don't need this. Instead we should have payflow
+// input.
 /* Take the flows and commit them to the chan_extra's . */
 void commit_flow_set(
 		const struct gossmap *gossmap,
diff --git a/plugins/renepay/pay.c b/plugins/renepay/pay.c
index bd60ac88d10c..b5e3f14e43a7 100644
--- a/plugins/renepay/pay.c
+++ b/plugins/renepay/pay.c
@@ -408,13 +408,17 @@ static struct command_result *flow_failed(struct command *cmd,
 	amount_msat_reduce(&p->total_delivering, flow_delivered(flow));
 	amount_msat_reduce(&p->total_sent, flow->amounts[0]);
 
+	/* flow failed, so we remove these HTLCs */
+	remove_htlc_payflow(pay_plugin->chan_extra_map,
+			    flow);
+
 	/* If nothing outstanding, don't wait for timer! */
 	if (amount_msat_eq(p->total_sent, AMOUNT_MSAT(0))) {
 		p->active_payment->rexmit_timer 
 			= tal_free(p->active_payment->rexmit_timer);
 		return try_paying(cmd, p, false);
 	}
-
+	
 	/* Still waiting for timer... */
 	return command_still_pending(cmd);
 }
@@ -445,6 +449,19 @@ static struct command_result *waitsendpay_succeeded(struct command *cmd,
 {
 	plugin_log(pay_plugin->plugin,LOG_DBG,"calling waitsendpay_succeeded");
 	debug_call(__PRETTY_FUNCTION__,MYLOG);
+	
+	/* payment succeeded, now remove the HTLCs */
+	remove_htlc_payflow(pay_plugin->chan_extra_map,
+			    flow);
+	
+	/* update the knowledge */
+	for (size_t i = 0; i < tal_count(flow->path_scids); i++)
+	{
+		chan_extra_sent_success(pay_plugin->chan_extra_map,
+					flow->path_scids[i],
+					flow->path_dirs[i],
+					flow->amounts[i]);
+	}
 	struct payment *p = flow->payment;
 	struct json_stream *response = jsonrpc_stream_success(cmd);
 	const jsmntok_t *preimagetok;
@@ -938,6 +955,8 @@ sendpay_flows(struct command *cmd,
 		/* Flow now owned by request */
 		tal_steal(req, flows[i]);
 		
+		/* record these HTLC along the flow path */
+		commit_htlc_payflow(pay_plugin->chan_extra_map,flows[i]);
 		send_outreq(cmd->plugin, req);
 	}
 	
@@ -984,6 +1003,7 @@ static struct command_result *try_paying(struct command *cmd,
 	 * than simply refuse.  Plus, models are not truth! */
 	pay_flows = get_payflows(p, remaining, feebudget, first_time,
 				 amount_msat_eq(p->total_delivering, AMOUNT_MSAT(0)));
+		
 	
 	// TODO(eduardo): remove this line
 	debug_payflows(pay_flows,MYLOG);
diff --git a/plugins/renepay/pay_flow.c b/plugins/renepay/pay_flow.c
index 4b0427d6b5eb..22c564e276ac 100644
--- a/plugins/renepay/pay_flow.c
+++ b/plugins/renepay/pay_flow.c
@@ -437,11 +437,11 @@ struct pay_flow **get_payflows(struct payment *p,
 					    disabled))
 			goto retry;
 		
-		/* Commit the flows to the chan_extra_map, 
-		 * update the htlc_total and num_htlcs. */
-		commit_flow_set(pay_plugin->gossmap,
-				pay_plugin->chan_extra_map,
-				flows);
+		// /* Commit the flows to the chan_extra_map, 
+		//  * update the htlc_total and num_htlcs. */
+		// commit_htlc_flow_set(pay_plugin->gossmap,
+		// 		pay_plugin->chan_extra_map,
+		// 		flows);
 		
 		
 		/* This can adjust amounts and final cltv for each flow,
@@ -534,3 +534,40 @@ const char* fmt_payflows(const tal_t *ctx,
 	size_t len;
 	return json_out_contents(jout,&len);
 }
+
+void remove_htlc_payflow(
+		struct chan_extra_map *chan_extra_map,
+		const struct pay_flow *flow)
+{
+	for (size_t i = 0; i < tal_count(flow->path_scids); i++) {
+		struct chan_extra_half *h = get_chan_extra_half_by_scid(
+							       chan_extra_map,
+							       flow->path_scids[i],
+							       flow->path_dirs[i]);
+		if (!amount_msat_sub(&h->htlc_total, h->htlc_total, flow->amounts[i]))
+		{
+			abort();
+		}
+		if (h->num_htlcs == 0)
+		{
+			abort();
+		}
+		h->num_htlcs--;
+	}
+}
+void commit_htlc_payflow(
+		struct chan_extra_map *chan_extra_map,
+		const struct pay_flow *flow)
+{
+	for (size_t i = 0; i < tal_count(flow->path_scids); i++) {
+		struct chan_extra_half *h = get_chan_extra_half_by_scid(
+							       chan_extra_map,
+							       flow->path_scids[i],
+							       flow->path_dirs[i]);
+		if (!amount_msat_add(&h->htlc_total, h->htlc_total, flow->amounts[i]))
+		{
+			abort();
+		}
+		h->num_htlcs++;
+	}
+}
diff --git a/plugins/renepay/pay_flow.h b/plugins/renepay/pay_flow.h
index a6cf313d71bb..2285207141d1 100644
--- a/plugins/renepay/pay_flow.h
+++ b/plugins/renepay/pay_flow.h
@@ -28,6 +28,14 @@ struct pay_flow **get_payflows(struct payment *p,
 			       bool unlikely_ok,
 			       bool is_entire_payment);
 
+void commit_htlc_payflow(
+		struct chan_extra_map *chan_extra_map,
+		const struct pay_flow *flow);
+
+void remove_htlc_payflow(
+		struct chan_extra_map *chan_extra_map,
+		const struct pay_flow *flow);
+
 const char* fmt_payflows(const tal_t *ctx,
 			 struct pay_flow ** flows);
 

From f400ed95a072cd96861589549585c63fc6abf7c4 Mon Sep 17 00:00:00 2001
From: Lagrang3 <eduardo.quintana@pm.me>
Date: Fri, 2 Jun 2023 09:15:14 +0100
Subject: [PATCH 44/64] renepay: use generic debug functions

---
 plugins/renepay/debug.c | 41 +++++------------------------------------
 plugins/renepay/debug.h | 29 ++++++++++++++++-------------
 2 files changed, 21 insertions(+), 49 deletions(-)

diff --git a/plugins/renepay/debug.c b/plugins/renepay/debug.c
index 5d88c16e0f51..51b19ef808f3 100644
--- a/plugins/renepay/debug.c
+++ b/plugins/renepay/debug.c
@@ -1,44 +1,13 @@
 #include <plugins/renepay/debug.h>
 
-
-void debug_knowledge(
-		struct chan_extra_map* chan_extra_map,
-		const char*fname)
-{
-	FILE *f = fopen(fname,"a");
-	fprintf(f,"Knowledge:\n");
-	struct chan_extra_map_iter it;
-	for(struct chan_extra *ch = chan_extra_map_first(chan_extra_map,&it);
-	    ch;
-	    ch=chan_extra_map_next(chan_extra_map,&it))
-	{
-		const char *scid_str = 
-			type_to_string(tmpctx,struct short_channel_id,&ch->scid);
-		for(int dir=0;dir<2;++dir)
-		{
-			fprintf(f,"%s[%d]:(%s,%s)\n",scid_str,dir,
-				type_to_string(tmpctx,struct amount_msat,&ch->half[dir].known_min),
-				type_to_string(tmpctx,struct amount_msat,&ch->half[dir].known_max));
-		}
-	}
-	fclose(f);
-}
-
-void debug_payflows(struct pay_flow **flows, const char* fname)
-{
-	FILE *f = fopen(fname,"a");
-	fprintf(f,"%s\n",fmt_payflows(tmpctx,flows));
-	fclose(f);
-}
-
-void debug_exec_branch(int lineno, const char* fun, const char* fname)
+void _debug_exec_branch(const char* fname,const char* fun, int lineno)
 {
 	FILE *f = fopen(fname,"a");
 	fprintf(f,"executing line: %d (%s)\n",lineno,fun);
 	fclose(f);
 }
 
-void debug_outreq(const struct out_req *req, const char*fname)
+void _debug_outreq(const char *fname, const struct out_req *req)
 {
 	FILE *f = fopen(fname,"a");
 	size_t len;
@@ -50,7 +19,7 @@ void debug_outreq(const struct out_req *req, const char*fname)
 	fclose(f);
 }
 
-void debug_call(const char* fun, const char* fname)
+void _debug_call(const char* fname, const char* fun)
 {
 	pthread_t tid = pthread_self();
 	FILE *f = fopen(fname,"a");
@@ -58,7 +27,7 @@ void debug_call(const char* fun, const char* fname)
 	fclose(f);
 }
 
-void debug_reply(const char* buf,const jsmntok_t *toks, const char*fname)
+void _debug_reply(const char* fname, const char* buf,const jsmntok_t *toks)
 {
 	FILE *f = fopen(fname,"a");
 	fprintf(f,"%.*s\n\n",
@@ -67,7 +36,7 @@ void debug_reply(const char* buf,const jsmntok_t *toks, const char*fname)
 	fclose(f);
 }
 
-void debug_info(const char* fname, const char *fmt, ...)
+void _debug_info(const char* fname, const char *fmt, ...)
 {
 	FILE *f = fopen(fname,"a");
 	
diff --git a/plugins/renepay/debug.h b/plugins/renepay/debug.h
index d25e5fc83f07..3449fce367ae 100644
--- a/plugins/renepay/debug.h
+++ b/plugins/renepay/debug.h
@@ -5,8 +5,6 @@
 #include <common/json_param.h>
 #include <common/json_stream.h>
 #include <plugins/libplugin.h>
-#include <plugins/renepay/flow.h>
-#include <plugins/renepay/pay_flow.h>
 #include <stdio.h>
 #include <wire/peer_wire.h>
 #include <ccan/json_out/json_out.h>
@@ -14,21 +12,26 @@
 
 #define MYLOG "/tmp/debug.txt"
 
-void debug_knowledge(
-		struct chan_extra_map* chan_extra_map,
-		const char*fname);
-		
-void debug_payflows(struct pay_flow **flows, const char* fname);
 
-void debug_exec_branch(int lineno, const char* fun, const char* fname);
+void _debug_outreq(const char *fname, const struct out_req *req);
+#define debug_outreq(req) \
+	_debug_outreq(MYLOG,req)
 
-void debug_outreq(const struct out_req *req, const char*fname);
+void _debug_reply(const char* fname, const char* buf,const jsmntok_t *toks);
+#define debug_reply(buf,toks) \
+	_debug_reply(MYLOG,buf,toks)
 
-void debug_call(const char* fun, const char* fname);
+void _debug_info(const char* fname, const char *fmt, ...);
+#define debug_info(...) \
+	_debug_info(MYLOG,__VA_ARGS__)
+	
 
-void debug_reply(const char* buf,const jsmntok_t *toks, const char*fname);
-
-void debug_info(const char* fname, const char *fmt, ...);
+void _debug_call(const char* fname, const char* fun);
+#define debug_call() \
+	_debug_call(MYLOG,__PRETTY_FUNCTION__)
 
+void _debug_exec_branch(const char* fname,const char* fun, int lineno);
+#define debug_exec_branch() \
+	_debug_exec_branch(MYLOG,__PRETTY_FUNCTION__,__LINE__)
 
 #endif

From fb6b195376a265e4ada7479563d79e6eca0026c0 Mon Sep 17 00:00:00 2001
From: Lagrang3 <eduardo.quintana@pm.me>
Date: Fri, 2 Jun 2023 09:18:10 +0100
Subject: [PATCH 45/64] renepay: remove mock functions from tests

---
 plugins/renepay/test/Makefile          | 15 +++----------
 plugins/renepay/test/run-dijkstra.c    | 31 --------------------------
 plugins/renepay/test/run-map.c         | 25 ---------------------
 plugins/renepay/test/run-mcf-diamond.c | 25 ---------------------
 plugins/renepay/test/run-mcf.c         | 22 ------------------
 plugins/renepay/test/run-testflow.c    | 22 ------------------
 6 files changed, 3 insertions(+), 137 deletions(-)

diff --git a/plugins/renepay/test/Makefile b/plugins/renepay/test/Makefile
index b012ff8c809b..4cf2bfa63349 100644
--- a/plugins/renepay/test/Makefile
+++ b/plugins/renepay/test/Makefile
@@ -9,18 +9,9 @@ ALL_TEST_PROGRAMS += $(PLUGIN_RENEPAY_TEST_PROGRAMS)
 $(PLUGIN_RENEPAY_TEST_OBJS): $(PLUGIN_RENEPAY_SRC)
 
 PLUGIN_RENEPAY_TEST_COMMON_OBJS :=		\
-	common/amount.o				\
-	common/autodata.o			\
-	common/fp16.o				\
-	common/gossmap.o			\
-	common/node_id.o			\
-	common/pseudorand.o			\
-	common/dijkstra.o			\
-	common/setup.o				\
-	common/type_to_string.o			\
-	common/utils.o				\
-	plugins/renepay/dijkstra.o
+	plugins/renepay/dijkstra.o		\
+	plugins/renepay/debug.o
 
-$(PLUGIN_RENEPAY_TEST_PROGRAMS): $(CCAN_OBJS) $(BITCOIN_OBJS) $(WIRE_OBJS) $(PLUGIN_RENEPAY_TEST_COMMON_OBJS)
+$(PLUGIN_RENEPAY_TEST_PROGRAMS): $(PLUGIN_RENEPAY_TEST_COMMON_OBJS) $(PLUGIN_LIB_OBJS) $(PLUGIN_COMMON_OBJS) $(JSMN_OBJS) $(CCAN_OBJS) bitcoin/chainparams.o common/gossmap.o common/fp16.o common/dijkstra.o common/bolt12.o common/bolt12_merkle.o wire/bolt12$(EXP)_wiregen.o
 
 check-renepay: $(PLUGIN_RENEPAY_TEST_PROGRAMS:%=unittest/%)
diff --git a/plugins/renepay/test/run-dijkstra.c b/plugins/renepay/test/run-dijkstra.c
index b97c2af686cd..7e9d208270ef 100644
--- a/plugins/renepay/test/run-dijkstra.c
+++ b/plugins/renepay/test/run-dijkstra.c
@@ -9,37 +9,6 @@
 
 #include <plugins/renepay/dijkstra.h>
 
-// #include <ccan/array_size/array_size.h>
-// #include <ccan/read_write_all/read_write_all.h>
-// #include <common/gossip_store.h>
-// #include <common/node_id.h>
-// #include <common/type_to_string.h>
-
-// #include <bitcoin/short_channel_id.h>
-// #include <ccan/htable/htable_type.h>
-
-/* AUTOGENERATED MOCKS START */
-/* Generated stub for fromwire_bigsize */
-bigsize_t fromwire_bigsize(const u8 **cursor UNNEEDED, size_t *max UNNEEDED)
-{ fprintf(stderr, "fromwire_bigsize called!\n"); abort(); }
-/* Generated stub for fromwire_channel_id */
-bool fromwire_channel_id(const u8 **cursor UNNEEDED, size_t *max UNNEEDED,
-			 struct channel_id *channel_id UNNEEDED)
-{ fprintf(stderr, "fromwire_channel_id called!\n"); abort(); }
-/* Generated stub for fromwire_wireaddr */
-bool fromwire_wireaddr(const u8 **cursor UNNEEDED, size_t *max UNNEEDED, struct wireaddr *addr UNNEEDED)
-{ fprintf(stderr, "fromwire_wireaddr called!\n"); abort(); }
-/* Generated stub for towire_bigsize */
-void towire_bigsize(u8 **pptr UNNEEDED, const bigsize_t val UNNEEDED)
-{ fprintf(stderr, "towire_bigsize called!\n"); abort(); }
-/* Generated stub for towire_channel_id */
-void towire_channel_id(u8 **pptr UNNEEDED, const struct channel_id *channel_id UNNEEDED)
-{ fprintf(stderr, "towire_channel_id called!\n"); abort(); }
-/* Generated stub for towire_wireaddr */
-void towire_wireaddr(u8 **pptr UNNEEDED, const struct wireaddr *addr UNNEEDED)
-{ fprintf(stderr, "towire_wireaddr called!\n"); abort(); }
-/* AUTOGENERATED MOCKS END */
-
 static void insertion_in_increasing_distance(const tal_t *ctx)
 {
 	dijkstra_malloc(ctx,10);
diff --git a/plugins/renepay/test/run-map.c b/plugins/renepay/test/run-map.c
index 2d70d529ab01..da568432fa1f 100644
--- a/plugins/renepay/test/run-map.c
+++ b/plugins/renepay/test/run-map.c
@@ -14,31 +14,6 @@
 #include <bitcoin/short_channel_id.h>
 #include <ccan/htable/htable_type.h>
 
-/* not_mcf sets NDEBUG, so assert() is useless */
-#define ASSERT(x) do { if (!(x)) abort(); } while(0)
-
-/* AUTOGENERATED MOCKS START */
-/* Generated stub for fromwire_bigsize */
-bigsize_t fromwire_bigsize(const u8 **cursor UNNEEDED, size_t *max UNNEEDED)
-{ fprintf(stderr, "fromwire_bigsize called!\n"); abort(); }
-/* Generated stub for fromwire_channel_id */
-bool fromwire_channel_id(const u8 **cursor UNNEEDED, size_t *max UNNEEDED,
-			 struct channel_id *channel_id UNNEEDED)
-{ fprintf(stderr, "fromwire_channel_id called!\n"); abort(); }
-/* Generated stub for fromwire_wireaddr */
-bool fromwire_wireaddr(const u8 **cursor UNNEEDED, size_t *max UNNEEDED, struct wireaddr *addr UNNEEDED)
-{ fprintf(stderr, "fromwire_wireaddr called!\n"); abort(); }
-/* Generated stub for towire_bigsize */
-void towire_bigsize(u8 **pptr UNNEEDED, const bigsize_t val UNNEEDED)
-{ fprintf(stderr, "towire_bigsize called!\n"); abort(); }
-/* Generated stub for towire_channel_id */
-void towire_channel_id(u8 **pptr UNNEEDED, const struct channel_id *channel_id UNNEEDED)
-{ fprintf(stderr, "towire_channel_id called!\n"); abort(); }
-/* Generated stub for towire_wireaddr */
-void towire_wireaddr(u8 **pptr UNNEEDED, const struct wireaddr *addr UNNEEDED)
-{ fprintf(stderr, "towire_wireaddr called!\n"); abort(); }
-/* AUTOGENERATED MOCKS END */
-
 struct chan_extra {
 	struct short_channel_id scid;
 	int value;
diff --git a/plugins/renepay/test/run-mcf-diamond.c b/plugins/renepay/test/run-mcf-diamond.c
index 69938327f4e4..ff2a37885453 100644
--- a/plugins/renepay/test/run-mcf-diamond.c
+++ b/plugins/renepay/test/run-mcf-diamond.c
@@ -12,31 +12,6 @@
 #include <common/wireaddr.h>
 #include <stdio.h>
 
-/* not_mcf sets NDEBUG, so assert() is useless */
-#define ASSERT(x) do { if (!(x)) abort(); } while(0)
-
-/* AUTOGENERATED MOCKS START */
-/* Generated stub for fromwire_bigsize */
-bigsize_t fromwire_bigsize(const u8 **cursor UNNEEDED, size_t *max UNNEEDED)
-{ fprintf(stderr, "fromwire_bigsize called!\n"); abort(); }
-/* Generated stub for fromwire_channel_id */
-bool fromwire_channel_id(const u8 **cursor UNNEEDED, size_t *max UNNEEDED,
-			 struct channel_id *channel_id UNNEEDED)
-{ fprintf(stderr, "fromwire_channel_id called!\n"); abort(); }
-/* Generated stub for fromwire_wireaddr */
-bool fromwire_wireaddr(const u8 **cursor UNNEEDED, size_t *max UNNEEDED, struct wireaddr *addr UNNEEDED)
-{ fprintf(stderr, "fromwire_wireaddr called!\n"); abort(); }
-/* Generated stub for towire_bigsize */
-void towire_bigsize(u8 **pptr UNNEEDED, const bigsize_t val UNNEEDED)
-{ fprintf(stderr, "towire_bigsize called!\n"); abort(); }
-/* Generated stub for towire_channel_id */
-void towire_channel_id(u8 **pptr UNNEEDED, const struct channel_id *channel_id UNNEEDED)
-{ fprintf(stderr, "towire_channel_id called!\n"); abort(); }
-/* Generated stub for towire_wireaddr */
-void towire_wireaddr(u8 **pptr UNNEEDED, const struct wireaddr *addr UNNEEDED)
-{ fprintf(stderr, "towire_wireaddr called!\n"); abort(); }
-/* AUTOGENERATED MOCKS END */
-
 static u8 empty_map[] = {
 	0
 };
diff --git a/plugins/renepay/test/run-mcf.c b/plugins/renepay/test/run-mcf.c
index 29714c55b5f4..031114659a32 100644
--- a/plugins/renepay/test/run-mcf.c
+++ b/plugins/renepay/test/run-mcf.c
@@ -12,28 +12,6 @@
 #include <inttypes.h>
 #include <stdio.h>
 
-/* AUTOGENERATED MOCKS START */
-/* Generated stub for fromwire_bigsize */
-bigsize_t fromwire_bigsize(const u8 **cursor UNNEEDED, size_t *max UNNEEDED)
-{ fprintf(stderr, "fromwire_bigsize called!\n"); abort(); }
-/* Generated stub for fromwire_channel_id */
-bool fromwire_channel_id(const u8 **cursor UNNEEDED, size_t *max UNNEEDED,
-			 struct channel_id *channel_id UNNEEDED)
-{ fprintf(stderr, "fromwire_channel_id called!\n"); abort(); }
-/* Generated stub for fromwire_wireaddr */
-bool fromwire_wireaddr(const u8 **cursor UNNEEDED, size_t *max UNNEEDED, struct wireaddr *addr UNNEEDED)
-{ fprintf(stderr, "fromwire_wireaddr called!\n"); abort(); }
-/* Generated stub for towire_bigsize */
-void towire_bigsize(u8 **pptr UNNEEDED, const bigsize_t val UNNEEDED)
-{ fprintf(stderr, "towire_bigsize called!\n"); abort(); }
-/* Generated stub for towire_channel_id */
-void towire_channel_id(u8 **pptr UNNEEDED, const struct channel_id *channel_id UNNEEDED)
-{ fprintf(stderr, "towire_channel_id called!\n"); abort(); }
-/* Generated stub for towire_wireaddr */
-void towire_wireaddr(u8 **pptr UNNEEDED, const struct wireaddr *addr UNNEEDED)
-{ fprintf(stderr, "towire_wireaddr called!\n"); abort(); }
-/* AUTOGENERATED MOCKS END */
-
 /* Canned gossmap, taken from tests/test_gossip.py's
  * setup_gossip_store_test via od -v -Anone -tx1 < /tmp/ltests-kaf30pn0/test_gossip_store_compact_noappend_1/lightning-2/regtest/gossip_store
  */
diff --git a/plugins/renepay/test/run-testflow.c b/plugins/renepay/test/run-testflow.c
index eb17d77de7e7..99ac9cdca7bf 100644
--- a/plugins/renepay/test/run-testflow.c
+++ b/plugins/renepay/test/run-testflow.c
@@ -15,28 +15,6 @@ static bool print_enable = true;
 #include <plugins/renepay/flow.h>
 #include "../flow.c"
 
-/* AUTOGENERATED MOCKS START */
-/* Generated stub for fromwire_bigsize */
-bigsize_t fromwire_bigsize(const u8 **cursor UNNEEDED, size_t *max UNNEEDED)
-{ fprintf(stderr, "fromwire_bigsize called!\n"); abort(); }
-/* Generated stub for fromwire_channel_id */
-bool fromwire_channel_id(const u8 **cursor UNNEEDED, size_t *max UNNEEDED,
-			 struct channel_id *channel_id UNNEEDED)
-{ fprintf(stderr, "fromwire_channel_id called!\n"); abort(); }
-/* Generated stub for fromwire_wireaddr */
-bool fromwire_wireaddr(const u8 **cursor UNNEEDED, size_t *max UNNEEDED, struct wireaddr *addr UNNEEDED)
-{ fprintf(stderr, "fromwire_wireaddr called!\n"); abort(); }
-/* Generated stub for towire_bigsize */
-void towire_bigsize(u8 **pptr UNNEEDED, const bigsize_t val UNNEEDED)
-{ fprintf(stderr, "towire_bigsize called!\n"); abort(); }
-/* Generated stub for towire_channel_id */
-void towire_channel_id(u8 **pptr UNNEEDED, const struct channel_id *channel_id UNNEEDED)
-{ fprintf(stderr, "towire_channel_id called!\n"); abort(); }
-/* Generated stub for towire_wireaddr */
-void towire_wireaddr(u8 **pptr UNNEEDED, const struct wireaddr *addr UNNEEDED)
-{ fprintf(stderr, "towire_wireaddr called!\n"); abort(); }
-/* AUTOGENERATED MOCKS END */
-	
 static const u8 canned_map[] = {
 0x0c, 0x80, 0x00, 0x01, 0xbc, 0x86, 0xe4, 0xbf, 0x95, 0x00, 0x00, 0x00, 0x00, 0x10, 0x08, 0x00, 
 0x00, 0x00, 0x00, 0x00, 0x0f, 0x42, 0x40, 0x01, 0xb0, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 

From c7b88094991217606762bc6031de90c746472092 Mon Sep 17 00:00:00 2001
From: Lagrang3 <eduardo.quintana@pm.me>
Date: Fri, 2 Jun 2023 09:19:22 +0100
Subject: [PATCH 46/64] renepay: add a print function for chan_extra_map

---
 plugins/renepay/flow.c | 26 ++++++++++++++++++++++++++
 plugins/renepay/flow.h |  4 ++++
 2 files changed, 30 insertions(+)

diff --git a/plugins/renepay/flow.c b/plugins/renepay/flow.c
index 759489e2a7d9..d602a5bbd37a 100644
--- a/plugins/renepay/flow.c
+++ b/plugins/renepay/flow.c
@@ -1,6 +1,8 @@
 #include "config.h"
 #include <assert.h>
 #include <ccan/asort/asort.h>
+#include <ccan/tal/tal.h>
+#include <ccan/tal/str/str.h>
 #include <common/gossmap.h>
 #include <math.h>
 #include <stdio.h>
@@ -13,6 +15,30 @@
 #define SUPERVERBOSE_ENABLED 1
 #endif
 
+const char *fmt_chan_extra_map(
+		const tal_t *ctx,
+		struct chan_extra_map* chan_extra_map)
+{
+	tal_t *this_ctx = tal(ctx,tal_t);
+	char *buff = tal_fmt(ctx,"Uncertainty network:\n");	
+	struct chan_extra_map_iter it;
+	for(struct chan_extra *ch = chan_extra_map_first(chan_extra_map,&it);
+	    ch;
+	    ch=chan_extra_map_next(chan_extra_map,&it))
+	{
+		const char *scid_str = 
+			type_to_string(this_ctx,struct short_channel_id,&ch->scid);
+		for(int dir=0;dir<2;++dir)
+		{
+			tal_append_fmt(&buff,"%s[%d]:(%s,%s)\n",scid_str,dir,
+				type_to_string(this_ctx,struct amount_msat,&ch->half[dir].known_min),
+				type_to_string(this_ctx,struct amount_msat,&ch->half[dir].known_max));
+		}
+	}
+	tal_free(this_ctx);
+	return buff;
+}
+
 struct chan_extra *new_chan_extra(
 		struct chan_extra_map *chan_extra_map,
 		const struct short_channel_id scid,
diff --git a/plugins/renepay/flow.h b/plugins/renepay/flow.h
index fbabf23dc58b..3792e375a409 100644
--- a/plugins/renepay/flow.h
+++ b/plugins/renepay/flow.h
@@ -112,6 +112,10 @@ HTABLE_DEFINE_TYPE(struct chan_extra,
  * 	Entropy increases unless it is already maximum.	
  * */
 
+const char *fmt_chan_extra_map(
+		const tal_t *ctx,
+		struct chan_extra_map* chan_extra_map);
+
 /* Creates a new chan_extra and adds it to the chan_extra_map. */
 struct chan_extra *new_chan_extra(
 		struct chan_extra_map *chan_extra_map,

From 09e1634043bd3f9b379e0ad8a5e4dbf839cb5ccd Mon Sep 17 00:00:00 2001
From: Lagrang3 <eduardo.quintana@pm.me>
Date: Fri, 2 Jun 2023 09:20:39 +0100
Subject: [PATCH 47/64] renepay: improve flow select in MCF

If two flows satisfy all bounds or no bounds, then we select the one
with the lowest fee. But if the fees are the same we prefer the highest
probability.
---
 plugins/renepay/mcf.c | 61 ++++++++++++++++++++++++++++++-------------
 1 file changed, 43 insertions(+), 18 deletions(-)

diff --git a/plugins/renepay/mcf.c b/plugins/renepay/mcf.c
index fee3d9f6a46c..48a0345f3d75 100644
--- a/plugins/renepay/mcf.c
+++ b/plugins/renepay/mcf.c
@@ -6,6 +6,7 @@
 #include <plugins/renepay/mcf.h>
 #include <plugins/renepay/flow.h>
 #include <plugins/renepay/dijkstra.h>
+#include <plugins/renepay/debug.h>
 #include <common/type_to_string.h>
 #include <stdint.h>
 #include <math.h>
@@ -1240,6 +1241,8 @@ static bool is_better(
 		struct amount_msat B_fee,
 		double B_prob)
 {
+	debug_call();
+	
 	bool A_fee_pass = amount_msat_less_eq(A_fee,max_fee);
 	bool B_fee_pass = amount_msat_less_eq(B_fee,max_fee);
 	bool A_prob_pass = A_prob >= min_probability;
@@ -1249,25 +1252,30 @@ static bool is_better(
 	if(A_fee_pass && B_fee_pass && A_prob_pass && B_prob_pass)
 	{
 		// prefer lower fees
-		return amount_msat_less_eq(A_fee,B_fee);
+		debug_info("all bounds are met\n");
+		goto fees_or_prob;
 	}
 	
 	// prefer the solution that satisfies both bounds
-	if((!A_fee_pass || !A_prob_pass) && (B_fee_pass && B_prob_pass))
+	if(!(A_fee_pass && A_prob_pass) && (B_fee_pass && B_prob_pass))
 	{
+		debug_info("B is better, satisfies all bounds\n");
 		return false;
 	}
 	// prefer the solution that satisfies both bounds
-	if((A_fee_pass && A_prob_pass) && (!B_fee_pass || !B_prob_pass))
+	if((A_fee_pass && A_prob_pass) && !(B_fee_pass && B_prob_pass))
 	{
+		debug_info("A is better, satisfies all bounds\n");
 		return true;
 	}
 	
 	// no solution satisfies both bounds
+	debug_info("Both bounds are not met.\n");	
 	
 	// bound on fee is met
 	if(A_fee_pass && B_fee_pass)
 	{
+		debug_info("Bound on fee is satisfies, select highest prob.\n");
 		// pick the highest prob.
 		return A_prob > B_prob;
 	}
@@ -1275,34 +1283,51 @@ static bool is_better(
 	// bound on prob. is met
 	if(A_prob_pass && B_prob_pass)
 	{
-		// pick the lowest fee
-		return amount_msat_less_eq(A_fee,B_fee);
+		debug_info("Bound on prob. is satisfies, select lowest fee.\n");
+		goto fees_or_prob;
 	}
 	
 	// prefer the solution that satisfies the bound on fees
-	if(A_fee_pass)
-	{
+	if(A_fee_pass && !B_fee_pass)
+	{	
+		debug_info("A satisfies bound on fees, B doesnt\n");	
 		return true;
 	}
-	if(B_fee_pass)
+	if(B_fee_pass && !A_fee_pass)
 	{
+		debug_info("B satisfies bound on fees, A doesnt\n");	
 		return false;
 	}
 	
 	// none of them satisfy the fee bound
+	debug_info("Fee bound is not met.\n");	
 	
 	// prefer the solution that satisfies the bound on prob.
-	if(A_prob_pass)
+	if(A_prob_pass && !B_prob_pass)
 	{
+		debug_info("A satisfies bound on prob., B doesnt\n");	
 		return true;
 	}
-	if(B_prob_pass)
+	if(B_prob_pass && !A_prob_pass)
 	{
+		debug_info("B satisfies bound on prob., A doesnt\n");	
 		return true;
 	}
 	
 	// no bound whatsoever is satisfied
+	debug_info("No bounds are met.\n");	
+	
+	fees_or_prob:
+	
+	// fees are the same, wins the highest prob.
+	if(amount_msat_eq(A_fee,B_fee))
+	{
+		debug_info("Fees are equal, select highest prob.\n");	
+		return A_prob > B_prob;
+	}
+	
 	// go for fees
+	debug_info("Select lowest fee.\n");	
 	return amount_msat_less_eq(A_fee,B_fee);
 }
 
@@ -1334,7 +1359,8 @@ struct flow** minflow(
 		double base_fee_penalty,
 		u32 prob_cost_factor )
 {
-	// printf("%s: starting\n",__PRETTY_FUNCTION__);
+	debug_call();
+	
 	tal_t *this_ctx = tal(tmpctx,tal_t);
 	
 	struct pay_parameters *params = tal(this_ctx,struct pay_parameters);	
@@ -1419,7 +1445,7 @@ struct flow** minflow(
 	{
 		
 		s64 mu = (mu_left + mu_right)/2;
-		// printf("%s: mu=%ld\n",__PRETTY_FUNCTION__,mu);
+		debug_info("mcf: mu=%ld\n",mu);
 		
 		combine_cost_function(linear_network,residual_network,mu);
 		
@@ -1436,8 +1462,9 @@ struct flow** minflow(
 						params->chan_extra_map);
 		struct amount_msat fee = flow_set_fee(flow_paths);
 		
-		// printf("prob %.2f, fee %s\n",prob_success,
-		//		type_to_string(this_ctx,struct amount_msat,&fee));
+		debug_info("mcf: prob %.2f, fee %s\n",
+			   prob_success,
+			   type_to_string(this_ctx,struct amount_msat,&fee));
 		
 		// is this better than the previous one?
 		if(!best_flow_paths || 
@@ -1455,12 +1482,12 @@ struct flow** minflow(
 		{
 			// too expensive
 			mu_left = mu+1;
-			// printf("%s: too expensive\n",__PRETTY_FUNCTION__);
+			debug_info("mcf too expensive");
 		}else if(prob_success < params->min_probability)
 		{
 			// too unlikely
 			mu_right = mu;
-			// printf("%s: too unlikely\n",__PRETTY_FUNCTION__);
+			debug_info("mcf too unlikely");
 		}else
 		{
 			// with mu constraints are satisfied, now let's optimize
@@ -1476,8 +1503,6 @@ struct flow** minflow(
 	
 	finish:
 	
-	// printf("%s: finished\n",__PRETTY_FUNCTION__);
-	
 	tal_free(this_ctx);
 	return best_flow_paths;
 }

From b92a220135fb1ffecfac59f8702b5fd5ad4ddbdf Mon Sep 17 00:00:00 2001
From: Lagrang3 <eduardo.quintana@pm.me>
Date: Fri, 9 Jun 2023 08:36:15 +0100
Subject: [PATCH 48/64] renepay: refactoring the payment mechanics

Signed-off-by: Lagrang3 <eduardo.quintana@pm.me>
---
 plugins/renepay/mcf.c      |   4 +-
 plugins/renepay/pay.c      | 733 +++++++++++++++++++++++++------------
 plugins/renepay/pay.h      | 148 ++++----
 plugins/renepay/pay_flow.c |   8 +-
 plugins/renepay/pay_flow.h |   7 +-
 5 files changed, 597 insertions(+), 303 deletions(-)

diff --git a/plugins/renepay/mcf.c b/plugins/renepay/mcf.c
index 48a0345f3d75..d31aa4e67874 100644
--- a/plugins/renepay/mcf.c
+++ b/plugins/renepay/mcf.c
@@ -1482,12 +1482,12 @@ struct flow** minflow(
 		{
 			// too expensive
 			mu_left = mu+1;
-			debug_info("mcf too expensive");
+			debug_info("mcf too expensive\n");
 		}else if(prob_success < params->min_probability)
 		{
 			// too unlikely
 			mu_right = mu;
-			debug_info("mcf too unlikely");
+			debug_info("mcf too unlikely\n");
 		}else
 		{
 			// with mu constraints are satisfied, now let's optimize
diff --git a/plugins/renepay/pay.c b/plugins/renepay/pay.c
index b5e3f14e43a7..8476cf1b80e7 100644
--- a/plugins/renepay/pay.c
+++ b/plugins/renepay/pay.c
@@ -22,9 +22,133 @@ struct pay_plugin *pay_plugin;
 
 
 static void renepay_cleanup(struct active_payment *ap);
+static void timer_kick(struct payment *p);
+
+static void payment_check_delivering_incomplete(struct payment *p)
+{
+	if(!amount_msat_less(p->total_delivering, p->amount))
+	{
+		plugin_err(
+			pay_plugin->plugin,
+			"Strange, delivering (%s) is not smaller than amount (%s)",
+			type_to_string(tmpctx,struct amount_msat,&p->total_delivering),
+			type_to_string(tmpctx,struct amount_msat,&p->amount));
+	}
+}
+static void payment_check_delivering_all(struct payment *p)
+{
+	if(amount_msat_less(p->total_delivering, p->amount))
+	{
+		plugin_err(
+			pay_plugin->plugin,
+			"Strange, delivering (%s) is less than amount (%s)",
+			type_to_string(tmpctx,struct amount_msat,&p->total_delivering),
+			type_to_string(tmpctx,struct amount_msat,&p->amount));
+	}
+}
+static struct command * payment_command(struct payment *p)
+{
+	return p->active_payment->cmd;
+}
+static u64 payment_parts(const struct payment *p)
+{
+	return p->active_payment->next_partid-1;
+}
+int payment_current_attempt(const struct payment *p)
+{
+	return p->active_payment->last_attempt;
+}
+static int payment_attempt_count(const struct payment *p)
+{
+	return p->active_payment->last_attempt+1;
+}
+static void payment_new_attempt(struct payment *p)
+{
+	p->status=PAYMENT_PENDING;
+	p->active_payment->last_attempt++;
+}
+static u64 payment_groupid(struct payment *p)
+{
+	return p->active_payment->groupid;
+}
+static void payment_initialize(
+		struct payment *p,
+		struct command *cmd)
+{
+	/* Owned by cmd, because this data is destroyed after the payment
+	 * completes, while p remains to get payment status history. */
+	p->active_payment = tal(cmd,struct active_payment);
+	tal_add_destructor(p->active_payment, renepay_cleanup);
+	
+	p->active_payment->cmd = cmd;
+ 	p->active_payment->local_gossmods = gossmap_localmods_new(p->active_payment);
+	p->active_payment->localmods_applied=false;
+	p->active_payment->disabled = tal_arr(p->active_payment, 
+					      struct short_channel_id, 
+					      0);
+	p->active_payment->rexmit_timer = NULL;
+	p->active_payment->last_attempt=-1;
+	p->active_payment->all_flows = tal(p->active_payment,tal_t);
+}
+// static struct amount_msat payment_fees(struct payment *p)
+// {
+// 	struct amount_msat fees;
+// 	struct amount_msat sent = payment_sent(p), 
+// 			   received = payment_amount(p);
+// 			   
+// 	if(!amount_msat_sub(&fees,sent,received))
+// 		plugin_err(payment_command(p)->plugin,
+// 			   "Strange, sent amount (%s) is less than received (%s), aborting.",
+// 			   type_to_string(tmpctx,struct amount_msat,&sent),
+// 			   type_to_string(tmpctx,struct amount_msat,&received));
+// 	return fees;
+// }
+
+static struct command_result *payment_success(struct payment *p)
+{
+	debug_call();
+	
+	payment_check_delivering_all(p);
+	
+	struct json_stream *response 
+		= jsonrpc_stream_success(payment_command(p));
+
+	/* Any one succeeding is success. */
+	json_add_preimage(response, "payment_preimage", p->preimage);
+	json_add_sha256(response, "payment_hash", &p->payment_hash);
+	json_add_timeabs(response, "created_at", p->start_time);
+	json_add_u32(response, "parts", payment_parts(p));
+	json_add_amount_msat_only(response, "amount_msat",
+				  p->amount);
+	json_add_amount_msat_only(response, "amount_sent_msat",
+				  p->total_sent);
+	json_add_string(response, "status", "complete");
+	json_add_node_id(response, "destination", &p->dest);
+	
+	return command_finished(payment_command(p), response);
+}
+
+// static struct command_result *payment_fail(struct payment *p)
+// {
+// 	plugin_log(pay_plugin->plugin,LOG_DBG,"calling %s",__PRETTY_FUNCTION__);
+// 	debug_call();
+// 	
+// 	p->status = PAYMENT_FAIL;
+// 	
+// 	/* In this version we do a MPP, which means either all payments fail or
+// 	 * all payments succeed. 
+// 	 * */
+// 	p->total_delivering = AMOUNT_MSAT(0);
+// 	p->total_sent = AMOUNT_MSAT(0);
+// 	
+// 	/* Still waiting for timer... */
+// 	return command_still_pending(payment_command(p));
+// }
 
 // TODO(eduardo): check if knowledge is updated after payment success
 // TODO(eduardo): how do we fail a payment
+// TODO(eduardo): improve loging
+// TODO(eduardo): handle hanging responses
 
 void paynote(struct payment *p, const char *fmt, ...)
 {
@@ -71,6 +195,36 @@ static void memleak_mark(struct plugin *p, struct htable *memtable)
 }
 #endif
 
+static void remove_htlc_payflow_and_update_knowlege(
+		struct pay_flow *flow,
+		struct chan_extra_map *chan_extra_map)
+{
+	debug_call();
+	
+	// TODO(eduardo): big assumption here, HTLCs can just simply dissapear
+	// if the MPP payment completes or the flow fails.
+	remove_htlc_payflow(flow,chan_extra_map);
+	
+	struct payment *p = flow->payment;
+	
+	switch(p->status)
+	{
+		case PAYMENT_SUCCESS:
+			for (size_t i = 0; i < tal_count(flow->path_scids); i++)
+			{
+				chan_extra_sent_success(pay_plugin->chan_extra_map,
+					flow->path_scids[i],
+					flow->path_dirs[i],
+					flow->amounts[i]);
+			}
+			break;
+		case PAYMENT_FAIL:
+		case PAYMENT_PENDING:
+		case PAYMENT_MPP_TIMEOUT:
+			break;
+	}
+}
+
 static const char *init(struct plugin *p,
 			const char *buf UNUSED, const jsmntok_t *config UNUSED)
 {
@@ -234,6 +388,7 @@ static void add_hintchan(struct payment *p,
 }
 
 /* Add routehints provided by bolt11 */
+// TODO(eduardo): check this again
 static void uncertainty_network_add_routehints(struct payment *p)
 {
 	struct bolt11 *b11;
@@ -398,93 +553,115 @@ static struct command_result *try_paying(struct command *cmd,
 					 struct payment *p,
 					 bool first_time);
 
-static struct command_result *flow_failed(struct command *cmd,
-					  const struct pay_flow *flow)
+static struct command_result *flow_failed(
+		struct command *cmd,
+		const struct pay_flow *flow)
 {
 	plugin_log(pay_plugin->plugin,LOG_DBG,"calling flow_failed");
-	debug_call(__PRETTY_FUNCTION__,MYLOG);
+	debug_call();
 	struct payment *p = flow->payment;
-
+	
+	
 	amount_msat_reduce(&p->total_delivering, flow_delivered(flow));
 	amount_msat_reduce(&p->total_sent, flow->amounts[0]);
-
-	/* flow failed, so we remove these HTLCs */
-	remove_htlc_payflow(pay_plugin->chan_extra_map,
-			    flow);
-
-	/* If nothing outstanding, don't wait for timer! */
-	if (amount_msat_eq(p->total_sent, AMOUNT_MSAT(0))) {
-		p->active_payment->rexmit_timer 
-			= tal_free(p->active_payment->rexmit_timer);
-		return try_paying(cmd, p, false);
-	}
+	tal_free(flow);
 	
 	/* Still waiting for timer... */
 	return command_still_pending(cmd);
 }
 
-/* Happens when timer goes off, but also works to arm timer if nothing to do */
-static void timer_kick(struct payment *p)
+
+// // TODO(eduardo): how do we treat the knowledge from old flow attempts?
+// // TODO(eduardo
+// static struct command_result payment_success(...)
+// {
+// 	for each pending flow:
+// 		if flow is current attempt
+// 			update knowledge flow was sent
+// 		free flow
+// 	
+// 	return command_finished
+// }
+
+
+static void payment_settimer(struct payment *p)
 {
-	debug_call(__PRETTY_FUNCTION__,MYLOG);
 	p->active_payment->rexmit_timer 
 		= tal_free(p->active_payment->rexmit_timer);
+	p->active_payment->rexmit_timer 
+		= plugin_timer(
+			pay_plugin->plugin,
+			time_from_msec(250),
+			timer_kick, p);
+}
 
-	/* Nothing has come back?  Re-arm timer. */
-	if (amount_msat_eq(p->total_delivering, p->amount)) {
-		// TODO(eduardo): what's the purpose of this?
-		// p->rexmit_timer = plugin_timer(p->cmd->plugin,
-		// 			       time_from_msec(250),
-		// 			       timer_kick, p);
-		return;
+/* Happens when timer goes off, but also works to arm timer if nothing to do */
+static void timer_kick(struct payment *p)
+{
+	debug_call();
+	
+	/* No, but payment hasn't gone through, so let's wait a bit. */
+	
+	switch(p->status)
+	{
+		/* Some flows succeeded, we finish the payment. */
+		case PAYMENT_SUCCESS:
+			payment_check_delivering_all(p);
+			payment_success(p);
+		break;
+		
+		/* Some flows failed, we retry. */
+		case PAYMENT_FAIL:
+			payment_check_delivering_incomplete(p);
+			try_paying(payment_command(p),p,false);
+		break;
+		
+		/* Nothing has returned yet, we have to wait. */
+		case PAYMENT_PENDING:
+			payment_check_delivering_all(p);
+			payment_settimer(p);
+		break;
+		/* Some flows timeout we wait until all of them have done so. */
+		case PAYMENT_MPP_TIMEOUT:
+			payment_check_delivering_incomplete(p);
+			if(amount_msat_eq(p->total_delivering,AMOUNT_MSAT(0)))
+			{
+				try_paying(payment_command(p),p,false);
+			}
+			payment_settimer(p);
+		break;
 	}
-
-	try_paying(p->active_payment->cmd, p, false);
 }
 
+/* Once this function is called we know that the payment succeeded and we
+ * terminate with payment_success. */
 static struct command_result *waitsendpay_succeeded(struct command *cmd,
 						    const char *buf,
 						    const jsmntok_t *result,
 						    struct pay_flow *flow)
 {
 	plugin_log(pay_plugin->plugin,LOG_DBG,"calling waitsendpay_succeeded");
-	debug_call(__PRETTY_FUNCTION__,MYLOG);
-	
-	/* payment succeeded, now remove the HTLCs */
-	remove_htlc_payflow(pay_plugin->chan_extra_map,
-			    flow);
+	debug_call();
 	
-	/* update the knowledge */
-	for (size_t i = 0; i < tal_count(flow->path_scids); i++)
-	{
-		chan_extra_sent_success(pay_plugin->chan_extra_map,
-					flow->path_scids[i],
-					flow->path_dirs[i],
-					flow->amounts[i]);
-	}
 	struct payment *p = flow->payment;
-	struct json_stream *response = jsonrpc_stream_success(cmd);
-	const jsmntok_t *preimagetok;
-
-	preimagetok = json_get_member(buf, result, "payment_preimage");
-	if (!preimagetok)
+	
+	/* This is a success. */
+	p->status = PAYMENT_SUCCESS;
+	
+	const jsmntok_t *preimagetok
+		= json_get_member(buf, result, "payment_preimage");
+	
+	struct preimage preimage;
+	
+	if (!preimagetok || !json_to_preimage(buf, preimagetok,&preimage))
 		plugin_err(cmd->plugin,
 			   "Strange return from waitsendpay: %.*s",
 			   json_tok_full_len(result),
 			   json_tok_full(buf, result));
-
-	/* Any one succeeding is success. */
-	json_add_tok(response, "payment_preimage", preimagetok, buf);
-	json_add_sha256(response, "payment_hash", &p->payment_hash);
-	json_add_timeabs(response, "created_at", p->start_time);
-	json_add_u32(response, "parts", p->next_partid);
-	json_add_amount_msat_only(response, "amount_msat",
-				  p->total_delivering);
-	json_add_amount_msat_only(response, "amount_sent_msat",
-				  p->total_sent);
-	json_add_string(response, "status", "complete");
-	json_add_node_id(response, "destination", &p->dest);
-	return command_finished(cmd, response);
+	
+	p->preimage = tal_dup_or_null(p, struct preimage, &preimage);
+	
+	return payment_success(p);
 }
 
 static const char *flow_path_to_str(const tal_t *ctx, const struct pay_flow *flow)
@@ -503,7 +680,7 @@ static struct command_result *handle_unhandleable_error(struct payment *p,
 							const struct pay_flow *flow,
 							const char *what)
 {
-	debug_call(__PRETTY_FUNCTION__,MYLOG);
+	debug_call();
 	size_t n = tal_count(flow);
 
 	/* We got a mangled reply.  We don't know who to penalize! */
@@ -540,125 +717,125 @@ struct addgossip {
 	const struct pay_flow *flow;
 };
 
-static struct command_result *addgossip_done(struct command *cmd,
-					     const char *buf,
-					     const jsmntok_t *err,
-					     struct addgossip *adg)
-{
-	debug_call(__PRETTY_FUNCTION__,MYLOG);
-	struct payment *p = adg->flow->payment;
-
-	/* Release this: if it's the last flow we'll retry immediately */
-	tal_free(adg);
-	timer_kick(p);
-
-	return command_still_pending(cmd);
-}
-
-static struct command_result *addgossip_failure(struct command *cmd,
-						const char *buf,
-						const jsmntok_t *err,
-						struct addgossip *adg)
-
-{
-	debug_call(__PRETTY_FUNCTION__,MYLOG);
-	struct payment *p = adg->flow->payment;
-
-	paynote(p, "addgossip failed, removing channel %s (%.*s)",
-		type_to_string(tmpctx, struct short_channel_id, &adg->scid),
-		err->end - err->start, buf + err->start);
-	tal_arr_expand(&p->active_payment->disabled, adg->scid);
-
-	return addgossip_done(cmd, buf, err, adg);
-}
-
-// TODO(eduardo): what is this?
-static struct command_result *submit_update(struct command *cmd,
-					    const struct pay_flow *flow,
-					    const u8 *update,
-					    struct short_channel_id errscid)
-{
-	plugin_log(pay_plugin->plugin,LOG_DBG,"calling submit_update");
-	debug_call(__PRETTY_FUNCTION__,MYLOG);
-	struct payment *p = flow->payment;
-	struct out_req *req;
-	struct addgossip *adg = tal(cmd, struct addgossip);
-
-	/* We need to stash scid in case this fails, and we need to hold flow so
-	 * we don't get a rexmit before this is complete. */
-	adg->scid = errscid;
-	adg->flow = tal_steal(adg, flow);
-	/* Disable re-xmit until this returns */
-	p->active_payment->rexmit_timer 
-		= tal_free(p->active_payment->rexmit_timer);
+// static struct command_result *addgossip_done(struct command *cmd,
+// 					     const char *buf,
+// 					     const jsmntok_t *err,
+// 					     struct addgossip *adg)
+// {
+// 	debug_call();
+// 	struct payment *p = adg->flow->payment;
+// 
+// 	/* Release this: if it's the last flow we'll retry immediately */
+// 	tal_free(adg);
+// 	timer_kick(p);
+// 
+// 	return command_still_pending(cmd);
+// }
 
-	paynote(p, "... extracted channel_update, telling gossipd");
-	plugin_log(pay_plugin->plugin, LOG_DBG, "(update = %s)", tal_hex(tmpctx, update));
+// static struct command_result *addgossip_failure(struct command *cmd,
+// 						const char *buf,
+// 						const jsmntok_t *err,
+// 						struct addgossip *adg)
+// 
+// {
+// 	debug_call();
+// 	struct payment *p = adg->flow->payment;
+// 
+// 	paynote(p, "addgossip failed, removing channel %s (%.*s)",
+// 		type_to_string(tmpctx, struct short_channel_id, &adg->scid),
+// 		err->end - err->start, buf + err->start);
+// 	tal_arr_expand(&p->active_payment->disabled, adg->scid);
+// 
+// 	return addgossip_done(cmd, buf, err, adg);
+// }
 
-	req = jsonrpc_request_start(pay_plugin->plugin, NULL, "addgossip",
-				    addgossip_done,
-				    addgossip_failure,
-				    adg);
-	json_add_hex_talarr(req->js, "message", update);
-	return send_outreq(pay_plugin->plugin, req);
-}
+// static struct command_result *submit_update(struct command *cmd,
+// 					    const struct pay_flow *flow,
+// 					    const u8 *update,
+// 					    struct short_channel_id errscid)
+// {
+// 	plugin_log(pay_plugin->plugin,LOG_DBG,"calling submit_update");
+// 	debug_call();
+// 	struct payment *p = flow->payment;
+// 	struct out_req *req;
+// 	struct addgossip *adg = tal(cmd, struct addgossip);
+// 
+// 	/* We need to stash scid in case this fails, and we need to hold flow so
+// 	 * we don't get a rexmit before this is complete. */
+// 	adg->scid = errscid;
+// 	adg->flow = tal_steal(adg, flow);
+// 	/* Disable re-xmit until this returns */
+// 	p->active_payment->rexmit_timer 
+// 		= tal_free(p->active_payment->rexmit_timer);
+// 
+// 	paynote(p, "... extracted channel_update, telling gossipd");
+// 	plugin_log(pay_plugin->plugin, LOG_DBG, "(update = %s)", tal_hex(tmpctx, update));
+// 
+// 	req = jsonrpc_request_start(pay_plugin->plugin, NULL, "addgossip",
+// 				    addgossip_done,
+// 				    addgossip_failure,
+// 				    adg);
+// 	json_add_hex_talarr(req->js, "message", update);
+// 	return send_outreq(pay_plugin->plugin, req);
+// }
 
 /* Fix up the channel_update to include the type if it doesn't currently have
  * one. See ElementsProject/lightning#1730 and lightningnetwork/lnd#1599 for the
  * in-depth discussion on why we break message parsing here... */
-static u8 *patch_channel_update(const tal_t *ctx, u8 *channel_update TAKES)
-{
-	u8 *fixed;
-	if (channel_update != NULL &&
-	    fromwire_peektype(channel_update) != WIRE_CHANNEL_UPDATE) {
-		/* This should be a channel_update, prefix with the
-		 * WIRE_CHANNEL_UPDATE type, but isn't. Let's prefix it. */
-		fixed = tal_arr(ctx, u8, 0);
-		towire_u16(&fixed, WIRE_CHANNEL_UPDATE);
-		towire(&fixed, channel_update, tal_bytelen(channel_update));
-		if (taken(channel_update))
-			tal_free(channel_update);
-		return fixed;
-	} else {
-		return tal_dup_talarr(ctx, u8, channel_update);
-	}
-}
+// static u8 *patch_channel_update(const tal_t *ctx, u8 *channel_update TAKES)
+// {
+// 	u8 *fixed;
+// 	if (channel_update != NULL &&
+// 	    fromwire_peektype(channel_update) != WIRE_CHANNEL_UPDATE) {
+// 		/* This should be a channel_update, prefix with the
+// 		 * WIRE_CHANNEL_UPDATE type, but isn't. Let's prefix it. */
+// 		fixed = tal_arr(ctx, u8, 0);
+// 		towire_u16(&fixed, WIRE_CHANNEL_UPDATE);
+// 		towire(&fixed, channel_update, tal_bytelen(channel_update));
+// 		if (taken(channel_update))
+// 			tal_free(channel_update);
+// 		return fixed;
+// 	} else {
+// 		return tal_dup_talarr(ctx, u8, channel_update);
+// 	}
+// }
+
 
 /* Return NULL if the wrapped onion error message has no channel_update field,
  * or return the embedded channel_update message otherwise. */
-static u8 *channel_update_from_onion_error(const tal_t *ctx,
-					   const char *buf,
-					   const jsmntok_t *onionmsgtok)
-{
-	u8 *channel_update = NULL;
-	struct amount_msat unused_msat;
-	u32 unused32;
-	u8 *onion_message = json_tok_bin_from_hex(tmpctx, buf, onionmsgtok);
-
-	/* Identify failcodes that have some channel_update.
-	 *
-	 * TODO > BOLT 1.0: Add new failcodes when updating to a
-	 * new BOLT version. */
-	if (!fromwire_temporary_channel_failure(ctx,
-						onion_message,
-						&channel_update) &&
-	    !fromwire_amount_below_minimum(ctx,
-					   onion_message, &unused_msat,
-					   &channel_update) &&
-	    !fromwire_fee_insufficient(ctx,
-		    		       onion_message, &unused_msat,
-				       &channel_update) &&
-	    !fromwire_incorrect_cltv_expiry(ctx,
-		    			    onion_message, &unused32,
-					    &channel_update) &&
-	    !fromwire_expiry_too_soon(ctx,
-		    		      onion_message,
-				      &channel_update))
-		/* No channel update. */
-		return NULL;
-
-	return patch_channel_update(ctx, take(channel_update));
-}
+// static u8 *channel_update_from_onion_error(const tal_t *ctx,
+// 					   const char *buf,
+// 					   const jsmntok_t *onionmsgtok)
+// {
+// 	u8 *channel_update = NULL;
+// 	struct amount_msat unused_msat;
+// 	u32 unused32;
+// 	u8 *onion_message = json_tok_bin_from_hex(tmpctx, buf, onionmsgtok);
+// 
+// 	/* Identify failcodes that have some channel_update.
+// 	 *
+// 	 * TODO > BOLT 1.0: Add new failcodes when updating to a
+// 	 * new BOLT version. */
+// 	if (!fromwire_temporary_channel_failure(ctx,
+// 						onion_message,
+// 						&channel_update) &&
+// 	    !fromwire_amount_below_minimum(ctx,
+// 					   onion_message, &unused_msat,
+// 					   &channel_update) &&
+// 	    !fromwire_fee_insufficient(ctx,
+// 		    		       onion_message, &unused_msat,
+// 				       &channel_update) &&
+// 	    !fromwire_incorrect_cltv_expiry(ctx,
+// 		    			    onion_message, &unused32,
+// 					    &channel_update) &&
+// 	    !fromwire_expiry_too_soon(ctx,
+// 		    		      onion_message,
+// 				      &channel_update))
+// 		/* No channel update. */
+// 		return NULL;
+// 
+// 	return patch_channel_update(ctx, take(channel_update));
+// }
 
 static struct command_result *waitsendpay_failed(struct command *cmd,
 						 const char *buf,
@@ -666,16 +843,31 @@ static struct command_result *waitsendpay_failed(struct command *cmd,
 						 struct pay_flow *flow)
 {
 	plugin_log(pay_plugin->plugin,LOG_DBG,"calling waitsendpay_failed");
-	debug_call(__PRETTY_FUNCTION__,MYLOG);
-	debug_reply(buf,err,MYLOG);
+	debug_call();
+	debug_reply(buf,err);
 	
 	struct payment *p = flow->payment;
+	
+	/* This is a fail. */
+	p->status = PAYMENT_FAIL;
+	
+	/* An old flow attempt. Just free the HTLCs. */
+	if(flow->attempt != payment_current_attempt(p))
+	{
+		// TODO(eduardo): I guess this should not happen unless some
+		// sendpay that would eventually fail gets stuck for some
+		// reason or some MPP part times out.
+		debug_info("waitsendpay_failed: received an old failure");
+	}
+
 	u64 errcode;
 	struct command_result *ret;
-	const jsmntok_t *datatok, *msgtok, *errchantok, *erridxtok, *failcodetok, *rawoniontok;
+	const jsmntok_t *datatok, *msgtok, *errchantok, *erridxtok, *failcodetok;
 	u32 onionerr, erridx;
 	struct short_channel_id errscid;
-	const u8 *update;
+	
+	// const u8 *update;
+	// const jsmntok_t *rawoniontok;
 
 	if (!json_to_u64(buf, json_get_member(buf, err, "code"), &errcode))
 		plugin_err(cmd->plugin, "Bad errcode from waitsendpay: %.*s",
@@ -683,17 +875,22 @@ static struct command_result *waitsendpay_failed(struct command *cmd,
 
 	switch (errcode) {
 	case PAY_UNPARSEABLE_ONION:
+		// TODO(eduardo): let's check this case carefully.
+		debug_info("waitsendpay_failed: PAY_UNPARSEABLE_ONION");
 		ret = handle_unhandleable_error(p, flow, "unparsable onion reply");
 		if (ret)
 			return ret;
 		goto done;
 	case PAY_DESTINATION_PERM_FAIL:
+		debug_info("waitsendpay_failed: PAY_DESTINATION_PERM_FAIL");
 		return command_fail(cmd, PAY_DESTINATION_PERM_FAIL,
 				    "Got an final failure from destination");
 	case PAY_TRY_OTHER_ROUTE:
+		debug_info("waitsendpay_failed: PAY_TRY_OTHER_ROUTE");
 		break;
 
 	default:
+		debug_info("waitsendpay_failed: unexpected errcode");
 		plugin_err(cmd->plugin,
 			   "Unexpected errcode from waitsendpay: %.*s",
 			   json_tok_full_len(err), json_tok_full(buf, err));
@@ -712,7 +909,12 @@ static struct command_result *waitsendpay_failed(struct command *cmd,
 	json_to_u32(buf, erridxtok, &erridx);
 	errchantok = json_get_member(buf, datatok, "erring_channel");
 	json_to_short_channel_id(buf, errchantok, &errscid);
-	if (!short_channel_id_eq(&errscid, &flow->path_scids[erridx]))
+	
+	/* TODO(eduardo): can we assume that if
+	 * erridx=tal_count(flow->path_scids), ie. one index past the last one, the failure 
+	 * is due to expired HTLCs? How can we handle this case? */
+	if (erridx<tal_count(flow->path_scids) 
+	    && !short_channel_id_eq(&errscid, &flow->path_scids[erridx]))
 		plugin_err(pay_plugin->plugin, "Erring channel %u/%zu was %s not %s (path %s)",
 			   erridx, tal_count(flow->path_scids),
 			   type_to_string(tmpctx, struct short_channel_id, &errscid),
@@ -723,17 +925,19 @@ static struct command_result *waitsendpay_failed(struct command *cmd,
 	json_to_u32(buf, failcodetok, &onionerr);
 
 	msgtok = json_get_member(buf, err, "message");
-	rawoniontok = json_get_member(buf, datatok, "raw_message");
-	
-	// const char *scid_str = type_to_string(tmpctx,struct short_channel_id,&errscid);
-	// debug(MYLOG,"failcode: %d, onion error: %s, err_idx: %d, scid: %s\n",
-	// 	onionerr,onion_wire_name(onionerr),erridx,scid_str);
+	// rawoniontok = json_get_member(buf, datatok, "raw_message");
 	
 	paynote(p, "onion error %s from node #%u %s: %.*s",
 		onion_wire_name(onionerr),
 		erridx,
 		type_to_string(tmpctx, struct short_channel_id, &errscid),
 		msgtok->end - msgtok->start, buf + msgtok->start);
+		
+	debug_info("onion error %s from node #%u %s: %.*s",
+		onion_wire_name(onionerr),
+		erridx,
+		type_to_string(tmpctx, struct short_channel_id, &errscid),
+		msgtok->end - msgtok->start, buf + msgtok->start);
 
 	/* All these parts succeeded, so we know something about min
 	 * capacity! */
@@ -763,6 +967,7 @@ static struct command_result *waitsendpay_failed(struct command *cmd,
 	case WIRE_INVALID_ONION_PAYLOAD:
 	case WIRE_INVALID_ONION_BLINDING:
 	case WIRE_EXPIRY_TOO_FAR:
+		debug_info("waitsendpay_failed: removing scid");
 		paynote(p, "... so we're removing scid");
 		tal_arr_expand(&p->active_payment->disabled, errscid);
 		goto done;
@@ -772,10 +977,12 @@ static struct command_result *waitsendpay_failed(struct command *cmd,
 	case WIRE_FEE_INSUFFICIENT:
 	case WIRE_INCORRECT_CLTV_EXPIRY:
 	case WIRE_EXPIRY_TOO_SOON:
+		debug_info("waitsendpay_failed: apply channel_update");
 		/* FIXME: Check scid! */
-		update = channel_update_from_onion_error(tmpctx, buf, rawoniontok);
-		if (update)
-			return submit_update(cmd, flow, update, errscid);
+		// TODO(eduardo): let's check this case carefully.
+		// update = channel_update_from_onion_error(tmpctx, buf, rawoniontok);
+		// if (update)
+		// 	return submit_update(cmd, flow, update, errscid);
 
 		paynote(p, "... missing an update, so we're removing scid");
 		tal_arr_expand(&p->active_payment->disabled, errscid);
@@ -783,6 +990,7 @@ static struct command_result *waitsendpay_failed(struct command *cmd,
 
 	/* Insufficient funds! */
 	case WIRE_TEMPORARY_CHANNEL_FAILURE: {
+		debug_info("waitsendpay_failed: Insufficient funds!");
 		/* OK new max is amount - 1 */
 		struct amount_msat max_possible;
 		if (!amount_msat_sub(&max_possible,
@@ -799,17 +1007,19 @@ static struct command_result *waitsendpay_failed(struct command *cmd,
 		goto done;
 	}
 
-	/* We ignore this, assuming we want to keep going. */
 	/* FIXME: We should probably pause until all parts returned, or at least
 	 * extend rexmit timer! */
 	case WIRE_MPP_TIMEOUT:
+		debug_info("waitsendpay_failed: WIRE_MPP_TIMEOUT");
 		paynote(p, "... will continue");
+		p->status = PAYMENT_MPP_TIMEOUT;
 		goto done;
 
 	/* These are from the final distination: fail */
 	case WIRE_INCORRECT_OR_UNKNOWN_PAYMENT_DETAILS:
 	case WIRE_FINAL_INCORRECT_CLTV_EXPIRY:
 	case WIRE_FINAL_INCORRECT_HTLC_AMOUNT:
+		debug_info("waitsendpay_failed: final destination fail");
 		paynote(p, "... fatal");
 		return command_fail(cmd, PAY_DESTINATION_PERM_FAIL,
 				    "Destination said %s: %.*s",
@@ -819,6 +1029,10 @@ static struct command_result *waitsendpay_failed(struct command *cmd,
 	}
 	/* Unknown response? */
 	if (erridx == tal_count(flow->path_nodes)) {
+		debug_info("waitsendpay_failed: unknown error code %u: %.*s",
+				    onionerr,
+				    msgtok->end - msgtok->start,
+				    buf + msgtok->start);
 		paynote(p, "... fatal");
 		return command_fail(cmd, PAY_DESTINATION_PERM_FAIL,
 				    "Destination gave unknown error code %u: %.*s",
@@ -833,6 +1047,12 @@ static struct command_result *waitsendpay_failed(struct command *cmd,
 				  &flow->path_nodes[erridx]),
 		   erridx, tal_count(flow->path_nodes),
 		   onionerr);
+	debug_info(
+		   "Node %s (%u/%zu) gave unknown error code %u",
+		   type_to_string(tmpctx, struct node_id,
+				  &flow->path_nodes[erridx]),
+		   erridx, tal_count(flow->path_nodes),
+		   onionerr);
 	tal_arr_expand(&p->active_payment->disabled, errscid);
 
 done:
@@ -846,7 +1066,8 @@ static struct command_result *flow_sent(struct command *cmd,
 					struct pay_flow *flow)
 {
 	plugin_log(pay_plugin->plugin,LOG_DBG,"calling flow_sent");
-	debug_call(__PRETTY_FUNCTION__,MYLOG);
+	debug_call();
+	
 	struct payment *p = flow->payment;
 	struct out_req *req;
 
@@ -855,12 +1076,10 @@ static struct command_result *flow_sent(struct command *cmd,
 				    waitsendpay_failed,
 				    cast_const(struct pay_flow *, flow));
 	json_add_sha256(req->js, "payment_hash", &p->payment_hash);
-	json_add_u64(req->js, "groupid", p->groupid);
+	json_add_u64(req->js, "groupid", payment_groupid(p));
 	json_add_u64(req->js, "partid", flow->partid);
 	/* FIXME: We don't set timeout... */
 
-	/* Now flow is owned by this request */
-	tal_steal(req, flow);
 	return send_outreq(cmd->plugin, req);
 }
 
@@ -874,8 +1093,12 @@ static struct command_result *flow_sendpay_failed(struct command *cmd,
 						  struct pay_flow *flow)
 {
 	plugin_log(pay_plugin->plugin,LOG_DBG,"calling flow_sendpay_failed");
-	debug_call(__PRETTY_FUNCTION__,MYLOG);
+	debug_call();
+	
 	struct payment *p = flow->payment;
+	/* This is a fail. */
+	p->status = PAYMENT_FAIL;
+	
 	u64 errcode;
 	const jsmntok_t *msg = json_get_member(buf, err, "message");
 
@@ -890,18 +1113,33 @@ static struct command_result *flow_sendpay_failed(struct command *cmd,
 	paynote(p,
 		"sendpay didn't like first hop, eliminated: %.*s",
 		msg->end - msg->start, buf + msg->start);
+		
+	/* There is no new knowledge from this kind of failure. 
+	 * We just disable this scid. */
 	tal_arr_expand(&p->active_payment->disabled, flow->path_scids[0]);
+	
 	return flow_failed(cmd, flow);
 }
 
+
 static struct command_result *
 sendpay_flows(struct command *cmd,
 	      struct payment *p,
 	      struct pay_flow **flows STEALS)
 {
 	plugin_log(pay_plugin->plugin,LOG_DBG,"calling sendpay_flows");
-	debug_call(__PRETTY_FUNCTION__,MYLOG);
+	debug_call();
 	paynote(p, "Sending out batch of %zu payments", tal_count(flows));
+	
+	// TODO(eduardo): mark here the status of the payment parts as either:
+	// 'pending', 'success' or 'fail'. If all of them fail then `try_paying`
+	// again, if instead all of them succeed then return `command_finished`,
+	// while there is one payment pending you return
+	// `command_still_pending`. Maybe the timer could be useful here.
+	// If all of the parts are processed but some are fail and some are
+	// success, then it means there is something wrong with my understanding
+	// and we need to rethink here.
+	
 	for (size_t i = 0; i < tal_count(flows); i++) {
 		struct out_req *req;
 		req = jsonrpc_request_start(cmd->plugin, cmd, "sendpay",
@@ -929,11 +1167,13 @@ sendpay_flows(struct command *cmd,
 		json_add_sha256(req->js, "payment_hash", &p->payment_hash);
 		json_add_secret(req->js, "payment_secret", p->payment_secret);
 		
+		/* TODO(eduardo): should this be p->amount or
+		 * p->total_delivering? */
 		json_add_amount_msat_only(req->js, "amount_msat", p->amount);
 			
 		json_add_u64(req->js, "partid", flows[i]->partid);
 		
-		json_add_u64(req->js, "groupid", p->groupid);
+		json_add_u64(req->js, "groupid", payment_groupid(p));
 		if (p->payment_metadata)
 			json_add_hex_talarr(req->js, "payment_metadata",
 					    p->payment_metadata);
@@ -946,25 +1186,36 @@ sendpay_flows(struct command *cmd,
 			json_add_string(req->js, "description", p->description);
 		
 		// TODO(eduardo): remove this line
-		debug_outreq(req,MYLOG);
+		debug_outreq(req);
 		
 		amount_msat_accumulate(&p->total_sent, flows[i]->amounts[0]);
 		amount_msat_accumulate(&p->total_delivering,
 				       flow_delivered(flows[i]));
 		
-		/* Flow now owned by request */
-		tal_steal(req, flows[i]);
+		/* Flow now owned by all_flows instead of req., in this way we
+		 * can control the destruction occurs before we remove temporary
+		 * channels from chan_extra_map. */
+		tal_steal(p->active_payment->all_flows,flows[i]);
 		
 		/* record these HTLC along the flow path */
 		commit_htlc_payflow(pay_plugin->chan_extra_map,flows[i]);
+		
+		/* Remove the HTLC from the chan_extra_map after finish. */
+		tal_add_destructor2(flows[i],
+				    remove_htlc_payflow_and_update_knowlege,
+				    pay_plugin->chan_extra_map);
+		
 		send_outreq(cmd->plugin, req);
 	}
 	
+	/* Safety check. */
+	payment_check_delivering_all(p);
+	
+	tal_free(flows);
+	
 	/* Get ready to process replies */
-	// TODO(eduardo): how is the timer mechanics?
-	timer_kick(p);
+	payment_settimer(p);
 	
-	// TODO(eduardo): what is this actually doing?
 	return command_still_pending(cmd);
 }
 
@@ -973,16 +1224,24 @@ static struct command_result *try_paying(struct command *cmd,
 					 bool first_time)
 {
 	plugin_log(pay_plugin->plugin,LOG_DBG,"calling try_paying");
-	debug_call(__PRETTY_FUNCTION__,MYLOG);
+	debug_call();
 	
-	// TODO(eduardo): we should move on only if all rpc requests are done
-	// (all threads join)
+	// TODO(eduardo): does it make sense to have this limit on attempts?
+	/* I am classifying the flows in attempt cycles. */
+	payment_new_attempt(p);
+	/* We try only MAX_NUM_ATTEMPTS, then we give up. */
+	if ( payment_attempt_count(p) > MAX_NUM_ATTEMPTS)
+	{
+		return command_fail(cmd, PAY_STOPPED_RETRYING,
+				    "Reached maximum number of attempts (%d)",
+				    MAX_NUM_ATTEMPTS);
+	}
 	
 	struct amount_msat feebudget, fees_spent, remaining;
-	struct pay_flow **pay_flows;
 
 	if (time_after(time_now(), p->stop_time))
 		return command_fail(cmd, PAY_STOPPED_RETRYING, "Timed out");
+	
 	/* Total feebudget  */
 	if (!amount_msat_sub(&feebudget, p->maxspend, p->amount))
 		abort();
@@ -999,15 +1258,20 @@ static struct command_result *try_paying(struct command *cmd,
 	if (!amount_msat_sub(&remaining, p->amount, p->total_delivering))
 		abort();
 
+	debug_info(fmt_chan_extra_map(tmpctx,pay_plugin->chan_extra_map));
+	
 	/* We let this return an unlikely path, as it's better to try once
 	 * than simply refuse.  Plus, models are not truth! */
-	pay_flows = get_payflows(p, remaining, feebudget, first_time,
+	// TODO(eduardo): some flows might be pending, even the flows that
+	// failed have commited some HTLCs that (?) reduce the throughput of the
+	// channels. Please do take that into account in the MCF.
+	struct pay_flow **pay_flows = get_payflows(p, remaining, feebudget, first_time,
 				 amount_msat_eq(p->total_delivering, AMOUNT_MSAT(0)));
-		
 	
-	// TODO(eduardo): remove this line
-	debug_payflows(pay_flows,MYLOG);
+	debug_info(fmt_payflows(tmpctx,pay_flows));
 	
+	/* MCF cannot find a feasible route, we stop. */
+	// TODO(eduardo): alternatively we can fallback to `pay`.
 	if (!pay_flows)
 	{
 		return command_fail(cmd, PAY_ROUTE_NOT_FOUND,
@@ -1018,6 +1282,7 @@ static struct command_result *try_paying(struct command *cmd,
 						   &feebudget));
 	}
 	/* Now begin making payments */
+	
 	return sendpay_flows(cmd, p, pay_flows);
 }
 
@@ -1025,7 +1290,7 @@ static struct command_result *
 listpeerchannels_done(struct command *cmd, const char *buf,
 	       const jsmntok_t *result, struct payment *p)
 {
-	debug_call(__PRETTY_FUNCTION__,MYLOG);
+	debug_call();
 	if (!update_uncertainty_network_from_listpeerchannels(cmd->plugin, p, buf, result))
 		return command_fail(cmd, LIGHTNINGD,
 				    "listpeerchannels malformed: %.*s",
@@ -1045,16 +1310,23 @@ listpeerchannels_done(struct command *cmd, const char *buf,
  * into a valid state before the next payment. */
 static void renepay_cleanup(struct active_payment *ap)
 {
-	debug_call(__PRETTY_FUNCTION__,MYLOG);
+	debug_call();
+
+	/* Free all pending flows, release their HTLCs and update knowledge. */
+	ap->all_flows = tal_free(ap->all_flows);
+	
 	/* Always remove our local mods (routehints) so others can use
 	 * gossmap. We do this only after the payment completes. */
 	if(ap->localmods_applied)
 		gossmap_remove_localmods(pay_plugin->gossmap,
 					 ap->local_gossmods);
-				 
+	ap->localmods_applied=false;
 	tal_free(ap->local_gossmods);
-	// TODO(eduardo): Am I missing other things?
-	// TODO(eduardo): When do we call this?
+	
+	debug_info(fmt_chan_extra_map(tmpctx,pay_plugin->chan_extra_map));
+	
+	// TODO(eduardo): what about private channels in `chan_extra_map`?
+	// Should they be removed?
 }
 
 static void destroy_payment(struct payment *p)
@@ -1096,6 +1368,12 @@ static struct command_result *json_paystatus(struct command *cmd,
 			json_add_string(ret, NULL, p->paynotes[i]);
 		json_array_end(ret);
 		json_object_end(ret);
+		
+		// TODO(eduardo): maybe we should add also:
+		// - preimage (as proof of payment)
+		// - payment_secret?
+		// - payment_metadata
+		// - status
 	}
 	json_array_end(ret);
 
@@ -1123,15 +1401,7 @@ static struct command_result *json_pay(struct command *cmd,
 
  	p = tal(cmd, struct payment);
 	
-	
 	p->paynotes = tal_arr(p, const char *, 0);
-	p->groupid = pseudorand_u64();
-	
-	// TODO(eduardo): If the partid starts counting from 1 we can do
-	// parallel payments MPP. A single payment route is done with partid=0.
-	// Can we assume that all payments are MPP, even in the case of an MPP
-	// with a single route?
-	p->next_partid = 1;
 	p->total_sent = AMOUNT_MSAT(0);
 	p->total_delivering = AMOUNT_MSAT(0);
 	
@@ -1170,18 +1440,7 @@ static struct command_result *json_pay(struct command *cmd,
 	tal_steal(pay_plugin,p);
 	tal_add_destructor(p, destroy_payment);
 	
-	/* Owned by cmd, because this data is destroyed after the payment
-	 * completes, while p remains to get payment status history. */
-	p->active_payment = tal(cmd,struct active_payment);
-	tal_add_destructor(p->active_payment, renepay_cleanup);
-	
-	p->active_payment->cmd = cmd;
- 	p->active_payment->local_gossmods = gossmap_localmods_new(p->active_payment);
-	p->active_payment->localmods_applied=false;
-	p->active_payment->disabled = tal_arr(p->active_payment, 
-					      struct short_channel_id, 
-					      0);
-	p->active_payment->rexmit_timer = NULL;
+	payment_initialize(p,cmd);
 	
 
 #if DEVELOPER
@@ -1191,6 +1450,12 @@ static struct command_result *json_pay(struct command *cmd,
 	p->use_shadow = true;
 #endif
  	
+	// TODO(eduardo): 
+	// - get time since last payment, 
+	// - forget a portion of the bounds
+	// - note that if sufficient time has passed, then we would forget
+	// everything.
+	
 	
 	plugin_log(pay_plugin->plugin,LOG_DBG,"Starting renepay");
 	bool gossmap_changed = gossmap_refresh(pay_plugin->gossmap, NULL);
@@ -1200,9 +1465,6 @@ static struct command_result *json_pay(struct command *cmd,
 			   strerror(errno));
 	
 	
-	// TODO(eduardo): remove this
-	debug_knowledge(pay_plugin->chan_extra_map,MYLOG);
-	
 	p->base_fee_penalty=*base_fee_penalty;
 	p->prob_cost_factor= *prob_cost_factor;
 	p->min_prob_success=*min_prob_success_millionths * 1e-6;
@@ -1217,6 +1479,7 @@ static struct command_result *json_pay(struct command *cmd,
 
 	p->start_time = time_now();
 	p->stop_time = timeabs_add(p->start_time, time_from_sec(*retryfor));
+	p->preimage=NULL;
 	tal_free(retryfor);
 	
 	bool invstr_is_b11=false;
@@ -1406,10 +1669,12 @@ static struct command_result *json_pay(struct command *cmd,
 		plugin_log(pay_plugin->plugin, 
 			   LOG_BROKEN,
 			   "uncertainty network invariants are violated");
-		
+	
+	/* We will try a single MPP payment. */
+	p->active_payment->groupid = pseudorand_u64();
+	p->active_payment->next_partid = 1;
 	
 	struct out_req *req;
-
 	/* Get local capacities... */
 	req = jsonrpc_request_start(cmd->plugin, cmd, "listpeerchannels",
 				    listpeerchannels_done,
diff --git a/plugins/renepay/pay.h b/plugins/renepay/pay.h
index 70921c3a5b1d..9afc05a4abaa 100644
--- a/plugins/renepay/pay.h
+++ b/plugins/renepay/pay.h
@@ -3,8 +3,16 @@
 #include "config.h"
 #include <ccan/list/list.h>
 #include <common/node_id.h>
+#include <plugins/libplugin.h>
 #include <plugins/renepay/flow.h>
 
+#define MAX_NUM_ATTEMPTS 10
+
+enum payment_status {
+        PAYMENT_PENDING, PAYMENT_SUCCESS, PAYMENT_FAIL,
+	PAYMENT_MPP_TIMEOUT
+};
+
 /* Our convenient global data, here in one place. */
 struct pay_plugin {
 	/* From libplugin */
@@ -32,12 +40,10 @@ struct pay_plugin {
 /* Set in init */
 extern struct pay_plugin *pay_plugin;
 
-// TODO(eduardo): does it make sense?
 /* Data only kept while the payment is being processed. */
 struct active_payment
 {
 	/* The command, and our owner (needed for timer func) */
-	// TODO(eduardo): is it used?
 	struct command *cmd;
 
 	/* Localmods to apply to gossip_map for our own use. */
@@ -45,111 +51,129 @@ struct active_payment
 	struct gossmap_localmods *local_gossmods;
 	
 	/* Channels we decided to disable for various reasons. */
-	// TODO(eduardo): is it used?
 	struct short_channel_id *disabled;
 
 	/* Timers. */
-	// TODO(eduardo): is it used?
 	struct plugin_timer *rexmit_timer;
+	
+	/* Keep track of the number of attempts. */
+	int last_attempt;
+	
+	/* Root to destroy pending flows */
+	tal_t *all_flows;
+	
+	/* Groupid, so listpays() can group them back together */
+	u64 groupid;
+	
+	/* Used in get_payflows to set ids to each pay_flow. */
+	u64 next_partid;
 };
 
 struct payment {
-	/* Data used while the payment is being processed. */
-	// TODO(eduardo): initialize this
-	// TODO(eduardo): add a destructor
-	struct active_payment *active_payment;
-
-	/* Inside pay_plugin->payments list */
-	// TODO(eduardo): is it used?
-	struct list_node list;
-
-	/* We promised this in pay() output */
-	// TODO(eduardo): is it used?
-	struct timeabs start_time;
-
+	/* Chatty description of attempts. */
+	const char **paynotes;
+	
+	/* Total sent, including fees. */
+	struct amount_msat total_sent;
+	
+	/* Total that is delivering (i.e. without fees) */
+	struct amount_msat total_delivering;
+	
 	/* invstring (bolt11 or bolt12) */
-	// TODO(eduardo): is it used?
 	const char *invstr;
-
+	
 	/* How much, what, where */
-	// TODO(eduardo): is it used?
+	struct amount_msat amount;
 	struct node_id dest;
 	struct sha256 payment_hash;
-	// TODO(eduardo): is it used?
-	struct amount_msat amount;
 	
-	// TODO(eduardo): is it used?
-	u32 final_cltv;
-
-	/* Total sent, including fees. */
-	struct amount_msat total_sent;
-	/* Total that is delivering (i.e. without fees) */
-	struct amount_msat total_delivering;
-
+	
+	/* Limits on what routes we'll accept. */
+	struct amount_msat maxspend;
+	
+	// TODO(eduardo): check out how this is used by get_payflows.
+	unsigned int maxdelay;
+	
+	/* We promised this in pay() output */
+	struct timeabs start_time;
+	
+	// TODO(eduardo): notice that stop_time = start_time + 60sec by default.
+	// On the other hand I am assuming that HTLCs are removed when a payment
+	// request fails or succeeds. A fail could happen when some other part
+	// of a MPP payment fails and the current part times-out. My question is
+	// what determines this time-out of the MPP part, is it encoded in the
+	// HTLCs themselves, thus measured in block numbers or is there another
+	// timer somewhere?
+	// If this time is in seconds, it would be desirable to be less than
+	// 30sec so that we can try at least two different MPP before the
+	// payment expires. On the other hand if HTLCs expire only after a
+	// certain block number, then we should keep track of these events in the
+	// uncertainty network somehow even after the payment terminates.
+	struct timeabs stop_time;
+	
+	/* Payment preimage, in case of success. */
+	const struct preimage *preimage;
+	
 	/* payment_secret, if specified by invoice. */
-	// TODO(eduardo): is it used?
+	// TODO(eduardo): isn't the preimage the payment secret?
 	struct secret *payment_secret;
+	
 	/* Payment metadata, if specified by invoice. */
-	// TODO(eduardo): is it used?
 	const u8 *payment_metadata;
+	
+	/* To know if the last attempt failed, succeeded or is it pending. */
+	enum payment_status status;	
+
+	// TODO(eduardo):
+	// 1. what is this?
+	// 2. what is it used for?
+	// 3. notice that we don't use it.
+	u32 final_cltv;
+
+	/* Inside pay_plugin->payments list */
+	struct list_node list;
 
 	/* Description and labels, if any. */
-	// TODO(eduardo): is it used?
-	// TODO(eduardo): could be NULL
 	const char *description, *label;
 
+	
 	/* Penalty for CLTV delays */
-	// TODO(eduardo): is it used?
 	double delay_feefactor;
 	
 	/* Penalty for base fee */
-	// TODO(eduardo): is it used?
 	double base_fee_penalty;
 	
-	/* linear fee cost = 
+	/* With these the effective linear fee cost is computed as
+	 * 
+	 * linear fee cost = 
 	 * 	millionths 
 	 * 	+ base_fee* base_fee_penalty
-	 * 	+delay*delay_feefactor;*/
+	 * 	+delay*delay_feefactor;
+	 * */
 
-	/* Limits on what routes we'll accept. */
-	// TODO(eduardo): is it used?
-	struct amount_msat maxspend;
-	// TODO(eduardo): is it used?
-	unsigned int maxdelay;
-	// TODO(eduardo): is it used?
-	struct timeabs stop_time;
-	
 	/* The minimum acceptable prob. of success */
-	// TODO(eduardo): is it used?
 	double min_prob_success;
 	
-	/* linear prob. cost = 
-	 * 	- prob_cost_factor * log prob. */
-	
 	/* Conversion from prob. cost to millionths */
-	// TODO(eduardo): is it used?
 	double prob_cost_factor;
+	/* linear prob. cost = 
+	 * 	- prob_cost_factor * log prob. */
 
-	/* Chatty description of attempts. */
-	// TODO(eduardo): is it used?
-	const char **paynotes;
 
-	/* Groupid, so listpays() can group them back together */
-	// TODO(eduardo): is it used?
-	u64 groupid;
-	// TODO(eduardo): is it used?
-	u64 next_partid;
 	/* If this is paying a local offer, this is the one (sendpay ensures we
 	 * don't pay twice for single-use offers) */
-	// TODO(eduardo): is it used?
-	// TODO(eduardo): could be NULL
+	// TODO(eduardo): this is not being used!
 	struct sha256 *local_offer_id;
 
 	/* DEVELOPER allows disabling shadow route */
-	// TODO(eduardo): is it used?
 	bool use_shadow;
+	
+	/* Data used while the payment is being processed. */
+	struct active_payment *active_payment;
 };
 
+int payment_current_attempt(const struct payment *p);
+
 void paynote(struct payment *p, const char *fmt, ...)
 	PRINTF_FMT(2,3);
 
diff --git a/plugins/renepay/pay_flow.c b/plugins/renepay/pay_flow.c
index 22c564e276ac..2f552895c22d 100644
--- a/plugins/renepay/pay_flow.c
+++ b/plugins/renepay/pay_flow.c
@@ -135,6 +135,7 @@ static u64 flow_delay(const struct flow *flow)
 }
 
 /* This enhances f->amounts, and returns per-flow cltvs */
+// TODO(eduardo): check out this again
 static u32 *shadow_additions(const tal_t *ctx,
 			     const struct gossmap *gossmap,
 			     struct payment *p,
@@ -239,6 +240,7 @@ static struct pay_flow **flows_to_pay_flows(struct payment *payment,
 		pf->amounts = tal_steal(pf, f->amounts);
 		pf->path_dirs = tal_steal(pf, f->dirs);
 		pf->success_prob = f->success_prob;
+		pf->attempt = payment_current_attempt(payment);
 	}
 	tal_free(flows);
 
@@ -453,7 +455,7 @@ struct pay_flow **get_payflows(struct payment *p,
 		 * pay_flows to outlive the current gossmap. */
 		pay_flows = flows_to_pay_flows(p, pay_plugin->gossmap,
 					       flows, final_cltvs,
-					       &p->next_partid);
+					       &p->active_payment->next_partid);
 		break;
 
 	retry:
@@ -536,8 +538,8 @@ const char* fmt_payflows(const tal_t *ctx,
 }
 
 void remove_htlc_payflow(
-		struct chan_extra_map *chan_extra_map,
-		const struct pay_flow *flow)
+		struct pay_flow *flow,
+		struct chan_extra_map *chan_extra_map)
 {
 	for (size_t i = 0; i < tal_count(flow->path_scids); i++) {
 		struct chan_extra_half *h = get_chan_extra_half_by_scid(
diff --git a/plugins/renepay/pay_flow.h b/plugins/renepay/pay_flow.h
index 2285207141d1..b25dfa7bf993 100644
--- a/plugins/renepay/pay_flow.h
+++ b/plugins/renepay/pay_flow.h
@@ -8,6 +8,9 @@ struct pay_flow {
 	/* So we can be an independent object for callbacks. */
 	struct payment *payment;
 
+	/* This flow belongs to some attempt. */
+	int attempt;
+	
 	/* Part id for RPC interface */
 	u64 partid;
 	/* The series of channels and nodes to traverse. */
@@ -33,8 +36,8 @@ void commit_htlc_payflow(
 		const struct pay_flow *flow);
 
 void remove_htlc_payflow(
-		struct chan_extra_map *chan_extra_map,
-		const struct pay_flow *flow);
+		struct pay_flow *flow,
+		struct chan_extra_map *chan_extra_map);
 
 const char* fmt_payflows(const tal_t *ctx,
 			 struct pay_flow ** flows);

From 8bd3d3e6493603fac8811f20eda93f88030c7623 Mon Sep 17 00:00:00 2001
From: Lagrang3 <eduardo.quintana@pm.me>
Date: Fri, 9 Jun 2023 10:54:01 +0100
Subject: [PATCH 49/64] renepay: update channel gossip from onion error

Signed-off-by: Lagrang3 <eduardo.quintana@pm.me>
---
 plugins/renepay/pay.c | 379 ++++++++++++++----------------------------
 plugins/renepay/pay.h |  14 ++
 2 files changed, 140 insertions(+), 253 deletions(-)

diff --git a/plugins/renepay/pay.c b/plugins/renepay/pay.c
index 8476cf1b80e7..a383c4c8ef95 100644
--- a/plugins/renepay/pay.c
+++ b/plugins/renepay/pay.c
@@ -23,6 +23,9 @@ struct pay_plugin *pay_plugin;
 
 static void renepay_cleanup(struct active_payment *ap);
 static void timer_kick(struct payment *p);
+static struct command_result *try_paying(struct command *cmd,
+					 struct payment *p,
+					 bool first_time);
 
 static void payment_check_delivering_incomplete(struct payment *p)
 {
@@ -128,25 +131,6 @@ static struct command_result *payment_success(struct payment *p)
 	return command_finished(payment_command(p), response);
 }
 
-// static struct command_result *payment_fail(struct payment *p)
-// {
-// 	plugin_log(pay_plugin->plugin,LOG_DBG,"calling %s",__PRETTY_FUNCTION__);
-// 	debug_call();
-// 	
-// 	p->status = PAYMENT_FAIL;
-// 	
-// 	/* In this version we do a MPP, which means either all payments fail or
-// 	 * all payments succeed. 
-// 	 * */
-// 	p->total_delivering = AMOUNT_MSAT(0);
-// 	p->total_sent = AMOUNT_MSAT(0);
-// 	
-// 	/* Still waiting for timer... */
-// 	return command_still_pending(payment_command(p));
-// }
-
-// TODO(eduardo): check if knowledge is updated after payment success
-// TODO(eduardo): how do we fail a payment
 // TODO(eduardo): improve loging
 // TODO(eduardo): handle hanging responses
 
@@ -268,87 +252,6 @@ static const char *init(struct plugin *p,
 	return NULL;
 }
 
-/* Create a entry for channel knowledge. */
-// static struct chan_extra* chan_knowledge_new(
-// 		struct gossmap* gossmap,
-// 		struct chan_extra_map *chan_extra_map,
-// 		struct short_channel_id scid)
-// {
-// 	struct chan_extra *chan = chan_extra_map_get(chan_extra_map,scid);
-// 	
-// 	return chan;
-// }
-		
-// TODO(eduardo): remember that if [a,b] is fixed on (c,dir) then [cap-b,cap-a]
-// is fixed on (c,!dir)
-// TODO(eduardo): I think this function does too many things.  I would prefer to
-// have finer grained control; like a function `chan_update_knowledge` that
-// improves the knowledge and also checks the bounds. But we also need to have
-// the possiblity to relax the knowledge in another function.
-/* We know something about this channel!  Update it! */
-// static void chan_update_knowledge(struct payment *p,
-// 				 struct short_channel_id scid,
-// 				 int dir,
-// 				 const struct amount_msat *min_capacity,
-// 				 const struct amount_msat *max_capacity)
-// {
-// 	plugin_log(pay_plugin->plugin,LOG_DBG,"Updating channel: %s [%s, %s]",
-// 		type_to_string(tmpctx,struct short_channel_id, &scid),
-// 		min_capacity ? type_to_string(tmpctx,struct amount_msat, min_capacity) : "-",
-// 		max_capacity ? type_to_string(tmpctx,struct amount_msat, max_capacity) : "-");
-// 	
-// 	
-// 	struct chan_extra_half *h;
-// 
-// 	/* This is assumed */
-// 	if (min_capacity && max_capacity)
-// 		assert(amount_msat_greater_eq(*max_capacity, *min_capacity));
-// 	
-// 	h = get_chan_extra_half_by_scid(pay_plugin->chan_extra_map, scid, dir);
-// 	if (!h)
-// 		h = new_chan_extra_half(pay_plugin->chan_extra_map, scid, dir,
-// 					*max_capacity);
-// 	if (min_capacity && amount_msat_greater(*min_capacity, h->known_min))
-// 		h->known_min = *min_capacity;
-// 	if (max_capacity && amount_msat_less(*max_capacity, h->known_max))
-// 		h->known_max = *max_capacity;
-// 
-// 	/* If we min > max, it means our previous assumptions are wrong
-// 	 * (i.e. things changed, or a we assumed full capacity for a routehint
-// 	 *  which didn't have it!) */
-// 	if (amount_msat_greater(h->known_min, h->known_max)) {
-// 		plugin_log(pay_plugin->plugin, LOG_BROKEN,
-// 			   "Updated %s capacity %s, now %s-%s! Resetting.",
-// 			   type_to_string(tmpctx, struct short_channel_id, &scid),
-// 			   min_capacity ? "min" : "max",
-// 			   type_to_string(tmpctx, struct amount_msat, &h->known_min),
-// 			   type_to_string(tmpctx, struct amount_msat, &h->known_max));
-// 
-// 		/* OK, assume *old* information is wrong: we can't have
-// 		 * just set both, since we assert() those are correct. */
-// 		if (min_capacity) {
-// 			const struct gossmap_chan *c;
-// 			struct amount_sat cap;
-// 
-// 			/* It might be a local channel; if we don't know better,
-// 			 * we reset max to infinite */
-// 			c = gossmap_find_chan(pay_plugin->gossmap, &scid);
-// 			if (!c
-// 			    || !gossmap_chan_get_capacity(pay_plugin->gossmap, c, &cap)
-// 			    || !amount_sat_to_msat(&h->known_max, cap))
-// 				h->known_max = p->maxspend;
-// 			plugin_log(pay_plugin->plugin, LOG_UNUSUAL,
-// 				   "... setting max to capacity (%s)",
-// 				   type_to_string(tmpctx, struct amount_msat,
-// 						  &h->known_max));
-// 		} else {
-// 			h->known_min = AMOUNT_MSAT(0);
-// 			plugin_log(pay_plugin->plugin, LOG_UNUSUAL,
-// 				   "... setting min to 0msat");
-// 		}
-// 	}
-// }
-
 static void add_hintchan(struct payment *p,
 			  const struct node_id *src,
 			  const struct node_id *dst,
@@ -356,7 +259,7 @@ static void add_hintchan(struct payment *p,
 			  const struct short_channel_id scid,
 			  u32 fee_base_msat,
 			  u32 fee_proportional_millionths,
-			  struct amount_msat amount)
+			  struct amount_msat amount UNUSED)
 {
 	int dir = node_id_cmp(src, dst) < 0 ? 0 : 1;
 	
@@ -365,6 +268,9 @@ static void add_hintchan(struct payment *p,
 	if(!ce)
 	{
 		/* this channel is not public, we don't know his capacity */
+		// TODO(eduardo): one possible solution is set the capacity to
+		// MAX_CAP and the state to [0,MAX_CAP]. Alternatively we set
+		// the capacity to amoung and state to [amount,amount].
 		ce = new_chan_extra(pay_plugin->chan_extra_map,
 				    scid,
 				    MAX_CAP);
@@ -388,7 +294,6 @@ static void add_hintchan(struct payment *p,
 }
 
 /* Add routehints provided by bolt11 */
-// TODO(eduardo): check this again
 static void uncertainty_network_add_routehints(struct payment *p)
 {
 	struct bolt11 *b11;
@@ -542,16 +447,12 @@ static bool update_uncertainty_network_from_listpeerchannels(
 	return false;
 }
 
-/* How much did does this flow deliver to dest? */
+/* How much does this flow deliver to dest? */
 static struct amount_msat flow_delivered(const struct pay_flow *flow)
 {
 	return flow->amounts[tal_count(flow->amounts)-1];
 }
 
-/* Mutual recursion */
-static struct command_result *try_paying(struct command *cmd,
-					 struct payment *p,
-					 bool first_time);
 
 static struct command_result *flow_failed(
 		struct command *cmd,
@@ -561,7 +462,6 @@ static struct command_result *flow_failed(
 	debug_call();
 	struct payment *p = flow->payment;
 	
-	
 	amount_msat_reduce(&p->total_delivering, flow_delivered(flow));
 	amount_msat_reduce(&p->total_sent, flow->amounts[0]);
 	tal_free(flow);
@@ -571,19 +471,6 @@ static struct command_result *flow_failed(
 }
 
 
-// // TODO(eduardo): how do we treat the knowledge from old flow attempts?
-// // TODO(eduardo
-// static struct command_result payment_success(...)
-// {
-// 	for each pending flow:
-// 		if flow is current attempt
-// 			update knowledge flow was sent
-// 		free flow
-// 	
-// 	return command_finished
-// }
-
-
 static void payment_settimer(struct payment *p)
 {
 	p->active_payment->rexmit_timer 
@@ -717,125 +604,126 @@ struct addgossip {
 	const struct pay_flow *flow;
 };
 
-// static struct command_result *addgossip_done(struct command *cmd,
-// 					     const char *buf,
-// 					     const jsmntok_t *err,
-// 					     struct addgossip *adg)
-// {
-// 	debug_call();
-// 	struct payment *p = adg->flow->payment;
-// 
-// 	/* Release this: if it's the last flow we'll retry immediately */
-// 	tal_free(adg);
-// 	timer_kick(p);
-// 
-// 	return command_still_pending(cmd);
-// }
+static struct command_result *addgossip_done(struct command *cmd,
+					     const char *buf,
+					     const jsmntok_t *err,
+					     struct addgossip *adg)
+{
+	debug_call();
+	struct payment *p = adg->flow->payment;
 
-// static struct command_result *addgossip_failure(struct command *cmd,
-// 						const char *buf,
-// 						const jsmntok_t *err,
-// 						struct addgossip *adg)
-// 
-// {
-// 	debug_call();
-// 	struct payment *p = adg->flow->payment;
-// 
-// 	paynote(p, "addgossip failed, removing channel %s (%.*s)",
-// 		type_to_string(tmpctx, struct short_channel_id, &adg->scid),
-// 		err->end - err->start, buf + err->start);
-// 	tal_arr_expand(&p->active_payment->disabled, adg->scid);
-// 
-// 	return addgossip_done(cmd, buf, err, adg);
-// }
+	/* Release this: if it's the last flow we'll retry immediately */
+	
+	tal_free(adg);
+	payment_settimer(p);
 
-// static struct command_result *submit_update(struct command *cmd,
-// 					    const struct pay_flow *flow,
-// 					    const u8 *update,
-// 					    struct short_channel_id errscid)
-// {
-// 	plugin_log(pay_plugin->plugin,LOG_DBG,"calling submit_update");
-// 	debug_call();
-// 	struct payment *p = flow->payment;
-// 	struct out_req *req;
-// 	struct addgossip *adg = tal(cmd, struct addgossip);
-// 
-// 	/* We need to stash scid in case this fails, and we need to hold flow so
-// 	 * we don't get a rexmit before this is complete. */
-// 	adg->scid = errscid;
-// 	adg->flow = tal_steal(adg, flow);
-// 	/* Disable re-xmit until this returns */
-// 	p->active_payment->rexmit_timer 
-// 		= tal_free(p->active_payment->rexmit_timer);
-// 
-// 	paynote(p, "... extracted channel_update, telling gossipd");
-// 	plugin_log(pay_plugin->plugin, LOG_DBG, "(update = %s)", tal_hex(tmpctx, update));
-// 
-// 	req = jsonrpc_request_start(pay_plugin->plugin, NULL, "addgossip",
-// 				    addgossip_done,
-// 				    addgossip_failure,
-// 				    adg);
-// 	json_add_hex_talarr(req->js, "message", update);
-// 	return send_outreq(pay_plugin->plugin, req);
-// }
+	return flow_failed(cmd,adg->flow);
+}
+
+static struct command_result *addgossip_failure(struct command *cmd,
+						const char *buf,
+						const jsmntok_t *err,
+						struct addgossip *adg)
+
+{
+	debug_call();
+	struct payment *p = adg->flow->payment;
+
+	paynote(p, "addgossip failed, removing channel %s (%.*s)",
+		type_to_string(tmpctx, struct short_channel_id, &adg->scid),
+		err->end - err->start, buf + err->start);
+	tal_arr_expand(&p->active_payment->disabled, adg->scid);
+
+	return addgossip_done(cmd, buf, err, adg);
+}
+
+static struct command_result *submit_update(struct command *cmd,
+					    const struct pay_flow *flow,
+					    const u8 *update,
+					    struct short_channel_id errscid)
+{
+	plugin_log(pay_plugin->plugin,LOG_DBG,"calling submit_update");
+	debug_call();
+	struct payment *p = flow->payment;
+	struct out_req *req;
+	struct addgossip *adg = tal(cmd, struct addgossip);
+
+	/* We need to stash scid in case this fails, and we need to hold flow so
+	 * we don't get a rexmit before this is complete. */
+	adg->scid = errscid;
+	adg->flow = flow;
+	/* Disable re-xmit until this returns */
+	p->active_payment->rexmit_timer 
+		= tal_free(p->active_payment->rexmit_timer);
+
+	paynote(p, "... extracted channel_update, telling gossipd");
+	plugin_log(pay_plugin->plugin, LOG_DBG, "(update = %s)", tal_hex(tmpctx, update));
+
+	req = jsonrpc_request_start(pay_plugin->plugin, NULL, "addgossip",
+				    addgossip_done,
+				    addgossip_failure,
+				    adg);
+	json_add_hex_talarr(req->js, "message", update);
+	return send_outreq(pay_plugin->plugin, req);
+}
 
 /* Fix up the channel_update to include the type if it doesn't currently have
  * one. See ElementsProject/lightning#1730 and lightningnetwork/lnd#1599 for the
  * in-depth discussion on why we break message parsing here... */
-// static u8 *patch_channel_update(const tal_t *ctx, u8 *channel_update TAKES)
-// {
-// 	u8 *fixed;
-// 	if (channel_update != NULL &&
-// 	    fromwire_peektype(channel_update) != WIRE_CHANNEL_UPDATE) {
-// 		/* This should be a channel_update, prefix with the
-// 		 * WIRE_CHANNEL_UPDATE type, but isn't. Let's prefix it. */
-// 		fixed = tal_arr(ctx, u8, 0);
-// 		towire_u16(&fixed, WIRE_CHANNEL_UPDATE);
-// 		towire(&fixed, channel_update, tal_bytelen(channel_update));
-// 		if (taken(channel_update))
-// 			tal_free(channel_update);
-// 		return fixed;
-// 	} else {
-// 		return tal_dup_talarr(ctx, u8, channel_update);
-// 	}
-// }
+static u8 *patch_channel_update(const tal_t *ctx, u8 *channel_update TAKES)
+{
+	u8 *fixed;
+	if (channel_update != NULL &&
+	    fromwire_peektype(channel_update) != WIRE_CHANNEL_UPDATE) {
+		/* This should be a channel_update, prefix with the
+		 * WIRE_CHANNEL_UPDATE type, but isn't. Let's prefix it. */
+		fixed = tal_arr(ctx, u8, 0);
+		towire_u16(&fixed, WIRE_CHANNEL_UPDATE);
+		towire(&fixed, channel_update, tal_bytelen(channel_update));
+		if (taken(channel_update))
+			tal_free(channel_update);
+		return fixed;
+	} else {
+		return tal_dup_talarr(ctx, u8, channel_update);
+	}
+}
 
 
 /* Return NULL if the wrapped onion error message has no channel_update field,
  * or return the embedded channel_update message otherwise. */
-// static u8 *channel_update_from_onion_error(const tal_t *ctx,
-// 					   const char *buf,
-// 					   const jsmntok_t *onionmsgtok)
-// {
-// 	u8 *channel_update = NULL;
-// 	struct amount_msat unused_msat;
-// 	u32 unused32;
-// 	u8 *onion_message = json_tok_bin_from_hex(tmpctx, buf, onionmsgtok);
-// 
-// 	/* Identify failcodes that have some channel_update.
-// 	 *
-// 	 * TODO > BOLT 1.0: Add new failcodes when updating to a
-// 	 * new BOLT version. */
-// 	if (!fromwire_temporary_channel_failure(ctx,
-// 						onion_message,
-// 						&channel_update) &&
-// 	    !fromwire_amount_below_minimum(ctx,
-// 					   onion_message, &unused_msat,
-// 					   &channel_update) &&
-// 	    !fromwire_fee_insufficient(ctx,
-// 		    		       onion_message, &unused_msat,
-// 				       &channel_update) &&
-// 	    !fromwire_incorrect_cltv_expiry(ctx,
-// 		    			    onion_message, &unused32,
-// 					    &channel_update) &&
-// 	    !fromwire_expiry_too_soon(ctx,
-// 		    		      onion_message,
-// 				      &channel_update))
-// 		/* No channel update. */
-// 		return NULL;
-// 
-// 	return patch_channel_update(ctx, take(channel_update));
-// }
+static u8 *channel_update_from_onion_error(const tal_t *ctx,
+					   const char *buf,
+					   const jsmntok_t *onionmsgtok)
+{
+	u8 *channel_update = NULL;
+	struct amount_msat unused_msat;
+	u32 unused32;
+	u8 *onion_message = json_tok_bin_from_hex(tmpctx, buf, onionmsgtok);
+
+	/* Identify failcodes that have some channel_update.
+	 *
+	 * TODO > BOLT 1.0: Add new failcodes when updating to a
+	 * new BOLT version. */
+	if (!fromwire_temporary_channel_failure(ctx,
+						onion_message,
+						&channel_update) &&
+	    !fromwire_amount_below_minimum(ctx,
+					   onion_message, &unused_msat,
+					   &channel_update) &&
+	    !fromwire_fee_insufficient(ctx,
+		    		       onion_message, &unused_msat,
+				       &channel_update) &&
+	    !fromwire_incorrect_cltv_expiry(ctx,
+		    			    onion_message, &unused32,
+					    &channel_update) &&
+	    !fromwire_expiry_too_soon(ctx,
+		    		      onion_message,
+				      &channel_update))
+		/* No channel update. */
+		return NULL;
+
+	return patch_channel_update(ctx, take(channel_update));
+}
 
 static struct command_result *waitsendpay_failed(struct command *cmd,
 						 const char *buf,
@@ -866,8 +754,8 @@ static struct command_result *waitsendpay_failed(struct command *cmd,
 	u32 onionerr, erridx;
 	struct short_channel_id errscid;
 	
-	// const u8 *update;
-	// const jsmntok_t *rawoniontok;
+	const u8 *update;
+	const jsmntok_t *rawoniontok;
 
 	if (!json_to_u64(buf, json_get_member(buf, err, "code"), &errcode))
 		plugin_err(cmd->plugin, "Bad errcode from waitsendpay: %.*s",
@@ -875,7 +763,7 @@ static struct command_result *waitsendpay_failed(struct command *cmd,
 
 	switch (errcode) {
 	case PAY_UNPARSEABLE_ONION:
-		// TODO(eduardo): let's check this case carefully.
+		// TODO(eduardo): when can this be triggered?
 		debug_info("waitsendpay_failed: PAY_UNPARSEABLE_ONION");
 		ret = handle_unhandleable_error(p, flow, "unparsable onion reply");
 		if (ret)
@@ -910,9 +798,6 @@ static struct command_result *waitsendpay_failed(struct command *cmd,
 	errchantok = json_get_member(buf, datatok, "erring_channel");
 	json_to_short_channel_id(buf, errchantok, &errscid);
 	
-	/* TODO(eduardo): can we assume that if
-	 * erridx=tal_count(flow->path_scids), ie. one index past the last one, the failure 
-	 * is due to expired HTLCs? How can we handle this case? */
 	if (erridx<tal_count(flow->path_scids) 
 	    && !short_channel_id_eq(&errscid, &flow->path_scids[erridx]))
 		plugin_err(pay_plugin->plugin, "Erring channel %u/%zu was %s not %s (path %s)",
@@ -925,7 +810,7 @@ static struct command_result *waitsendpay_failed(struct command *cmd,
 	json_to_u32(buf, failcodetok, &onionerr);
 
 	msgtok = json_get_member(buf, err, "message");
-	// rawoniontok = json_get_member(buf, datatok, "raw_message");
+	rawoniontok = json_get_member(buf, datatok, "raw_message");
 	
 	paynote(p, "onion error %s from node #%u %s: %.*s",
 		onion_wire_name(onionerr),
@@ -979,10 +864,9 @@ static struct command_result *waitsendpay_failed(struct command *cmd,
 	case WIRE_EXPIRY_TOO_SOON:
 		debug_info("waitsendpay_failed: apply channel_update");
 		/* FIXME: Check scid! */
-		// TODO(eduardo): let's check this case carefully.
-		// update = channel_update_from_onion_error(tmpctx, buf, rawoniontok);
-		// if (update)
-		// 	return submit_update(cmd, flow, update, errscid);
+		update = channel_update_from_onion_error(tmpctx, buf, rawoniontok);
+		if (update)
+			return submit_update(cmd, flow, update, errscid);
 
 		paynote(p, "... missing an update, so we're removing scid");
 		tal_arr_expand(&p->active_payment->disabled, errscid);
@@ -1131,15 +1015,6 @@ sendpay_flows(struct command *cmd,
 	debug_call();
 	paynote(p, "Sending out batch of %zu payments", tal_count(flows));
 	
-	// TODO(eduardo): mark here the status of the payment parts as either:
-	// 'pending', 'success' or 'fail'. If all of them fail then `try_paying`
-	// again, if instead all of them succeed then return `command_finished`,
-	// while there is one payment pending you return
-	// `command_still_pending`. Maybe the timer could be useful here.
-	// If all of the parts are processed but some are fail and some are
-	// success, then it means there is something wrong with my understanding
-	// and we need to rethink here.
-	
 	for (size_t i = 0; i < tal_count(flows); i++) {
 		struct out_req *req;
 		req = jsonrpc_request_start(cmd->plugin, cmd, "sendpay",
@@ -1167,8 +1042,7 @@ sendpay_flows(struct command *cmd,
 		json_add_sha256(req->js, "payment_hash", &p->payment_hash);
 		json_add_secret(req->js, "payment_secret", p->payment_secret);
 		
-		/* TODO(eduardo): should this be p->amount or
-		 * p->total_delivering? */
+		// TODO(eduardo): should this be p->amount or p->total_delivering? 
 		json_add_amount_msat_only(req->js, "amount_msat", p->amount);
 			
 		json_add_u64(req->js, "partid", flows[i]->partid);
@@ -1185,7 +1059,6 @@ sendpay_flows(struct command *cmd,
 		if (p->description)
 			json_add_string(req->js, "description", p->description);
 		
-		// TODO(eduardo): remove this line
 		debug_outreq(req);
 		
 		amount_msat_accumulate(&p->total_sent, flows[i]->amounts[0]);
diff --git a/plugins/renepay/pay.h b/plugins/renepay/pay.h
index 9afc05a4abaa..d51efb42857d 100644
--- a/plugins/renepay/pay.h
+++ b/plugins/renepay/pay.h
@@ -8,6 +8,20 @@
 
 #define MAX_NUM_ATTEMPTS 10
 
+// TODO(eduardo): Test ideas
+// - make a payment to a node that is hidden behind private channels, check that
+// private channels are removed from the gossmap and chan_extra_map
+// - one payment route hangs, and the rest keep waiting, eventually all MPP
+// should timeout and we retry excluding the unresponsive path (are we able to
+// identify it?)
+// - a particular route fails because fees are wrong, we update the gossip
+// information and redo the path.
+// - a MPP in which several parts have a common intermediary node 
+// 	source -MANY- o -MANY- dest
+// - a MPP in which several parts have a common intermediary channel
+// 	source -MANY- o--o -MANY- dest
+// - a payment with a direct channel to the destination
+
 enum payment_status {
         PAYMENT_PENDING, PAYMENT_SUCCESS, PAYMENT_FAIL,
 	PAYMENT_MPP_TIMEOUT

From a1a88ee803188f678089723537fcccd6ace87a22 Mon Sep 17 00:00:00 2001
From: Lagrang3 <eduardo.quintana@pm.me>
Date: Tue, 13 Jun 2023 16:56:15 +0100
Subject: [PATCH 50/64] renepay: remove bug that crashed pay after success

Signed-off-by: Lagrang3 <eduardo.quintana@pm.me>
---
 plugins/renepay/pay.c         | 4 +++-
 plugins/renepay/pay.h         | 6 ++++++
 plugins/renepay/test/Makefile | 3 +++
 3 files changed, 12 insertions(+), 1 deletion(-)

diff --git a/plugins/renepay/pay.c b/plugins/renepay/pay.c
index a383c4c8ef95..50c992d97f03 100644
--- a/plugins/renepay/pay.c
+++ b/plugins/renepay/pay.c
@@ -494,7 +494,7 @@ static void timer_kick(struct payment *p)
 		/* Some flows succeeded, we finish the payment. */
 		case PAYMENT_SUCCESS:
 			payment_check_delivering_all(p);
-			payment_success(p);
+			// payment_success(p);
 		break;
 		
 		/* Some flows failed, we retry. */
@@ -1198,6 +1198,8 @@ static void renepay_cleanup(struct active_payment *ap)
 	
 	debug_info(fmt_chan_extra_map(tmpctx,pay_plugin->chan_extra_map));
 	
+	ap->rexmit_timer = tal_free(ap->rexmit_timer);
+	
 	// TODO(eduardo): what about private channels in `chan_extra_map`?
 	// Should they be removed?
 }
diff --git a/plugins/renepay/pay.h b/plugins/renepay/pay.h
index d51efb42857d..5f0279fffb77 100644
--- a/plugins/renepay/pay.h
+++ b/plugins/renepay/pay.h
@@ -21,6 +21,12 @@
 // - a MPP in which several parts have a common intermediary channel
 // 	source -MANY- o--o -MANY- dest
 // - a payment with a direct channel to the destination
+// - payment failures:
+// 	- destination is not in the gossmap
+// 	- destination is offline
+// 	- with current knowledge there is no flow solution to destination
+// 	- path to destination is too expensive
+// 	- path to destination is too unlikely
 
 enum payment_status {
         PAYMENT_PENDING, PAYMENT_SUCCESS, PAYMENT_FAIL,
diff --git a/plugins/renepay/test/Makefile b/plugins/renepay/test/Makefile
index 4cf2bfa63349..37860ef9eeeb 100644
--- a/plugins/renepay/test/Makefile
+++ b/plugins/renepay/test/Makefile
@@ -15,3 +15,6 @@ PLUGIN_RENEPAY_TEST_COMMON_OBJS :=		\
 $(PLUGIN_RENEPAY_TEST_PROGRAMS): $(PLUGIN_RENEPAY_TEST_COMMON_OBJS) $(PLUGIN_LIB_OBJS) $(PLUGIN_COMMON_OBJS) $(JSMN_OBJS) $(CCAN_OBJS) bitcoin/chainparams.o common/gossmap.o common/fp16.o common/dijkstra.o common/bolt12.o common/bolt12_merkle.o wire/bolt12$(EXP)_wiregen.o
 
 check-renepay: $(PLUGIN_RENEPAY_TEST_PROGRAMS:%=unittest/%)
+	bash plugins/renepay/test/regtest1.sh 
+	bash plugins/renepay/test/regtest2.sh 
+	bash plugins/renepay/test/regtest3.sh 

From 8498ccd6bede386ca0c90700f17a5de17f52597b Mon Sep 17 00:00:00 2001
From: Lagrang3 <eduardo.quintana@pm.me>
Date: Thu, 15 Jun 2023 10:52:26 +0100
Subject: [PATCH 51/64] renepay: debug message are sent to the log file

Signed-off-by: Lagrang3 <eduardo.quintana@pm.me>
---
 plugins/renepay/debug.h | 35 +++++++++++++++++++----
 plugins/renepay/mcf.c   | 27 +-----------------
 plugins/renepay/pay.c   | 61 +++++++++++++++++++++++++----------------
 plugins/renepay/pay.h   |  7 ++++-
 4 files changed, 73 insertions(+), 57 deletions(-)

diff --git a/plugins/renepay/debug.h b/plugins/renepay/debug.h
index 3449fce367ae..acccff436ffe 100644
--- a/plugins/renepay/debug.h
+++ b/plugins/renepay/debug.h
@@ -10,28 +10,51 @@
 #include <ccan/json_out/json_out.h>
 #include <pthread.h>
 
-#define MYLOG "/tmp/debug.txt"
+void _debug_outreq(const char *fname, const struct out_req *req);
+void _debug_reply(const char* fname, const char* buf,const jsmntok_t *toks);
+void _debug_info(const char* fname, const char *fmt, ...);
+void _debug_call(const char* fname, const char* fun);
+void _debug_exec_branch(const char* fname,const char* fun, int lineno);
 
+#ifndef MYLOCALDEBUG
 
-void _debug_outreq(const char *fname, const struct out_req *req);
+#define debug_reply(buf,toks) \
+	if(pay_plugin->debug_payflow) \
+	plugin_log(pay_plugin->plugin,LOG_DBG,\
+	"%.*s",json_tok_full_len(toks),json_tok_full(buf,toks))
+
+#define debug_info(...) \
+	if(pay_plugin->debug_payflow) \
+	plugin_log(pay_plugin->plugin,LOG_DBG,__VA_ARGS__)
+	
+#define debug_call() \
+	if(pay_plugin->debug_payflow) \
+	plugin_log(pay_plugin->plugin,LOG_DBG,"calling %s",__PRETTY_FUNCTION__)
+
+#define debug_exec_branch() \
+	if(pay_plugin->debug_payflow) \
+	plugin_log(pay_plugin->plugin,LOG_DBG,"executing line: %d (%s)",\
+		__LINE__,__PRETTY_FUNCTION__)
+
+#else
+
+#define MYLOG "/tmp/debug.txt"
 #define debug_outreq(req) \
 	_debug_outreq(MYLOG,req)
 
-void _debug_reply(const char* fname, const char* buf,const jsmntok_t *toks);
 #define debug_reply(buf,toks) \
 	_debug_reply(MYLOG,buf,toks)
 
-void _debug_info(const char* fname, const char *fmt, ...);
 #define debug_info(...) \
 	_debug_info(MYLOG,__VA_ARGS__)
 	
 
-void _debug_call(const char* fname, const char* fun);
 #define debug_call() \
 	_debug_call(MYLOG,__PRETTY_FUNCTION__)
 
-void _debug_exec_branch(const char* fname,const char* fun, int lineno);
 #define debug_exec_branch() \
 	_debug_exec_branch(MYLOG,__PRETTY_FUNCTION__,__LINE__)
 
+
+#endif
 #endif
diff --git a/plugins/renepay/mcf.c b/plugins/renepay/mcf.c
index d31aa4e67874..6603a965d5d5 100644
--- a/plugins/renepay/mcf.c
+++ b/plugins/renepay/mcf.c
@@ -6,7 +6,6 @@
 #include <plugins/renepay/mcf.h>
 #include <plugins/renepay/flow.h>
 #include <plugins/renepay/dijkstra.h>
-#include <plugins/renepay/debug.h>
 #include <common/type_to_string.h>
 #include <stdint.h>
 #include <math.h>
@@ -1241,8 +1240,6 @@ static bool is_better(
 		struct amount_msat B_fee,
 		double B_prob)
 {
-	debug_call();
-	
 	bool A_fee_pass = amount_msat_less_eq(A_fee,max_fee);
 	bool B_fee_pass = amount_msat_less_eq(B_fee,max_fee);
 	bool A_prob_pass = A_prob >= min_probability;
@@ -1252,30 +1249,25 @@ static bool is_better(
 	if(A_fee_pass && B_fee_pass && A_prob_pass && B_prob_pass)
 	{
 		// prefer lower fees
-		debug_info("all bounds are met\n");
 		goto fees_or_prob;
 	}
 	
 	// prefer the solution that satisfies both bounds
 	if(!(A_fee_pass && A_prob_pass) && (B_fee_pass && B_prob_pass))
 	{
-		debug_info("B is better, satisfies all bounds\n");
 		return false;
 	}
 	// prefer the solution that satisfies both bounds
 	if((A_fee_pass && A_prob_pass) && !(B_fee_pass && B_prob_pass))
 	{
-		debug_info("A is better, satisfies all bounds\n");
 		return true;
 	}
 	
 	// no solution satisfies both bounds
-	debug_info("Both bounds are not met.\n");	
 	
 	// bound on fee is met
 	if(A_fee_pass && B_fee_pass)
 	{
-		debug_info("Bound on fee is satisfies, select highest prob.\n");
 		// pick the highest prob.
 		return A_prob > B_prob;
 	}
@@ -1283,51 +1275,42 @@ static bool is_better(
 	// bound on prob. is met
 	if(A_prob_pass && B_prob_pass)
 	{
-		debug_info("Bound on prob. is satisfies, select lowest fee.\n");
 		goto fees_or_prob;
 	}
 	
 	// prefer the solution that satisfies the bound on fees
 	if(A_fee_pass && !B_fee_pass)
 	{	
-		debug_info("A satisfies bound on fees, B doesnt\n");	
 		return true;
 	}
 	if(B_fee_pass && !A_fee_pass)
 	{
-		debug_info("B satisfies bound on fees, A doesnt\n");	
 		return false;
 	}
 	
 	// none of them satisfy the fee bound
-	debug_info("Fee bound is not met.\n");	
 	
 	// prefer the solution that satisfies the bound on prob.
 	if(A_prob_pass && !B_prob_pass)
 	{
-		debug_info("A satisfies bound on prob., B doesnt\n");	
 		return true;
 	}
 	if(B_prob_pass && !A_prob_pass)
 	{
-		debug_info("B satisfies bound on prob., A doesnt\n");	
 		return true;
 	}
 	
 	// no bound whatsoever is satisfied
-	debug_info("No bounds are met.\n");	
 	
 	fees_or_prob:
 	
 	// fees are the same, wins the highest prob.
 	if(amount_msat_eq(A_fee,B_fee))
 	{
-		debug_info("Fees are equal, select highest prob.\n");	
 		return A_prob > B_prob;
 	}
 	
 	// go for fees
-	debug_info("Select lowest fee.\n");	
 	return amount_msat_less_eq(A_fee,B_fee);
 }
 
@@ -1359,8 +1342,6 @@ struct flow** minflow(
 		double base_fee_penalty,
 		u32 prob_cost_factor )
 {
-	debug_call();
-	
 	tal_t *this_ctx = tal(tmpctx,tal_t);
 	
 	struct pay_parameters *params = tal(this_ctx,struct pay_parameters);	
@@ -1445,7 +1426,6 @@ struct flow** minflow(
 	{
 		
 		s64 mu = (mu_left + mu_right)/2;
-		debug_info("mcf: mu=%ld\n",mu);
 		
 		combine_cost_function(linear_network,residual_network,mu);
 		
@@ -1462,10 +1442,6 @@ struct flow** minflow(
 						params->chan_extra_map);
 		struct amount_msat fee = flow_set_fee(flow_paths);
 		
-		debug_info("mcf: prob %.2f, fee %s\n",
-			   prob_success,
-			   type_to_string(this_ctx,struct amount_msat,&fee));
-		
 		// is this better than the previous one?
 		if(!best_flow_paths || 
 			is_better(params->max_fee,params->min_probability,
@@ -1482,12 +1458,11 @@ struct flow** minflow(
 		{
 			// too expensive
 			mu_left = mu+1;
-			debug_info("mcf too expensive\n");
+		
 		}else if(prob_success < params->min_probability)
 		{
 			// too unlikely
 			mu_right = mu;
-			debug_info("mcf too unlikely\n");
 		}else
 		{
 			// with mu constraints are satisfied, now let's optimize
diff --git a/plugins/renepay/pay.c b/plugins/renepay/pay.c
index 50c992d97f03..be9f32c06082 100644
--- a/plugins/renepay/pay.c
+++ b/plugins/renepay/pay.c
@@ -14,12 +14,12 @@
 #include <errno.h>
 #include <plugins/libplugin.h>
 #include <plugins/renepay/pay.h>
-#include <plugins/renepay/debug.h>
 #include <plugins/renepay/pay_flow.h>
+#include <plugins/renepay/debug.h>
 
 /* Set in init */
-struct pay_plugin *pay_plugin;
-
+static struct pay_plugin the_pay_plugin;
+struct pay_plugin * const pay_plugin = &the_pay_plugin;
 
 static void renepay_cleanup(struct active_payment *ap);
 static void timer_kick(struct payment *p);
@@ -109,8 +109,7 @@ static void payment_initialize(
 
 static struct command_result *payment_success(struct payment *p)
 {
-	debug_call();
-	
+	debug_call();	
 	payment_check_delivering_all(p);
 	
 	struct json_stream *response 
@@ -174,7 +173,7 @@ void amount_msat_reduce_(struct amount_msat *dst,
 #if DEVELOPER
 static void memleak_mark(struct plugin *p, struct htable *memtable)
 {
-	memleak_scan_obj(memtable, pay_plugin);
+	memleak_scan_obj(memtable, pay_plugin->ctx);
 	memleak_scan_htable(memtable, &pay_plugin->chan_extra_map->raw);
 }
 #endif
@@ -214,7 +213,8 @@ static const char *init(struct plugin *p,
 {
 	size_t num_channel_updates_rejected;
 
-	pay_plugin = tal(p, struct pay_plugin);
+	// pay_plugin = tal(p, struct pay_plugin);
+	pay_plugin->ctx = tal(p,tal_t);
 	pay_plugin->plugin = p;
 
 	rpc_scan(p, "getinfo", take(json_out_obj(NULL, NULL, NULL)),
@@ -224,17 +224,18 @@ static const char *init(struct plugin *p,
 		 take(json_out_obj(NULL, NULL, NULL)),
 		 "{max-locktime-blocks:%,experimental-offers:%}",
 		 JSON_SCAN(json_to_number, &pay_plugin->maxdelay_default),
-		 JSON_SCAN(json_to_bool, &pay_plugin->exp_offers));
+		 JSON_SCAN(json_to_bool, &pay_plugin->exp_offers)
+		 );
 
 	list_head_init(&pay_plugin->payments);
 	
-	pay_plugin->chan_extra_map = tal(pay_plugin,struct chan_extra_map);
+	pay_plugin->chan_extra_map = tal(pay_plugin->ctx,struct chan_extra_map);
 	chan_extra_map_init(pay_plugin->chan_extra_map);
 
-	pay_plugin->gossmap = gossmap_load(pay_plugin,
+	pay_plugin->gossmap = gossmap_load(pay_plugin->ctx,
 					   GOSSIP_STORE_FILENAME,
 					   &num_channel_updates_rejected);
-	
+
 	if (!pay_plugin->gossmap)
 		plugin_err(p, "Could not load gossmap %s: %s",
 			   GOSSIP_STORE_FILENAME, strerror(errno));
@@ -459,7 +460,6 @@ static struct command_result *flow_failed(
 		const struct pay_flow *flow)
 {
 	plugin_log(pay_plugin->plugin,LOG_DBG,"calling flow_failed");
-	debug_call();
 	struct payment *p = flow->payment;
 	
 	amount_msat_reduce(&p->total_delivering, flow_delivered(flow));
@@ -527,7 +527,6 @@ static struct command_result *waitsendpay_succeeded(struct command *cmd,
 						    const jsmntok_t *result,
 						    struct pay_flow *flow)
 {
-	plugin_log(pay_plugin->plugin,LOG_DBG,"calling waitsendpay_succeeded");
 	debug_call();
 	
 	struct payment *p = flow->payment;
@@ -642,7 +641,6 @@ static struct command_result *submit_update(struct command *cmd,
 					    const u8 *update,
 					    struct short_channel_id errscid)
 {
-	plugin_log(pay_plugin->plugin,LOG_DBG,"calling submit_update");
 	debug_call();
 	struct payment *p = flow->payment;
 	struct out_req *req;
@@ -730,7 +728,6 @@ static struct command_result *waitsendpay_failed(struct command *cmd,
 						 const jsmntok_t *err,
 						 struct pay_flow *flow)
 {
-	plugin_log(pay_plugin->plugin,LOG_DBG,"calling waitsendpay_failed");
 	debug_call();
 	debug_reply(buf,err);
 	
@@ -949,7 +946,6 @@ static struct command_result *flow_sent(struct command *cmd,
 					const jsmntok_t *result,
 					struct pay_flow *flow)
 {
-	plugin_log(pay_plugin->plugin,LOG_DBG,"calling flow_sent");
 	debug_call();
 	
 	struct payment *p = flow->payment;
@@ -976,7 +972,6 @@ static struct command_result *flow_sendpay_failed(struct command *cmd,
 						  const jsmntok_t *err,
 						  struct pay_flow *flow)
 {
-	plugin_log(pay_plugin->plugin,LOG_DBG,"calling flow_sendpay_failed");
 	debug_call();
 	
 	struct payment *p = flow->payment;
@@ -1011,7 +1006,6 @@ sendpay_flows(struct command *cmd,
 	      struct payment *p,
 	      struct pay_flow **flows STEALS)
 {
-	plugin_log(pay_plugin->plugin,LOG_DBG,"calling sendpay_flows");
 	debug_call();
 	paynote(p, "Sending out batch of %zu payments", tal_count(flows));
 	
@@ -1059,7 +1053,7 @@ sendpay_flows(struct command *cmd,
 		if (p->description)
 			json_add_string(req->js, "description", p->description);
 		
-		debug_outreq(req);
+		// debug_outreq(req);
 		
 		amount_msat_accumulate(&p->total_sent, flows[i]->amounts[0]);
 		amount_msat_accumulate(&p->total_delivering,
@@ -1096,7 +1090,6 @@ static struct command_result *try_paying(struct command *cmd,
 					 struct payment *p,
 					 bool first_time)
 {
-	plugin_log(pay_plugin->plugin,LOG_DBG,"calling try_paying");
 	debug_call();
 	
 	// TODO(eduardo): does it make sense to have this limit on attempts?
@@ -1311,8 +1304,14 @@ static struct command_result *json_pay(struct command *cmd,
 		   NULL))
 		return command_param_failed();
 	
+	debug_info("json_pay, renepay-debug-mcf option is set to %s\n",
+		   pay_plugin->debug_mcf ? "true" : "false");
+	debug_info("json_pay, renepay-debug-payflow option is set to %s\n",
+		   pay_plugin->debug_payflow ? "true" : "false");
+
+	
 	
-	tal_steal(pay_plugin,p);
+	tal_steal(pay_plugin->ctx,p);
 	tal_add_destructor(p, destroy_payment);
 	
 	payment_initialize(p,cmd);
@@ -1577,7 +1576,21 @@ static const struct plugin_command commands[] = {
 int main(int argc, char *argv[])
 {
 	setup_locale();
-	plugin_main(argv, init, PLUGIN_RESTARTABLE, true, NULL, commands,
-		    ARRAY_SIZE(commands), NULL, 0, NULL, 0, NULL, 0,
-		    NULL);
+	plugin_main(
+		argv, 
+		init, 
+		PLUGIN_RESTARTABLE, 
+		/* init_rpc */ true, 
+		/* features */ NULL, 
+		commands, ARRAY_SIZE(commands), 
+		/* notifications */ NULL, 0, 
+		/* hooks */ NULL, 0, 
+		/* notification topics */ NULL, 0,
+		plugin_option("renepay-debug-mcf", "flag",
+			"Enable renepay MCF debug info.",
+			flag_option, &pay_plugin->debug_mcf),
+		plugin_option("renepay-debug-payflow", "flag",
+			"Enable renepay payment flows debug info.",
+			flag_option, &pay_plugin->debug_payflow),
+		NULL);
 }
diff --git a/plugins/renepay/pay.h b/plugins/renepay/pay.h
index 5f0279fffb77..d5f20173f05a 100644
--- a/plugins/renepay/pay.h
+++ b/plugins/renepay/pay.h
@@ -55,10 +55,15 @@ struct pay_plugin {
 
 	/* Per-channel metadata: some persists between payments */
 	struct chan_extra_map *chan_extra_map;
+
+	bool debug_mcf; 
+	bool debug_payflow; 
+	
+	tal_t *ctx;
 };
 
 /* Set in init */
-extern struct pay_plugin *pay_plugin;
+extern struct pay_plugin * const pay_plugin;
 
 /* Data only kept while the payment is being processed. */
 struct active_payment

From 66e71ac061686a3c135f8a138e3b14564c130b95 Mon Sep 17 00:00:00 2001
From: Lagrang3 <eduardo.quintana@pm.me>
Date: Tue, 20 Jun 2023 09:42:41 +0100
Subject: [PATCH 52/64] renepay: add renepay to default plugins

Signed-off-by: Lagrang3 <eduardo.quintana@pm.me>
---
 plugins/Makefile | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/plugins/Makefile b/plugins/Makefile
index 4ed25c9669f3..754359866a07 100644
--- a/plugins/Makefile
+++ b/plugins/Makefile
@@ -100,7 +100,8 @@ C_PLUGINS :=					\
 	plugins/offers				\
 	plugins/pay				\
 	plugins/txprepare			\
-	plugins/spenderp
+	plugins/spenderp			\
+	plugins/cln-renepay
 
 ifeq ($(HAVE_SQLITE3),1)
 C_PLUGINS += plugins/sql

From b94ebe0c725aba7afdf6a596cb78779939b48cc9 Mon Sep 17 00:00:00 2001
From: Lagrang3 <eduardo.quintana@pm.me>
Date: Tue, 20 Jun 2023 11:15:57 +0100
Subject: [PATCH 53/64] renepay: add pyln-testing

Signed-off-by: Lagrang3 <eduardo.quintana@pm.me>
---
 plugins/renepay/pay.c | 14 ++++++++---
 tests/test_renepay.py | 55 +++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 66 insertions(+), 3 deletions(-)
 create mode 100644 tests/test_renepay.py

diff --git a/plugins/renepay/pay.c b/plugins/renepay/pay.c
index be9f32c06082..8639e2afde86 100644
--- a/plugins/renepay/pay.c
+++ b/plugins/renepay/pay.c
@@ -17,7 +17,8 @@
 #include <plugins/renepay/pay_flow.h>
 #include <plugins/renepay/debug.h>
 
-/* Set in init */
+// TODO(eduardo): the state of plugin is stored in data (instead of the heap) so
+// that we can load lightningd options directly into before `init` is called.
 static struct pay_plugin the_pay_plugin;
 struct pay_plugin * const pay_plugin = &the_pay_plugin;
 
@@ -213,8 +214,11 @@ static const char *init(struct plugin *p,
 {
 	size_t num_channel_updates_rejected;
 
-	// pay_plugin = tal(p, struct pay_plugin);
-	pay_plugin->ctx = tal(p,tal_t);
+	// TODO(eduardo): this is a weird fix I can't grasp yet. Why is it that
+	// `plugin_main` doesn't release `struct plugin *p` at shutdown?
+	// Where can I put my global state destructors now if `struct plugin *p`
+	// is not expected to free at shutdown?
+	pay_plugin->ctx = notleak_with_children(tal(p,tal_t));
 	pay_plugin->plugin = p;
 
 	rpc_scan(p, "getinfo", take(json_out_obj(NULL, NULL, NULL)),
@@ -1593,4 +1597,8 @@ int main(int argc, char *argv[])
 			"Enable renepay payment flows debug info.",
 			flag_option, &pay_plugin->debug_payflow),
 		NULL);
+	
+	// TODO(eduardo): I think this is actually never executed
+	tal_free(pay_plugin->ctx);
+	return 0;
 }
diff --git a/tests/test_renepay.py b/tests/test_renepay.py
new file mode 100644
index 000000000000..a8a40b6da2e7
--- /dev/null
+++ b/tests/test_renepay.py
@@ -0,0 +1,55 @@
+from fixtures import *  # noqa: F401,F403
+from fixtures import TEST_NETWORK
+from pathlib import Path
+from io import BytesIO
+from pyln.client import RpcError, Millisatoshi
+from pyln.proto.onion import TlvPayload
+from pyln.testing.utils import EXPERIMENTAL_DUAL_FUND, FUNDAMOUNT, scid_to_int
+from utils import (
+    DEVELOPER, wait_for, only_one, sync_blockheight, TIMEOUT,
+    EXPERIMENTAL_FEATURES, VALGRIND, mine_funding_to_announce, first_scid
+)
+import copy
+import os
+import pytest
+import random
+import re
+import string
+import struct
+import subprocess
+import time
+import unittest
+
+def test_pay_to_peer(node_factory):
+    '''Testing simply paying a peer.'''
+    l1, l2 = node_factory.line_graph(2)
+    inv = l2.rpc.invoice(123000, 'test_renepay', 'description')['bolt11']
+    before = int(time.time())
+    details = l1.rpc.call('renepay',{'invstring':inv})
+    after = time.time()
+    assert details['status'] == 'complete'
+    assert details['amount_msat'] == Millisatoshi(123000)
+    assert details['destination'] == l2.info['id']
+
+def test_mpp(node_factory):
+    '''Test paying a remote node using two routes.
+    1----2----4
+    |         |
+    3----5----6
+    Try paying 1.2M sats from 1 to 6.
+    '''
+    opts = [
+        {'disable-mpp': None, 'fee-base':0, 'fee-per-satoshi':0},
+    ]
+    l1, l2, l3, l4, l5,l6 = node_factory.get_nodes(6, opts=opts*6)
+    node_factory.join_nodes([l1, l2, l4, l6],
+        wait_for_announce=True,fundamount=1000000)
+    node_factory.join_nodes([l1, l3, l5, l6], 
+        wait_for_announce=True,fundamount=1000000)
+    
+    send_amount = Millisatoshi('1200000sat')
+    inv = l6.rpc.invoice(send_amount, 'test_renepay', 'description')['bolt11']
+    details = l1.rpc.call('renepay',{'invstring':inv})
+    assert details['status'] == 'complete'
+    assert details['amount_msat'] == send_amount
+    assert details['destination'] == l6.info['id']

From c77e743de8b635fa05313652bb0b3acca48670e0 Mon Sep 17 00:00:00 2001
From: Lagrang3 <eduardo.quintana@pm.me>
Date: Wed, 21 Jun 2023 14:43:39 +0100
Subject: [PATCH 54/64] renepay: testing different ways to cleanup at shutdown

---
 plugins/renepay/pay.c | 88 +++++++++++++++++++++++++++----------------
 1 file changed, 55 insertions(+), 33 deletions(-)

diff --git a/plugins/renepay/pay.c b/plugins/renepay/pay.c
index 8639e2afde86..b02952f0a2df 100644
--- a/plugins/renepay/pay.c
+++ b/plugins/renepay/pay.c
@@ -174,8 +174,19 @@ void amount_msat_reduce_(struct amount_msat *dst,
 #if DEVELOPER
 static void memleak_mark(struct plugin *p, struct htable *memtable)
 {
-	memleak_scan_obj(memtable, pay_plugin->ctx);
-	memleak_scan_htable(memtable, &pay_plugin->chan_extra_map->raw);
+	/* TODO(eduardo): understand the purpose of memleak_scan_obj, why use it
+	 * instead of tal_free?
+	 * 1st problem: this is executed before the plugin can process the
+	 * shutdown notification, 
+	 * 2nd problem: memleak_scan_obj does not propagate to children. 
+	 * For the moment let's just (incorrectly) do tal_free here
+	 * */
+	pay_plugin->ctx = tal_free(pay_plugin->ctx);
+	
+	// memleak_scan_obj(memtable, pay_plugin->ctx);
+	// memleak_scan_obj(memtable, pay_plugin->gossmap);
+	// memleak_scan_obj(memtable, pay_plugin->chan_extra_map);
+	// memleak_scan_htable(memtable, &pay_plugin->chan_extra_map->raw);
 }
 #endif
 
@@ -214,11 +225,7 @@ static const char *init(struct plugin *p,
 {
 	size_t num_channel_updates_rejected;
 
-	// TODO(eduardo): this is a weird fix I can't grasp yet. Why is it that
-	// `plugin_main` doesn't release `struct plugin *p` at shutdown?
-	// Where can I put my global state destructors now if `struct plugin *p`
-	// is not expected to free at shutdown?
-	pay_plugin->ctx = notleak_with_children(tal(p,tal_t));
+	pay_plugin->ctx = tal(p,tal_t);
 	pay_plugin->plugin = p;
 
 	rpc_scan(p, "getinfo", take(json_out_obj(NULL, NULL, NULL)),
@@ -746,7 +753,7 @@ static struct command_result *waitsendpay_failed(struct command *cmd,
 		// TODO(eduardo): I guess this should not happen unless some
 		// sendpay that would eventually fail gets stuck for some
 		// reason or some MPP part times out.
-		debug_info("waitsendpay_failed: received an old failure");
+		plugin_log(pay_plugin->plugin,LOG_DBG,"waitsendpay_failed: received an old failure");
 	}
 
 	u64 errcode;
@@ -765,24 +772,24 @@ static struct command_result *waitsendpay_failed(struct command *cmd,
 	switch (errcode) {
 	case PAY_UNPARSEABLE_ONION:
 		// TODO(eduardo): when can this be triggered?
-		debug_info("waitsendpay_failed: PAY_UNPARSEABLE_ONION");
+		plugin_log(pay_plugin->plugin,LOG_DBG,"waitsendpay_failed: PAY_UNPARSEABLE_ONION");
 		ret = handle_unhandleable_error(p, flow, "unparsable onion reply");
 		if (ret)
 			return ret;
 		goto done;
 	case PAY_DESTINATION_PERM_FAIL:
-		debug_info("waitsendpay_failed: PAY_DESTINATION_PERM_FAIL");
+		plugin_log(pay_plugin->plugin,LOG_DBG,"waitsendpay_failed: PAY_DESTINATION_PERM_FAIL");
 		return command_fail(cmd, PAY_DESTINATION_PERM_FAIL,
 				    "Got an final failure from destination");
 	case PAY_TRY_OTHER_ROUTE:
-		debug_info("waitsendpay_failed: PAY_TRY_OTHER_ROUTE");
+		plugin_log(pay_plugin->plugin,LOG_DBG,"waitsendpay_failed: PAY_TRY_OTHER_ROUTE");
 		break;
-
+	
 	default:
-		debug_info("waitsendpay_failed: unexpected errcode");
-		plugin_err(cmd->plugin,
+		plugin_log(pay_plugin->plugin,LOG_DBG,
 			   "Unexpected errcode from waitsendpay: %.*s",
 			   json_tok_full_len(err), json_tok_full(buf, err));
+		return command_fail(cmd, errcode, "Unexpected errcode from waitsendpay.");
 	}
 
 	datatok = json_get_member(buf, err, "data");
@@ -819,7 +826,8 @@ static struct command_result *waitsendpay_failed(struct command *cmd,
 		type_to_string(tmpctx, struct short_channel_id, &errscid),
 		msgtok->end - msgtok->start, buf + msgtok->start);
 		
-	debug_info("onion error %s from node #%u %s: %.*s",
+	plugin_log(pay_plugin->plugin,LOG_DBG,
+		"onion error %s from node #%u %s: %.*s",
 		onion_wire_name(onionerr),
 		erridx,
 		type_to_string(tmpctx, struct short_channel_id, &errscid),
@@ -853,7 +861,7 @@ static struct command_result *waitsendpay_failed(struct command *cmd,
 	case WIRE_INVALID_ONION_PAYLOAD:
 	case WIRE_INVALID_ONION_BLINDING:
 	case WIRE_EXPIRY_TOO_FAR:
-		debug_info("waitsendpay_failed: removing scid");
+		plugin_log(pay_plugin->plugin,LOG_DBG,"waitsendpay_failed: removing scid");
 		paynote(p, "... so we're removing scid");
 		tal_arr_expand(&p->active_payment->disabled, errscid);
 		goto done;
@@ -863,7 +871,7 @@ static struct command_result *waitsendpay_failed(struct command *cmd,
 	case WIRE_FEE_INSUFFICIENT:
 	case WIRE_INCORRECT_CLTV_EXPIRY:
 	case WIRE_EXPIRY_TOO_SOON:
-		debug_info("waitsendpay_failed: apply channel_update");
+		plugin_log(pay_plugin->plugin,LOG_DBG,"waitsendpay_failed: apply channel_update");
 		/* FIXME: Check scid! */
 		update = channel_update_from_onion_error(tmpctx, buf, rawoniontok);
 		if (update)
@@ -875,7 +883,7 @@ static struct command_result *waitsendpay_failed(struct command *cmd,
 
 	/* Insufficient funds! */
 	case WIRE_TEMPORARY_CHANNEL_FAILURE: {
-		debug_info("waitsendpay_failed: Insufficient funds!");
+		plugin_log(pay_plugin->plugin,LOG_DBG,"waitsendpay_failed: Insufficient funds!");
 		/* OK new max is amount - 1 */
 		struct amount_msat max_possible;
 		if (!amount_msat_sub(&max_possible,
@@ -895,7 +903,7 @@ static struct command_result *waitsendpay_failed(struct command *cmd,
 	/* FIXME: We should probably pause until all parts returned, or at least
 	 * extend rexmit timer! */
 	case WIRE_MPP_TIMEOUT:
-		debug_info("waitsendpay_failed: WIRE_MPP_TIMEOUT");
+		plugin_log(pay_plugin->plugin,LOG_DBG,"waitsendpay_failed: WIRE_MPP_TIMEOUT");
 		paynote(p, "... will continue");
 		p->status = PAYMENT_MPP_TIMEOUT;
 		goto done;
@@ -904,7 +912,7 @@ static struct command_result *waitsendpay_failed(struct command *cmd,
 	case WIRE_INCORRECT_OR_UNKNOWN_PAYMENT_DETAILS:
 	case WIRE_FINAL_INCORRECT_CLTV_EXPIRY:
 	case WIRE_FINAL_INCORRECT_HTLC_AMOUNT:
-		debug_info("waitsendpay_failed: final destination fail");
+		plugin_log(pay_plugin->plugin,LOG_DBG,"waitsendpay_failed: final destination fail");
 		paynote(p, "... fatal");
 		return command_fail(cmd, PAY_DESTINATION_PERM_FAIL,
 				    "Destination said %s: %.*s",
@@ -914,7 +922,7 @@ static struct command_result *waitsendpay_failed(struct command *cmd,
 	}
 	/* Unknown response? */
 	if (erridx == tal_count(flow->path_nodes)) {
-		debug_info("waitsendpay_failed: unknown error code %u: %.*s",
+		plugin_log(pay_plugin->plugin,LOG_DBG,"waitsendpay_failed: unknown error code %u: %.*s",
 				    onionerr,
 				    msgtok->end - msgtok->start,
 				    buf + msgtok->start);
@@ -932,12 +940,6 @@ static struct command_result *waitsendpay_failed(struct command *cmd,
 				  &flow->path_nodes[erridx]),
 		   erridx, tal_count(flow->path_nodes),
 		   onionerr);
-	debug_info(
-		   "Node %s (%u/%zu) gave unknown error code %u",
-		   type_to_string(tmpctx, struct node_id,
-				  &flow->path_nodes[erridx]),
-		   erridx, tal_count(flow->path_nodes),
-		   onionerr);
 	tal_arr_expand(&p->active_payment->disabled, errscid);
 
 done:
@@ -1128,7 +1130,7 @@ static struct command_result *try_paying(struct command *cmd,
 	if (!amount_msat_sub(&remaining, p->amount, p->total_delivering))
 		abort();
 
-	debug_info(fmt_chan_extra_map(tmpctx,pay_plugin->chan_extra_map));
+	plugin_log(pay_plugin->plugin,LOG_DBG,fmt_chan_extra_map(tmpctx,pay_plugin->chan_extra_map));
 	
 	/* We let this return an unlikely path, as it's better to try once
 	 * than simply refuse.  Plus, models are not truth! */
@@ -1138,7 +1140,7 @@ static struct command_result *try_paying(struct command *cmd,
 	struct pay_flow **pay_flows = get_payflows(p, remaining, feebudget, first_time,
 				 amount_msat_eq(p->total_delivering, AMOUNT_MSAT(0)));
 	
-	debug_info(fmt_payflows(tmpctx,pay_flows));
+	plugin_log(pay_plugin->plugin,LOG_DBG,fmt_payflows(tmpctx,pay_flows));
 	
 	/* MCF cannot find a feasible route, we stop. */
 	// TODO(eduardo): alternatively we can fallback to `pay`.
@@ -1193,7 +1195,7 @@ static void renepay_cleanup(struct active_payment *ap)
 	ap->localmods_applied=false;
 	tal_free(ap->local_gossmods);
 	
-	debug_info(fmt_chan_extra_map(tmpctx,pay_plugin->chan_extra_map));
+	plugin_log(pay_plugin->plugin,LOG_DBG,fmt_chan_extra_map(tmpctx,pay_plugin->chan_extra_map));
 	
 	ap->rexmit_timer = tal_free(ap->rexmit_timer);
 	
@@ -1308,9 +1310,9 @@ static struct command_result *json_pay(struct command *cmd,
 		   NULL))
 		return command_param_failed();
 	
-	debug_info("json_pay, renepay-debug-mcf option is set to %s\n",
+	plugin_log(pay_plugin->plugin,LOG_DBG,"json_pay, renepay-debug-mcf option is set to %s\n",
 		   pay_plugin->debug_mcf ? "true" : "false");
-	debug_info("json_pay, renepay-debug-payflow option is set to %s\n",
+	plugin_log(pay_plugin->plugin,LOG_DBG,"json_pay, renepay-debug-payflow option is set to %s\n",
 		   pay_plugin->debug_payflow ? "true" : "false");
 
 	
@@ -1560,6 +1562,19 @@ static struct command_result *json_pay(struct command *cmd,
 	return send_outreq(cmd->plugin, req);
 }
 
+static struct command_result *json_shutdown(struct command *cmd,
+					         const char *buf,
+					         const jsmntok_t *params)
+{
+	/* TODO(eduardo): 
+	 * 1. at shutdown the `struct plugin *p` is not freed,
+	 * 2. `memleak_check` is called before we have the chance to get this
+	 * notification. */
+	plugin_log(pay_plugin->plugin,LOG_DBG,"received shutdown notification, freeing data.");
+	pay_plugin->ctx = tal_free(pay_plugin->ctx);
+	return notification_handled(cmd);
+}
+
 static const struct plugin_command commands[] = {
 	{
 		"renepaystatus",
@@ -1577,6 +1592,13 @@ static const struct plugin_command commands[] = {
 	},
 };
 
+static const struct plugin_notification notifications[] = {
+	{
+		"shutdown",
+		json_shutdown,
+	}
+};
+
 int main(int argc, char *argv[])
 {
 	setup_locale();
@@ -1587,7 +1609,7 @@ int main(int argc, char *argv[])
 		/* init_rpc */ true, 
 		/* features */ NULL, 
 		commands, ARRAY_SIZE(commands), 
-		/* notifications */ NULL, 0, 
+		notifications, ARRAY_SIZE(notifications),
 		/* hooks */ NULL, 0, 
 		/* notification topics */ NULL, 0,
 		plugin_option("renepay-debug-mcf", "flag",

From fbca98c8a9ccb328575976e318dc00231ab6aad3 Mon Sep 17 00:00:00 2001
From: Lagrang3 <eduardo.quintana@pm.me>
Date: Thu, 22 Jun 2023 10:37:00 +0100
Subject: [PATCH 55/64] renepay: on critical error log messages

---
 plugins/renepay/debug.h                |  42 ++---
 plugins/renepay/flow.c                 | 175 +++++++++++++-------
 plugins/renepay/pay.c                  |  68 +++++---
 plugins/renepay/pay_flow.c             |  20 ++-
 plugins/renepay/test/Makefile          |   3 -
 plugins/renepay/test/run-map.c         | 220 -------------------------
 plugins/renepay/test/run-mcf-diamond.c |  37 +++--
 plugins/renepay/test/run-mcf.c         |  36 ++--
 plugins/renepay/test/run-testflow.c    |   5 +-
 9 files changed, 244 insertions(+), 362 deletions(-)
 delete mode 100644 plugins/renepay/test/run-map.c

diff --git a/plugins/renepay/debug.h b/plugins/renepay/debug.h
index acccff436ffe..875ea0048bf8 100644
--- a/plugins/renepay/debug.h
+++ b/plugins/renepay/debug.h
@@ -16,45 +16,29 @@ void _debug_info(const char* fname, const char *fmt, ...);
 void _debug_call(const char* fname, const char* fun);
 void _debug_exec_branch(const char* fname,const char* fun, int lineno);
 
-#ifndef MYLOCALDEBUG
+#ifndef MYLOG
+#define MYLOG "/tmp/debug.txt"
+#endif
 
-#define debug_reply(buf,toks) \
-	if(pay_plugin->debug_payflow) \
-	plugin_log(pay_plugin->plugin,LOG_DBG,\
-	"%.*s",json_tok_full_len(toks),json_tok_full(buf,toks))
+#ifdef FLOW_UNITTEST
 
 #define debug_info(...) \
-	if(pay_plugin->debug_payflow) \
-	plugin_log(pay_plugin->plugin,LOG_DBG,__VA_ARGS__)
-	
-#define debug_call() \
-	if(pay_plugin->debug_payflow) \
-	plugin_log(pay_plugin->plugin,LOG_DBG,"calling %s",__PRETTY_FUNCTION__)
+	_debug_info(MYLOG,__VA_ARGS__)
 
-#define debug_exec_branch() \
-	if(pay_plugin->debug_payflow) \
-	plugin_log(pay_plugin->plugin,LOG_DBG,"executing line: %d (%s)",\
-		__LINE__,__PRETTY_FUNCTION__)
+#define debug_err(...) \
+	_debug_info(MYLOG,__VA_ARGS__); abort()
 
 #else
 
-#define MYLOG "/tmp/debug.txt"
-#define debug_outreq(req) \
-	_debug_outreq(MYLOG,req)
-
-#define debug_reply(buf,toks) \
-	_debug_reply(MYLOG,buf,toks)
+#include <plugins/renepay/pay.h>
 
 #define debug_info(...) \
-	_debug_info(MYLOG,__VA_ARGS__)
-	
-
-#define debug_call() \
-	_debug_call(MYLOG,__PRETTY_FUNCTION__)
-
-#define debug_exec_branch() \
-	_debug_exec_branch(MYLOG,__PRETTY_FUNCTION__,__LINE__)
+	plugin_log(pay_plugin->plugin,LOG_DBG,__VA_ARGS__)
 
+#define debug_err(...) \
+	plugin_err(pay_plugin->plugin,__VA_ARGS__)
 
 #endif
+
+	
 #endif
diff --git a/plugins/renepay/flow.c b/plugins/renepay/flow.c
index d602a5bbd37a..f49d223aaeec 100644
--- a/plugins/renepay/flow.c
+++ b/plugins/renepay/flow.c
@@ -7,6 +7,7 @@
 #include <math.h>
 #include <stdio.h>
 #include <plugins/renepay/flow.h>
+#include <plugins/renepay/debug.h>
 #include <common/type_to_string.h>
 
 #ifndef SUPERVERBOSE
@@ -92,9 +93,21 @@ void chan_extra_adjust_half(struct chan_extra *ce,
 			    int dir)
 {
 	if(!amount_msat_sub(&ce->half[dir].known_max,ce->capacity,ce->half[!dir].known_min))
-		abort();
+	{
+		debug_err("%s cannot substract capacity=%s and known_min=%s",
+			__PRETTY_FUNCTION__,
+			type_to_string(tmpctx,struct amount_msat,&ce->capacity),
+			type_to_string(tmpctx,struct amount_msat,&ce->half[!dir].known_min)
+			);
+	}
 	if(!amount_msat_sub(&ce->half[dir].known_min,ce->capacity,ce->half[!dir].known_max))
-		abort();
+	{
+		debug_err("%s cannot substract capacity=%s and known_max=%s",
+			__PRETTY_FUNCTION__,
+			type_to_string(tmpctx,struct amount_msat,&ce->capacity),
+			type_to_string(tmpctx,struct amount_msat,&ce->half[!dir].known_max)
+			);
+	}
 }
 
 
@@ -106,8 +119,11 @@ static void chan_extra_can_send_(
 {
 	if(amount_msat_greater(x,ce->capacity))
 	{
-		// It should never happen thatn x>capacity
-		abort();
+		debug_err("%s unexpected capacity=%s is less than x=%s",
+			__PRETTY_FUNCTION__,
+			type_to_string(tmpctx,struct amount_msat,&ce->capacity),
+			type_to_string(tmpctx,struct amount_msat,&x)
+			);
 		x = ce->capacity;
 	}
 	
@@ -125,7 +141,10 @@ void chan_extra_can_send(
 	struct chan_extra *ce = chan_extra_map_get(chan_extra_map,
 						   scid);
 	if(!ce)
-		abort();
+	{
+		debug_err("%s unexpected chan_extra ce is NULL",
+			__PRETTY_FUNCTION__);
+	}
 	chan_extra_can_send_(ce,dir,x);
 }
 /* Update the knowledge that this (channel,direction) cannot send x msat.*/
@@ -136,8 +155,10 @@ static void chan_extra_cannot_send_(
 {
 	if(!amount_msat_sub(&x,x,AMOUNT_MSAT(1)))
 	{
-		// It should never happen that x==0
-		abort();
+		debug_err("%s unexpected x=%s is less than 0msat",
+			__PRETTY_FUNCTION__,
+			type_to_string(tmpctx,struct amount_msat,&x)
+			);
 		x = AMOUNT_MSAT(0);
 	}
 	
@@ -155,7 +176,10 @@ void chan_extra_cannot_send(
 	struct chan_extra *ce = chan_extra_map_get(chan_extra_map,
 						   scid);
 	if(!ce)
-		abort();
+	{
+		debug_err("%s unexpected chan_extra ce is NULL",
+			__PRETTY_FUNCTION__);
+	}
 	chan_extra_cannot_send_(ce,dir,x);
 }
 /* Update the knowledge that this (channel,direction) has liquidity x.*/
@@ -166,8 +190,11 @@ static void chan_extra_set_liquidity_(
 {
 	if(amount_msat_greater(x,ce->capacity))
 	{
-		// It should never happen thatn x>capacity
-		abort();
+		debug_err("%s unexpected capacity=%s is less than x=%s",
+			__PRETTY_FUNCTION__,
+			type_to_string(tmpctx,struct amount_msat,&ce->capacity),
+			type_to_string(tmpctx,struct amount_msat,&x)
+			);
 		x = ce->capacity;
 	}
 	
@@ -185,7 +212,10 @@ void chan_extra_set_liquidity(
 	struct chan_extra *ce = chan_extra_map_get(chan_extra_map,
 						   scid);
 	if(!ce)
-		abort();
+	{
+		debug_err("%s unexpected chan_extra ce is NULL",
+			__PRETTY_FUNCTION__);
+	}
 	chan_extra_set_liquidity_(ce,dir,x);
 }
 /* Update the knowledge that this (channel,direction) has sent x msat.*/
@@ -196,8 +226,11 @@ static void chan_extra_sent_success_(
 {
 	if(amount_msat_greater(x,ce->capacity))
 	{
-		// It should never happen thatn x>capacity
-		abort();
+		debug_err("%s unexpected capacity=%s is less than x=%s",
+			__PRETTY_FUNCTION__,
+			type_to_string(tmpctx,struct amount_msat,&ce->capacity),
+			type_to_string(tmpctx,struct amount_msat,&x)
+			);
 		x = ce->capacity;
 	}
 	
@@ -222,7 +255,10 @@ void chan_extra_sent_success(
 	struct chan_extra *ce = chan_extra_map_get(chan_extra_map,
 						   scid);
 	if(!ce)
-		abort();
+	{
+		debug_err("%s unexpected chan_extra ce is NULL",
+			__PRETTY_FUNCTION__);
+	}
 	chan_extra_sent_success_(ce,dir,x);
 }
 /* Forget a bit about this (channel,direction) state. */
@@ -254,7 +290,10 @@ void chan_extra_relax(
 	struct chan_extra *ce = chan_extra_map_get(chan_extra_map,
 						   scid);
 	if(!ce)
-		abort();
+	{
+		debug_err("%s unexpected chan_extra ce is NULL",
+			__PRETTY_FUNCTION__);
+	}
 	chan_extra_relax_(ce,dir,x,y);
 }
 
@@ -307,8 +346,9 @@ void uncertainty_network_update(
  		struct chan_extra *ce = chan_extra_map_get(chan_extra_map,del_list[i]);
 		if(!ce)
 		{
-			SUPERVERBOSE("%s: aborting\n",__PRETTY_FUNCTION__);
-			abort();
+			debug_err("%s (line %d) unexpected chan_extra ce is NULL",
+				__PRETTY_FUNCTION__,
+				__LINE__);
 		}
 		chan_extra_map_del(chan_extra_map, ce);
 		tal_free(ce);
@@ -331,13 +371,15 @@ void uncertainty_network_update(
 			
 			if(!gossmap_chan_get_capacity(gossmap,chan,&cap))
 			{
-				SUPERVERBOSE("%s: aborting\n",__PRETTY_FUNCTION__);
-				abort();
+				debug_err("%s (line %d) unable to fetch channel capacity",
+					__PRETTY_FUNCTION__,
+					__LINE__);
 			}
 			if(!amount_sat_to_msat(&cap_msat,cap))
 			{
-				SUPERVERBOSE("%s: aborting\n",__PRETTY_FUNCTION__);
-				abort();
+				debug_err("%s (line %d) unable convert sat to msat",
+					__PRETTY_FUNCTION__,
+					__LINE__);
 			}
 			new_chan_extra(chan_extra_map,scid,cap_msat);
 		}
@@ -397,8 +439,10 @@ get_chan_extra_half_by_chan_verify(
 		if (!gossmap_chan_get_capacity(gossmap,chan, &cap) || 
 		    !amount_sat_to_msat(&cap_msat, cap))
 		{
-			SUPERVERBOSE("%s: aborting\n",__PRETTY_FUNCTION__);
-			abort();
+			debug_err("%s (line %d) unable convert sat to msat or "
+				"get channel capacity",
+				__PRETTY_FUNCTION__,
+				__LINE__);
 		}
 		h = & new_chan_extra(chan_extra_map,scid,cap_msat)->half[dir];
 		
@@ -487,18 +531,20 @@ static double edge_probability(struct amount_msat min, struct amount_msat max,
 	// one past the last known value, makes computations simpler
 	if(!amount_msat_add(&B,B,one))
 	{
-		SUPERVERBOSE("%s: aborting, cannot add %s + %s\n",__PRETTY_FUNCTION__,
-			   type_to_string(this_ctx, struct amount_msat, &B),
-			   type_to_string(this_ctx, struct amount_msat, &one));
-		abort();
+		debug_err("%s (line %d) cannot add B=%s and %s",
+			__PRETTY_FUNCTION__,
+			__LINE__,
+			type_to_string(this_ctx, struct amount_msat, &B),
+			type_to_string(this_ctx, struct amount_msat, &one));
 	}	
 	// in_flight cannot be greater than max
 	if(!amount_msat_sub(&B,B,in_flight))
 	{
-		SUPERVERBOSE("%s: aborting, cannot substract %s - %s\n",__PRETTY_FUNCTION__,
-			   type_to_string(this_ctx, struct amount_msat, &B),
-			   type_to_string(this_ctx, struct amount_msat, &in_flight));
-		abort();
+		debug_err("%s (line %d) in_flight=%s cannot be greater than B=%s",
+			__PRETTY_FUNCTION__,
+			__LINE__,
+			type_to_string(this_ctx, struct amount_msat, &in_flight),
+			type_to_string(this_ctx, struct amount_msat, &B));
 	}	
 	struct amount_msat A=min; // = MAX(0,min-in_flight);
 	
@@ -510,10 +556,11 @@ static double edge_probability(struct amount_msat min, struct amount_msat max,
 	// B cannot be smaller than or equal A
 	if(!amount_msat_sub(&denominator,B,A) || amount_msat_less_eq(B,A))
 	{
-		SUPERVERBOSE("%s: aborting, cannot substract %s - %s\n",__PRETTY_FUNCTION__,
-			   type_to_string(this_ctx, struct amount_msat, &B),
-			   type_to_string(this_ctx, struct amount_msat, &A));
-		abort();
+		debug_err("%s (line %d) B=%s must be greater than A=%s",
+			__PRETTY_FUNCTION__,
+			__LINE__,
+			type_to_string(this_ctx, struct amount_msat, &B),
+			type_to_string(this_ctx, struct amount_msat, &A));
 	}
 	struct amount_msat numerator; // MAX(0,B-f)
 	
@@ -529,7 +576,7 @@ static double edge_probability(struct amount_msat min, struct amount_msat max,
 
 
 
-
+// TODO(eduardo): remove this function, is a duplicate
 void remove_completed_flow(const struct gossmap *gossmap,
 			   struct chan_extra_map *chan_extra_map,
 			   struct flow *flow)
@@ -541,17 +588,23 @@ void remove_completed_flow(const struct gossmap *gossmap,
 							       flow->dirs[i]);
 		if (!amount_msat_sub(&h->htlc_total, h->htlc_total, flow->amounts[i]))
 		{
-			SUPERVERBOSE("%s: aborting\n",__PRETTY_FUNCTION__);
-			abort();
+			debug_err("%s could not substract HTLC amounts, "
+				   "half total htlc amount = %s, "
+				   "flow->amounts[%lld] = %s.",
+				   __PRETTY_FUNCTION__,
+				   type_to_string(tmpctx, struct amount_msat, &h->htlc_total),
+				   i,
+				   type_to_string(tmpctx, struct amount_msat, &flow->amounts[i]));
 		}
 		if (h->num_htlcs == 0)
 		{
-			SUPERVERBOSE("%s: aborting\n",__PRETTY_FUNCTION__);
-			abort();
+			debug_err("%s could not decrease HTLC count.",
+				   __PRETTY_FUNCTION__);
 		}
 		h->num_htlcs--;
 	}
 }
+// TODO(eduardo): remove this function, is a duplicate
 void remove_completed_flow_set(
 		const struct gossmap *gossmap,
 		struct chan_extra_map *chan_extra_map,
@@ -563,6 +616,7 @@ void remove_completed_flow_set(
 	}
 }
 
+// TODO(eduardo): remove this function, is a duplicate
 void commit_flow(
 		const struct gossmap *gossmap,
 		struct chan_extra_map *chan_extra_map,
@@ -575,12 +629,16 @@ void commit_flow(
 							       flow->dirs[i]);
 		if (!amount_msat_add(&h->htlc_total, h->htlc_total, flow->amounts[i]))
 		{
-			SUPERVERBOSE("%s: aborting\n",__PRETTY_FUNCTION__);
-			abort();
+			debug_err("%s could not add HTLC amounts, "
+				   "flow->amounts[%lld] = %s.",
+				   __PRETTY_FUNCTION__,
+				   i,
+				   type_to_string(tmpctx, struct amount_msat, &flow->amounts[i]));
 		}
 		h->num_htlcs++;
 	}
 }
+// TODO(eduardo): remove this function, is a duplicate
 void commit_flow_set(
 		const struct gossmap *gossmap,
 		struct chan_extra_map *chan_extra_map,
@@ -616,8 +674,8 @@ void flow_complete(struct flow *flow,
 		
 		if(!h)
 		{
-			SUPERVERBOSE("%s: aborting\n",__PRETTY_FUNCTION__);
-			abort();
+			debug_err("%s unexpected chan_extra_half is NULL",
+				__PRETTY_FUNCTION__);
 		}
 		
 		flow->amounts[i] = delivered;
@@ -630,8 +688,8 @@ void flow_complete(struct flow *flow,
 					 flow_edge(flow, i)->base_fee,
 					 flow_edge(flow, i)->proportional_fee))
 		{
-			SUPERVERBOSE("%s: aborting\n",__PRETTY_FUNCTION__);
-			abort();
+			debug_err("%s fee overflow",
+				__PRETTY_FUNCTION__);
 		}
 	}
 }
@@ -701,8 +759,9 @@ double flow_set_probability(
 			struct amount_msat prev_flow;
 			if(!amount_msat_add(&prev_flow,h->htlc_total,in_flight[c_idx].half[c_dir]))
 			{
-				SUPERVERBOSE("%s: aborting\n",__PRETTY_FUNCTION__);
-				abort();
+				debug_err("%s (line %d) in-flight amount_msat overflow",
+					__PRETTY_FUNCTION__,
+					__LINE__);
 			}
 			
 			prob *= edge_probability(h->known_min,h->known_max,
@@ -712,8 +771,9 @@ double flow_set_probability(
 					in_flight[c_idx].half[c_dir],
 					deliver))
 			{
-				SUPERVERBOSE("%s: aborting\n",__PRETTY_FUNCTION__);
-				abort();
+				debug_err("%s (line %d) in-flight amount_msat overflow",
+					__PRETTY_FUNCTION__,
+					__LINE__);
 			}
 		}
 	}
@@ -793,8 +853,9 @@ static void get_medians(const struct gossmap *gossmap,
 		*median_capacity = amount;
 	else if (!amount_sat_to_msat(median_capacity, caps[num_caps / 2]))
 	{
-		SUPERVERBOSE("%s: aborting\n",__PRETTY_FUNCTION__);
-		abort();
+		debug_err("%s (line %d) amount_msat overflow",
+			__PRETTY_FUNCTION__,
+			__LINE__);
 	}
 	asort(fees, num_fees, cmp_amount_msat, NULL);
 	if (!num_caps)
@@ -859,13 +920,15 @@ struct amount_msat flow_set_fee(struct flow **flows)
 				     flows[i]->amounts[0],
 				     flows[i]->amounts[n-1]))
 		{
-			SUPERVERBOSE("%s: aborting\n",__PRETTY_FUNCTION__);
-			abort();
+			debug_err("%s (line %d) amount_msat overflow",
+				__PRETTY_FUNCTION__,
+				__LINE__);
 		}
 		if(!amount_msat_add(&fee, this_fee,fee))
 		{
-			SUPERVERBOSE("%s: aborting\n",__PRETTY_FUNCTION__);
-			abort();
+			debug_err("%s (line %d) amount_msat overflow",
+				__PRETTY_FUNCTION__,
+				__LINE__);
 		}
 	}
 	return fee;
diff --git a/plugins/renepay/pay.c b/plugins/renepay/pay.c
index b02952f0a2df..a13a83adaf76 100644
--- a/plugins/renepay/pay.c
+++ b/plugins/renepay/pay.c
@@ -110,7 +110,7 @@ static void payment_initialize(
 
 static struct command_result *payment_success(struct payment *p)
 {
-	debug_call();	
+	plugin_log(pay_plugin->plugin,LOG_DBG,"calling %s",__PRETTY_FUNCTION__);
 	payment_check_delivering_all(p);
 	
 	struct json_stream *response 
@@ -194,7 +194,7 @@ static void remove_htlc_payflow_and_update_knowlege(
 		struct pay_flow *flow,
 		struct chan_extra_map *chan_extra_map)
 {
-	debug_call();
+	plugin_log(pay_plugin->plugin,LOG_DBG,"calling %s",__PRETTY_FUNCTION__);
 	
 	// TODO(eduardo): big assumption here, HTLCs can just simply dissapear
 	// if the MPP payment completes or the flow fails.
@@ -225,7 +225,7 @@ static const char *init(struct plugin *p,
 {
 	size_t num_channel_updates_rejected;
 
-	pay_plugin->ctx = tal(p,tal_t);
+	pay_plugin->ctx = notleak_with_children(tal(p,tal_t));
 	pay_plugin->plugin = p;
 
 	rpc_scan(p, "getinfo", take(json_out_obj(NULL, NULL, NULL)),
@@ -496,7 +496,7 @@ static void payment_settimer(struct payment *p)
 /* Happens when timer goes off, but also works to arm timer if nothing to do */
 static void timer_kick(struct payment *p)
 {
-	debug_call();
+	plugin_log(pay_plugin->plugin,LOG_DBG,"calling %s",__PRETTY_FUNCTION__);
 	
 	/* No, but payment hasn't gone through, so let's wait a bit. */
 	
@@ -538,7 +538,7 @@ static struct command_result *waitsendpay_succeeded(struct command *cmd,
 						    const jsmntok_t *result,
 						    struct pay_flow *flow)
 {
-	debug_call();
+	plugin_log(pay_plugin->plugin,LOG_DBG,"calling %s",__PRETTY_FUNCTION__);
 	
 	struct payment *p = flow->payment;
 	
@@ -577,7 +577,7 @@ static struct command_result *handle_unhandleable_error(struct payment *p,
 							const struct pay_flow *flow,
 							const char *what)
 {
-	debug_call();
+	plugin_log(pay_plugin->plugin,LOG_DBG,"calling %s",__PRETTY_FUNCTION__);
 	size_t n = tal_count(flow);
 
 	/* We got a mangled reply.  We don't know who to penalize! */
@@ -619,7 +619,7 @@ static struct command_result *addgossip_done(struct command *cmd,
 					     const jsmntok_t *err,
 					     struct addgossip *adg)
 {
-	debug_call();
+	plugin_log(pay_plugin->plugin,LOG_DBG,"calling %s",__PRETTY_FUNCTION__);
 	struct payment *p = adg->flow->payment;
 
 	/* Release this: if it's the last flow we'll retry immediately */
@@ -636,7 +636,7 @@ static struct command_result *addgossip_failure(struct command *cmd,
 						struct addgossip *adg)
 
 {
-	debug_call();
+	plugin_log(pay_plugin->plugin,LOG_DBG,"calling %s",__PRETTY_FUNCTION__);
 	struct payment *p = adg->flow->payment;
 
 	paynote(p, "addgossip failed, removing channel %s (%.*s)",
@@ -652,7 +652,7 @@ static struct command_result *submit_update(struct command *cmd,
 					    const u8 *update,
 					    struct short_channel_id errscid)
 {
-	debug_call();
+	plugin_log(pay_plugin->plugin,LOG_DBG,"calling %s",__PRETTY_FUNCTION__);
 	struct payment *p = flow->payment;
 	struct out_req *req;
 	struct addgossip *adg = tal(cmd, struct addgossip);
@@ -739,8 +739,10 @@ static struct command_result *waitsendpay_failed(struct command *cmd,
 						 const jsmntok_t *err,
 						 struct pay_flow *flow)
 {
-	debug_call();
-	debug_reply(buf,err);
+	plugin_log(pay_plugin->plugin,LOG_DBG,"calling %s",__PRETTY_FUNCTION__);
+	plugin_log(pay_plugin->plugin,LOG_DBG,"waitsendpay failed with reply %.*s",
+		   json_tok_full_len(err),
+		   json_tok_full(buf, err));
 	
 	struct payment *p = flow->payment;
 	
@@ -952,7 +954,7 @@ static struct command_result *flow_sent(struct command *cmd,
 					const jsmntok_t *result,
 					struct pay_flow *flow)
 {
-	debug_call();
+	plugin_log(pay_plugin->plugin,LOG_DBG,"calling %s",__PRETTY_FUNCTION__);
 	
 	struct payment *p = flow->payment;
 	struct out_req *req;
@@ -978,7 +980,7 @@ static struct command_result *flow_sendpay_failed(struct command *cmd,
 						  const jsmntok_t *err,
 						  struct pay_flow *flow)
 {
-	debug_call();
+	plugin_log(pay_plugin->plugin,LOG_DBG,"calling %s",__PRETTY_FUNCTION__);
 	
 	struct payment *p = flow->payment;
 	/* This is a fail. */
@@ -1012,7 +1014,7 @@ sendpay_flows(struct command *cmd,
 	      struct payment *p,
 	      struct pay_flow **flows STEALS)
 {
-	debug_call();
+	plugin_log(pay_plugin->plugin,LOG_DBG,"calling %s",__PRETTY_FUNCTION__);
 	paynote(p, "Sending out batch of %zu payments", tal_count(flows));
 	
 	for (size_t i = 0; i < tal_count(flows); i++) {
@@ -1096,7 +1098,7 @@ static struct command_result *try_paying(struct command *cmd,
 					 struct payment *p,
 					 bool first_time)
 {
-	debug_call();
+	plugin_log(pay_plugin->plugin,LOG_DBG,"calling %s",__PRETTY_FUNCTION__);
 	
 	// TODO(eduardo): does it make sense to have this limit on attempts?
 	/* I am classifying the flows in attempt cycles. */
@@ -1116,19 +1118,43 @@ static struct command_result *try_paying(struct command *cmd,
 	
 	/* Total feebudget  */
 	if (!amount_msat_sub(&feebudget, p->maxspend, p->amount))
-		abort();
+	{
+		plugin_err(pay_plugin->plugin,
+			   "%s could not substract maxspend=%s and amount=%s.",
+			   __PRETTY_FUNCTION__,
+			   type_to_string(tmpctx, struct amount_msat, &p->maxspend),
+			   type_to_string(tmpctx, struct amount_msat, &p->amount));
+	}
 
 	/* Fees spent so far */
 	if (!amount_msat_sub(&fees_spent, p->total_sent, p->total_delivering))
-		abort();
+	{
+		plugin_err(pay_plugin->plugin,
+			   "%s could not substract total_sent=%s and total_delivering=%s.",
+			   __PRETTY_FUNCTION__,
+			   type_to_string(tmpctx, struct amount_msat, &p->total_sent),
+			   type_to_string(tmpctx, struct amount_msat, &p->total_delivering));
+	}
 
 	/* Remaining fee budget. */
 	if (!amount_msat_sub(&feebudget, feebudget, fees_spent))
-		abort();
+	{
+		plugin_err(pay_plugin->plugin,
+			   "%s could not substract feebudget=%s and fees_spent=%s.",
+			   __PRETTY_FUNCTION__,
+			   type_to_string(tmpctx, struct amount_msat, &feebudget),
+			   type_to_string(tmpctx, struct amount_msat, &fees_spent));
+	}
 
 	/* How much are we still trying to send? */
 	if (!amount_msat_sub(&remaining, p->amount, p->total_delivering))
-		abort();
+	{
+		plugin_err(pay_plugin->plugin,
+			   "%s could not substract amount=%s and total_delivering=%s.",
+			   __PRETTY_FUNCTION__,
+			   type_to_string(tmpctx, struct amount_msat, &p->amount),
+			   type_to_string(tmpctx, struct amount_msat, &p->total_delivering));
+	}
 
 	plugin_log(pay_plugin->plugin,LOG_DBG,fmt_chan_extra_map(tmpctx,pay_plugin->chan_extra_map));
 	
@@ -1162,7 +1188,7 @@ static struct command_result *
 listpeerchannels_done(struct command *cmd, const char *buf,
 	       const jsmntok_t *result, struct payment *p)
 {
-	debug_call();
+	plugin_log(pay_plugin->plugin,LOG_DBG,"calling %s",__PRETTY_FUNCTION__);
 	if (!update_uncertainty_network_from_listpeerchannels(cmd->plugin, p, buf, result))
 		return command_fail(cmd, LIGHTNINGD,
 				    "listpeerchannels malformed: %.*s",
@@ -1182,7 +1208,7 @@ listpeerchannels_done(struct command *cmd, const char *buf,
  * into a valid state before the next payment. */
 static void renepay_cleanup(struct active_payment *ap)
 {
-	debug_call();
+	plugin_log(pay_plugin->plugin,LOG_DBG,"calling %s",__PRETTY_FUNCTION__);
 
 	/* Free all pending flows, release their HTLCs and update knowledge. */
 	ap->all_flows = tal_free(ap->all_flows);
diff --git a/plugins/renepay/pay_flow.c b/plugins/renepay/pay_flow.c
index 2f552895c22d..a54b6be4132b 100644
--- a/plugins/renepay/pay_flow.c
+++ b/plugins/renepay/pay_flow.c
@@ -548,11 +548,20 @@ void remove_htlc_payflow(
 							       flow->path_dirs[i]);
 		if (!amount_msat_sub(&h->htlc_total, h->htlc_total, flow->amounts[i]))
 		{
-			abort();
+			plugin_err(pay_plugin->plugin,
+				   "%s could not substract HTLC amounts, "
+				   "half total htlc amount = %s, "
+				   "flow->amounts[%lld] = %s.",
+				   __PRETTY_FUNCTION__,
+				   type_to_string(tmpctx, struct amount_msat, &h->htlc_total),
+				   i,
+				   type_to_string(tmpctx, struct amount_msat, &flow->amounts[i]));
 		}
 		if (h->num_htlcs == 0)
 		{
-			abort();
+			plugin_err(pay_plugin->plugin,
+				   "%s could not decrease HTLC count.",
+				   __PRETTY_FUNCTION__);
 		}
 		h->num_htlcs--;
 	}
@@ -568,7 +577,12 @@ void commit_htlc_payflow(
 							       flow->path_dirs[i]);
 		if (!amount_msat_add(&h->htlc_total, h->htlc_total, flow->amounts[i]))
 		{
-			abort();
+			plugin_err(pay_plugin->plugin,
+				   "%s could not add HTLC amounts, "
+				   "flow->amounts[%lld] = %s.",
+				   __PRETTY_FUNCTION__,
+				   i,
+				   type_to_string(tmpctx, struct amount_msat, &flow->amounts[i]));
 		}
 		h->num_htlcs++;
 	}
diff --git a/plugins/renepay/test/Makefile b/plugins/renepay/test/Makefile
index 37860ef9eeeb..4cf2bfa63349 100644
--- a/plugins/renepay/test/Makefile
+++ b/plugins/renepay/test/Makefile
@@ -15,6 +15,3 @@ PLUGIN_RENEPAY_TEST_COMMON_OBJS :=		\
 $(PLUGIN_RENEPAY_TEST_PROGRAMS): $(PLUGIN_RENEPAY_TEST_COMMON_OBJS) $(PLUGIN_LIB_OBJS) $(PLUGIN_COMMON_OBJS) $(JSMN_OBJS) $(CCAN_OBJS) bitcoin/chainparams.o common/gossmap.o common/fp16.o common/dijkstra.o common/bolt12.o common/bolt12_merkle.o wire/bolt12$(EXP)_wiregen.o
 
 check-renepay: $(PLUGIN_RENEPAY_TEST_PROGRAMS:%=unittest/%)
-	bash plugins/renepay/test/regtest1.sh 
-	bash plugins/renepay/test/regtest2.sh 
-	bash plugins/renepay/test/regtest3.sh 
diff --git a/plugins/renepay/test/run-map.c b/plugins/renepay/test/run-map.c
deleted file mode 100644
index da568432fa1f..000000000000
--- a/plugins/renepay/test/run-map.c
+++ /dev/null
@@ -1,220 +0,0 @@
-#include "config.h"
-#include <ccan/array_size/array_size.h>
-#include <ccan/read_write_all/read_write_all.h>
-#include <common/bigsize.h>
-#include <common/channel_id.h>
-#include <common/gossip_store.h>
-#include <common/node_id.h>
-#include <common/setup.h>
-#include <common/type_to_string.h>
-#include <common/wireaddr.h>
-#include <stdio.h>
-#include <assert.h>
-
-#include <bitcoin/short_channel_id.h>
-#include <ccan/htable/htable_type.h>
-
-struct chan_extra {
-	struct short_channel_id scid;
-	int value;
-};
-
-static inline const struct short_channel_id
-chan_extra_scid(const struct chan_extra *cd)
-{
-	return cd->scid;
-}
-
-static inline size_t hash_scid(const struct short_channel_id scid)
-{
-	/* scids cost money to generate, so simple hash works here */
-	return (scid.u64 >> 32)
-		^ (scid.u64 >> 16)
-		^ scid.u64;
-}
-
-static inline bool chan_extra_eq_scid(const struct chan_extra *cd,
-				      const struct short_channel_id scid)
-{
-	return short_channel_id_eq(&scid, &cd->scid);
-}
-
-HTABLE_DEFINE_TYPE(struct chan_extra,
-		   chan_extra_scid, hash_scid, chan_extra_eq_scid,
-		   chan_extra_map);
-
-
-static void destroy_chan_extra(struct chan_extra *ce,
-			       struct chan_extra_map *chan_extra_map)
-{
-	printf("calling %s with value = %d\n",__PRETTY_FUNCTION__,ce->value);
-	chan_extra_map_del(chan_extra_map, ce);
-}
-static inline void new_chan_extra_value(
-	const tal_t *ctx,
-	struct chan_extra_map *chan_extra_map,
-	const struct short_channel_id scid,
-	int value)
-{
-	// struct chan_extra *ce = tal(chan_extra_map, struct chan_extra);
-	struct chan_extra *ce = tal(ctx, struct chan_extra);
-	
-	ce->scid = scid;
-	ce->value=value;
-	
-	/* Remove self from map when done */
-	chan_extra_map_add(chan_extra_map, ce);
-	tal_add_destructor2(ce, destroy_chan_extra, chan_extra_map);
-}
-
-static struct chan_extra*
-get_chan_extra_value(struct chan_extra_map *chan_extra_map,
-			    const struct short_channel_id scid)
-{
-	struct chan_extra *ce;
-
-	ce = chan_extra_map_get(chan_extra_map, scid);
-	if (!ce)
-		return NULL;
-	return ce;
-}
-
-static void valgrind_ok1(void)
-{
-	struct short_channel_id scid1,scid2;
-	
-	tal_t *this_ctx = tal(tmpctx,tal_t);
-	
-	struct chan_extra_map *chan_extra_map
-		= tal(this_ctx, struct chan_extra_map);
-	
-	chan_extra_map_init(chan_extra_map);
-	
-	assert(short_channel_id_from_str("1x1x0", 7, &scid1));
-	assert(short_channel_id_from_str("2x1x0", 7, &scid2));
-	
-	{
-		tal_t *local_ctx = tal(this_ctx,tal_t);
-		
-		/* in this case: elements are allocated in a local context, and freed at
-		 * the end, i.e. before chan_extra_map is freed. */
-		new_chan_extra_value(local_ctx,chan_extra_map,scid1,11);
-		new_chan_extra_value(local_ctx,chan_extra_map,scid2,21);
-		
-		struct chan_extra *x1 = get_chan_extra_value(chan_extra_map,scid1);
-		struct chan_extra *x2 = get_chan_extra_value(chan_extra_map,scid2);
-		
-		assert(x1->value==11);
-		assert(x2->value==21);
-		
-		tal_free(local_ctx);
-	}
-	
-	tal_free(this_ctx);
-
-}
-static void valgrind_ok2(void)
-{
-	struct short_channel_id scid1,scid2;
-	
-	tal_t *this_ctx = tal(tmpctx,tal_t);
-	
-	struct chan_extra_map *chan_extra_map
-		= tal(this_ctx, struct chan_extra_map);
-	
-	chan_extra_map_init(chan_extra_map);
-	
-	assert(short_channel_id_from_str("1x2x0", 7, &scid1));
-	assert(short_channel_id_from_str("2x2x0", 7, &scid2));
-	
-	
-	/* in this case: elements are allocated with chan_extra_map as parent.
-	 * the end of this function. i.e. before chan_extra_map is freed. */
-	new_chan_extra_value(chan_extra_map,chan_extra_map,scid1,12);
-	new_chan_extra_value(chan_extra_map,chan_extra_map,scid2,22);
-	
-	struct chan_extra *x1 = get_chan_extra_value(chan_extra_map,scid1);
-	struct chan_extra *x2 = get_chan_extra_value(chan_extra_map,scid2);
-	
-	assert(x1->value==12);
-	assert(x2->value==22);
-	
-	/* elements are freed before chan_extra_map is freed */
-	tal_free(x1);
-	tal_free(x2);
-	
-	tal_free(this_ctx);
-
-}
-static void valgrind_fail3(void)
-{
-	struct short_channel_id scid1,scid2;
-	
-	tal_t *this_ctx = tal(tmpctx,tal_t);
-	
-	struct chan_extra_map *chan_extra_map
-		= tal(this_ctx, struct chan_extra_map);
-	
-	chan_extra_map_init(chan_extra_map);
-	
-	assert(short_channel_id_from_str("1x3x0", 7, &scid1));
-	assert(short_channel_id_from_str("2x3x0", 7, &scid2));
-	
-	
-	/* in this case: elements are allocated with chan_extra_map as parent.
-	 * the end of this function. i.e. before chan_extra_map is freed. */
-	new_chan_extra_value(chan_extra_map,chan_extra_map,scid1,13);
-	new_chan_extra_value(chan_extra_map,chan_extra_map,scid2,23);
-	
-	struct chan_extra *x1 = get_chan_extra_value(chan_extra_map,scid1);
-	struct chan_extra *x2 = get_chan_extra_value(chan_extra_map,scid2);
-	
-	assert(x1->value==13);
-	assert(x2->value==23);
-	
-	/* Valgrind complains. It seems that the hash table is trying to remove
-	 * the element at a moment when its memory has already been released. 
-	 * If the following two lines are not commented the error disapears. */
-	tal_free(x1);
-	tal_free(x2);
-	
-	tal_free(this_ctx);
-}
-
-static void destroy_int(int* x)
-{
-	fprintf(stderr,"Destroying %d\n",*x);
-}
-
-/* This clearly shows that the parent is destroyed before the children. 
- * That's the reason why the chan_extra destructor fails. */
-static void test_order(void)
-{
-	const tal_t *this_ctx = tal(tmpctx,tal_t);
-	
-	int *parent = tal(this_ctx,int);
-	tal_add_destructor(parent,destroy_int);
-	*parent = 1;
-	
-	int *child = tal(parent,int);
-	tal_add_destructor(child,destroy_int);
-	*child=2;
-	
-	tal_free(this_ctx);
-	
-	// prints:
-	// Destroying 1
-	// Destroying 2
-}
-
-int main(int argc, char *argv[])
-{
-	common_setup(argv[0]);
-	valgrind_ok1();
-	valgrind_ok2();
-	valgrind_fail3();
-	
-	test_order();
-	common_shutdown();
-}
-
diff --git a/plugins/renepay/test/run-mcf-diamond.c b/plugins/renepay/test/run-mcf-diamond.c
index ff2a37885453..7112c004aa91 100644
--- a/plugins/renepay/test/run-mcf-diamond.c
+++ b/plugins/renepay/test/run-mcf-diamond.c
@@ -1,6 +1,9 @@
 #include "config.h"
+
+#define FLOW_UNITTEST // logs are written in /tmp/debug.txt
 #include "../flow.c"
 #include "../mcf.c"
+
 #include <ccan/array_size/array_size.h>
 #include <ccan/read_write_all/read_write_all.h>
 #include <common/bigsize.h>
@@ -16,35 +19,40 @@ static u8 empty_map[] = {
 	0
 };
 
-static void print_flows(const char *desc,
-			const struct gossmap *gossmap,
-			struct chan_extra_map* chan_extra_map,
-			struct flow **flows)
+static const char* print_flows(
+		const tal_t *ctx,
+		const char *desc,
+		const struct gossmap *gossmap,
+		struct chan_extra_map* chan_extra_map,
+		struct flow **flows)
 {
+	tal_t *this_ctx = tal(ctx,tal_t);
 	double tot_prob = flow_set_probability(flows,gossmap,chan_extra_map);
-	printf("%s: %zu subflows, prob %.2lf\n", desc, tal_count(flows),tot_prob);
+	char *buff = tal_fmt(ctx,"%s: %zu subflows, prob %2lf\n", desc, tal_count(flows),tot_prob);
 	for (size_t i = 0; i < tal_count(flows); i++) {
 		struct amount_msat fee, delivered;
-		printf("   ");
+		tal_append_fmt(&buff,"   ");
 		for (size_t j = 0; j < tal_count(flows[i]->path); j++) {
 			struct short_channel_id scid
 				= gossmap_chan_scid(gossmap,
 						    flows[i]->path[j]);
-			printf("%s%s", j ? "->" : "",
-			       type_to_string(tmpctx, struct short_channel_id, &scid));
+			tal_append_fmt(&buff,"%s%s", j ? "->" : "",
+			       type_to_string(this_ctx, struct short_channel_id, &scid));
 		}
 		delivered = flows[i]->amounts[tal_count(flows[i]->amounts)-1];
 		if (!amount_msat_sub(&fee, flows[i]->amounts[0], delivered))
 		{
-			fprintf(stderr,"%s: flow[i]->amount[0]<delivered\n",
+			debug_err("%s: flow[i]->amount[0]<delivered\n",
 				__PRETTY_FUNCTION__);
-			abort();
 		}
-		printf(" prob %.2f, %s delivered with fee %s\n",
+		tal_append_fmt(&buff," prob %.2f, %s delivered with fee %s\n",
 		       flows[i]->success_prob,
-		       type_to_string(tmpctx, struct amount_msat, &delivered),
-		       type_to_string(tmpctx, struct amount_msat, &fee));
+		       type_to_string(this_ctx, struct amount_msat, &delivered),
+		       type_to_string(this_ctx, struct amount_msat, &fee));
 	}
+	
+	tal_free(this_ctx);
+	return buff;
 }
 
 int main(int argc, char *argv[])
@@ -139,7 +147,8 @@ int main(int argc, char *argv[])
 			 /* base fee penalty */ 0,
 			 /* prob cost factor = */ 1);
 	
-	print_flows("Simple minflow", gossmap,chan_extra_map, flows);
+	debug_info("%s\n",
+		print_flows(tmpctx,"Simple minflow", gossmap,chan_extra_map, flows));
 
 	common_shutdown();
 }
diff --git a/plugins/renepay/test/run-mcf.c b/plugins/renepay/test/run-mcf.c
index 031114659a32..92a0ac5adf61 100644
--- a/plugins/renepay/test/run-mcf.c
+++ b/plugins/renepay/test/run-mcf.c
@@ -1,6 +1,9 @@
 #include "config.h"
+
+#define FLOW_UNITTEST // logs are written in /tmp/debug.txt
 #include "../flow.c"
 #include "../mcf.c"
+
 #include <ccan/array_size/array_size.h>
 #include <ccan/read_write_all/read_write_all.h>
 #include <common/bigsize.h>
@@ -225,29 +228,34 @@ static u8 canned_map[] = {
 /* not_mcf sets NDEBUG, so assert() is useless */
 #define ASSERT(x) do { if (!(x)) abort(); } while(0)
 
-static void print_flows(const char *desc,
-			const struct gossmap *gossmap,
-			struct flow **flows)
+static const char *print_flows(
+		const tal_t *ctx,
+		const char *desc,
+		const struct gossmap *gossmap,
+		struct flow **flows)
 {
-	printf("%s: %zu subflows\n", desc, tal_count(flows));
+	tal_t *this_ctx = tal(ctx,tal_t);
+	char *buff = tal_fmt(ctx,"%s: %zu subflows\n", desc, tal_count(flows));
 	for (size_t i = 0; i < tal_count(flows); i++) {
 		struct amount_msat fee, delivered;
-		printf("   ");
+		tal_append_fmt(&buff,"   ");
 		for (size_t j = 0; j < tal_count(flows[i]->path); j++) {
 			struct short_channel_id scid
 				= gossmap_chan_scid(gossmap,
 						    flows[i]->path[j]);
-			printf("%s%s", j ? "->" : "",
-			       type_to_string(tmpctx, struct short_channel_id, &scid));
+			tal_append_fmt(&buff,"%s%s", j ? "->" : "",
+			       type_to_string(this_ctx, struct short_channel_id, &scid));
 		}
 		delivered = flows[i]->amounts[tal_count(flows[i]->amounts)-1];
 		if (!amount_msat_sub(&fee, flows[i]->amounts[0], delivered))
 			abort();
-		printf(" prob %.2f, %s delivered with fee %s\n",
+		tal_append_fmt(&buff," prob %.2f, %s delivered with fee %s\n",
 		       flows[i]->success_prob,
-		       type_to_string(tmpctx, struct amount_msat, &delivered),
-		       type_to_string(tmpctx, struct amount_msat, &fee));
+		       type_to_string(this_ctx, struct amount_msat, &delivered),
+		       type_to_string(this_ctx, struct amount_msat, &fee));
 	}
+	tal_free(this_ctx);
+	return buff;
 }
 
 int main(int argc, char *argv[])
@@ -267,6 +275,7 @@ int main(int argc, char *argv[])
 
 	gossmap = gossmap_load(tmpctx, gossfile, NULL);
 	assert(gossmap);
+	remove(gossfile);
 
 	/* There is a public channel 2<->3 (103x1x0), and private
 	 * 1<->2 (110x1x1). */
@@ -292,7 +301,8 @@ int main(int argc, char *argv[])
 			 /* base fee penalty */ 1,
 			 /* prob cost factor = */ 10);
 	commit_flow_set(gossmap,chan_extra_map,flows);
-	print_flows("Flow via single path l1->l2->l3", gossmap, flows);
+	debug_info("%s\n",
+		print_flows(tmpctx,"Flow via single path l1->l2->l3", gossmap, flows));
 	
 	
 	
@@ -436,7 +446,8 @@ int main(int argc, char *argv[])
 			 /* delay fee factor = */ 1,
 			 /* base fee penalty */ 1,
 			 /* prob cost factor = */ 10);
-	print_flows("Flow via two paths, high mu", gossmap, flows2);
+	debug_info("%s\n",
+		print_flows(tmpctx,"Flow via two paths, high mu", gossmap, flows2));
 	assert(tal_count(flows2) == 2);
 	assert(tal_count(flows2[0]->path) == 1);
 	assert(tal_count(flows2[1]->path) == 2);
@@ -453,6 +464,5 @@ int main(int argc, char *argv[])
 	assert(flows2[0]->amounts[0].millisatoshis + flows2[1]->amounts[0].millisatoshis
 	       > flows2[0]->amounts[0].millisatoshis - flows2[1]->amounts[0].millisatoshis);
 
-
 	common_shutdown();
 }
diff --git a/plugins/renepay/test/run-testflow.c b/plugins/renepay/test/run-testflow.c
index 99ac9cdca7bf..4c1a9a7ebf3b 100644
--- a/plugins/renepay/test/run-testflow.c
+++ b/plugins/renepay/test/run-testflow.c
@@ -9,9 +9,8 @@
 #include <common/node_id.h>
 #include <ccan/read_write_all/read_write_all.h>
 
-static bool print_enable = true;
-#define SUPERVERBOSE(...) do { if (print_enable) printf(__VA_ARGS__); } while(0)
-
+#define MYLOG "/tmp/debug.txt"
+#define FLOW_UNITTEST // logs are written in MYLOG
 #include <plugins/renepay/flow.h>
 #include "../flow.c"
 

From 2ed8fc88432295435ce3c26b0ec42ea69ebe6a80 Mon Sep 17 00:00:00 2001
From: Lagrang3 <eduardo.quintana@pm.me>
Date: Thu, 22 Jun 2023 18:28:44 +0100
Subject: [PATCH 56/64] renepay: call listsendpay before payment

We've just replicated the workflow of the standard pay command.
We first query listsendpays for previous attempts to pay for the same
hash.
---
 plugins/renepay/pay.c      | 161 +++++++++++++++++++++++++++++++++----
 plugins/renepay/pay.h      |   9 ++-
 plugins/renepay/pay_flow.c |   2 +-
 3 files changed, 153 insertions(+), 19 deletions(-)

diff --git a/plugins/renepay/pay.c b/plugins/renepay/pay.c
index a13a83adaf76..658c790b6d0b 100644
--- a/plugins/renepay/pay.c
+++ b/plugins/renepay/pay.c
@@ -73,7 +73,7 @@ static void payment_new_attempt(struct payment *p)
 }
 static u64 payment_groupid(struct payment *p)
 {
-	return p->active_payment->groupid;
+	return p->groupid;
 }
 static void payment_initialize(
 		struct payment *p,
@@ -126,7 +126,7 @@ static struct command_result *payment_success(struct payment *p)
 	json_add_amount_msat_only(response, "amount_sent_msat",
 				  p->total_sent);
 	json_add_string(response, "status", "complete");
-	json_add_node_id(response, "destination", &p->dest);
+	json_add_node_id(response, "destination", &p->destination);
 	
 	return command_finished(payment_command(p), response);
 }
@@ -323,7 +323,7 @@ static void uncertainty_network_add_routehints(struct payment *p)
 	for (size_t i = 0; i < tal_count(b11->routes); i++) {
 		/* Each one, presumably, leads to the destination */
 		const struct route_info *r = b11->routes[i];
-		const struct node_id *end = & p->dest;
+		const struct node_id *end = & p->destination;
 		for (int j = tal_count(r)-1; j >= 0; j--) {
 			// TODO(eduardo): amount to send to the add_hintchan
 			// should consider fees.
@@ -459,7 +459,7 @@ static bool update_uncertainty_network_from_listpeerchannels(
 	return false;
 }
 
-/* How much does this flow deliver to dest? */
+/* How much does this flow deliver to destination? */
 static struct amount_msat flow_delivered(const struct pay_flow *flow)
 {
 	return flow->amounts[tal_count(flow->amounts)-1];
@@ -1262,7 +1262,7 @@ static struct command_result *json_paystatus(struct command *cmd,
 			json_add_invstring(ret, p->invstr);
 		json_add_amount_msat_only(ret, "amount_msat", p->amount);
 
-		json_add_node_id(ret, "destination", &p->dest);
+		json_add_node_id(ret, "destination", &p->destination);
 		json_array_start(ret, "notes");
 		for (size_t i = 0; i < tal_count(p->paynotes); i++)
 			json_add_string(ret, NULL, p->paynotes[i]);
@@ -1280,6 +1280,136 @@ static struct command_result *json_paystatus(struct command *cmd,
 	return command_finished(cmd, ret);
 }
 
+/* Taken from ./plugins/pay.c
+ * 
+ * We are interested in any prior attempts to pay this payment_hash /
+ * invoice so we can set the `groupid` correctly and ensure we don't
+ * already have a pending payment running. We also collect the summary
+ * about an eventual previous complete payment so we can return that
+ * as a no-op. */
+static struct command_result *
+payment_listsendpays_previous(struct command *cmd, const char *buf,
+			      const jsmntok_t *result, struct payment *p)
+{
+	size_t i;
+	const jsmntok_t *t, *arr, *err;
+	/* What was the groupid of an eventual previous attempt? */
+	u64 last_group = 0;
+	/* Do we have pending sendpays for the previous attempt? */
+	bool pending = false;
+
+	/* Group ID of the first pending payment, this will be the one
+	 * who's result gets replayed if we end up suspending. */
+	u64 pending_group_id = 0;
+	/* Did a prior attempt succeed? */
+	bool completed = false;
+
+	/* Metadata for a complete payment, if one exists. */
+	struct json_stream *ret;
+	u32 parts = 0;
+	struct preimage preimage;
+	struct amount_msat sent, msat;
+	struct node_id destination;
+	u32 created_at;
+
+	err = json_get_member(buf, result, "error");
+	if (err)
+		return command_fail(
+			   cmd, LIGHTNINGD,
+			   "Error retrieving previous pay attempts: %s",
+			   json_strdup(tmpctx, buf, err));
+
+	arr = json_get_member(buf, result, "payments");
+	if (!arr || arr->type != JSMN_ARRAY)
+		return command_fail(
+		    cmd, LIGHTNINGD,
+		    "Unexpected non-array result from listsendpays");
+
+	/* We iterate through all prior sendpays, looking for the
+	 * latest group and remembering what its state is. */
+	json_for_each_arr(i, t, arr)
+	{
+		u64 groupid;
+		const jsmntok_t *status, *grouptok;
+		struct amount_msat diff_sent, diff_msat;
+		grouptok = json_get_member(buf, t, "groupid");
+		json_to_u64(buf, grouptok, &groupid);
+
+		/* New group, reset what we collected. */
+		if (last_group != groupid) {
+			completed = false;
+			pending = false;
+			last_group = groupid;
+
+			parts = 1;
+			json_scan(tmpctx, buf, t,
+				  "{destination:%"
+				  ",created_at:%"
+				  ",amount_msat:%"
+				  ",amount_sent_msat:%"
+				  ",payment_preimage:%}",
+				  JSON_SCAN(json_to_node_id, &destination),
+				  JSON_SCAN(json_to_u32, &created_at),
+				  JSON_SCAN(json_to_msat, &msat),
+				  JSON_SCAN(json_to_msat, &sent),
+				  JSON_SCAN(json_to_preimage, &preimage));
+		} else {
+			json_scan(tmpctx, buf, t,
+				  "{amount_msat:%"
+				  ",amount_sent_msat:%}",
+				  JSON_SCAN(json_to_msat, &diff_msat),
+				  JSON_SCAN(json_to_msat, &diff_sent));
+			if (!amount_msat_add(&msat, msat, diff_msat) ||
+			    !amount_msat_add(&sent, sent, diff_sent))
+				debug_err("msat overflow adding up parts");
+			parts++;
+		}
+
+		status = json_get_member(buf, t, "status");
+		completed |= json_tok_streq(buf, status, "complete");
+		pending |= json_tok_streq(buf, status, "pending");
+
+		/* Remember the group id of the first pending group so
+		 * we can replay its result later. */
+		if (!pending_group_id && pending)
+			pending_group_id = groupid;
+	}
+
+	if (completed) {
+		ret = jsonrpc_stream_success(cmd);
+		json_add_preimage(ret, "payment_preimage", &preimage);
+		json_add_string(ret, "status", "complete");
+		json_add_amount_msat_compat(ret, msat, "msatoshi",
+					    "amount_msat");
+		json_add_amount_msat_compat(ret, sent, "msatoshi_sent",
+					    "amount_sent_msat");
+		json_add_node_id(ret, "destination", &p->destination);
+		json_add_sha256(ret, "payment_hash", &p->payment_hash);
+		json_add_u32(ret, "created_at", created_at);
+		json_add_num(ret, "parts", parts);
+		return command_finished(cmd, ret);
+	} else if (pending) {
+		/* We suspend this call and wait for the
+		 * `on_payment_success` or `on_payment_failure`
+		 * handler of the currently running payment to notify
+		 * us about its completion. We latch on to the result
+		 * from the call we extracted above. */
+		p->groupid = pending_group_id;
+		
+		// TODO(eduardo): for this to work we need to subscribe to
+		// pay_success and pay_failure
+		return command_still_pending(cmd);
+	}
+	p->groupid = last_group + 1;
+	
+	struct out_req *req;
+	/* Get local capacities... */
+	req = jsonrpc_request_start(cmd->plugin, cmd, "listpeerchannels",
+				    listpeerchannels_done,
+				    listpeerchannels_done, p);
+	return send_outreq(cmd->plugin, req);
+}
+
 static struct command_result *json_pay(struct command *cmd,
 				       const char *buf,
 				       const jsmntok_t *params)
@@ -1404,7 +1534,7 @@ static struct command_result *json_pay(struct command *cmd,
 		invmsat = b11->msat;
 		invexpiry = b11->timestamp + b11->expiry;
 
-		p->dest = b11->receiver_id;
+		p->destination = b11->receiver_id;
 		p->payment_hash = b11->payment_hash;
 		p->payment_secret =
 			tal_dup_or_null(p, struct secret, b11->payment_secret);
@@ -1468,7 +1598,7 @@ static struct command_result *json_pay(struct command *cmd,
 		} else
 			invmsat = NULL;
 
-		node_id_from_pubkey(&p->dest, b12->offer_node_id);
+		node_id_from_pubkey(&p->destination, b12->offer_node_id);
 		p->payment_hash = *b12->invoice_payment_hash;
 		if (b12->invreq_recurrence_counter && !p->label)
 			return command_fail(
@@ -1503,7 +1633,7 @@ static struct command_result *json_pay(struct command *cmd,
 			invexpiry = *b12->invoice_created_at + BOLT12_DEFAULT_REL_EXPIRY;
 	}
 	
-	if (node_id_eq(&pay_plugin->my_id, &p->dest))
+	if (node_id_eq(&pay_plugin->my_id, &p->destination))
 		return command_fail(cmd, JSONRPC2_INVALID_PARAMS,
 				    "This payment is destined for ourselves. "
 				    "Self-payments are not supported");
@@ -1577,14 +1707,17 @@ static struct command_result *json_pay(struct command *cmd,
 			   "uncertainty network invariants are violated");
 	
 	/* We will try a single MPP payment. */
-	p->active_payment->groupid = pseudorand_u64();
+	// p->active_payment->groupid = pseudorand_u64();
 	p->active_payment->next_partid = 1;
 	
-	struct out_req *req;
-	/* Get local capacities... */
-	req = jsonrpc_request_start(cmd->plugin, cmd, "listpeerchannels",
-				    listpeerchannels_done,
-				    listpeerchannels_done, p);
+	/* Next, request listsendpays for previous payments that use the same
+	 * hash. */
+	struct out_req *req
+		= jsonrpc_request_start(cmd->plugin, cmd, "listsendpays",
+			payment_listsendpays_previous,
+			payment_listsendpays_previous, p);
+
+	json_add_sha256(req->js, "payment_hash", &p->payment_hash);
 	return send_outreq(cmd->plugin, req);
 }
 
diff --git a/plugins/renepay/pay.h b/plugins/renepay/pay.h
index d5f20173f05a..4292d3cc86d8 100644
--- a/plugins/renepay/pay.h
+++ b/plugins/renepay/pay.h
@@ -87,9 +87,6 @@ struct active_payment
 	/* Root to destroy pending flows */
 	tal_t *all_flows;
 	
-	/* Groupid, so listpays() can group them back together */
-	u64 groupid;
-	
 	/* Used in get_payflows to set ids to each pay_flow. */
 	u64 next_partid;
 };
@@ -109,7 +106,7 @@ struct payment {
 	
 	/* How much, what, where */
 	struct amount_msat amount;
-	struct node_id dest;
+	struct node_id destination;
 	struct sha256 payment_hash;
 	
 	
@@ -195,6 +192,10 @@ struct payment {
 	
 	/* Data used while the payment is being processed. */
 	struct active_payment *active_payment;
+	
+	/* Groupid, so listpays() can group them back together */
+	u64 groupid;
+	
 };
 
 int payment_current_attempt(const struct payment *p);
diff --git a/plugins/renepay/pay_flow.c b/plugins/renepay/pay_flow.c
index a54b6be4132b..047368528880 100644
--- a/plugins/renepay/pay_flow.c
+++ b/plugins/renepay/pay_flow.c
@@ -359,7 +359,7 @@ struct pay_flow **get_payflows(struct payment *p,
 		paynote(p, "We don't have any channels?");
 		goto fail;
 	}
-	dst = gossmap_find_node(pay_plugin->gossmap, &p->dest);
+	dst = gossmap_find_node(pay_plugin->gossmap, &p->destination);
 	if (!src) {
 		paynote(p, "No trace of destination in network gossip");
 		goto fail;

From 0067712f3fc437ecd8368268b4fad81cbde27b6c Mon Sep 17 00:00:00 2001
From: Lagrang3 <eduardo.quintana@pm.me>
Date: Thu, 22 Jun 2023 20:58:33 +0100
Subject: [PATCH 57/64] renepay: don't record twice for the same payment

---
 plugins/renepay/pay.c | 22 +++++++++++++---------
 1 file changed, 13 insertions(+), 9 deletions(-)

diff --git a/plugins/renepay/pay.c b/plugins/renepay/pay.c
index 658c790b6d0b..b7ae36c3bc63 100644
--- a/plugins/renepay/pay.c
+++ b/plugins/renepay/pay.c
@@ -1374,7 +1374,7 @@ payment_listsendpays_previous(struct command *cmd, const char *buf,
 		if (!pending_group_id && pending)
 			pending_group_id = groupid;
 	}
-
+	
 	if (completed) {
 		ret = jsonrpc_stream_success(cmd);
 		json_add_preimage(ret, "payment_preimage", &preimage);
@@ -1387,19 +1387,23 @@ payment_listsendpays_previous(struct command *cmd, const char *buf,
 		json_add_sha256(ret, "payment_hash", &p->payment_hash);
 		json_add_u32(ret, "created_at", created_at);
 		json_add_num(ret, "parts", parts);
+		
+		/* This payment was already completed, we don't keep record of
+		 * it twice. */
+		tal_free(p);
+		
 		return command_finished(cmd, ret);
 	} else if (pending) {
-		/* We suspend this call and wait for the
-		 * `on_payment_success` or `on_payment_failure`
-		 * handler of the currently running payment to notify
-		 * us about its completion. We latch on to the result
-		 * from the call we extracted above. */
 		p->groupid = pending_group_id;
+		/* Someone else is trying to get this payment through. I'll fail
+		 * this attempt. */
+		tal_free(p);
 		
-		// TODO(eduardo): for this to work we need to subscribe to
-		// pay_success and pay_failure
-		return command_still_pending(cmd);
+		// TODO(eduardo): is this an acceptable behaviour?
+		return command_fail(cmd, PAY_IN_PROGRESS,
+				    "Payment is pending by some other request.");
 	}
+	
 	p->groupid = last_group + 1;
 	
 	struct out_req *req;

From 55e8dabf5a001c37e49f241e9a1af4529c839e27 Mon Sep 17 00:00:00 2001
From: Lagrang3 <eduardo.quintana@pm.me>
Date: Fri, 23 Jun 2023 18:56:14 +0100
Subject: [PATCH 58/64] renepay: code review, remove old comments

---
 plugins/renepay/pay.c      |  2 +-
 plugins/renepay/pay.h      | 10 +++---
 plugins/renepay/pay_flow.c | 74 +++++++++++++++-----------------------
 3 files changed, 34 insertions(+), 52 deletions(-)

diff --git a/plugins/renepay/pay.c b/plugins/renepay/pay.c
index b7ae36c3bc63..3c21738f0b57 100644
--- a/plugins/renepay/pay.c
+++ b/plugins/renepay/pay.c
@@ -1454,7 +1454,7 @@ static struct command_result *json_pay(struct command *cmd,
 		   	&min_prob_success_millionths,100000), // default is 10%
 	
 		   p_opt_def("riskfactor", param_millionths,
-			     &riskfactor_millionths, 1.0),
+			     &riskfactor_millionths, 1),
 
 		   p_opt_def("maxdelay", param_number, &maxdelay,
 			     /* We're initially called to probe usage, before init! */
diff --git a/plugins/renepay/pay.h b/plugins/renepay/pay.h
index 4292d3cc86d8..d1cb358cfb70 100644
--- a/plugins/renepay/pay.h
+++ b/plugins/renepay/pay.h
@@ -6,6 +6,9 @@
 #include <plugins/libplugin.h>
 #include <plugins/renepay/flow.h>
 
+// TODO(eduardo): check if paynotes are meaningful
+// TODO(eduardo): remove assertions, introduce LOG_BROKEN messages
+
 #define MAX_NUM_ATTEMPTS 10
 
 // TODO(eduardo): Test ideas
@@ -113,7 +116,7 @@ struct payment {
 	/* Limits on what routes we'll accept. */
 	struct amount_msat maxspend;
 	
-	// TODO(eduardo): check out how this is used by get_payflows.
+	/* Max accepted HTLC delay.*/
 	unsigned int maxdelay;
 	
 	/* We promised this in pay() output */
@@ -137,7 +140,6 @@ struct payment {
 	const struct preimage *preimage;
 	
 	/* payment_secret, if specified by invoice. */
-	// TODO(eduardo): isn't the preimage the payment secret?
 	struct secret *payment_secret;
 	
 	/* Payment metadata, if specified by invoice. */
@@ -146,10 +148,6 @@ struct payment {
 	/* To know if the last attempt failed, succeeded or is it pending. */
 	enum payment_status status;	
 
-	// TODO(eduardo):
-	// 1. what is this?
-	// 2. what is it used for?
-	// 3. notice that we don't use it.
 	u32 final_cltv;
 
 	/* Inside pay_plugin->payments list */
diff --git a/plugins/renepay/pay_flow.c b/plugins/renepay/pay_flow.c
index 047368528880..bf965c8150aa 100644
--- a/plugins/renepay/pay_flow.c
+++ b/plugins/renepay/pay_flow.c
@@ -135,7 +135,6 @@ static u64 flow_delay(const struct flow *flow)
 }
 
 /* This enhances f->amounts, and returns per-flow cltvs */
-// TODO(eduardo): check out this again
 static u32 *shadow_additions(const tal_t *ctx,
 			     const struct gossmap *gossmap,
 			     struct payment *p,
@@ -330,16 +329,6 @@ static bool disable_htlc_violations(struct payment *p,
 	return disabled_some;
 }
 
-// /* We're not using these flows after all: clean up chan_extra_map */
-// static void remove_flows(const struct gossmap *gossmap,
-// 			 struct chan_extra_map *chan_extra_map,
-// 			 struct flow **flows)
-// {
-// 	for (size_t i = 0; i < tal_count(flows); i++)
-// 		remove_completed_flow(gossmap, chan_extra_map, flows[i]);
-// }
-
-// TODO(eduardo): check this
 /* Get some payment flows to get this amount to destination, or NULL. */
 struct pay_flow **get_payflows(struct payment *p,
 			       struct amount_msat amount,
@@ -373,8 +362,6 @@ struct pay_flow **get_payflows(struct payment *p,
 		bool too_unlikely, too_expensive, too_delayed;
 		const u32 *final_cltvs;
 		
-		/* Note!  This actually puts flows in chan_extra_map, so
-		 * flows must be removed if not used! */
 		flows = minflow(tmpctx, pay_plugin->gossmap, src, dst,
 				pay_plugin->chan_extra_map, disabled,
 				amount, 
@@ -400,7 +387,7 @@ struct pay_flow **get_payflows(struct payment *p,
 		if (too_unlikely && !unlikely_ok)
 		{
 			paynote(p, "Flows too unlikely, P() = %f%%", prob * 100);
-			goto fail_path;
+			goto fail;
 		}
 		too_expensive = amount_msat_greater(fee, feebudget);
 		if (too_expensive)
@@ -408,28 +395,26 @@ struct pay_flow **get_payflows(struct payment *p,
 			paynote(p, "Flows too expensive, fee = %s (max %s)",
 				type_to_string(tmpctx, struct amount_msat, &fee),
 				type_to_string(tmpctx, struct amount_msat, &feebudget));
-			goto fail_path;
+			goto fail;
 		}
 		too_delayed = (delay > p->maxdelay);
 		if (too_delayed) {
 			paynote(p, "Flows too delayed, delay = %"PRIu64" (max %u)",
 				delay, p->maxdelay);
-			// /* FIXME: What is a sane limit? */
-			// if (p->delay_feefactor > 1000) {
-			// 	paynote(p, "Giving up!");
-			// 	goto fail_path;
-			// }
-
-			// p->delay_feefactor *= 2;
-			// paynote(p, "Doubling delay_feefactor to %f",
-			// 	p->delay_feefactor);
-			// 
-			// goto retry;
 			
-			goto fail_path;
+			/* FIXME: What is a sane limit? */
+			if (p->delay_feefactor > 1000) {
+				paynote(p, "Giving up!");
+				goto fail;
+			}
+
+			p->delay_feefactor *= 2;
+			paynote(p, "Doubling delay_feefactor to %f",
+				p->delay_feefactor);
+			
+			continue; // retry
 		}
 
-	// seems_ok:
 		/* Now we check for min/max htlc violations, and
 		 * excessive htlc counts.  It would be more efficient
 		 * to do this inside minflow(), but the diagnostics here
@@ -437,39 +422,26 @@ struct pay_flow **get_payflows(struct payment *p,
 		 * *actually* made us reconsider. */
 		if (disable_htlc_violations(p, flows, pay_plugin->gossmap,
 					    disabled))
-			goto retry;
-		
-		// /* Commit the flows to the chan_extra_map, 
-		//  * update the htlc_total and num_htlcs. */
-		// commit_htlc_flow_set(pay_plugin->gossmap,
-		// 		pay_plugin->chan_extra_map,
-		// 		flows);
-		
+		{
+			continue; // retry
+		}
 		
 		/* This can adjust amounts and final cltv for each flow,
 		 * to make it look like it's going elsewhere */
 		final_cltvs = shadow_additions(tmpctx, pay_plugin->gossmap,
 					       p, flows, is_entire_payment);
-
 		/* OK, we are happy with these flows: convert to
 		 * pay_flows to outlive the current gossmap. */
 		pay_flows = flows_to_pay_flows(p, pay_plugin->gossmap,
 					       flows, final_cltvs,
 					       &p->active_payment->next_partid);
 		break;
-
-	retry:
-		continue;
-	fail_path:
-		goto fail;
 	}
 
-out:
 	return pay_flows;
 
 fail:
-	pay_flows = NULL;
-	goto out;
+	return NULL;
 }
 
 const char* fmt_payflows(const tal_t *ctx,
@@ -546,6 +518,12 @@ void remove_htlc_payflow(
 							       chan_extra_map,
 							       flow->path_scids[i],
 							       flow->path_dirs[i]);
+		if(!h)
+		{
+			plugin_err(pay_plugin->plugin,
+				   "%s could not resolve chan_extra_half",
+				   __PRETTY_FUNCTION__);
+		}
 		if (!amount_msat_sub(&h->htlc_total, h->htlc_total, flow->amounts[i]))
 		{
 			plugin_err(pay_plugin->plugin,
@@ -575,6 +553,12 @@ void commit_htlc_payflow(
 							       chan_extra_map,
 							       flow->path_scids[i],
 							       flow->path_dirs[i]);
+		if(!h)
+		{
+			plugin_err(pay_plugin->plugin,
+				   "%s could not resolve chan_extra_half",
+				   __PRETTY_FUNCTION__);
+		}
 		if (!amount_msat_add(&h->htlc_total, h->htlc_total, flow->amounts[i]))
 		{
 			plugin_err(pay_plugin->plugin,

From ddaf64f093859499086c815295dd4a6c0d161275 Mon Sep 17 00:00:00 2001
From: Lagrang3 <eduardo.quintana@pm.me>
Date: Sun, 25 Jun 2023 09:24:18 +0100
Subject: [PATCH 59/64] renepay: add msat precision payments

---
 plugins/renepay/mcf.c      |  110 +-
 plugins/renepay/pay.c      |    5 +-
 plugins/renepay/pay.h      |   16 +-
 plugins/renepay/pay_flow.c |    2 -
 tests/test_renepay.py      | 5386 +++++++++++++++++++++++++++++++++++-
 5 files changed, 5457 insertions(+), 62 deletions(-)

diff --git a/plugins/renepay/mcf.c b/plugins/renepay/mcf.c
index 6603a965d5d5..35fdd98f5c51 100644
--- a/plugins/renepay/mcf.c
+++ b/plugins/renepay/mcf.c
@@ -6,6 +6,7 @@
 #include <plugins/renepay/mcf.h>
 #include <plugins/renepay/flow.h>
 #include <plugins/renepay/dijkstra.h>
+#include <plugins/renepay/debug.h>
 #include <common/type_to_string.h>
 #include <stdint.h>
 #include <math.h>
@@ -432,7 +433,12 @@ static void linearize_channel(
 							c,
 							dir);
 	
-	assert(extra_half);
+	if(!extra_half)
+	{
+		debug_err("%s (line %d) unexpected, extra_half is NULL",
+			__PRETTY_FUNCTION__,
+			__LINE__);
+	}
 	
 	s64 a = extra_half->known_min.millisatoshis/1000,
 	    b = 1 + extra_half->known_max.millisatoshis/1000;
@@ -446,11 +452,6 @@ static void linearize_channel(
 		cost[i] = params->cost_fraction[i]
 		          *params->amount.millisatoshis
 		          *params->prob_cost_factor*1.0/(b-a);
-		
-		// printf("channel part: %ld, cost_fraction: %lf, cost: %ld\n",
-		// 	i,params->cost_fraction[i],cost[i]);
-		// printf("prob_cost_factor: %d, amount_msat: %ld\n",
-		// 	params->prob_cost_factor,params->amount.millisatoshis);
 	}
 }
 
@@ -495,7 +496,6 @@ static void combine_cost_function(
 		struct residual_network *residual_network,
 		s64 mu)
 {
-	// printf("Report on arcs cost\n");
 	for(u32 arc_idx=0;arc_idx<linear_network->max_num_arcs;++arc_idx)
 	{
 		arc_t arc = (arc_t){.idx=arc_idx};
@@ -511,17 +511,7 @@ static void combine_cost_function(
 		residual_network->cost[arc_idx]
 			= mu==0 ? pcost : 
 			          (mu==(MU_MAX-1) ? fcost : combined);
-		
-		// printf("arc_idx: %d, comb. cost: %ld, res. cap: %ld\n",arc_idx,
-		// 	residual_network->cost[arc_idx],
-		// 	residual_network->cap[arc_idx]);
-		// printf("cap: %ld, prob_cost: %ld, fee_cost: %ld, mu: %ld\n\n",
-		// 	linear_network->capacity[arc_idx],
-		// 	linear_network->arc_prob_cost[arc_idx],
-		// 	linear_network->arc_fee_cost[arc_idx],
-		// 	mu);
 	}
-	// printf("\n\n");
 }
 
 static void linear_network_add_adjacenct_arc(
@@ -983,15 +973,16 @@ static int optimize_mcf(
 			// see page 323 of Ahuja-Magnanti-Orlin
 			residual_network->potential[n] -= MIN(distance[target],distance[n]);
 			
-			// Notice:
-			// if node i is permanently labeled we have
-			// 	d_i<=d_t 
-			// which implies
-			// 	MIN(d_i,d_t) = d_i
-			// if node i is temporarily labeled we have
-			// 	d_i>=d_t
-			// which implies
-			// 	MIN(d_i,d_t) = d_t
+			/* Notice:
+			 * if node i is permanently labeled we have
+			 * 	d_i<=d_t 
+			 * which implies
+			 * 	MIN(d_i,d_t) = d_i
+			 * if node i is temporarily labeled we have
+			 * 	d_i>=d_t
+			 * which implies
+			 * 	MIN(d_i,d_t) = d_t 
+			 * */
 		}
 	}
 	tal_free(this_ctx);
@@ -1040,7 +1031,6 @@ static u32 find_positive_balance(
 				= gossmap_nth_chan(gossmap,
 				                   cur,i,&dir);
 			
-			// TODO(eduardo): do we check if the channel is public?
 			if (!gossmap_chan_set(c,dir))
 				continue;
 			
@@ -1065,8 +1055,6 @@ static u32 find_positive_balance(
 		
 		assert(updated_idx!=INVALID_INDEX);
 		assert(updated_idx!=final_idx);
-		// printf("%s: balance[%d] = %ld\n",__PRETTY_FUNCTION__,
-		//	updated_idx,balance[updated_idx]);
 		
 		final_idx = updated_idx;
 	}
@@ -1091,9 +1079,14 @@ static struct flow **
 		// track of htlcs and in_flight sats.
 		struct chan_extra_map *chan_extra_map,
 		const struct linear_network *linear_network,
-		const struct residual_network *residual_network)
+		const struct residual_network *residual_network,
+		
+		// how many msats in excess we paid for not having msat accuracy
+		// in the MCF solver
+		struct amount_msat excess)
 {
-	// printf("%s: starting\n",__PRETTY_FUNCTION__);
+	assert(excess.millisatoshis < 1000);
+	
 	tal_t *this_ctx = tal(tmpctx,tal_t);
 	
 	const size_t max_num_chans = gossmap_max_chan_idx(gossmap);
@@ -1172,7 +1165,9 @@ static struct flow **
 				
 				// TODO(eduardo) does htlc_max has any relevance
 				// here?
-				delta = MIN(delta,chan_flow[c_idx].half[dir]);
+				// HINT: delta=MIN(delta,htlc_max);
+				// however this might not work because often we
+				// move delta+fees
 			}
 			
 			
@@ -1203,9 +1198,22 @@ static struct flow **
 				chan_flow[c_idx].half[prev_dir[cur_idx]]-=delta;
 			}
 			
+			assert(delta>0);
+			
+			// substract the excess of msats for not having msat
+			// accuracy
+			struct amount_msat delivered = amount_msat(delta*1000);
+			if(!amount_msat_sub(&delivered,delivered,excess))
+			{
+				debug_err("%s (line %d) unable to substract excess.",
+					__PRETTY_FUNCTION__,
+					__LINE__);
+			}
+			excess = amount_msat(0);
+			
 			// complete the flow path by adding real fees and
 			// probabilities.
-			flow_complete(fp,gossmap,chan_extra_map,amount_msat(delta*1000));
+			flow_complete(fp,gossmap,chan_extra_map,delivered);
 			
 			// add fp to list
 			ld = tal(list_ctx,struct list_data);
@@ -1224,7 +1232,6 @@ static struct flow **
 	}
 	
 	tal_free(this_ctx);
-	// printf("%s: done\n",__PRETTY_FUNCTION__);
 	return flows;
 }
 
@@ -1391,27 +1398,43 @@ struct flow** minflow(
 	
 	init_residual_network(linear_network,residual_network);
 	
-	// printf("%s: done with allocation and initialization\n",__PRETTY_FUNCTION__);
-	
 	struct amount_msat best_fee;
 	double best_prob_success;
 	struct flow **best_flow_paths = NULL;
 	
-	// printf("%s: searching for a feasible flow\n",__PRETTY_FUNCTION__);
+	/* TODO(eduardo):
+	 * Some MCF algorithms' performance depend on the size of maxflow. If we
+	 * were to work in units of msats we 1. risking overflow when computing
+	 * costs and 2. we risk a performance overhead for no good reason. 
+	 * 
+	 * Working in units of sats was my first choice, but maybe working in
+	 * units of 10, or 100 sats could be even better. 
+	 * 
+	 * IDEA: define the size of our precision as some parameter got at
+	 * runtime that depends on the size of the payment and adjust the MCF
+	 * accordingly. 
+	 * For example if we are trying to pay 1M sats our precision could be
+	 * set to 1000sat, then channels that had capacity for 3M sats become 3k
+	 * flow units. */
+	const u64 pay_amount_msats = params->amount.millisatoshis % 1000;
+	const u64 pay_amount_sats = params->amount.millisatoshis/1000 
+			+ (pay_amount_msats ? 1 : 0);
+	const struct amount_msat excess 
+		= amount_msat(pay_amount_msats ? 1000 - pay_amount_msats : 0);
+	
 	int err = find_feasible_flow(linear_network,residual_network,source_idx,target_idx,
-	                             params->amount.millisatoshis/1000);
+	                             pay_amount_sats);
 	
 	if(err!=RENEPAY_ERR_OK)
 	{
 		// there is no flow that satisfy the constraints, we stop here
-		// printf("%s: feasible flow not found\n",__PRETTY_FUNCTION__);
 		goto finish;
 	}
-	// printf("%s: found a feasible flow\n",__PRETTY_FUNCTION__);
 	
 	// first flow found
 	best_flow_paths = get_flow_paths(ctx,params->gossmap,params->chan_extra_map,
-	                            linear_network,residual_network);
+	                            linear_network,residual_network,
+				    excess);
 	best_prob_success = flow_set_probability(best_flow_paths,
 						params->gossmap,
 						params->chan_extra_map);
@@ -1430,11 +1453,12 @@ struct flow** minflow(
 		combine_cost_function(linear_network,residual_network,mu);
 		
 		optimize_mcf(linear_network,residual_network,
-				source_idx,target_idx,params->amount.millisatoshis/1000);
+				source_idx,target_idx,pay_amount_sats);
 		
 		struct flow **flow_paths;
 		flow_paths = get_flow_paths(this_ctx,params->gossmap,params->chan_extra_map,
-		                            linear_network,residual_network);
+		                            linear_network,residual_network,
+					    excess);
 		
 		double prob_success = flow_set_probability(
 						flow_paths,
diff --git a/plugins/renepay/pay.c b/plugins/renepay/pay.c
index 3c21738f0b57..10048aeb5c06 100644
--- a/plugins/renepay/pay.c
+++ b/plugins/renepay/pay.c
@@ -1156,13 +1156,10 @@ static struct command_result *try_paying(struct command *cmd,
 			   type_to_string(tmpctx, struct amount_msat, &p->total_delivering));
 	}
 
-	plugin_log(pay_plugin->plugin,LOG_DBG,fmt_chan_extra_map(tmpctx,pay_plugin->chan_extra_map));
+	// plugin_log(pay_plugin->plugin,LOG_DBG,fmt_chan_extra_map(tmpctx,pay_plugin->chan_extra_map));
 	
 	/* We let this return an unlikely path, as it's better to try once
 	 * than simply refuse.  Plus, models are not truth! */
-	// TODO(eduardo): some flows might be pending, even the flows that
-	// failed have commited some HTLCs that (?) reduce the throughput of the
-	// channels. Please do take that into account in the MCF.
 	struct pay_flow **pay_flows = get_payflows(p, remaining, feebudget, first_time,
 				 amount_msat_eq(p->total_delivering, AMOUNT_MSAT(0)));
 	
diff --git a/plugins/renepay/pay.h b/plugins/renepay/pay.h
index d1cb358cfb70..657ad4e3740b 100644
--- a/plugins/renepay/pay.h
+++ b/plugins/renepay/pay.h
@@ -6,6 +6,9 @@
 #include <plugins/libplugin.h>
 #include <plugins/renepay/flow.h>
 
+// TODO(eduardo): add an option entry for maxfeepercent
+// TODO(eduardo): I think this plugin can't handle a msat accuracy payment.
+// TODO(eduardo): write a man entry for renepay
 // TODO(eduardo): check if paynotes are meaningful
 // TODO(eduardo): remove assertions, introduce LOG_BROKEN messages
 
@@ -122,18 +125,7 @@ struct payment {
 	/* We promised this in pay() output */
 	struct timeabs start_time;
 	
-	// TODO(eduardo): notice that stop_time = start_time + 60sec by default.
-	// On the other hand I am assuming that HTLCs are removed when a payment
-	// request fails or succeeds. A fail could happen when some other part
-	// of a MPP payment fails and the current part times-out. My question is
-	// what determines this time-out of the MPP part, is it encoded in the
-	// HTLCs themselves, thus measured in block numbers or is there another
-	// timer somewhere?
-	// If this time is in seconds, it would be desirable to be less than
-	// 30sec so that we can try at least two different MPP before the
-	// payment expires. On the other hand if HTLCs expire only after a
-	// certain block number, then we should keep track of these events in the
-	// uncertainty network somehow even after the payment terminates.
+	/* We stop trying after this time is reached. */
 	struct timeabs stop_time;
 	
 	/* Payment preimage, in case of success. */
diff --git a/plugins/renepay/pay_flow.c b/plugins/renepay/pay_flow.c
index bf965c8150aa..540017189c53 100644
--- a/plugins/renepay/pay_flow.c
+++ b/plugins/renepay/pay_flow.c
@@ -336,8 +336,6 @@ struct pay_flow **get_payflows(struct payment *p,
 			       bool unlikely_ok,
 			       bool is_entire_payment)
 {
-	// double frugality = 1.0;
-	// bool was_too_expensive = false;
 	bitmap *disabled;
 	struct pay_flow **pay_flows;
 	const struct gossmap_node *src, *dst;
diff --git a/tests/test_renepay.py b/tests/test_renepay.py
index a8a40b6da2e7..fc45ed3d2632 100644
--- a/tests/test_renepay.py
+++ b/tests/test_renepay.py
@@ -20,7 +20,7 @@
 import time
 import unittest
 
-def test_pay_to_peer(node_factory):
+def test_simple(node_factory):
     '''Testing simply paying a peer.'''
     l1, l2 = node_factory.line_graph(2)
     inv = l2.rpc.invoice(123000, 'test_renepay', 'description')['bolt11']
@@ -53,3 +53,5387 @@ def test_mpp(node_factory):
     assert details['status'] == 'complete'
     assert details['amount_msat'] == send_amount
     assert details['destination'] == l6.info['id']
+
+### SAME TESTS USED IN THE STANDARD PAY COMMAND ###
+
+@pytest.mark.developer("needs to deactivate shadow routing")
+@pytest.mark.openchannel('v1')
+@pytest.mark.openchannel('v2')
+def test_pay(node_factory):
+    l1, l2 = node_factory.line_graph(2)
+
+    inv = l2.rpc.invoice(123000, 'test_pay', 'description')['bolt11']
+    before = int(time.time())
+    details = l1.rpc.call('renepay',{'invstring':inv,'use_shadow':False})
+    after = time.time()
+    preimage = details['payment_preimage']
+    assert details['status'] == 'complete'
+    assert details['amount_msat'] == Millisatoshi(123000)
+    assert details['destination'] == l2.info['id']
+    assert details['created_at'] >= before
+    assert details['created_at'] <= after
+
+    invoices = l2.rpc.listinvoices('test_pay')['invoices']
+    assert len(invoices) == 1
+    invoice = invoices[0]
+    assert invoice['status'] == 'paid' and invoice['paid_at'] >= before and invoice['paid_at'] <= after
+
+    # Repeat payments are NOPs (if valid): we can hand null.
+    l1.rpc.call('renepay',{'invstring':inv,'use_shadow':False})
+    # This won't work: can't provide an amount (even if correct!)
+    with pytest.raises(RpcError):
+        l1.rpc.call('renepay',{'invstring':inv,'amount_msat':123000000})
+    with pytest.raises(RpcError):
+        l1.rpc.call('renepay',{'invstring':inv,'amount_msat':122000000})
+
+    # Check pay_index is not null
+    outputs = l2.db_query('SELECT pay_index IS NOT NULL AS q FROM invoices WHERE label="label";')
+    assert len(outputs) == 1 and outputs[0]['q'] != 0
+
+    # Check payment of any-amount invoice.
+    for i in range(5):
+        label = "any{}".format(i)
+        inv2 = l2.rpc.invoice("any", label, 'description')['bolt11']
+        # Must provide an amount!
+        with pytest.raises(RpcError):
+            l1.rpc.call('renepay',{'invstring':inv2,'use_shadow':False})
+        
+        # TODO(eduardo): renepay can't handle a msat precision, fix this and add
+        # a test to check it
+        l1.rpc.call('renepay',{'invstring':inv2,'use_shadow':False,'amount_msat':random.randint(1000,999999)})
+
+    # Should see 6 completed payments
+    assert len(l1.rpc.listsendpays()['payments']) == 6
+
+    # Test listsendpays indexed by bolt11.
+    payments = l1.rpc.listsendpays(inv)['payments']
+    assert len(payments) == 1 and payments[0]['payment_preimage'] == preimage
+
+    # Check channels apy summary view of channel activity
+    apys_1 = l1.rpc.bkpr_channelsapy()['channels_apy']
+    apys_2 = l2.rpc.bkpr_channelsapy()['channels_apy']
+
+    assert apys_1[0]['channel_start_balance_msat'] == apys_2[0]['channel_start_balance_msat']
+    assert apys_1[0]['channel_start_balance_msat'] == apys_1[0]['our_start_balance_msat']
+    assert apys_2[0]['our_start_balance_msat'] == Millisatoshi(0)
+    assert apys_1[0]['routed_out_msat'] == apys_2[0]['routed_in_msat']
+    assert apys_1[0]['routed_in_msat'] == apys_2[0]['routed_out_msat']
+
+
+@pytest.mark.developer("needs to deactivate shadow routing")
+def test_amounts(node_factory):
+    l1, l2 = node_factory.line_graph(2)
+    inv = l2.rpc.invoice(Millisatoshi("123sat"), 'test_pay_amounts', 'description')['bolt11']
+
+    invoice = only_one(l2.rpc.listinvoices('test_pay_amounts')['invoices'])
+
+    assert isinstance(invoice['amount_msat'], Millisatoshi)
+    assert invoice['amount_msat'] == Millisatoshi(123000)
+
+    l1.rpc.call('renepay',{'invstring':inv, 'use_shadow':False})
+
+    invoice = only_one(l2.rpc.listinvoices('test_pay_amounts')['invoices'])
+    assert isinstance(invoice['amount_received_msat'], Millisatoshi)
+    assert invoice['amount_received_msat'] >= Millisatoshi(123000)
+
+
+@pytest.mark.developer("needs to deactivate shadow routing")
+def test_limits(node_factory):
+    """Test that we enforce fee max percentage and max delay"""
+    l1, l2, l3 = node_factory.line_graph(3, wait_for_announce=True)
+
+    # FIXME: pylightning should define these!
+    # PAY_STOPPED_RETRYING = 210
+    PAY_ROUTE_NOT_FOUND=205
+
+    inv = l3.rpc.invoice("any", "any", 'description')
+
+    # Fee too high.
+    err = r'Failed to find a route for 100000msat with budget 1msat'
+    with pytest.raises(RpcError, match=err) as err:
+        l1.rpc.call('renepay', {'invstring': inv['bolt11'], 'amount_msat': 100000, 'maxfee': 1})
+
+    # TODO(eduardo): which error code shall we use here?
+    assert err.value.error['code'] == PAY_ROUTE_NOT_FOUND
+    # assert err.value.error['code'] == PAY_STOPPED_RETRYING
+
+    # TODO(eduardo): shall we list attempts in renepay?
+    # It should have retried two more times (one without routehint and one with routehint)
+    # status = l1.rpc.call('renepaystatus', {'invstring':inv['bolt11']})['paystatus'][0]['attempts']
+
+    # We have an internal test to see if we can reach the destination directly
+    # without a routehint, that will enable a NULL-routehint. We will then try
+    # with the provided routehint, and the NULL routehint, resulting in 2
+    # attempts.
+    # assert(len(status) == 1)
+    # assert(status[0]['failure']['code'] == 205)
+
+#     failmsg = r'CLTV delay exceeds our CLTV budget'
+#     # Delay too high.
+#     with pytest.raises(RpcError, match=failmsg) as err:
+#         l1.rpc.call('pay', {'bolt11': inv['bolt11'], 'amount_msat': 100000, 'maxdelay': 0})
+# 
+#     assert err.value.error['code'] == PAY_STOPPED_RETRYING
+#     # Should also have retried two more times.
+#     status = l1.rpc.call('paystatus', {'bolt11': inv['bolt11']})['pay'][1]['attempts']
+# 
+#     assert(len(status) == 2)
+#     assert(status[0]['failure']['code'] == 205)
+# 
+#     # This fails!
+#     err = r'Fee exceeds our fee budget: 2msat > 1msat, discarding route'
+#     with pytest.raises(RpcError, match=err) as err:
+#         l1.rpc.pay(bolt11=inv['bolt11'], amount_msat=100000, maxfee=1)
+# 
+#     # This works, because fee is less than exemptfee.
+#     l1.dev_pay(inv['bolt11'], amount_msat=100000, maxfeepercent=0.0001,
+#                exemptfee=2000, use_shadow=False)
+#     status = l1.rpc.call('paystatus', {'bolt11': inv['bolt11']})['pay'][3]['attempts']
+#     assert len(status) == 1
+#     assert status[0]['strategy'] == "Initial attempt"
+# 
+# 
+# @pytest.mark.developer("Gossip is too slow without developer")
+# def test_pay_exclude_node(node_factory, bitcoind):
+#     """Test excluding the node if there's the NODE-level error in the failure_code
+#     """
+#     # FIXME: Remove our reliance on HTLCs failing on startup and the need for
+#     #        this plugin
+#     opts = [
+#         {'disable-mpp': None},
+#         {'plugin': os.path.join(os.getcwd(), 'tests/plugins/fail_htlcs.py')},
+#         {},
+#         {'fee-base': 100, 'fee-per-satoshi': 1000},
+#         {}
+#     ]
+#     l1, l2, l3, l4, l5 = node_factory.get_nodes(5, opts=opts)
+#     node_factory.join_nodes([l1, l2, l3], wait_for_announce=True)
+#     amount = 10**8
+# 
+#     inv = l3.rpc.invoice(amount, "test1", 'description')['bolt11']
+#     with pytest.raises(RpcError):
+#         l1.rpc.pay(inv)
+# 
+#     # It should have retried (once without routehint, too)
+#     status = l1.rpc.call('paystatus', {'bolt11': inv})['pay'][0]['attempts']
+# 
+#     # Excludes channel, then ignores routehint which includes that, then
+#     # it excludes other channel.
+#     assert len(status) == 2
+#     assert status[0]['strategy'] == "Initial attempt"
+#     assert status[0]['failure']['data']['failcodename'] == 'WIRE_TEMPORARY_NODE_FAILURE'
+#     assert 'failure' in status[1]
+# 
+#     # Get a fresh invoice, but do it before other routes exist, so routehint
+#     # will be via l2.
+#     inv = l3.rpc.invoice(amount, "test2", 'description')['bolt11']
+#     assert only_one(l1.rpc.decodepay(inv)['routes'])[0]['pubkey'] == l2.info['id']
+# 
+#     # l1->l4->l5->l3 is the longer route. This makes sure this route won't be
+#     # tried for the first pay attempt. Just to be sure we also raise the fees
+#     # that l4 leverages.
+#     l1.rpc.connect(l4.info['id'], 'localhost', l4.port)
+#     l4.rpc.connect(l5.info['id'], 'localhost', l5.port)
+#     l5.rpc.connect(l3.info['id'], 'localhost', l3.port)
+#     scid14, _ = l1.fundchannel(l4, 10**6, wait_for_active=False)
+#     scid45, _ = l4.fundchannel(l5, 10**6, wait_for_active=False)
+#     scid53, _ = l5.fundchannel(l3, 10**6, wait_for_active=False)
+#     mine_funding_to_announce(bitcoind, [l1, l2, l3, l4, l5])
+# 
+#     l1.daemon.wait_for_logs([r'update for channel {}/0 now ACTIVE'
+#                              .format(scid14),
+#                              r'update for channel {}/1 now ACTIVE'
+#                              .format(scid14),
+#                              r'update for channel {}/0 now ACTIVE'
+#                              .format(scid45),
+#                              r'update for channel {}/1 now ACTIVE'
+#                              .format(scid45),
+#                              r'update for channel {}/0 now ACTIVE'
+#                              .format(scid53),
+#                              r'update for channel {}/1 now ACTIVE'
+#                              .format(scid53)])
+# 
+#     # This `pay` will work
+#     l1.rpc.pay(inv)
+# 
+#     # It should have retried (once without routehint, too)
+#     status = l1.rpc.call('paystatus', {'bolt11': inv})['pay'][0]['attempts']
+# 
+#     # Excludes channel, then ignores routehint which includes that, then
+#     # it excludes other channel.
+#     assert len(status) == 2
+#     assert status[0]['strategy'] == "Initial attempt"
+#     assert status[0]['failure']['data']['failcodename'] == 'WIRE_TEMPORARY_NODE_FAILURE'
+#     assert 'success' in status[1]
+# 
+# 
+# def test_pay0(node_factory):
+#     """Test paying 0 amount
+#     """
+#     l1, l2 = node_factory.line_graph(2)
+#     chanid = l1.get_channel_scid(l2)
+# 
+#     # Get any-amount invoice
+#     inv = l2.rpc.invoice("any", "any", 'description')
+#     rhash = inv['payment_hash']
+# 
+#     routestep = {
+#         'amount_msat': 0,
+#         'id': l2.info['id'],
+#         'delay': 10,
+#         'channel': chanid
+#     }
+# 
+#     # Amount must be nonzero!
+#     l1.rpc.sendpay([routestep], rhash, payment_secret=inv['payment_secret'])
+#     with pytest.raises(RpcError, match=r'WIRE_AMOUNT_BELOW_MINIMUM'):
+#         l1.rpc.waitsendpay(rhash)
+# 
+# 
+# @pytest.mark.developer("needs DEVELOPER=1")
+# def test_pay_disconnect(node_factory, bitcoind):
+#     """If the remote node has disconnected, we fail payment, but can try again when it reconnects"""
+#     l1, l2 = node_factory.line_graph(2, opts={'dev-max-fee-multiplier': 5,
+#                                               'may_reconnect': True,
+#                                               'allow_warning': True})
+# 
+#     # Dummy payment to kick off update_fee messages
+#     l1.pay(l2, 1000)
+# 
+#     inv = l2.rpc.invoice(123000, 'test_pay_disconnect', 'description')
+#     rhash = inv['payment_hash']
+# 
+#     wait_for(lambda: [c['active'] for c in l1.rpc.listchannels()['channels']] == [True, True])
+# 
+#     # Can't use `pay` since that'd notice that we can't route, due to disabling channel_update
+#     route = l1.rpc.getroute(l2.info['id'], 123000, 1)["route"]
+# 
+#     l2.stop()
+#     # Make sure channeld has exited!
+#     wait_for(lambda: 'owner' not in only_one(l1.rpc.listpeerchannels(l2.info['id'])['channels']))
+# 
+#     # Can't pay while its offline.
+#     with pytest.raises(RpcError, match=r'failed: WIRE_TEMPORARY_CHANNEL_FAILURE \(First peer not ready\)'):
+#         l1.rpc.sendpay(route, rhash, payment_secret=inv['payment_secret'])
+# 
+#     l2.start()
+#     l1.daemon.wait_for_log('peer_out WIRE_CHANNEL_REESTABLISH')
+# 
+#     # Make l2 upset by asking for crazy fee.
+#     l1.set_feerates((10**6, 10**6, 10**6, 10**6), False)
+# 
+#     # Wait for l1 notice
+#     l1.daemon.wait_for_log(r'Peer transient failure in CHANNELD_NORMAL: channeld WARNING: .*: update_fee \d+ outside range 1875-75000')
+# 
+#     # Make l2 fail hard.
+#     l2.rpc.close(l1.info['id'], unilateraltimeout=1)
+#     l2.daemon.wait_for_log('sendrawtx exit')
+#     bitcoind.generate_block(1, wait_for_mempool=1)
+#     sync_blockheight(bitcoind, [l1, l2])
+# 
+#     # Should fail due to permenant channel fail
+#     with pytest.raises(RpcError, match=r'WIRE_UNKNOWN_NEXT_PEER'):
+#         l1.rpc.sendpay(route, rhash, payment_secret=inv['payment_secret'])
+# 
+#     assert not l1.daemon.is_in_log('Payment is still in progress')
+# 
+#     # After it sees block, someone should close channel.
+#     l1.daemon.wait_for_log('ONCHAIN')
+# 
+# 
+# @pytest.mark.developer("needs DEVELOPER=1 for dev_suppress_gossip")
+# def test_pay_get_error_with_update(node_factory):
+#     """We should process an update inside a temporary_channel_failure"""
+#     l1, l2, l3 = node_factory.line_graph(3, opts={'log-level': 'io'}, fundchannel=True, wait_for_announce=True)
+#     chanid2 = l2.get_channel_scid(l3)
+# 
+#     inv = l3.rpc.invoice(123000, 'test_pay_get_error_with_update', 'description')
+# 
+#     # Make sure l2 doesn't tell l1 directly that channel is disabled.
+#     l2.rpc.dev_suppress_gossip()
+#     l3.stop()
+# 
+#     # Make sure that l2 has seen disconnect, considers channel disabled.
+#     wait_for(lambda: [c['active'] for c in l2.rpc.listchannels(chanid2)['channels']] == [False, False])
+# 
+#     assert(l1.is_channel_active(chanid2))
+#     with pytest.raises(RpcError, match=r'WIRE_TEMPORARY_CHANNEL_FAILURE'):
+#         l1.rpc.pay(inv['bolt11'])
+# 
+#     # Make sure we get an onionreply, without the type prefix of the nested
+#     # channel_update, and it should patch it to include a type prefix. The
+#     # prefix 0x0102 should be in the channel_update, but not in the
+#     # onionreply (negation of 0x0102 in the RE)
+#     l1.daemon.wait_for_log(r'Extracted channel_update 0102.*from onionreply 1007008a[0-9a-fA-F]{276}$')
+# 
+#     # And now monitor for l1 to apply the channel_update we just extracted
+#     wait_for(lambda: not l1.is_channel_active(chanid2))
+# 
+# 
+# @pytest.mark.developer("needs DEVELOPER=1 for dev_suppress_gossip, dev-routes")
+# def test_pay_error_update_fees(node_factory):
+#     """We should process an update inside a temporary_channel_failure"""
+#     l1, l2, l3 = node_factory.line_graph(3, fundchannel=True, wait_for_announce=True)
+# 
+#     # Don't include any routehints in first invoice.
+#     inv1 = l3.dev_invoice(amount_msat=123000,
+#                           label='test_pay_error_update_fees',
+#                           description='description',
+#                           dev_routes=[])
+# 
+#     inv2 = l3.rpc.invoice(123000, 'test_pay_error_update_fees2', 'desc')  # noqa: F841
+# 
+#     # Make sure l2 doesn't tell l1 directly that channel fee is changed.
+#     l2.rpc.dev_suppress_gossip()
+#     l2.rpc.setchannel(l3.info['id'], 1337, 137, enforcedelay=0)
+# 
+#     # Should bounce off and retry...
+#     l1.rpc.pay(inv1['bolt11'])
+#     attempts = only_one(l1.rpc.paystatus(inv1['bolt11'])['pay'])['attempts']
+#     assert len(attempts) == 2
+#     # WIRE_FEE_INSUFFICIENT = UPDATE|12
+#     assert attempts[0]['failure']['data']['failcode'] == 4108
+# 
+#     # FIXME: We *DO NOT* handle misleading routehints!
+#     # # Should ignore old routehint and do the same...
+#     # l1.rpc.pay(inv2['bolt11'])
+#     # attempts = only_one(l1.rpc.paystatus(inv2['bolt11'])['pay'])['attempts']
+#     # assert len(attempts) == 2
+#     # # WIRE_FEE_INSUFFICIENT = UPDATE|12
+#     # assert attempts[0]['failure']['data']['failcode'] == 4108
+# 
+# 
+# @pytest.mark.developer("needs to deactivate shadow routing")
+# def test_pay_optional_args(node_factory):
+#     l1, l2 = node_factory.line_graph(2)
+# 
+#     inv1 = l2.rpc.invoice(123000, 'test_pay', 'desc')['bolt11']
+#     l1.dev_pay(inv1, label='desc', use_shadow=False)
+#     payment1 = l1.rpc.listsendpays(inv1)['payments']
+#     assert len(payment1) and payment1[0]['amount_sent_msat'] == 123000
+#     assert payment1[0]['label'] == 'desc'
+# 
+#     inv2 = l2.rpc.invoice(321000, 'test_pay2', 'description')['bolt11']
+#     l1.dev_pay(inv2, riskfactor=5.0, use_shadow=False)
+#     payment2 = l1.rpc.listsendpays(inv2)['payments']
+#     assert(len(payment2) == 1)
+#     # The pay plugin uses `sendonion` since 0.9.0 and `lightningd` doesn't
+#     # learn about the amount we intended to send (that's why we annotate the
+#     # root of a payment tree with the bolt11 invoice).
+# 
+#     anyinv = l2.rpc.invoice('any', 'any_pay', 'desc')['bolt11']
+#     l1.dev_pay(anyinv, label='desc', amount_msat=500, use_shadow=False)
+#     payment3 = l1.rpc.listsendpays(anyinv)['payments']
+#     assert len(payment3) == 1
+#     assert payment3[0]['label'] == 'desc'
+# 
+#     # Should see 3 completed transactions
+#     assert len(l1.rpc.listsendpays()['payments']) == 3
+# 
+# 
+# @pytest.mark.developer("needs to deactivate shadow routing")
+# @pytest.mark.openchannel('v1')
+# @pytest.mark.openchannel('v2')
+# def test_payment_success_persistence(node_factory, bitcoind, executor):
+#     # Start two nodes and open a channel.. die during payment.
+#     # Feerates identical so we don't get gratuitous commit to update them
+#     disconnect = ['+WIRE_COMMITMENT_SIGNED']
+#     if EXPERIMENTAL_DUAL_FUND:
+#         # We have to add an extra 'wire-commitment-signed' because
+#         # dual funding uses this for channel establishment also
+#         disconnect = ['=WIRE_COMMITMENT_SIGNED'] + disconnect
+# 
+#     l1 = node_factory.get_node(disconnect=disconnect,
+#                                options={'dev-no-reconnect': None},
+#                                may_reconnect=True,
+#                                feerates=(7500, 7500, 7500, 7500))
+#     l2 = node_factory.get_node(may_reconnect=True)
+#     l1.rpc.connect(l2.info['id'], 'localhost', l2.port)
+# 
+#     chanid, _ = l1.fundchannel(l2, 100000)
+# 
+#     inv1 = l2.rpc.invoice(1000, 'inv1', 'inv1')
+# 
+#     # Fire off a pay request, it'll get interrupted by a restart
+#     executor.submit(l1.dev_pay, inv1['bolt11'], use_shadow=False)
+# 
+#     l1.daemon.wait_for_log(r'dev_disconnect: \+WIRE_COMMITMENT_SIGNED')
+# 
+#     print("Killing l1 in mid HTLC")
+#     l1.daemon.kill()
+# 
+#     # Restart l1, without disconnect stuff.
+#     del l1.daemon.opts['dev-no-reconnect']
+#     del l1.daemon.opts['dev-disconnect']
+# 
+#     # Should reconnect, and sort the payment out.
+#     l1.start()
+# 
+#     wait_for(lambda: l1.rpc.listsendpays()['payments'][0]['status'] != 'pending')
+# 
+#     payments = l1.rpc.listsendpays()['payments']
+#     invoices = l2.rpc.listinvoices('inv1')['invoices']
+#     assert len(payments) == 1 and payments[0]['status'] == 'complete'
+#     assert len(invoices) == 1 and invoices[0]['status'] == 'paid'
+# 
+#     l1.wait_channel_active(chanid)
+# 
+#     # A duplicate should succeed immediately (nop) and return correct preimage.
+#     preimage = l1.dev_pay(
+#         inv1['bolt11'],
+#         use_shadow=False
+#     )['payment_preimage']
+#     assert l1.rpc.dev_rhash(preimage)['rhash'] == inv1['payment_hash']
+# 
+# 
+# @pytest.mark.developer("needs DEVELOPER=1")
+# @pytest.mark.openchannel('v1')
+# @pytest.mark.openchannel('v2')
+# def test_payment_failed_persistence(node_factory, executor):
+#     # Start two nodes and open a channel.. die during payment.
+#     # Feerates identical so we don't get gratuitous commit to update them
+#     disconnect = ['+WIRE_COMMITMENT_SIGNED']
+#     if EXPERIMENTAL_DUAL_FUND:
+#         # We have to add an extra 'wire-commitment-signed' because
+#         # dual funding uses this for channel establishment also
+#         disconnect = ['=WIRE_COMMITMENT_SIGNED'] + disconnect
+#     l1 = node_factory.get_node(disconnect=disconnect,
+#                                options={'dev-no-reconnect': None},
+#                                may_reconnect=True,
+#                                feerates=(7500, 7500, 7500, 7500))
+#     l2 = node_factory.get_node(may_reconnect=True)
+#     l1.rpc.connect(l2.info['id'], 'localhost', l2.port)
+# 
+#     l1.fundchannel(l2, 100000)
+# 
+#     # Expires almost immediately, so it will fail.
+#     inv1 = l2.rpc.invoice(1000, 'inv1', 'inv1', 5)
+# 
+#     # Fire off a pay request, it'll get interrupted by a restart
+#     executor.submit(l1.rpc.pay, inv1['bolt11'])
+# 
+#     l1.daemon.wait_for_log(r'dev_disconnect: \+WIRE_COMMITMENT_SIGNED')
+# 
+#     print("Killing l1 in mid HTLC")
+#     l1.daemon.kill()
+# 
+#     # Restart l1, without disconnect stuff.
+#     del l1.daemon.opts['dev-no-reconnect']
+#     del l1.daemon.opts['dev-disconnect']
+# 
+#     # Make sure invoice has expired.
+#     time.sleep(5 + 1)
+# 
+#     # Should reconnect, and fail the payment
+#     l1.start()
+# 
+#     wait_for(lambda: l1.rpc.listsendpays()['payments'][0]['status'] != 'pending')
+# 
+#     payments = l1.rpc.listsendpays()['payments']
+#     invoices = l2.rpc.listinvoices('inv1')['invoices']
+#     assert len(invoices) == 1 and invoices[0]['status'] == 'expired'
+#     assert len(payments) == 1 and payments[0]['status'] == 'failed'
+# 
+#     # Another attempt should also fail.
+#     with pytest.raises(RpcError):
+#         l1.rpc.pay(inv1['bolt11'])
+# 
+# 
+# @pytest.mark.developer("needs DEVELOPER=1")
+# def test_payment_duplicate_uncommitted(node_factory, executor):
+#     # We want to test two payments at the same time, before we send commit
+#     l1 = node_factory.get_node(options={'dev-disable-commit-after': 0})
+#     l2 = node_factory.get_node()
+# 
+#     l1.rpc.connect(l2.info['id'], 'localhost', l2.port)
+# 
+#     l1.fundchannel(l2, 100000)
+# 
+#     inv1 = l2.rpc.invoice(1000, 'inv1', 'inv1')
+# 
+#     # Start first payment, but not yet in db.
+#     fut = executor.submit(l1.rpc.pay, inv1['bolt11'])
+# 
+#     # Make sure that's started...
+#     l1.daemon.wait_for_log('peer_out WIRE_UPDATE_ADD_HTLC')
+# 
+#     # We should see it in listsendpays
+#     payments = l1.rpc.listsendpays()['payments']
+#     assert len(payments) == 1
+#     assert payments[0]['status'] == 'pending' and payments[0]['payment_hash'] == inv1['payment_hash']
+# 
+#     # Second one will succeed eventually.
+#     fut2 = executor.submit(l1.rpc.pay, inv1['bolt11'])
+# 
+#     # Now, let it commit.
+#     l1.rpc.dev_reenable_commit(l2.info['id'])
+# 
+#     # These should succeed.
+#     fut.result(TIMEOUT)
+#     fut2.result(TIMEOUT)
+# 
+# 
+# @pytest.mark.developer("Too slow without --dev-fast-gossip")
+# def test_pay_maxfee_shadow(node_factory):
+#     """Test that we respect maxfeepercent for shadow routing."""
+#     l1, l2, l3 = node_factory.line_graph(3, fundchannel=True,
+#                                          wait_for_announce=True)
+#     # We use this to search for shadow routes
+#     wait_for(
+#         lambda: len(l1.rpc.listchannels(source=l2.info["id"])["channels"]) > 1
+#     )
+# 
+#     # shadow routes are random, so run multiple times.
+#     for i in range(5):
+#         # A tiny amount, we must not add the base_fee between l2 and l3
+#         amount = 2
+#         bolt11 = l2.rpc.invoice(amount, "tiny.{}".format(i), "tiny")["bolt11"]
+#         pay_status = l1.rpc.pay(bolt11)
+#         assert pay_status["amount_msat"] == Millisatoshi(amount)
+# 
+#     # shadow routes are random, so run multiple times.
+#     for i in range(5):
+#         # A bigger amount, shadow routing could have been used but we set a low
+#         # maxfeepercent.
+#         amount = 20000
+#         bolt11 = l2.rpc.invoice(amount, "big.{}".format(i), "bigger")["bolt11"]
+#         pay_status = l1.rpc.pay(bolt11, maxfeepercent=0.001)
+#         assert pay_status["amount_msat"] == Millisatoshi(amount)
+# 
+# 
+# def test_sendpay(node_factory):
+#     l1, l2 = node_factory.line_graph(2, fundamount=10**6)
+# 
+#     amt = 200000000
+#     inv = l2.rpc.invoice(amt, 'testpayment2', 'desc')
+#     rhash = inv['payment_hash']
+# 
+#     def invoice_unpaid(dst, label):
+#         invoices = dst.rpc.listinvoices(label)['invoices']
+#         return len(invoices) == 1 and invoices[0]['status'] == 'unpaid'
+# 
+#     routestep = {
+#         'amount_msat': amt,
+#         'id': l2.info['id'],
+#         'delay': 5,
+#         'channel': first_scid(l1, l2)
+#     }
+# 
+#     # Insufficient funds.
+#     with pytest.raises(RpcError):
+#         rs = copy.deepcopy(routestep)
+#         rs['amount_msat'] = rs['amount_msat'] - 1
+#         l1.rpc.sendpay([rs], rhash, payment_secret=inv['payment_secret'])
+#         l1.rpc.waitsendpay(rhash)
+#     assert invoice_unpaid(l2, 'testpayment2')
+# 
+#     # Gross overpayment (more than factor of 2)
+#     with pytest.raises(RpcError):
+#         rs = copy.deepcopy(routestep)
+#         rs['amount_msat'] = rs['amount_msat'] * 2 + 1
+#         l1.rpc.sendpay([rs], rhash, payment_secret=inv['payment_secret'])
+#         l1.rpc.waitsendpay(rhash)
+#     assert invoice_unpaid(l2, 'testpayment2')
+# 
+#     # Insufficient delay.
+#     with pytest.raises(RpcError):
+#         rs = copy.deepcopy(routestep)
+#         rs['delay'] = rs['delay'] - 2
+#         l1.rpc.sendpay([rs], rhash, payment_secret=inv['payment_secret'])
+#         l1.rpc.waitsendpay(rhash)
+#     assert invoice_unpaid(l2, 'testpayment2')
+# 
+#     # Bad ID.
+#     l1.rpc.check_request_schemas = False
+#     with pytest.raises(RpcError):
+#         rs = copy.deepcopy(routestep)
+#         rs['id'] = '00000000000000000000000000000000'
+#         l1.rpc.sendpay([rs], rhash, payment_secret=inv['payment_secret'])
+#     assert invoice_unpaid(l2, 'testpayment2')
+#     l1.rpc.check_request_schemas = True
+# 
+#     # Bad payment_secret
+#     l1.rpc.sendpay([routestep], rhash, payment_secret="00" * 32)
+#     with pytest.raises(RpcError):
+#         l1.rpc.waitsendpay(rhash)
+#     assert invoice_unpaid(l2, 'testpayment2')
+# 
+#     # Missing payment_secret
+#     l1.rpc.sendpay([routestep], rhash)
+#     with pytest.raises(RpcError):
+#         l1.rpc.waitsendpay(rhash)
+#     assert invoice_unpaid(l2, 'testpayment2')
+# 
+#     # FIXME: test paying via another node, should fail to pay twice.
+#     c1 = only_one(l1.rpc.listpeerchannels(l2.info['id'])['channels'])
+#     c2 = only_one(l2.rpc.listpeerchannels(l1.info['id'])['channels'])
+#     assert c1['to_us_msat'] == 10**6 * 1000
+#     assert c1['total_msat'] == 10**6 * 1000
+#     assert c2['to_us_msat'] == 0
+#     assert c2['total_msat'] == 10**6 * 1000
+# 
+#     # This works.
+#     before = int(time.time())
+#     details = l1.rpc.sendpay([routestep], rhash, payment_secret=inv['payment_secret'])
+#     after = int(time.time())
+#     preimage = l1.rpc.waitsendpay(rhash)['payment_preimage']
+#     # Check details
+#     assert details['payment_hash'] == rhash
+#     assert details['destination'] == l2.info['id']
+#     assert details['amount_msat'] == amt
+#     assert details['created_at'] >= before
+#     assert details['created_at'] <= after
+#     # Check receiver
+#     assert only_one(l2.rpc.listinvoices('testpayment2')['invoices'])['status'] == 'paid'
+#     assert only_one(l2.rpc.listinvoices('testpayment2')['invoices'])['pay_index'] == 1
+#     assert only_one(l2.rpc.listinvoices('testpayment2')['invoices'])['amount_received_msat'] == rs['amount_msat']
+#     assert only_one(l2.rpc.listinvoices('testpayment2')['invoices'])['payment_preimage'] == preimage
+# 
+#     # Balances should reflect it.
+#     def check_balances():
+#         c1 = only_one(l1.rpc.listpeerchannels(l2.info['id'])['channels'])
+#         c2 = only_one(l2.rpc.listpeerchannels(l1.info['id'])['channels'])
+#         return (
+#             c1['to_us_msat'] == 10**6 * 1000 - amt
+#             and c1['total_msat'] == 10**6 * 1000
+#             and c2['to_us_msat'] == amt
+#             and c2['total_msat'] == 10**6 * 1000
+#         )
+#     wait_for(check_balances)
+# 
+#     # Repeat will "succeed", but won't actually send anything (duplicate)
+#     assert not l1.daemon.is_in_log('Payment ./.: .* COMPLETE')
+#     details = l1.rpc.sendpay([routestep], rhash, payment_secret=inv['payment_secret'])
+#     assert details['status'] == "complete"
+#     preimage2 = details['payment_preimage']
+#     assert preimage == preimage2
+#     l1.daemon.wait_for_log('Payment ./.: .* COMPLETE')
+#     assert only_one(l2.rpc.listinvoices('testpayment2')['invoices'])['status'] == 'paid'
+#     assert only_one(l2.rpc.listinvoices('testpayment2')['invoices'])['amount_received_msat'] == rs['amount_msat']
+# 
+#     # Overpaying by "only" a factor of 2 succeeds.
+#     inv = l2.rpc.invoice(amt, 'testpayment3', 'desc')
+#     rhash = inv['payment_hash']
+#     assert only_one(l2.rpc.listinvoices('testpayment3')['invoices'])['status'] == 'unpaid'
+#     routestep = {'amount_msat': amt * 2, 'id': l2.info['id'], 'delay': 5, 'channel': first_scid(l1, l2)}
+#     l1.rpc.sendpay([routestep], rhash, payment_secret=inv['payment_secret'])
+#     preimage3 = l1.rpc.waitsendpay(rhash)['payment_preimage']
+#     assert only_one(l2.rpc.listinvoices('testpayment3')['invoices'])['status'] == 'paid'
+#     assert only_one(l2.rpc.listinvoices('testpayment3')['invoices'])['amount_received_msat'] == amt * 2
+# 
+#     # Test listsendpays
+#     payments = l1.rpc.listsendpays()['payments']
+#     assert len(payments) == 7  # Failed attempts also create entries, but with a different groupid
+# 
+#     invoice2 = only_one(l2.rpc.listinvoices('testpayment2')['invoices'])
+#     payments = l1.rpc.listsendpays(payment_hash=invoice2['payment_hash'])['payments']
+#     assert len(payments) == 6  # Failed attempts also create entries, but with a different groupid
+# 
+#     assert payments[-1]['status'] == 'complete'
+#     assert payments[-1]['payment_preimage'] == preimage2
+# 
+#     invoice3 = only_one(l2.rpc.listinvoices('testpayment3')['invoices'])
+#     payments = l1.rpc.listsendpays(payment_hash=invoice3['payment_hash'])['payments']
+#     assert len(payments) == 1
+# 
+#     assert payments[-1]['status'] == 'complete'
+#     assert payments[-1]['payment_preimage'] == preimage3
+# 
+# 
+# @unittest.skipIf(TEST_NETWORK != 'regtest', "The reserve computation is bitcoin specific")
+# def test_sendpay_cant_afford(node_factory):
+#     # Set feerates the same so we don't have to wait for update.
+#     l1, l2 = node_factory.line_graph(2, fundamount=10**6,
+#                                      opts={'feerates': (15000, 15000, 15000, 15000)})
+# 
+#     # Can't pay more than channel capacity.
+#     with pytest.raises(RpcError):
+#         l1.pay(l2, 10**9 + 1)
+# 
+#     # Reserve is 1%.
+#     reserve = 10**7
+# 
+#     # # This is how we recalc constants (v. v. slow!)
+#     # minimum = 1
+#     # maximum = 10**9
+#     # while maximum - minimum > 1:
+#     #     l1, l2 = node_factory.line_graph(2, fundamount=10**6,
+#     #                                      opts={'feerates': (15000, 15000, 15000, 15000)})
+#     #     try:
+#     #         l1.pay(l2, (minimum + maximum) // 2)
+#     #         minimum = (minimum + maximum) // 2
+#     #     except RpcError:
+#     #         maximum = (minimum + maximum) // 2
+#     #     print("{} - {}".format(minimum, maximum))
+#     # assert False
+# 
+#     # This is the fee, which needs to be taken into account for l1.
+#     if EXPERIMENTAL_FEATURES:
+#         # option_anchor_outputs
+#         available = 10**9 - 44700000
+#     else:
+#         available = 10**9 - 32040000
+# 
+#     # Can't pay past reserve.
+#     with pytest.raises(RpcError):
+#         l1.pay(l2, available)
+#     with pytest.raises(RpcError):
+#         l1.pay(l2, available - reserve + 1)
+# 
+#     # Can pay up to reserve (1%)
+#     l1.pay(l2, available - reserve)
+# 
+#     # And now it can't pay back, due to its own reserve.
+#     with pytest.raises(RpcError):
+#         l2.pay(l1, available - reserve)
+# 
+#     # But this should work.
+#     l2.pay(l1, available - reserve * 2)
+# 
+# 
+# def test_decodepay(node_factory):
+#     l1 = node_factory.get_node()
+# 
+#     # BOLT #11:
+#     # > ### Please make a donation of any amount using payment_hash 0001020304050607080900010203040506070809000102030405060708090102 to me @03e7156ae33b0a208d0744199163177e909e80176e55d97a2f221ede0f934dd9ad
+#     # > lnbc1pvjluezpp5qqqsyqcyq5rqwzqfqqqsyqcyq5rqwzqfqqqsyqcyq5rqwzqfqypqdpl2pkx2ctnv5sxxmmwwd5kgetjypeh2ursdae8g6twvus8g6rfwvs8qun0dfjkxaq8rkx3yf5tcsyz3d73gafnh3cax9rn449d9p5uxz9ezhhypd0elx87sjle52x86fux2ypatgddc6k63n7erqz25le42c4u4ecky03ylcqca784w
+#     #
+#     # Breakdown:
+#     #
+#     # * `lnbc`: prefix, lightning on bitcoin mainnet
+#     # * `1`: Bech32 separator
+#     # * `pvjluez`: timestamp (1496314658)
+#     # * `p`: payment hash
+#     #   * `p5`: `data_length` (`p` = 1, `5` = 20. 1 * 32 + 20 == 52)
+#     #   * `qqqsyqcyq5rqwzqfqqqsyqcyq5rqwzqfqqqsyqcyq5rqwzqfqypq`: payment hash 0001020304050607080900010203040506070809000102030405060708090102
+#     # * `d`: short description
+#     #   * `pl`: `data_length` (`p` = 1, `l` = 31. 1 * 32 + 31 == 63)
+#     #   * `2pkx2ctnv5sxxmmwwd5kgetjypeh2ursdae8g6twvus8g6rfwvs8qun0dfjkxaq`: 'Please consider supporting this project'
+#     # * `32vjcgqxyuj7nqphl3xmmhls2rkl3t97uan4j0xa87gj5779czc8p0z58zf5wpt9ggem6adl64cvawcxlef9djqwp2jzzfvs272504sp`: signature
+#     # * `0lkg3c`: Bech32 checksum
+#     b11 = l1.rpc.decodepay(
+#         'lnbc1pvjluezpp5qqqsyqcyq5rqwzqfqqqsyqcyq5rqwzqfqqqsyqcyq5rqwzqfqypqd'
+#         'pl2pkx2ctnv5sxxmmwwd5kgetjypeh2ursdae8g6twvus8g6rfwvs8qun0dfjkxaq8rk'
+#         'x3yf5tcsyz3d73gafnh3cax9rn449d9p5uxz9ezhhypd0elx87sjle52x86fux2ypatg'
+#         'ddc6k63n7erqz25le42c4u4ecky03ylcqca784w'
+#     )
+#     assert b11['currency'] == 'bc'
+#     assert b11['created_at'] == 1496314658
+#     assert b11['payment_hash'] == '0001020304050607080900010203040506070809000102030405060708090102'
+#     assert b11['description'] == 'Please consider supporting this project'
+#     assert b11['expiry'] == 3600
+#     assert b11['payee'] == '03e7156ae33b0a208d0744199163177e909e80176e55d97a2f221ede0f934dd9ad'
+# 
+#     # BOLT #11:
+#     # > ### Please send $3 for a cup of coffee to the same peer, within 1 minute
+#     # > lnbc2500u1pvjluezpp5qqqsyqcyq5rqwzqfqqqsyqcyq5rqwzqfqqqsyqcyq5rqwzqfqypqdq5xysxxatsyp3k7enxv4jsxqzpuaztrnwngzn3kdzw5hydlzf03qdgm2hdq27cqv3agm2awhz5se903vruatfhq77w3ls4evs3ch9zw97j25emudupq63nyw24cg27h2rspfj9srp
+#     #
+#     # Breakdown:
+#     #
+#     # * `lnbc`: prefix, lightning on bitcoin mainnet
+#     # * `2500u`: amount (2500 micro-bitcoin)
+#     # * `1`: Bech32 separator
+#     # * `pvjluez`: timestamp (1496314658)
+#     # * `p`: payment hash...
+#     # * `d`: short description
+#     #   * `q5`: `data_length` (`q` = 0, `5` = 20. 0 * 32 + 20 == 20)
+#     #   * `xysxxatsyp3k7enxv4js`: '1 cup coffee'
+#     # * `x`: expiry time
+#     #   * `qz`: `data_length` (`q` = 0, `z` = 2. 0 * 32 + 2 == 2)
+#     #   * `pu`: 60 seconds (`p` = 1, `u` = 28.  1 * 32 + 28 == 60)
+#     # * `azh8qt5w7qeewkmxtv55khqxvdfs9zzradsvj7rcej9knpzdwjykcq8gv4v2dl705pjadhpsc967zhzdpuwn5qzjm0s4hqm2u0vuhhqq`: signature
+#     # * `7vc09u`: Bech32 checksum
+#     b11 = l1.rpc.decodepay(
+#         'lnbc2500u1pvjluezpp5qqqsyqcyq5rqwzqfqqqsyqcyq5rqwzqfqqqsyqcyq5rqwzqf'
+#         'qypqdq5xysxxatsyp3k7enxv4jsxqzpuaztrnwngzn3kdzw5hydlzf03qdgm2hdq27cq'
+#         'v3agm2awhz5se903vruatfhq77w3ls4evs3ch9zw97j25emudupq63nyw24cg27h2rsp'
+#         'fj9srp'
+#     )
+#     assert b11['currency'] == 'bc'
+#     assert b11['amount_msat'] == Millisatoshi(2500 * 10**11 // 1000000)
+#     assert b11['created_at'] == 1496314658
+#     assert b11['payment_hash'] == '0001020304050607080900010203040506070809000102030405060708090102'
+#     assert b11['description'] == '1 cup coffee'
+#     assert b11['expiry'] == 60
+#     assert b11['payee'] == '03e7156ae33b0a208d0744199163177e909e80176e55d97a2f221ede0f934dd9ad'
+# 
+#     # BOLT #11:
+#     # > ### Now send $24 for an entire list of things (hashed)
+#     # > lnbc20m1pvjluezpp5qqqsyqcyq5rqwzqfqqqsyqcyq5rqwzqfqqqsyqcyq5rqwzqfqypqhp58yjmdan79s6qqdhdzgynm4zwqd5d7xmw5fk98klysy043l2ahrqscc6gd6ql3jrc5yzme8v4ntcewwz5cnw92tz0pc8qcuufvq7khhr8wpald05e92xw006sq94mg8v2ndf4sefvf9sygkshp5zfem29trqq2yxxz7
+#     #
+#     # Breakdown:
+#     #
+#     # * `lnbc`: prefix, lightning on bitcoin mainnet
+#     # * `20m`: amount (20 milli-bitcoin)
+#     # * `1`: Bech32 separator
+#     # * `pvjluez`: timestamp (1496314658)
+#     # * `p`: payment hash...
+#     # * `h`: tagged field: hash of description
+#     # * `p5`: `data_length` (`p` = 1, `5` = 20. 1 * 32 + 20 == 52)
+#     # * `8yjmdan79s6qqdhdzgynm4zwqd5d7xmw5fk98klysy043l2ahrqs`: SHA256 of 'One piece of chocolate cake, one icecream cone, one pickle, one slice of swiss cheese, one slice of salami, one lollypop, one piece of cherry pie, one sausage, one cupcake, and one slice of watermelon'
+#     # * `vjfls3ljx9e93jkw0kw40yxn4pevgzflf83qh2852esjddv4xk4z70nehrdcxa4fk0t6hlcc6vrxywke6njenk7yzkzw0quqcwxphkcp`: signature
+#     # * `vam37w`: Bech32 checksum
+#     b11 = l1.rpc.decodepay(
+#         'lnbc20m1pvjluezpp5qqqsyqcyq5rqwzqfqqqsyqcyq5rqwzqfqqqsyqcyq5rqwzqfqy'
+#         'pqhp58yjmdan79s6qqdhdzgynm4zwqd5d7xmw5fk98klysy043l2ahrqscc6gd6ql3jr'
+#         'c5yzme8v4ntcewwz5cnw92tz0pc8qcuufvq7khhr8wpald05e92xw006sq94mg8v2ndf'
+#         '4sefvf9sygkshp5zfem29trqq2yxxz7',
+#         'One piece of chocolate cake, one icecream cone, one pickle, one slic'
+#         'e of swiss cheese, one slice of salami, one lollypop, one piece of c'
+#         'herry pie, one sausage, one cupcake, and one slice of watermelon'
+#     )
+#     assert b11['currency'] == 'bc'
+#     assert b11['amount_msat'] == Millisatoshi(str(20 * 10**11 // 1000) + 'msat')
+#     assert b11['created_at'] == 1496314658
+#     assert b11['payment_hash'] == '0001020304050607080900010203040506070809000102030405060708090102'
+#     assert b11['expiry'] == 3600
+#     assert b11['payee'] == '03e7156ae33b0a208d0744199163177e909e80176e55d97a2f221ede0f934dd9ad'
+# 
+#     # > ### The same, on testnet, with a fallback address mk2QpYatsKicvFVuTAQLBryyccRXMUaGHP
+#     # > lntb20m1pvjluezhp58yjmdan79s6qqdhdzgynm4zwqd5d7xmw5fk98klysy043l2ahrqspp5qqqsyqcyq5rqwzqfqqqsyqcyq5rqwzqfqqqsyqcyq5rqwzqfqypqfpp3x9et2e20v6pu37c5d9vax37wxq72un98kmzzhznpurw9sgl2v0nklu2g4d0keph5t7tj9tcqd8rexnd07ux4uv2cjvcqwaxgj7v4uwn5wmypjd5n69z2xm3xgksg28nwht7f6zspwp3f9t
+#     #
+#     # Breakdown:
+#     #
+#     # * `lntb`: prefix, lightning on bitcoin testnet
+#     # * `20m`: amount (20 milli-bitcoin)
+#     # * `1`: Bech32 separator
+#     # * `pvjluez`: timestamp (1496314658)
+#     # * `p`: payment hash...
+#     # * `f`: tagged field: fallback address
+#     # * `pp`: `data_length` (`p` = 1. 1 * 32 + 1 == 33)
+#     # * `3x9et2e20v6pu37c5d9vax37wxq72un98`: `3` = 17, so P2PKH address
+#     # * `h`: tagged field: hash of description...
+#     # * `qh84fmvn2klvglsjxfy0vq2mz6t9kjfzlxfwgljj35w2kwa60qv49k7jlsgx43yhs9nuutllkhhnt090mmenuhp8ue33pv4klmrzlcqp`: signature
+#     # * `us2s2r`: Bech32 checksum
+#     b11 = l1.rpc.decodepay(
+#         'lntb20m1pvjluezhp58yjmdan79s6qqdhdzgynm4zwqd5d7xmw5fk98klysy043l2ahr'
+#         'qspp5qqqsyqcyq5rqwzqfqqqsyqcyq5rqwzqfqqqsyqcyq5rqwzqfqypqfpp3x9et2e2'
+#         '0v6pu37c5d9vax37wxq72un98kmzzhznpurw9sgl2v0nklu2g4d0keph5t7tj9tcqd8r'
+#         'exnd07ux4uv2cjvcqwaxgj7v4uwn5wmypjd5n69z2xm3xgksg28nwht7f6zspwp3f9t',
+#         'One piece of chocolate cake, one icecream cone, one pickle, one slic'
+#         'e of swiss cheese, one slice of salami, one lollypop, one piece of c'
+#         'herry pie, one sausage, one cupcake, and one slice of watermelon'
+#     )
+#     assert b11['currency'] == 'tb'
+#     assert b11['amount_msat'] == Millisatoshi(20 * 10**11 // 1000)
+#     assert b11['created_at'] == 1496314658
+#     assert b11['payment_hash'] == '0001020304050607080900010203040506070809000102030405060708090102'
+#     assert b11['expiry'] == 3600
+#     assert b11['payee'] == '03e7156ae33b0a208d0744199163177e909e80176e55d97a2f221ede0f934dd9ad'
+#     assert len(b11['fallbacks']) == 1
+#     assert b11['fallbacks'][0]['type'] == 'P2PKH'
+#     assert b11['fallbacks'][0]['addr'] == 'mk2QpYatsKicvFVuTAQLBryyccRXMUaGHP'
+# 
+#     # > ### On mainnet, with fallback address 1RustyRX2oai4EYYDpQGWvEL62BBGqN9T with extra routing info to go via nodes 029e03a901b85534ff1e92c43c74431f7ce72046060fcf7a95c37e148f78c77255 then 039e03a901b85534ff1e92c43c74431f7ce72046060fcf7a95c37e148f78c77255
+#     # > lnbc20m1pvjluezpp5qqqsyqcyq5rqwzqfqqqsyqcyq5rqwzqfqqqsyqcyq5rqwzqfqypqhp58yjmdan79s6qqdhdzgynm4zwqd5d7xmw5fk98klysy043l2ahrqsfpp3qjmp7lwpagxun9pygexvgpjdc4jdj85fr9yq20q82gphp2nflc7jtzrcazrra7wwgzxqc8u7754cdlpfrmccae92qgzqvzq2ps8pqqqqqqpqqqqq9qqqvpeuqafqxu92d8lr6fvg0r5gv0heeeqgcrqlnm6jhphu9y00rrhy4grqszsvpcgpy9qqqqqqgqqqqq7qqzqj9n4evl6mr5aj9f58zp6fyjzup6ywn3x6sk8akg5v4tgn2q8g4fhx05wf6juaxu9760yp46454gpg5mtzgerlzezqcqvjnhjh8z3g2qqdhhwkj
+#     #
+#     # Breakdown:
+#     #
+#     # * `lnbc`: prefix, lightning on bitcoin mainnet
+#     # * `20m`: amount (20 milli-bitcoin)
+#     # * `1`: Bech32 separator
+#     # * `pvjluez`: timestamp (1496314658)
+#     # * `p`: payment hash...
+#     # * `h`: tagged field: hash of description...
+#     # * `f`: tagged field: fallback address
+#     #   * `pp`: `data_length` (`p` = 1. 1 * 32 + 1 == 33)
+#     #   * `3` = 17, so P2PKH address
+#     #   * `qjmp7lwpagxun9pygexvgpjdc4jdj85f`: 160 bit P2PKH address
+#     # * `r`: tagged field: route information
+#     #   * `9y`: `data_length` (`9` = 5, `y` = 4.  5 * 32 + 4 = 164)
+#     #     `q20q82gphp2nflc7jtzrcazrra7wwgzxqc8u7754cdlpfrmccae92qgzqvzq2ps8pqqqqqqqqqqqq9qqqvpeuqafqxu92d8lr6fvg0r5gv0heeeqgcrqlnm6jhphu9y00rrhy4grqszsvpcgpy9qqqqqqqqqqqq7qqzq`: pubkey `029e03a901b85534ff1e92c43c74431f7ce72046060fcf7a95c37e148f78c77255`, `short_channel_id` 0102030405060708, `fee_base_msat` 1 millisatoshi, `fee_proportional_millionths` 20, `cltv_expiry_delta` 3.  pubkey `039e03a901b85534ff1e92c43c74431f7ce72046060fcf7a95c37e148f78c77255`, `short_channel_id` 030405060708090a, `fee_base_msat` 2 millisatoshi, `fee_proportional_millionths` 30, `cltv_expiry_delta` 4.
+#     # * `j9n4evl6mr5aj9f58zp6fyjzup6ywn3x6sk8akg5v4tgn2q8g4fhx05wf6juaxu9760yp46454gpg5mtzgerlzezqcqvjnhjh8z3g2qq`: signature
+#     # * `dhhwkj`: Bech32 checksum
+#     b11 = l1.rpc.decodepay('lnbc20m1pvjluezpp5qqqsyqcyq5rqwzqfqqqsyqcyq5rqwzqfqqqsyqcyq5rqwzqfqypqhp58yjmdan79s6qqdhdzgynm4zwqd5d7xmw5fk98klysy043l2ahrqsfpp3qjmp7lwpagxun9pygexvgpjdc4jdj85fr9yq20q82gphp2nflc7jtzrcazrra7wwgzxqc8u7754cdlpfrmccae92qgzqvzq2ps8pqqqqqqpqqqqq9qqqvpeuqafqxu92d8lr6fvg0r5gv0heeeqgcrqlnm6jhphu9y00rrhy4grqszsvpcgpy9qqqqqqgqqqqq7qqzqj9n4evl6mr5aj9f58zp6fyjzup6ywn3x6sk8akg5v4tgn2q8g4fhx05wf6juaxu9760yp46454gpg5mtzgerlzezqcqvjnhjh8z3g2qqdhhwkj', 'One piece of chocolate cake, one icecream cone, one pickle, one slice of swiss cheese, one slice of salami, one lollypop, one piece of cherry pie, one sausage, one cupcake, and one slice of watermelon')
+#     assert b11['currency'] == 'bc'
+#     assert b11['amount_msat'] == Millisatoshi(20 * 10**11 // 1000)
+#     assert b11['created_at'] == 1496314658
+#     assert b11['payment_hash'] == '0001020304050607080900010203040506070809000102030405060708090102'
+#     assert b11['expiry'] == 3600
+#     assert b11['payee'] == '03e7156ae33b0a208d0744199163177e909e80176e55d97a2f221ede0f934dd9ad'
+#     assert len(b11['fallbacks']) == 1
+#     assert b11['fallbacks'][0]['type'] == 'P2PKH'
+#     assert b11['fallbacks'][0]['addr'] == '1RustyRX2oai4EYYDpQGWvEL62BBGqN9T'
+#     assert len(b11['routes']) == 1
+#     assert len(b11['routes'][0]) == 2
+#     assert b11['routes'][0][0]['pubkey'] == '029e03a901b85534ff1e92c43c74431f7ce72046060fcf7a95c37e148f78c77255'
+#     # 0x010203:0x040506:0x0708
+#     assert b11['routes'][0][0]['short_channel_id'] == '66051x263430x1800'
+#     assert b11['routes'][0][0]['fee_base_msat'] == 1
+#     assert b11['routes'][0][0]['fee_proportional_millionths'] == 20
+#     assert b11['routes'][0][0]['cltv_expiry_delta'] == 3
+# 
+#     assert b11['routes'][0][1]['pubkey'] == '039e03a901b85534ff1e92c43c74431f7ce72046060fcf7a95c37e148f78c77255'
+#     # 0x030405:0x060708:0x090a
+#     assert b11['routes'][0][1]['short_channel_id'] == '197637x395016x2314'
+#     assert b11['routes'][0][1]['fee_base_msat'] == 2
+#     assert b11['routes'][0][1]['fee_proportional_millionths'] == 30
+#     assert b11['routes'][0][1]['cltv_expiry_delta'] == 4
+# 
+#     # > ### On mainnet, with fallback (P2SH) address 3EktnHQD7RiAE6uzMj2ZifT9YgRrkSgzQX
+#     # > lnbc20m1pvjluezhp58yjmdan79s6qqdhdzgynm4zwqd5d7xmw5fk98klysy043l2ahrqspp5qqqsyqcyq5rqwzqfqqqsyqcyq5rqwzqfqqqsyqcyq5rqwzqfqypqfppj3a24vwu6r8ejrss3axul8rxldph2q7z9kmrgvr7xlaqm47apw3d48zm203kzcq357a4ls9al2ea73r8jcceyjtya6fu5wzzpe50zrge6ulk4nvjcpxlekvmxl6qcs9j3tz0469gq5g658y
+#     #
+#     # Breakdown:
+#     #
+#     # * `lnbc`: prefix, lightning on bitcoin mainnet
+#     # * `20m`: amount (20 milli-bitcoin)
+#     # * `1`: Bech32 separator
+#     # * `pvjluez`: timestamp (1496314658)
+#     # * `p`: payment hash...
+#     # * `f`: tagged field: fallback address.
+#     # * `pp`: `data_length` (`p` = 1. 1 * 32 + 1 == 33)
+#     # * `j3a24vwu6r8ejrss3axul8rxldph2q7z9`: `j` = 18, so P2SH address
+#     # * `h`: tagged field: hash of description...
+#     # * `2jhz8j78lv2jynuzmz6g8ve53he7pheeype33zlja5azae957585uu7x59w0f2l3rugyva6zpu394y4rh093j6wxze0ldsvk757a9msq`: signature
+#     # * `mf9swh`: Bech32 checksum
+#     b11 = l1.rpc.decodepay('lnbc20m1pvjluezhp58yjmdan79s6qqdhdzgynm4zwqd5d7xmw5fk98klysy043l2ahrqspp5qqqsyqcyq5rqwzqfqqqsyqcyq5rqwzqfqqqsyqcyq5rqwzqfqypqfppj3a24vwu6r8ejrss3axul8rxldph2q7z9kmrgvr7xlaqm47apw3d48zm203kzcq357a4ls9al2ea73r8jcceyjtya6fu5wzzpe50zrge6ulk4nvjcpxlekvmxl6qcs9j3tz0469gq5g658y', 'One piece of chocolate cake, one icecream cone, one pickle, one slice of swiss cheese, one slice of salami, one lollypop, one piece of cherry pie, one sausage, one cupcake, and one slice of watermelon')
+#     assert b11['currency'] == 'bc'
+#     assert b11['amount_msat'] == Millisatoshi(20 * 10**11 // 1000)
+#     assert b11['created_at'] == 1496314658
+#     assert b11['payment_hash'] == '0001020304050607080900010203040506070809000102030405060708090102'
+#     assert b11['expiry'] == 3600
+#     assert b11['payee'] == '03e7156ae33b0a208d0744199163177e909e80176e55d97a2f221ede0f934dd9ad'
+#     assert len(b11['fallbacks']) == 1
+#     assert b11['fallbacks'][0]['type'] == 'P2SH'
+#     assert b11['fallbacks'][0]['addr'] == '3EktnHQD7RiAE6uzMj2ZifT9YgRrkSgzQX'
+# 
+#     # > ### On mainnet, with fallback (P2WPKH) address bc1qw508d6qejxtdg4y5r3zarvary0c5xw7kv8f3t4
+#     # > lnbc20m1pvjluezhp58yjmdan79s6qqdhdzgynm4zwqd5d7xmw5fk98klysy043l2ahrqspp5qqqsyqcyq5rqwzqfqqqsyqcyq5rqwzqfqqqsyqcyq5rqwzqfqypqfppqw508d6qejxtdg4y5r3zarvary0c5xw7kepvrhrm9s57hejg0p662ur5j5cr03890fa7k2pypgttmh4897d3raaq85a293e9jpuqwl0rnfuwzam7yr8e690nd2ypcq9hlkdwdvycqa0qza8
+#     #
+#     # * `lnbc`: prefix, lightning on bitcoin mainnet
+#     # * `20m`: amount (20 milli-bitcoin)
+#     # * `1`: Bech32 separator
+#     # * `pvjluez`: timestamp (1496314658)
+#     # * `p`: payment hash...
+#     # * `f`: tagged field: fallback address.
+#     # * `pp`: `data_length` (`p` = 1. 1 * 32 + 1 == 33)
+#     # * `q`: 0, so witness version 0.
+#     # * `qw508d6qejxtdg4y5r3zarvary0c5xw7k`: 160 bits = P2WPKH.
+#     # * `h`: tagged field: hash of description...
+#     # * `gw6tk8z0p0qdy9ulggx65lvfsg3nxxhqjxuf2fvmkhl9f4jc74gy44d5ua9us509prqz3e7vjxrftn3jnk7nrglvahxf7arye5llphgq`: signature
+#     # * `qdtpa4`: Bech32 checksum
+#     b11 = l1.rpc.decodepay('lnbc20m1pvjluezhp58yjmdan79s6qqdhdzgynm4zwqd5d7xmw5fk98klysy043l2ahrqspp5qqqsyqcyq5rqwzqfqqqsyqcyq5rqwzqfqqqsyqcyq5rqwzqfqypqfppqw508d6qejxtdg4y5r3zarvary0c5xw7kepvrhrm9s57hejg0p662ur5j5cr03890fa7k2pypgttmh4897d3raaq85a293e9jpuqwl0rnfuwzam7yr8e690nd2ypcq9hlkdwdvycqa0qza8', 'One piece of chocolate cake, one icecream cone, one pickle, one slice of swiss cheese, one slice of salami, one lollypop, one piece of cherry pie, one sausage, one cupcake, and one slice of watermelon')
+#     assert b11['currency'] == 'bc'
+#     assert b11['amount_msat'] == Millisatoshi(20 * 10**11 // 1000)
+#     assert b11['created_at'] == 1496314658
+#     assert b11['payment_hash'] == '0001020304050607080900010203040506070809000102030405060708090102'
+#     assert b11['expiry'] == 3600
+#     assert b11['payee'] == '03e7156ae33b0a208d0744199163177e909e80176e55d97a2f221ede0f934dd9ad'
+#     assert len(b11['fallbacks']) == 1
+#     assert b11['fallbacks'][0]['type'] == 'P2WPKH'
+#     assert b11['fallbacks'][0]['addr'] == 'bc1qw508d6qejxtdg4y5r3zarvary0c5xw7kv8f3t4'
+# 
+#     # > ### On mainnet, with fallback (P2WSH) address bc1qrp33g0q5c5txsp9arysrx4k6zdkfs4nce4xj0gdcccefvpysxf3qccfmv3
+#     # > lnbc20m1pvjluezhp58yjmdan79s6qqdhdzgynm4zwqd5d7xmw5fk98klysy043l2ahrqspp5qqqsyqcyq5rqwzqfqqqsyqcyq5rqwzqfqqqsyqcyq5rqwzqfqypqfp4qrp33g0q5c5txsp9arysrx4k6zdkfs4nce4xj0gdcccefvpysxf3q28j0v3rwgy9pvjnd48ee2pl8xrpxysd5g44td63g6xcjcu003j3qe8878hluqlvl3km8rm92f5stamd3jw763n3hck0ct7p8wwj463cql26ava
+#     #
+#     # * `lnbc`: prefix, lightning on bitcoin mainnet
+#     # * `20m`: amount (20 milli-bitcoin)
+#     # * `1`: Bech32 separator
+#     # * `pvjluez`: timestamp (1496314658)
+#     # * `p`: payment hash...
+#     # * `f`: tagged field: fallback address.
+#     # * `p4`: `data_length` (`p` = 1, `4` = 21. 1 * 32 + 21 == 53)
+#     # * `q`: 0, so witness version 0.
+#     # * `rp33g0q5c5txsp9arysrx4k6zdkfs4nce4xj0gdcccefvpysxf3q`: 260 bits = P2WSH.
+#     # * `h`: tagged field: hash of description...
+#     # * `5yps56lmsvgcrf476flet6js02m93kgasews8q3jhtp7d6cqckmh70650maq4u65tk53ypszy77v9ng9h2z3q3eqhtc3ewgmmv2grasp`: signature
+#     # * `akvd7y`: Bech32 checksum
+#     b11 = l1.rpc.decodepay('lnbc20m1pvjluezhp58yjmdan79s6qqdhdzgynm4zwqd5d7xmw5fk98klysy043l2ahrqspp5qqqsyqcyq5rqwzqfqqqsyqcyq5rqwzqfqqqsyqcyq5rqwzqfqypqfp4qrp33g0q5c5txsp9arysrx4k6zdkfs4nce4xj0gdcccefvpysxf3q28j0v3rwgy9pvjnd48ee2pl8xrpxysd5g44td63g6xcjcu003j3qe8878hluqlvl3km8rm92f5stamd3jw763n3hck0ct7p8wwj463cql26ava', 'One piece of chocolate cake, one icecream cone, one pickle, one slice of swiss cheese, one slice of salami, one lollypop, one piece of cherry pie, one sausage, one cupcake, and one slice of watermelon')
+#     assert b11['currency'] == 'bc'
+#     assert b11['amount_msat'] == Millisatoshi(20 * 10**11 // 1000)
+#     assert b11['created_at'] == 1496314658
+#     assert b11['payment_hash'] == '0001020304050607080900010203040506070809000102030405060708090102'
+#     assert b11['expiry'] == 3600
+#     assert b11['payee'] == '03e7156ae33b0a208d0744199163177e909e80176e55d97a2f221ede0f934dd9ad'
+#     assert len(b11['fallbacks']) == 1
+#     assert b11['fallbacks'][0]['type'] == 'P2WSH'
+#     assert b11['fallbacks'][0]['addr'] == 'bc1qrp33g0q5c5txsp9arysrx4k6zdkfs4nce4xj0gdcccefvpysxf3qccfmv3'
+# 
+#     # > ### Please send $30 for coffee beans to the same peer, which supports features 1 and 9
+#     # > lnbc25m1pvjluezpp5qqqsyqcyq5rqwzqfqqqsyqcyq5rqwzqfqqqsyqcyq5rqwzqfqypqdq5vdhkven9v5sxyetpdees9qzsze992adudgku8p05pstl6zh7av6rx2f297pv89gu5q93a0hf3g7lynl3xq56t23dpvah6u7y9qey9lccrdml3gaqwc6nxsl5ktzm464sq73t7cl
+#     #
+#     # Breakdown:
+#     #
+#     # * `lnbc`: prefix, Lightning on Bitcoin mainnet
+#     # * `25m`: amount (25 milli-bitcoin)
+#     # * `1`: Bech32 separator
+#     # * `pvjluez`: timestamp (1496314658)
+#     # * `p`: payment hash...
+#     # * `d`: short description
+#     #   * `q5`: `data_length` (`q` = 0, `5` = 20; 0 * 32 + 20 == 20)
+#     #   * `vdhkven9v5sxyetpdees`: 'coffee beans'
+#     # * `9`: features
+#     #   * `qz`: `data_length` (`q` = 0, `z` = 2; 0 * 32 + 2 == 2)
+#     #   * `sz`: b1000000010
+#     # * `e992adudgku8p05pstl6zh7av6rx2f297pv89gu5q93a0hf3g7lynl3xq56t23dpvah6u7y9qey9lccrdml3gaqwc6nxsl5ktzm464sq`: signature
+#     # * `73t7cl`: Bech32 checksum
+#     b11 = l1.rpc.decodepay('lnbc25m1pvjluezpp5qqqsyqcyq5rqwzqfqqqsyqcyq5rqwzqfqqqsyqcyq5rqwzqfqypqdq5vdhkven9v5sxyetpdees9qzsze992adudgku8p05pstl6zh7av6rx2f297pv89gu5q93a0hf3g7lynl3xq56t23dpvah6u7y9qey9lccrdml3gaqwc6nxsl5ktzm464sq73t7cl')
+#     assert b11['currency'] == 'bc'
+#     assert b11['amount_msat'] == Millisatoshi(25 * 10**11 // 1000)
+#     assert b11['created_at'] == 1496314658
+#     assert b11['payment_hash'] == '0001020304050607080900010203040506070809000102030405060708090102'
+#     assert b11['description'] == 'coffee beans'
+#     assert b11['expiry'] == 3600
+#     assert b11['payee'] == '03e7156ae33b0a208d0744199163177e909e80176e55d97a2f221ede0f934dd9ad'
+#     assert b11['features'] == '0202'
+# 
+#     # > # Same, but using invalid unknown feature 100
+#     # > lnbc25m1pvjluezpp5qqqsyqcyq5rqwzqfqqqsyqcyq5rqwzqfqqqsyqcyq5rqwzqfqypqdq5vdhkven9v5sxyetpdees9q4pqqqqqqqqqqqqqqqqqqszk3ed62snp73037h4py4gry05eltlp0uezm2w9ajnerhmxzhzhsu40g9mgyx5v3ad4aqwkmvyftzk4k9zenz90mhjcy9hcevc7r3lx2sphzfxz7
+#     #
+#     # Breakdown:
+#     #
+#     # * `lnbc`: prefix, Lightning on Bitcoin mainnet
+#     # * `25m`: amount (25 milli-bitcoin)
+#     # * `1`: Bech32 separator
+#     # * `pvjluez`: timestamp (1496314658)
+#     # * `p`: payment hash...
+#     # * `d`: short description
+#     #   * `q5`: `data_length` (`q` = 0, `5` = 20; 0 * 32 + 20 == 20)
+#     #   * `vdhkven9v5sxyetpdees`: 'coffee beans'
+#     # * `9`: features
+#     #   * `q4`: `data_length` (`q` = 0, `4` = 21; 0 * 32 + 21 == 21)
+#     #   * `pqqqqqqqqqqqqqqqqqqsz`: b00001...(90 zeroes)...1000000010
+#     # * `k3ed62snp73037h4py4gry05eltlp0uezm2w9ajnerhmxzhzhsu40g9mgyx5v3ad4aqwkmvyftzk4k9zenz90mhjcy9hcevc7r3lx2sp`: signature
+#     # * `hzfxz7`: Bech32 checksum
+#     with pytest.raises(RpcError, match='unknown feature.*100'):
+#         l1.rpc.decodepay('lnbc25m1pvjluezpp5qqqsyqcyq5rqwzqfqqqsyqcyq5rqwzqfqqqsyqcyq5rqwzqfqypqdq5vdhkven9v5sxyetpdees9q4pqqqqqqqqqqqqqqqqqqszk3ed62snp73037h4py4gry05eltlp0uezm2w9ajnerhmxzhzhsu40g9mgyx5v3ad4aqwkmvyftzk4k9zenz90mhjcy9hcevc7r3lx2sphzfxz7')
+# 
+#     # Example of an invoice without a multiplier suffix to the amount. This
+#     # should then be interpreted as 7 BTC according to the spec:
+#     #
+#     #   `amount`: optional number in that currency, followed by an optional
+#     #   `multiplier` letter. The unit encoded here is the 'social' convention of
+#     #   a payment unit -- in the case of Bitcoin the unit is 'bitcoin' NOT
+#     #   satoshis.
+#     b11 = "lnbcrt71p0g4u8upp5xn4k45tsp05akmn65s5k2063d5fyadhjse9770xz5sk7u4x6vcmqdqqcqzynxqrrssx94cf4p727jamncsvcd8m99n88k423ruzq4dxwevfatpp5gx2mksj2swshjlx4pe3j5w9yed5xjktrktzd3nc2a04kq8yu84l7twhwgpxjn3pw"
+#     b11 = l1.rpc.decodepay(b11)
+#     sat_per_btc = 10**8
+#     assert(b11['amount_msat'] == 7 * sat_per_btc * 1000)
+# 
+#     with pytest.raises(RpcError):
+#         l1.rpc.decodepay('1111111')
+# 
+# 
+# @pytest.mark.developer("Too slow without --dev-fast-gossip")
+# def test_forward(node_factory, bitcoind):
+#     # Connect 1 -> 2 -> 3.
+#     l1, l2, l3 = node_factory.line_graph(3, wait_for_announce=True)
+# 
+#     # If they're at different block heights we can get spurious errors.
+#     sync_blockheight(bitcoind, [l1, l2, l3])
+# 
+#     chanid1 = only_one(l1.rpc.listpeerchannels(l2.info['id'])['channels'])['short_channel_id']
+#     chanid2 = only_one(l2.rpc.listpeerchannels(l3.info['id'])['channels'])['short_channel_id']
+#     assert only_one(l2.rpc.listpeerchannels(l1.info['id'])['channels'])['short_channel_id'] == chanid1
+#     assert only_one(l3.rpc.listpeerchannels(l2.info['id'])['channels'])['short_channel_id'] == chanid2
+# 
+#     inv = l3.rpc.invoice(100000000, 'testpayment1', 'desc')
+#     rhash = inv['payment_hash']
+#     assert only_one(l3.rpc.listinvoices('testpayment1')['invoices'])['status'] == 'unpaid'
+# 
+#     # Fee for node2 is 10 millionths, plus 1.
+#     amt = 100000000
+#     fee = amt * 10 // 1000000 + 1
+# 
+#     baseroute = [{'amount_msat': amt + fee,
+#                   'id': l2.info['id'],
+#                   'delay': 12,
+#                   'channel': chanid1},
+#                  {'amount_msat': amt,
+#                   'id': l3.info['id'],
+#                   'delay': 6,
+#                   'channel': chanid2}]
+# 
+#     # Unknown other peer
+#     route = copy.deepcopy(baseroute)
+#     route[1]['id'] = '031a8dc444e41bb989653a4501e11175a488a57439b0c4947704fd6e3de5dca607'
+#     l1.rpc.sendpay(route, rhash, payment_secret=inv['payment_secret'])
+#     with pytest.raises(RpcError):
+#         l1.rpc.waitsendpay(rhash)
+# 
+#     # Delay too short (we always add one internally anyway, so subtract 2 here).
+#     route = copy.deepcopy(baseroute)
+#     route[0]['delay'] = 8
+#     l1.rpc.sendpay(route, rhash, payment_secret=inv['payment_secret'])
+#     with pytest.raises(RpcError):
+#         l1.rpc.waitsendpay(rhash)
+# 
+#     # Final delay too short
+#     route = copy.deepcopy(baseroute)
+#     route[1]['delay'] = 3
+#     l1.rpc.sendpay(route, rhash, payment_secret=inv['payment_secret'])
+#     with pytest.raises(RpcError):
+#         l1.rpc.waitsendpay(rhash)
+# 
+#     # This one works
+#     route = copy.deepcopy(baseroute)
+#     l1.rpc.sendpay(route, rhash, payment_secret=inv['payment_secret'])
+#     l1.rpc.waitsendpay(rhash)
+# 
+#     # Check that invoice payment and fee are tracked appropriately
+#     l1.daemon.wait_for_log('coin_move .* [(]invoice[)]')
+#     l1.rpc.bkpr_dumpincomecsv('koinly', 'koinly.csv')
+# 
+#     koinly_path = os.path.join(l1.daemon.lightning_dir, TEST_NETWORK, 'koinly.csv')
+#     koinly_csv = open(koinly_path, 'rb').read()
+#     expected_line = r'0.00100000000,.*,,,0.00000001001,.*,invoice'
+#     assert only_one(re.findall(expected_line, str(koinly_csv)))
+# 
+# 
+# @pytest.mark.developer("needs --dev-fast-gossip")
+# def test_forward_different_fees_and_cltv(node_factory, bitcoind):
+#     # FIXME: Check BOLT quotes here too
+#     # BOLT #7:
+#     # ```
+#     #    B
+#     #   / \
+#     #  /   \
+#     # A     C
+#     #  \   /
+#     #   \ /
+#     #    D
+#     # ```
+#     #
+#     # Each advertises the following `cltv_expiry_delta` on its end of every
+#     # channel:
+#     #
+#     # 1. A: 10 blocks
+#     # 2. B: 20 blocks
+#     # 3. C: 30 blocks
+#     # 4. D: 40 blocks
+#     #
+#     # C also uses a minimum `cltv_expiry` of 9 (the default) when requesting
+#     # payments.
+#     #
+#     # Also, each node has the same fee scheme which it uses for each of its
+#     # channels:
+#     #
+#     # 1. A: 100 base + 1000 millionths
+#     # 1. B: 200 base + 2000 millionths
+#     # 1. C: 300 base + 3000 millionths
+#     # 1. D: 400 base + 4000 millionths
+# 
+#     # We don't do D yet.
+#     l1, l2, l3 = node_factory.get_nodes(3, opts=[{'cltv-delta': 10, 'fee-base': 100, 'fee-per-satoshi': 1000},
+#                                                  {'cltv-delta': 20, 'fee-base': 200, 'fee-per-satoshi': 2000},
+#                                                  {'cltv-delta': 30, 'cltv-final': 9, 'fee-base': 300, 'fee-per-satoshi': 3000}])
+# 
+#     ret = l1.rpc.connect(l2.info['id'], 'localhost', l2.port)
+#     assert ret['id'] == l2.info['id']
+# 
+#     l1.daemon.wait_for_log('Handed peer, entering loop')
+#     l2.daemon.wait_for_log('Handed peer, entering loop')
+# 
+#     ret = l2.rpc.connect(l3.info['id'], 'localhost', l3.port)
+#     assert ret['id'] == l3.info['id']
+# 
+#     l2.daemon.wait_for_log('Handed peer, entering loop')
+#     l3.daemon.wait_for_log('Handed peer, entering loop')
+# 
+#     c1, _ = l1.fundchannel(l2, 10**6)
+#     c2, _ = l2.fundchannel(l3, 10**6)
+#     mine_funding_to_announce(bitcoind, [l1, l2, l3])
+# 
+#     # Make sure l1 has seen announce for all channels.
+#     l1.wait_channel_active(c1)
+#     l1.wait_channel_active(c2)
+# 
+#     # BOLT #7:
+#     #
+#     # If B were to send 4,999,999 millisatoshi directly to C, it wouldn't
+#     # charge itself a fee nor add its own `cltv_expiry_delta`, so it would
+#     # use C's requested `cltv_expiry` of 9.  We also assume it adds a
+#     # "shadow route" to give an extra CLTV of 42.  It could also add extra
+#     # cltv deltas at other hops, as these values are a minimum, but we don't
+#     # here for simplicity:
+# 
+#     # FIXME: Add shadow route
+#     shadow_route = 0
+#     route = l2.rpc.getroute(l3.info['id'], 4999999, 1)["route"]
+#     assert len(route) == 1
+# 
+#     # BOLT #7:
+#     #
+#     #    * `amount_msat`: 4999999
+#     #    * `cltv_expiry`: current-block-height + 9 + 42
+#     #    * `onion_routing_packet`:
+#     #      * `amt_to_forward` = 4999999
+#     #      * `outgoing_cltv_value` = current-block-height + 9 + 42
+#     #
+#     assert route[0]['amount_msat'] == 4999999
+#     assert route[0]['delay'] == 9 + shadow_route
+# 
+#     # BOLT #7:
+#     # If A were to send 4,999,999 millisatoshi to C via B, it needs to
+#     # pay B the fee it specified in the B->C `channel_update`, calculated as
+#     # per [HTLC Fees](#htlc_fees):
+#     #
+#     # 200 + 4999999 * 2000 / 1000000 = 10199
+#     #
+#     # Similarly, it would need to add the `cltv_expiry` from B->C's
+#     # `channel_update` (20), plus C's requested minimum (9), plus 42 for the
+#     # "shadow route".  Thus the `update_add_htlc` message from A to B would
+#     # be:
+#     #
+#     #    * `amount_msat`: 5010198
+#     #    * `cltv_expiry`: current-block-height + 20 + 9 + 42
+#     #    * `onion_routing_packet`:
+#     #      * `amt_to_forward` = 4999999
+#     #      * `outgoing_cltv_value` = current-block-height + 9 + 42
+#     route = l1.rpc.getroute(l3.info['id'], 4999999, 1)["route"]
+#     assert len(route) == 2
+# 
+#     assert route[0]['amount_msat'] == 5010198
+#     assert route[0]['delay'] == 20 + 9 + shadow_route
+#     assert route[1]['amount_msat'] == 4999999
+#     assert route[1]['delay'] == 9 + shadow_route
+# 
+#     inv = l3.rpc.invoice(4999999, 'test_forward_different_fees_and_cltv', 'desc')
+#     rhash = inv['payment_hash']
+#     assert only_one(l3.rpc.listinvoices('test_forward_different_fees_and_cltv')['invoices'])['status'] == 'unpaid'
+# 
+#     # This should work.
+#     l1.rpc.sendpay(route, rhash, payment_secret=inv['payment_secret'])
+#     l1.rpc.waitsendpay(rhash)
+# 
+#     # We add one to the blockcount for a bit of fuzz (FIXME: Shadowroute would fix this!)
+#     shadow_route = 1
+#     l1.daemon.wait_for_log("Adding HTLC 0 amount=5010198msat cltv={} gave CHANNEL_ERR_ADD_OK"
+#                            .format(bitcoind.rpc.getblockcount() + 20 + 9 + shadow_route))
+#     l2.daemon.wait_for_log("Adding HTLC 0 amount=4999999msat cltv={} gave CHANNEL_ERR_ADD_OK"
+#                            .format(bitcoind.rpc.getblockcount() + 9 + shadow_route))
+#     l3.daemon.wait_for_log("Resolved invoice 'test_forward_different_fees_and_cltv' with amount 4999999msat")
+#     assert only_one(l3.rpc.listinvoices('test_forward_different_fees_and_cltv')['invoices'])['status'] == 'paid'
+# 
+#     # Check that we see all the channels
+#     shortids = set(c['short_channel_id'] for c in l2.rpc.listchannels()['channels'])
+#     for scid in shortids:
+#         c = l1.rpc.listchannels(scid)['channels']
+#         # We get one entry for each direction.
+#         assert len(c) == 2
+#         assert c[0]['short_channel_id'] == scid
+#         assert c[1]['short_channel_id'] == scid
+#         assert c[0]['source'] == c[1]['destination']
+#         assert c[1]['source'] == c[0]['destination']
+# 
+# 
+# @pytest.mark.developer("too slow without --dev-fast-gossip")
+# def test_forward_pad_fees_and_cltv(node_factory, bitcoind):
+#     """Test that we are allowed extra locktime delta, and fees"""
+# 
+#     l1, l2, l3 = node_factory.get_nodes(3, opts=[{'cltv-delta': 10, 'fee-base': 100, 'fee-per-satoshi': 1000},
+#                                                  {'cltv-delta': 20, 'fee-base': 200, 'fee-per-satoshi': 2000},
+#                                                  {'cltv-delta': 30, 'cltv-final': 9, 'fee-base': 300, 'fee-per-satoshi': 3000}])
+# 
+#     ret = l1.rpc.connect(l2.info['id'], 'localhost', l2.port)
+#     assert ret['id'] == l2.info['id']
+# 
+#     l1.daemon.wait_for_log('Handed peer, entering loop')
+#     l2.daemon.wait_for_log('Handed peer, entering loop')
+# 
+#     ret = l2.rpc.connect(l3.info['id'], 'localhost', l3.port)
+#     assert ret['id'] == l3.info['id']
+# 
+#     l2.daemon.wait_for_log('Handed peer, entering loop')
+#     l3.daemon.wait_for_log('Handed peer, entering loop')
+# 
+#     c1, _ = l1.fundchannel(l2, 10**6)
+#     c2, _ = l2.fundchannel(l3, 10**6)
+#     mine_funding_to_announce(bitcoind, [l1, l2, l3])
+# 
+#     # Make sure l1 has seen announce for all channels.
+#     l1.wait_channel_active(c1)
+#     l1.wait_channel_active(c2)
+# 
+#     route = l1.rpc.getroute(l3.info['id'], 4999999, 1)["route"]
+#     assert len(route) == 2
+# 
+#     assert route[0]['amount_msat'] == 5010198
+#     assert route[0]['delay'] == 20 + 9
+#     assert route[1]['amount_msat'] == 4999999
+#     assert route[1]['delay'] == 9
+# 
+#     # Modify so we overpay, overdo the cltv.
+#     route[0]['amount_msat'] += 2000
+#     route[0]['delay'] += 20
+#     route[1]['amount_msat'] += 1000
+#     route[1]['delay'] += 10
+# 
+#     # This should work.
+#     inv = l3.rpc.invoice(4999999, 'test_forward_pad_fees_and_cltv', 'desc')
+#     rhash = inv['payment_hash']
+#     l1.rpc.sendpay(route, rhash, payment_secret=inv['payment_secret'])
+#     l1.rpc.waitsendpay(rhash)
+#     assert only_one(l3.rpc.listinvoices('test_forward_pad_fees_and_cltv')['invoices'])['status'] == 'paid'
+# 
+#     # Do some checks of the bookkeeper's records
+#     def _income_tagset(node, tagset):
+#         incomes = node.rpc.bkpr_listincome()['income_events']
+#         return [e for e in incomes if e['tag'] in tagset]
+# 
+#     tags = ['invoice', 'invoice_fee']
+#     wait_for(lambda: len(_income_tagset(l1, tags)) == 2)
+#     incomes = _income_tagset(l1, tags)
+#     # the balance on l3 should equal the invoice
+#     accts = l3.rpc.bkpr_listbalances()['accounts']
+#     assert len(accts) == 2
+#     wallet = accts[0]
+#     chan_acct = accts[1]
+#     assert wallet['account'] == 'wallet'
+#     assert only_one(wallet['balances'])['balance_msat'] == Millisatoshi(0)
+#     assert incomes[0]['tag'] == 'invoice'
+#     assert only_one(chan_acct['balances'])['balance_msat'] == incomes[0]['debit_msat']
+#     inve = only_one([e for e in l1.rpc.bkpr_listaccountevents()['events'] if e['tag'] == 'invoice'])
+#     assert inve['debit_msat'] == incomes[0]['debit_msat'] + incomes[1]['debit_msat']
+# 
+# 
+# @pytest.mark.developer("needs DEVELOPER=1 for dev_ignore_htlcs")
+# def test_forward_stats(node_factory, bitcoind):
+#     """Check that we track forwarded payments correctly.
+# 
+#     We wire up the network to have l1 as payment initiator, l2 as
+#     forwarded (the one we check) and l3-l5 as payment recipients. l3
+#     accepts correctly, l4 rejects (because it doesn't know the payment
+#     hash) and l5 will keep the HTLC dangling by disconnecting.
+# 
+#     """
+#     amount = 10**5
+#     l1, l2, l3, l4, l5 = node_factory.get_nodes(5, opts=[{}] * 4 + [{'may_fail': True}])
+#     node_factory.join_nodes([l1, l2, l3], wait_for_announce=False)
+#     l2.openchannel(l4, 10**6, wait_for_announce=False)
+#     l2.openchannel(l5, 10**6, wait_for_announce=False)
+# 
+#     mine_funding_to_announce(bitcoind, [l1, l2, l3, l4, l5])
+# 
+#     wait_for(lambda: len(l1.rpc.listchannels()['channels']) == 8)
+# 
+#     inv = l3.rpc.invoice(amount, "first", "desc")
+#     payment_hash = inv['payment_hash']
+#     route = l1.rpc.getroute(l3.info['id'], amount, 1)['route']
+# 
+#     l1.rpc.sendpay(route, payment_hash, payment_secret=inv['payment_secret'])
+#     l1.rpc.waitsendpay(payment_hash)
+# 
+#     # l4 rejects since it doesn't know the payment_hash
+#     route = l1.rpc.getroute(l4.info['id'], amount, 1)['route']
+#     payment_hash = "F" * 64
+#     with pytest.raises(RpcError):
+#         l1.rpc.sendpay(route, payment_hash, payment_secret=inv['payment_secret'])
+#         l1.rpc.waitsendpay(payment_hash)
+# 
+#     # l5 will hold the HTLC hostage.
+#     l5.rpc.dev_ignore_htlcs(id=l2.info['id'], ignore=True)
+#     route = l1.rpc.getroute(l5.info['id'], amount, 1)['route']
+#     inv = l5.rpc.invoice(amount, "first", "desc")
+#     payment_hash = inv['payment_hash']
+#     l1.rpc.sendpay(route, payment_hash, payment_secret=inv['payment_secret'])
+# 
+#     l5.daemon.wait_for_log(r'their htlc .* dev_ignore_htlcs')
+# 
+#     # Select all forwardings, ordered by htlc_id to ensure the order
+#     # matches below
+#     forwardings = l2.db_query("SELECT *, in_msatoshi - out_msatoshi as fee "
+#                               "FROM forwards "
+#                               "ORDER BY in_htlc_id;")
+#     assert(len(forwardings) == 3)
+#     states = [f['state'] for f in forwardings]
+#     assert(states == [1, 2, 0])  # settled, failed, offered
+# 
+#     inchan = l2.rpc.listpeerchannels(l1.info['id'])['channels'][0]
+#     outchan = l2.rpc.listpeerchannels(l3.info['id'])['channels'][0]
+# 
+#     # Check that we correctly account channel changes
+#     assert inchan['in_payments_offered'] == 3
+#     assert inchan['in_payments_fulfilled'] == 1
+#     assert inchan['in_offered_msat'] >= Millisatoshi(3 * amount)
+#     assert inchan['in_fulfilled_msat'] >= Millisatoshi(amount)
+# 
+#     assert outchan['out_payments_offered'] == 1
+#     assert outchan['out_payments_fulfilled'] == 1
+#     assert outchan['out_offered_msat'] >= Millisatoshi(amount)
+#     assert outchan['out_offered_msat'] == outchan['out_fulfilled_msat']
+# 
+#     assert outchan['out_fulfilled_msat'] < inchan['in_fulfilled_msat']
+# 
+#     stats = l2.rpc.listforwards()
+# 
+#     assert [f['status'] for f in stats['forwards']] == ['settled', 'failed', 'offered']
+#     assert l2.rpc.getinfo()['fees_collected_msat'] == 1 + amount // 100000
+#     assert l1.rpc.getinfo()['fees_collected_msat'] == 0
+#     assert l3.rpc.getinfo()['fees_collected_msat'] == 0
+#     assert stats['forwards'][0]['received_time'] <= stats['forwards'][0]['resolved_time']
+#     assert stats['forwards'][1]['received_time'] <= stats['forwards'][1]['resolved_time']
+#     assert 'received_time' in stats['forwards'][2] and 'resolved_time' not in stats['forwards'][2]
+# 
+# 
+# @pytest.mark.developer("too slow without --dev-fast-gossip")
+# @pytest.mark.slow_test
+# def test_forward_local_failed_stats(node_factory, bitcoind, executor):
+#     """Check that we track forwarded payments correctly.
+# 
+#     We wire up the network to have l1 and l6 as payment initiator, l2 as
+#     forwarded (the one we check) and l3-l5 as payment recipients.
+# 
+#     There 5 cases for FORWARD_LOCAL_FAILED status:
+#     1. When Msater resolves the reply about the next peer infor(sent
+#        by Gossipd), and need handle unknown next peer failure in
+#        channel_resolve_reply(). For this case, we ask l1 pay to l3
+#        through l2 but close the channel between l2 and l3 after
+#        getroute(), the payment will fail in l2 because of
+#        WIRE_UNKNOWN_NEXT_PEER;
+#     2. When Master handle the forward process with the htlc_in and
+#        the id of next hop, it tries to drive a new htlc_out but fails
+#        in forward_htlc(). For this case, we ask l1 pay to 14 through
+#        with no fee, so the payment will fail in l2 becase of
+#        WIRE_FEE_INSUFFICIENT;
+#     3. When we send htlc_out, Master asks Channeld to add a new htlc
+#        into the outgoing channel but Channeld fails. Master need
+#        handle and store this failure in rcvd_htlc_reply(). For this
+#        case, we ask l1 pay to l5 with 10**8 sat though the channel
+#        (l2-->l5) with the max capacity of 10**4 msat , the payment
+#        will fail in l2 because of CHANNEL_ERR_MAX_HTLC_VALUE_EXCEEDED;
+#     4. When Channeld receives a new revoke message, if the state of
+#        corresponding htlc is RCVD_ADD_ACK_REVOCATION, Master will tries
+#        to resolve onionpacket and handle the failure before resolving
+#        the next hop in peer_got_revoke(). For this case, we ask l6 pay
+#        to l4 though l1 and l2, but we replace the second node_id in route
+#        with the wrong one, so the payment will fail in l2 because of
+#        WIRE_INVALID_ONION_KEY;
+#     5. When Onchaind finds the htlc time out or missing htlc, Master
+#        need handle these failure as FORWARD_LOCAL_FAILED in if it's forward
+#        payment case. For this case, we ask l1 pay to l4 though l2 with the
+#        amount less than the invoice(the payment must fail in l4), and we
+#        also ask l5 disconnected before sending update_fail_htlc, so the
+#        htlc will be holding until l2 meets timeout and handle it as local_fail.
+#     """
+# 
+#     amount = 10**8
+# 
+#     disconnects = ['-WIRE_UPDATE_FAIL_HTLC', 'permfail']
+# 
+#     l1, l2, l3, l4, l5, l6 = node_factory.get_nodes(6, opts=[{},
+#                                                              {},
+#                                                              {},
+#                                                              {'disconnect': disconnects},
+#                                                              {},
+#                                                              {}])
+# 
+#     l1.rpc.connect(l2.info['id'], 'localhost', l2.port)
+#     l2.rpc.connect(l3.info['id'], 'localhost', l3.port)
+#     l2.rpc.connect(l4.info['id'], 'localhost', l4.port)
+#     l2.rpc.connect(l5.info['id'], 'localhost', l5.port)
+#     l6.rpc.connect(l1.info['id'], 'localhost', l1.port)
+#     c12, _ = l1.fundchannel(l2, 10**6)
+#     c23, _ = l2.fundchannel(l3, 10**6)
+#     c24, _ = l2.fundchannel(l4, 10**6)
+#     c25, _ = l2.fundchannel(l5, 10**4 * 3)
+#     l6.fundchannel(l1, 10**6)
+# 
+#     # Make sure routes finalized.
+#     mine_funding_to_announce(bitcoind, [l1, l2, l3, l4, l5, l6])
+#     l1.wait_channel_active(c23)
+#     l1.wait_channel_active(c24)
+#     l1.wait_channel_active(c25)
+#     l6.wait_channel_active(c24)
+# 
+#     wait_for(lambda: len(l1.rpc.listchannels()['channels']) == 10)
+# 
+#     """1. When Msater resolves the reply about the next peer infor(sent
+#        by Gossipd), and need handle unknown next peer failure in
+#        channel_resolve_reply();
+# 
+#        For this case, we ask l1 pay to l3 through l2 but close the channel
+#        between l2 and l3 after getroute(), the payment will fail in l2
+#        because of WIRE_UNKNOWN_NEXT_PEER;
+#     """
+# 
+#     inv = l3.rpc.invoice(amount, "first", "desc")
+#     payment_hash = inv['payment_hash']
+#     route = l1.rpc.getroute(l3.info['id'], amount, 1)['route']
+# 
+#     l2.rpc.close(c23, 1)
+# 
+#     with pytest.raises(RpcError):
+#         l1.rpc.sendpay(route, payment_hash, payment_secret=inv['payment_secret'])
+#         l1.rpc.waitsendpay(payment_hash)
+# 
+#     """2. When Master handle the forward process with the htlc_in and
+#        the id of next hop, it tries to drive a new htlc_out but fails
+#        in forward_htlc();
+# 
+#        For this case, we ask l1 pay to 14 through with no fee, so the
+#        payment will fail in l2 becase of WIRE_FEE_INSUFFICIENT;
+#     """
+# 
+#     inv = l4.rpc.invoice(amount, "third", "desc")
+#     payment_hash = inv['payment_hash']
+#     fee = amount * 10 // 1000000 + 1
+# 
+#     route = [{'amount_msat': amount,
+#               'id': l2.info['id'],
+#               'delay': 12,
+#               'channel': c12},
+#              {'amount_msat': amount,
+#               'id': l4.info['id'],
+#               'delay': 6,
+#               'channel': c24}]
+# 
+#     with pytest.raises(RpcError):
+#         l1.rpc.sendpay(route, payment_hash, payment_secret=inv['payment_secret'])
+#         l1.rpc.waitsendpay(payment_hash)
+# 
+#     """3. When we send htlc_out, Master asks Channeld to add a new htlc
+#        into the outgoing channel but Channeld fails. Master need
+#        handle and store this failure in rcvd_htlc_reply();
+# 
+#        For this case, we ask l1 pay to l5 with 10**8 sat though the channel
+#        (l2-->l5) with the max capacity of 10**4 msat , the payment will
+#        fail in l2 because of CHANNEL_ERR_MAX_HTLC_VALUE_EXCEEDED;
+#     """
+# 
+#     inv = l5.rpc.invoice(amount, "second", "desc")
+#     payment_hash = inv['payment_hash']
+#     fee = amount * 10 // 1000000 + 1
+# 
+#     route = [{'amount_msat': amount + fee,
+#               'id': l2.info['id'],
+#               'delay': 12,
+#               'channel': c12},
+#              {'amount_msat': amount,
+#               'id': l5.info['id'],
+#               'delay': 6,
+#               'channel': c25}]
+# 
+#     with pytest.raises(RpcError):
+#         l1.rpc.sendpay(route, payment_hash, payment_secret=inv['payment_secret'])
+#         l1.rpc.waitsendpay(payment_hash)
+# 
+#     """4. When Channeld receives a new revoke message, if the state of
+#        corresponding htlc is RCVD_ADD_ACK_REVOCATION, Master will tries
+#        to resolve onionpacket and handle the failure before resolving
+#        the next hop in peer_got_revoke();
+# 
+#        For this case, we ask l6 pay to l4 though l1 and l2, but we replace
+#        the second node_id in route with the wrong one, so the payment will
+#        fail in l2 because of WIRE_INVALID_ONION_KEY;
+#     """
+# 
+#     inv = l4.rpc.invoice(amount, 'fourth', 'desc')
+#     payment_hash = inv['payment_hash']
+#     route = l6.rpc.getroute(l4.info['id'], amount, 1)['route']
+# 
+#     mangled_nodeid = '0265b6ab5ec860cd257865d61ef0bbf5b3339c36cbda8b26b74e7f1dca490b6510'
+# 
+#     # Replace id with a different pubkey, so onion encoded badly at l2 hop.
+#     route[1]['id'] = mangled_nodeid
+# 
+#     with pytest.raises(RpcError):
+#         l6.rpc.sendpay(route, payment_hash, payment_secret=inv['payment_secret'])
+#         l6.rpc.waitsendpay(payment_hash)
+# 
+#     """5. When Onchaind finds the htlc time out or missing htlc, Master
+#        need handle these failure as FORWARD_LOCAL_FAILED in if it's forward
+#        payment case.
+# 
+#        For this case, we ask l1 pay to l4 though l2 with the amount less than
+#        the invoice(the payment must fail in l4), and we also ask l5 disconnected
+#        before sending update_fail_htlc, so the htlc will be holding until l2
+#        meets timeout and handle it as local_fail.
+#     """
+#     inv = l4.rpc.invoice(amount, 'onchain_timeout', 'desc')
+#     payment_hash = inv['payment_hash']
+#     fee = amount * 10 // 1000000 + 1
+# 
+#     # We underpay, so it fails.
+#     route = [{'amount_msat': amount + fee - 1,
+#               'id': l2.info['id'],
+#               'delay': 12,
+#               'channel': c12},
+#              {'amount_msat': amount - 1,
+#               'id': l4.info['id'],
+#               'delay': 5,
+#               'channel': c24}]
+# 
+#     executor.submit(l1.rpc.sendpay, route, payment_hash, payment_secret=inv['payment_secret'])
+# 
+#     l4.daemon.wait_for_log('permfail')
+#     l4.wait_for_channel_onchain(l2.info['id'])
+#     l2.bitcoin.generate_block(1)
+#     l2.daemon.wait_for_log(' to ONCHAIN')
+#     l4.daemon.wait_for_log(' to ONCHAIN')
+# 
+#     # Wait for timeout.
+#     l2.daemon.wait_for_log('Propose handling THEIR_UNILATERAL/OUR_HTLC by OUR_HTLC_TIMEOUT_TO_US .* after 6 blocks')
+#     bitcoind.generate_block(6)
+# 
+#     l2.wait_for_onchaind_broadcast('OUR_HTLC_TIMEOUT_TO_US',
+#                                    'THEIR_UNILATERAL/OUR_HTLC')
+# 
+#     bitcoind.generate_block(1)
+#     l2.daemon.wait_for_log('Resolved THEIR_UNILATERAL/OUR_HTLC by our proposal OUR_HTLC_TIMEOUT_TO_US')
+#     l4.daemon.wait_for_log('Ignoring output.*: OUR_UNILATERAL/THEIR_HTLC')
+# 
+#     bitcoind.generate_block(100)
+#     sync_blockheight(bitcoind, [l2])
+# 
+#     # give time to let l2 store the local_failed stats
+#     time.sleep(5)
+# 
+#     # Select all forwardings, and check the status
+#     stats = l2.rpc.listforwards()
+# 
+#     assert [f['status'] for f in stats['forwards']] == ['local_failed', 'local_failed', 'local_failed', 'local_failed', 'local_failed']
+#     assert l2.rpc.getinfo()['fees_collected_msat'] == 0
+# 
+#     assert 'received_time' in stats['forwards'][0] and 'resolved_time' not in stats['forwards'][0]
+#     assert 'received_time' in stats['forwards'][1] and 'resolved_time' not in stats['forwards'][1]
+#     assert 'received_time' in stats['forwards'][2] and 'resolved_time' not in stats['forwards'][2]
+#     assert 'received_time' in stats['forwards'][3] and 'resolved_time' not in stats['forwards'][3]
+#     assert 'received_time' in stats['forwards'][3] and 'resolved_time' not in stats['forwards'][4]
+# 
+#     # Correct in and out channels
+#     assert [s['in_channel'] for s in stats['forwards']] == [c12] * 5
+#     assert [s.get('out_channel') for s in stats['forwards']] == [c23, c24, c25, None, c24]
+# 
+# 
+# @pytest.mark.developer("too slow without --dev-fast-gossip")
+# @pytest.mark.slow_test
+# def test_htlcs_cltv_only_difference(node_factory, bitcoind):
+#     # l1 -> l2 -> l3 -> l4
+#     # l4 ignores htlcs, so they stay.
+#     # l3 will see a reconnect from l4 when l4 restarts.
+#     l1, l2, l3, l4 = node_factory.line_graph(4, wait_for_announce=True, opts=[{}] * 2 + [{'dev-no-reconnect': None, 'may_reconnect': True}] * 2)
+# 
+#     inv = l4.rpc.invoice(amount_msat=10**8, label='x', description='desc')
+#     h = inv['payment_hash']
+#     l4.rpc.dev_ignore_htlcs(id=l3.info['id'], ignore=True)
+# 
+#     # L2 tries to pay
+#     r = l2.rpc.getroute(l4.info['id'], 10**8, 1)["route"]
+#     l2.rpc.sendpay(r, h, payment_secret=inv['payment_secret'])
+# 
+#     # Now increment CLTV
+#     bitcoind.generate_block(1)
+#     sync_blockheight(bitcoind, [l1, l2, l3, l4])
+# 
+#     # L1 tries to pay
+#     r = l1.rpc.getroute(l4.info['id'], 10**8, 1)["route"]
+#     l1.rpc.sendpay(r, h, payment_secret=inv['payment_secret'])
+# 
+#     # Now increment CLTV
+#     bitcoind.generate_block(1)
+#     sync_blockheight(bitcoind, [l1, l2, l3, l4])
+# 
+#     # L3 tries to pay
+#     r = l3.rpc.getroute(l4.info['id'], 10**8, 1)["route"]
+#     l3.rpc.sendpay(r, h, payment_secret=inv['payment_secret'])
+# 
+#     # Give them time to go through.
+#     time.sleep(5)
+# 
+#     # Will all be connected OK.
+#     assert only_one(l1.rpc.listpeers(l2.info['id'])['peers'])['connected']
+#     assert only_one(l2.rpc.listpeers(l3.info['id'])['peers'])['connected']
+#     assert only_one(l3.rpc.listpeers(l4.info['id'])['peers'])['connected']
+# 
+#     # TODO Remove our reliance on HTLCs failing on startup and the need for
+#     #      this plugin
+#     l4.daemon.opts['plugin'] = os.path.join(os.getcwd(), 'tests/plugins/fail_htlcs.py')
+# 
+#     # Restarting tail node will stop it ignoring HTLCs (it will actually
+#     # fail them immediately).
+#     l4.restart()
+#     l3.rpc.connect(l4.info['id'], 'localhost', l4.port)
+# 
+#     wait_for(lambda: only_one(l1.rpc.listsendpays()['payments'])['status'] == 'failed')
+#     wait_for(lambda: only_one(l2.rpc.listsendpays()['payments'])['status'] == 'failed')
+#     wait_for(lambda: only_one(l3.rpc.listsendpays()['payments'])['status'] == 'failed')
+# 
+#     # Should all still be connected.
+#     assert only_one(l1.rpc.listpeers(l2.info['id'])['peers'])['connected']
+#     assert only_one(l2.rpc.listpeers(l3.info['id'])['peers'])['connected']
+#     assert only_one(l3.rpc.listpeers(l4.info['id'])['peers'])['connected']
+# 
+# 
+# def test_pay_variants(node_factory):
+#     l1, l2 = node_factory.line_graph(2)
+# 
+#     # Upper case is allowed
+#     b11 = l2.rpc.invoice(123000, 'test_pay_variants upper', 'description')['bolt11'].upper()
+#     l1.rpc.decodepay(b11)
+#     l1.rpc.pay(b11)
+# 
+#     # lightning: prefix is allowed
+#     b11 = 'lightning:' + l2.rpc.invoice(123000, 'test_pay_variants with prefix', 'description')['bolt11']
+#     l1.rpc.decodepay(b11)
+#     l1.rpc.pay(b11)
+# 
+#     # BOTH is allowed.
+#     b11 = 'LIGHTNING:' + l2.rpc.invoice(123000, 'test_pay_variants upper with prefix', 'description')['bolt11'].upper()
+#     l1.rpc.decodepay(b11)
+#     l1.rpc.pay(b11)
+# 
+# 
+# @pytest.mark.developer("gossip without DEVELOPER=1 is slow")
+# @pytest.mark.slow_test
+# def test_pay_retry(node_factory, bitcoind, executor, chainparams):
+#     """Make sure pay command retries properly. """
+# 
+#     def exhaust_channel(opener, peer, scid, already_spent=0):
+#         """Spend all available capacity (10^6 - 1%) of channel
+#         """
+#         chan = only_one(opener.rpc.listpeerchannels(peer.info['id'])["channels"])
+#         maxpay = chan['spendable_msat']
+#         lbl = ''.join(random.choice(string.ascii_letters) for _ in range(20))
+#         inv = peer.rpc.invoice(maxpay, lbl, "exhaust_channel")
+#         routestep = {
+#             'amount_msat': maxpay,
+#             'id': peer.info['id'],
+#             'delay': 10,
+#             'channel': scid
+#         }
+#         opener.rpc.sendpay([routestep], inv['payment_hash'], payment_secret=inv['payment_secret'])
+#         opener.rpc.waitsendpay(inv['payment_hash'])
+# 
+#     # We connect every node to l5; in a line and individually.
+#     # Keep fixed fees so we can easily calculate exhaustion
+#     l1, l2, l3, l4, l5 = node_factory.line_graph(5, fundchannel=False,
+#                                                  opts={'feerates': (7500, 7500, 7500, 7500), 'disable-mpp': None})
+# 
+#     # scid12
+#     l1.fundchannel(l2, 10**6, wait_for_active=False)
+#     # scid23
+#     l2.fundchannel(l3, 10**6, wait_for_active=False)
+#     # scid34
+#     l3.fundchannel(l4, 10**6, wait_for_active=False)
+#     scid45, _ = l4.fundchannel(l5, 10**6, wait_for_active=False)
+# 
+#     l1.rpc.connect(l5.info['id'], 'localhost', l5.port)
+#     scid15, _ = l1.fundchannel(l5, 10**6, wait_for_active=False)
+#     l2.rpc.connect(l5.info['id'], 'localhost', l5.port)
+#     scid25, _ = l2.fundchannel(l5, 10**6, wait_for_active=False)
+#     l3.rpc.connect(l5.info['id'], 'localhost', l5.port)
+#     scid35, _ = l3.fundchannel(l5, 10**6, wait_for_active=False)
+# 
+#     # Make sure l1 sees all 7 channels
+#     mine_funding_to_announce(bitcoind, [l1, l2, l3, l4, l5])
+#     wait_for(lambda: len(l1.rpc.listchannels()['channels']) == 14)
+# 
+#     # Exhaust shortcut channels one at a time, to force retries.
+#     exhaust_channel(l1, l5, scid15)
+#     exhaust_channel(l2, l5, scid25)
+#     exhaust_channel(l3, l5, scid35)
+# 
+#     def listpays_nofail(b11):
+#         while True:
+#             pays = l1.rpc.listpays(b11)['pays']
+#             if len(pays) != 0:
+#                 if only_one(pays)['status'] == 'complete':
+#                     return
+#                 assert only_one(pays)['status'] != 'failed'
+# 
+#     inv = l5.rpc.invoice(10**8, 'test_retry', 'test_retry')
+# 
+#     # Make sure listpays doesn't transiently show failure while pay
+#     # is retrying.
+#     fut = executor.submit(listpays_nofail, inv['bolt11'])
+# 
+#     # Pay l1->l5 should succeed via straight line (eventually)
+#     l1.dev_pay(inv['bolt11'], use_shadow=False)
+# 
+#     # This should be OK.
+#     fut.result()
+# 
+#     # This should make it fail.
+#     exhaust_channel(l4, l5, scid45, 10**8)
+# 
+#     # It won't try l1->l5, since it knows that's under capacity.
+#     # It will try l1->l2->l5, which fails.
+#     # It will try l1->l2->l3->l5, which fails.
+#     # It will try l1->l2->l3->l4->l5, which fails.
+#     # Finally, fails to find a route.
+#     inv = l5.rpc.invoice(10**8, 'test_retry2', 'test_retry2')['bolt11']
+#     with pytest.raises(RpcError, match=r'4 attempts'):
+#         l1.dev_pay(inv, use_shadow=False)
+# 
+# 
+# @pytest.mark.developer("needs DEVELOPER=1 otherwise gossip takes 5 minutes!")
+# @pytest.mark.slow_test
+# def test_pay_routeboost(node_factory, bitcoind):
+#     """Make sure we can use routeboost information. """
+#     # l1->l2->l3--private-->l4
+#     l1, l2 = node_factory.line_graph(2, announce_channels=True, wait_for_announce=True)
+#     l3, l4, l5 = node_factory.line_graph(3, announce_channels=False, wait_for_announce=False)
+# 
+#     # This should a "could not find a route" because that's true.
+#     error = r'Destination [a-f0-9]{66} is not reachable directly and all routehints were unusable'
+# 
+#     with pytest.raises(RpcError, match=error):
+#         l1.rpc.pay(l5.rpc.invoice(10**8, 'test_retry', 'test_retry')['bolt11'])
+# 
+#     l2.rpc.connect(l3.info['id'], 'localhost', l3.port)
+#     scidl2l3, _ = l2.fundchannel(l3, 10**6)
+# 
+#     # Make sure l1 knows about the 2->3 channel.
+#     mine_funding_to_announce(bitcoind, [l1, l2, l3, l4, l5])
+#     l1.daemon.wait_for_logs([r'update for channel {}/0 now ACTIVE'
+#                              .format(scidl2l3),
+#                              r'update for channel {}/1 now ACTIVE'
+#                              .format(scidl2l3)])
+#     # Make sure l4 knows about 2->3 channel too so it's not a dead-end.
+#     l4.daemon.wait_for_logs([r'update for channel {}/0 now ACTIVE'
+#                              .format(scidl2l3),
+#                              r'update for channel {}/1 now ACTIVE'
+#                              .format(scidl2l3)])
+# 
+#     # Get an l4 invoice; it should put the private channel in routeboost.
+#     inv = l4.rpc.invoice(10**5, 'test_pay_routeboost', 'test_pay_routeboost',
+#                          exposeprivatechannels=True)
+#     assert 'warning_capacity' not in inv
+#     assert 'warning_offline' not in inv
+#     assert only_one(only_one(l1.rpc.decodepay(inv['bolt11'])['routes']))
+# 
+#     # Now we should be able to pay it.
+#     l1.dev_pay(inv['bolt11'], use_shadow=False)
+# 
+#     # Status should show all the gory details.
+#     status = l1.rpc.call('paystatus', [inv['bolt11']])
+#     assert only_one(status['pay'])['bolt11'] == inv['bolt11']
+#     assert only_one(status['pay'])['amount_msat'] == Millisatoshi(10**5)
+#     assert only_one(status['pay'])['destination'] == l4.info['id']
+#     assert 'label' not in only_one(status['pay'])
+#     assert 'routehint_modifications' not in only_one(status['pay'])
+#     assert 'local_exclusions' not in only_one(status['pay'])
+#     attempts = only_one(status['pay'])['attempts']
+#     scid34 = l3.rpc.listpeerchannels(l4.info['id'])['channels'][0]['alias']['local']
+#     assert(len(attempts) == 1)
+#     a = attempts[0]
+#     assert(a['strategy'] == "Initial attempt")
+#     assert('success' in a)
+#     assert('payment_preimage' in a['success'])
+# 
+#     # With dev-route option we can test longer routehints.
+#     if DEVELOPER:
+#         scid45 = l4.rpc.listpeerchannels(l5.info['id'])['channels'][0]['alias']['local']
+#         routel3l4l5 = [{'id': l3.info['id'],
+#                         'short_channel_id': scid34,
+#                         'fee_base_msat': 1000,
+#                         'fee_proportional_millionths': 10,
+#                         'cltv_expiry_delta': 6},
+#                        {'id': l4.info['id'],
+#                         'short_channel_id': scid45,
+#                         'fee_base_msat': 1000,
+#                         'fee_proportional_millionths': 10,
+#                         'cltv_expiry_delta': 6}]
+#         inv = l5.dev_invoice(amount_msat=10**5,
+#                              label='test_pay_routeboost2',
+#                              description='test_pay_routeboost2',
+#                              dev_routes=[routel3l4l5])
+#         l1.dev_pay(inv['bolt11'], use_shadow=False)
+#         status = l1.rpc.call('paystatus', [inv['bolt11']])
+#         pay = only_one(status['pay'])
+#         attempts = pay['attempts']
+#         assert(len(attempts) == 1)
+#         assert 'failure' not in attempts[0]
+#         assert 'success' in attempts[0]
+# 
+#         # Finally, it should fall back to second routehint if first fails.
+#         # (Note, this is not public because it's not 6 deep)
+#         l3.rpc.connect(l5.info['id'], 'localhost', l5.port)
+#         scid35, _ = l3.fundchannel(l5, 10**6)
+#         l4.stop()
+#         routel3l5 = [{'id': l3.info['id'],
+#                       'short_channel_id': scid35,
+#                       'fee_base_msat': 1000,
+#                       'fee_proportional_millionths': 10,
+#                       'cltv_expiry_delta': 6}]
+#         inv = l5.dev_invoice(amount_msat=10**5,
+#                              label='test_pay_routeboost5',
+#                              description='test_pay_routeboost5',
+#                              dev_routes=[routel3l4l5, routel3l5])
+#         l1.dev_pay(inv['bolt11'], label="paying test_pay_routeboost5",
+#                    use_shadow=False)
+# 
+#         status = l1.rpc.call('paystatus', [inv['bolt11']])
+#         assert only_one(status['pay'])['bolt11'] == inv['bolt11']
+#         assert only_one(status['pay'])['destination'] == l5.info['id']
+#         assert only_one(status['pay'])['label'] == "paying test_pay_routeboost5"
+#         assert 'routehint_modifications' not in only_one(status['pay'])
+#         assert 'local_exclusions' not in only_one(status['pay'])
+#         attempts = only_one(status['pay'])['attempts']
+# 
+#         # First one fails, second one succeeds, no routehint would come last.
+#         assert len(attempts) == 2
+#         assert 'success' not in attempts[0]
+#         assert 'success' in attempts[1]
+#         # TODO Add assertion on the routehint once we add them to the pay
+#         # output
+# 
+# 
+# @pytest.mark.developer("updates are delayed without --dev-fast-gossip")
+# def test_setchannel_usage(node_factory, bitcoind):
+#     # TEST SETUP
+#     #
+#     # [l1] ---> [l2]  (channel funded)
+#     #   |
+#     #   o - - > [l3]  (only connected)
+#     #
+#     # - check initial SQL values
+#     # - check setchannel can be used
+#     # - checks command's return object format
+#     # - check custom SQL fee values
+#     # - check values in local nodes listchannels output
+#     # - json throws exception on negative values
+#     # - checks if peer id can be used instead of scid
+#     # - checks fee_base_msat and fee_proportional_millionths in `listpeers` out
+#     DEF_BASE = 10
+#     DEF_BASE_MSAT = Millisatoshi(DEF_BASE)
+#     DEF_PPM = 100
+#     # Minus reserve
+#     MAX_HTLC = Millisatoshi(int(FUNDAMOUNT * 1000 * 0.99))
+# 
+#     l1, l2, l3 = node_factory.get_nodes(3,
+#                                         opts={'fee-base': DEF_BASE, 'fee-per-satoshi': DEF_PPM})
+#     node_factory.join_nodes([l1, l2])
+#     l1.rpc.connect(l3.info['id'], 'localhost', l3.port)
+# 
+#     def channel_get_config(scid):
+#         return l1.db.query(
+#             'SELECT feerate_base, feerate_ppm, htlc_minimum_msat, htlc_maximum_msat FROM channels '
+#             'WHERE scid={};'.format(scid_to_int(scid)))
+# 
+#     # get short channel id
+#     scid = l1.get_channel_scid(l2)
+# 
+#     # feerates should be init with global config
+#     db_fees = l1.db_query('SELECT feerate_base, feerate_ppm, htlc_maximum_msat FROM channels;')
+#     assert(db_fees[0]['feerate_base'] == DEF_BASE)
+#     assert(db_fees[0]['feerate_ppm'] == DEF_PPM)
+#     # This will be the capacity - reserves:
+#     assert(db_fees[0]['htlc_maximum_msat'] == MAX_HTLC)
+#     # this is also what listpeers should return
+#     channel = only_one(l1.rpc.listpeerchannels()['channels'])
+#     assert channel['fee_base_msat'] == DEF_BASE_MSAT
+#     assert channel['fee_proportional_millionths'] == DEF_PPM
+#     assert channel['maximum_htlc_out_msat'] == MAX_HTLC
+# 
+#     # custom setchannel scid <feebase> <feeppm> <htlcmin> <htlcmax>
+#     result = l1.rpc.setchannel(scid, 1337, 137, 17, 133337)
+# 
+#     # check result format
+#     assert(len(result['channels']) == 1)
+#     assert(re.match('^[0-9a-f]{64}$', result['channels'][0]['channel_id']))
+#     assert(result['channels'][0]['peer_id'] == l2.info['id'])
+#     assert(result['channels'][0]['short_channel_id'] == scid)
+#     assert(result['channels'][0]['fee_base_msat'] == 1337)
+#     assert(result['channels'][0]['fee_proportional_millionths'] == 137)
+#     assert(result['channels'][0]['minimum_htlc_out_msat'] == 17)
+#     assert(result['channels'][0]['maximum_htlc_out_msat'] == 133337)
+# 
+#     # check if custom values made it into the database
+#     db_fees = channel_get_config(scid)
+#     assert(db_fees[0]['feerate_base'] == 1337)
+#     assert(db_fees[0]['feerate_ppm'] == 137)
+#     assert(db_fees[0]['htlc_minimum_msat'] == 17)
+#     assert(db_fees[0]['htlc_maximum_msat'] == 133337)
+#     # also check for updated values in `listpeers`
+#     channel = only_one(l1.rpc.listpeerchannels()['channels'])
+#     assert channel['fee_base_msat'] == Millisatoshi(1337)
+#     assert channel['fee_proportional_millionths'] == 137
+#     assert channel['minimum_htlc_out_msat'] == 17
+#     assert channel['maximum_htlc_out_msat'] == 133337
+# 
+#     # wait for gossip and check if l1 sees new fees in listchannels
+#     wait_for(lambda: [c['base_fee_millisatoshi'] for c in l1.rpc.listchannels(scid)['channels']] == [DEF_BASE, 1337])
+#     wait_for(lambda: [c['fee_per_millionth'] for c in l1.rpc.listchannels(scid)['channels']] == [DEF_PPM, 137])
+#     wait_for(lambda: [c['htlc_minimum_msat'] for c in l1.rpc.listchannels(scid)['channels']] == [0, 17])
+#     wait_for(lambda: [c['htlc_maximum_msat'] for c in l1.rpc.listchannels(scid)['channels']] == [MAX_HTLC, 133337])
+# 
+#     # also test with named and missing parameters
+#     result = l1.rpc.setchannel(feeppm=42, id=scid)
+#     assert(len(result['channels']) == 1)
+#     assert(re.match('^[0-9a-f]{64}$', result['channels'][0]['channel_id']))
+#     assert(result['channels'][0]['short_channel_id'] == scid)
+#     assert(result['channels'][0]['fee_base_msat'] == 1337)
+#     assert(result['channels'][0]['fee_proportional_millionths'] == 42)
+#     assert result['channels'][0]['minimum_htlc_out_msat'] == 17
+#     assert(result['channels'][0]['maximum_htlc_out_msat'] == 133337)
+# 
+#     result = l1.rpc.setchannel(feebase=43, id=scid)
+#     assert(len(result['channels']) == 1)
+#     assert(re.match('^[0-9a-f]{64}$', result['channels'][0]['channel_id']))
+#     assert(result['channels'][0]['short_channel_id'] == scid)
+#     assert(result['channels'][0]['fee_base_msat'] == 43)
+#     assert(result['channels'][0]['fee_proportional_millionths'] == 42)
+#     assert result['channels'][0]['minimum_htlc_out_msat'] == 17
+#     assert(result['channels'][0]['maximum_htlc_out_msat'] == 133337)
+# 
+#     result = l1.rpc.setchannel(htlcmin=45, id=scid)
+#     assert(len(result['channels']) == 1)
+#     assert(re.match('^[0-9a-f]{64}$', result['channels'][0]['channel_id']))
+#     assert(result['channels'][0]['short_channel_id'] == scid)
+#     assert(result['channels'][0]['fee_base_msat'] == 43)
+#     assert(result['channels'][0]['fee_proportional_millionths'] == 42)
+#     assert result['channels'][0]['minimum_htlc_out_msat'] == 45
+#     assert(result['channels'][0]['maximum_htlc_out_msat'] == 133337)
+# 
+#     result = l1.rpc.setchannel(htlcmax=43333, id=scid)
+#     assert(len(result['channels']) == 1)
+#     assert(re.match('^[0-9a-f]{64}$', result['channels'][0]['channel_id']))
+#     assert(result['channels'][0]['short_channel_id'] == scid)
+#     assert(result['channels'][0]['fee_base_msat'] == 43)
+#     assert(result['channels'][0]['fee_proportional_millionths'] == 42)
+#     assert result['channels'][0]['minimum_htlc_out_msat'] == 45
+#     assert(result['channels'][0]['maximum_htlc_out_msat'] == 43333)
+# 
+#     # check if negative fees raise error and DB keeps values
+#     # JSONRPC2_INVALID_PARAMS := -32602
+#     from pyln.client import LightningRpc
+#     with pytest.raises(RpcError, match=r'-32602'):
+#         # Need to bypass pyln since it'd check args locally. We also
+#         # have to sidestep the schema validation, it attempts to
+#         # instantiate Millisatoshis and fails due to the non-negative
+#         # constraint.
+#         LightningRpc.call(l1.rpc, 'setchannel', {
+#             "id": scid,
+#             "feebase": -1,
+#             "feeppm": -1
+#         })
+# 
+#     # test if zero fees is possible
+#     result = l1.rpc.setchannel(scid, 0, 0)
+#     assert(result['channels'][0]['short_channel_id'] == scid)
+#     assert(result['channels'][0]['fee_base_msat'] == 0)
+#     assert(result['channels'][0]['fee_proportional_millionths'] == 0)
+# 
+#     db_fees = channel_get_config(scid)
+#     assert(db_fees[0]['feerate_base'] == 0)
+#     assert(db_fees[0]['feerate_ppm'] == 0)
+#     # also check for updated values in `listpeers`
+#     channel = only_one(l1.rpc.listpeerchannels()['channels'])
+#     assert channel['fee_base_msat'] == Millisatoshi(0)
+#     assert channel['fee_proportional_millionths'] == 0
+# 
+#     # check also peer id can be used
+#     result = l1.rpc.setchannel(l2.info['id'], 142, 143)
+#     assert(len(result['channels']) == 1)
+#     assert(result['channels'][0]['peer_id'] == l2.info['id'])
+#     assert(result['channels'][0]['short_channel_id'] == scid)
+#     assert(result['channels'][0]['fee_base_msat'] == 142)
+#     assert(result['channels'][0]['fee_proportional_millionths'] == 143)
+# 
+#     db_fees = channel_get_config(scid)
+#     assert(db_fees[0]['feerate_base'] == 142)
+#     assert(db_fees[0]['feerate_ppm'] == 143)
+# 
+#     # check if invalid scid raises proper error
+#     with pytest.raises(RpcError, match=r'-1.*Could not find any active channels of peer with that id'):
+#         result = l1.rpc.setchannel(l3.info['id'], 42, 43)
+#     with pytest.raises(RpcError, match=r'-32602.*id: should be a channel ID or short channel ID: invalid token'):
+#         result = l1.rpc.setchannel('f42' + scid[3:], 42, 43)
+# 
+#     # check if 'base' unit can be modified to satoshi
+#     result = l1.rpc.setchannel(scid, '1sat')
+#     assert(len(result['channels']) == 1)
+#     assert(result['channels'][0]['peer_id'] == l2.info['id'])
+#     assert(result['channels'][0]['short_channel_id'] == scid)
+#     assert(result['channels'][0]['fee_base_msat'] == 1000)
+#     db_fees = channel_get_config(scid)
+#     assert(db_fees[0]['feerate_base'] == 1000)
+# 
+#     # check if 'ppm' values greater than u32_max fail
+#     with pytest.raises(RpcError, match=r'-32602.*ppm: should be an integer: invalid token'):
+#         LightningRpc.call(l1.rpc, 'setchannel', payload={
+#             "id": scid,
+#             'feebase': 0,
+#             'feeppm': 2**32,
+#         })
+# 
+#     # check if 'base' values greater than u32_max fail
+#     with pytest.raises(RpcError, match=r'-32602.*base: exceeds u32 max: invalid token'):
+#         LightningRpc.call(l1.rpc, 'setchannel', payload={
+#             "id": scid,
+#             "feebase": 2**32,
+#         })
+# 
+# 
+# @pytest.mark.developer("gossip without DEVELOPER=1 is slow")
+# def test_setchannel_state(node_factory, bitcoind):
+#     # TEST SETUP
+#     #
+#     # [l0] --> [l1] --> [l2]
+#     #
+#     # Initiate channel [l1,l2] and try to set feerates other states than
+#     # CHANNELD_NORMAL or CHANNELD_AWAITING_LOCKIN. Should raise error.
+#     # Use l0 to make a forward through l1/l2 for testing.
+#     DEF_BASE = 0
+#     DEF_PPM = 0
+# 
+#     l0, l1, l2 = node_factory.get_nodes(3, opts={
+#         'fee-base': DEF_BASE,
+#         'fee-per-satoshi': DEF_PPM
+#     })
+# 
+#     # connection and funding
+#     l0.rpc.connect(l1.info['id'], 'localhost', l1.port)
+#     l0.fundchannel(l1, 1000000, wait_for_active=True)
+#     l1.rpc.connect(l2.info['id'], 'localhost', l2.port)
+#     scid, _ = l1.fundchannel(l2, 1000000, wait_for_active=False)
+# 
+#     # try setting the fee in state AWAITING_LOCKIN should be possible
+#     # assert(l1.channel_state(l2) == "CHANNELD_AWAITING_LOCKIN")
+#     result = l1.rpc.setchannel(l2.info['id'], 42, 0)
+#     assert(result['channels'][0]['peer_id'] == l2.info['id'])
+#     # cid = result['channels'][0]['channel_id']
+# 
+#     # test routing correct new fees once routing is established
+#     mine_funding_to_announce(bitcoind, [l0, l1, l2])
+# 
+#     l0.wait_for_route(l2)
+#     inv = l2.rpc.invoice(100000, 'test_setchannel_state', 'desc')['bolt11']
+#     result = l0.dev_pay(inv, use_shadow=False)
+#     assert result['status'] == 'complete'
+#     assert result['amount_sent_msat'] == 100042
+# 
+#     # Disconnect and unilaterally close from l2 to l1
+#     l2.rpc.disconnect(l1.info['id'], force=True)
+#     result = l2.rpc.close(scid, 1)
+#     assert result['type'] == 'unilateral'
+# 
+#     # wait for l1 to see unilateral close via bitcoin network
+#     while l1.channel_state(l2) == "CHANNELD_NORMAL":
+#         bitcoind.generate_block(1)
+#     # assert l1.channel_state(l2) == "FUNDING_SPEND_SEEN"
+# 
+#     # Try to setchannel in order to raise expected error.
+#     # To reduce false positive flakes, only test if state is not NORMAL anymore.
+#     with pytest.raises(RpcError, match=r'-1.*'):
+#         # l1.rpc.setchannel(l2.info['id'], 10, 1)
+#         l1.rpc.setchannel(l2.info['id'], 10, 1)
+# 
+# 
+# @pytest.mark.developer("gossip without DEVELOPER=1 is slow")
+# def test_setchannel_routing(node_factory, bitcoind):
+#     # TEST SETUP
+#     #
+#     # [l1] <--default_fees--> [l2] <--specific_fees--> [l3]
+#     #
+#     # - json listchannels is able to see the new values in foreign node
+#     # - routing calculates fees correctly
+#     # - payment can be done using specific fees
+#     # - channel specific fees can be disabled again
+#     # - payment can be done using global fees
+#     # - htlc max is honored
+#     DEF_BASE = 1
+#     DEF_PPM = 10
+#     MAX_HTLC = Millisatoshi(int(FUNDAMOUNT * 1000 * 0.99))
+#     MIN_HTLC = Millisatoshi(0)
+# 
+#     l1, l2, l3 = node_factory.line_graph(
+#         3, announce_channels=True, wait_for_announce=True,
+#         opts={'fee-base': DEF_BASE, 'fee-per-satoshi': DEF_PPM,
+#               'disable-mpp': None})
+# 
+#     # get short channel id for 2->3
+#     scid = l2.get_channel_scid(l3)
+# 
+#     # TEST CUSTOM VALUES
+#     l2.rpc.setchannel(scid, 1337, 137, 17, 4000000, enforcedelay=0)
+# 
+#     # wait for l1 to see updated channel via gossip
+#     wait_for(lambda: [c['base_fee_millisatoshi'] for c in l1.rpc.listchannels(scid)['channels']] == [1337, DEF_BASE])
+#     wait_for(lambda: [c['fee_per_millionth'] for c in l1.rpc.listchannels(scid)['channels']] == [137, DEF_PPM])
+#     wait_for(lambda: [c['htlc_minimum_msat'] for c in l1.rpc.listchannels(scid)['channels']] == [17, MIN_HTLC])
+#     wait_for(lambda: [c['htlc_maximum_msat'] for c in l1.rpc.listchannels(scid)['channels']] == [4000000, MAX_HTLC])
+# 
+#     # test fees are applied to HTLC forwards
+#     #
+#     # BOLT #7:
+#     # If l1 were to send 4,999,999 millisatoshi to l3 via l2, it needs to
+#     # pay l2 the fee it specified in the l2->l3 `channel_update`, calculated as
+#     # per [HTLC Fees](#htlc_fees):  base + amt * pm / 10**6
+# 
+#     # Note: we use fp16 internally for channel max, so we overestimate:
+#     # from devtools/fp16 4000000: fp16:5fa1 min 3999744 max 4001791
+#     # Since it rounds up, it will use 4001792 as max capacity.
+# 
+#     # Should refuse to route this!
+#     with pytest.raises(RpcError, match=r'Could not find a route'):
+#         l1.rpc.getroute(l3.info['id'], 4001793, 1, fuzzpercent=0)["route"]
+# 
+#     # We should consider this unroutable!  (MPP is disabled!)
+#     inv = l3.dev_invoice(amount_msat=4001793,
+#                          label='test_setchannel_1',
+#                          description='desc',
+#                          dev_routes=[])
+#     with pytest.raises(RpcError) as routefail:
+#         l1.dev_pay(inv['bolt11'], use_shadow=False)
+#     assert routefail.value.error['attempts'][0]['failreason'] == 'No path found'
+# 
+#     # 1337 + 4000000 * 137 / 1000000 = 1885
+#     route_ok = l1.rpc.getroute(l3.info['id'], 4000000, 1)["route"]
+#     assert len(route_ok) == 2
+#     assert route_ok[0]['amount_msat'] == 4001885
+#     assert route_ok[1]['amount_msat'] == 4000000
+# 
+#     # Make variant that tries to pay more than allowed htlc!
+#     route_bad = copy.deepcopy(route_ok)
+#     route_bad[0]['amount_msat'] = Millisatoshi(4001887)
+#     route_bad[1]['amount_msat'] = Millisatoshi(4000001)
+#     assert route_bad != route_ok
+# 
+#     # In case l3 includes a routehint, we need to make sure they also know
+#     # about the new fees, otherwise we may end up with the old feerate
+#     wait_for(lambda: [(c['base_fee_millisatoshi'], c['fee_per_millionth'], c['htlc_minimum_msat'], c['htlc_maximum_msat'], c['active']) for c in l3.rpc.listchannels(scid)['channels']] == [(1337, 137, 17, 4000000, True), (DEF_BASE, DEF_PPM, MIN_HTLC, MAX_HTLC, True)])
+# 
+#     # do and check actual payment
+#     inv = l3.rpc.invoice(4000000, 'test_setchannel_2', 'desc')
+#     # Check that routehint from l3 incorporated new feerate!
+#     decoded = l1.rpc.decodepay(inv['bolt11'])
+#     assert decoded['routes'] == [[{'pubkey': l2.info['id'], 'short_channel_id': scid, 'fee_base_msat': 1337, 'fee_proportional_millionths': 137, 'cltv_expiry_delta': 6}]]
+# 
+#     # This will fail.
+#     l1.rpc.sendpay(route_bad, inv['payment_hash'], payment_secret=inv['payment_secret'])
+#     with pytest.raises(RpcError, match='WIRE_TEMPORARY_CHANNEL_FAILURE'):
+#         l1.rpc.waitsendpay(inv['payment_hash'])
+# 
+#     # This will succeed
+#     l1.rpc.sendpay(route_ok, inv['payment_hash'], payment_secret=inv['payment_secret'])
+#     l1.rpc.waitsendpay(inv['payment_hash'])
+# 
+#     # Now try below minimum
+#     route_ok = l1.rpc.getroute(l3.info['id'], 17, 1)["route"]
+#     assert len(route_ok) == 2
+#     assert route_ok[0]['amount_msat'] == 1337 + 17
+#     assert route_ok[1]['amount_msat'] == 17
+# 
+#     route_bad = copy.deepcopy(route_ok)
+#     route_bad[0]['amount_msat'] = Millisatoshi(1337 + 16)
+#     route_bad[1]['amount_msat'] = Millisatoshi(16)
+#     assert route_bad != route_ok
+# 
+#     inv = l3.rpc.invoice(17, 'test_setchannel_3', 'desc')
+# 
+#     # This will fail.
+#     l1.rpc.sendpay(route_bad, inv['payment_hash'], payment_secret=inv['payment_secret'])
+#     with pytest.raises(RpcError, match='WIRE_TEMPORARY_CHANNEL_FAILURE'):
+#         l1.rpc.waitsendpay(inv['payment_hash'])
+# 
+#     # This will succeed
+#     l1.rpc.sendpay(route_ok, inv['payment_hash'], payment_secret=inv['payment_secret'])
+#     l1.rpc.waitsendpay(inv['payment_hash'])
+# 
+#     # Check that this one warns about capacity!
+#     inv = l3.rpc.call('invoice', {'amount_msat': 4001793,
+#                                   'label': 'test_setchannel_4',
+#                                   'description': 'desc'})
+#     assert 'warning_capacity' in inv
+# 
+# 
+# @pytest.mark.developer("gossip without DEVELOPER=1 is slow")
+# def test_setchannel_zero(node_factory, bitcoind):
+#     # TEST SETUP
+#     #
+#     # [l1] <--default_fees--> [l2] <--specific_fees--> [l3]
+#     #
+#     # - json listchannels is able to see the new values in foreign node
+#     # - routing calculates fees correctly
+#     # - payment can be done using zero fees
+#     DEF_BASE = 1
+#     DEF_PPM = 10
+#     MAX_HTLC = Millisatoshi(int(FUNDAMOUNT * 1000 * 0.99))
+# 
+#     l1, l2, l3 = node_factory.line_graph(
+#         3, announce_channels=True, wait_for_announce=True,
+#         opts={'fee-base': DEF_BASE, 'fee-per-satoshi': DEF_PPM})
+# 
+#     # get short channel id for 2->3
+#     scid = l2.get_channel_scid(l3)
+# 
+#     # TEST ZERO fees possible
+#     l2.rpc.setchannel(scid, 0, 0)
+#     wait_for(lambda: [c['base_fee_millisatoshi'] for c in l1.rpc.listchannels(scid)['channels']] == [0, DEF_BASE])
+#     wait_for(lambda: [c['fee_per_millionth'] for c in l1.rpc.listchannels(scid)['channels']] == [0, DEF_PPM])
+# 
+#     # test if zero fees are applied
+#     route = l1.rpc.getroute(l3.info['id'], 4999999, 1)["route"]
+#     assert len(route) == 2
+#     assert route[0]['amount_msat'] == 4999999
+#     assert route[1]['amount_msat'] == 4999999
+# 
+#     # Wait for l3 to know about our low-balling, otherwise they'll add a wrong
+#     # routehint to the invoice.
+#     wait_for(lambda: [(c['base_fee_millisatoshi'], c['fee_per_millionth'], c['active']) for c in l3.rpc.listchannels(scid)['channels']] == [(0, 0, True), (DEF_BASE, DEF_PPM, True)])
+# 
+#     # do and check actual payment
+#     inv = l3.rpc.invoice(4999999, 'test_setchannel_3', 'desc')['bolt11']
+#     result = l1.dev_pay(inv, use_shadow=False)
+#     assert result['status'] == 'complete'
+#     assert result['amount_sent_msat'] == 4999999
+# 
+#     # FIXME: hack something up to advertize min_htlc > 0, then test mintoolow.
+#     with pytest.raises(RpcError, match="htlcmax cannot be less than htlcmin"):
+#         l2.rpc.setchannel(scid, htlcmin=100000, htlcmax=99999)
+# 
+#     ret = l2.rpc.setchannel(scid, htlcmax=FUNDAMOUNT * 1000)
+#     assert 'warning_htlcmax_too_high' in only_one(ret['channels'])
+#     assert only_one(ret['channels'])['maximum_htlc_out_msat'] == MAX_HTLC
+# 
+# 
+# @pytest.mark.developer("gossip without DEVELOPER=1 is slow")
+# def test_setchannel_restart(node_factory, bitcoind):
+#     # TEST SETUP
+#     #
+#     # [l1] <--default_fees--> [l2] <--specific_fees--> [l3]
+#     #
+#     # - l2 sets fees to custom values and restarts
+#     # - l1 routing can be made with the custom fees
+#     # - l2 sets fees to UIN32_MAX (db update default) and restarts
+#     # - l1 routing can be made to l3 and global (1 10) fees are applied
+#     DEF_BASE = 1
+#     DEF_PPM = 10
+#     MIN_HTLC = Millisatoshi(0)
+#     MAX_HTLC = Millisatoshi(int(FUNDAMOUNT * 1000 * 0.99))
+#     OPTS = {'may_reconnect': True, 'fee-base': DEF_BASE, 'fee-per-satoshi': DEF_PPM}
+# 
+#     l1, l2, l3 = node_factory.line_graph(3, announce_channels=True, wait_for_announce=True, opts=OPTS)
+# 
+#     # get short channel idS
+#     scid12 = l1.get_channel_scid(l2)
+#     scid23 = l2.get_channel_scid(l3)
+# 
+#     # l2 set custom fees
+#     l2.rpc.setchannel(scid23, 1337, 137, 17, 500001)
+# 
+#     # restart l2 and reconnect
+#     l2.restart()
+#     l1.rpc.connect(l2.info['id'], 'localhost', l2.port)
+#     l2.rpc.connect(l3.info['id'], 'localhost', l3.port)
+# 
+#     # Make sure l1's gossipd registered channeld activating channel.
+#     wait_for(lambda: [c['active'] for c in l1.rpc.listchannels(scid12)['channels']] == [True, True])
+# 
+#     # l1 wait for channel update from l2
+#     wait_for(lambda: [(c['base_fee_millisatoshi'], c['fee_per_millionth'], c['htlc_minimum_msat'], c['htlc_maximum_msat'], c['active']) for c in l1.rpc.listchannels(scid23)['channels']] == [(1337, 137, 17, 500001, True), (DEF_BASE, DEF_PPM, MIN_HTLC, MAX_HTLC, True)])
+# 
+#     # In case l3 includes a routehint, we need to make sure they also know
+#     # about the new fees, otherwise we may end up with the old feerate
+#     wait_for(lambda: [(c['base_fee_millisatoshi'], c['fee_per_millionth'], c['htlc_minimum_msat'], c['htlc_maximum_msat'], c['active']) for c in l3.rpc.listchannels(scid23)['channels']] == [(1337, 137, 17, 500001, True), (DEF_BASE, DEF_PPM, MIN_HTLC, MAX_HTLC, True)])
+# 
+#     # l1 can make payment to l3 with custom fees being applied
+#     # Note: BOLT #7 math works out to 1405 msat fees
+#     inv = l3.rpc.invoice(499999, 'test_setchannel_1', 'desc')['bolt11']
+#     result = l1.dev_pay(inv, use_shadow=False)
+#     assert result['status'] == 'complete'
+#     assert result['amount_sent_msat'] == 501404
+# 
+# 
+# @pytest.mark.developer("updates are delayed without --dev-fast-gossip")
+# def test_setchannel_all(node_factory, bitcoind):
+#     # TEST SETUP
+#     #
+#     # [l1]----> [l2]
+#     #   |
+#     #   o-----> [l3]
+#     DEF_BASE = 10
+#     DEF_PPM = 100
+# 
+#     l1, l2, l3 = node_factory.get_nodes(3, opts={'fee-base': DEF_BASE, 'fee-per-satoshi': DEF_PPM})
+#     l1.rpc.connect(l2.info['id'], 'localhost', l2.port)
+#     l1.rpc.connect(l3.info['id'], 'localhost', l3.port)
+#     l1.fundchannel(l2, 1000000)
+#     l1.fundchannel(l3, 1000000)
+# 
+#     # get short channel id
+#     scid2 = l1.get_channel_scid(l2)
+#     scid3 = l1.get_channel_scid(l3)
+# 
+#     # now try to set all (two) channels using wildcard syntax
+#     result = l1.rpc.setchannel("all", 0xDEAD, 0xBEEF, 0xBAD, 0xCAFE)
+# 
+#     wait_for(lambda: [c['base_fee_millisatoshi'] for c in l1.rpc.listchannels(scid2)['channels']] == [DEF_BASE, 0xDEAD])
+#     wait_for(lambda: [c['fee_per_millionth'] for c in l1.rpc.listchannels(scid2)['channels']] == [DEF_PPM, 0xBEEF])
+#     wait_for(lambda: [c['base_fee_millisatoshi'] for c in l1.rpc.listchannels(scid3)['channels']] == [0xDEAD, DEF_BASE])
+#     wait_for(lambda: [c['fee_per_millionth'] for c in l1.rpc.listchannels(scid3)['channels']] == [0xBEEF, DEF_PPM])
+# 
+#     # Don't assume order!
+#     assert len(result['channels']) == 2
+#     if result['channels'][0]['peer_id'] == l3.info['id']:
+#         result['channels'] = [result['channels'][1], result['channels'][0]]
+#     assert result['channels'][0]['peer_id'] == l2.info['id']
+#     assert result['channels'][0]['short_channel_id'] == scid2
+#     assert result['channels'][0]['fee_base_msat'] == 0xDEAD
+#     assert result['channels'][0]['fee_proportional_millionths'] == 0xBEEF
+#     assert result['channels'][0]['minimum_htlc_out_msat'] == 0xBAD
+#     assert result['channels'][0]['maximum_htlc_out_msat'] == 0xCAFE
+#     assert result['channels'][1]['peer_id'] == l3.info['id']
+#     assert result['channels'][1]['short_channel_id'] == scid3
+#     assert result['channels'][1]['fee_base_msat'] == 0xDEAD
+#     assert result['channels'][1]['fee_proportional_millionths'] == 0xBEEF
+#     assert result['channels'][1]['minimum_htlc_out_msat'] == 0xBAD
+#     assert result['channels'][1]['maximum_htlc_out_msat'] == 0xCAFE
+# 
+# 
+# @pytest.mark.developer("updates are delayed without --dev-fast-gossip")
+# def test_setchannel_startup_opts(node_factory, bitcoind):
+#     """Tests that custom config/cmdline options are applied correctly when set
+#     """
+#     opts = {
+#         'fee-base': 2,
+#         'fee-per-satoshi': 3,
+#         'htlc-minimum-msat': '4msat',
+#         'htlc-maximum-msat': '5msat'
+#     }
+#     l1, l2 = node_factory.line_graph(2, opts=opts, wait_for_announce=True)
+# 
+#     result = l2.rpc.listchannels()['channels']
+#     assert result[0]['base_fee_millisatoshi'] == 2
+#     assert result[0]['fee_per_millionth'] == 3
+#     assert result[0]['htlc_minimum_msat'] == Millisatoshi(4)
+#     assert result[0]['htlc_maximum_msat'] == Millisatoshi(5)
+#     assert result[1]['base_fee_millisatoshi'] == 2
+#     assert result[1]['fee_per_millionth'] == 3
+#     assert result[1]['htlc_minimum_msat'] == Millisatoshi(4)
+#     assert result[1]['htlc_maximum_msat'] == Millisatoshi(5)
+# 
+# 
+# @pytest.mark.developer("gossip without DEVELOPER=1 is slow")
+# def test_channel_spendable(node_factory, bitcoind):
+#     """Test that spendable_msat is accurate"""
+#     sats = 10**6
+#     l1, l2 = node_factory.line_graph(2, fundamount=sats, wait_for_announce=True,
+#                                      opts={'plugin': os.path.join(os.getcwd(), 'tests/plugins/hold_invoice.py'), 'holdtime': '30'})
+# 
+#     inv = l2.rpc.invoice('any', 'inv', 'for testing')
+#     payment_hash = inv['payment_hash']
+# 
+#     # We should be able to spend this much, and not one msat more!
+#     amount = l1.rpc.listpeerchannels()['channels'][0]['spendable_msat']
+#     route = l1.rpc.getroute(l2.info['id'], amount + 1, riskfactor=1, fuzzpercent=0)['route']
+#     l1.rpc.sendpay(route, payment_hash, payment_secret=inv['payment_secret'])
+# 
+#     # This should fail locally with "capacity exceeded"
+#     with pytest.raises(RpcError, match=r"Capacity exceeded.*'erring_index': 0"):
+#         l1.rpc.waitsendpay(payment_hash, TIMEOUT)
+# 
+#     # Exact amount should succeed.
+#     route = l1.rpc.getroute(l2.info['id'], amount, riskfactor=1, fuzzpercent=0)['route']
+#     l1.rpc.sendpay(route, payment_hash, payment_secret=inv['payment_secret'])
+# 
+#     # Amount should drop to 0 once HTLC is sent; we have time, thanks to
+#     # hold_invoice.py plugin.
+#     wait_for(lambda: len(l1.rpc.listpeerchannels()['channels'][0]['htlcs']) == 1)
+#     assert l1.rpc.listpeerchannels()['channels'][0]['spendable_msat'] == Millisatoshi(0)
+#     l1.rpc.waitsendpay(payment_hash, TIMEOUT)
+# 
+#     # Make sure l2 thinks it's all over.
+#     wait_for(lambda: len(l2.rpc.listpeerchannels()['channels'][0]['htlcs']) == 0)
+#     # Now, reverse should work similarly.
+#     inv = l1.rpc.invoice('any', 'inv', 'for testing')
+#     payment_hash = inv['payment_hash']
+#     amount = l2.rpc.listpeerchannels()['channels'][0]['spendable_msat']
+# 
+#     # Turns out we won't route this, as it's over max - reserve:
+#     route = l2.rpc.getroute(l1.info['id'], amount + 1, riskfactor=1, fuzzpercent=0)['route']
+#     l2.rpc.sendpay(route, payment_hash, payment_secret=inv['payment_secret'])
+# 
+#     # This should fail locally with "capacity exceeded"
+#     with pytest.raises(RpcError, match=r"Capacity exceeded.*'erring_index': 0"):
+#         l2.rpc.waitsendpay(payment_hash, TIMEOUT)
+# 
+#     # Exact amount should succeed.
+#     route = l2.rpc.getroute(l1.info['id'], amount, riskfactor=1, fuzzpercent=0)['route']
+#     l2.rpc.sendpay(route, payment_hash, payment_secret=inv['payment_secret'])
+# 
+#     # Amount should drop to 0 once HTLC is sent; we have time, thanks to
+#     # hold_invoice.py plugin.
+#     wait_for(lambda: len(l2.rpc.listpeerchannels()['channels'][0]['htlcs']) == 1)
+#     assert l2.rpc.listpeerchannels()['channels'][0]['spendable_msat'] == Millisatoshi(0)
+#     l2.rpc.waitsendpay(payment_hash, TIMEOUT)
+# 
+# 
+# @pytest.mark.developer("gossip without DEVELOPER=1 is slow")
+# def test_channel_receivable(node_factory, bitcoind):
+#     """Test that receivable_msat is accurate"""
+#     sats = 10**6
+#     l1, l2 = node_factory.line_graph(2, fundamount=sats, wait_for_announce=True,
+#                                      opts={'plugin': os.path.join(os.getcwd(), 'tests/plugins/hold_invoice.py'), 'holdtime': '30'})
+# 
+#     inv = l2.rpc.invoice('any', 'inv', 'for testing')
+#     payment_hash = inv['payment_hash']
+# 
+#     # We should be able to receive this much, and not one msat more!
+#     amount = l2.rpc.listpeerchannels()['channels'][0]['receivable_msat']
+#     route = l1.rpc.getroute(l2.info['id'], amount + 1, riskfactor=1, fuzzpercent=0)['route']
+#     l1.rpc.sendpay(route, payment_hash, payment_secret=inv['payment_secret'])
+# 
+#     # This should fail locally with "capacity exceeded"
+#     with pytest.raises(RpcError, match=r"Capacity exceeded.*'erring_index': 0"):
+#         l1.rpc.waitsendpay(payment_hash, TIMEOUT)
+# 
+#     # Exact amount should succeed.
+#     route = l1.rpc.getroute(l2.info['id'], amount, riskfactor=1, fuzzpercent=0)['route']
+#     l1.rpc.sendpay(route, payment_hash, payment_secret=inv['payment_secret'])
+# 
+#     # Amount should drop to 0 once HTLC is sent; we have time, thanks to
+#     # hold_invoice.py plugin.
+#     wait_for(lambda: len(l2.rpc.listpeerchannels()['channels'][0]['htlcs']) == 1)
+#     assert l2.rpc.listpeerchannels()['channels'][0]['receivable_msat'] == Millisatoshi(0)
+#     l1.rpc.waitsendpay(payment_hash, TIMEOUT)
+# 
+#     # Make sure both think it's all over.
+#     wait_for(lambda: len(l1.rpc.listpeerchannels()['channels'][0]['htlcs']) == 0)
+#     wait_for(lambda: len(l2.rpc.listpeerchannels()['channels'][0]['htlcs']) == 0)
+#     # Now, reverse should work similarly.
+#     inv = l1.rpc.invoice('any', 'inv', 'for testing')
+#     payment_hash = inv['payment_hash']
+#     amount = l1.rpc.listpeerchannels()['channels'][0]['receivable_msat']
+# 
+#     # Turns out we won't route this, as it's over max - reserve:
+#     route = l2.rpc.getroute(l1.info['id'], amount + 1, riskfactor=1, fuzzpercent=0)['route']
+#     l2.rpc.sendpay(route, payment_hash, payment_secret=inv['payment_secret'])
+# 
+#     # This should fail locally with "capacity exceeded"
+#     with pytest.raises(RpcError, match=r"Capacity exceeded.*'erring_index': 0"):
+#         l2.rpc.waitsendpay(payment_hash, TIMEOUT)
+# 
+#     # Exact amount should succeed.
+#     route = l2.rpc.getroute(l1.info['id'], amount, riskfactor=1, fuzzpercent=0)['route']
+#     l2.rpc.sendpay(route, payment_hash, payment_secret=inv['payment_secret'])
+# 
+#     # Amount should drop to 0 once HTLC is sent; we have time, thanks to
+#     # hold_invoice.py plugin.
+#     wait_for(lambda: len(l1.rpc.listpeerchannels()['channels'][0]['htlcs']) == 1)
+#     assert l1.rpc.listpeerchannels()['channels'][0]['receivable_msat'] == Millisatoshi(0)
+#     l2.rpc.waitsendpay(payment_hash, TIMEOUT)
+# 
+# 
+# @pytest.mark.developer("gossip without DEVELOPER=1 is slow")
+# def test_channel_spendable_large(node_factory, bitcoind):
+#     """Test that spendable_msat is accurate for large channels"""
+#     # This is almost the max allowable spend.
+#     sats = 4294967
+#     l1, l2 = node_factory.line_graph(
+#         2,
+#         fundamount=sats,
+#         wait_for_announce=True,
+#         opts={
+#             'plugin': os.path.join(os.getcwd(), 'tests/plugins/hold_invoice.py'),
+#             'holdtime': '30'
+#         }
+#     )
+# 
+#     inv = l2.rpc.invoice('any', 'inv', 'for testing')
+#     payment_hash = inv['payment_hash']
+# 
+#     # We should be able to spend this much, and not one msat more!
+#     spendable = l1.rpc.listpeerchannels()['channels'][0]['spendable_msat']
+# 
+#     # receivable from the other side should calculate to the exact same amount
+#     receivable = l2.rpc.listpeerchannels()['channels'][0]['receivable_msat']
+#     assert spendable == receivable
+# 
+#     # route or waitsendpay fill fail.
+#     with pytest.raises(RpcError):
+#         route = l1.rpc.getroute(l2.info['id'], spendable + 1, riskfactor=1, fuzzpercent=0)['route']
+#         l1.rpc.sendpay(route, payment_hash, payment_secret=inv['payment_secret'])
+#         l1.rpc.waitsendpay(payment_hash, TIMEOUT)
+# 
+#     # Exact amount should succeed.
+#     route = l1.rpc.getroute(l2.info['id'], spendable, riskfactor=1, fuzzpercent=0)['route']
+#     l1.rpc.sendpay(route, payment_hash, payment_secret=inv['payment_secret'])
+#     l1.rpc.waitsendpay(payment_hash, TIMEOUT)
+# 
+# 
+# def test_channel_spendable_receivable_capped(node_factory, bitcoind):
+#     """Test that spendable_msat and receivable_msat is capped at 2^32-1"""
+#     sats = 16777215
+#     l1, l2 = node_factory.line_graph(2, fundamount=sats, wait_for_announce=False)
+#     assert l1.rpc.listpeerchannels()['channels'][0]['spendable_msat'] == Millisatoshi(0xFFFFFFFF)
+#     assert l2.rpc.listpeerchannels()['channels'][0]['receivable_msat'] == Millisatoshi(0xFFFFFFFF)
+# 
+# 
+# @unittest.skipIf(True, "Test is extremely flaky")
+# @unittest.skipIf(not DEVELOPER and VALGRIND, "Doesn't raise exception, needs better sync")
+# def test_lockup_drain(node_factory, bitcoind):
+#     """Try to get channel into a state where opener can't afford fees on additional HTLC, so peer can't add HTLC"""
+#     l1, l2 = node_factory.line_graph(2, opts={'may_reconnect': True})
+# 
+#     # l1 sends all the money to l2 until even 1 msat can't get through.
+#     total = l1.drain(l2)
+# 
+#     # Even if feerate now increases 2x (30000), l2 should be able to send
+#     # non-dust HTLC to l1.
+#     l1.force_feerates(30000)
+#     l2.pay(l1, total // 2)
+# 
+#     # reset fees and send all back again
+#     l1.force_feerates(15000)
+#     l1.drain(l2)
+# 
+#     # But if feerate increase just a little more, l2 should not be able to send
+#     # non-fust HTLC to l1
+#     l1.force_feerates(30002)  # TODO: Why does 30001 fail? off by one in C code?
+#     wait_for(lambda: l1.rpc.listpeers()['peers'][0]['connected'])
+#     with pytest.raises(RpcError, match=r".*Capacity exceeded.*"):
+#         l2.pay(l1, total // 2)
+# 
+# 
+# @pytest.mark.developer("needs DEVELOPER=1 for dev_ignore_htlcs")
+# def test_htlc_too_dusty_outgoing(node_factory, bitcoind, chainparams):
+#     """ Try to hit the 'too much dust' limit, should fail the HTLC """
+#     feerate = 10000
+# 
+#     # elements txs are bigger so they become dusty faster
+#     max_dust_limit_sat = 100000 if chainparams['elements'] else 50000
+#     non_dust_htlc_val_sat = 20000 if chainparams['elements'] else 10000
+#     htlc_val_sat = 10000 if chainparams['elements'] else 5000
+# 
+#     l1, l2 = node_factory.line_graph(2, opts={'may_reconnect': True,
+#                                               'feerates': (feerate, feerate, feerate, feerate),
+#                                               'max-dust-htlc-exposure-msat': '{}sat'.format(max_dust_limit_sat),
+#                                               'allow_warning': True})
+# 
+#     # l2 holds all of l1's htlcs hostage
+#     l2.rpc.dev_ignore_htlcs(id=l1.info['id'], ignore=True)
+# 
+#     # l2's max dust limit is set to 100k
+#     htlc_val_msat = htlc_val_sat * 1000
+#     num_dusty_htlcs = max_dust_limit_sat // htlc_val_sat
+# 
+#     # add a some non-dusty htlcs, these will fail when we raise the dust limit
+#     route = l1.rpc.getroute(l2.info['id'], non_dust_htlc_val_sat * 1000, 1)['route']
+#     for i in range(0, 3):
+#         inv = l2.rpc.invoice((non_dust_htlc_val_sat * 1000), str(i + 100), str(i + 100))
+#         l1.rpc.sendpay(route, inv['payment_hash'], payment_secret=inv['payment_secret'])
+#         l2.daemon.wait_for_log(r'their htlc .* dev_ignore_htlcs')
+#         res = only_one(l1.rpc.listsendpays(payment_hash=inv['payment_hash'])['payments'])
+#         assert res['status'] == 'pending'
+# 
+#     # add some dusty-htlcs
+#     route = l1.rpc.getroute(l2.info['id'], htlc_val_msat, 1)['route']
+#     for i in range(0, num_dusty_htlcs):
+#         inv = l2.rpc.invoice(htlc_val_msat, str(i), str(i))
+#         l1.rpc.sendpay(route, inv['payment_hash'], payment_secret=inv['payment_secret'])
+#         l2.daemon.wait_for_log(r'their htlc .* dev_ignore_htlcs')
+#         res = only_one(l1.rpc.listsendpays(payment_hash=inv['payment_hash'])['payments'])
+#         assert res['status'] == 'pending'
+# 
+#     # one more should tip it over, and return a payment failure
+#     inv = l2.rpc.invoice(htlc_val_msat, str(num_dusty_htlcs), str(num_dusty_htlcs))
+#     l1.rpc.sendpay(route, inv['payment_hash'], payment_secret=inv['payment_secret'])
+#     l1.daemon.wait_for_log('CHANNEL_ERR_DUST_FAILURE')
+#     wait_for(lambda: only_one(l1.rpc.listsendpays(payment_hash=inv['payment_hash'])['payments'])['status'] == 'failed')
+# 
+#     # but we can still add a non dust htlc
+#     route = l1.rpc.getroute(l2.info['id'], non_dust_htlc_val_sat * 1000, 1)['route']
+#     inv = l2.rpc.invoice((10000 * 1000), str(120), str(120))
+#     l1.rpc.sendpay(route, inv['payment_hash'], payment_secret=inv['payment_secret'])
+#     l2.daemon.wait_for_log(r'their htlc .* dev_ignore_htlcs')
+#     res = only_one(l1.rpc.listsendpays(payment_hash=inv['payment_hash'])['payments'])
+#     assert res['status'] == 'pending'
+# 
+#     # Ok, adjust our feerate upward, so the non-dust htlcs are now dust
+#     # note that this is above the buffer we've been keeping, so the channel
+#     # should automatically fail
+#     l1.set_feerates([feerate * 2] * 4, False)
+#     l1.restart()
+# 
+#     # Make sure fails before we try sending htlc!
+#     l1.daemon.wait_for_log('Too much dust to update fee')
+# 
+#     # the channel should start warning -- too much dust
+#     inv = l2.rpc.invoice(htlc_val_msat, str(num_dusty_htlcs + 1), str(num_dusty_htlcs + 1))
+#     with pytest.raises(RpcError, match=r'WIRE_TEMPORARY_CHANNEL_FAILURE'):
+#         l1.rpc.sendpay(route, inv['payment_hash'], payment_secret=inv['payment_secret'])
+# 
+# 
+# @pytest.mark.developer("needs DEVELOPER=1 for dev_ignore_htlcs")
+# def test_htlc_too_dusty_incoming(node_factory, bitcoind):
+#     """ Try to hit the 'too much dust' limit, should fail the HTLC """
+#     feerate = 30000
+#     l1, l2, l3 = node_factory.line_graph(3, opts=[{'may_reconnect': True,
+#                                                    'feerates': (feerate, feerate, feerate, feerate),
+#                                                    'max-dust-htlc-exposure-msat': '200000sat'},
+#                                                   {'may_reconnect': True,
+#                                                    'feerates': (feerate, feerate, feerate, feerate),
+#                                                    'max-dust-htlc-exposure-msat': '100000sat',
+#                                                    'fee-base': 0,
+#                                                    'fee-per-satoshi': 0},
+#                                                   {'max-dust-htlc-exposure-msat': '500000sat'}],
+#                                          wait_for_announce=True)
+# 
+#     # on the l2->l3, and l3 holds all the htlcs hostage
+#     # have l3 hold onto all the htlcs and not fulfill them
+#     l3.rpc.dev_ignore_htlcs(id=l2.info['id'], ignore=True)
+# 
+#     # l2's max dust limit is set to 100k
+#     max_dust_limit_sat = 100000
+#     htlc_val_sat = 10000
+#     htlc_val_msat = htlc_val_sat * 1000
+#     num_dusty_htlcs = max_dust_limit_sat // htlc_val_sat
+#     route = l1.rpc.getroute(l3.info['id'], htlc_val_msat, 1)['route']
+# 
+#     # l1 sends as much money as it can
+#     for i in range(0, num_dusty_htlcs):
+#         inv = l3.rpc.invoice(htlc_val_msat, str(i), str(i))
+#         l1.rpc.sendpay(route, inv['payment_hash'], payment_secret=inv['payment_secret'])
+#         l3.daemon.wait_for_log(r'their htlc .* dev_ignore_htlcs')
+#         res = only_one(l1.rpc.listsendpays(payment_hash=inv['payment_hash'])['payments'])
+#         assert res['status'] == 'pending'
+# 
+#     # one more should tip it over, and return a payment failure
+#     inv = l3.rpc.invoice(htlc_val_msat, str(num_dusty_htlcs), str(num_dusty_htlcs))
+#     l1.rpc.sendpay(route, inv['payment_hash'], payment_secret=inv['payment_secret'])
+#     l2.daemon.wait_for_log('failing immediately, as requested')
+#     wait_for(lambda: only_one(l1.rpc.listsendpays(payment_hash=inv['payment_hash'])['payments'])['status'] == 'failed')
+# 
+# 
+# def test_error_returns_blockheight(node_factory, bitcoind):
+#     """Test that incorrect_or_unknown_payment_details returns block height"""
+#     l1, l2 = node_factory.line_graph(2)
+# 
+#     l1.rpc.sendpay([{'amount_msat': 100,
+#                      'id': l2.info['id'],
+#                      'delay': 10,
+#                      'channel': l1.get_channel_scid(l2)}],
+#                    '00' * 32, payment_secret='00' * 32)
+# 
+#     with pytest.raises(RpcError, match=r"INCORRECT_OR_UNKNOWN_PAYMENT_DETAILS.*'erring_index': 1") as err:
+#         l1.rpc.waitsendpay('00' * 32, TIMEOUT)
+# 
+#     # BOLT #4:
+#     # 1. type: PERM|15 (`incorrect_or_unknown_payment_details`)
+#     # 2. data:
+#     #    * [`u64`:`htlc_msat`]
+#     #    * [`u32`:`height`]
+#     assert (err.value.error['data']['raw_message']
+#             == '400f{:016x}{:08x}'.format(100, bitcoind.rpc.getblockcount()))
+# 
+# 
+# @pytest.mark.developer('Needs dev-routes')
+# def test_tlv_or_legacy(node_factory, bitcoind):
+#     # Ideally we'd test with l2 NOT var-onion, but then it can no longer connect
+#     # to us!
+#     l1, l2, l3 = node_factory.line_graph(3,
+#                                          opts={'plugin': os.path.join(os.getcwd(), 'tests/plugins/print_htlc_onion.py')})
+# 
+#     scid12 = l1.get_channel_scid(l2)
+#     scid23 = l2.get_channel_scid(l3)
+# 
+#     # We need to force l3 to provide route hint from l2 (it won't normally,
+#     # since it sees l2 as a dead end).
+#     inv = l3.dev_invoice(amount_msat=10000,
+#                          label="test_tlv1",
+#                          description="test_tlv1",
+#                          dev_routes=[[{'id': l2.info['id'],
+#                                        'short_channel_id': scid23,
+#                                        'fee_base_msat': 1,
+#                                        'fee_proportional_millionths': 10,
+#                                        'cltv_expiry_delta': 6}]])['bolt11']
+#     l1.rpc.pay(inv)
+# 
+#     # Since L1 hasn't seen broadcast, it doesn't know L2 isn't TLV, but invoice tells it about L3
+#     l2.daemon.wait_for_log("Got onion.*'type': 'tlv'")
+#     l3.daemon.wait_for_log("Got onion.*'type': 'tlv'")
+# 
+#     # We need 5 more blocks to announce l1->l2 channel.
+#     mine_funding_to_announce(bitcoind, [l1, l2, l3])
+# 
+#     # Make sure l1 knows about l2
+#     wait_for(lambda: 'alias' in l1.rpc.listnodes(l2.info['id'])['nodes'][0])
+# 
+#     # Make sure l3 knows about l1->l2, so it will add route hint now.
+#     wait_for(lambda: len(l3.rpc.listchannels(scid12)['channels']) > 0)
+# 
+#     # Now it should send TLV to l2.
+#     inv = l3.rpc.invoice(10000, "test_tlv2", "test_tlv2")['bolt11']
+# 
+#     l1.rpc.pay(inv)
+#     l2.daemon.wait_for_log("Got onion.*'type': 'tlv'")
+#     l3.daemon.wait_for_log("Got onion.*'type': 'tlv'")
+# 
+# 
+# @unittest.skipIf(TEST_NETWORK != 'regtest', "Invoice is network specific")
+# def test_pay_no_secret(node_factory, bitcoind):
+#     l1, l2 = node_factory.line_graph(2, wait_for_announce=True)
+# 
+#     l2.rpc.invoice(100000, "test_pay_no_secret", "test_pay_no_secret",
+#                    preimage='00' * 32, expiry=2000000000)
+# 
+#     # Produced from modified version (different secret!).
+#     inv_badsecret = 'lnbcrt1u1pwuedm6pp5ve584t0cv27hwmy0cx9ca8uwyqyfw9y9dm3r8vus9fv36r2l9yjsdqaw3jhxazlwpshjhmwda0hxetrwfjhgxq8pmnt9qqcqp9sp52au0npwmw4xxv2rfrat04kh9p3jlmklgavhfxqukx0l05pw5tccs9qypqsqa286dmt2xh3jy8cd8ndeyr845q8a7nhgjkerdqjns76jraux6j25ddx9f5k5r2ey0kk942x3uhaff66794kyjxxcd48uevf7p6ja53gqjj5ur7'
+#     with pytest.raises(RpcError, match=r"INCORRECT_OR_UNKNOWN_PAYMENT_DETAILS.*'erring_index': 1"):
+#         l1.rpc.pay(inv_badsecret)
+# 
+#     # Produced from old version (no secret!)
+#     inv_nosecret = 'lnbcrt1u1pwue4vapp5ve584t0cv27hwmy0cx9ca8uwyqyfw9y9dm3r8vus9fv36r2l9yjsdqaw3jhxazlwpshjhmwda0hxetrwfjhgxq8pmnt9qqcqp9570xsjyykvssa6ty8fjth6f2y8h09myngad9utesttwjwclv95fz3lgd402f9e5yzpnxmkypg55rkvpg522gcz4ymsjl2w3m4jhw4jsp55m7tl'
+#     with pytest.raises(RpcError, match=r"INCORRECT_OR_UNKNOWN_PAYMENT_DETAILS.*'erring_index': 1"):
+#         l1.rpc.pay(inv_nosecret)
+# 
+# 
+# def test_shadow_routing(node_factory):
+#     """
+#     Test the value randomization through shadow routing
+# 
+#     Note there is a very low (0.5**10) probability that it fails.
+#     """
+#     # We need l3 for random walk
+#     l1, l2, l3 = node_factory.line_graph(3, wait_for_announce=True)
+# 
+#     amount = 10000
+#     total_amount = 0
+#     n_payments = 10
+#     for i in range(n_payments):
+#         inv = l3.rpc.invoice(amount, "{}".format(i), "test")["bolt11"]
+#         total_amount += l1.rpc.pay(inv)["amount_sent_msat"]
+# 
+#     assert total_amount.millisatoshis > n_payments * amount
+#     # Test that the added amount isn't absurd
+#     assert total_amount.millisatoshis < (n_payments * amount) * (1 + 0.01)
+# 
+#     # FIXME: Test cltv delta too ?
+# 
+# 
+# def test_createonion_rpc(node_factory):
+#     l1 = node_factory.get_node()
+# 
+#     # From bolt04/onion-test.json:
+#     hops = [{
+#         "pubkey": "02eec7245d6b7d2ccb30380bfbe2a3648cd7a942653f5aa340edcea1f283686619",
+#         "payload": "1202023a98040205dc06080000000000000001"
+#     }, {
+#         "pubkey": "0324653eac434488002cc06bbfb7f10fe18991e35f9fe4302dbea6d2353dc0ab1c",
+#         "payload": "52020236b00402057806080000000000000002fd02013c0102030405060708090a0b0c0d0e0f0102030405060708090a0b0c0d0e0f0102030405060708090a0b0c0d0e0f0102030405060708090a0b0c0d0e0f"
+#     }, {
+#         "pubkey": "027f31ebc5462c1fdce1b737ecff52d37d75dea43ce11c74d25aa297165faa2007",
+#         "payload": "12020230d4040204e206080000000000000003"
+#     }, {
+#         "pubkey": "032c0b7cf95324a07d05398b240174dc0c2be444d96b159aa6c7f7b1e668680991",
+#         "payload": "1202022710040203e806080000000000000004"
+#     }, {
+#         "pubkey": "02edabbd16b41c8371b92ef2f04c1185b4f03b6dcd52ba9b78d9d7c89c8f221145",
+#         "payload": "fd011002022710040203e8082224a33562c54507a9334e79f0dc4f17d407e6d7c61f0e2f3d0d38599502f617042710fd012de02a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a"
+#     }]
+# 
+#     res = l1.rpc.createonion(hops=hops, assocdata="BB" * 32)
+#     assert(len(res['onion']) == 2 * 1366)
+#     assert(len(res['shared_secrets']) == len(hops))
+# 
+#     res = l1.rpc.createonion(hops=hops, assocdata="42" * 32,
+#                              session_key="41" * 32)
+#     # The trailer is generated using the filler and can be ued as a
+#     # checksum. This trailer is from the test-vector in the specs.
+#     assert(res['onion'].endswith('9126aaefb627719f421e20'))
+# 
+# 
+# @pytest.mark.developer("gossip propagation is slow without DEVELOPER=1")
+# def test_sendonion_rpc(node_factory):
+#     l1, l2, l3, l4 = node_factory.line_graph(4, wait_for_announce=True)
+#     amt = 10**3
+#     route = l1.rpc.getroute(l4.info['id'], 10**3, 10)['route']
+#     inv = l4.rpc.invoice(amt, "lbl", "desc")
+# 
+#     first_hop = route[0]
+#     blockheight = l1.rpc.getinfo()['blockheight']
+# 
+#     def truncate_encode(i: int):
+#         """Encode a tu64 (or tu32 etc) value"""
+#         ret = struct.pack("!Q", i)
+#         while ret.startswith(b'\0'):
+#             ret = ret[1:]
+#         return ret
+# 
+#     def serialize_payload_tlv(n):
+#         block, tx, out = n['channel'].split('x')
+# 
+#         payload = TlvPayload()
+#         # BOLT #4:
+#         #     1. type: 2 (`amt_to_forward`)
+#         #     2. data:
+#         #         * [`tu64`:`amt_to_forward`]
+#         b = BytesIO()
+#         b.write(truncate_encode(int(n['amount_msat'])))
+#         payload.add_field(2, b.getvalue())
+#         # BOLT #4:
+#         #    1. type: 4 (`outgoing_cltv_value`)
+#         #    2. data:
+#         #        * [`tu32`:`outgoing_cltv_value`]
+#         b = BytesIO()
+#         b.write(truncate_encode(blockheight + n['delay']))
+#         payload.add_field(4, b.getvalue())
+#         # BOLT #4:
+#         #    1. type: 6 (`short_channel_id`)
+#         #    2. data:
+#         #        * [`short_channel_id`:`short_channel_id`]
+#         b = BytesIO()
+#         b.write(struct.pack("!Q", int(block) << 40 | int(tx) << 16 | int(out)))
+#         payload.add_field(6, b.getvalue())
+#         return payload.to_bytes().hex()
+# 
+#     def serialize_payload_final_tlv(n, payment_secret: str):
+#         payload = TlvPayload()
+#         # BOLT #4:
+#         #     1. type: 2 (`amt_to_forward`)
+#         #     2. data:
+#         #         * [`tu64`:`amt_to_forward`]
+#         b = BytesIO()
+#         b.write(truncate_encode(int(n['amount_msat'])))
+#         payload.add_field(2, b.getvalue())
+#         # BOLT #4:
+#         #    1. type: 4 (`outgoing_cltv_value`)
+#         #    2. data:
+#         #        * [`tu32`:`outgoing_cltv_value`]
+#         b = BytesIO()
+#         b.write(truncate_encode(blockheight + n['delay']))
+#         payload.add_field(4, b.getvalue())
+#         # BOLT #4:
+#         #    1. type: 8 (`payment_data`)
+#         #    2. data:
+#         #        * [`32*byte`:`payment_secret`]
+#         #        * [`tu64`:`total_msat`]
+#         b = BytesIO()
+#         b.write(bytes.fromhex(payment_secret))
+#         b.write(truncate_encode(int(n['amount_msat'])))
+#         payload.add_field(8, b.getvalue())
+#         return payload.to_bytes().hex()
+# 
+#     # Need to shift the parameters by one hop
+#     hops = []
+#     for h, n in zip(route[:-1], route[1:]):
+#         # We tell the node h about the parameters to use for n (a.k.a. h + 1)
+#         hops.append({
+#             "pubkey": h['id'],
+#             "payload": serialize_payload_tlv(n)
+#         })
+# 
+#     # The last hop has a special payload:
+#     hops.append({
+#         "pubkey": route[-1]['id'],
+#         "payload": serialize_payload_final_tlv(route[-1], inv['payment_secret'])
+#     })
+# 
+#     onion = l1.rpc.createonion(hops=hops, assocdata=inv['payment_hash'])
+# 
+#     l1.rpc.sendonion(onion=onion['onion'], first_hop=first_hop,
+#                      payment_hash=inv['payment_hash'])
+# 
+#     l1.rpc.waitsendpay(payment_hash=inv['payment_hash'])
+#     invs = l4.rpc.listinvoices(label="lbl")['invoices']
+#     assert(len(invs) == 1 and invs[0]['status'] == 'paid')
+# 
+#     pays = l1.rpc.listsendpays()['payments']
+#     assert(len(pays) == 1 and pays[0]['status'] == 'complete'
+#            and pays[0]['payment_hash'] == inv['payment_hash'])
+# 
+#     # And now for a failing payment, using a payment_hash that doesn't match an
+#     # invoice
+#     payment_hash = "00" * 32
+#     onion = l1.rpc.createonion(hops=hops, assocdata=payment_hash)
+#     l1.rpc.sendonion(onion=onion['onion'], first_hop=first_hop,
+#                      payment_hash=payment_hash)
+# 
+#     try:
+#         l1.rpc.waitsendpay(payment_hash=payment_hash)
+#         raise ValueError()
+#     except RpcError as e:
+#         assert(e.error['code'] == 202)
+#         assert(e.error['message'] == "Malformed error reply")
+# 
+#     pays = l1.rpc.listsendpays(payment_hash=payment_hash)['payments']
+#     assert(len(pays) == 1 and pays[0]['status'] == 'failed'
+#            and pays[0]['payment_hash'] == payment_hash)
+#     assert('erroronion' in pays[0])
+# 
+#     # Fail onion is msg + padding = 256 + 2*2 byte lengths + 32 byte HMAC
+#     assert(len(pays[0]['erroronion']) == (256 + 32 + 2 + 2) * 2)
+# 
+#     # Let's try that again, this time we give it the shared_secrets so it
+#     # should be able to decode the error.
+#     payment_hash = "01" * 32
+#     onion = l1.rpc.createonion(hops=hops, assocdata=payment_hash)
+#     l1.rpc.sendonion(onion=onion['onion'], first_hop=first_hop,
+#                      payment_hash=payment_hash,
+#                      shared_secrets=onion['shared_secrets'])
+# 
+#     try:
+#         l1.rpc.waitsendpay(payment_hash=payment_hash)
+#     except RpcError as e:
+#         assert(e.error['code'] == 204)
+#         assert(e.error['data']['raw_message'] == "400f00000000000003e80000006c")
+# 
+# 
+# @pytest.mark.developer("needs dev-disable-commit-after, dev-no-htlc-timeout")
+# @pytest.mark.openchannel('v1')
+# @pytest.mark.openchannel('v2')
+# def test_partial_payment(node_factory, bitcoind, executor):
+#     # We want to test two payments at the same time, before we send commit
+#     l1, l2, l3, l4 = node_factory.get_nodes(4, [{}] + [{'dev-disable-commit-after': 0, 'dev-no-htlc-timeout': None}] * 2 + [{'plugin': os.path.join(os.getcwd(), 'tests/plugins/print_htlc_onion.py')}])
+# 
+#     # Two routes to l4: one via l2, and one via l3.
+#     l1.rpc.connect(l2.info['id'], 'localhost', l2.port)
+#     l1.fundchannel(l2, 100000)
+#     l1.rpc.connect(l3.info['id'], 'localhost', l3.port)
+#     l1.fundchannel(l3, 100000)
+#     l2.rpc.connect(l4.info['id'], 'localhost', l4.port)
+#     scid24, _ = l2.fundchannel(l4, 100000)
+#     l3.rpc.connect(l4.info['id'], 'localhost', l4.port)
+#     scid34, _ = l3.fundchannel(l4, 100000)
+#     mine_funding_to_announce(bitcoind, [l1, l2, l3, l4])
+# 
+#     # Wait until l1 knows about all channels.
+#     wait_for(lambda: len(l1.rpc.listchannels()['channels']) == 8)
+# 
+#     inv = l4.rpc.invoice(1000, 'inv', 'inv')
+#     paysecret = l4.rpc.decodepay(inv['bolt11'])['payment_secret']
+# 
+#     # Separate routes for each part of the payment.
+#     r134 = l1.rpc.getroute(l4.info['id'], 501, 1, exclude=[scid24 + '/0', scid24 + '/1'])['route']
+#     r124 = l1.rpc.getroute(l4.info['id'], 499, 1, exclude=[scid34 + '/0', scid34 + '/1'])['route']
+# 
+#     # These can happen in parallel.
+#     l1.rpc.sendpay(
+#         route=r134,
+#         payment_hash=inv['payment_hash'],
+#         amount_msat=1000,
+#         bolt11=inv['bolt11'],
+#         payment_secret=paysecret,
+#         partid=1,
+#         groupid=1
+#     )
+# 
+#     # Can't mix non-parallel payment!
+#     with pytest.raises(RpcError, match=r'Already have parallel payment in progress'):
+#         l1.rpc.sendpay(
+#             route=r124,
+#             payment_hash=inv['payment_hash'],
+#             amount_msat=499,
+#             payment_secret=paysecret,
+#             groupid=1,
+#         )
+# 
+#     # It will not allow a parallel with different msatoshi!
+#     with pytest.raises(RpcError, match=r'msatoshi was previously 1000msat, now 999msat'):
+#         l1.rpc.sendpay(
+#             route=r124,
+#             payment_hash=inv['payment_hash'],
+#             amount_msat=999,
+#             bolt11=inv['bolt11'],
+#             payment_secret=paysecret,
+#             partid=2,
+#             groupid=1,
+#         )
+# 
+#     # This will work fine.
+#     l1.rpc.sendpay(
+#         route=r124,
+#         payment_hash=inv['payment_hash'],
+#         amount_msat=1000,
+#         bolt11=inv['bolt11'],
+#         payment_secret=paysecret,
+#         partid=2,
+#         groupid=1,
+#     )
+# 
+#     # Any more would exceed total payment
+#     with pytest.raises(RpcError, match=r'Already have 1000msat of 1000msat payments in progress'):
+#         l1.rpc.sendpay(
+#             route=r124,
+#             payment_hash=inv['payment_hash'],
+#             amount_msat=1000,
+#             bolt11=inv['bolt11'],
+#             payment_secret=paysecret,
+#             partid=3,
+#             groupid=1,
+#         )
+# 
+#     # But repeat is a NOOP, as long as they're exactly the same!
+#     with pytest.raises(RpcError, match=r'Already pending with amount 501msat \(not 499msat\)'):
+#         l1.rpc.sendpay(
+#             route=r124,
+#             payment_hash=inv['payment_hash'],
+#             amount_msat=1000,
+#             bolt11=inv['bolt11'],
+#             payment_secret=paysecret,
+#             partid=1,
+#             groupid=1,
+#         )
+# 
+#     l1.rpc.sendpay(
+#         route=r134,
+#         payment_hash=inv['payment_hash'],
+#         amount_msat=1000,
+#         bolt11=inv['bolt11'],
+#         payment_secret=paysecret,
+#         partid=1,
+#         groupid=1,
+#     )
+# 
+#     l1.rpc.sendpay(
+#         route=r124,
+#         payment_hash=inv['payment_hash'],
+#         amount_msat=1000,
+#         bolt11=inv['bolt11'],
+#         payment_secret=paysecret,
+#         partid=2,
+#         groupid=1,
+#     )
+# 
+#     # Make sure they've got the HTLCs before we unsuppress
+#     l2.daemon.wait_for_logs('peer_in WIRE_UPDATE_ADD_HTLC')
+#     l3.daemon.wait_for_log('peer_in WIRE_UPDATE_ADD_HTLC')
+# 
+#     # Now continue, payments will succeed due to MPP.
+#     l2.rpc.dev_reenable_commit(l4.info['id'])
+#     l3.rpc.dev_reenable_commit(l4.info['id'])
+#     l2.rpc.dev_reenable_commit(l1.info['id'])
+#     l3.rpc.dev_reenable_commit(l1.info['id'])
+# 
+#     res = l1.rpc.waitsendpay(payment_hash=inv['payment_hash'], partid=1)
+#     assert res['partid'] == 1
+#     res = l1.rpc.waitsendpay(payment_hash=inv['payment_hash'], partid=2)
+#     assert res['partid'] == 2
+# 
+#     for i in range(2):
+#         line = l4.daemon.wait_for_log('print_htlc_onion.py: Got onion')
+#         assert "'type': 'tlv'" in line
+#         assert "'forward_msat': 499" in line or "'forward_msat': 501" in line
+#         assert "'total_msat': 1000" in line
+#         assert "'payment_secret': '{}'".format(paysecret) in line
+# 
+#     pay = only_one(l1.rpc.listpays()['pays'])
+#     assert pay['bolt11'] == inv['bolt11']
+#     assert pay['status'] == 'complete'
+#     assert pay['number_of_parts'] == 2
+#     assert pay['amount_sent_msat'] == Millisatoshi(1002)
+# 
+#     # It will immediately succeed if we pay again.
+#     pay = l1.rpc.sendpay(
+#         route=r124,
+#         payment_hash=inv['payment_hash'],
+#         amount_msat=1000,
+#         bolt11=inv['bolt11'],
+#         payment_secret=paysecret,
+#         partid=2,
+#         groupid=1,
+#     )
+#     assert pay['status'] == 'complete'
+# 
+#     # If we try with an unknown partid, it will refuse.
+#     with pytest.raises(RpcError, match=r'Already succeeded'):
+#         l1.rpc.sendpay(
+#             route=r124,
+#             payment_hash=inv['payment_hash'],
+#             amount_msat=1000,
+#             bolt11=inv['bolt11'],
+#             payment_secret=paysecret,
+#             partid=3,
+#             groupid=1)
+# 
+# 
+# def test_partial_payment_timeout(node_factory, bitcoind):
+#     l1, l2 = node_factory.line_graph(2)
+# 
+#     inv = l2.rpc.invoice(1000, 'inv', 'inv')
+#     paysecret = l2.rpc.decodepay(inv['bolt11'])['payment_secret']
+# 
+#     route = l1.rpc.getroute(l2.info['id'], 500, 1)['route']
+#     l1.rpc.sendpay(
+#         route=route,
+#         payment_hash=inv['payment_hash'],
+#         amount_msat=1000,
+#         bolt11=inv['bolt11'],
+#         payment_secret=paysecret,
+#         partid=1,
+#         groupid=1,
+#     )
+# 
+#     with pytest.raises(RpcError, match=r'WIRE_MPP_TIMEOUT'):
+#         l1.rpc.waitsendpay(
+#             payment_hash=inv['payment_hash'],
+#             timeout=70 + TIMEOUT // 4,
+#             partid=1,
+#             groupid=1,
+#         )
+#     l2.daemon.wait_for_log(r'HTLC set contains 1 HTLCs, for a total of 500msat out of 1000msat \(payment_secret\)')
+# 
+#     # We can still pay it normally.
+#     l1.rpc.sendpay(
+#         route=route,
+#         payment_hash=inv['payment_hash'],
+#         amount_msat=1000,
+#         bolt11=inv['bolt11'],
+#         payment_secret=paysecret,
+#         partid=1,
+#         groupid=2
+#     )
+#     l1.rpc.sendpay(
+#         route=route,
+#         payment_hash=inv['payment_hash'],
+#         amount_msat=1000,
+#         bolt11=inv['bolt11'],
+#         payment_secret=paysecret,
+#         partid=2,
+#         groupid=2
+#     )
+#     l1.rpc.waitsendpay(payment_hash=inv['payment_hash'], timeout=TIMEOUT, partid=1, groupid=2)
+#     l1.rpc.waitsendpay(payment_hash=inv['payment_hash'], timeout=TIMEOUT, partid=2, groupid=2)
+#     l2.daemon.wait_for_log(r'HTLC set contains 1 HTLCs, for a total of 500msat out of 1000msat \(payment_secret\)')
+#     l2.daemon.wait_for_log(r'HTLC set contains 2 HTLCs, for a total of 1000msat out of 1000msat \(payment_secret\)')
+# 
+# 
+# def test_partial_payment_restart(node_factory, bitcoind):
+#     """Test that we recover a set when we restart"""
+#     l1, l2, l3 = node_factory.line_graph(3, wait_for_announce=True,
+#                                          opts=[{}]
+#                                          + [{'may_reconnect': True}] * 2)
+# 
+#     inv = l3.rpc.invoice(1000, 'inv', 'inv')
+#     paysecret = l3.rpc.decodepay(inv['bolt11'])['payment_secret']
+# 
+#     route = l1.rpc.getroute(l3.info['id'], 500, 1)['route']
+# 
+#     l1.rpc.sendpay(
+#         route=route,
+#         payment_hash=inv['payment_hash'],
+#         amount_msat=1000,
+#         bolt11=inv['bolt11'],
+#         payment_secret=paysecret,
+#         partid=1,
+#         groupid=1,
+#     )
+# 
+#     wait_for(lambda: [f['status'] for f in l2.rpc.listforwards()['forwards']] == ['offered'])
+# 
+#     # Restart, and make sure it's reconnected to l2.
+#     l3.restart()
+#     print(l2.rpc.listpeers())
+#     wait_for(lambda: [p['connected'] for p in l2.rpc.listpeers()['peers']] == [True, True])
+# 
+#     # Pay second part.
+#     l1.rpc.sendpay(
+#         route=route,
+#         payment_hash=inv['payment_hash'],
+#         amount_msat=1000,
+#         bolt11=inv['bolt11'],
+#         payment_secret=paysecret,
+#         partid=2,
+#         groupid=1,
+#     )
+# 
+#     l1.rpc.waitsendpay(payment_hash=inv['payment_hash'], timeout=TIMEOUT, partid=1)
+#     l1.rpc.waitsendpay(payment_hash=inv['payment_hash'], timeout=TIMEOUT, partid=2)
+# 
+# 
+# @pytest.mark.developer("needs dev-disconnect")
+# def test_partial_payment_htlc_loss(node_factory, bitcoind):
+#     """Test that we discard a set when the HTLC is lost"""
+#     # We want l2 to fail once it has completed first htlc.
+#     l1, l2, l3 = node_factory.line_graph(3, wait_for_announce=True,
+#                                          opts=[{},
+#                                                {'disconnect': ['=WIRE_UPDATE_ADD_HTLC', '+WIRE_REVOKE_AND_ACK']},
+#                                                {}])
+# 
+#     inv = l3.rpc.invoice(1000, 'inv', 'inv')
+#     paysecret = l3.rpc.decodepay(inv['bolt11'])['payment_secret']
+# 
+#     route = l1.rpc.getroute(l3.info['id'], 500, 1)['route']
+# 
+#     l1.rpc.sendpay(route=route, payment_hash=inv['payment_hash'], amount_msat=1000, bolt11=inv['bolt11'], payment_secret=paysecret, partid=1)
+# 
+#     wait_for(lambda: not only_one(l2.rpc.listpeers(l3.info['id'])['peers'])['connected'])
+#     l2.rpc.dev_fail(l3.info['id'])
+# 
+#     # Since HTLC is missing from commit (dust), it's closed as soon as l2 sees
+#     # it onchain.  l3 shouldn't crash though.
+#     bitcoind.generate_block(1, wait_for_mempool=1)
+# 
+#     with pytest.raises(RpcError,
+#                        match=r'WIRE_PERMANENT_CHANNEL_FAILURE \(reply from remote\)'):
+#         l1.rpc.waitsendpay(payment_hash=inv['payment_hash'], timeout=TIMEOUT, partid=1)
+# 
+# 
+# def test_createonion_limits(node_factory):
+#     l1, = node_factory.get_nodes(1)
+#     hops = [{
+#         # privkey: 41bfd2660762506c9933ade59f1debf7e6495b10c14a92dbcd2d623da2507d3d
+#         "pubkey": "0266e4598d1d3c415f572a8488830b60f7e744ed9235eb0b1ba93283b315c03518",
+#         "payload": bytes([227] + [0] * 227).hex(),
+#     }, {
+#         "pubkey": "0324653eac434488002cc06bbfb7f10fe18991e35f9fe4302dbea6d2353dc0ab1c",
+#         "payload": bytes([227] + [0] * 227).hex(),
+#     }, {
+#         "pubkey": "027f31ebc5462c1fdce1b737ecff52d37d75dea43ce11c74d25aa297165faa2007",
+#         "payload": bytes([227] + [0] * 227).hex(),
+#     }, {
+#         "pubkey": "032c0b7cf95324a07d05398b240174dc0c2be444d96b159aa6c7f7b1e668680991",
+#         "payload": bytes([227] + [0] * 227).hex(),
+#     }, {
+#         "pubkey": "02edabbd16b41c8371b92ef2f04c1185b4f03b6dcd52ba9b78d9d7c89c8f221145",
+#         "payload": bytes([227] + [0] * 227).hex(),
+#     }]
+# 
+#     # This should success since it's right at the edge
+#     l1.rpc.createonion(hops=hops, assocdata="BB" * 32)
+# 
+#     # This one should fail however
+#     with pytest.raises(RpcError, match=r'Payloads exceed maximum onion packet size.'):
+#         hops[0]['payload'] = bytes([228] + [0] * 228).hex()
+#         l1.rpc.createonion(hops=hops, assocdata="BB" * 32)
+# 
+#     # But with a larger onion, it will work!
+#     oniontool = os.path.join(os.path.dirname(__file__), "..", "devtools", "onion")
+#     onion = l1.rpc.createonion(hops=hops, assocdata="BB" * 32, onion_size=1301)['onion']
+# 
+#     # Oniontool wants a filename :(
+#     onionfile = os.path.join(l1.daemon.lightning_dir, 'onion')
+#     with open(onionfile, "w") as f:
+#         f.write(onion)
+# 
+#     subprocess.check_output(
+#         [oniontool, '--assoc-data', "BB" * 32,
+#          'decode', onionfile, "41bfd2660762506c9933ade59f1debf7e6495b10c14a92dbcd2d623da2507d3d"]
+#     )
+# 
+# 
+# @pytest.mark.developer("needs use_shadow")
+# def test_blockheight_disagreement(node_factory, bitcoind, executor):
+#     """
+#     While a payment is in-transit from payer to payee, a block
+#     might be mined, so that the blockheight the payer used to
+#     initiate the payment is no longer the blockheight when the
+#     payee receives it.
+#     This leads to a failure which *used* to be
+#     `final_expiry_too_soon`, a non-permanent failure, but
+#     which is *now* `incorrect_or_unknown_payment_details`,
+#     a permanent failure.
+#     `pay` treats permanent failures as, well, permanent, and
+#     gives up on receiving such failure from the payee, but
+#     this particular subcase of blockheight disagreement is
+#     actually a non-permanent failure (the payer only needs
+#     to synchronize to the same blockheight as the payee).
+#     """
+#     l1, l2 = node_factory.line_graph(2)
+# 
+#     sync_blockheight(bitcoind, [l1, l2])
+# 
+#     # Arrange l1 to stop getting new blocks.
+#     def no_more_blocks(req):
+#         return {"result": None,
+#                 "error": {"code": -8, "message": "Block height out of range"}, "id": req['id']}
+#     l1.daemon.rpcproxy.mock_rpc('getblockhash', no_more_blocks)
+# 
+#     # Increase blockheight and make sure l2 knows it.
+#     # Why 2? Because `pay` uses min_final_cltv_expiry + 1.
+#     # But 2 blocks coming in close succession, plus slow
+#     # forwarding nodes and block propagation, are still
+#     # possible on the mainnet, thus this test.
+#     bitcoind.generate_block(2)
+#     sync_blockheight(bitcoind, [l2])
+# 
+#     # Have l2 make an invoice.
+#     inv = l2.rpc.invoice(1000, 'l', 'd')['bolt11']
+# 
+#     # Have l1 pay l2
+#     def pay(l1, inv):
+#         l1.dev_pay(inv, use_shadow=False)
+#     fut = executor.submit(pay, l1, inv)
+# 
+#     # Make sure l1 sends out the HTLC.
+#     l1.daemon.wait_for_logs([r'NEW:: HTLC LOCAL'])
+# 
+#     height = bitcoind.rpc.getblockchaininfo()['blocks']
+#     l1.daemon.wait_for_log('Remote node appears to be on a longer chain.*catch up to block {}'.format(height))
+# 
+#     # Unblock l1 from new blocks.
+#     l1.daemon.rpcproxy.mock_rpc('getblockhash', None)
+# 
+#     # pay command should complete without error
+#     fut.result()
+# 
+# 
+# def test_sendpay_msatoshi_arg(node_factory):
+#     """sendpay msatoshi arg was used for non-MPP to indicate the amount
+# they asked for.  But using it with anything other than the final amount
+# caused a crash in 0.8.0, so we then disallowed it.
+#     """
+#     l1, l2 = node_factory.line_graph(2)
+# 
+#     inv = l2.rpc.invoice(1000, 'inv', 'inv')
+# 
+#     # Can't send non-MPP payment which specifies msatoshi != final.
+#     with pytest.raises(RpcError, match=r'Do not specify msatoshi \(1001msat\) without'
+#                        ' partid: if you do, it must be exactly'
+#                        r' the final amount \(1000msat\)'):
+#         l1.rpc.sendpay(route=l1.rpc.getroute(l2.info['id'], 1000, 1)['route'], payment_hash=inv['payment_hash'], amount_msat=1001, bolt11=inv['bolt11'], payment_secret=inv['payment_secret'])
+#     with pytest.raises(RpcError, match=r'Do not specify msatoshi \(999msat\) without'
+#                        ' partid: if you do, it must be exactly'
+#                        r' the final amount \(1000msat\)'):
+#         l1.rpc.sendpay(route=l1.rpc.getroute(l2.info['id'], 1000, 1)['route'], payment_hash=inv['payment_hash'], amount_msat=999, bolt11=inv['bolt11'], payment_secret=inv['payment_secret'])
+# 
+#     # Can't send MPP payment which pays any more than amount.
+#     with pytest.raises(RpcError, match=r'Final amount 1001msat is greater than 1000msat, despite MPP'):
+#         l1.rpc.sendpay(route=l1.rpc.getroute(l2.info['id'], 1001, 1)['route'], payment_hash=inv['payment_hash'], amount_msat=1000, bolt11=inv['bolt11'], partid=1, payment_secret=inv['payment_secret'])
+# 
+#     # But this works
+#     l1.rpc.sendpay(route=l1.rpc.getroute(l2.info['id'], 1001, 1)['route'], payment_hash=inv['payment_hash'], amount_msat=1001, bolt11=inv['bolt11'], payment_secret=inv['payment_secret'])
+#     l1.rpc.waitsendpay(inv['payment_hash'])
+# 
+#     inv = only_one(l2.rpc.listinvoices('inv')['invoices'])
+#     assert inv['status'] == 'paid'
+#     assert inv['amount_received_msat'] == Millisatoshi(1001)
+# 
+# 
+# def test_reject_invalid_payload(node_factory):
+#     """Send an onion payload with an unknown even type.
+# 
+#     Recipient l2 should reject it the incoming HTLC with an invalid onion
+#     payload error.
+# 
+#     """
+# 
+#     l1, l2 = node_factory.line_graph(2)
+#     amt = 10**3
+#     route = l1.rpc.getroute(l2.info['id'], amt, 10)['route']
+#     inv = l2.rpc.invoice(amt, "lbl", "desc")
+# 
+#     first_hop = route[0]
+# 
+#     # A TLV payload with an unknown even type:
+#     payload = TlvPayload()
+#     payload.add_field(0xB000, b'Hi there')
+#     hops = [{"pubkey": l2.info['id'], "payload": payload.to_hex()}]
+#     onion = l1.rpc.createonion(hops=hops, assocdata=inv['payment_hash'])
+#     l1.rpc.sendonion(onion=onion['onion'],
+#                      first_hop=first_hop,
+#                      payment_hash=inv['payment_hash'],
+#                      shared_secrets=onion['shared_secrets'])
+# 
+#     l2.daemon.wait_for_log(r'Failing HTLC because of an invalid payload')
+# 
+#     with pytest.raises(RpcError, match=r'WIRE_INVALID_ONION_PAYLOAD'):
+#         l1.rpc.waitsendpay(inv['payment_hash'])
+# 
+# 
+# @pytest.mark.skip("Needs to be updated for modern onion")
+# @unittest.skipIf(not EXPERIMENTAL_FEATURES, "Needs blinding args to sendpay")
+# def test_sendpay_blinding(node_factory):
+#     l1, l2, l3, l4 = node_factory.line_graph(4)
+# 
+#     blindedpathtool = os.path.join(os.path.dirname(__file__), "..", "devtools", "blindedpath")
+# 
+#     # Create blinded path l2->l4
+#     output = subprocess.check_output(
+#         [blindedpathtool, '--simple-output', 'create',
+#          l2.info['id'] + "/" + l2.get_channel_scid(l3),
+#          l3.info['id'] + "/" + l3.get_channel_scid(l4),
+#          l4.info['id']]
+#     ).decode('ASCII').strip()
+# 
+#     # First line is blinding, then <peerid> then <encblob>.
+#     blinding, p1, p1enc, p2, p2enc, p3 = output.split('\n')
+#     # First hop can't be blinded!
+#     assert p1 == l2.info['id']
+# 
+#     amt = 10**3
+#     inv = l4.rpc.invoice(amt, "lbl", "desc")
+# 
+#     route = [{'id': l2.info['id'],
+#               'channel': l1.get_channel_scid(l2),
+#               'amount_msat': Millisatoshi(1002),
+#               'delay': 21,
+#               'blinding': blinding,
+#               'enctlv': p1enc},
+#              {'id': p2,
+#               'amount_msat': Millisatoshi(1001),
+#               'delay': 15,
+#               # FIXME: this is a dummy!
+#               'channel': '0x0x0',
+#               'enctlv': p2enc},
+#              {'id': p3,
+#               # FIXME: this is a dummy!
+#               'channel': '0x0x0',
+#               'amount_msat': Millisatoshi(1000),
+#               'delay': 9,
+#               'style': 'tlv'}]
+#     l1.rpc.sendpay(route=route,
+#                    payment_hash=inv['payment_hash'],
+#                    bolt11=inv['bolt11'], payment_secret=inv['payment_secret'])
+#     l1.rpc.waitsendpay(inv['payment_hash'])
+# 
+# 
+# def test_excluded_adjacent_routehint(node_factory, bitcoind):
+#     """Test case where we try have a routehint which leads to an adjacent
+#     node, but the result exceeds our maxfee; we crashed trying to find
+#     what part of the path was most expensive in that case
+# 
+#     """
+#     l1, l2, l3 = node_factory.line_graph(3)
+# 
+#     # We'll be forced to use routehint, since we don't know about l3.
+#     wait_for(lambda: len(l3.rpc.listchannels(source=l2.info['id'])['channels']) == 1)
+#     inv = l3.rpc.invoice(10**3, "lbl", "desc", exposeprivatechannels=l2.get_channel_scid(l3))
+# 
+#     # This will make it reject the routehint.
+#     err = r'Fee exceeds our fee budget: 1msat > 0msat, discarding route'
+#     with pytest.raises(RpcError, match=err):
+#         l1.rpc.pay(bolt11=inv['bolt11'], maxfeepercent=0, exemptfee=0)
+# 
+# 
+# def test_keysend(node_factory):
+#     amt = 10000
+#     l1, l2, l3, l4 = node_factory.line_graph(
+#         4,
+#         wait_for_announce=True,
+#         opts=[{}, {}, {}, {'disable-plugin': 'keysend'}]
+#     )
+# 
+#     # The keysend featurebit must be set in the announcement, i.e., l1 should
+#     # learn that l3 supports keysends.
+#     features = l1.rpc.listnodes(l3.info['id'])['nodes'][0]['features']
+#     assert(int(features, 16) >> 55 & 0x01 == 1)
+# 
+#     # If we disable keysend, then the featurebit must not be set,
+#     # i.e., l4 doesn't support it.
+#     features = l1.rpc.listnodes(l4.info['id'])['nodes'][0]['features']
+#     assert(int(features, 16) >> 55 & 0x01 == 0)
+# 
+#     # Self-sends are not allowed (see #4438)
+#     with pytest.raises(RpcError, match=r'We are the destination.'):
+#         l1.rpc.keysend(l1.info['id'], amt)
+# 
+#     # Send an indirect one from l1 to l3
+#     l1.rpc.keysend(l3.info['id'], amt)
+#     invs = l3.rpc.listinvoices()['invoices']
+#     assert(len(invs) == 1)
+# 
+#     inv = invs[0]
+#     print(inv)
+#     assert(inv['amount_received_msat'] >= Millisatoshi(amt))
+# 
+#     # Now send a direct one instead from l1 to l2
+#     l1.rpc.keysend(l2.info['id'], amt)
+#     invs = l2.rpc.listinvoices()['invoices']
+#     assert(len(invs) == 1)
+# 
+#     inv = invs[0]
+#     assert(inv['amount_received_msat'] >= Millisatoshi(amt))
+# 
+#     # And finally try to send a keysend payment to l4, which doesn't
+#     # support it. It MUST fail.
+#     with pytest.raises(RpcError, match=r"Recipient [0-9a-f]{66} reported an invalid payload"):
+#         l3.rpc.keysend(l4.info['id'], amt)
+# 
+# 
+# def test_keysend_strip_tlvs(node_factory):
+#     """Use the extratlvs option to deliver a message with sphinx' TLV type, which keysend strips.
+#     """
+#     amt = 10**7
+#     l1, l2 = node_factory.line_graph(
+#         2,
+#         wait_for_announce=True,
+#         opts=[
+#             {
+#                 # Not needed, just for listconfigs test.
+#                 'accept-htlc-tlv-types': '133773310,99990',
+#                 "plugin": os.path.join(os.path.dirname(__file__), "plugins/sphinx-receiver.py"),
+#             },
+#             {
+#                 "plugin": os.path.join(os.path.dirname(__file__), "plugins/sphinx-receiver.py"),
+#             },
+#         ]
+#     )
+# 
+#     # Make sure listconfigs works here
+#     assert l1.rpc.listconfigs()['accept-htlc-tlv-types'] == '133773310,99990'
+# 
+#     # l1 is configured to accept, so l2 should still filter them out
+#     l1.rpc.keysend(l2.info['id'], amt, extratlvs={133773310: 'FEEDC0DE'})
+#     inv = only_one(l2.rpc.listinvoices()['invoices'])
+#     assert not l2.daemon.is_in_log(r'plugin-sphinx-receiver.py.*extratlvs.*133773310.*feedc0de')
+# 
+#     assert(inv['amount_received_msat'] >= Millisatoshi(amt))
+#     assert inv['description'] == 'keysend'
+#     l2.rpc.delinvoice(inv['label'], 'paid')
+# 
+#     # Now try again with the TLV type in extra_tlvs as string:
+#     l1.rpc.keysend(l2.info['id'], amt, extratlvs={133773310: b'hello there'.hex()})
+#     inv = only_one(l2.rpc.listinvoices()['invoices'])
+#     assert inv['description'] == 'keysend: hello there'
+#     l2.daemon.wait_for_log('Keysend payment uses illegal even field 133773310: stripping')
+#     l2.rpc.delinvoice(inv['label'], 'paid')
+# 
+#     # We can (just!) fit a giant description in.
+#     l1.rpc.keysend(l2.info['id'], amt, extratlvs={133773310: (b'a' * 1100).hex()})
+#     inv = only_one(l2.rpc.listinvoices()['invoices'])
+#     assert inv['description'] == 'keysend: ' + 'a' * 1100
+#     l2.rpc.delinvoice(inv['label'], 'paid')
+#     l2.daemon.wait_for_log('Keysend payment uses illegal even field 133773310: stripping')
+# 
+#     # Now try with some special characters
+#     ksinfo = """💕 ₿"'
+# More info
+# """
+#     # Since we're at it, use this to test string-keyed TLVs
+#     l1.rpc.keysend(l2.info['id'], amt, extratlvs={"133773310": bytes(ksinfo, encoding='utf8').hex()})
+#     inv = only_one(l2.rpc.listinvoices()['invoices'])
+#     assert inv['description'] == 'keysend: ' + ksinfo
+#     l2.daemon.wait_for_log('Keysend payment uses illegal even field 133773310: stripping')
+# 
+#     # Now reverse the direction. l1 accepts 133773310, but filters out
+#     # other even unknown types (like 133773312).
+#     l2.rpc.keysend(l1.info['id'], amt, extratlvs={
+#         "133773310": b"helloworld".hex(),  # This one is allowlisted
+#         "133773312": b"filterme".hex(),  # This one will get stripped
+#     })
+# 
+#     # The invoice_payment hook must contain the allowlisted TLV type,
+#     # but not the stripped one.
+#     assert l1.daemon.wait_for_log(r'plugin-sphinx-receiver.py: invoice_payment.*extratlvs.*133773310')
+#     assert not l1.daemon.is_in_log(r'plugin-sphinx-receiver.py: invoice_payment.*extratlvs.*133773312')
+# 
+# 
+# def test_keysend_routehint(node_factory):
+#     """Test whether we can deliver a keysend by adding a routehint on the cli
+#     """
+#     amt = 10000
+#     l1, l2 = node_factory.line_graph(2, wait_for_announce=True)
+#     l3 = node_factory.get_node()
+#     l2.connect(l3)
+#     l2.fundchannel(l3, announce_channel=False)
+# 
+#     dest = l3.info['id']
+#     routehints = [
+#         [
+#             {
+#                 'scid': only_one(l3.rpc.listpeerchannels()['channels'])['alias']['remote'],
+#                 'id': l2.info['id'],
+#                 'feebase': '1msat',
+#                 'feeprop': 10,
+#                 'expirydelta': 9,
+#             }
+#         ],
+#         [  # Dummy
+#             {
+#                 'scid': '1x2x3',
+#                 'id': '02' * 33,
+#                 'feebase': 1,
+#                 'feeprop': 1,
+#                 'expirydelta': 9,
+#             },
+#         ],
+#     ]
+# 
+#     # Without any hints we should fail:
+#     with pytest.raises(RpcError):
+#         l1.rpc.call("keysend", payload={'destination': dest, 'amount_msat': amt})
+# 
+#     # We should also fail with only non-working hints:
+#     with pytest.raises(RpcError):
+#         l1.rpc.call("keysend", payload={'destination': dest, 'amount_msat': amt, 'routehints': routehints[1:]})
+# 
+#     l1.rpc.call("keysend", payload={'destination': dest, 'amount_msat': amt, 'routehints': routehints})
+#     invs = l3.rpc.listinvoices()['invoices']
+#     assert(len(invs) == 1)
+# 
+#     inv = invs[0]
+#     assert(inv['amount_received_msat'] >= Millisatoshi(amt))
+# 
+# 
+# def test_invalid_onion_channel_update(node_factory):
+#     '''
+#     Some onion failures "should" send a `channel_update`.
+# 
+#     This test checks to see if we handle things correctly
+#     even if some remote node does not send the required
+#     `channel_update`.
+#     '''
+#     plugin = os.path.join(os.getcwd(), 'tests/plugins/fail_htlcs_invalid.py')
+#     l1, l2, l3 = node_factory.line_graph(3,
+#                                          opts=[{},
+#                                                {'plugin': plugin},
+#                                                {}],
+#                                          wait_for_announce=True)
+# 
+#     l1id = l1.info['id']
+# 
+#     inv = l3.rpc.invoice(12345, 'inv', 'inv')['bolt11']
+#     # Should fail, since l2 will always fail to forward.
+#     with pytest.raises(RpcError):
+#         l1.rpc.pay(inv)
+# 
+#     # l1 should still be alive afterwards.
+#     assert l1.rpc.getinfo()['id'] == l1id
+# 
+# 
+# @pytest.mark.developer("Requires use_shadow")
+# def test_pay_exemptfee(node_factory):
+#     """Tiny payment, huge fee
+# 
+#     l1 -> l2 -> l3
+# 
+#     Create a tiny invoice for 1 msat, it'll be dominated by the base_fee on
+#     the l2->l3 channel. So it'll get rejected on the first attempt if we set
+#     the exemptfee way too low. The default fee exemption threshold is
+#     5000msat, so 5001msat is not exempted by default and a 5001msat fee on
+#     l2->l3 should trigger this.
+# 
+#     """
+#     l1, l2, l3 = node_factory.line_graph(
+#         3,
+#         opts=[{}, {'fee-base': 5001, 'fee-per-satoshi': 0}, {}],
+#         wait_for_announce=True
+#     )
+# 
+#     err = r'Ran out of routes to try'
+# 
+#     with pytest.raises(RpcError, match=err):
+#         l1.dev_pay(l3.rpc.invoice(1, "lbl1", "desc")['bolt11'], use_shadow=False)
+# 
+#     # If we tell our node that 5001msat is ok this should work
+#     l1.dev_pay(l3.rpc.invoice(1, "lbl2", "desc")['bolt11'], use_shadow=False, exemptfee=5001)
+# 
+#     # Given the above network this is the smallest amount that passes without
+#     # the fee-exemption (notice that we let it through on equality).
+#     threshold = int(5001 / 0.05)
+# 
+#     # This should be just below the fee-exemption and is the first value that is allowed through
+#     with pytest.raises(RpcError, match=err):
+#         l1.dev_pay(l3.rpc.invoice(threshold - 1, "lbl3", "desc")['bolt11'], use_shadow=False)
+# 
+#     # While this'll work just fine
+#     l1.dev_pay(l3.rpc.invoice(int(5001 * 200), "lbl4", "desc")['bolt11'], use_shadow=False)
+# 
+# 
+# @pytest.mark.developer("Requires use_shadow flag")
+# def test_pay_peer(node_factory, bitcoind):
+#     """If we have a direct channel to the destination we should use that.
+# 
+#     This is complicated a bit by not having sufficient capacity, but the
+#     channel_hints can help us there.
+# 
+#     l1 -> l2
+#      |   ^
+#      v  /
+#      l3
+#     """
+#     # Set the dust exposure higher, this gets triggered on liquid
+#     l1, l2, l3 = node_factory.get_nodes(3, opts={'max-dust-htlc-exposure-msat': '100000sat'})
+#     node_factory.join_nodes([l1, l2])
+#     node_factory.join_nodes([l1, l3])
+#     node_factory.join_nodes([l3, l2], wait_for_announce=True)
+# 
+#     wait_for(lambda: len(l1.rpc.listchannels()['channels']) == 6)
+# 
+#     def spendable(n1, n2):
+#         chan = n1.rpc.listpeerchannels(n2.info['id'])['channels'][0]
+#         avail = chan['spendable_msat']
+#         return avail
+# 
+#     amt = Millisatoshi(10**8)
+#     # How many payments do we expect to go through directly?
+#     direct = spendable(l1, l2).millisatoshis // amt.millisatoshis
+# 
+#     # Remember the l1 -> l3 capacity, it should not change until we run out of
+#     # direct capacity.
+#     l1l3cap = spendable(l1, l3)
+# 
+#     for i in range(0, direct):
+#         inv = l2.rpc.invoice(amt.millisatoshis, "lbl{}".format(i),
+#                              "desc{}".format(i))['bolt11']
+#         l1.dev_pay(inv, use_shadow=False)
+# 
+#     # We should not have more than amt in the direct channel anymore
+#     assert(spendable(l1, l2) < amt)
+#     assert(spendable(l1, l3) == l1l3cap)
+# 
+#     # Next one should take the alternative, but it should still work
+#     inv = l2.rpc.invoice(amt.millisatoshis, "final", "final")['bolt11']
+#     l1.dev_pay(inv, use_shadow=False)
+# 
+# 
+# def test_mpp_presplit(node_factory):
+#     """Make a rather large payment of 5*10ksat and see it being split.
+#     """
+#     MPP_TARGET_SIZE = 10**7  # Taken from libpluin-pay.c
+#     amt = 5 * MPP_TARGET_SIZE
+# 
+#     # Assert that the amount we're going to send is indeed larger than our
+#     # split size.
+#     assert(MPP_TARGET_SIZE < amt)
+# 
+#     l1, l2, l3 = node_factory.line_graph(
+#         3, fundamount=10**8, wait_for_announce=True,
+#         opts={'wumbo': None,
+#               'max-dust-htlc-exposure-msat': '500000sat'}
+#     )
+# 
+#     inv = l3.rpc.invoice(amt, 'lbl', 'desc')['bolt11']
+#     p = l1.rpc.pay(inv)
+# 
+#     assert(p['parts'] >= 5)
+#     inv = l3.rpc.listinvoices()['invoices'][0]
+# 
+#     assert(inv['amount_msat'] == inv['amount_received_msat'])
+# 
+#     # Make sure that bolt11 isn't duplicated for every part
+#     bolt11s = 0
+#     count = 0
+#     for p in l1.rpc.listsendpays()['payments']:
+#         if 'bolt11' in p:
+#             bolt11s += 1
+#         count += 1
+# 
+#     # You were supposed to mpp!
+#     assert count > 1
+#     # Not every one should have the bolt11 string
+#     assert bolt11s < count
+#     # In fact, only one should
+#     assert bolt11s == 1
+# 
+#     # But listpays() gathers it:
+#     assert only_one(l1.rpc.listpays()['pays'])['bolt11'] == inv['bolt11']
+# 
+# 
+# def test_mpp_adaptive(node_factory, bitcoind):
+#     """We have two paths, both too small on their own, let's combine them.
+# 
+#     ```dot
+#     digraph {
+#       l1 -> l2 [label="scid=103x1x1, cap=amt-1"];
+#       l2 -> l4 [label="scid=105x1x1, cap=max"];
+#       l1 -> l3 [label="scid=107x1x1, cap=max"];
+#       l3 -> l4 [label="scid=109x1x1, cap=amt-1"];
+#     }
+#     """
+#     amt = 10**7 - 1
+#     l1, l2, l3, l4 = node_factory.get_nodes(4)
+# 
+#     l1.connect(l2)
+#     l2.connect(l4)
+#     l1.connect(l3)
+#     l3.connect(l4)
+# 
+#     # First roadblock right away on an outgoing channel
+#     l2.fundchannel(l1, amt)
+#     l2.fundchannel(l4, amt, wait_for_active=True)
+#     l2.rpc.pay(l1.rpc.invoice(
+#         amt + 99999000 - 1,  # Slightly less than amt + reserve
+#         label="reb l1->l2",
+#         description="Rebalance l1 -> l2"
+#     )['bolt11'])
+# 
+#     # Second path fails only after the first hop
+#     l1.fundchannel(l3, amt)
+#     l4.fundchannel(l3, amt, wait_for_active=True)
+#     l4.rpc.pay(l3.rpc.invoice(
+#         amt + 99999000 - 1,  # Slightly less than amt + reserve
+#         label="reb l3->l4",
+#         description="Rebalance l3 -> l4"
+#     )['bolt11'])
+#     l1.rpc.listpeers()
+# 
+#     # Make sure neither channel can fit the payment by itself.
+#     c12 = l1.rpc.listpeerchannels(l2.info['id'])['channels'][0]
+#     c34 = l3.rpc.listpeerchannels(l4.info['id'])['channels'][0]
+#     assert(c12['spendable_msat'].millisatoshis < amt)
+#     assert(c34['spendable_msat'].millisatoshis < amt)
+# 
+#     # Make sure all HTLCs entirely resolved before we mine more blocks!
+#     def all_htlcs(n):
+#         htlcs = []
+#         for p in n.rpc.listpeers()['peers']:
+#             for c in n.rpc.listpeerchannels(p['id'])['channels']:
+#                 htlcs += c['htlcs']
+#         return htlcs
+# 
+#     wait_for(lambda: all([all_htlcs(n) == [] for n in [l1, l2, l3, l4]]))
+# 
+#     mine_funding_to_announce(bitcoind, [l1, l2, l3, l4])
+#     wait_for(lambda: len(l1.rpc.listchannels()['channels']) == 8)
+# 
+#     inv = l4.rpc.invoice(
+#         amt,
+#         label="splittest",
+#         description="Needs to be split into at least 2"
+#     )['bolt11']
+# 
+#     p = l1.rpc.pay(inv)
+#     from pprint import pprint
+#     pprint(p)
+#     pprint(l1.rpc.paystatus(inv))
+# 
+#     # Make sure that bolt11 isn't duplicated for every part
+#     bolt11s = 0
+#     count = 0
+#     for p in l1.rpc.listsendpays()['payments']:
+#         if 'bolt11' in p:
+#             bolt11s += 1
+#         count += 1
+# 
+#     # You were supposed to mpp!
+#     assert count > 1
+#     # Not every one should have the bolt11 string
+#     assert bolt11s < count
+# 
+#     # listpays() shows bolt11 string
+#     assert 'bolt11' in only_one(l1.rpc.listpays()['pays'])
+# 
+# 
+# def test_pay_fail_unconfirmed_channel(node_factory, bitcoind):
+#     '''
+#     Replicate #3855.
+#     `pay` crash when any direct channel is still
+#     unconfirmed.
+#     '''
+#     l1, l2 = node_factory.get_nodes(2)
+# 
+#     amount_sat = 10 ** 6
+# 
+#     # create l2->l1 channel.
+#     l2.fundwallet(amount_sat * 5)
+#     l1.rpc.connect(l2.info['id'], 'localhost', l2.port)
+#     l2.rpc.fundchannel(l1.info['id'], amount_sat * 3)
+#     # channel is still unconfirmed.
+# 
+#     # Attempt to pay from l1 to l2.
+#     # This should fail since the channel capacities are wrong.
+#     invl2 = l2.rpc.invoice(Millisatoshi(amount_sat * 1000), 'i', 'i')['bolt11']
+#     with pytest.raises(RpcError):
+#         l1.rpc.pay(invl2)
+# 
+#     # Let the channel confirm.
+#     bitcoind.generate_block(6)
+#     sync_blockheight(bitcoind, [l1, l2])
+# 
+#     # Now give enough capacity so l1 can pay.
+#     invl1 = l1.rpc.invoice(Millisatoshi(amount_sat * 2 * 1000), 'j', 'j')['bolt11']
+#     l2.rpc.pay(invl1)
+# 
+#     # Wait for us to recognize that the channel is available
+#     wait_for(lambda: l1.rpc.listpeerchannels()['channels'][0]['spendable_msat'].millisatoshis > amount_sat * 1000)
+# 
+#     # Now l1 can pay to l2.
+#     l1.rpc.pay(invl2)
+# 
+# 
+# def test_bolt11_null_after_pay(node_factory, bitcoind):
+#     l1, l2 = node_factory.get_nodes(2)
+# 
+#     amount_sat = 10 ** 6
+#     # pay a generic bolt11 and test if the label bol11 is null
+#     # inside the command listpays
+# 
+#     # create l2->l1 channel.
+#     l2.fundwallet(amount_sat * 5)
+#     l1.rpc.connect(l2.info['id'], 'localhost', l2.port)
+#     # Make sure l2 considers it fully connected too!
+#     wait_for(lambda: l2.rpc.listpeers(l1.info['id']) != {'peers': []})
+#     l2.rpc.fundchannel(l1.info['id'], amount_sat * 3)
+# 
+#     # Let the channel confirm.
+#     bitcoind.generate_block(6)
+#     sync_blockheight(bitcoind, [l1, l2])
+#     wait_for(lambda: only_one(l1.rpc.listpeerchannels()['channels'])['state'] == 'CHANNELD_NORMAL')
+# 
+#     amt = Millisatoshi(amount_sat * 2 * 1000)
+#     invl1 = l1.rpc.invoice(amt, 'j', 'j')['bolt11']
+#     l2.rpc.pay(invl1)
+# 
+#     pays = l2.rpc.listpays()["pays"]
+#     assert(pays[0]["bolt11"] == invl1)
+#     assert('amount_msat' in pays[0] and pays[0]['amount_msat'] == amt)
+#     assert('created_at' in pays[0])
+#     assert('completed_at' in pays[0])
+# 
+# 
+# def test_mpp_presplit_routehint_conflict(node_factory, bitcoind):
+#     '''
+#     We had a bug where pre-splitting the payment prevents *any*
+#     routehints from being taken.
+#     We tickle that bug here by building l1->l2->l3, but with
+#     l2->l3 as an unpublished channel.
+#     If the payment is large enough to trigger pre-splitting, the
+#     routehints are not applied in any of the splits.
+#     '''
+#     l1, l2, l3 = node_factory.get_nodes(3)
+# 
+#     l1.rpc.connect(l2.info['id'], 'localhost', l2.port)
+#     l1l2, _ = l1.fundchannel(l2, 10**7, announce_channel=True)
+#     l2.rpc.connect(l3.info['id'], 'localhost', l3.port)
+#     l2.fundchannel(l3, 10**7, announce_channel=False)
+# 
+#     mine_funding_to_announce(bitcoind, [l1, l2, l3])
+# 
+#     # Wait for l3 to learn about l1->l2, otherwise it will think
+#     # l2 is a deadend and not add it to the routehint.
+#     wait_for(lambda: len(l3.rpc.listchannels(l1l2)['channels']) >= 2)
+# 
+#     inv = l3.rpc.invoice(Millisatoshi(2 * 10000 * 1000), 'i', 'i', exposeprivatechannels=True)['bolt11']
+# 
+#     l1.rpc.pay(inv)
+# 
+# 
+# def test_delpay_argument_invalid(node_factory, bitcoind):
+#     """
+#     This test includes all possible combinations of input error inside the
+#     delpay command.
+#     """
+# 
+#     # Create the line graph l2 -> l1 with a channel of 10 ** 5 sat!
+#     l2, l1 = node_factory.line_graph(2, fundamount=10**5, wait_for_announce=True)
+# 
+#     l2.rpc.check_request_schemas = False
+#     with pytest.raises(RpcError):
+#         l2.rpc.delpay()
+#     l2.rpc.check_request_schemas = True
+# 
+#     # sanity check
+#     inv = l1.rpc.invoice(10 ** 5, 'inv', 'inv')
+#     payment_hash = "AA" * 32
+#     with pytest.raises(RpcError):
+#         l2.rpc.delpay(payment_hash, 'complete')
+# 
+#     l2.rpc.pay(inv['bolt11'])
+# 
+#     wait_for(lambda: l2.rpc.listpays(inv['bolt11'])['pays'][0]['status'] == 'complete')
+# 
+#     payment_hash = inv['payment_hash']
+# 
+#     # payment paid with wrong status (pending status is a illegal input)
+#     l2.rpc.check_request_schemas = False
+#     with pytest.raises(RpcError):
+#         l2.rpc.delpay(payment_hash, 'pending')
+# 
+#     with pytest.raises(RpcError):
+#         l2.rpc.delpay(payment_hash, 'invalid_status')
+#     l2.rpc.check_request_schemas = True
+# 
+#     with pytest.raises(RpcError):
+#         l2.rpc.delpay(payment_hash, 'failed')
+# 
+#     # test if the node is still ready
+#     payments = l2.rpc.delpay(payment_hash, 'complete')
+# 
+#     assert payments['payments'][0]['bolt11'] == inv['bolt11']
+#     assert len(payments['payments']) == 1
+#     assert len(l2.rpc.listpays()['pays']) == 0
+# 
+# 
+# def test_delpay_payment_split(node_factory, bitcoind):
+#     """
+#     Test behavior of delpay with an MPP
+#     """
+#     MPP_TARGET_SIZE = 10**7  # Taken from libpluin-pay.c
+#     amt = 4 * MPP_TARGET_SIZE
+# 
+#     l1, l2, l3 = node_factory.line_graph(3, fundamount=10**5,
+#                                          wait_for_announce=True)
+#     inv = l3.rpc.invoice(amt, 'lbl', 'desc')
+#     l1.rpc.pay(inv['bolt11'])
+# 
+#     assert len(l1.rpc.listpays()['pays']) == 1
+#     delpay_result = l1.rpc.delpay(inv['payment_hash'], 'complete')['payments']
+#     assert len(delpay_result) >= 4
+#     assert len(l1.rpc.listpays()['pays']) == 0
+# 
+# 
+# def test_listpay_result_with_paymod(node_factory, bitcoind):
+#     """
+#     The object of this test is to verify the correct behavior
+#     of the RPC command listpay e with two different type of
+#     payment, such as: keysend (without invoice) and pay (with invoice).
+#     l1 -> keysend -> l2
+#     l2 -> pay invoice -> l3
+#     """
+# 
+#     amount_sat = 10 ** 6
+# 
+#     l1, l2, l3 = node_factory.line_graph(3, wait_for_announce=True)
+# 
+#     invl2 = l2.rpc.invoice(amount_sat * 2, "inv_l2", "inv_l2")
+#     l1.rpc.pay(invl2['bolt11'])
+# 
+#     l2.rpc.keysend(l3.info['id'], amount_sat * 2, "keysend_l3")
+# 
+#     assert 'bolt11' in l1.rpc.listpays()['pays'][0]
+#     assert 'bolt11' not in l2.rpc.listpays()['pays'][0]
+#     assert 'payment_hash' in l2.rpc.listpays()['pays'][0]
+#     assert 'payment_hash' in l1.rpc.listpays()['pays'][0]
+#     assert 'destination' in l1.rpc.listpays()['pays'][0]
+#     assert 'destination' in l2.rpc.listpays()['pays'][0]
+# 
+# 
+# def test_listsendpays_and_listpays_order(node_factory):
+#     """listsendpays should be in increasing id order, listpays in created_at"""
+#     l1, l2 = node_factory.line_graph(2)
+#     for i in range(5):
+#         inv = l2.rpc.invoice(1000 - i, "test {}".format(i), "test")['bolt11']
+#         l1.rpc.pay(inv)
+# 
+#     ids = [p['id'] for p in l1.rpc.listsendpays()['payments']]
+#     assert ids == sorted(ids)
+# 
+#     created_at = [p['created_at'] for p in l1.rpc.listpays()['pays']]
+#     assert created_at == sorted(created_at)
+# 
+# 
+# @pytest.mark.developer("needs use_shadow")
+# def test_mpp_waitblockheight_routehint_conflict(node_factory, bitcoind, executor):
+#     '''
+#     We have a bug where a blockheight disagreement between us and
+#     the receiver causes us to advance through the routehints a bit
+#     too aggressively.
+#     '''
+#     l1, l2, l3 = node_factory.get_nodes(3)
+# 
+#     l1.rpc.connect(l2.info['id'], 'localhost', l2.port)
+#     l1l2, _ = l1.fundchannel(l2, 10**7, announce_channel=True)
+#     l2.rpc.connect(l3.info['id'], 'localhost', l3.port)
+#     l2.fundchannel(l3, 10**7, announce_channel=False)
+# 
+#     mine_funding_to_announce(bitcoind, [l1, l2, l3])
+# 
+#     # Wait for l3 to learn about l1->l2, otherwise it will think
+#     # l2 is a deadend and not add it to the routehint.
+#     wait_for(lambda: len(l3.rpc.listchannels(l1l2)['channels']) >= 2)
+# 
+#     # Now make the l1 payer stop receiving blocks.
+#     def no_more_blocks(req):
+#         return {"result": None,
+#                 "error": {"code": -8, "message": "Block height out of range"}, "id": req['id']}
+#     l1.daemon.rpcproxy.mock_rpc('getblockhash', no_more_blocks)
+# 
+#     # Increase blockheight by 2, like in test_blockheight_disagreement.
+#     bitcoind.generate_block(2)
+#     sync_blockheight(bitcoind, [l3])
+# 
+#     inv = l3.rpc.invoice(Millisatoshi(2 * 10000 * 1000), 'i', 'i', exposeprivatechannels=True)['bolt11']
+# 
+#     # Have l1 pay l3
+#     def pay(l1, inv):
+#         l1.dev_pay(inv, use_shadow=False)
+#     fut = executor.submit(pay, l1, inv)
+# 
+#     # Make sure l1 sends out the HTLC.
+#     l1.daemon.wait_for_logs([r'NEW:: HTLC LOCAL'])
+# 
+#     # Unblock l1 from new blocks.
+#     l1.daemon.rpcproxy.mock_rpc('getblockhash', None)
+# 
+#     # pay command should complete without error
+#     fut.result(TIMEOUT)
+# 
+# 
+# @pytest.mark.developer("channel setup very slow (~10 minutes) if not DEVELOPER")
+# @pytest.mark.slow_test
+# @pytest.mark.openchannel('v1')
+# @pytest.mark.openchannel('v2')
+# @unittest.skipIf(True, "Temporarily disabled while flake diagnosed: blame Rusty!")
+# def test_mpp_interference_2(node_factory, bitcoind, executor):
+#     '''
+#     We create a "public network" that looks like so.
+#     Each channel is perfectly balanced, with 7 * unit
+#     funds on each side.
+# 
+#         4 -- 5
+#         |   /|
+#         |  / |
+#         | /  |
+#         |/   |
+#         6 -- 7
+# 
+#     l1 is the payee, who will later issue some invoices.
+#     It arranges unpublished channels from the above public
+#     network:
+# 
+#         l5->l1: 7 * unit
+#         l6->l1: 5 * unit
+#         l4->l1: 3 * unit
+#         l7->l1: 2 * unit
+# 
+#     l2 and l3 are payers.
+#     They create some unpublished channels to the public network:
+# 
+#         l2->l4, l2->l6: 6 * unit each
+#         l3->l7, l3->l6: 6 * unit each
+# 
+#     Finally, l1 issues 6 * unit invoices, simultaneously, to l2 and l3.
+#     Both of them perform `pay` simultaneously, in order to test if
+#     they interfere with each other.
+# 
+#     This test then tries to check if both of them can pay, given
+#     that there is sufficient incoming capacity, and then some,
+#     to the payee, and the public network is perfectly balanced
+#     with more than sufficient capacity, as well.
+#     '''
+#     opts = {'feerates': (1000, 1000, 1000, 1000)}
+# 
+#     l1, l2, l3, l4, l5, l6, l7 = node_factory.get_nodes(7, opts=opts)
+# 
+#     # Unit
+#     unit = Millisatoshi(11000 * 1000)
+# 
+#     # Build the public network.
+#     public_network = [l4.fundbalancedchannel(l5, unit * 14),
+#                       l4.fundbalancedchannel(l6, unit * 14),
+#                       l5.fundbalancedchannel(l6, unit * 14),
+#                       l5.fundbalancedchannel(l7, unit * 14),
+#                       l6.fundbalancedchannel(l7, unit * 14)]
+# 
+#     # Build unpublished channels to the merchant l1.
+#     l4.rpc.connect(l1.info['id'], 'localhost', l1.port)
+#     l5.rpc.connect(l1.info['id'], 'localhost', l1.port)
+#     l6.rpc.connect(l1.info['id'], 'localhost', l1.port)
+#     l7.rpc.connect(l1.info['id'], 'localhost', l1.port)
+# 
+#     # If we're 'dual-funding', turn off the reciprocal funding
+#     # so that we can fund channels without making them balanced
+#     if EXPERIMENTAL_DUAL_FUND:
+#         for n in [l1, l2, l3, l4, l5, l6, l7]:
+#             n.rpc.call('funderupdate', {'fund_probability': 0})
+# 
+#     # The order in which the routes are built should not matter so
+#     # shuffle them.
+#     incoming_builders = [lambda: l5.fundchannel(l1, int((unit * 7).to_satoshi()), announce_channel=False),
+#                          lambda: l6.fundchannel(l1, int((unit * 5).to_satoshi()), announce_channel=False),
+#                          lambda: l4.fundchannel(l1, int((unit * 3).to_satoshi()), announce_channel=False),
+#                          lambda: l7.fundchannel(l1, int((unit * 2).to_satoshi()), announce_channel=False)]
+#     random.shuffle(incoming_builders)
+#     for b in incoming_builders:
+#         b()
+# 
+#     # Build unpublished channels from the buyers l2 and l3.
+#     l2.rpc.connect(l4.info['id'], 'localhost', l4.port)
+#     l2.rpc.connect(l6.info['id'], 'localhost', l6.port)
+#     l3.rpc.connect(l7.info['id'], 'localhost', l7.port)
+#     l3.rpc.connect(l6.info['id'], 'localhost', l6.port)
+#     l2.fundchannel(l4, int((unit * 6).to_satoshi()), announce_channel=False)
+#     l2.fundchannel(l6, int((unit * 6).to_satoshi()), announce_channel=False)
+#     l3.fundchannel(l7, int((unit * 6).to_satoshi()), announce_channel=False)
+#     l3.fundchannel(l6, int((unit * 6).to_satoshi()), announce_channel=False)
+# 
+#     # Now wait for the buyers to learn the entire public network.
+#     mine_funding_to_announce(bitcoind, [l1, l2, l3, l4, l5, l6, l7])
+#     for channel in public_network:
+#         wait_for(lambda: len(l2.rpc.listchannels(channel)['channels']) == 2)
+#         wait_for(lambda: len(l3.rpc.listchannels(channel)['channels']) == 2)
+# 
+#     # At this point, we have the following incoming channel capacities:
+#     # 74094000, 52314000, 30318000, 19318000
+# 
+#     # We *always* rotate through, since we have no published channels,
+#     # but we can select badly and get an overlap. e.g. first invoice
+#     # takes 30318000, 19318000 and 74094000.  Second will then take
+#     # 52314000, and have to reuse 30318000, which gets exhausted by the
+#     # first payer, thus leaving them unable to pay 66000000.
+# 
+#     # So we re-do this until we only have 4 or fewer routehints.
+#     while True:
+#         # Buyers check out some purchaseable stuff from the merchant.
+#         i2 = l1.rpc.invoice(unit * 6, ''.join(random.choice(string.ascii_letters + string.digits) for _ in range(20)), 'i2')['bolt11']
+#         i3 = l1.rpc.invoice(unit * 6, ''.join(random.choice(string.ascii_letters + string.digits) for _ in range(20)), 'i3')['bolt11']
+#         if len(l1.rpc.decodepay(i2)['routes'] + l1.rpc.decodepay(i3)['routes']) <= 4:
+#             break
+# 
+#     # Pay simultaneously!
+#     p2 = executor.submit(l2.rpc.pay, i2)
+#     p3 = executor.submit(l3.rpc.pay, i3)
+# 
+#     # Both payments should succeed.
+#     p2.result(TIMEOUT)
+#     p3.result(TIMEOUT)
+# 
+# 
+# def test_large_mpp_presplit(node_factory):
+#     """Make sure that ludicrous amounts don't saturate channels
+# 
+#     We aim to have at most PRESPLIT_MAX_SPLITS HTLCs created directly from the
+#     `presplit` modifier. The modifier will scale up its target size to
+#     guarantee this, while still bucketizing payments that are in the following
+#     range:
+# 
+#     ```
+#     target_size = PRESPLIT_MAX_SPLITS^{n} + MPP_TARGET_SIZE
+#     target_size < amount <= target_size * PRESPLIT_MAX_SPLITS
+#     ```
+# 
+#     """
+#     PRESPLIT_MAX_SPLITS = 16
+#     MPP_TARGET_SIZE = 10 ** 7
+#     amt = 400 * MPP_TARGET_SIZE
+# 
+#     l1, l2, l3 = node_factory.line_graph(
+#         3, fundamount=10**8, wait_for_announce=True,
+#         opts={'wumbo': None}
+#     )
+# 
+#     inv = l3.rpc.invoice(amt, 'lbl', 'desc')['bolt11']
+#     p = l1.rpc.pay(inv)
+# 
+#     assert(p['parts'] <= PRESPLIT_MAX_SPLITS)
+#     inv = l3.rpc.listinvoices()['invoices'][0]
+# 
+#     assert(inv['amount_msat'] == inv['amount_received_msat'])
+# 
+# 
+# @pytest.mark.developer("builds large network, which is slow if not DEVELOPER")
+# @pytest.mark.slow_test
+# def test_mpp_overload_payee(node_factory, bitcoind):
+#     """
+#     We had a bug where if the payer is unusually well-connected compared
+#     to the payee, the payee is unable to accept a large payment since the
+#     payer will split it into lots of tiny payments, which would choke the
+#     max-concurrent-htlcs limit of the payee.
+#     """
+#     # Default value as of this writing.
+#     # However, with anchor commitments we might be able to safely lift this
+#     # default limit in the future, so explicitly put this value here, since
+#     # that is what our test assumes.
+#     opts = {'max-concurrent-htlcs': 30}
+# 
+#     l1, l2, l3, l4, l5, l6 = node_factory.get_nodes(6, opts=opts)
+# 
+#     # Respect wumbo.
+#     # Using max-sized channels shows that the issue is not capacity
+#     # but rather max-concurrent-htlcs.
+#     # This is grade-school level.
+#     amt = 2**24 - 1
+# 
+#     # Build the public network.
+#     # l1 is the very well-connected payer.
+#     # l2 is the poorly-connected payee.
+#     # l3->l6 are well-connected hop nodes.
+#     public_network = [l1.fundbalancedchannel(l3, amt),
+#                       l1.fundbalancedchannel(l4, amt),
+#                       l1.fundbalancedchannel(l5, amt),
+#                       l1.fundbalancedchannel(l6, amt),
+#                       l2.fundbalancedchannel(l6, amt),
+#                       l3.fundbalancedchannel(l4, amt),
+#                       l3.fundbalancedchannel(l5, amt),
+#                       l3.fundbalancedchannel(l6, amt),
+#                       l4.fundbalancedchannel(l5, amt),
+#                       l5.fundbalancedchannel(l6, amt)]
+# 
+#     # Ensure l1 knows the entire public network.
+#     mine_funding_to_announce(bitcoind, [l1, l2, l3, l4, l5, l6])
+#     for c in public_network:
+#         wait_for(lambda: len(l1.rpc.listchannels(c)['channels']) >= 2)
+# 
+#     # Now create a 400,000-sat invoice.
+#     # This assumes the MPP presplitter strongly prefers to
+#     # create lot sizes of 10,000 sats each.
+#     # This leads the presplitter to prefer to split into
+#     # around 40 HTLCs of 10,000 sats each, but since
+#     # max-concurrent-htlcs is set to 30, l2 would be unable
+#     # to receive.
+#     inv = l2.rpc.invoice(Millisatoshi(400000 * 1000), 'i', 'i')['bolt11']
+# 
+#     # pay.
+#     l1.rpc.pay(inv)
+# 
+# 
+# @unittest.skipIf(EXPERIMENTAL_FEATURES, "this is always on with EXPERIMENTAL_FEATURES")
+# def test_offer_needs_option(node_factory):
+#     """Make sure we don't make offers without offer command"""
+#     l1 = node_factory.get_node()
+#     with pytest.raises(RpcError, match='experimental-offers not enabled'):
+#         l1.rpc.call('offer', {'amount': '1msat', 'description': 'test'})
+#     with pytest.raises(RpcError, match='experimental-offers not enabled'):
+#         l1.rpc.call('invoicerequest', {'amount': '2msat',
+#                                        'description': 'simple test'})
+#     with pytest.raises(RpcError, match='Unknown command'):
+#         l1.rpc.call('fetchinvoice', {'offer': 'aaaa'})
+# 
+#     # Decode still works though
+#     assert l1.rpc.decode('lno1qgsqvgnwgcg35z6ee2h3yczraddm72xrfua9uve2rlrm9deu7xyfzrcgqyqs5pr5v4ehg93pqfnwgkvdr57yzh6h92zg3qctvrm7w38djg67kzcm4yeg8vc4cq63s')['valid']
+# 
+# 
+# def test_offer(node_factory, bitcoind):
+#     plugin = os.path.join(os.path.dirname(__file__), 'plugins/currencyUSDAUD5000.py')
+#     l1 = node_factory.get_node(options={'plugin': plugin, 'experimental-offers': None})
+# 
+#     # Try empty description
+#     ret = l1.rpc.call('offer', [9, ''])
+#     l1.rpc.decode(ret['bolt12'])
+# 
+#     bolt12tool = os.path.join(os.path.dirname(__file__), "..", "devtools", "bolt12-cli")
+#     # Try different amount strings
+#     for amount in ['1msat', '0.1btc', 'any', '1USD', '1.10AUD']:
+#         ret = l1.rpc.call('offer', {'amount': amount,
+#                                     'description': 'test for ' + amount})
+#         offer = only_one(l1.rpc.call('listoffers', [ret['offer_id']])['offers'])
+# 
+#         assert offer['bolt12'] == ret['bolt12']
+#         assert offer['offer_id'] == ret['offer_id']
+# 
+#         output = subprocess.check_output([bolt12tool, 'decode',
+#                                           offer['bolt12']]).decode('ASCII')
+#         if amount == 'any':
+#             assert 'amount' not in output
+#         else:
+#             assert 'amount' in output
+# 
+#     # Try wrong amount precision:
+#     with pytest.raises(RpcError, match='Currency AUD requires 2 minor units'):
+#         l1.rpc.call('offer', {'amount': '1.100AUD',
+#                               'description': 'test for invalid amount'})
+# 
+#     with pytest.raises(RpcError, match='Currency AUD requires 2 minor units'):
+#         l1.rpc.call('offer', {'amount': '1.1AUD',
+#                               'description': 'test for invalid amount'})
+# 
+#     # Make sure it fails on unknown currencies.
+#     with pytest.raises(RpcError, match='No values available for currency EUR'):
+#         l1.rpc.call('offer', {'amount': '1.00EUR',
+#                               'description': 'test for unknown currency'})
+# 
+#     # Test label and description
+#     weird_label = 'label \\ " \t \n'
+#     weird_desc = 'description \\ " \t \n ナンセンス 1杯'
+#     ret = l1.rpc.call('offer', {'amount': '0.1btc',
+#                                 'description': weird_desc,
+#                                 'label': weird_label})
+#     offer = only_one(l1.rpc.call('listoffers', [ret['offer_id']])['offers'])
+#     assert offer['label'] == weird_label
+# 
+#     output = subprocess.check_output([bolt12tool, 'decode',
+#                                       offer['bolt12']]).decode('UTF-8')
+#     assert 'description: ' + weird_desc in output
+# 
+#     # Test issuer
+#     weird_issuer = 'description \\ " \t \n ナンセンス 1杯'
+#     ret = l1.rpc.call('offer', {'amount': '100000sat',
+#                                 'description': 'issuer test',
+#                                 'issuer': weird_issuer})
+#     offer = only_one(l1.rpc.call('listoffers', [ret['offer_id']])['offers'])
+# 
+#     output = subprocess.check_output([bolt12tool, 'decode',
+#                                       offer['bolt12']]).decode('UTF-8')
+#     assert 'issuer: ' + weird_issuer in output
+# 
+#     # Test quantity
+#     ret = l1.rpc.call('offer', {'amount': '100000sat',
+#                                 'description': 'quantity_max existence test',
+#                                 'quantity_max': 0})
+#     offer = only_one(l1.rpc.call('listoffers', [ret['offer_id']])['offers'])
+#     output = subprocess.check_output([bolt12tool, 'decode',
+#                                       offer['bolt12']]).decode('UTF-8')
+#     assert 'quantity_max: 0' in output
+# 
+#     ret = l1.rpc.call('offer', {'amount': '100000sat',
+#                                 'description': 'quantity_max test',
+#                                 'quantity_max': 2})
+#     offer = only_one(l1.rpc.call('listoffers', [ret['offer_id']])['offers'])
+#     output = subprocess.check_output([bolt12tool, 'decode',
+#                                       offer['bolt12']]).decode('UTF-8')
+#     assert 'quantity_max: 2' in output
+# 
+#     # Test absolute_expiry
+#     exp = int(time.time() + 2)
+#     ret = l1.rpc.call('offer', {'amount': '100000sat',
+#                                 'description': 'quantity_max test',
+#                                 'absolute_expiry': exp})
+#     offer = only_one(l1.rpc.call('listoffers', [ret['offer_id']])['offers'])
+#     output = subprocess.check_output([bolt12tool, 'decode',
+#                                       offer['bolt12']]).decode('UTF-8')
+#     assert 'absolute_expiry: {}'.format(exp) in output
+# 
+#     # Recurrence tests!
+#     for r in [['1second', 'seconds', 1],
+#               ['10seconds', 'seconds', 10],
+#               ['1minute', 'seconds', 60],
+#               ['10minutes', 'seconds', 600],
+#               ['1hour', 'seconds', 3600],
+#               ['10hours', 'seconds', 36000],
+#               ['1day', 'days', 1],
+#               ['10days', 'days', 10],
+#               ['1week', 'days', 7],
+#               ['10weeks', 'days', 70],
+#               ['1month', 'months', 1],
+#               ['10months', 'months', 10],
+#               ['1year', 'years', 1],
+#               ['10years', 'years', 10]]:
+#         ret = l1.rpc.call('offer', {'amount': '100000sat',
+#                                     'description': 'quantity_max test',
+#                                     'recurrence': r[0]})
+#         offer = only_one(l1.rpc.call('listoffers', [ret['offer_id']])['offers'])
+#         output = subprocess.check_output([bolt12tool, 'decode',
+#                                           offer['bolt12']]).decode('UTF-8')
+#         assert 'recurrence: every {} {}\n'.format(r[2], r[1]) in output
+# 
+#     # Test limit
+#     ret = l1.rpc.call('offer', {'amount': '100000sat',
+#                                 'description': 'quantity_max test',
+#                                 'recurrence': '10minutes',
+#                                 'recurrence_limit': 5})
+#     offer = only_one(l1.rpc.call('listoffers', [ret['offer_id']])['offers'])
+#     output = subprocess.check_output([bolt12tool, 'decode',
+#                                       offer['bolt12']]).decode('UTF-8')
+#     assert 'recurrence: every 600 seconds limit 5\n' in output
+# 
+#     # Test base
+#     # (1456740000 == 10:00:00 (am) UTC on 29 February, 2016)
+#     ret = l1.rpc.call('offer', {'amount': '100000sat',
+#                                 'description': 'quantity_max test',
+#                                 'recurrence': '10minutes',
+#                                 'recurrence_base': '@1456740000'})
+#     offer = only_one(l1.rpc.call('listoffers', [ret['offer_id']])['offers'])
+#     output = subprocess.check_output([bolt12tool, 'decode',
+#                                       offer['bolt12']]).decode('UTF-8')
+#     assert 'recurrence: every 600 seconds start 1456740000' in output
+#     assert '(can start any period)' not in output
+# 
+#     ret = l1.rpc.call('offer', {'amount': '100000sat',
+#                                 'description': 'quantity_max test',
+#                                 'recurrence': '10minutes',
+#                                 'recurrence_base': 1456740000})
+#     offer = only_one(l1.rpc.call('listoffers', [ret['offer_id']])['offers'])
+#     output = subprocess.check_output([bolt12tool, 'decode',
+#                                       offer['bolt12']]).decode('UTF-8')
+#     assert 'recurrence: every 600 seconds start 1456740000' in output
+#     assert '(can start any period)' in output
+# 
+#     # Test paywindow
+#     ret = l1.rpc.call('offer', {'amount': '100000sat',
+#                                 'description': 'quantity_max test',
+#                                 'recurrence': '10minutes',
+#                                 'recurrence_paywindow': '-10+20'})
+#     offer = only_one(l1.rpc.call('listoffers', [ret['offer_id']])['offers'])
+#     output = subprocess.check_output([bolt12tool, 'decode',
+#                                       offer['bolt12']]).decode('UTF-8')
+#     assert 'recurrence: every 600 seconds paywindow -10 to +20\n' in output
+# 
+#     ret = l1.rpc.call('offer', {'amount': '100000sat',
+#                                 'description': 'quantity_max test',
+#                                 'recurrence': '10minutes',
+#                                 'recurrence_paywindow': '-10+600%'})
+#     offer = only_one(l1.rpc.call('listoffers', [ret['offer_id']])['offers'])
+#     output = subprocess.check_output([bolt12tool, 'decode',
+#                                       offer['bolt12']]).decode('UTF-8')
+#     assert 'recurrence: every 600 seconds paywindow -10 to +600 (pay proportional)\n' in output
+# 
+# 
+# def test_offer_deprecated_api(node_factory, bitcoind):
+#     l1, l2 = node_factory.line_graph(2, opts={'experimental-offers': None,
+#                                               'allow-deprecated-apis': True})
+# 
+#     offer = l2.rpc.call('offer', {'amount': '2msat',
+#                                   'description': 'test_offer_deprecated_api'})
+#     inv = l1.rpc.call('fetchinvoice', {'offer': offer['bolt12']})
+# 
+#     # Deprecated fields make schema checker upset.
+#     l1.rpc.jsonschemas = {}
+#     l1.rpc.pay(inv['invoice'])
+# 
+# 
+# @pytest.mark.developer("dev-no-modern-onion is DEVELOPER-only")
+# def test_fetchinvoice_3hop(node_factory, bitcoind):
+#     l1, l2, l3, l4 = node_factory.line_graph(4, wait_for_announce=True,
+#                                              opts={'experimental-offers': None,
+#                                                    'may_reconnect': True,
+#                                                    'dev-no-reconnect': None})
+#     offer1 = l4.rpc.call('offer', {'amount': '2msat',
+#                                    'description': 'simple test'})
+#     assert offer1['created'] is True
+# 
+#     l1.rpc.call('fetchinvoice', {'offer': offer1['bolt12']})
+# 
+# 
+# def test_fetchinvoice(node_factory, bitcoind):
+#     # We remove the conversion plugin on l3, causing it to get upset.
+#     l1, l2, l3 = node_factory.line_graph(3, wait_for_announce=True,
+#                                          opts=[{'experimental-offers': None},
+#                                                {'experimental-offers': None},
+#                                                {'experimental-offers': None,
+#                                                 'allow_broken_log': True}])
+# 
+#     # Simple offer first.
+#     offer1 = l3.rpc.call('offer', {'amount': '2msat',
+#                                    'description': 'simple test'})
+#     assert offer1['created'] is True
+# 
+#     inv1 = l1.rpc.call('fetchinvoice', {'offer': offer1['bolt12']})
+#     inv2 = l1.rpc.call('fetchinvoice', {'offer': offer1['bolt12'],
+#                                         'payer_note': 'Thanks for the fish!'})
+#     assert inv1 != inv2
+#     assert 'next_period' not in inv1
+#     assert 'next_period' not in inv2
+#     assert only_one(l3.rpc.call('listoffers', [offer1['offer_id']])['offers'])['used'] is False
+#     l1.rpc.pay(inv1['invoice'])
+#     assert only_one(l3.rpc.call('listoffers', [offer1['offer_id']])['offers'])['used'] is True
+#     l1.rpc.pay(inv2['invoice'])
+#     assert only_one(l3.rpc.call('listoffers', [offer1['offer_id']])['offers'])['used'] is True
+# 
+#     # listinvoices will show these on l3
+#     assert [x['local_offer_id'] for x in l3.rpc.listinvoices()['invoices']] == [offer1['offer_id'], offer1['offer_id']]
+# 
+#     assert 'invreq_payer_note' not in only_one(l3.rpc.call('listinvoices', {'invstring': inv1['invoice']})['invoices'])
+#     assert only_one(l3.rpc.call('listinvoices', {'invstring': inv2['invoice']})['invoices'])['invreq_payer_note'] == 'Thanks for the fish!'
+# 
+#     # BTW, test listinvoices-by-offer_id:
+#     assert len(l3.rpc.listinvoices(offer_id=offer1['offer_id'])['invoices']) == 2
+# 
+#     # We can also set the amount explicitly, to tip.
+#     inv1 = l1.rpc.call('fetchinvoice', {'offer': offer1['bolt12'], 'amount_msat': 3})
+#     assert l1.rpc.call('decode', [inv1['invoice']])['invoice_amount_msat'] == 3
+#     l1.rpc.pay(inv1['invoice'])
+# 
+#     # More than ~5x expected is rejected as absurd (it's actually a divide test,
+#     # which means we need 15 here, not 11).
+#     with pytest.raises(RpcError, match="Remote node sent failure message.*Amount vastly exceeds 2msat"):
+#         l1.rpc.call('fetchinvoice', {'offer': offer1['bolt12'], 'amount_msat': 15})
+# 
+#     # Underpay is rejected.
+#     with pytest.raises(RpcError, match="Remote node sent failure message.*Amount must be at least 2msat"):
+#         l1.rpc.call('fetchinvoice', {'offer': offer1['bolt12'], 'amount_msat': 1})
+# 
+#     # If no amount is specified in offer, one must be in invoice.
+#     offer_noamount = l3.rpc.call('offer', {'amount': 'any',
+#                                            'description': 'any amount test'})
+#     with pytest.raises(RpcError, match="msatoshi parameter required"):
+#         l1.rpc.call('fetchinvoice', {'offer': offer_noamount['bolt12']})
+#     inv1 = l1.rpc.call('fetchinvoice', {'offer': offer_noamount['bolt12'], 'amount_msat': 100})
+#     # But amount won't appear in changes
+#     assert 'msat' not in inv1['changes']
+# 
+#     # Single-use invoice can be fetched multiple times, only paid once.
+#     offer2 = l3.rpc.call('offer', {'amount': '1msat',
+#                                    'description': 'single-use test',
+#                                    'single_use': True})['bolt12']
+# 
+#     inv1 = l1.rpc.call('fetchinvoice', {'offer': offer2})
+#     inv2 = l1.rpc.call('fetchinvoice', {'offer': offer2})
+#     assert inv1 != inv2
+#     assert 'next_period' not in inv1
+#     assert 'next_period' not in inv2
+# 
+#     l1.rpc.pay(inv1['invoice'])
+# 
+#     # We can't pay the other one now.
+#     # FIXME: Even dummy blinded paths always return WIRE_INVALID_ONION_BLINDING!
+#     with pytest.raises(RpcError, match="INVALID_ONION_BLINDING.*'erring_node': '{}'".format(l3.info['id'])):
+#         l1.rpc.pay(inv2['invoice'])
+# 
+#     # We can't reuse the offer, either.
+#     with pytest.raises(RpcError, match='Offer no longer available'):
+#         l1.rpc.call('fetchinvoice', {'offer': offer2})
+# 
+#     # Recurring offer.
+#     offer3 = l2.rpc.call('offer', {'amount': '1msat',
+#                                    'description': 'recurring test',
+#                                    'recurrence': '1minutes'})
+#     assert only_one(l2.rpc.call('listoffers', [offer3['offer_id']])['offers'])['used'] is False
+# 
+#     ret = l1.rpc.call('fetchinvoice', {'offer': offer3['bolt12'],
+#                                        'recurrence_counter': 0,
+#                                        'recurrence_label': 'test recurrence'})
+#     period1 = ret['next_period']
+#     assert period1['counter'] == 1
+#     assert period1['endtime'] == period1['starttime'] + 59
+#     assert period1['paywindow_start'] == period1['starttime'] - 60
+#     assert period1['paywindow_end'] == period1['endtime']
+#     assert only_one(l2.rpc.call('listoffers', [offer3['offer_id']])['offers'])['used'] is False
+# 
+#     l1.rpc.pay(ret['invoice'], label='test recurrence')
+#     assert only_one(l2.rpc.call('listoffers', [offer3['offer_id']])['offers'])['used'] is True
+# 
+#     ret = l1.rpc.call('fetchinvoice', {'offer': offer3['bolt12'],
+#                                        'recurrence_counter': 1,
+#                                        'recurrence_label': 'test recurrence'})
+#     period2 = ret['next_period']
+#     assert period2['counter'] == 2
+#     assert period2['starttime'] == period1['endtime'] + 1
+#     assert period2['endtime'] == period2['starttime'] + 59
+#     assert period2['paywindow_start'] == period2['starttime'] - 60
+#     assert period2['paywindow_end'] == period2['endtime']
+# 
+#     # Can't request 2 before paying 1.
+#     with pytest.raises(RpcError, match='previous invoice has not been paid'):
+#         l1.rpc.call('fetchinvoice', {'offer': offer3['bolt12'],
+#                                      'recurrence_counter': 2,
+#                                      'recurrence_label': 'test recurrence'})
+# 
+#     l1.rpc.pay(ret['invoice'], label='test recurrence')
+# 
+#     # Now we can, but it's too early:
+#     with pytest.raises(RpcError, match="Too early: can't send until time {}".format(period1['starttime'])):
+#         l1.rpc.call('fetchinvoice', {'offer': offer3['bolt12'],
+#                                      'recurrence_counter': 2,
+#                                      'recurrence_label': 'test recurrence'})
+# 
+#     # Wait until the correct moment.
+#     while time.time() < period1['starttime']:
+#         time.sleep(1)
+# 
+#     l1.rpc.call('fetchinvoice', {'offer': offer3['bolt12'],
+#                                  'recurrence_counter': 2,
+#                                  'recurrence_label': 'test recurrence'})
+# 
+#     # Check we can request invoice without a channel.
+#     l4 = node_factory.get_node(options={'experimental-offers': None})
+#     l4.rpc.connect(l2.info['id'], 'localhost', l2.port)
+#     # ... even if we can't find ourselves.
+#     l4.rpc.call('fetchinvoice', {'offer': offer3['bolt12'],
+#                                  'recurrence_counter': 0,
+#                                  'recurrence_label': 'test nochannel'})
+#     # ... even if we know it from gossmap
+#     wait_for(lambda: l4.rpc.listnodes(l3.info['id'])['nodes'] != [])
+#     l4.rpc.connect(l3.info['id'], 'localhost', l3.port)
+#     l4.rpc.call('fetchinvoice', {'offer': offer1['bolt12']})
+# 
+#     # Now, test amount in different currency!
+#     plugin = os.path.join(os.path.dirname(__file__), 'plugins/currencyUSDAUD5000.py')
+#     l3.rpc.plugin_start(plugin)
+# 
+#     offerusd = l3.rpc.call('offer', {'amount': '10.05USD',
+#                                      'description': 'USD test'})['bolt12']
+# 
+#     inv = l1.rpc.call('fetchinvoice', {'offer': offerusd})
+#     assert inv['changes']['amount_msat'] == Millisatoshi(int(10.05 * 5000))
+# 
+#     # If we remove plugin, it can no longer give us an invoice.
+#     l3.rpc.plugin_stop(plugin)
+# 
+#     with pytest.raises(RpcError, match="Internal error"):
+#         l1.rpc.call('fetchinvoice', {'offer': offerusd})
+#     l3.daemon.wait_for_log("Unknown command 'currencyconvert'")
+#     # But we can still pay the (already-converted) invoice.
+#     l1.rpc.pay(inv['invoice'])
+# 
+#     # Identical creation gives it again, just with created false.
+#     offer1 = l3.rpc.call('offer', {'amount': '2msat',
+#                                    'description': 'simple test'})
+#     assert offer1['created'] is False
+#     l3.rpc.call('disableoffer', {'offer_id': offer1['offer_id']})
+#     with pytest.raises(RpcError, match="1000.*Already exists, but isn't active"):
+#         l3.rpc.call('offer', {'amount': '2msat',
+#                               'description': 'simple test'})
+# 
+#     # Test timeout.
+#     l3.stop()
+#     with pytest.raises(RpcError, match='Timeout waiting for response'):
+#         l1.rpc.call('fetchinvoice', {'offer': offer1['bolt12'], 'timeout': 10})
+# 
+#     # Now try an offer with a more complex paywindow (only 10 seconds before)
+#     offer = l2.rpc.call('offer', {'amount': '1msat',
+#                                   'description': 'paywindow test',
+#                                   'recurrence': '20seconds',
+#                                   'recurrence_paywindow': '-10+0'})['bolt12']
+# 
+#     ret = l1.rpc.call('fetchinvoice', {'offer': offer,
+#                                        'recurrence_counter': 0,
+#                                        'recurrence_label': 'test paywindow'})
+#     period3 = ret['next_period']
+#     assert period3['counter'] == 1
+#     assert period3['endtime'] == period3['starttime'] + 19
+#     assert period3['paywindow_start'] == period3['starttime'] - 10
+#     assert period3['paywindow_end'] == period3['starttime']
+#     l1.rpc.pay(ret['invoice'], label='test paywindow')
+# 
+#     # We can get another invoice, as many times as we want.
+#     # (It may return the same one!).
+#     while int(time.time()) <= period3['paywindow_start']:
+#         time.sleep(1)
+# 
+#     l1.rpc.call('fetchinvoice', {'offer': offer,
+#                                  'recurrence_counter': 1,
+#                                  'recurrence_label': 'test paywindow'})
+#     l1.rpc.call('fetchinvoice', {'offer': offer,
+#                                  'recurrence_counter': 1,
+#                                  'recurrence_label': 'test paywindow'})
+# 
+#     # Wait until too late!
+#     while int(time.time()) <= period3['paywindow_end']:
+#         time.sleep(1)
+# 
+#     with pytest.raises(RpcError, match="Too late: expired time {}".format(period3['paywindow_end'])):
+#         l1.rpc.call('fetchinvoice', {'offer': offer,
+#                                      'recurrence_counter': 1,
+#                                      'recurrence_label': 'test paywindow'})
+# 
+# 
+# @pytest.mark.developer("Needs dev-allow-localhost for autoconnect, dev-force-features to avoid routing onionmsgs")
+# def test_fetchinvoice_autoconnect(node_factory, bitcoind):
+#     """We should autoconnect if we need to, to route."""
+# 
+#     if EXPERIMENTAL_FEATURES:
+#         # We have to force option_onion_messages off!
+#         opts1 = {'dev-force-features': '-39'}
+#     else:
+#         opts1 = {}
+#     l1, l2 = node_factory.line_graph(2, wait_for_announce=True,
+#                                      opts=[opts1,
+#                                            {'experimental-offers': None,
+#                                             'dev-allow-localhost': None}])
+# 
+#     l3 = node_factory.get_node(options={'experimental-offers': None})
+#     l3.rpc.connect(l1.info['id'], 'localhost', l1.port)
+#     wait_for(lambda: l3.rpc.listnodes(l2.info['id'])['nodes'] != [])
+# 
+#     offer = l2.rpc.call('offer', {'amount': '2msat',
+#                                   'description': 'simple test'})
+#     l3.rpc.call('fetchinvoice', {'offer': offer['bolt12']})
+#     assert l3.rpc.listpeers(l2.info['id'])['peers'] != []
+# 
+#     # Similarly for an invoice_request.
+#     l3.rpc.disconnect(l2.info['id'])
+#     invreq = l2.rpc.call('invoicerequest', {'amount': '2msat',
+#                                             'description': 'simple test'})
+#     # Ofc l2 can't actually pay it!
+#     with pytest.raises(RpcError, match='pay attempt failed: "Ran out of routes to try'):
+#         l3.rpc.call('sendinvoice', {'invreq': invreq['bolt12'], 'label': 'payme!'})
+# 
+#     assert l3.rpc.listpeers(l2.info['id'])['peers'] != []
+# 
+#     # But if we create a channel l3->l1->l2 (and balance!), l2 can!
+#     node_factory.join_nodes([l3, l1], wait_for_announce=True)
+#     # Make sure l2 knows about it
+#     wait_for(lambda: l2.rpc.listnodes(l3.info['id'])['nodes'] != [])
+# 
+#     l3.rpc.pay(l2.rpc.invoice(FUNDAMOUNT * 500, 'balancer', 'balancer')['bolt11'])
+#     # Make sure l2 has capacity (can be still resolving!).
+#     wait_for(lambda: only_one(l2.rpc.listpeerchannels(l1.info['id'])['channels'])['spendable_msat'] != Millisatoshi(0))
+# 
+#     l3.rpc.disconnect(l2.info['id'])
+#     l3.rpc.call('sendinvoice', {'invreq': invreq['bolt12'], 'label': 'payme for real!'})
+#     # It will have autoconnected, to send invoice (since l1 says it doesn't do onion messages!)
+#     assert l3.rpc.listpeers(l2.info['id'])['peers'] != []
+# 
+# 
+# def test_pay_waitblockheight_timeout(node_factory, bitcoind):
+#     plugin = os.path.join(os.path.dirname(__file__), 'plugins', 'endlesswaitblockheight.py')
+#     l1, l2 = node_factory.line_graph(2, opts=[{}, {'plugin': plugin}])
+# 
+#     sync_blockheight(bitcoind, [l1, l2])
+#     inv = l2.rpc.invoice(42, 'lbl', 'desc')['bolt11']
+# 
+#     with pytest.raises(RpcError, match=r'WIRE_INCORRECT_OR_UNKNOWN_PAYMENT_DETAILS'):
+#         l1.rpc.pay(inv)
+# 
+#     # Post mortem checks that we tried only once.
+#     status = l1.rpc.paystatus(inv)
+# 
+#     # Should have only one attempt that triggered the wait, which then failed.
+#     assert len(status['pay']) == 1
+#     assert len(status['pay'][0]['attempts']) == 1
+# 
+# 
+# @pytest.mark.developer("dev-rawrequest is DEVELOPER-only")
+# def test_dev_rawrequest(node_factory):
+#     l1, l2 = node_factory.line_graph(2, fundchannel=False,
+#                                      opts={'experimental-offers': None})
+# 
+#     offer = l2.rpc.call('offer', {'amount': '2msat',
+#                                   'description': 'simple test'})
+#     # Get fetchinvoice to make us an invoice_request
+#     l1.rpc.call('fetchinvoice', {'offer': offer['bolt12']})
+# 
+#     m = re.search(r'invoice_request: \\"([a-z0-9]*)\\"', l1.daemon.is_in_log('invoice_request:'))
+#     ret = l1.rpc.call('dev-rawrequest', {'invreq': m.group(1),
+#                                          'nodeid': l2.info['id'],
+#                                          'timeout': 10})
+#     assert 'invoice' in ret
+# 
+# 
+# def test_sendinvoice(node_factory, bitcoind):
+#     l2opts = {'experimental-offers': None}
+#     l1, l2 = node_factory.line_graph(2, wait_for_announce=True,
+#                                      opts=[{'experimental-offers': None},
+#                                            l2opts])
+# 
+#     # Simple offer to send money (balances channel a little)
+#     invreq = l1.rpc.call('invoicerequest', {'amount': '100000sat',
+#                                             'description': 'simple test'})
+# 
+#     # Fetchinvoice will refuse, since it's not an offer.
+#     with pytest.raises(RpcError, match='unexpected prefix lnr'):
+#         l2.rpc.call('fetchinvoice', {'offer': invreq['bolt12']})
+# 
+#     # Pay will refuse, since it's not an invoice.
+#     with pytest.raises(RpcError, match='unexpected prefix lnr'):
+#         l2.rpc.call('fetchinvoice', {'offer': invreq['bolt12']})
+# 
+#     # used will be false
+#     assert only_one(l1.rpc.call('listinvoicerequests', [invreq['invreq_id']])['invoicerequests'])['used'] is False
+# 
+#     # sendinvoice should work.
+#     out = l2.rpc.call('sendinvoice', {'invreq': invreq['bolt12'],
+#                                       'label': 'test sendinvoice 1'})
+#     assert out['label'] == 'test sendinvoice 1'
+#     assert out['description'] == 'simple test'
+#     assert 'bolt12' in out
+#     assert 'payment_hash' in out
+#     assert out['status'] == 'paid'
+#     assert 'payment_preimage' in out
+#     assert 'expires_at' in out
+#     assert out['amount_msat'] == Millisatoshi(100000000)
+#     assert 'pay_index' in out
+#     assert out['amount_received_msat'] == Millisatoshi(100000000)
+# 
+#     # Note, if we're slow, this fails with "Offer no longer available",
+#     # *but* if it hasn't heard about payment success yet, l2 will fail
+#     # simply because payments are already pending.
+#     with pytest.raises(RpcError, match='no longer available|pay attempt failed'):
+#         l2.rpc.call('sendinvoice', {'invreq': invreq['bolt12'],
+#                                     'label': 'test sendinvoice 2'})
+# 
+#     # Technically, l1 may not have gotten payment success, so we need to wait.
+#     wait_for(lambda: only_one(l1.rpc.call('listinvoicerequests', [invreq['invreq_id']])['invoicerequests'])['used'] is True)
+# 
+#     # Offer with issuer: we must copy issuer into our invoice!
+#     invreq = l1.rpc.call('invoicerequest', {'amount': '10000sat',
+#                                             'description': 'simple test',
+#                                             'issuer': "clightning test suite"})
+# 
+#     out = l2.rpc.call('sendinvoice', {'invreq': invreq['bolt12'],
+#                                       'label': 'test sendinvoice 3'})
+#     assert out['label'] == 'test sendinvoice 3'
+#     assert out['description'] == 'simple test'
+#     assert 'issuer' not in out
+#     assert 'bolt12' in out
+#     assert 'payment_hash' in out
+#     assert out['status'] == 'paid'
+#     assert 'payment_preimage' in out
+#     assert 'expires_at' in out
+#     assert out['amount_msat'] == Millisatoshi(10000000)
+#     assert 'pay_index' in out
+#     assert out['amount_received_msat'] == Millisatoshi(10000000)
+# 
+# 
+# def test_self_pay(node_factory):
+#     """Repro test for issue 4345: pay ourselves via the pay plugin.
+# 
+#     """
+#     l1, l2 = node_factory.line_graph(2, wait_for_announce=True)
+# 
+#     inv = l1.rpc.invoice(10000, 'test', 'test')['bolt11']
+# 
+#     with pytest.raises(RpcError):
+#         l1.rpc.pay(inv)
+# 
+# 
+# @unittest.skipIf(TEST_NETWORK != 'regtest', "Canned invoice is network specific")
+# def test_unreachable_routehint(node_factory, bitcoind):
+#     """Test that we discard routehints that we can't reach.
+# 
+#     Reachability is tested by checking whether we can reach the
+#     entrypoint of the routehint, i.e., the first node in the
+#     routehint. The network we create is partitioned on purpose for
+#     this: first we attempt with an unknown destination and an unknown
+#     routehint entrypoint, later we make them known, but still
+#     unreachable, by connecting them without a channel.
+# 
+#     """
+# 
+#     # Create a partitioned network, first without connecting it, then
+#     # connecting it without a channel so they can sync gossip. Notice
+#     # that l4 is there only to trick the deadend heuristic.
+#     l1, l2 = node_factory.line_graph(2, wait_for_announce=True)
+#     l3, l4, l5 = node_factory.line_graph(3, wait_for_announce=True)
+#     entrypoint = '0382ce59ebf18be7d84677c2e35f23294b9992ceca95491fcf8a56c6cb2d9de199'
+# 
+#     # Generate an invoice with exactly one routehint.
+#     for i in range(100):
+#         invoice = l5.rpc.invoice(10, 'attempt{}'.format(i), 'description')['bolt11']
+#         decoded = l1.rpc.decodepay(invoice)
+#         if 'routes' in decoded and len(decoded['routes']) == 1:
+#             break
+# 
+#     assert('routes' in decoded and len(decoded['routes']) == 1)
+# 
+#     with pytest.raises(RpcError, match=r'Destination [a-f0-9]{66} is not reachable'):
+#         l1.rpc.pay(invoice)
+# 
+#     l1.daemon.wait_for_log(
+#         r"Removed routehint 0 because entrypoint {entrypoint} is unknown.".format(
+#             entrypoint=entrypoint
+#         )
+#     )
+# 
+#     # Now connect l2 to l3 to create a bridge, but without a
+#     # channel. The entrypoint will become known, but still
+#     # unreachable, resulting in a slightly different error message,
+#     # but the routehint will still be removed.
+#     l2.connect(l3)
+#     wait_for(lambda: len(l1.rpc.listnodes(entrypoint)['nodes']) == 1)
+# 
+#     with pytest.raises(RpcError, match=r'Destination [a-f0-9]{66} is not reachable') as excinfo:
+#         l1.rpc.pay(invoice)
+# 
+#     # Verify that we failed for the correct reason.
+#     l1.daemon.wait_for_log(
+#         r"Removed routehint 0 because entrypoint {entrypoint} is unreachable.".format(
+#             entrypoint=entrypoint
+#         )
+#     )
+# 
+#     # Since we aborted once we realized the destination is unreachable
+#     # both directly, and via the routehints we should now just have a
+#     # single attempt.
+#     assert(len(excinfo.value.error['attempts']) == 1)
+# 
+# 
+# def test_routehint_tous(node_factory, bitcoind):
+#     """
+# Test bug where trying to pay an invoice from an *offline* node which
+# gives a routehint straight to us causes an issue
+# """
+# 
+#     # Existence of l1 makes l3 use l2 for routehint (otherwise it sees deadend)
+#     l1, l2 = node_factory.line_graph(2, wait_for_announce=True)
+#     l3 = node_factory.get_node()
+#     l3.rpc.connect(l2.info['id'], 'localhost', l2.port)
+#     scid23, _ = l2.fundchannel(l3, 1000000, announce_channel=False)
+#     # Make sure l3 sees l1->l2 channel.
+#     wait_for(lambda: l3.rpc.listnodes(l1.info['id'])['nodes'] != [])
+# 
+#     inv = l3.rpc.invoice(10, "test", "test")['bolt11']
+#     decoded = l3.rpc.decodepay(inv)
+#     assert(only_one(only_one(decoded['routes']))['short_channel_id']
+#            == only_one(l3.rpc.listpeerchannels()['channels'])['alias']['remote'])
+# 
+#     l3.stop()
+#     with pytest.raises(RpcError, match=r'Destination .* is not reachable directly and all routehints were unusable'):
+#         l2.rpc.pay(inv)
+# 
+# 
+# def test_pay_low_max_htlcs(node_factory):
+#     """Test we can pay if *any* HTLC slots are available"""
+# 
+#     l1, l2, l3 = node_factory.line_graph(3,
+#                                          opts={'max-concurrent-htlcs': 1},
+#                                          wait_for_announce=True)
+#     l1.rpc.pay(l3.rpc.invoice(FUNDAMOUNT * 50, "test", "test")['bolt11'])
+#     l1.daemon.wait_for_log(
+#         r'Number of pre-split HTLCs \([0-9]+\) exceeds our HTLC budget \([0-9]+\), skipping pre-splitter'
+#     )
+# 
+# 
+# def test_setchannel_enforcement_delay(node_factory, bitcoind):
+#     # Fees start at 1msat + 1%
+#     l1, l2, l3 = node_factory.line_graph(3, wait_for_announce=True,
+#                                          opts={'fee-base': 1,
+#                                                'fee-per-satoshi': 10000})
+# 
+#     chanid1 = only_one(l1.rpc.listpeerchannels(l2.info['id'])['channels'])['short_channel_id']
+#     chanid2 = only_one(l2.rpc.listpeerchannels(l3.info['id'])['channels'])['short_channel_id']
+# 
+#     route = [{'amount_msat': 1011,
+#               'id': l2.info['id'],
+#               'delay': 20,
+#               'channel': chanid1},
+#              {'amount_msat': 1000,
+#               'id': l3.info['id'],
+#               'delay': 10,
+#               'channel': chanid2}]
+# 
+#     # This works.
+#     inv = l3.rpc.invoice(1000, "test1", "test1")
+#     l1.rpc.sendpay(route,
+#                    payment_hash=inv['payment_hash'],
+#                    payment_secret=inv['payment_secret'])
+#     l1.rpc.waitsendpay(inv['payment_hash'])
+# 
+#     # Increase fee immediately; l1 payment rejected.
+#     l2.rpc.setchannel("all", 2, 10000, enforcedelay=0)
+# 
+#     inv = l3.rpc.invoice(1000, "test2", "test2")
+#     l1.rpc.sendpay(route,
+#                    payment_hash=inv['payment_hash'],
+#                    payment_secret=inv['payment_secret'])
+#     with pytest.raises(RpcError, match=r'WIRE_FEE_INSUFFICIENT'):
+#         l1.rpc.waitsendpay(inv['payment_hash'])
+# 
+#     # Test increased amount.
+#     route[0]['amount_msat'] += 1
+#     inv = l3.rpc.invoice(1000, "test3", "test3")
+#     l1.rpc.sendpay(route,
+#                    payment_hash=inv['payment_hash'],
+#                    payment_secret=inv['payment_secret'])
+#     l1.rpc.waitsendpay(inv['payment_hash'])
+# 
+#     # Now, give us 30 seconds please.
+#     l2.rpc.setchannel("all", 3, 10000, enforcedelay=30)
+#     inv = l3.rpc.invoice(1000, "test4", "test4")
+#     l1.rpc.sendpay(route,
+#                    payment_hash=inv['payment_hash'],
+#                    payment_secret=inv['payment_secret'])
+#     l1.rpc.waitsendpay(inv['payment_hash'])
+#     l2.daemon.wait_for_log("Allowing payment using older feerate")
+# 
+#     time.sleep(30)
+#     inv = l3.rpc.invoice(1000, "test5", "test5")
+#     l1.rpc.sendpay(route,
+#                    payment_hash=inv['payment_hash'],
+#                    payment_secret=inv['payment_secret'])
+#     with pytest.raises(RpcError, match=r'WIRE_FEE_INSUFFICIENT'):
+#         l1.rpc.waitsendpay(inv['payment_hash'])
+# 
+# 
+# def test_listpays_with_filter_by_status(node_factory, bitcoind):
+#     """
+#     This test check if the filtering by status of the command listpays
+#     has some mistakes.
+#     """
+# 
+#     # Create the line graph l2 -> l1 with a channel of 10 ** 5 sat!
+#     l2, l1 = node_factory.line_graph(2, fundamount=10**5, wait_for_announce=True)
+# 
+#     inv = l1.rpc.invoice(10 ** 5, 'inv', 'inv')
+#     l2.rpc.pay(inv['bolt11'])
+# 
+#     wait_for(lambda: l2.rpc.listpays(inv['bolt11'])['pays'][0]['status'] == 'complete')
+# 
+#     # test if the node is still ready
+#     payments = l2.rpc.listpays(status='failed')
+# 
+#     assert len(payments['pays']) == 0
+# 
+#     payments = l2.rpc.listpays()
+#     assert len(l2.rpc.listpays()['pays']) == 1
+# 
+# 
+# def test_sendpay_grouping(node_factory, bitcoind):
+#     """Paying an invoice multiple times, listpays should list them individually
+#     """
+#     l1, l2, l3 = node_factory.line_graph(
+#         3,
+#         wait_for_announce=True,
+#         opts=[
+#             {},
+#             {'may_reconnect': True},
+#             {'may_reconnect': True},
+#         ],
+#     )
+#     wait_for(lambda: len(l1.rpc.listnodes()['nodes']) == 3)
+# 
+#     inv = l3.rpc.invoice(amount_msat='any', label='lbl1', description='desc')['bolt11']
+#     l3.stop()  # Stop recipient so the first attempt fails
+# 
+#     assert(len(l1.db.query("SELECT * FROM payments")) == 0)
+#     assert(len(l1.rpc.listpays()['pays']) == 0)
+# 
+#     with pytest.raises(RpcError, match=r'Ran out of routes to try after [0-9]+ attempts'):
+#         l1.rpc.pay(inv, amount_msat='100000msat')
+# 
+#     # After this one invocation we have one entry in `listpays`
+#     assert(len(l1.rpc.listpays()['pays']) == 1)
+# 
+#     with pytest.raises(RpcError, match=r'Ran out of routes to try after [0-9]+ attempts'):
+#         l1.rpc.pay(inv, amount_msat='200000msat')
+# 
+#     # Surprise: we should have 2 entries after 2 invocations
+#     assert(len(l1.rpc.listpays()['pays']) == 2)
+# 
+#     l3.start()
+#     invoices = l3.rpc.listinvoices()['invoices']
+#     assert(len(invoices) == 1)
+#     assert(invoices[0]['status'] == 'unpaid')
+#     # Will reconnect automatically
+#     wait_for(lambda: only_one(l3.rpc.listpeers()['peers'])['connected'] is True)
+#     scid = l3.rpc.listpeerchannels()['channels'][0]['short_channel_id']
+#     wait_for(lambda: [c['active'] for c in l1.rpc.listchannels(scid)['channels']] == [True, True])
+#     l1.rpc.pay(inv, amount_msat='420000msat')
+# 
+#     # And finally we should have all 3 attempts to pay the invoice
+#     pays = l1.rpc.listpays()['pays']
+#     assert(len(pays) == 3)
+#     assert([p['status'] for p in pays] == ['failed', 'failed', 'complete'])
+# 
+# 
+# def test_pay_manual_exclude(node_factory, bitcoind):
+#     l1, l2, l3 = node_factory.line_graph(3, wait_for_announce=True)
+#     l1_id = l1.rpc.getinfo()['id']
+#     l2_id = l2.rpc.getinfo()['id']
+#     l3_id = l3.rpc.getinfo()['id']
+#     chan12 = l1.rpc.listpeerchannels(l2_id)['channels'][0]
+#     chan23 = l2.rpc.listpeerchannels(l3_id)['channels'][0]
+#     scid12 = chan12['short_channel_id'] + '/' + str(chan12['direction'])
+#     scid23 = chan23['short_channel_id'] + '/' + str(chan23['direction'])
+#     inv = l3.rpc.invoice(amount_msat=123000, label='label1', description='desc')['bolt11']
+#     # Exclude the payer node id
+#     with pytest.raises(RpcError, match=r'Payer is manually excluded'):
+#         l1.rpc.pay(inv, exclude=[l1_id])
+#     # Exclude the direct payee node id
+#     with pytest.raises(RpcError, match=r'Payee is manually excluded'):
+#         l2.rpc.pay(inv, exclude=[l3_id])
+#     # Exclude intermediate node id
+#     with pytest.raises(RpcError, match=r'is not reachable directly and all routehints were unusable.'):
+#         l1.rpc.pay(inv, exclude=[l2_id])
+#     # Exclude intermediate channel id
+#     with pytest.raises(RpcError, match=r'is not reachable directly and all routehints were unusable.'):
+#         l1.rpc.pay(inv, exclude=[scid12])
+#     # Exclude direct channel id
+#     with pytest.raises(RpcError, match=r'is not reachable directly and all routehints were unusable.'):
+#         l2.rpc.pay(inv, exclude=[scid23])
+# 
+# 
+# @unittest.skipIf(TEST_NETWORK != 'regtest', "Invoice is network specific")
+# def test_pay_bolt11_metadata(node_factory, bitcoind):
+#     l1, l2 = node_factory.line_graph(2)
+# 
+#     # BOLT #11:
+#     # > ### Please send 0.01 BTC with payment metadata 0x01fafaf0
+#     # > lnbc10m1pvjluezpp5qqqsyqcyq5rqwzqfqqqsyqcyq5rqwzqfqqqsyqcyq5rqwzqfqypqdp9wpshjmt9de6zqmt9w3skgct5vysxjmnnd9jx2mq8q8a04uqsp5zyg3zyg3zyg3zyg3zyg3zyg3zyg3zyg3zyg3zyg3zyg3zyg3zygs9q2gqqqqqqsgq7hf8he7ecf7n4ffphs6awl9t6676rrclv9ckg3d3ncn7fct63p6s365duk5wrk202cfy3aj5xnnp5gs3vrdvruverwwq7yzhkf5a3xqpd05wjc
+# 
+#     b11 = l1.rpc.decode('lnbc10m1pvjluezpp5qqqsyqcyq5rqwzqfqqqsyqcyq5rqwzqfqqqsyqcyq5rqwzqfqypqdp9wpshjmt9de6zqmt9w3skgct5vysxjmnnd9jx2mq8q8a04uqsp5zyg3zyg3zyg3zyg3zyg3zyg3zyg3zyg3zyg3zyg3zyg3zyg3zygs9q2gqqqqqqsgq7hf8he7ecf7n4ffphs6awl9t6676rrclv9ckg3d3ncn7fct63p6s365duk5wrk202cfy3aj5xnnp5gs3vrdvruverwwq7yzhkf5a3xqpd05wjc')
+#     assert b11['payment_metadata'] == '01fafaf0'
+# 
+#     # I previously hacked lightningd to add "this is metadata" to metadata.
+#     # After CI started failing, I *also* hacked it to set expiry to BIGNUM.
+#     inv = "lnbcrt1230n1p3yzgcxsp5q8g040f9rl9mu2unkjuj0vn262s6nyrhz5hythk3ueu2lfzahmzspp5ve584t0cv27hwmy0cx9ca8uwyqyfw9y9dm3r8vus9fv36r2l9yjsdq8v3jhxccmq6w35xjueqd9ejqmt9w3skgct5vyxqxra2q2qcqp99q2sqqqqqysgqfw6efxpzk5x5vfj8se46yg667x5cvhyttnmuqyk0q7rmhx3gs249qhtdggnek8c5adm2pztkjddlwyn2art2zg9xap2ckczzl3fzz4qqsej6mf"
+#     # Make l2 "know" about this invoice.
+#     l2.rpc.invoice(amount_msat=123000, label='label1', description='desc', preimage='00' * 32)
+# 
+#     with pytest.raises(RpcError, match=r'WIRE_INVALID_ONION_PAYLOAD'):
+#         l1.rpc.pay(inv)
+# 
+#     l2.daemon.wait_for_log("Unexpected payment_metadata {}".format(b'this is metadata'.hex()))
+# 
+# 
+# @pytest.mark.developer("needs to dev-disconnect")
+# def test_pay_middle_fail(node_factory, bitcoind, executor):
+#     """Test the case where a HTLC is failed, but not on peer's side, then
+#     we go onchain"""
+#     # Set feerates the same so we don't have update_fee interfering.
+#     # We want to disconnect on revoke-and-ack we send for failing htlc.
+#     l1, l2, l3 = node_factory.line_graph(3, wait_for_announce=True,
+#                                          opts=[{'feerates': (1500,) * 4},
+#                                                {'feerates': (1500,) * 4},
+#                                                {'feerates': (1500,) * 4,
+#                                                 'disconnect': ['-WIRE_REVOKE_AND_ACK*2']}])
+# 
+#     chanid12 = only_one(l1.rpc.listpeerchannels(l2.info['id'])['channels'])['short_channel_id']
+#     chanid23 = only_one(l2.rpc.listpeerchannels(l3.info['id'])['channels'])['short_channel_id']
+# 
+#     # Make a failing payment.
+#     route = [{'amount_msat': 1011,
+#               'id': l2.info['id'],
+#               'delay': 20,
+#               'channel': chanid12},
+#              {'amount_msat': 1000,
+#               'id': l3.info['id'],
+#               'delay': 10,
+#               'channel': chanid23}]
+# 
+#     # Start payment, it will fail.
+#     l1.rpc.sendpay(route, payment_hash='00' * 32)
+# 
+#     wait_for(lambda: only_one(l3.rpc.listpeers(l2.info['id'])['peers'])['connected'] is False)
+# 
+#     # After this (cltv is actually +11, and we give it 1 block grace)
+#     # l2 will go onchain since HTLC is not resolved.
+#     bitcoind.generate_block(12)
+#     sync_blockheight(bitcoind, [l1, l2, l3])
+#     wait_for(lambda: only_one(l2.rpc.listpeerchannels(l3.info['id'])['channels'])['state'] == 'AWAITING_UNILATERAL')
+# 
+#     # Three blocks and it will resolve the parent.
+#     bitcoind.generate_block(3, wait_for_mempool=1)
+# 
+#     # And that will fail upstream
+#     with pytest.raises(RpcError, match=r'WIRE_INCORRECT_OR_UNKNOWN_PAYMENT_DETAILS'):
+#         l1.rpc.waitsendpay('00' * 32)
+# 
+# 
+# def test_sendpay_dual_amounts(node_factory):
+#     """Test that handing *both* msatoshi and amount_msat to sendpay works"""
+#     l1 = node_factory.get_node(options={'allow-deprecated-apis': True})
+# 
+#     route = [{'amount_msat': 1011,
+#               'msatoshi': 1011,
+#               'id': '022d223620a359a47ff7f7ac447c85c46c923da53389221a0054c11c1e3ca31d59',
+#               'delay': 20,
+#               'channel': '1x1x1'},
+#              {'amount_msat': 1000,
+#               'msatoshi': 1000,
+#               'id': '035d2b1192dfba134e10e540875d366ebc8bc353d5aa766b80c090b39c3a5d885d',
+#               'delay': 10,
+#               'channel': '2x2x2'}]
+#     l1.rpc.check("sendpay", route=route, payment_hash="00" * 32)
+# 
+#     with pytest.raises(RpcError, match=r'No connection to first peer found'):
+#         l1.rpc.sendpay(route=route, payment_hash="00" * 32)
+# 
+# 
+# @unittest.skipIf(TEST_NETWORK != 'regtest', "Invoice is network specific")
+# @pytest.mark.developer("needs createinvoicerequest which allows unsigned invoice containing payerinfo")
+# @pytest.mark.slow_test
+# def test_payerkey(node_factory):
+#     """payerkey calculation should not change across releases!"""
+#     nodes = node_factory.get_nodes(7)
+# 
+#     expected_keys = ["02294ec1cd3f100947fe859d71a42cb87932e36e7771abf2d50b02a7a92be8e4d5",
+#                      "026a4a3b6b0c694da6f14629ca5140713fc703591a6d8aae5c79ba9b5556fc5723",
+#                      "03defd2b1f3004b0145351f469f34512c6fa4d02fe891a977bafdb34fe7b73ea48",
+#                      "03eccb00c0a3c760465bb69a6297d7cfa5bcbd989d5a88e435bd8d6e4c723013cd",
+#                      "021b4bfa652f0df7498d734b0ca888b4e3b07f59e1a974ec7d4a9d6046e8e5ab92",
+#                      "03fc91d60b061e517f9182e3e40ea14c27df520c51db204f1409ff50e5cf9a5e4d",
+#                      "03a3bbda0137722ba62207b9d3e5e6cc2a11e58480f801892093e01383aacb7fb2"]
+# 
+#     for n, k in zip(nodes, expected_keys):
+#         b12 = n.rpc.createinvoicerequest('lnr1qqgz2d7u2smys9dc5q2447e8thjlgq3qqc3xu3s3rg94nj40zfsy866mhu5vxne6tcej5878k2mneuvgjy8ssqvepgz5zsjrg3z3vggzvkm2khkgvrxj27r96c00pwl4kveecdktm29jdd6w0uwu5jgtv5v9qgqxyfhyvyg6pdvu4tcjvpp7kkal9rp57wj7xv4pl3ajku70rzy3pu', False)['bolt12']
+#         assert n.rpc.decode(b12)['invreq_payer_id'] == k
+# 
+# 
+# def test_pay_multichannel_use_zeroconf(bitcoind, node_factory):
+#     """Check that we use the zeroconf direct channel to pay when we need to"""
+#     # 0. Setup normal channel, 200k sats.
+#     zeroconf_plugin = Path(__file__).parent / "plugins" / "zeroconf-selective.py"
+#     l1, l2 = node_factory.line_graph(2, wait_for_announce=False,
+#                                      fundamount=200_000,
+#                                      opts=[{},
+#                                            {'plugin': zeroconf_plugin,
+#                                             'zeroconf-allow': 'any'}])
+# 
+#     # 1. Open a zeoconf channel l1 -> l2
+#     zeroconf_sats = 1_000_000
+# 
+#     # 1.1 Add funds to l1's wallet for the channel open
+#     l1.fundwallet(zeroconf_sats * 2)  # This will mine a block!
+#     sync_blockheight(bitcoind, [l1, l2])
+# 
+#     # 1.2 Open the zeroconf channel
+#     l1.rpc.fundchannel(l2.info['id'], zeroconf_sats, announce=False, mindepth=0)
+# 
+#     # 1.3 Wait until all channels active.
+#     wait_for(lambda: all([c['state'] == 'CHANNELD_NORMAL' for c in l1.rpc.listpeerchannels()['channels'] + l2.rpc.listpeerchannels()['channels']]))
+# 
+#     # 2. Have l2 generate an invoice to be paid
+#     invoice_sats = "500000sat"
+#     inv = l2.rpc.invoice(invoice_sats, "test", "test")
+# 
+#     # 3. Send a payment over the zeroconf channel
+#     riskfactor = 0
+#     l1.rpc.pay(inv['bolt11'], riskfactor=riskfactor)
+# 
+# 
+# @pytest.mark.developer("needs dev-no-reconnect, dev-routes to force failover")
+# def test_delpay_works(node_factory, bitcoind):
+#     """
+#     One failure, one success; deleting the success works (groupid=1, partid=2)
+#     """
+#     l1, l2, l3 = node_factory.line_graph(3, fundamount=10**5,
+#                                          wait_for_announce=True)
+#     # Expensive route!
+#     l4 = node_factory.get_node(options={'fee-per-satoshi': 1000,
+#                                         'fee-base': 2000})
+#     node_factory.join_nodes([l1, l4, l3], wait_for_announce=True)
+# 
+#     # Don't give a hint, so l1 chooses cheapest.
+#     inv = l3.dev_invoice(10**5, 'lbl', 'desc', dev_routes=[])
+#     l3.rpc.disconnect(l2.info['id'], force=True)
+#     l1.rpc.pay(inv['bolt11'])
+# 
+#     assert len(l1.rpc.listsendpays()['payments']) == 2
+#     failed = [p for p in l1.rpc.listsendpays()['payments'] if p['status'] == 'complete'][0]
+#     l1.rpc.delpay(payment_hash=failed['payment_hash'],
+#                   status=failed['status'],
+#                   groupid=failed['groupid'],
+#                   partid=failed['partid'])
+# 
+#     with pytest.raises(RpcError, match=r'No payment for that payment_hash'):
+#         l1.rpc.delpay(payment_hash=failed['payment_hash'],
+#                       status=failed['status'],
+#                       groupid=failed['groupid'],
+#                       partid=failed['partid'])

From 0e6e5d6db6ddf24985c19b27841b7d8699e3d182 Mon Sep 17 00:00:00 2001
From: Lagrang3 <eduardo.quintana@pm.me>
Date: Sun, 25 Jun 2023 17:13:15 +0100
Subject: [PATCH 60/64] renepay: add error messages from pay_flow

---
 plugins/renepay/pay.c      | 15 ++++---
 plugins/renepay/pay_flow.c | 21 +++++++++-
 plugins/renepay/pay_flow.h |  3 +-
 tests/test_renepay.py      | 86 ++++++++++++++++++--------------------
 4 files changed, 69 insertions(+), 56 deletions(-)

diff --git a/plugins/renepay/pay.c b/plugins/renepay/pay.c
index 10048aeb5c06..f0bba49c8807 100644
--- a/plugins/renepay/pay.c
+++ b/plugins/renepay/pay.c
@@ -1158,10 +1158,13 @@ static struct command_result *try_paying(struct command *cmd,
 
 	// plugin_log(pay_plugin->plugin,LOG_DBG,fmt_chan_extra_map(tmpctx,pay_plugin->chan_extra_map));
 	
+	char const * err_msg;
+	
 	/* We let this return an unlikely path, as it's better to try once
 	 * than simply refuse.  Plus, models are not truth! */
 	struct pay_flow **pay_flows = get_payflows(p, remaining, feebudget, first_time,
-				 amount_msat_eq(p->total_delivering, AMOUNT_MSAT(0)));
+				 		   amount_msat_eq(p->total_delivering, AMOUNT_MSAT(0)),
+						   &err_msg);
 	
 	plugin_log(pay_plugin->plugin,LOG_DBG,fmt_payflows(tmpctx,pay_flows));
 	
@@ -1170,11 +1173,8 @@ static struct command_result *try_paying(struct command *cmd,
 	if (!pay_flows)
 	{
 		return command_fail(cmd, PAY_ROUTE_NOT_FOUND,
-				    "Failed to find a route for %s with budget %s",
-				    type_to_string(tmpctx, struct amount_msat,
-						   &remaining),
-				    type_to_string(tmpctx, struct amount_msat,
-						   &feebudget));
+				    "Failed to find a route, %s",
+				    err_msg);
 	}
 	/* Now begin making payments */
 	
@@ -1501,10 +1501,9 @@ static struct command_result *json_pay(struct command *cmd,
 		plugin_err(pay_plugin->plugin, "Failed to refresh gossmap: %s",
 			   strerror(errno));
 	
-	
 	p->base_fee_penalty=*base_fee_penalty;
 	p->prob_cost_factor= *prob_cost_factor;
-	p->min_prob_success=*min_prob_success_millionths * 1e-6;
+	p->min_prob_success = *min_prob_success_millionths / 1000000.0;
  	p->delay_feefactor = *riskfactor_millionths / 1000000.0;
 	p->maxdelay = *maxdelay;
 	
diff --git a/plugins/renepay/pay_flow.c b/plugins/renepay/pay_flow.c
index 540017189c53..9e76945b1c78 100644
--- a/plugins/renepay/pay_flow.c
+++ b/plugins/renepay/pay_flow.c
@@ -334,7 +334,8 @@ struct pay_flow **get_payflows(struct payment *p,
 			       struct amount_msat amount,
 			       struct amount_msat feebudget,
 			       bool unlikely_ok,
-			       bool is_entire_payment)
+			       bool is_entire_payment,
+			       char const ** err_msg)
 {
 	bitmap *disabled;
 	struct pay_flow **pay_flows;
@@ -344,11 +345,13 @@ struct pay_flow **get_payflows(struct payment *p,
 	src = gossmap_find_node(pay_plugin->gossmap, &pay_plugin->my_id);
 	if (!src) {
 		paynote(p, "We don't have any channels?");
+		*err_msg = tal_fmt(tmpctx,"We don't have any channels.");
 		goto fail;
 	}
 	dst = gossmap_find_node(pay_plugin->gossmap, &p->destination);
 	if (!src) {
 		paynote(p, "No trace of destination in network gossip");
+		*err_msg = tal_fmt(tmpctx,"Destination is unreacheable in the network gossip.");
 		goto fail;
 	}
 
@@ -381,10 +384,15 @@ struct pay_flow **get_payflows(struct payment *p,
 		fee = flow_set_fee(flows);
 		delay = flows_worst_delay(flows) + p->final_cltv;
 
-		too_unlikely = (prob < 0.01);
+		too_unlikely = (prob < p->min_prob_success);
 		if (too_unlikely && !unlikely_ok)
 		{
 			paynote(p, "Flows too unlikely, P() = %f%%", prob * 100);
+			*err_msg = tal_fmt(tmpctx,
+					  "Probability is too small, "
+					  "Prob = %f%% (min = %f%%)",
+					  prob*100,
+					  p->min_prob_success*100);
 			goto fail;
 		}
 		too_expensive = amount_msat_greater(fee, feebudget);
@@ -393,6 +401,11 @@ struct pay_flow **get_payflows(struct payment *p,
 			paynote(p, "Flows too expensive, fee = %s (max %s)",
 				type_to_string(tmpctx, struct amount_msat, &fee),
 				type_to_string(tmpctx, struct amount_msat, &feebudget));
+			*err_msg = tal_fmt(tmpctx,
+					  "Fee exceeds our fee budget, "
+					  "fee = %s (maxfee = %s)",
+					  type_to_string(tmpctx, struct amount_msat, &fee),
+					  type_to_string(tmpctx, struct amount_msat, &feebudget));
 			goto fail;
 		}
 		too_delayed = (delay > p->maxdelay);
@@ -403,6 +416,10 @@ struct pay_flow **get_payflows(struct payment *p,
 			/* FIXME: What is a sane limit? */
 			if (p->delay_feefactor > 1000) {
 				paynote(p, "Giving up!");
+				*err_msg = tal_fmt(tmpctx,
+						  "CLTV delay exceeds our CLTV budget, "
+						  "delay = %"PRIu64" (maxdelay = %u)",
+						  delay,p->maxdelay);
 				goto fail;
 			}
 
diff --git a/plugins/renepay/pay_flow.h b/plugins/renepay/pay_flow.h
index b25dfa7bf993..ef905ad7425b 100644
--- a/plugins/renepay/pay_flow.h
+++ b/plugins/renepay/pay_flow.h
@@ -29,7 +29,8 @@ struct pay_flow **get_payflows(struct payment *p,
 			       struct amount_msat amount,
 			       struct amount_msat feebudget,
 			       bool unlikely_ok,
-			       bool is_entire_payment);
+			       bool is_entire_payment,
+			       char const ** err_msg);
 
 void commit_htlc_payflow(
 		struct chan_extra_map *chan_extra_map,
diff --git a/tests/test_renepay.py b/tests/test_renepay.py
index fc45ed3d2632..d72c2c5237b0 100644
--- a/tests/test_renepay.py
+++ b/tests/test_renepay.py
@@ -123,76 +123,72 @@ def test_pay(node_factory):
 @pytest.mark.developer("needs to deactivate shadow routing")
 def test_amounts(node_factory):
     l1, l2 = node_factory.line_graph(2)
-    inv = l2.rpc.invoice(Millisatoshi("123sat"), 'test_pay_amounts', 'description')['bolt11']
+    inv = l2.rpc.invoice(Millisatoshi(123456), 'test_pay_amounts', 'description')['bolt11']
 
     invoice = only_one(l2.rpc.listinvoices('test_pay_amounts')['invoices'])
 
     assert isinstance(invoice['amount_msat'], Millisatoshi)
-    assert invoice['amount_msat'] == Millisatoshi(123000)
+    assert invoice['amount_msat'] == Millisatoshi(123456)
 
     l1.rpc.call('renepay',{'invstring':inv, 'use_shadow':False})
 
     invoice = only_one(l2.rpc.listinvoices('test_pay_amounts')['invoices'])
     assert isinstance(invoice['amount_received_msat'], Millisatoshi)
-    assert invoice['amount_received_msat'] >= Millisatoshi(123000)
+    assert invoice['amount_received_msat'] >= Millisatoshi(123456)
 
 
 @pytest.mark.developer("needs to deactivate shadow routing")
 def test_limits(node_factory):
-    """Test that we enforce fee max percentage and max delay"""
-    l1, l2, l3 = node_factory.line_graph(3, wait_for_announce=True)
-
+    '''
+    Topology:
+    1----2----4
+    |         |
+    3----5----6
+    '''
+    opts = [
+        {'disable-mpp': None, 'fee-base':0, 'fee-per-satoshi':100},
+    ]
+    l1, l2, l3, l4, l5,l6 = node_factory.get_nodes(6, opts=opts*6)
+    node_factory.join_nodes([l1, l2, l4, l6],
+        wait_for_announce=True,fundamount=1000000)
+    node_factory.join_nodes([l1, l3, l5, l6], 
+        wait_for_announce=True,fundamount=1000000)
+    
+    inv = l4.rpc.invoice('any','any','description')
+    l2.rpc.call('pay', {'bolt11': inv['bolt11'], 'amount_msat':500000000})
+    inv = l5.rpc.invoice('any','any','description')
+    l3.rpc.call('pay', {'bolt11': inv['bolt11'], 'amount_msat':500000000})
+    
     # FIXME: pylightning should define these!
     # PAY_STOPPED_RETRYING = 210
     PAY_ROUTE_NOT_FOUND=205
 
-    inv = l3.rpc.invoice("any", "any", 'description')
+    inv = l6.rpc.invoice("any", "any", 'description')
 
     # Fee too high.
-    err = r'Failed to find a route for 100000msat with budget 1msat'
-    with pytest.raises(RpcError, match=err) as err:
-        l1.rpc.call('renepay', {'invstring': inv['bolt11'], 'amount_msat': 100000, 'maxfee': 1})
-
-    # TODO(eduardo): which error code shall we use here?
+    failmsg = r'Fee exceeds our fee budget'
+    with pytest.raises(RpcError, match=failmsg) as err:
+        l1.rpc.call('renepay', {'invstring': inv['bolt11'], 'amount_msat': 1000000, 'maxfee': 1})
     assert err.value.error['code'] == PAY_ROUTE_NOT_FOUND
-    # assert err.value.error['code'] == PAY_STOPPED_RETRYING
+    # TODO(eduardo): which error code shall we use here?
 
     # TODO(eduardo): shall we list attempts in renepay?
-    # It should have retried two more times (one without routehint and one with routehint)
     # status = l1.rpc.call('renepaystatus', {'invstring':inv['bolt11']})['paystatus'][0]['attempts']
 
-    # We have an internal test to see if we can reach the destination directly
-    # without a routehint, that will enable a NULL-routehint. We will then try
-    # with the provided routehint, and the NULL routehint, resulting in 2
-    # attempts.
-    # assert(len(status) == 1)
-    # assert(status[0]['failure']['code'] == 205)
+    failmsg = r'CLTV delay exceeds our CLTV budget'
+    # Delay too high.
+    with pytest.raises(RpcError, match=failmsg) as err:
+        l1.rpc.call('renepay', {'invstring': inv['bolt11'], 'amount_msat': 1000000, 'maxdelay': 0})
+    assert err.value.error['code'] == PAY_ROUTE_NOT_FOUND
+    
+    failmsg = r'Probability is too small'
+    # Delay too high.
+    with pytest.raises(RpcError, match=failmsg) as err:
+        l1.rpc.call('renepay', {'invstring': inv['bolt11'], 
+            'amount_msat': 800000000, 
+            'min_prob_success': '0.5'})
+    assert err.value.error['code'] == PAY_ROUTE_NOT_FOUND
 
-#     failmsg = r'CLTV delay exceeds our CLTV budget'
-#     # Delay too high.
-#     with pytest.raises(RpcError, match=failmsg) as err:
-#         l1.rpc.call('pay', {'bolt11': inv['bolt11'], 'amount_msat': 100000, 'maxdelay': 0})
-# 
-#     assert err.value.error['code'] == PAY_STOPPED_RETRYING
-#     # Should also have retried two more times.
-#     status = l1.rpc.call('paystatus', {'bolt11': inv['bolt11']})['pay'][1]['attempts']
-# 
-#     assert(len(status) == 2)
-#     assert(status[0]['failure']['code'] == 205)
-# 
-#     # This fails!
-#     err = r'Fee exceeds our fee budget: 2msat > 1msat, discarding route'
-#     with pytest.raises(RpcError, match=err) as err:
-#         l1.rpc.pay(bolt11=inv['bolt11'], amount_msat=100000, maxfee=1)
-# 
-#     # This works, because fee is less than exemptfee.
-#     l1.dev_pay(inv['bolt11'], amount_msat=100000, maxfeepercent=0.0001,
-#                exemptfee=2000, use_shadow=False)
-#     status = l1.rpc.call('paystatus', {'bolt11': inv['bolt11']})['pay'][3]['attempts']
-#     assert len(status) == 1
-#     assert status[0]['strategy'] == "Initial attempt"
-# 
-# 
 # @pytest.mark.developer("Gossip is too slow without developer")
 # def test_pay_exclude_node(node_factory, bitcoind):
 #     """Test excluding the node if there's the NODE-level error in the failure_code

From 0926bf084c298482b63874d097c9c773a6ff4c8a Mon Sep 17 00:00:00 2001
From: Lagrang3 <eduardo.quintana@pm.me>
Date: Mon, 26 Jun 2023 07:38:39 +0100
Subject: [PATCH 61/64] renepay: can complete pending MPP

---
 plugins/renepay/pay.c | 123 +++++++++++++++++++++++++++++++-----------
 plugins/renepay/pay.h |   8 +--
 tests/test_renepay.py |  11 ++--
 3 files changed, 104 insertions(+), 38 deletions(-)

diff --git a/plugins/renepay/pay.c b/plugins/renepay/pay.c
index f0bba49c8807..8340979e69a2 100644
--- a/plugins/renepay/pay.c
+++ b/plugins/renepay/pay.c
@@ -17,6 +17,9 @@
 #include <plugins/renepay/pay_flow.h>
 #include <plugins/renepay/debug.h>
 
+#define INVALID_ID UINT64_MAX 
+#define MAX(a,b) ((a)>(b)? (a) : (b))
+
 // TODO(eduardo): the state of plugin is stored in data (instead of the heap) so
 // that we can load lightningd options directly into before `init` is called.
 static struct pay_plugin the_pay_plugin;
@@ -1044,7 +1047,6 @@ sendpay_flows(struct command *cmd,
 		json_add_sha256(req->js, "payment_hash", &p->payment_hash);
 		json_add_secret(req->js, "payment_secret", p->payment_secret);
 		
-		// TODO(eduardo): should this be p->amount or p->total_delivering? 
 		json_add_amount_msat_only(req->js, "amount_msat", p->amount);
 			
 		json_add_u64(req->js, "partid", flows[i]->partid);
@@ -1061,8 +1063,6 @@ sendpay_flows(struct command *cmd,
 		if (p->description)
 			json_add_string(req->js, "description", p->description);
 		
-		// debug_outreq(req);
-		
 		amount_msat_accumulate(&p->total_sent, flows[i]->amounts[0]);
 		amount_msat_accumulate(&p->total_delivering,
 				       flow_delivered(flows[i]));
@@ -1120,8 +1120,9 @@ static struct command_result *try_paying(struct command *cmd,
 	if (!amount_msat_sub(&feebudget, p->maxspend, p->amount))
 	{
 		plugin_err(pay_plugin->plugin,
-			   "%s could not substract maxspend=%s and amount=%s.",
+			   "%s (line %d) could not substract maxspend=%s and amount=%s.",
 			   __PRETTY_FUNCTION__,
+			   __LINE__,
 			   type_to_string(tmpctx, struct amount_msat, &p->maxspend),
 			   type_to_string(tmpctx, struct amount_msat, &p->amount));
 	}
@@ -1130,8 +1131,9 @@ static struct command_result *try_paying(struct command *cmd,
 	if (!amount_msat_sub(&fees_spent, p->total_sent, p->total_delivering))
 	{
 		plugin_err(pay_plugin->plugin,
-			   "%s could not substract total_sent=%s and total_delivering=%s.",
+			   "%s (line %d) could not substract total_sent=%s and total_delivering=%s.",
 			   __PRETTY_FUNCTION__,
+			   __LINE__,
 			   type_to_string(tmpctx, struct amount_msat, &p->total_sent),
 			   type_to_string(tmpctx, struct amount_msat, &p->total_delivering));
 	}
@@ -1140,8 +1142,9 @@ static struct command_result *try_paying(struct command *cmd,
 	if (!amount_msat_sub(&feebudget, feebudget, fees_spent))
 	{
 		plugin_err(pay_plugin->plugin,
-			   "%s could not substract feebudget=%s and fees_spent=%s.",
+			   "%s (line %d) could not substract feebudget=%s and fees_spent=%s.",
 			   __PRETTY_FUNCTION__,
+			   __LINE__,
 			   type_to_string(tmpctx, struct amount_msat, &feebudget),
 			   type_to_string(tmpctx, struct amount_msat, &fees_spent));
 	}
@@ -1150,8 +1153,9 @@ static struct command_result *try_paying(struct command *cmd,
 	if (!amount_msat_sub(&remaining, p->amount, p->total_delivering))
 	{
 		plugin_err(pay_plugin->plugin,
-			   "%s could not substract amount=%s and total_delivering=%s.",
+			   "%s (line %d) could not substract amount=%s and total_delivering=%s.",
 			   __PRETTY_FUNCTION__,
+			   __LINE__,
 			   type_to_string(tmpctx, struct amount_msat, &p->amount),
 			   type_to_string(tmpctx, struct amount_msat, &p->total_delivering));
 	}
@@ -1218,7 +1222,7 @@ static void renepay_cleanup(struct active_payment *ap)
 	ap->localmods_applied=false;
 	tal_free(ap->local_gossmods);
 	
-	plugin_log(pay_plugin->plugin,LOG_DBG,fmt_chan_extra_map(tmpctx,pay_plugin->chan_extra_map));
+	// plugin_log(pay_plugin->plugin,LOG_DBG,fmt_chan_extra_map(tmpctx,pay_plugin->chan_extra_map));
 	
 	ap->rexmit_timer = tal_free(ap->rexmit_timer);
 	
@@ -1297,7 +1301,9 @@ payment_listsendpays_previous(struct command *cmd, const char *buf,
 
 	/* Group ID of the first pending payment, this will be the one
 	 * who's result gets replayed if we end up suspending. */
-	u64 pending_group_id = 0;
+	u64 first_pending_group_id = INVALID_ID;
+	u64 last_pending_group_id = INVALID_ID;
+	u64 last_partid=0;
 	/* Did a prior attempt succeed? */
 	bool completed = false;
 
@@ -1326,12 +1332,23 @@ payment_listsendpays_previous(struct command *cmd, const char *buf,
 	 * latest group and remembering what its state is. */
 	json_for_each_arr(i, t, arr)
 	{
-		u64 groupid;
-		const jsmntok_t *status, *grouptok;
+		bool this_pending = false;
+		u64 partid, groupid;
+		struct amount_msat this_delivering, this_sent;
+		
+		const jsmntok_t *status;
 		struct amount_msat diff_sent, diff_msat;
-		grouptok = json_get_member(buf, t, "groupid");
-		json_to_u64(buf, grouptok, &groupid);
-
+		
+		json_scan(tmpctx,buf,t,
+			  "{partid:%"
+			  ",groupid:%"
+			  ",amount_msat:%"
+			  ",amount_sent_msat:%}",
+			  JSON_SCAN(json_to_u64,&partid),
+			  JSON_SCAN(json_to_u64,&groupid),
+			  JSON_SCAN(json_to_msat,&this_delivering),
+			  JSON_SCAN(json_to_msat,&this_sent));
+		
 		/* New group, reset what we collected. */
 		if (last_group != groupid) {
 			completed = false;
@@ -1365,11 +1382,42 @@ payment_listsendpays_previous(struct command *cmd, const char *buf,
 		status = json_get_member(buf, t, "status");
 		completed |= json_tok_streq(buf, status, "complete");
 		pending |= json_tok_streq(buf, status, "pending");
+		
+		this_pending |= json_tok_streq(buf, status, "pending");
 
-		/* Remember the group id of the first pending group so
-		 * we can replay its result later. */
-		if (!pending_group_id && pending)
-			pending_group_id = groupid;
+		if(this_pending)
+		{
+			if(first_pending_group_id==INVALID_ID || 
+			   last_pending_group_id==INVALID_ID)
+				first_pending_group_id = last_pending_group_id = groupid;
+			
+			if(groupid > last_pending_group_id)
+			{
+				last_pending_group_id = groupid;
+				p->total_sent = AMOUNT_MSAT(0);
+				p->total_delivering = AMOUNT_MSAT(0);
+				last_partid = partid;
+			}
+			if(groupid < first_pending_group_id)
+			{
+				first_pending_group_id = groupid;
+			}
+			if(groupid == last_pending_group_id)
+			{
+				amount_msat_accumulate(&p->total_sent, 
+						       this_sent);
+				amount_msat_accumulate(&p->total_delivering,
+						       this_delivering);
+				plugin_log(pay_plugin->plugin,LOG_DBG,
+					"pending deliver increased by %s",
+					type_to_string(tmpctx,struct amount_msat,&this_delivering));
+			}
+		}
+		
+		/* Let's get the very last id in the last pending group. */
+		if(groupid == last_pending_group_id)		
+			last_partid = MAX(last_partid,partid);
+				
 	}
 	
 	if (completed) {
@@ -1391,14 +1439,32 @@ payment_listsendpays_previous(struct command *cmd, const char *buf,
 		
 		return command_finished(cmd, ret);
 	} else if (pending) {
-		p->groupid = pending_group_id;
-		/* Someone else is trying to get this payment through. I'll fail
-		 * this attempt. */
-		tal_free(p);
+		p->groupid = last_pending_group_id;
+		p->active_payment->next_partid = last_partid+1;	
+		
+		plugin_log(pay_plugin->plugin,LOG_DBG,
+			   "There are pending sendpays to this invoice. "
+			   "groupids = %ld or %ld, "
+			   "delivering = %s, "
+			   "last_partid = %ld",
+			   first_pending_group_id,
+			   last_pending_group_id,
+			   type_to_string(tmpctx,struct amount_msat,&p->total_delivering),
+			   last_partid);
 		
-		// TODO(eduardo): is this an acceptable behaviour?
-		return command_fail(cmd, PAY_IN_PROGRESS,
-				    "Payment is pending by some other request.");
+		if( 
+		/* TODO(eduardo): If two group_id are trying to complete the
+		 * payment, could this lead to overpayment? */
+		first_pending_group_id != last_pending_group_id
+			|| 
+		amount_msat_greater_eq(p->total_delivering,p->amount))
+		{
+			tal_free(p);
+			return command_fail(cmd, PAY_IN_PROGRESS,
+					    "Payment is pending by some other request.");
+		}
+		
+		return try_paying(cmd,p,true);
 	}
 	
 	p->groupid = last_group + 1;
@@ -1467,13 +1533,6 @@ static struct command_result *json_pay(struct command *cmd,
 		   NULL))
 		return command_param_failed();
 	
-	plugin_log(pay_plugin->plugin,LOG_DBG,"json_pay, renepay-debug-mcf option is set to %s\n",
-		   pay_plugin->debug_mcf ? "true" : "false");
-	plugin_log(pay_plugin->plugin,LOG_DBG,"json_pay, renepay-debug-payflow option is set to %s\n",
-		   pay_plugin->debug_payflow ? "true" : "false");
-
-	
-	
 	tal_steal(pay_plugin->ctx,p);
 	tal_add_destructor(p, destroy_payment);
 	
diff --git a/plugins/renepay/pay.h b/plugins/renepay/pay.h
index 657ad4e3740b..0902c59d0c58 100644
--- a/plugins/renepay/pay.h
+++ b/plugins/renepay/pay.h
@@ -6,8 +6,12 @@
 #include <plugins/libplugin.h>
 #include <plugins/renepay/flow.h>
 
+// TODO(eduardo): check if we are doing correctly the knowledge update: if a
+// channel is not able to forward X it doesn't mean MAX<X, it means that MAX<X+Y
+// where Y is the sum of all HTLCs.
+// TODO(eduardo): check a problem with param_millionths(), if I input an integer
+// should or shouldn't be multiplied by 10^6?
 // TODO(eduardo): add an option entry for maxfeepercent
-// TODO(eduardo): I think this plugin can't handle a msat accuracy payment.
 // TODO(eduardo): write a man entry for renepay
 // TODO(eduardo): check if paynotes are meaningful
 // TODO(eduardo): remove assertions, introduce LOG_BROKEN messages
@@ -31,8 +35,6 @@
 // 	- destination is not in the gossmap
 // 	- destination is offline
 // 	- with current knowledge there is no flow solution to destination
-// 	- path to destination is too expensive
-// 	- path to destination is too unlikely
 
 enum payment_status {
         PAYMENT_PENDING, PAYMENT_SUCCESS, PAYMENT_FAIL,
diff --git a/tests/test_renepay.py b/tests/test_renepay.py
index d72c2c5237b0..14ac7a0b9dff 100644
--- a/tests/test_renepay.py
+++ b/tests/test_renepay.py
@@ -181,13 +181,18 @@ def test_limits(node_factory):
         l1.rpc.call('renepay', {'invstring': inv['bolt11'], 'amount_msat': 1000000, 'maxdelay': 0})
     assert err.value.error['code'] == PAY_ROUTE_NOT_FOUND
     
+    inv2 = l6.rpc.invoice("800000sat", "inv2", 'description')
     failmsg = r'Probability is too small'
     # Delay too high.
     with pytest.raises(RpcError, match=failmsg) as err:
-        l1.rpc.call('renepay', {'invstring': inv['bolt11'], 
-            'amount_msat': 800000000, 
-            'min_prob_success': '0.5'})
+        l1.rpc.call('renepay', {'invstring': inv2['bolt11'],'min_prob_success': '0.5'})
     assert err.value.error['code'] == PAY_ROUTE_NOT_FOUND
+    
+    # if we try again we can finish this payment
+    l1.rpc.call('renepay', {'invstring': inv2['bolt11']})
+    invoice = only_one(l6.rpc.listinvoices('inv2')['invoices'])
+    assert isinstance(invoice['amount_received_msat'], Millisatoshi)
+    assert invoice['amount_received_msat'] >= Millisatoshi('800000sat')
 
 # @pytest.mark.developer("Gossip is too slow without developer")
 # def test_pay_exclude_node(node_factory, bitcoind):

From 3125d2604265b6d7dfcd7642d2a19ce96ea8defe Mon Sep 17 00:00:00 2001
From: Lagrang3 <eduardo.quintana@pm.me>
Date: Tue, 27 Jun 2023 08:50:57 +0100
Subject: [PATCH 62/64] renepay: bugfix

Knowledge update must consider total amount of liquidity commited to a
channel in the form of HTLC.
---
 plugins/renepay/debug.h |  6 +++++
 plugins/renepay/flow.c  | 57 +++++++++++++++++++++++++++--------------
 plugins/renepay/flow.h  |  4 ++-
 plugins/renepay/pay.c   | 25 +++++++++---------
 plugins/renepay/pay.h   | 15 ++++++++---
 5 files changed, 72 insertions(+), 35 deletions(-)

diff --git a/plugins/renepay/debug.h b/plugins/renepay/debug.h
index 875ea0048bf8..7c61e6eea4d6 100644
--- a/plugins/renepay/debug.h
+++ b/plugins/renepay/debug.h
@@ -28,6 +28,9 @@ void _debug_exec_branch(const char* fname,const char* fun, int lineno);
 #define debug_err(...) \
 	_debug_info(MYLOG,__VA_ARGS__); abort()
 
+#define debug_paynote(p,...) \
+	_debug_info(MYLOG,__VA_ARGS__);
+
 #else
 
 #include <plugins/renepay/pay.h>
@@ -38,6 +41,9 @@ void _debug_exec_branch(const char* fname,const char* fun, int lineno);
 #define debug_err(...) \
 	plugin_err(pay_plugin->plugin,__VA_ARGS__)
 
+#define debug_paynote(p,...) \
+	paynote(p,__VA_ARGS__);
+
 #endif
 
 	
diff --git a/plugins/renepay/flow.c b/plugins/renepay/flow.c
index f49d223aaeec..f448a9529906 100644
--- a/plugins/renepay/flow.c
+++ b/plugins/renepay/flow.c
@@ -145,18 +145,46 @@ void chan_extra_can_send(
 		debug_err("%s unexpected chan_extra ce is NULL",
 			__PRETTY_FUNCTION__);
 	}
+	if(!amount_msat_add(&x,x,ce->half[dir].htlc_total))
+	{
+		debug_err("%s (line %d) cannot add x=%s and htlc_total=%s",
+			__PRETTY_FUNCTION__,__LINE__,
+			type_to_string(tmpctx,struct amount_msat,&x),
+			type_to_string(tmpctx,struct amount_msat,&ce->half[dir].htlc_total));
+	}
 	chan_extra_can_send_(ce,dir,x);
 }
+
 /* Update the knowledge that this (channel,direction) cannot send x msat.*/
-static void chan_extra_cannot_send_(
-		struct chan_extra *ce, 
+void chan_extra_cannot_send(
+		struct payment *p,
+		struct chan_extra_map *chan_extra_map,
+		struct short_channel_id scid, 
 		int dir,
 		struct amount_msat x)
 {
+	struct chan_extra *ce = chan_extra_map_get(chan_extra_map,
+						   scid);
+	if(!ce)
+	{
+		debug_err("%s (line %d) unexpected chan_extra ce is NULL",
+			__PRETTY_FUNCTION__,__LINE__);
+	}
+	
+	/* If a channel cannot send x it means that the upper bound for the
+	 * liquidity is MAX_L < x + htlc_total */
+	if(!amount_msat_add(&x,x,ce->half[dir].htlc_total))
+	{
+		debug_err("%s (line %d) cannot add x=%s and htlc_total=%s",
+			__PRETTY_FUNCTION__,__LINE__,
+			type_to_string(tmpctx,struct amount_msat,&x),
+			type_to_string(tmpctx,struct amount_msat,&ce->half[dir].htlc_total));
+	}
+	
 	if(!amount_msat_sub(&x,x,AMOUNT_MSAT(1)))
 	{
-		debug_err("%s unexpected x=%s is less than 0msat",
-			__PRETTY_FUNCTION__,
+		debug_err("%s (line %d) unexpected x=%s is less than 0msat",
+			__PRETTY_FUNCTION__,__LINE__,
 			type_to_string(tmpctx,struct amount_msat,&x)
 			);
 		x = AMOUNT_MSAT(0);
@@ -165,23 +193,14 @@ static void chan_extra_cannot_send_(
 	ce->half[dir].known_min = amount_msat_min(ce->half[dir].known_min,x);
 	ce->half[dir].known_max = amount_msat_min(ce->half[dir].known_max,x);
 	
+	debug_paynote(p,"Update chan knowledge scid=%s, dir=%d: [%s,%s]",
+		type_to_string(tmpctx,struct short_channel_id,&scid),
+		dir,
+		type_to_string(tmpctx,struct amount_msat,&ce->half[dir].known_min),
+		type_to_string(tmpctx,struct amount_msat,&ce->half[dir].known_max));
+	
 	chan_extra_adjust_half(ce,!dir);
 }
-void chan_extra_cannot_send(
-		struct chan_extra_map *chan_extra_map,
-		struct short_channel_id scid, 
-		int dir,
-		struct amount_msat x)
-{
-	struct chan_extra *ce = chan_extra_map_get(chan_extra_map,
-						   scid);
-	if(!ce)
-	{
-		debug_err("%s unexpected chan_extra ce is NULL",
-			__PRETTY_FUNCTION__);
-	}
-	chan_extra_cannot_send_(ce,dir,x);
-}
 /* Update the knowledge that this (channel,direction) has liquidity x.*/
 static void chan_extra_set_liquidity_(
 		struct chan_extra *ce, 
diff --git a/plugins/renepay/flow.h b/plugins/renepay/flow.h
index 3792e375a409..1e5f64750320 100644
--- a/plugins/renepay/flow.h
+++ b/plugins/renepay/flow.h
@@ -6,6 +6,7 @@
 #include <common/amount.h>
 #include <common/gossmap.h>
 
+struct payment;
 
 // TODO(eduardo): a hard coded constant to indicate a limit on any channel
 // capacity. Channels for which the capacity is unknown (because they are not
@@ -156,7 +157,8 @@ void chan_extra_can_send(struct chan_extra_map *chan_extra_map,
 			 struct amount_msat x);
 
 /* Update the knowledge that this (channel,direction) cannot send x msat.*/
-void chan_extra_cannot_send(struct chan_extra_map *chan_extra_map,
+void chan_extra_cannot_send(struct payment* p,
+			    struct chan_extra_map *chan_extra_map,
 			    struct short_channel_id scid, 
 			    int dir,
 			    struct amount_msat x);
diff --git a/plugins/renepay/pay.c b/plugins/renepay/pay.c
index 8340979e69a2..3a2c8a345fc1 100644
--- a/plugins/renepay/pay.c
+++ b/plugins/renepay/pay.c
@@ -845,7 +845,12 @@ static struct command_result *waitsendpay_failed(struct command *cmd,
 		chan_extra_can_send(pay_plugin->chan_extra_map,
 				    flow->path_scids[i],
 				    flow->path_dirs[i],
-				    flow->amounts[i]);
+				    
+				    /* This channel can send all that was
+				     * commited in HTLCs.
+				     * Had we removed the commited amount then
+				     * we would have to put here flow->amounts[i]. */
+				    AMOUNT_MSAT(0));
 	}
 	switch ((enum onion_wire)onionerr) {
 	/* These definitely mean eliminate channel */
@@ -889,19 +894,15 @@ static struct command_result *waitsendpay_failed(struct command *cmd,
 	/* Insufficient funds! */
 	case WIRE_TEMPORARY_CHANNEL_FAILURE: {
 		plugin_log(pay_plugin->plugin,LOG_DBG,"waitsendpay_failed: Insufficient funds!");
-		/* OK new max is amount - 1 */
-		struct amount_msat max_possible;
-		if (!amount_msat_sub(&max_possible,
-				     flow->amounts[erridx],
-				     AMOUNT_MSAT(1)))
-			max_possible = AMOUNT_MSAT(0);
-		paynote(p, "... assuming that max capacity is %s",
-			type_to_string(tmpctx, struct amount_msat,
-				       &max_possible));
-		chan_extra_cannot_send(pay_plugin->chan_extra_map,
+		
+		chan_extra_cannot_send(p,pay_plugin->chan_extra_map,
 				       flow->path_scids[erridx],
 				       flow->path_dirs[erridx],
-				       flow->amounts[erridx]);
+				    /* This channel can't send all that was
+				     * commited in HTLCs.
+				     * Had we removed the commited amount then
+				     * we would have to put here flow->amounts[erridx]. */
+				       AMOUNT_MSAT(0));
 		goto done;
 	}
 
diff --git a/plugins/renepay/pay.h b/plugins/renepay/pay.h
index 0902c59d0c58..576019953d94 100644
--- a/plugins/renepay/pay.h
+++ b/plugins/renepay/pay.h
@@ -6,9 +6,18 @@
 #include <plugins/libplugin.h>
 #include <plugins/renepay/flow.h>
 
-// TODO(eduardo): check if we are doing correctly the knowledge update: if a
-// channel is not able to forward X it doesn't mean MAX<X, it means that MAX<X+Y
-// where Y is the sum of all HTLCs.
+// TODO(eduardo): renepaystatus should be similar to paystatus
+
+// TODO(eduardo): notice that payment must outlive the command!
+
+// TODO(eduardo): MCF should consider pending HTLCs occupy some capacity in the
+// routing channels.
+
+// TODO(eduardo): renepaystatus sometimes shows a truncated invoice string
+// 
+// TODO(eduardo): for some reason cln-renepay does not terminate inmediately:
+// 2023-06-26T07:32:36.064Z DEBUG   lightningd: cln-renepay: failed to self-terminate in time, killing.
+// 
 // TODO(eduardo): check a problem with param_millionths(), if I input an integer
 // should or shouldn't be multiplied by 10^6?
 // TODO(eduardo): add an option entry for maxfeepercent

From a05f5041b3405446e6bb6d4b84a55f4db5aef24f Mon Sep 17 00:00:00 2001
From: Lagrang3 <eduardo.quintana@pm.me>
Date: Fri, 30 Jun 2023 08:22:26 +0100
Subject: [PATCH 63/64] renepay: restructure the code (read below)

- add a renepay object that lives throughout the life of the command.
- make a clear distinction between struct renepay and struct payment.
- add a payment module to keep the methods of renepay and payment.
- add a uncertainty_network module
- waitsendpay calls now are detached from the command, allowing to
  receive responses even after the command finishes
  (yet I think this could have been done in a more robust way using
  notifications instead of waitsendpay).
---
 plugins/renepay/Makefile               |    4 +-
 plugins/renepay/debug.h                |   15 +-
 plugins/renepay/flow.c                 |  117 +--
 plugins/renepay/flow.h                 |   19 +-
 plugins/renepay/pay.c                  | 1168 ++++++++++--------------
 plugins/renepay/pay.h                  |  157 +---
 plugins/renepay/pay_flow.c             |   85 +-
 plugins/renepay/pay_flow.h             |   16 +-
 plugins/renepay/payment.c              |  225 +++++
 plugins/renepay/payment.h              |  163 ++++
 plugins/renepay/test/run-mcf-diamond.c |    4 +-
 plugins/renepay/test/run-mcf.c         |    4 +-
 plugins/renepay/test/run-testflow.c    |    6 +-
 plugins/renepay/uncertainty_network.c  |  346 +++++++
 plugins/renepay/uncertainty_network.h  |   46 +
 tests/test_renepay.py                  |   16 +-
 16 files changed, 1385 insertions(+), 1006 deletions(-)
 create mode 100644 plugins/renepay/payment.c
 create mode 100644 plugins/renepay/payment.h
 create mode 100644 plugins/renepay/uncertainty_network.c
 create mode 100644 plugins/renepay/uncertainty_network.h

diff --git a/plugins/renepay/Makefile b/plugins/renepay/Makefile
index b026f625dd2a..b0149fb1c124 100644
--- a/plugins/renepay/Makefile
+++ b/plugins/renepay/Makefile
@@ -1,7 +1,7 @@
 PLUGIN_RENEPAY_SRC := plugins/renepay/pay.c plugins/renepay/pay_flow.c plugins/renepay/flow.c plugins/renepay/mcf.c plugins/renepay/dijkstra.c \
-	plugins/renepay/debug.c
+	plugins/renepay/debug.c plugins/renepay/payment.c plugins/renepay/uncertainty_network.c
 PLUGIN_RENEPAY_HDRS := plugins/renepay/pay.h plugins/renepay/pay_flow.h plugins/renepay/flow.h plugins/renepay/mcf.h plugins/renepay/heap.h plugins/renepay/dijkstra.h \
-	plugins/renepay/debug.h
+	plugins/renepay/debug.h plugins/renepay/payment.h plugins/renepay/uncertainty_network.h
 PLUGIN_RENEPAY_OBJS := $(PLUGIN_RENEPAY_SRC:.c=.o)
 
 # Make sure these depend on everything.
diff --git a/plugins/renepay/debug.h b/plugins/renepay/debug.h
index 7c61e6eea4d6..d8f221539455 100644
--- a/plugins/renepay/debug.h
+++ b/plugins/renepay/debug.h
@@ -20,20 +20,24 @@ void _debug_exec_branch(const char* fname,const char* fun, int lineno);
 #define MYLOG "/tmp/debug.txt"
 #endif
 
-#ifdef FLOW_UNITTEST
+
+/* All debug information goes to a file. */
+#ifdef RENEPAY_UNITTEST
 
 #define debug_info(...) \
 	_debug_info(MYLOG,__VA_ARGS__)
 
 #define debug_err(...) \
-	_debug_info(MYLOG,__VA_ARGS__); abort()
+	{_debug_info(MYLOG,__VA_ARGS__); abort();}
 
 #define debug_paynote(p,...) \
-	_debug_info(MYLOG,__VA_ARGS__);
+	{payment_note(p,__VA_ARGS__);_debug_info(MYLOG,__VA_ARGS__);}
 
 #else
+/* Debugin information goes either to payment notes or to lightningd log. */
 
 #include <plugins/renepay/pay.h>
+#include <plugins/renepay/payment.h>
 
 #define debug_info(...) \
 	plugin_log(pay_plugin->plugin,LOG_DBG,__VA_ARGS__)
@@ -42,9 +46,12 @@ void _debug_exec_branch(const char* fname,const char* fun, int lineno);
 	plugin_err(pay_plugin->plugin,__VA_ARGS__)
 
 #define debug_paynote(p,...) \
-	paynote(p,__VA_ARGS__);
+	payment_note(p,__VA_ARGS__);
 
 #endif
 
+#define debug_assert(expr) \
+	if(!(expr)) debug_err("Assertion failed %s, file: %s, line %d", #expr,__FILE__,__LINE__)
+
 	
 #endif
diff --git a/plugins/renepay/flow.c b/plugins/renepay/flow.c
index f448a9529906..b1679195afbe 100644
--- a/plugins/renepay/flow.c
+++ b/plugins/renepay/flow.c
@@ -16,6 +16,12 @@
 #define SUPERVERBOSE_ENABLED 1
 #endif
 
+bool chan_extra_is_busy(struct chan_extra const * const ce)
+{
+	if(ce==NULL)return false;
+	return ce->half[0].num_htlcs || ce->half[1].num_htlcs;
+}
+
 const char *fmt_chan_extra_map(
 		const tal_t *ctx,
 		struct chan_extra_map* chan_extra_map)
@@ -66,26 +72,6 @@ struct chan_extra *new_chan_extra(
 	return ce;
 }
 
-bool chan_extra_check_invariants(struct chan_extra *ce)
-{
-	bool all_ok = true;
-	for(int i=0;i<2;++i)
-	{
-		all_ok &= amount_msat_less_eq(ce->half[i].known_min,
-					      ce->half[i].known_max);
-		all_ok &= amount_msat_less_eq(ce->half[i].known_max,
-					      ce->capacity);
-	}
-	struct amount_msat diff_cb,diff_ca;
-	
-	all_ok &= amount_msat_sub(&diff_cb,ce->capacity,ce->half[1].known_max);
-	all_ok &= amount_msat_sub(&diff_ca,ce->capacity,ce->half[1].known_min);
-	
-	all_ok &= amount_msat_eq(ce->half[0].known_min,diff_cb);
-	all_ok &= amount_msat_eq(ce->half[0].known_max,diff_ca);
-	return all_ok;
-}
-
 /* This helper function preserves the uncertainty network invariant after the
  * knowledge is updated. It assumes that the (channel,!dir) knowledge is
  * correct. */
@@ -316,97 +302,6 @@ void chan_extra_relax(
 	chan_extra_relax_(ce,dir,x,y);
 }
 
-/* Checks the entire uncertainty network for invariant violations. */
-bool uncertainty_network_check_invariants(struct chan_extra_map *chan_extra_map)
-{
-	bool all_ok = true;
-	
-	struct chan_extra_map_iter it;
-	for(struct chan_extra *ce = chan_extra_map_first(chan_extra_map,&it);
-	    ce && all_ok;
-	    ce=chan_extra_map_next(chan_extra_map,&it))
-	{
-		all_ok &= chan_extra_check_invariants(ce);
-	}
-		
-	return all_ok;
-}
-
-/* Mirror the gossmap in the public uncertainty network.
- * result: Every channel in gossmap must have associated data in chan_extra_map,
- * while every channel in chan_extra_map is also registered in gossmap.
- * */
-void uncertainty_network_update(
-		const struct gossmap *gossmap,
-		struct chan_extra_map *chan_extra_map)
-{
-	const tal_t* this_ctx = tal(tmpctx,tal_t);
-	
-	// For each chan in chan_extra_map remove if not in the gossmap
-	struct short_channel_id *del_list
-		= tal_arr(this_ctx,struct short_channel_id,0);
-		
-	struct chan_extra_map_iter it;
-	for(struct chan_extra *ce = chan_extra_map_first(chan_extra_map,&it);
-	    ce;
-	    ce=chan_extra_map_next(chan_extra_map,&it))
-	{
-		struct gossmap_chan * chan = gossmap_find_chan(gossmap,&ce->scid);
-		if(!chan)
-		{
-			// TODO(eduardo): is this efficiently implemented?
-			// otherwise i'll use a ccan list
-			tal_arr_expand(&del_list, ce->scid);
-		}
-	}
-	
-	for(size_t i=0;i<tal_count(del_list);++i)
-	{
- 		struct chan_extra *ce = chan_extra_map_get(chan_extra_map,del_list[i]);
-		if(!ce)
-		{
-			debug_err("%s (line %d) unexpected chan_extra ce is NULL",
-				__PRETTY_FUNCTION__,
-				__LINE__);
-		}
-		chan_extra_map_del(chan_extra_map, ce);
-		tal_free(ce);
-	}
-	
-	// For each channel in the gossmap, create a extra data in
-	// chan_extra_map
-	for(struct gossmap_chan *chan = gossmap_first_chan(gossmap);
-	    chan;
-	    chan=gossmap_next_chan(gossmap,chan))
-	{
-		struct short_channel_id scid =
-			gossmap_chan_scid(gossmap,chan);
-		struct chan_extra *ce = chan_extra_map_get(chan_extra_map,
-							   gossmap_chan_scid(gossmap,chan));	
-		if(!ce)
-		{
-			struct amount_sat cap;
-			struct amount_msat cap_msat;
-			
-			if(!gossmap_chan_get_capacity(gossmap,chan,&cap))
-			{
-				debug_err("%s (line %d) unable to fetch channel capacity",
-					__PRETTY_FUNCTION__,
-					__LINE__);
-			}
-			if(!amount_sat_to_msat(&cap_msat,cap))
-			{
-				debug_err("%s (line %d) unable convert sat to msat",
-					__PRETTY_FUNCTION__,
-					__LINE__);
-			}
-			new_chan_extra(chan_extra_map,scid,cap_msat);
-		}
-	}
-	
-	
-	tal_free(this_ctx);
-}
 
 /* Returns either NULL, or an entry from the hash */
 struct chan_extra_half *
diff --git a/plugins/renepay/flow.h b/plugins/renepay/flow.h
index 1e5f64750320..06bffe154b21 100644
--- a/plugins/renepay/flow.h
+++ b/plugins/renepay/flow.h
@@ -5,8 +5,7 @@
 #include <ccan/htable/htable_type.h>
 #include <common/amount.h>
 #include <common/gossmap.h>
-
-struct payment;
+#include <plugins/renepay/payment.h>
 
 // TODO(eduardo): a hard coded constant to indicate a limit on any channel
 // capacity. Channels for which the capacity is unknown (because they are not
@@ -41,6 +40,8 @@ struct chan_extra {
 	} half[2];
 };
 
+bool chan_extra_is_busy(struct chan_extra const * const ce);
+
 static inline const struct short_channel_id
 chan_extra_scid(const struct chan_extra *cd)
 {
@@ -123,11 +124,6 @@ struct chan_extra *new_chan_extra(
 		const struct short_channel_id scid,
 		struct amount_msat capacity);
 
-/* Checks for this chan_extra if the invariants are satisfied. */
-bool chan_extra_check_invariants(struct chan_extra *ce);
-
-/* Checks the entire uncertainty network for invariant violations. */
-bool uncertainty_network_check_invariants(struct chan_extra_map *chan_extra_map);
 
 /* This helper function preserves the uncertainty network invariant after the
  * knowledge is updated. It assumes that the (channel,!dir) knowledge is
@@ -182,15 +178,6 @@ void chan_extra_relax(struct chan_extra_map *chan_extra_map,
 		      struct amount_msat down,
 		      struct amount_msat up);
 
-/* Mirror the gossmap in the public uncertainty network.
- * result: Every channel in gossmap must have associated data in chan_extra_map,
- * while every channel in chan_extra_map is also registered in gossmap.
- * */
-void uncertainty_network_update(
-		const struct gossmap *gossmap,
-		struct chan_extra_map *chan_extra_map);
-
-
 
 /* Returns either NULL, or an entry from the hash */
 struct chan_extra_half *get_chan_extra_half_by_scid(struct chan_extra_map *chan_extra_map,
diff --git a/plugins/renepay/pay.c b/plugins/renepay/pay.c
index 3a2c8a345fc1..4216f8acc967 100644
--- a/plugins/renepay/pay.c
+++ b/plugins/renepay/pay.c
@@ -16,138 +16,28 @@
 #include <plugins/renepay/pay.h>
 #include <plugins/renepay/pay_flow.h>
 #include <plugins/renepay/debug.h>
+#include <plugins/renepay/payment.h>
+#include <plugins/renepay/uncertainty_network.h>
+
+// TODO(eduardo): understand the use of struct command_result as a return.
+
+// TODO(eduardo): maybe there are too many debug_err and plugin_err and
+// plugin_log(...,LOG_BROKEN,...) that could be resolved with a command_fail
 
 #define INVALID_ID UINT64_MAX 
 #define MAX(a,b) ((a)>(b)? (a) : (b))
 
-// TODO(eduardo): the state of plugin is stored in data (instead of the heap) so
-// that we can load lightningd options directly into before `init` is called.
 static struct pay_plugin the_pay_plugin;
 struct pay_plugin * const pay_plugin = &the_pay_plugin;
 
-static void renepay_cleanup(struct active_payment *ap);
-static void timer_kick(struct payment *p);
+static void timer_kick(struct renepay * renepay);
 static struct command_result *try_paying(struct command *cmd,
-					 struct payment *p,
+					 struct renepay * renepay,
 					 bool first_time);
 
-static void payment_check_delivering_incomplete(struct payment *p)
-{
-	if(!amount_msat_less(p->total_delivering, p->amount))
-	{
-		plugin_err(
-			pay_plugin->plugin,
-			"Strange, delivering (%s) is not smaller than amount (%s)",
-			type_to_string(tmpctx,struct amount_msat,&p->total_delivering),
-			type_to_string(tmpctx,struct amount_msat,&p->amount));
-	}
-}
-static void payment_check_delivering_all(struct payment *p)
-{
-	if(amount_msat_less(p->total_delivering, p->amount))
-	{
-		plugin_err(
-			pay_plugin->plugin,
-			"Strange, delivering (%s) is less than amount (%s)",
-			type_to_string(tmpctx,struct amount_msat,&p->total_delivering),
-			type_to_string(tmpctx,struct amount_msat,&p->amount));
-	}
-}
-static struct command * payment_command(struct payment *p)
-{
-	return p->active_payment->cmd;
-}
-static u64 payment_parts(const struct payment *p)
-{
-	return p->active_payment->next_partid-1;
-}
-int payment_current_attempt(const struct payment *p)
-{
-	return p->active_payment->last_attempt;
-}
-static int payment_attempt_count(const struct payment *p)
-{
-	return p->active_payment->last_attempt+1;
-}
-static void payment_new_attempt(struct payment *p)
-{
-	p->status=PAYMENT_PENDING;
-	p->active_payment->last_attempt++;
-}
-static u64 payment_groupid(struct payment *p)
-{
-	return p->groupid;
-}
-static void payment_initialize(
-		struct payment *p,
-		struct command *cmd)
-{
-	/* Owned by cmd, because this data is destroyed after the payment
-	 * completes, while p remains to get payment status history. */
-	p->active_payment = tal(cmd,struct active_payment);
-	tal_add_destructor(p->active_payment, renepay_cleanup);
-	
-	p->active_payment->cmd = cmd;
- 	p->active_payment->local_gossmods = gossmap_localmods_new(p->active_payment);
-	p->active_payment->localmods_applied=false;
-	p->active_payment->disabled = tal_arr(p->active_payment, 
-					      struct short_channel_id, 
-					      0);
-	p->active_payment->rexmit_timer = NULL;
-	p->active_payment->last_attempt=-1;
-	p->active_payment->all_flows = tal(p->active_payment,tal_t);
-}
-// static struct amount_msat payment_fees(struct payment *p)
-// {
-// 	struct amount_msat fees;
-// 	struct amount_msat sent = payment_sent(p), 
-// 			   received = payment_amount(p);
-// 			   
-// 	if(!amount_msat_sub(&fees,sent,received))
-// 		plugin_err(payment_command(p)->plugin,
-// 			   "Strange, sent amount (%s) is less than received (%s), aborting.",
-// 			   type_to_string(tmpctx,struct amount_msat,&sent),
-// 			   type_to_string(tmpctx,struct amount_msat,&received));
-// 	return fees;
-// }
-
-static struct command_result *payment_success(struct payment *p)
-{
-	plugin_log(pay_plugin->plugin,LOG_DBG,"calling %s",__PRETTY_FUNCTION__);
-	payment_check_delivering_all(p);
-	
-	struct json_stream *response 
-		= jsonrpc_stream_success(payment_command(p));
-
-	/* Any one succeeding is success. */
-	json_add_preimage(response, "payment_preimage", p->preimage);
-	json_add_sha256(response, "payment_hash", &p->payment_hash);
-	json_add_timeabs(response, "created_at", p->start_time);
-	json_add_u32(response, "parts", payment_parts(p));
-	json_add_amount_msat_only(response, "amount_msat",
-				  p->amount);
-	json_add_amount_msat_only(response, "amount_sent_msat",
-				  p->total_sent);
-	json_add_string(response, "status", "complete");
-	json_add_node_id(response, "destination", &p->destination);
-	
-	return command_finished(payment_command(p), response);
-}
-
-// TODO(eduardo): improve loging
-// TODO(eduardo): handle hanging responses
-
-void paynote(struct payment *p, const char *fmt, ...)
-{
-	va_list ap;
-	const char *str;
-
-	va_start(ap, fmt);
-	str = tal_vfmt(p->paynotes, fmt, ap);
-	va_end(ap);
-	tal_arr_expand(&p->paynotes, str);
-	plugin_log(pay_plugin->plugin, LOG_DBG, "%s", str);
-}
+// TODO(eduardo): maybe we don't need these
+static void background_timer_kick(void*p UNUSED);
+static void background_settimer(void);
 
 void amount_msat_accumulate_(struct amount_msat *dst,
 			     struct amount_msat src,
@@ -156,7 +46,7 @@ void amount_msat_accumulate_(struct amount_msat *dst,
 {
 	if (amount_msat_add(dst, *dst, src))
 		return;
-	plugin_err(pay_plugin->plugin, "Overflow adding %s (%s) into %s (%s)",
+	debug_err("Overflow adding %s (%s) into %s (%s)",
 		   srcname, type_to_string(tmpctx, struct amount_msat, &src),
 		   dstname, type_to_string(tmpctx, struct amount_msat, dst));
 }
@@ -168,7 +58,7 @@ void amount_msat_reduce_(struct amount_msat *dst,
 {
 	if (amount_msat_sub(dst, *dst, src))
 		return;
-	plugin_err(pay_plugin->plugin, "Underflow subtracting %s (%s) from %s (%s)",
+	debug_err("Underflow subtracting %s (%s) from %s (%s)",
 		   srcname, type_to_string(tmpctx, struct amount_msat, &src),
 		   dstname, type_to_string(tmpctx, struct amount_msat, dst));
 }
@@ -193,36 +83,6 @@ static void memleak_mark(struct plugin *p, struct htable *memtable)
 }
 #endif
 
-static void remove_htlc_payflow_and_update_knowlege(
-		struct pay_flow *flow,
-		struct chan_extra_map *chan_extra_map)
-{
-	plugin_log(pay_plugin->plugin,LOG_DBG,"calling %s",__PRETTY_FUNCTION__);
-	
-	// TODO(eduardo): big assumption here, HTLCs can just simply dissapear
-	// if the MPP payment completes or the flow fails.
-	remove_htlc_payflow(flow,chan_extra_map);
-	
-	struct payment *p = flow->payment;
-	
-	switch(p->status)
-	{
-		case PAYMENT_SUCCESS:
-			for (size_t i = 0; i < tal_count(flow->path_scids); i++)
-			{
-				chan_extra_sent_success(pay_plugin->chan_extra_map,
-					flow->path_scids[i],
-					flow->path_dirs[i],
-					flow->amounts[i]);
-			}
-			break;
-		case PAYMENT_FAIL:
-		case PAYMENT_PENDING:
-		case PAYMENT_MPP_TIMEOUT:
-			break;
-	}
-}
-
 static const char *init(struct plugin *p,
 			const char *buf UNUSED, const jsmntok_t *config UNUSED)
 {
@@ -230,6 +90,7 @@ static const char *init(struct plugin *p,
 
 	pay_plugin->ctx = notleak_with_children(tal(p,tal_t));
 	pay_plugin->plugin = p;
+	pay_plugin->rexmit_timer=NULL;
 
 	rpc_scan(p, "getinfo", take(json_out_obj(NULL, NULL, NULL)),
 		 "{id:%}", JSON_SCAN(json_to_node_id, &pay_plugin->my_id));
@@ -263,273 +124,89 @@ static const char *init(struct plugin *p,
 #if DEVELOPER
 	plugin_set_memleak_handler(p, memleak_mark);
 #endif
-
-	return NULL;
-}
-
-static void add_hintchan(struct payment *p,
-			  const struct node_id *src,
-			  const struct node_id *dst,
-			  u16 cltv_expiry_delta,
-			  const struct short_channel_id scid,
-			  u32 fee_base_msat,
-			  u32 fee_proportional_millionths,
-			  struct amount_msat amount UNUSED)
-{
-	int dir = node_id_cmp(src, dst) < 0 ? 0 : 1;
 	
-	struct chan_extra *ce = chan_extra_map_get(pay_plugin->chan_extra_map,
-						   scid);
-	if(!ce)
-	{
-		/* this channel is not public, we don't know his capacity */
-		// TODO(eduardo): one possible solution is set the capacity to
-		// MAX_CAP and the state to [0,MAX_CAP]. Alternatively we set
-		// the capacity to amoung and state to [amount,amount].
-		ce = new_chan_extra(pay_plugin->chan_extra_map,
-				    scid,
-				    MAX_CAP);
-		/* FIXME: features? */
-		gossmap_local_addchan(p->active_payment->local_gossmods, 
-				      src, dst, &scid, NULL);
-		gossmap_local_updatechan(p->active_payment->local_gossmods,
-					 &scid,
-					 /* We assume any HTLC is allowed */
-					 AMOUNT_MSAT(0), MAX_CAP,
-					 fee_base_msat, fee_proportional_millionths,
-					 cltv_expiry_delta,
-					 true,
-					 dir);
-	}
-	
-	/* It is wrong to assume that this channel has sufficient capacity! 
-	 * Doing so leads to knowledge updates in which the known min liquidity
-	 * is greater than the channel's capacity. */
-	// chan_extra_can_send(pay_plugin->chan_extra_map,scid,dir,amount);
+	background_settimer();
+	return NULL;
 }
 
-/* Add routehints provided by bolt11 */
-static void uncertainty_network_add_routehints(struct payment *p)
-{
-	struct bolt11 *b11;
-	char *fail;
-
-	b11 =
-	    bolt11_decode(tmpctx, p->invstr, 
-	    		  plugin_feature_set(p->active_payment->cmd->plugin),
-			  p->description, chainparams, &fail);
-	if (b11 == NULL)
-		plugin_log(pay_plugin->plugin, LOG_BROKEN,
-				    "add_routehints: Invalid bolt11: %s", fail);
-		
 
-	for (size_t i = 0; i < tal_count(b11->routes); i++) {
-		/* Each one, presumably, leads to the destination */
-		const struct route_info *r = b11->routes[i];
-		const struct node_id *end = & p->destination;
-		for (int j = tal_count(r)-1; j >= 0; j--) {
-			// TODO(eduardo): amount to send to the add_hintchan
-			// should consider fees.
-			
-			add_hintchan(p, &r[j].pubkey, end,
-				      r[j].cltv_expiry_delta,
-				      r[j].short_channel_id,
-				      r[j].fee_base_msat,
-				      r[j].fee_proportional_millionths,
-				      p->maxspend);
-			end = &r[j].pubkey;
-		}
-	}
-}
-
-/* listpeerchannels gives us the certainty on local channels' capacity.  Of course,
- * this is racy and transient, but better than nothing! */
-static bool update_uncertainty_network_from_listpeerchannels(
-		struct plugin *plugin,
-		struct payment *p,
-		const char *buf,
-		const jsmntok_t *toks)
+// /* TODO(eduardo): an example of an RPC call that is not bound to any command. */
+//static 
+//struct command_result* getinfo_done(struct command *cmd UNUSED,
+//			 const char *buf,
+//			 const jsmntok_t *result,
+//			 void* pp  UNUSED)
+//{
+//	struct node_id id;
+//	const jsmntok_t *id_tok = json_get_member(buf,result,"id");
+//	json_to_node_id(buf,id_tok,&id);
+//	
+//	plugin_log(pay_plugin->plugin,LOG_DBG,
+//		"calling %s, nodeid = %s",
+//		__PRETTY_FUNCTION__,
+//		type_to_string(tmpctx,struct node_id,&id));
+//	
+//	return command_still_pending(NULL);
+//}
+
+static void background_settimer(void)
 {
-	const jsmntok_t *channels, *channel;
-	size_t i;
-
-	if (json_get_member(buf, toks, "error"))
-		goto malformed;
-
-	channels = json_get_member(buf, toks, "channels");
-	if (!channels)
-		goto malformed;
-
-	json_for_each_arr(i, channel, channels) {
-		struct short_channel_id scid;
-		const jsmntok_t *scidtok = json_get_member(buf, channel, "short_channel_id");
-		/* If channel is still opening, this won't be there. 
-		 * Also it won't be in the gossmap, so there is 
-		 * no need to mark it as disabled. */
-		if (!scidtok)
-			continue;
-		if (!json_to_short_channel_id(buf, scidtok, &scid))
-			goto malformed;
-	
-		bool connected;
-		if(!json_to_bool(buf,
-				 json_get_member(buf,channel,"peer_connected"),
-				 &connected))
-			goto malformed;
-	
-		if (!connected) {
-			paynote(p, "local channel %s disabled:"
-				" peer disconnected",
-				type_to_string(tmpctx,
-					       struct short_channel_id,
-					       &scid));
-			tal_arr_expand(&p->active_payment->disabled, scid);
-			continue;
-		}
-		
-		const jsmntok_t *spendabletok, *dirtok,*statetok, *totaltok, 
-			*peeridtok;
-		struct amount_msat spendable,capacity;
-		int dir;
-		
-		const struct node_id src=pay_plugin->my_id;
-		struct node_id dst;
-		
-		spendabletok = json_get_member(buf, channel, "spendable_msat");
-		dirtok = json_get_member(buf, channel, "direction");
-		statetok = json_get_member(buf, channel, "state");
-		totaltok = json_get_member(buf, channel, "total_msat");
-		peeridtok = json_get_member(buf,channel,"peer_id");
-		
-		if(spendabletok==NULL || dirtok==NULL || statetok==NULL || 
-		   totaltok==NULL || peeridtok==NULL)
-			goto malformed;
-		if (!json_to_msat(buf, spendabletok, &spendable))
-			goto malformed;
-		if (!json_to_msat(buf, totaltok, &capacity))
-			goto malformed;
-		if (!json_to_int(buf, dirtok,&dir))
-			goto malformed;
-		if(!json_to_node_id(buf,peeridtok,&dst))
-			goto malformed;
-			
-		/* Don't report opening/closing channels */
-		if (!json_tok_streq(buf, statetok, "CHANNELD_NORMAL")) {
-			tal_arr_expand(&p->active_payment->disabled, scid);
-			continue;
-		}
-		
-		struct chan_extra *ce = chan_extra_map_get(pay_plugin->chan_extra_map,
-							   scid);
-			  
-		if(!ce)
-		{
-			/* this channel is not public, but it belongs to us */
-			ce = new_chan_extra(pay_plugin->chan_extra_map,
-					    scid,
-					    capacity);
-			/* FIXME: features? */
-			gossmap_local_addchan(p->active_payment->local_gossmods, 
-					      &src, &dst, &scid, NULL);
-			gossmap_local_updatechan(p->active_payment->local_gossmods,
-						 &scid,
-						 
-						 /* TODO(eduardo): does it
-						  * matter to consider HTLC
-						  * limits in our own channel? */
-						 AMOUNT_MSAT(0),capacity,
-						 
-						 /* fees = */0,0,
-						 
-						 /* TODO(eduardo): does it
-						  * matter to set this delay? */
-						 /*delay=*/0,
-						 true,
-						 dir);
-		}
-		
-		/* We know min and max liquidity exactly now! */
-		chan_extra_set_liquidity(pay_plugin->chan_extra_map,
-					 scid,dir,spendable);
-	}
-	return true;
-
-malformed:
-	plugin_log(plugin, LOG_BROKEN,
-		   "listpeerchannels malformed: %.*s",
-		   json_tok_full_len(toks),
-		   json_tok_full(buf, toks));
-	return false;
-}
-
-/* How much does this flow deliver to destination? */
-static struct amount_msat flow_delivered(const struct pay_flow *flow)
-{
-	return flow->amounts[tal_count(flow->amounts)-1];
+	pay_plugin->rexmit_timer 
+		= tal_free(pay_plugin->rexmit_timer);
+	pay_plugin->rexmit_timer 
+		= plugin_timer(
+			pay_plugin->plugin,
+			time_from_msec(2000),
+			background_timer_kick, NULL);
 }
 
-
-static struct command_result *flow_failed(
-		struct command *cmd,
-		const struct pay_flow *flow)
+static void background_timer_kick(void * p UNUSED)
 {
-	plugin_log(pay_plugin->plugin,LOG_DBG,"calling flow_failed");
-	struct payment *p = flow->payment;
-	
-	amount_msat_reduce(&p->total_delivering, flow_delivered(flow));
-	amount_msat_reduce(&p->total_sent, flow->amounts[0]);
-	tal_free(flow);
-	
-	/* Still waiting for timer... */
-	return command_still_pending(cmd);
+	// plugin_log(pay_plugin->plugin,LOG_DBG,"calling %s",__PRETTY_FUNCTION__);
+	background_settimer();
+	
+	// /* TODO(eduardo): an example of an RPC call that is not bound to any command. */
+	// struct out_req * req = jsonrpc_request_start(pay_plugin->plugin,
+	// 				NULL, 
+	// 				"getinfo",
+	// 			    	getinfo_done,
+	// 			    	getinfo_done,
+	// 			    	NULL);
+	// send_outreq(pay_plugin->plugin, req);
 }
 
-
-static void payment_settimer(struct payment *p)
+static void renepay_settimer(struct renepay * renepay)
 {
-	p->active_payment->rexmit_timer 
-		= tal_free(p->active_payment->rexmit_timer);
-	p->active_payment->rexmit_timer 
-		= plugin_timer(
+	renepay->rexmit_timer = tal_free(renepay->rexmit_timer);
+	renepay->rexmit_timer = plugin_timer(
 			pay_plugin->plugin,
-			time_from_msec(250),
-			timer_kick, p);
+			time_from_msec(TIMER_COLLECT_FAILURES_MSEC),
+			timer_kick, renepay);
 }
 
 /* Happens when timer goes off, but also works to arm timer if nothing to do */
-static void timer_kick(struct payment *p)
+static void timer_kick(struct renepay * renepay)
 {
+	struct payment * const p = renepay->payment;
 	plugin_log(pay_plugin->plugin,LOG_DBG,"calling %s",__PRETTY_FUNCTION__);
 	
-	/* No, but payment hasn't gone through, so let's wait a bit. */
-	
 	switch(p->status)
 	{
 		/* Some flows succeeded, we finish the payment. */
 		case PAYMENT_SUCCESS:
-			payment_check_delivering_all(p);
-			// payment_success(p);
+			renepay_success(renepay);
 		break;
 		
 		/* Some flows failed, we retry. */
 		case PAYMENT_FAIL:
-			payment_check_delivering_incomplete(p);
-			try_paying(payment_command(p),p,false);
+			payment_assert_delivering_incomplete(p);
+			try_paying(renepay->cmd,renepay,false);
 		break;
 		
 		/* Nothing has returned yet, we have to wait. */
 		case PAYMENT_PENDING:
-			payment_check_delivering_all(p);
-			payment_settimer(p);
-		break;
-		/* Some flows timeout we wait until all of them have done so. */
-		case PAYMENT_MPP_TIMEOUT:
-			payment_check_delivering_incomplete(p);
-			if(amount_msat_eq(p->total_delivering,AMOUNT_MSAT(0)))
-			{
-				try_paying(payment_command(p),p,false);
-			}
-			payment_settimer(p);
+			payment_assert_delivering_all(p);
+			renepay_settimer(renepay);
 		break;
 	}
 }
@@ -542,11 +219,8 @@ static struct command_result *waitsendpay_succeeded(struct command *cmd,
 						    struct pay_flow *flow)
 {
 	plugin_log(pay_plugin->plugin,LOG_DBG,"calling %s",__PRETTY_FUNCTION__);
-	
-	struct payment *p = flow->payment;
-	
-	/* This is a success. */
-	p->status = PAYMENT_SUCCESS;
+	struct payment * const p = flow->payment;
+	payment_success(p);
 	
 	const jsmntok_t *preimagetok
 		= json_get_member(buf, result, "payment_preimage");
@@ -559,39 +233,37 @@ static struct command_result *waitsendpay_succeeded(struct command *cmd,
 			   json_tok_full_len(result),
 			   json_tok_full(buf, result));
 	
-	p->preimage = tal_dup_or_null(p, struct preimage, &preimage);
-	
-	return payment_success(p);
-}
+	p->preimage = tal_dup_or_null(p,struct preimage,&preimage);
 
-static const char *flow_path_to_str(const tal_t *ctx, const struct pay_flow *flow)
-{
-	char *s = tal_strdup(ctx, "");
-	for (size_t i = 0; i < tal_count(flow->path_scids); i++) {
-		tal_append_fmt(&s, "-%s->",
-			       type_to_string(tmpctx, struct short_channel_id,
-					      &flow->path_scids[i]));
-	}
-	return s;
+	uncertainty_network_flow_success(pay_plugin->chan_extra_map,flow);
+	tal_free(flow);
+	return command_still_pending(cmd);
 }
 
+
 /* Sometimes we don't know exactly who to blame... */
-static struct command_result *handle_unhandleable_error(struct payment *p,
-							const struct pay_flow *flow,
+static struct command_result *handle_unhandleable_error(struct renepay * renepay,
+							struct pay_flow *flow,
 							const char *what)
 {
+	struct payment * const p = renepay->payment;
 	plugin_log(pay_plugin->plugin,LOG_DBG,"calling %s",__PRETTY_FUNCTION__);
 	size_t n = tal_count(flow);
 
 	/* We got a mangled reply.  We don't know who to penalize! */
-	paynote(p, "%s on route %s", what, flow_path_to_str(tmpctx, flow));
+	debug_paynote(p, "%s on route %s", what, flow_path_to_str(tmpctx, flow));
+	
+	// TODO(eduardo): does LOG_BROKEN finish the plugin execution?
 	plugin_log(pay_plugin->plugin, LOG_BROKEN,
 		   "%s on route %s",
 		   what, flow_path_to_str(tmpctx, flow));
 
 	if (n == 1)
-		return command_fail(p->active_payment->cmd, PAY_UNPARSEABLE_ONION,
+	{
+		payflow_fail(flow);
+		return renepay_fail(renepay, PAY_UNPARSEABLE_ONION,
 				    "Got %s from the destination", what);
+	}
 	/* FIXME: check chan_extra_map, since we might have succeeded though
 	 * this node before? */
 
@@ -603,8 +275,8 @@ static struct command_result *handle_unhandleable_error(struct payment *p,
 		/* Assume it's not the destination */
 		n = pseudorand(n-1);
 
-	tal_arr_expand(&p->active_payment->disabled, flow->path_scids[n]);
-	paynote(p, "... eliminated %s",
+	tal_arr_expand(&renepay->disabled, flow->path_scids[n]);
+	debug_paynote(p, "... eliminated %s",
 		type_to_string(tmpctx, struct short_channel_id,
 			       &flow->path_scids[n]));
 	return NULL;
@@ -614,7 +286,7 @@ static struct command_result *handle_unhandleable_error(struct payment *p,
  * gossipd to receive the channel_update we got from the error. */
 struct addgossip {
 	struct short_channel_id scid;
-	const struct pay_flow *flow;
+	struct pay_flow *flow;
 };
 
 static struct command_result *addgossip_done(struct command *cmd,
@@ -623,14 +295,15 @@ static struct command_result *addgossip_done(struct command *cmd,
 					     struct addgossip *adg)
 {
 	plugin_log(pay_plugin->plugin,LOG_DBG,"calling %s",__PRETTY_FUNCTION__);
-	struct payment *p = adg->flow->payment;
+	struct renepay * renepay = adg->flow->payment->renepay;
 
 	/* Release this: if it's the last flow we'll retry immediately */
 	
+	payflow_fail(adg->flow);
 	tal_free(adg);
-	payment_settimer(p);
-
-	return flow_failed(cmd,adg->flow);
+	renepay_settimer(renepay);
+	
+	return command_still_pending(cmd);
 }
 
 static struct command_result *addgossip_failure(struct command *cmd,
@@ -640,23 +313,25 @@ static struct command_result *addgossip_failure(struct command *cmd,
 
 {
 	plugin_log(pay_plugin->plugin,LOG_DBG,"calling %s",__PRETTY_FUNCTION__);
-	struct payment *p = adg->flow->payment;
+	struct payment * p = adg->flow->payment;
+	struct renepay * renepay = p->renepay;
 
-	paynote(p, "addgossip failed, removing channel %s (%.*s)",
+	debug_paynote(p, "addgossip failed, removing channel %s (%.*s)",
 		type_to_string(tmpctx, struct short_channel_id, &adg->scid),
 		err->end - err->start, buf + err->start);
-	tal_arr_expand(&p->active_payment->disabled, adg->scid);
+	tal_arr_expand(&renepay->disabled, adg->scid);
 
 	return addgossip_done(cmd, buf, err, adg);
 }
 
 static struct command_result *submit_update(struct command *cmd,
-					    const struct pay_flow *flow,
+					    struct pay_flow *flow,
 					    const u8 *update,
 					    struct short_channel_id errscid)
 {
 	plugin_log(pay_plugin->plugin,LOG_DBG,"calling %s",__PRETTY_FUNCTION__);
-	struct payment *p = flow->payment;
+	struct payment * p = flow->payment;
+	struct renepay * renepay = p->renepay;
 	struct out_req *req;
 	struct addgossip *adg = tal(cmd, struct addgossip);
 
@@ -665,10 +340,10 @@ static struct command_result *submit_update(struct command *cmd,
 	adg->scid = errscid;
 	adg->flow = flow;
 	/* Disable re-xmit until this returns */
-	p->active_payment->rexmit_timer 
-		= tal_free(p->active_payment->rexmit_timer);
+	renepay->rexmit_timer 
+		= tal_free(renepay->rexmit_timer);
 
-	paynote(p, "... extracted channel_update, telling gossipd");
+	debug_paynote(p, "... extracted channel_update, telling gossipd");
 	plugin_log(pay_plugin->plugin, LOG_DBG, "(update = %s)", tal_hex(tmpctx, update));
 
 	req = jsonrpc_request_start(pay_plugin->plugin, NULL, "addgossip",
@@ -737,6 +412,69 @@ static u8 *channel_update_from_onion_error(const tal_t *ctx,
 	return patch_channel_update(ctx, take(channel_update));
 }
 
+/* This is a waitsendpay_fail branch that happens in the background (ie. after
+ * renepay command returned). */
+static struct command_result *waitsendpay_failed_background(
+		const char *buf,
+		const jsmntok_t *err,
+		struct pay_flow *flow)
+{
+	plugin_log(pay_plugin->plugin,LOG_DBG,"calling %s",__PRETTY_FUNCTION__);
+	struct payment * const p = flow->payment;
+	
+	payment_fail(p);
+	
+	u64 errcode;
+	if (!json_to_u64(buf, json_get_member(buf, err, "code"), &errcode))
+		goto done;
+	
+	if(errcode!=PAY_TRY_OTHER_ROUTE)
+		goto done;
+
+	const jsmntok_t* datatok = json_get_member(buf, err, "data");
+
+	/* OK, we expect an onion error reply. */
+	const jsmntok_t * erridxtok = json_get_member(buf, datatok, "erring_index");
+	if (!erridxtok)
+		goto done;
+	
+	u32 erridx;
+	json_to_u32(buf, erridxtok, &erridx);
+	const jsmntok_t *errchantok = json_get_member(buf, datatok, "erring_channel");
+	struct short_channel_id errscid;
+	json_to_short_channel_id(buf, errchantok, &errscid);
+	
+	if (erridx<tal_count(flow->path_scids) 
+	    && !short_channel_id_eq(&errscid, &flow->path_scids[erridx]))
+	    	goto done;
+
+	const jsmntok_t *failcodetok = json_get_member(buf, datatok, "failcode");
+	u32 onionerr;
+	json_to_u32(buf, failcodetok, &onionerr);
+
+	uncertainty_network_channel_can_send(
+			pay_plugin->chan_extra_map,
+			flow,
+			erridx);
+	
+	/* Insufficient funds! */
+	if((enum onion_wire)onionerr == WIRE_TEMPORARY_CHANNEL_FAILURE){	
+		plugin_log(pay_plugin->plugin,LOG_DBG,"waitsendpay_failed: Insufficient funds!");
+		
+		chan_extra_cannot_send(p,pay_plugin->chan_extra_map,
+				       flow->path_scids[erridx],
+				       flow->path_dirs[erridx],
+				    /* This channel can't send all that was
+				     * commited in HTLCs.
+				     * Had we removed the commited amount then
+				     * we would have to put here flow->amounts[erridx]. */
+				       AMOUNT_MSAT(0));
+	}
+done:
+	payflow_fail(flow);
+	return command_still_pending(NULL);
+}
+
 static struct command_result *waitsendpay_failed(struct command *cmd,
 						 const char *buf,
 						 const jsmntok_t *err,
@@ -747,27 +485,19 @@ static struct command_result *waitsendpay_failed(struct command *cmd,
 		   json_tok_full_len(err),
 		   json_tok_full(buf, err));
 	
-	struct payment *p = flow->payment;
+	struct payment * const p = flow->payment;
+	struct renepay * const renepay = p->renepay;
 	
-	/* This is a fail. */
-	p->status = PAYMENT_FAIL;
+	payment_fail(p);
+	if(!renepay)
+		return waitsendpay_failed_background(buf,err,flow);
 	
-	/* An old flow attempt. Just free the HTLCs. */
-	if(flow->attempt != payment_current_attempt(p))
-	{
-		// TODO(eduardo): I guess this should not happen unless some
-		// sendpay that would eventually fail gets stuck for some
-		// reason or some MPP part times out.
-		plugin_log(pay_plugin->plugin,LOG_DBG,"waitsendpay_failed: received an old failure");
-	}
-
 	u64 errcode;
 	struct command_result *ret;
 	const jsmntok_t *datatok, *msgtok, *errchantok, *erridxtok, *failcodetok;
 	u32 onionerr, erridx;
 	struct short_channel_id errscid;
 	
-	const u8 *update;
 	const jsmntok_t *rawoniontok;
 
 	if (!json_to_u64(buf, json_get_member(buf, err, "code"), &errcode))
@@ -776,25 +506,29 @@ static struct command_result *waitsendpay_failed(struct command *cmd,
 
 	switch (errcode) {
 	case PAY_UNPARSEABLE_ONION:
-		// TODO(eduardo): when can this be triggered?
 		plugin_log(pay_plugin->plugin,LOG_DBG,"waitsendpay_failed: PAY_UNPARSEABLE_ONION");
-		ret = handle_unhandleable_error(p, flow, "unparsable onion reply");
+		ret = handle_unhandleable_error(renepay, flow, "unparsable onion reply");
 		if (ret)
 			return ret;
 		goto done;
 	case PAY_DESTINATION_PERM_FAIL:
-		plugin_log(pay_plugin->plugin,LOG_DBG,"waitsendpay_failed: PAY_DESTINATION_PERM_FAIL");
-		return command_fail(cmd, PAY_DESTINATION_PERM_FAIL,
+		plugin_log(pay_plugin->plugin,LOG_DBG,
+			"waitsendpay_failed: PAY_DESTINATION_PERM_FAIL");
+		
+		/* Inmediate failure. No information. */
+		payflow_fail(flow);
+		return renepay_fail(renepay,PAY_DESTINATION_PERM_FAIL,
 				    "Got an final failure from destination");
 	case PAY_TRY_OTHER_ROUTE:
 		plugin_log(pay_plugin->plugin,LOG_DBG,"waitsendpay_failed: PAY_TRY_OTHER_ROUTE");
 		break;
-	
 	default:
 		plugin_log(pay_plugin->plugin,LOG_DBG,
 			   "Unexpected errcode from waitsendpay: %.*s",
 			   json_tok_full_len(err), json_tok_full(buf, err));
-		return command_fail(cmd, errcode, "Unexpected errcode from waitsendpay.");
+		/* Inmediate failure. No information. */
+		payflow_fail(flow);
+		return renepay_fail(renepay,errcode, "Unexpected errcode from waitsendpay.");
 	}
 
 	datatok = json_get_member(buf, err, "data");
@@ -802,7 +536,7 @@ static struct command_result *waitsendpay_failed(struct command *cmd,
 	/* OK, we expect an onion error reply. */
 	erridxtok = json_get_member(buf, datatok, "erring_index");
 	if (!erridxtok) {
-		ret = handle_unhandleable_error(p, flow, "Missing erring_index");
+		ret = handle_unhandleable_error(renepay, flow, "Missing erring_index");
 		if (ret)
 			return ret;
 		goto done;
@@ -825,7 +559,7 @@ static struct command_result *waitsendpay_failed(struct command *cmd,
 	msgtok = json_get_member(buf, err, "message");
 	rawoniontok = json_get_member(buf, datatok, "raw_message");
 	
-	paynote(p, "onion error %s from node #%u %s: %.*s",
+	debug_paynote(p, "onion error %s from node #%u %s: %.*s",
 		onion_wire_name(onionerr),
 		erridx,
 		type_to_string(tmpctx, struct short_channel_id, &errscid),
@@ -837,21 +571,12 @@ static struct command_result *waitsendpay_failed(struct command *cmd,
 		erridx,
 		type_to_string(tmpctx, struct short_channel_id, &errscid),
 		msgtok->end - msgtok->start, buf + msgtok->start);
-
-	/* All these parts succeeded, so we know something about min
-	 * capacity! */
-	for (size_t i = 0; i < erridx; i++)
-	{
-		chan_extra_can_send(pay_plugin->chan_extra_map,
-				    flow->path_scids[i],
-				    flow->path_dirs[i],
-				    
-				    /* This channel can send all that was
-				     * commited in HTLCs.
-				     * Had we removed the commited amount then
-				     * we would have to put here flow->amounts[i]. */
-				    AMOUNT_MSAT(0));
-	}
+	
+	uncertainty_network_channel_can_send(
+			pay_plugin->chan_extra_map,
+			flow,
+			erridx);
+	
 	switch ((enum onion_wire)onionerr) {
 	/* These definitely mean eliminate channel */
 	case WIRE_PERMANENT_CHANNEL_FAILURE:
@@ -872,8 +597,8 @@ static struct command_result *waitsendpay_failed(struct command *cmd,
 	case WIRE_INVALID_ONION_BLINDING:
 	case WIRE_EXPIRY_TOO_FAR:
 		plugin_log(pay_plugin->plugin,LOG_DBG,"waitsendpay_failed: removing scid");
-		paynote(p, "... so we're removing scid");
-		tal_arr_expand(&p->active_payment->disabled, errscid);
+		debug_paynote(p, "... so we're removing scid");
+		tal_arr_expand(&renepay->disabled, errscid);
 		goto done;
 
 	/* These can be fixed (maybe) by applying the included channel_update */
@@ -883,12 +608,13 @@ static struct command_result *waitsendpay_failed(struct command *cmd,
 	case WIRE_EXPIRY_TOO_SOON:
 		plugin_log(pay_plugin->plugin,LOG_DBG,"waitsendpay_failed: apply channel_update");
 		/* FIXME: Check scid! */
-		update = channel_update_from_onion_error(tmpctx, buf, rawoniontok);
+		// TODO(eduardo): check
+		const u8 *update = channel_update_from_onion_error(tmpctx, buf, rawoniontok);
 		if (update)
 			return submit_update(cmd, flow, update, errscid);
 
-		paynote(p, "... missing an update, so we're removing scid");
-		tal_arr_expand(&p->active_payment->disabled, errscid);
+		debug_paynote(p, "... missing an update, so we're removing scid");
+		tal_arr_expand(&renepay->disabled, errscid);
 		goto done;
 
 	/* Insufficient funds! */
@@ -910,8 +636,7 @@ static struct command_result *waitsendpay_failed(struct command *cmd,
 	 * extend rexmit timer! */
 	case WIRE_MPP_TIMEOUT:
 		plugin_log(pay_plugin->plugin,LOG_DBG,"waitsendpay_failed: WIRE_MPP_TIMEOUT");
-		paynote(p, "... will continue");
-		p->status = PAYMENT_MPP_TIMEOUT;
+		debug_paynote(p, "... will continue");
 		goto done;
 
 	/* These are from the final distination: fail */
@@ -919,8 +644,10 @@ static struct command_result *waitsendpay_failed(struct command *cmd,
 	case WIRE_FINAL_INCORRECT_CLTV_EXPIRY:
 	case WIRE_FINAL_INCORRECT_HTLC_AMOUNT:
 		plugin_log(pay_plugin->plugin,LOG_DBG,"waitsendpay_failed: final destination fail");
-		paynote(p, "... fatal");
-		return command_fail(cmd, PAY_DESTINATION_PERM_FAIL,
+		debug_paynote(p, "... fatal");
+		/* Inmediate failure. No information. */
+		payflow_fail(flow);
+		return renepay_fail(renepay,PAY_DESTINATION_PERM_FAIL,
 				    "Destination said %s: %.*s",
 				    onion_wire_name(onionerr),
 				    msgtok->end - msgtok->start,
@@ -932,24 +659,26 @@ static struct command_result *waitsendpay_failed(struct command *cmd,
 				    onionerr,
 				    msgtok->end - msgtok->start,
 				    buf + msgtok->start);
-		paynote(p, "... fatal");
-		return command_fail(cmd, PAY_DESTINATION_PERM_FAIL,
+		debug_paynote(p, "... fatal");
+		payflow_fail(flow);
+		return renepay_fail(renepay,PAY_DESTINATION_PERM_FAIL,
 				    "Destination gave unknown error code %u: %.*s",
 				    onionerr,
 				    msgtok->end - msgtok->start,
 				    buf + msgtok->start);
 	}
-	paynote(p, "... eliminating and continuing");
+	debug_paynote(p, "... eliminating and continuing");
 	plugin_log(pay_plugin->plugin, LOG_BROKEN,
 		   "Node %s (%u/%zu) gave unknown error code %u",
 		   type_to_string(tmpctx, struct node_id,
 				  &flow->path_nodes[erridx]),
 		   erridx, tal_count(flow->path_nodes),
 		   onionerr);
-	tal_arr_expand(&p->active_payment->disabled, errscid);
+	tal_arr_expand(&renepay->disabled, errscid);
 
 done:
-	return flow_failed(cmd, flow);
+	payflow_fail(flow);
+	return command_still_pending(cmd);
 }
 
 /* Once we've sent it, we immediate wait for reply. */
@@ -962,13 +691,20 @@ static struct command_result *flow_sent(struct command *cmd,
 	
 	struct payment *p = flow->payment;
 	struct out_req *req;
-
-	req = jsonrpc_request_start(cmd->plugin, cmd, "waitsendpay",
+	
+	// TODO(eduardo): instead of calling this waitsendpay rpc request, we
+	// could subscribe to sendpay_success and sendpay_failure. We should
+	// investigate if the received data contains payment_hash+groupid+partid
+	// so that we can connect it to a corresponding pay_flow.
+	req = jsonrpc_request_start(cmd->plugin,
+				    /* cmd = */ NULL /* allow us to receive
+				    responses after command finishes. */, 
+				    "waitsendpay",
 				    waitsendpay_succeeded,
 				    waitsendpay_failed,
 				    cast_const(struct pay_flow *, flow));
 	json_add_sha256(req->js, "payment_hash", &p->payment_hash);
-	json_add_u64(req->js, "groupid", payment_groupid(p));
+	json_add_u64(req->js, "groupid", p->groupid);
 	json_add_u64(req->js, "partid", flow->partid);
 	/* FIXME: We don't set timeout... */
 
@@ -987,8 +723,11 @@ static struct command_result *flow_sendpay_failed(struct command *cmd,
 	plugin_log(pay_plugin->plugin,LOG_DBG,"calling %s",__PRETTY_FUNCTION__);
 	
 	struct payment *p = flow->payment;
+	struct renepay * renepay = p->renepay;
+	debug_assert(renepay);
+	
 	/* This is a fail. */
-	p->status = PAYMENT_FAIL;
+	payment_fail(p);
 	
 	u64 errcode;
 	const jsmntok_t *msg = json_get_member(buf, err, "message");
@@ -1001,25 +740,28 @@ static struct command_result *flow_sendpay_failed(struct command *cmd,
 		plugin_err(cmd->plugin, "Strange error from sendpay: %.*s",
 			   json_tok_full_len(err), json_tok_full(buf, err));
 
-	paynote(p,
+	debug_paynote(p,
 		"sendpay didn't like first hop, eliminated: %.*s",
 		msg->end - msg->start, buf + msg->start);
 		
 	/* There is no new knowledge from this kind of failure. 
 	 * We just disable this scid. */
-	tal_arr_expand(&p->active_payment->disabled, flow->path_scids[0]);
+	tal_arr_expand(&renepay->disabled, flow->path_scids[0]);
 	
-	return flow_failed(cmd, flow);
+	payflow_fail(flow);
+	return command_still_pending(cmd);
 }
 
 
 static struct command_result *
 sendpay_flows(struct command *cmd,
-	      struct payment *p,
+	      struct renepay * renepay,
 	      struct pay_flow **flows STEALS)
 {
+	struct payment * const p = renepay->payment;
+	
 	plugin_log(pay_plugin->plugin,LOG_DBG,"calling %s",__PRETTY_FUNCTION__);
-	paynote(p, "Sending out batch of %zu payments", tal_count(flows));
+	debug_paynote(p, "Sending out batch of %zu payments", tal_count(flows));
 	
 	for (size_t i = 0; i < tal_count(flows); i++) {
 		struct out_req *req;
@@ -1052,7 +794,7 @@ sendpay_flows(struct command *cmd,
 			
 		json_add_u64(req->js, "partid", flows[i]->partid);
 		
-		json_add_u64(req->js, "groupid", payment_groupid(p));
+		json_add_u64(req->js, "groupid", p->groupid);
 		if (p->payment_metadata)
 			json_add_hex_talarr(req->js, "payment_metadata",
 					    p->payment_metadata);
@@ -1066,48 +808,52 @@ sendpay_flows(struct command *cmd,
 		
 		amount_msat_accumulate(&p->total_sent, flows[i]->amounts[0]);
 		amount_msat_accumulate(&p->total_delivering,
-				       flow_delivered(flows[i]));
+				       payflow_delivered(flows[i]));
 		
 		/* Flow now owned by all_flows instead of req., in this way we
 		 * can control the destruction occurs before we remove temporary
 		 * channels from chan_extra_map. */
-		tal_steal(p->active_payment->all_flows,flows[i]);
+		tal_steal(pay_plugin->ctx,flows[i]);
 		
 		/* record these HTLC along the flow path */
 		commit_htlc_payflow(pay_plugin->chan_extra_map,flows[i]);
 		
 		/* Remove the HTLC from the chan_extra_map after finish. */
+		// TODO(eduardo): I think flows should be contained in a data
+		// structure, so this constructor should also remove them from
+		// that data structure.
 		tal_add_destructor2(flows[i],
-				    remove_htlc_payflow_and_update_knowlege,
+				    remove_htlc_payflow,
 				    pay_plugin->chan_extra_map);
 		
 		send_outreq(cmd->plugin, req);
 	}
 	
 	/* Safety check. */
-	payment_check_delivering_all(p);
+	payment_assert_delivering_all(p);
 	
 	tal_free(flows);
 	
 	/* Get ready to process replies */
-	payment_settimer(p);
+	renepay_settimer(renepay);
 	
 	return command_still_pending(cmd);
 }
 
 static struct command_result *try_paying(struct command *cmd,
-					 struct payment *p,
+					 struct renepay *renepay,
 					 bool first_time)
 {
+	struct payment * const p = renepay->payment;
 	plugin_log(pay_plugin->plugin,LOG_DBG,"calling %s",__PRETTY_FUNCTION__);
 	
 	// TODO(eduardo): does it make sense to have this limit on attempts?
 	/* I am classifying the flows in attempt cycles. */
-	payment_new_attempt(p);
+	renepay_new_attempt(renepay);
 	/* We try only MAX_NUM_ATTEMPTS, then we give up. */
-	if ( payment_attempt_count(p) > MAX_NUM_ATTEMPTS)
+	if ( renepay_attempt_count(renepay) > MAX_NUM_ATTEMPTS)
 	{
-		return command_fail(cmd, PAY_STOPPED_RETRYING,
+		return renepay_fail(renepay, PAY_STOPPED_RETRYING,
 				    "Reached maximum number of attempts (%d)",
 				    MAX_NUM_ATTEMPTS);
 	}
@@ -1115,7 +861,7 @@ static struct command_result *try_paying(struct command *cmd,
 	struct amount_msat feebudget, fees_spent, remaining;
 
 	if (time_after(time_now(), p->stop_time))
-		return command_fail(cmd, PAY_STOPPED_RETRYING, "Timed out");
+		return renepay_fail(renepay, PAY_STOPPED_RETRYING, "Timed out");
 	
 	/* Total feebudget  */
 	if (!amount_msat_sub(&feebudget, p->maxspend, p->amount))
@@ -1167,9 +913,18 @@ static struct command_result *try_paying(struct command *cmd,
 	
 	/* We let this return an unlikely path, as it's better to try once
 	 * than simply refuse.  Plus, models are not truth! */
-	struct pay_flow **pay_flows = get_payflows(p, remaining, feebudget, first_time,
-				 		   amount_msat_eq(p->total_delivering, AMOUNT_MSAT(0)),
-						   &err_msg);
+	struct pay_flow **pay_flows = get_payflows(
+						renepay, 
+						remaining, feebudget, 
+						
+						/* would you accept unlikely
+						 * payments? */
+						first_time,
+						
+				 		/* is entire payment? */
+						amount_msat_eq(p->total_delivering, AMOUNT_MSAT(0)),
+						
+						&err_msg);
 	
 	plugin_log(pay_plugin->plugin,LOG_DBG,fmt_payflows(tmpctx,pay_flows));
 	
@@ -1177,22 +932,29 @@ static struct command_result *try_paying(struct command *cmd,
 	// TODO(eduardo): alternatively we can fallback to `pay`.
 	if (!pay_flows)
 	{
-		return command_fail(cmd, PAY_ROUTE_NOT_FOUND,
+		return renepay_fail(renepay, PAY_ROUTE_NOT_FOUND,
 				    "Failed to find a route, %s",
 				    err_msg);
 	}
 	/* Now begin making payments */
 	
-	return sendpay_flows(cmd, p, pay_flows);
+	return sendpay_flows(cmd, renepay, pay_flows);
 }
 
-static struct command_result *
-listpeerchannels_done(struct command *cmd, const char *buf,
-	       const jsmntok_t *result, struct payment *p)
+static struct command_result *listpeerchannels_done(
+		struct command *cmd, 
+		const char *buf,
+		const jsmntok_t *result, 
+		struct renepay *renepay)
 {
 	plugin_log(pay_plugin->plugin,LOG_DBG,"calling %s",__PRETTY_FUNCTION__);
-	if (!update_uncertainty_network_from_listpeerchannels(cmd->plugin, p, buf, result))
-		return command_fail(cmd, LIGHTNINGD,
+	if (!uncertainty_network_update_from_listpeerchannels(
+			pay_plugin->chan_extra_map,
+			pay_plugin->my_id,
+			renepay, 
+			buf, 
+			result))
+		return renepay_fail(renepay,LIGHTNINGD,
 				    "listpeerchannels malformed: %.*s",
 				    json_tok_full_len(result),
 				    json_tok_full(buf, result));
@@ -1201,35 +963,11 @@ listpeerchannels_done(struct command *cmd, const char *buf,
 	// TODO(eduardo): check that there won't be a prob. cost associated with
 	// any gossmap local chan. The same way there aren't fees to pay for my
 	// local channels.
-	gossmap_apply_localmods(pay_plugin->gossmap,p->active_payment->local_gossmods);
-	p->active_payment->localmods_applied=true;
-	return try_paying(cmd, p, true);
+	gossmap_apply_localmods(pay_plugin->gossmap,renepay->local_gossmods);
+	renepay->localmods_applied=true;
+	return try_paying(cmd, renepay, true);
 }
 
-/* Either the payment succeeded or failed, we need to cleanup/set the plugin
- * into a valid state before the next payment. */
-static void renepay_cleanup(struct active_payment *ap)
-{
-	plugin_log(pay_plugin->plugin,LOG_DBG,"calling %s",__PRETTY_FUNCTION__);
-
-	/* Free all pending flows, release their HTLCs and update knowledge. */
-	ap->all_flows = tal_free(ap->all_flows);
-	
-	/* Always remove our local mods (routehints) so others can use
-	 * gossmap. We do this only after the payment completes. */
-	if(ap->localmods_applied)
-		gossmap_remove_localmods(pay_plugin->gossmap,
-					 ap->local_gossmods);
-	ap->localmods_applied=false;
-	tal_free(ap->local_gossmods);
-	
-	// plugin_log(pay_plugin->plugin,LOG_DBG,fmt_chan_extra_map(tmpctx,pay_plugin->chan_extra_map));
-	
-	ap->rexmit_timer = tal_free(ap->rexmit_timer);
-	
-	// TODO(eduardo): what about private channels in `chan_extra_map`?
-	// Should they be removed?
-}
 
 static void destroy_payment(struct payment *p)
 {
@@ -1261,10 +999,35 @@ static struct command_result *json_paystatus(struct command *cmd,
 			json_add_string(ret, "label", p->label);
 
 		if (p->invstr)
-			json_add_invstring(ret, p->invstr);
+			json_add_invstring(ret,p->invstr);
+		
 		json_add_amount_msat_only(ret, "amount_msat", p->amount);
-
+		json_add_sha256(ret, "payment_hash", &p->payment_hash);
 		json_add_node_id(ret, "destination", &p->destination);
+		
+		if (p->description)
+			json_add_string(ret, "description", p->description);
+		
+		json_add_timeabs(ret,"created_at",p->start_time);
+		json_add_u64(ret,"groupid",p->groupid);
+		
+		switch(p->status)
+		{
+			case PAYMENT_SUCCESS:
+				json_add_string(ret,"status","complete");
+				debug_assert(p->preimage);
+				json_add_preimage(ret,"payment_preimage",p->preimage);
+				json_add_amount_msat_only(ret, "amount_sent_msat", p->total_sent);
+			
+			break;
+			case PAYMENT_FAIL:
+				json_add_string(ret,"status","failed");
+			
+			break;
+			default:
+				json_add_string(ret,"status","pending");
+		}
+		
 		json_array_start(ret, "notes");
 		for (size_t i = 0; i < tal_count(p->paynotes); i++)
 			json_add_string(ret, NULL, p->paynotes[i]);
@@ -1272,10 +1035,9 @@ static struct command_result *json_paystatus(struct command *cmd,
 		json_object_end(ret);
 		
 		// TODO(eduardo): maybe we should add also:
-		// - preimage (as proof of payment)
 		// - payment_secret?
-		// - payment_metadata
-		// - status
+		// - payment_metadata?
+		// - number of parts?
 	}
 	json_array_end(ret);
 
@@ -1290,31 +1052,38 @@ static struct command_result *json_paystatus(struct command *cmd,
  * about an eventual previous complete payment so we can return that
  * as a no-op. */
 static struct command_result *
-payment_listsendpays_previous(struct command *cmd, const char *buf,
-			      const jsmntok_t *result, struct payment *p)
+payment_listsendpays_previous(
+		struct command *cmd, 
+		const char *buf,
+		const jsmntok_t *result, 
+		struct renepay * renepay)
 {
+	debug_info("calling %s",__PRETTY_FUNCTION__);
+	struct payment * p = renepay->payment;
+	
 	size_t i;
 	const jsmntok_t *t, *arr, *err;
-	/* What was the groupid of an eventual previous attempt? */
-	u64 last_group = 0;
+	
 	/* Do we have pending sendpays for the previous attempt? */
 	bool pending = false;
-
 	/* Group ID of the first pending payment, this will be the one
 	 * who's result gets replayed if we end up suspending. */
 	u64 first_pending_group_id = INVALID_ID;
 	u64 last_pending_group_id = INVALID_ID;
-	u64 last_partid=0;
+	u64 last_pending_partid=0;
+	struct amount_msat pending_sent = AMOUNT_MSAT(0),
+			   pending_msat = AMOUNT_MSAT(0);
+	
 	/* Did a prior attempt succeed? */
 	bool completed = false;
-
 	/* Metadata for a complete payment, if one exists. */
-	struct json_stream *ret;
-	u32 parts = 0;
-	struct preimage preimage;
-	struct amount_msat sent, msat;
-	struct node_id destination;
-	u32 created_at;
+	u32 complete_parts = 0;
+	struct preimage complete_preimage;
+	struct amount_msat complete_sent = AMOUNT_MSAT(0), 
+			   complete_msat = AMOUNT_MSAT(0);
+	u32 complete_created_at;
+
+	u64 last_group=INVALID_ID;
 
 	err = json_get_member(buf, result, "error");
 	if (err)
@@ -1333,13 +1102,12 @@ payment_listsendpays_previous(struct command *cmd, const char *buf,
 	 * latest group and remembering what its state is. */
 	json_for_each_arr(i, t, arr)
 	{
-		bool this_pending = false;
 		u64 partid, groupid;
-		struct amount_msat this_delivering, this_sent;
+		struct amount_msat this_msat, this_sent;
 		
 		const jsmntok_t *status;
-		struct amount_msat diff_sent, diff_msat;
 		
+		// TODO(eduardo): assuming amount_msat is always known.
 		json_scan(tmpctx,buf,t,
 			  "{partid:%"
 			  ",groupid:%"
@@ -1347,47 +1115,39 @@ payment_listsendpays_previous(struct command *cmd, const char *buf,
 			  ",amount_sent_msat:%}",
 			  JSON_SCAN(json_to_u64,&partid),
 			  JSON_SCAN(json_to_u64,&groupid),
-			  JSON_SCAN(json_to_msat,&this_delivering),
+			  JSON_SCAN(json_to_msat,&this_msat),
 			  JSON_SCAN(json_to_msat,&this_sent));
 		
-		/* New group, reset what we collected. */
-		if (last_group != groupid) {
-			completed = false;
-			pending = false;
-			last_group = groupid;
-
-			parts = 1;
+		/* status could be completed, pending or failed */
+		
+		
+		status = json_get_member(buf, t, "status");
+		
+		if(json_tok_streq(buf,status,"failed"))
+			continue;
+		
+		if(json_tok_streq(buf,status,"complete"))
+		{
+			/* Now we know the payment completed. */
+			completed = true;
+			if(!amount_msat_add(&complete_msat,complete_msat,this_msat))
+				debug_err("%s (line %d) msat overflow.",
+					__PRETTY_FUNCTION__,__LINE__);
+			if(!amount_msat_add(&complete_sent,complete_sent,this_sent))
+				debug_err("%s (line %d) msat overflow.",
+					__PRETTY_FUNCTION__,__LINE__);
 			json_scan(tmpctx, buf, t,
-				  "{destination:%"
-				  ",created_at:%"
-				  ",amount_msat:%"
-				  ",amount_sent_msat:%"
+				  "{created_at:%"
 				  ",payment_preimage:%}",
-				  JSON_SCAN(json_to_node_id, &destination),
-				  JSON_SCAN(json_to_u32, &created_at),
-				  JSON_SCAN(json_to_msat, &msat),
-				  JSON_SCAN(json_to_msat, &sent),
-				  JSON_SCAN(json_to_preimage, &preimage));
-		} else {
-			json_scan(tmpctx, buf, t,
-				  "{amount_msat:%"
-				  ",amount_sent_msat:%}",
-				  JSON_SCAN(json_to_msat, &diff_msat),
-				  JSON_SCAN(json_to_msat, &diff_sent));
-			if (!amount_msat_add(&msat, msat, diff_msat) ||
-			    !amount_msat_add(&sent, sent, diff_sent))
-				debug_err("msat overflow adding up parts");
-			parts++;
+				  JSON_SCAN(json_to_u32, &complete_created_at),
+				  JSON_SCAN(json_to_preimage, &complete_preimage));
+			complete_parts ++;
 		}
-
-		status = json_get_member(buf, t, "status");
-		completed |= json_tok_streq(buf, status, "complete");
-		pending |= json_tok_streq(buf, status, "pending");
 		
-		this_pending |= json_tok_streq(buf, status, "pending");
-
-		if(this_pending)
+		if(json_tok_streq(buf,status,"pending"))
 		{
+			pending = true; // there are parts pending
+			
 			if(first_pending_group_id==INVALID_ID || 
 			   last_pending_group_id==INVALID_ID)
 				first_pending_group_id = last_pending_group_id = groupid;
@@ -1395,9 +1155,9 @@ payment_listsendpays_previous(struct command *cmd, const char *buf,
 			if(groupid > last_pending_group_id)
 			{
 				last_pending_group_id = groupid;
-				p->total_sent = AMOUNT_MSAT(0);
-				p->total_delivering = AMOUNT_MSAT(0);
-				last_partid = partid;
+				last_pending_partid = partid;
+				pending_msat = AMOUNT_MSAT(0);
+				pending_sent = AMOUNT_MSAT(0);
 			}
 			if(groupid < first_pending_group_id)
 			{
@@ -1405,76 +1165,80 @@ payment_listsendpays_previous(struct command *cmd, const char *buf,
 			}
 			if(groupid == last_pending_group_id)
 			{
-				amount_msat_accumulate(&p->total_sent, 
-						       this_sent);
-				amount_msat_accumulate(&p->total_delivering,
-						       this_delivering);
+				amount_msat_accumulate(&pending_sent,this_sent);
+				amount_msat_accumulate(&pending_msat,this_msat);
+				
+				last_pending_partid = MAX(last_pending_partid,partid);
 				plugin_log(pay_plugin->plugin,LOG_DBG,
 					"pending deliver increased by %s",
-					type_to_string(tmpctx,struct amount_msat,&this_delivering));
+					type_to_string(tmpctx,struct amount_msat,&this_msat));
 			}
-		}
 		
-		/* Let's get the very last id in the last pending group. */
-		if(groupid == last_pending_group_id)		
-			last_partid = MAX(last_partid,partid);
-				
+		}
 	}
 	
 	if (completed) {
-		ret = jsonrpc_stream_success(cmd);
-		json_add_preimage(ret, "payment_preimage", &preimage);
+		struct json_stream *ret = jsonrpc_stream_success(cmd);
+		json_add_preimage(ret, "payment_preimage", &complete_preimage);
 		json_add_string(ret, "status", "complete");
-		json_add_amount_msat_compat(ret, msat, "msatoshi",
+		json_add_amount_msat_compat(ret, complete_msat, "msatoshi",
 					    "amount_msat");
-		json_add_amount_msat_compat(ret, sent, "msatoshi_sent",
+		json_add_amount_msat_compat(ret, complete_sent, "msatoshi_sent",
 					    "amount_sent_msat");
 		json_add_node_id(ret, "destination", &p->destination);
 		json_add_sha256(ret, "payment_hash", &p->payment_hash);
-		json_add_u32(ret, "created_at", created_at);
-		json_add_num(ret, "parts", parts);
+		json_add_u32(ret, "created_at", complete_created_at);
+		json_add_num(ret, "parts", complete_parts);
 		
 		/* This payment was already completed, we don't keep record of
 		 * it twice. */
-		tal_free(p);
+		renepay->payment = tal_free(renepay->payment);
 		
 		return command_finished(cmd, ret);
 	} else if (pending) {
 		p->groupid = last_pending_group_id;
-		p->active_payment->next_partid = last_partid+1;	
+		renepay->next_partid = last_pending_partid+1;	
 		
+		p->total_sent = pending_sent;
+		p->total_delivering = pending_msat;
+	
 		plugin_log(pay_plugin->plugin,LOG_DBG,
 			   "There are pending sendpays to this invoice. "
-			   "groupids = %ld or %ld, "
+			   "groupid = %ld, "
 			   "delivering = %s, "
 			   "last_partid = %ld",
-			   first_pending_group_id,
 			   last_pending_group_id,
 			   type_to_string(tmpctx,struct amount_msat,&p->total_delivering),
-			   last_partid);
+			   last_pending_partid);
 		
-		if( 
-		/* TODO(eduardo): If two group_id are trying to complete the
-		 * payment, could this lead to overpayment? */
-		first_pending_group_id != last_pending_group_id
-			|| 
-		amount_msat_greater_eq(p->total_delivering,p->amount))
+		if( first_pending_group_id != last_pending_group_id)
 		{
-			tal_free(p);
-			return command_fail(cmd, PAY_IN_PROGRESS,
+			/* At least two pending groups for the same invoice,
+			 * this is weird, we better stop. */
+			renepay->payment = tal_free(renepay->payment);
+			return renepay_fail(renepay, PAY_IN_PROGRESS,
 					    "Payment is pending by some other request.");
 		}
-		
-		return try_paying(cmd,p,true);
+		if(amount_msat_greater_eq(p->total_delivering,p->amount))
+		{
+			/* Pending payment already pays the full amount, we
+			 * better stop. */
+			renepay->payment = tal_free(renepay->payment);
+			return renepay_fail(renepay, PAY_IN_PROGRESS,
+					    "Payment is pending with full amount already commited");
+		}
+	}else
+	{
+		p->groupid = (last_group==INVALID_ID  ? 1 : (last_group+1)) ;
+		renepay->next_partid=1;
 	}
 	
-	p->groupid = last_group + 1;
 	
 	struct out_req *req;
 	/* Get local capacities... */
 	req = jsonrpc_request_start(cmd->plugin, cmd, "listpeerchannels",
 				    listpeerchannels_done,
-				    listpeerchannels_done, p);
+				    listpeerchannels_done, renepay);
 	return send_outreq(cmd->plugin, req);
 }
 
@@ -1482,77 +1246,85 @@ static struct command_result *json_pay(struct command *cmd,
 				       const char *buf,
 				       const jsmntok_t *params)
 {
- 	struct payment *p;
-	
+	const char *invstr;
+	const char *label;
+	const char *description;
+	struct sha256 * local_offer_id;
  	u64 invexpiry;
  	struct amount_msat *msat, *invmsat;
 	struct amount_msat *maxfee;
 	u64 *riskfactor_millionths;
- 	u32 *maxdelay;
-	u64 *base_fee_penalty, *prob_cost_factor;
+	u32 *maxdelay;
+	u64 *base_fee_penalty;
+	u64 *prob_cost_factor;
 	u64 *min_prob_success_millionths;
-	unsigned int *retryfor;
+	u32 *retryfor;
 
 #if DEVELOPER
 	bool *use_shadow;
 #endif
-
- 	p = tal(cmd, struct payment);
-	
-	p->paynotes = tal_arr(p, const char *, 0);
-	p->total_sent = AMOUNT_MSAT(0);
-	p->total_delivering = AMOUNT_MSAT(0);
 	
 	if (!param(cmd, buf, params,
-		   p_req("invstring", param_string, &p->invstr),
+		   p_req("invstring", param_string, &invstr),
  		   p_opt("amount_msat", param_msat, &msat),
  		   p_opt("maxfee", param_msat, &maxfee),
- 		   
+		   
 		   // MCF parameters
+		   // TODO(eduardo): are these parameters read correctly?
 		   p_opt_def("base_fee_penalty", param_millionths, &base_fee_penalty,10),
  		   p_opt_def("prob_cost_factor", param_millionths, &prob_cost_factor,10),
- 		   
-		   // TODO(eduardo): probability of success as a ppm parameter
-		   // or a real number?
 		   p_opt_def("min_prob_success", param_millionths, 
-		   	&min_prob_success_millionths,100000), // default is 10%
-	
-		   p_opt_def("riskfactor", param_millionths,
-			     &riskfactor_millionths, 1),
-
+		   	&min_prob_success_millionths,100000),// default is 10%
+		   
+		   p_opt_def("riskfactor", param_millionths,&riskfactor_millionths,1),
+		   
 		   p_opt_def("maxdelay", param_number, &maxdelay,
 			     /* We're initially called to probe usage, before init! */
 			     pay_plugin ? pay_plugin->maxdelay_default : 0),
-
- 		   p_opt_def("retry_for", param_number, &retryfor, 60),
- 		   p_opt("localofferid", param_sha256, &p->local_offer_id),
- 		   p_opt("description", param_string, &p->description),
- 		   p_opt("label", param_string, &p->label),
+ 		   
+		   
+ 		   p_opt_def("retry_for", param_number, &retryfor, 60), // 60 seconds
+ 		   p_opt("localofferid", param_sha256, &local_offer_id),
+ 		   p_opt("description", param_string, &description),
+ 		   p_opt("label", param_string, &label),
 #if DEVELOPER
 		   p_opt_def("use_shadow", param_bool, &use_shadow, true),
 #endif
 		   NULL))
 		return command_param_failed();
 	
-	tal_steal(pay_plugin->ctx,p);
-	tal_add_destructor(p, destroy_payment);
+	/* renepay is bound to the command, if the command finishes renepay is
+	 * freed. */
+	struct renepay * renepay = renepay_new(cmd);
+	tal_add_destructor2(renepay,
+			    renepay_cleanup,
+			    pay_plugin->gossmap);
+ 	struct payment * p = renepay->payment;
+
+	p->invstr = tal_steal(p,invstr);
+	p->description = tal_steal(p,description);
+	p->label = tal_steal(p,label);
+	p->local_offer_id = tal_steal(p,local_offer_id);
+
+	p->base_fee_penalty = *base_fee_penalty; 
+	base_fee_penalty = tal_free(base_fee_penalty);
 	
-	payment_initialize(p,cmd);
+	p->prob_cost_factor = *prob_cost_factor;
+	prob_cost_factor = tal_free(prob_cost_factor);
 	
-
-#if DEVELOPER
-	p->use_shadow = *use_shadow;
-	tal_free(use_shadow);
-#else
-	p->use_shadow = true;
-#endif
- 	
-	// TODO(eduardo): 
-	// - get time since last payment, 
-	// - forget a portion of the bounds
-	// - note that if sufficient time has passed, then we would forget
-	// everything.
+	p->min_prob_success = *min_prob_success_millionths/1e6;
+	min_prob_success_millionths = tal_free(min_prob_success_millionths);
+	
+	p->delay_feefactor = *riskfactor_millionths/1e6;
+	riskfactor_millionths = tal_free(riskfactor_millionths);
+	
+	p->maxdelay = *maxdelay;
+	maxdelay = tal_free(maxdelay);
 	
+	/* We inmediately add this payment to the payment list. */
+	tal_steal(pay_plugin->ctx,p);
+	list_add_tail(&pay_plugin->payments, &p->list);
+	tal_add_destructor(p, destroy_payment);
 	
 	plugin_log(pay_plugin->plugin,LOG_DBG,"Starting renepay");
 	bool gossmap_changed = gossmap_refresh(pay_plugin->gossmap, NULL);
@@ -1561,21 +1333,8 @@ static struct command_result *json_pay(struct command *cmd,
 		plugin_err(pay_plugin->plugin, "Failed to refresh gossmap: %s",
 			   strerror(errno));
 	
-	p->base_fee_penalty=*base_fee_penalty;
-	p->prob_cost_factor= *prob_cost_factor;
-	p->min_prob_success = *min_prob_success_millionths / 1000000.0;
- 	p->delay_feefactor = *riskfactor_millionths / 1000000.0;
-	p->maxdelay = *maxdelay;
-	
-	tal_free(base_fee_penalty);
-	tal_free(prob_cost_factor);
-	tal_free(min_prob_success_millionths);
-	tal_free(riskfactor_millionths);
-	tal_free(maxdelay);
-
 	p->start_time = time_now();
 	p->stop_time = timeabs_add(p->start_time, time_from_sec(*retryfor));
-	p->preimage=NULL;
 	tal_free(retryfor);
 	
 	bool invstr_is_b11=false;
@@ -1587,7 +1346,7 @@ static struct command_result *json_pay(struct command *cmd,
 		    bolt11_decode(tmpctx, p->invstr, plugin_feature_set(cmd->plugin),
 				  p->description, chainparams, &fail);
 		if (b11 == NULL)
-			return command_fail(cmd, JSONRPC2_INVALID_PARAMS,
+			return renepay_fail(renepay, JSONRPC2_INVALID_PARAMS,
 					    "Invalid bolt11: %s", fail);
 		invstr_is_b11=true;
 		
@@ -1620,12 +1379,12 @@ static struct command_result *json_pay(struct command *cmd,
 		 */
 		if (!b11->description) {
 			if (!b11->description_hash) {
-				return command_fail(cmd,
+				return renepay_fail(renepay,
 						    JSONRPC2_INVALID_PARAMS,
 						    "Invalid bolt11: missing description");
 			}
 			if (!p->description)
-				return command_fail(cmd,
+				return renepay_fail(renepay,
 						    JSONRPC2_INVALID_PARAMS,
 						    "bolt11 uses description_hash, but you did not provide description parameter");
 		}
@@ -1637,20 +1396,20 @@ static struct command_result *json_pay(struct command *cmd,
 				     plugin_feature_set(cmd->plugin),
 				     chainparams, &fail);
 		if (b12 == NULL)
-			return command_fail(cmd, JSONRPC2_INVALID_PARAMS,
+			return renepay_fail(renepay, JSONRPC2_INVALID_PARAMS,
 					    "Invalid bolt12: %s", fail);
 		if (!pay_plugin->exp_offers)
-			return command_fail(cmd, JSONRPC2_INVALID_PARAMS,
+			return renepay_fail(renepay, JSONRPC2_INVALID_PARAMS,
 					    "experimental-offers disabled");
 
 		if (!b12->offer_node_id)
-			return command_fail(cmd, JSONRPC2_INVALID_PARAMS,
+			return renepay_fail(renepay, JSONRPC2_INVALID_PARAMS,
 					    "invoice missing offer_node_id");
 		if (!b12->invoice_payment_hash)
-			return command_fail(cmd, JSONRPC2_INVALID_PARAMS,
+			return renepay_fail(renepay, JSONRPC2_INVALID_PARAMS,
 					    "invoice missing payment_hash");
 		if (!b12->invoice_created_at)
-			return command_fail(cmd, JSONRPC2_INVALID_PARAMS,
+			return renepay_fail(renepay, JSONRPC2_INVALID_PARAMS,
 					    "invoice missing created_at");
 		if (b12->invoice_amount) {
 			invmsat = tal(cmd, struct amount_msat);
@@ -1661,8 +1420,8 @@ static struct command_result *json_pay(struct command *cmd,
 		node_id_from_pubkey(&p->destination, b12->offer_node_id);
 		p->payment_hash = *b12->invoice_payment_hash;
 		if (b12->invreq_recurrence_counter && !p->label)
-			return command_fail(
-			    cmd, JSONRPC2_INVALID_PARAMS,
+			return renepay_fail(
+			    renepay, JSONRPC2_INVALID_PARAMS,
 			    "recurring invoice requires a label");
 		/* FIXME payment_secret should be signature! */
 		{
@@ -1694,17 +1453,16 @@ static struct command_result *json_pay(struct command *cmd,
 	}
 	
 	if (node_id_eq(&pay_plugin->my_id, &p->destination))
-		return command_fail(cmd, JSONRPC2_INVALID_PARAMS,
+		return renepay_fail(renepay, JSONRPC2_INVALID_PARAMS,
 				    "This payment is destined for ourselves. "
 				    "Self-payments are not supported");
 
-	list_add_tail(&pay_plugin->payments, &p->list);
 	
 	// set the payment amount
 	if (invmsat) {
 		// amount is written in the invoice
 		if (msat) {
-			return command_fail(cmd, JSONRPC2_INVALID_PARAMS,
+			return renepay_fail(renepay, JSONRPC2_INVALID_PARAMS,
 					    "amount_msat parameter unnecessary");
 		}
 		p->amount = *invmsat;
@@ -1712,7 +1470,7 @@ static struct command_result *json_pay(struct command *cmd,
 	} else {
 		// amount is not written in the invoice
 		if (!msat) {
-			return command_fail(cmd, JSONRPC2_INVALID_PARAMS,
+			return renepay_fail(renepay, JSONRPC2_INVALID_PARAMS,
 					    "amount_msat parameter required");
 		}
 		p->amount = *msat;
@@ -1728,14 +1486,14 @@ static struct command_result *json_pay(struct command *cmd,
 	}
 
 	if (!amount_msat_add(&p->maxspend, p->amount, *maxfee)) {
-		return command_fail(
-			cmd, JSONRPC2_INVALID_PARAMS,
+		return renepay_fail(
+			renepay, JSONRPC2_INVALID_PARAMS,
 			"Overflow when computing fee budget, fee far too high.");
 	}
 	tal_free(maxfee);
 	
 	if (time_now().ts.tv_sec > invexpiry)
-		return command_fail(cmd, PAY_INVOICE_EXPIRED, "Invoice expired");
+		return renepay_fail(renepay, PAY_INVOICE_EXPIRED, "Invoice expired");
 	
 	
 	/* To construct the uncertainty network we need to perform the following
@@ -1759,26 +1517,28 @@ static struct command_result *json_pay(struct command *cmd,
 	// TODO(eduardo): are there route hints for B12?
 	// Add any extra hidden channel revealed by the routehints to the uncertainty network.
 	if(invstr_is_b11)
-		uncertainty_network_add_routehints(p);
+		uncertainty_network_add_routehints(pay_plugin->chan_extra_map,renepay);
 	
 	if(!uncertainty_network_check_invariants(pay_plugin->chan_extra_map))
 		plugin_log(pay_plugin->plugin, 
 			   LOG_BROKEN,
 			   "uncertainty network invariants are violated");
 	
-	/* We will try a single MPP payment. */
-	// p->active_payment->groupid = pseudorand_u64();
-	p->active_payment->next_partid = 1;
-	
 	/* Next, request listsendpays for previous payments that use the same
 	 * hash. */
 	struct out_req *req
 		= jsonrpc_request_start(cmd->plugin, cmd, "listsendpays",
 			payment_listsendpays_previous,
-			payment_listsendpays_previous, p);
+			payment_listsendpays_previous, renepay);
 
 	json_add_sha256(req->js, "payment_hash", &p->payment_hash);
 	return send_outreq(cmd->plugin, req);
+	
+	// TODO(eduardo): 
+	// - get time since last payment, 
+	// - forget a portion of the bounds
+	// - note that if sufficient time has passed, then we would forget
+	// everything use TIMER_FORGET_SEC.
 }
 
 static struct command_result *json_shutdown(struct command *cmd,
diff --git a/plugins/renepay/pay.h b/plugins/renepay/pay.h
index 576019953d94..1a2b79fa5a29 100644
--- a/plugins/renepay/pay.h
+++ b/plugins/renepay/pay.h
@@ -5,6 +5,8 @@
 #include <common/node_id.h>
 #include <plugins/libplugin.h>
 #include <plugins/renepay/flow.h>
+#include <plugins/renepay/payment.h>
+#include <plugins/renepay/uncertainty_network.h>
 
 // TODO(eduardo): renepaystatus should be similar to paystatus
 
@@ -13,8 +15,6 @@
 // TODO(eduardo): MCF should consider pending HTLCs occupy some capacity in the
 // routing channels.
 
-// TODO(eduardo): renepaystatus sometimes shows a truncated invoice string
-// 
 // TODO(eduardo): for some reason cln-renepay does not terminate inmediately:
 // 2023-06-26T07:32:36.064Z DEBUG   lightningd: cln-renepay: failed to self-terminate in time, killing.
 // 
@@ -27,6 +27,13 @@
 
 #define MAX_NUM_ATTEMPTS 10
 
+/* Time lapse used to wait for failed sendpays before try_paying. */
+#define TIMER_COLLECT_FAILURES_MSEC 250
+
+/* Knowledge is proportionally decreased with time up to TIMER_FORGET_SEC when
+ * we forget everything. */
+#define TIMER_FORGET_SEC 3600 
+
 // TODO(eduardo): Test ideas
 // - make a payment to a node that is hidden behind private channels, check that
 // private channels are removed from the gossmap and chan_extra_map
@@ -45,11 +52,6 @@
 // 	- destination is offline
 // 	- with current knowledge there is no flow solution to destination
 
-enum payment_status {
-        PAYMENT_PENDING, PAYMENT_SUCCESS, PAYMENT_FAIL,
-	PAYMENT_MPP_TIMEOUT
-};
-
 /* Our convenient global data, here in one place. */
 struct pay_plugin {
 	/* From libplugin */
@@ -76,134 +78,31 @@ struct pay_plugin {
 	bool debug_mcf; 
 	bool debug_payflow; 
 	
+	/* I'll allocate all global (controlled by pay_plugin) variables tied to
+	 * this tal_t. */
 	tal_t *ctx;
-};
-
-/* Set in init */
-extern struct pay_plugin * const pay_plugin;
-
-/* Data only kept while the payment is being processed. */
-struct active_payment
-{
-	/* The command, and our owner (needed for timer func) */
-	struct command *cmd;
-
-	/* Localmods to apply to gossip_map for our own use. */
-	bool localmods_applied;
-	struct gossmap_localmods *local_gossmods;
 	
-	/* Channels we decided to disable for various reasons. */
-	struct short_channel_id *disabled;
-
+	// TODO(eduardo): pending flows have HTLCs (in-flight) liquidity
+	// attached that is reflected in the uncertainty network. When
+	// waitsendpay returns either fail or success that flow is destroyed and
+	// the liquidity is restored. A payment command could end before all
+	// flows are destroyed, therefore it is important to delegate the
+	// ownership of the waitsendpay request to pay_plugin->ctx so that the
+	// request is kept alive. One more thing: to double check that flows are
+	// not accumulating ad-infinitum I would insert them into a data
+	// structure here so that once in a while a timer kicks and verifies the
+	// list of pending flows.
+	// TODO(eduardo): notice that pending attempts performed with another
+	// pay plugin are not considered by the uncertainty network in renepay,
+	// it would be nice if listsendpay would give us the route of pending
+	// sendpays.
+	
 	/* Timers. */
 	struct plugin_timer *rexmit_timer;
-	
-	/* Keep track of the number of attempts. */
-	int last_attempt;
-	
-	/* Root to destroy pending flows */
-	tal_t *all_flows;
-	
-	/* Used in get_payflows to set ids to each pay_flow. */
-	u64 next_partid;
 };
 
-struct payment {
-	/* Chatty description of attempts. */
-	const char **paynotes;
-	
-	/* Total sent, including fees. */
-	struct amount_msat total_sent;
-	
-	/* Total that is delivering (i.e. without fees) */
-	struct amount_msat total_delivering;
-	
-	/* invstring (bolt11 or bolt12) */
-	const char *invstr;
-	
-	/* How much, what, where */
-	struct amount_msat amount;
-	struct node_id destination;
-	struct sha256 payment_hash;
-	
-	
-	/* Limits on what routes we'll accept. */
-	struct amount_msat maxspend;
-	
-	/* Max accepted HTLC delay.*/
-	unsigned int maxdelay;
-	
-	/* We promised this in pay() output */
-	struct timeabs start_time;
-	
-	/* We stop trying after this time is reached. */
-	struct timeabs stop_time;
-	
-	/* Payment preimage, in case of success. */
-	const struct preimage *preimage;
-	
-	/* payment_secret, if specified by invoice. */
-	struct secret *payment_secret;
-	
-	/* Payment metadata, if specified by invoice. */
-	const u8 *payment_metadata;
-	
-	/* To know if the last attempt failed, succeeded or is it pending. */
-	enum payment_status status;	
-
-	u32 final_cltv;
-
-	/* Inside pay_plugin->payments list */
-	struct list_node list;
-
-	/* Description and labels, if any. */
-	const char *description, *label;
-
-	
-	/* Penalty for CLTV delays */
-	double delay_feefactor;
-	
-	/* Penalty for base fee */
-	double base_fee_penalty;
-	
-	/* With these the effective linear fee cost is computed as
-	 * 
-	 * linear fee cost = 
-	 * 	millionths 
-	 * 	+ base_fee* base_fee_penalty
-	 * 	+delay*delay_feefactor;
-	 * */
-
-	/* The minimum acceptable prob. of success */
-	double min_prob_success;
-	
-	/* Conversion from prob. cost to millionths */
-	double prob_cost_factor;
-	/* linear prob. cost = 
-	 * 	- prob_cost_factor * log prob. */
-
-
-	/* If this is paying a local offer, this is the one (sendpay ensures we
-	 * don't pay twice for single-use offers) */
-	// TODO(eduardo): this is not being used!
-	struct sha256 *local_offer_id;
-
-	/* DEVELOPER allows disabling shadow route */
-	bool use_shadow;
-	
-	/* Data used while the payment is being processed. */
-	struct active_payment *active_payment;
-	
-	/* Groupid, so listpays() can group them back together */
-	u64 groupid;
-	
-};
-
-int payment_current_attempt(const struct payment *p);
-
-void paynote(struct payment *p, const char *fmt, ...)
-	PRINTF_FMT(2,3);
-
+/* Set in init */
+extern struct pay_plugin * const pay_plugin;
 
 /* Accumulate or panic on overflow */
 #define amount_msat_accumulate(dst, src) \
diff --git a/plugins/renepay/pay_flow.c b/plugins/renepay/pay_flow.c
index 9e76945b1c78..c8d3de7dc888 100644
--- a/plugins/renepay/pay_flow.c
+++ b/plugins/renepay/pay_flow.c
@@ -137,10 +137,11 @@ static u64 flow_delay(const struct flow *flow)
 /* This enhances f->amounts, and returns per-flow cltvs */
 static u32 *shadow_additions(const tal_t *ctx,
 			     const struct gossmap *gossmap,
-			     struct payment *p,
+			     struct renepay *renepay,
 			     struct flow **flows,
 			     bool is_entire_payment)
 {
+	struct payment * p = renepay->payment;
 	u32 *final_cltvs;
 
 	/* Set these up now in case we decide to do nothing */
@@ -159,7 +160,7 @@ static u32 *shadow_additions(const tal_t *ctx,
 		shadow_delay = shadow_one_flow(gossmap, flows[i],
 					       &shadow_fee);
 		if (flow_delay(flows[i]) + shadow_delay > p->maxdelay) {
-			paynote(p, "No shadow for flow %zu/%zu:"
+			debug_paynote(p, "No shadow for flow %zu/%zu:"
 				" delay would add %u to %"PRIu64", exceeding max delay.",
 				i, tal_count(flows),
 				shadow_delay,
@@ -174,7 +175,7 @@ static u32 *shadow_additions(const tal_t *ctx,
 		if (is_entire_payment && tal_count(flows) == 1) {
 			if (!add_to_amounts(gossmap, flows[i], p->maxspend,
 					    shadow_fee)) {
-				paynote(p, "No shadow fee for flow %zu/%zu:"
+				debug_paynote(p, "No shadow fee for flow %zu/%zu:"
 					" fee would add %s to %s, exceeding budget %s.",
 					i, tal_count(flows),
 					type_to_string(tmpctx, struct amount_msat,
@@ -184,14 +185,14 @@ static u32 *shadow_additions(const tal_t *ctx,
 					type_to_string(tmpctx, struct amount_msat,
 						       &p->maxspend));
 			} else {
-				paynote(p, "No MPP, so added %s shadow fee",
+				debug_paynote(p, "No MPP, so added %s shadow fee",
 					type_to_string(tmpctx, struct amount_msat,
 						       &shadow_fee));
 			}
 		}
 
 		final_cltvs[i] += shadow_delay;
-		paynote(p, "Shadow route on flow %zu/%zu added %u block delay. now %u",
+		debug_paynote(p, "Shadow route on flow %zu/%zu added %u block delay. now %u",
 			i, tal_count(flows), shadow_delay, final_cltvs[i]);
 	}
 
@@ -239,7 +240,7 @@ static struct pay_flow **flows_to_pay_flows(struct payment *payment,
 		pf->amounts = tal_steal(pf, f->amounts);
 		pf->path_dirs = tal_steal(pf, f->dirs);
 		pf->success_prob = f->success_prob;
-		pf->attempt = payment_current_attempt(payment);
+		pf->attempt = renepay_current_attempt(payment->renepay);
 	}
 	tal_free(flows);
 
@@ -275,11 +276,12 @@ static u64 flows_worst_delay(struct flow **flows)
 }
 
 /* FIXME: If only path has channels marked disabled, we should try... */
-static bool disable_htlc_violations_oneflow(struct payment *p,
+static bool disable_htlc_violations_oneflow(struct renepay * renepay,
 					    const struct flow *flow,
 					    const struct gossmap *gossmap,
 					    bitmap *disabled)
 {
+	struct payment * p = renepay->payment;
 	bool disabled_some = false;
 
 	for (size_t i = 0; i < tal_count(flow->path); i++) {
@@ -297,12 +299,12 @@ static bool disable_htlc_violations_oneflow(struct payment *p,
 			continue;
 
 		scid = gossmap_chan_scid(gossmap, flow->path[i]);
-		paynote(p, "...disabling channel %s: %s",
+		debug_paynote(p, "...disabling channel %s: %s",
 			type_to_string(tmpctx, struct short_channel_id, &scid),
 			reason);
 
 		/* Add this for future searches for this payment. */
-		tal_arr_expand(&p->active_payment->disabled, scid);
+		tal_arr_expand(&renepay->disabled, scid);
 		/* Add to existing bitmap */
 		bitmap_set_bit(disabled,
 			       gossmap_chan_idx(gossmap, flow->path[i]));
@@ -313,7 +315,7 @@ static bool disable_htlc_violations_oneflow(struct payment *p,
 
 /* If we can't use one of these flows because we hit limits, we disable that
  * channel for future searches and return false */
-static bool disable_htlc_violations(struct payment *p,
+static bool disable_htlc_violations(struct renepay *renepay,
 				    struct flow **flows,
 				    const struct gossmap *gossmap,
 				    bitmap *disabled)
@@ -322,7 +324,7 @@ static bool disable_htlc_violations(struct payment *p,
 
 	/* We continue through all of them, to disable many at once. */
 	for (size_t i = 0; i < tal_count(flows); i++) {
-		disabled_some |= disable_htlc_violations_oneflow(p, flows[i],
+		disabled_some |= disable_htlc_violations_oneflow(renepay, flows[i],
 								 gossmap,
 								 disabled);
 	}
@@ -330,27 +332,28 @@ static bool disable_htlc_violations(struct payment *p,
 }
 
 /* Get some payment flows to get this amount to destination, or NULL. */
-struct pay_flow **get_payflows(struct payment *p,
+struct pay_flow **get_payflows(struct renepay * renepay,
 			       struct amount_msat amount,
 			       struct amount_msat feebudget,
 			       bool unlikely_ok,
 			       bool is_entire_payment,
 			       char const ** err_msg)
 {
+	struct payment * p = renepay->payment;
 	bitmap *disabled;
 	struct pay_flow **pay_flows;
 	const struct gossmap_node *src, *dst;
 	
-	disabled = make_disabled_bitmap(tmpctx, pay_plugin->gossmap, p->active_payment->disabled);
+	disabled = make_disabled_bitmap(tmpctx, pay_plugin->gossmap, renepay->disabled);
 	src = gossmap_find_node(pay_plugin->gossmap, &pay_plugin->my_id);
 	if (!src) {
-		paynote(p, "We don't have any channels?");
+		debug_paynote(p, "We don't have any channels?");
 		*err_msg = tal_fmt(tmpctx,"We don't have any channels.");
 		goto fail;
 	}
 	dst = gossmap_find_node(pay_plugin->gossmap, &p->destination);
 	if (!src) {
-		paynote(p, "No trace of destination in network gossip");
+		debug_paynote(p, "No trace of destination in network gossip");
 		*err_msg = tal_fmt(tmpctx,"Destination is unreacheable in the network gossip.");
 		goto fail;
 	}
@@ -372,7 +375,7 @@ struct pay_flow **get_payflows(struct payment *p,
 				p->base_fee_penalty,
 				p->prob_cost_factor);
 		if (!flows) {
-			paynote(p, "Failed to find any paths for %s",
+			debug_paynote(p, "Failed to find any paths for %s",
 				type_to_string(tmpctx,
 					       struct amount_msat,
 					       &amount));
@@ -387,7 +390,7 @@ struct pay_flow **get_payflows(struct payment *p,
 		too_unlikely = (prob < p->min_prob_success);
 		if (too_unlikely && !unlikely_ok)
 		{
-			paynote(p, "Flows too unlikely, P() = %f%%", prob * 100);
+			debug_paynote(p, "Flows too unlikely, P() = %f%%", prob * 100);
 			*err_msg = tal_fmt(tmpctx,
 					  "Probability is too small, "
 					  "Prob = %f%% (min = %f%%)",
@@ -398,7 +401,7 @@ struct pay_flow **get_payflows(struct payment *p,
 		too_expensive = amount_msat_greater(fee, feebudget);
 		if (too_expensive)
 		{
-			paynote(p, "Flows too expensive, fee = %s (max %s)",
+			debug_paynote(p, "Flows too expensive, fee = %s (max %s)",
 				type_to_string(tmpctx, struct amount_msat, &fee),
 				type_to_string(tmpctx, struct amount_msat, &feebudget));
 			*err_msg = tal_fmt(tmpctx,
@@ -410,12 +413,12 @@ struct pay_flow **get_payflows(struct payment *p,
 		}
 		too_delayed = (delay > p->maxdelay);
 		if (too_delayed) {
-			paynote(p, "Flows too delayed, delay = %"PRIu64" (max %u)",
+			debug_paynote(p, "Flows too delayed, delay = %"PRIu64" (max %u)",
 				delay, p->maxdelay);
 			
 			/* FIXME: What is a sane limit? */
 			if (p->delay_feefactor > 1000) {
-				paynote(p, "Giving up!");
+				debug_paynote(p, "Giving up!");
 				*err_msg = tal_fmt(tmpctx,
 						  "CLTV delay exceeds our CLTV budget, "
 						  "delay = %"PRIu64" (maxdelay = %u)",
@@ -424,7 +427,7 @@ struct pay_flow **get_payflows(struct payment *p,
 			}
 
 			p->delay_feefactor *= 2;
-			paynote(p, "Doubling delay_feefactor to %f",
+			debug_paynote(p, "Doubling delay_feefactor to %f",
 				p->delay_feefactor);
 			
 			continue; // retry
@@ -435,7 +438,7 @@ struct pay_flow **get_payflows(struct payment *p,
 		 * to do this inside minflow(), but the diagnostics here
 		 * are far better, since we can report min/max which
 		 * *actually* made us reconsider. */
-		if (disable_htlc_violations(p, flows, pay_plugin->gossmap,
+		if (disable_htlc_violations(renepay, flows, pay_plugin->gossmap,
 					    disabled))
 		{
 			continue; // retry
@@ -444,12 +447,12 @@ struct pay_flow **get_payflows(struct payment *p,
 		/* This can adjust amounts and final cltv for each flow,
 		 * to make it look like it's going elsewhere */
 		final_cltvs = shadow_additions(tmpctx, pay_plugin->gossmap,
-					       p, flows, is_entire_payment);
+					       renepay, flows, is_entire_payment);
 		/* OK, we are happy with these flows: convert to
 		 * pay_flows to outlive the current gossmap. */
-		pay_flows = flows_to_pay_flows(p, pay_plugin->gossmap,
+		pay_flows = flows_to_pay_flows(renepay->payment, pay_plugin->gossmap,
 					       flows, final_cltvs,
-					       &p->active_payment->next_partid);
+					       &renepay->next_partid);
 		break;
 	}
 
@@ -459,6 +462,17 @@ struct pay_flow **get_payflows(struct payment *p,
 	return NULL;
 }
 
+const char *flow_path_to_str(const tal_t *ctx, const struct pay_flow *flow)
+{
+	char *s = tal_strdup(ctx, "");
+	for (size_t i = 0; i < tal_count(flow->path_scids); i++) {
+		tal_append_fmt(&s, "-%s->",
+			       type_to_string(tmpctx, struct short_channel_id,
+					      &flow->path_scids[i]));
+	}
+	return s;
+}
+
 const char* fmt_payflows(const tal_t *ctx,
 			 struct pay_flow ** flows)
 {
@@ -586,3 +600,24 @@ void commit_htlc_payflow(
 		h->num_htlcs++;
 	}
 }
+
+/* How much does this flow deliver to destination? */
+struct amount_msat payflow_delivered(const struct pay_flow *flow)
+{
+	return flow->amounts[tal_count(flow->amounts)-1];
+}
+
+void payflow_fail(struct pay_flow *flow)
+{
+	debug_assert(flow);
+	struct payment * p = flow->payment;
+	
+	payment_fail(p);
+	amount_msat_reduce(&p->total_delivering, payflow_delivered(flow));
+	amount_msat_reduce(&p->total_sent, flow->amounts[0]);
+
+	/* Release the HTLCs in the uncertainty_network. */
+	tal_free(flow);
+}
+
+
diff --git a/plugins/renepay/pay_flow.h b/plugins/renepay/pay_flow.h
index ef905ad7425b..26db4180a34c 100644
--- a/plugins/renepay/pay_flow.h
+++ b/plugins/renepay/pay_flow.h
@@ -1,12 +1,14 @@
 #ifndef LIGHTNING_PLUGINS_RENEPAY_PAY_FLOW_H
 #define LIGHTNING_PLUGINS_RENEPAY_PAY_FLOW_H
 #include <ccan/short_types/short_types.h>
+#include <plugins/renepay/payment.h>
+#include <plugins/renepay/debug.h>
 
 /* This is like a struct flow, but independent of gossmap, and contains
  * all we need to actually send the part payment. */
 struct pay_flow {
 	/* So we can be an independent object for callbacks. */
-	struct payment *payment;
+	struct payment * payment;
 
 	/* This flow belongs to some attempt. */
 	int attempt;
@@ -25,7 +27,7 @@ struct pay_flow {
 	double success_prob;
 };
 
-struct pay_flow **get_payflows(struct payment *p,
+struct pay_flow **get_payflows(struct renepay * renepay,
 			       struct amount_msat amount,
 			       struct amount_msat feebudget,
 			       bool unlikely_ok,
@@ -40,7 +42,17 @@ void remove_htlc_payflow(
 		struct pay_flow *flow,
 		struct chan_extra_map *chan_extra_map);
 
+const char *flow_path_to_str(const tal_t *ctx, const struct pay_flow *flow);
+
 const char* fmt_payflows(const tal_t *ctx,
 			 struct pay_flow ** flows);
 
+/* How much does this flow deliver to destination? */
+struct amount_msat payflow_delivered(const struct pay_flow *flow);
+
+/* Removes amounts from payment and frees flow pointer. 
+ * A possible destructor for flow would remove HTLCs from the
+ * uncertainty_network and remove the flow from any data structure. */
+void payflow_fail(struct pay_flow *flow);
+
 #endif /* LIGHTNING_PLUGINS_RENEPAY_PAY_FLOW_H */
diff --git a/plugins/renepay/payment.c b/plugins/renepay/payment.c
new file mode 100644
index 000000000000..3528366cc810
--- /dev/null
+++ b/plugins/renepay/payment.c
@@ -0,0 +1,225 @@
+#include <plugins/renepay/payment.h>
+#include <plugins/renepay/debug.h>
+#include <ccan/ccan/tal/str/str.h>
+
+struct payment * payment_new(struct renepay * renepay)
+{
+	struct payment *p = tal(renepay,struct payment);
+	p->renepay = renepay;
+	p->paynotes = tal_arr(p, const char *, 0);
+	
+	p->total_sent = AMOUNT_MSAT(0);
+	p->total_delivering = AMOUNT_MSAT(0);
+	
+	p->invstr=NULL;
+	
+	p->amount = AMOUNT_MSAT(0);
+	// p->destination=
+	// p->payment_hash
+	p->maxspend = AMOUNT_MSAT(0);
+	p->maxdelay=0;
+	// p->start_time=
+	// p->stop_time=
+	p->preimage = NULL;
+	p->payment_secret=NULL;
+	p->payment_metadata=NULL;
+	p->status=PAYMENT_PENDING;
+	p->final_cltv=0;
+	// p->list=
+	p->description=NULL;
+	p->label=NULL;
+	
+	p->delay_feefactor=0;
+	p->base_fee_penalty=0;
+	p->prob_cost_factor=0;
+	p->min_prob_success=0;
+	
+	p->local_offer_id=NULL;
+	p->use_shadow=true;
+	p->groupid=1;
+	
+	p->result = NULL;
+	return p;
+}
+
+struct renepay * renepay_new(struct command *cmd)
+{
+	struct renepay *renepay = tal(cmd,struct renepay);
+	
+	renepay->cmd = cmd;
+	renepay->payment = payment_new(renepay);
+	renepay->localmods_applied=false;
+ 	renepay->local_gossmods = gossmap_localmods_new(renepay);
+	renepay->disabled = tal_arr(renepay,struct short_channel_id,0);
+	renepay->rexmit_timer = NULL;
+	renepay->next_attempt=1;
+	renepay->next_partid=1;
+	renepay->all_flows = tal(renepay,tal_t);
+	
+	return renepay;
+}
+
+
+void payment_fail(struct payment * p)
+{
+	/* If the payment already succeeded this function call must correspond
+	 * to an old sendpay. */
+	if(p->status == PAYMENT_SUCCESS)return;
+	p->status=PAYMENT_FAIL;
+}
+void payment_success(struct payment * p)
+{
+	p->status=PAYMENT_SUCCESS;
+}
+
+struct amount_msat payment_sent(struct payment const * p)
+{
+	return p->total_sent;
+}
+struct amount_msat payment_delivered(struct payment const * p)
+{
+	return p->total_delivering;
+}
+struct amount_msat payment_amount(struct payment const * p)
+{
+	return p->amount;
+}
+
+struct amount_msat payment_fees(struct payment const*p)
+{
+	struct amount_msat fees;
+	struct amount_msat sent = payment_sent(p), 
+			   delivered = payment_delivered(p);
+			   
+	if(!amount_msat_sub(&fees,sent,delivered))
+		debug_err( "Strange, sent amount (%s) is less than delivered (%s), aborting.",
+			   type_to_string(tmpctx,struct amount_msat,&sent),
+			   type_to_string(tmpctx,struct amount_msat,&delivered));
+	return fees;
+}
+
+void payment_note(struct payment *p, const char *fmt, ...)
+{
+	va_list ap;
+	const char *str;
+
+	va_start(ap, fmt);
+	str = tal_vfmt(p->paynotes, fmt, ap);
+	va_end(ap);
+	tal_arr_expand(&p->paynotes, str);
+	debug_info("%s",str);
+}
+
+void payment_assert_delivering_incomplete(struct payment const * p)
+{
+	if(!amount_msat_less(p->total_delivering, p->amount))
+	{
+		debug_err(
+			"Strange, delivering (%s) is not smaller than amount (%s)",
+			type_to_string(tmpctx,struct amount_msat,&p->total_delivering),
+			type_to_string(tmpctx,struct amount_msat,&p->amount));
+	}
+}
+void payment_assert_delivering_all(struct payment const * p)
+{
+	if(amount_msat_less(p->total_delivering, p->amount))
+	{
+		debug_err(
+			"Strange, delivering (%s) is less than amount (%s)",
+			type_to_string(tmpctx,struct amount_msat,&p->total_delivering),
+			type_to_string(tmpctx,struct amount_msat,&p->amount));
+	}
+}
+
+
+int renepay_current_attempt(const struct renepay * renepay)
+{
+	return renepay->next_attempt-1;
+}
+int renepay_attempt_count(const struct renepay * renepay)
+{
+	return renepay->next_attempt-1;
+}
+void renepay_new_attempt(struct renepay * renepay)
+{
+	renepay->payment->status=PAYMENT_PENDING;
+	renepay->next_attempt++;
+}
+struct command_result *renepay_success(struct renepay * renepay)
+{
+	debug_info("calling %s",__PRETTY_FUNCTION__);
+	struct payment *p = renepay->payment;
+	
+	payment_success(p);
+	payment_assert_delivering_all(p);
+	
+	struct json_stream *response 
+		= jsonrpc_stream_success(renepay->cmd);
+
+	/* Any one succeeding is success. */
+	json_add_preimage(response, "payment_preimage", p->preimage);
+	json_add_sha256(response, "payment_hash", &p->payment_hash);
+	json_add_timeabs(response, "created_at", p->start_time);
+	json_add_u32(response, "parts", renepay_parts(renepay));
+	json_add_amount_msat_only(response, "amount_msat",
+				  p->amount);
+	json_add_amount_msat_only(response, "amount_sent_msat",
+				  p->total_sent);
+	json_add_string(response, "status", "complete");
+	json_add_node_id(response, "destination", &p->destination);
+	
+	return command_finished(renepay->cmd, response);
+}
+
+struct command_result *renepay_fail(
+	struct renepay * renepay,
+	enum jsonrpc_errcode code,
+	const char *fmt, ...)
+{
+	/* renepay_fail is called after command finished. */
+	if(renepay==NULL)
+	{
+		return command_still_pending(NULL);
+	}
+	payment_fail(renepay->payment);
+	
+	va_list args;
+	va_start(args, fmt);
+	char *message = tal_vfmt(tmpctx,fmt,args);
+	va_end(args);
+	
+	debug_paynote(renepay->payment,"%s",message);
+	
+	return command_fail(renepay->cmd,code,"%s",message);
+}
+
+u64 renepay_parts(struct renepay const * renepay)
+{
+	return renepay->next_partid-1;
+}
+
+/* Either the payment succeeded or failed, we need to cleanup/set the plugin
+ * into a valid state before the next payment. */
+void renepay_cleanup(
+		struct renepay * renepay,
+		struct gossmap * gossmap)
+{
+	debug_info("calling %s",__PRETTY_FUNCTION__);
+	/* Always remove our local mods (routehints) so others can use
+	 * gossmap. We do this only after the payment completes. */
+	if(renepay->localmods_applied)
+		gossmap_remove_localmods(gossmap,
+					 renepay->local_gossmods);
+	// TODO(eduardo): I wonder if it is possible to have two instances of
+	// renepay at the same time. 
+	// 1st problem: dijkstra datastructure is global, this can be fixed,
+	// 2nd problem: we don't know if gossmap_apply_localmods and gossmap_remove_localmods,
+	// 	can handle different local_gossmods applied to the same gossmap.
+	renepay->localmods_applied=false;
+	tal_free(renepay->local_gossmods);
+	
+	renepay->rexmit_timer = tal_free(renepay->rexmit_timer);
+	
+	if(renepay->payment)
+		renepay->payment->renepay = NULL;
+}
diff --git a/plugins/renepay/payment.h b/plugins/renepay/payment.h
new file mode 100644
index 000000000000..d46ddc7b9c4a
--- /dev/null
+++ b/plugins/renepay/payment.h
@@ -0,0 +1,163 @@
+#ifndef LIGHTNING_PLUGINS_RENEPAY_PAYMENT_H
+#define LIGHTNING_PLUGINS_RENEPAY_PAYMENT_H
+#include "config.h"
+#include <plugins/libplugin.h>
+#include <common/gossmap.h>
+
+enum payment_status {
+        PAYMENT_PENDING, PAYMENT_SUCCESS, PAYMENT_FAIL
+};
+
+
+struct payment {
+	struct renepay * renepay;
+	
+	/* Chatty description of attempts. */
+	const char **paynotes;
+	
+	/* Total sent, including fees. */
+	struct amount_msat total_sent;
+	
+	/* Total that is delivering (i.e. without fees) */
+	struct amount_msat total_delivering;
+	
+	/* invstring (bolt11 or bolt12) */
+	const char *invstr;
+	
+	/* How much, what, where */
+	struct amount_msat amount;
+	struct node_id destination;
+	struct sha256 payment_hash;
+	
+	
+	/* Limits on what routes we'll accept. */
+	struct amount_msat maxspend;
+	
+	/* Max accepted HTLC delay.*/
+	unsigned int maxdelay;
+	
+	/* We promised this in pay() output */
+	struct timeabs start_time;
+	
+	/* We stop trying after this time is reached. */
+	struct timeabs stop_time;
+	
+	/* Payment preimage, in case of success. */
+	const struct preimage *preimage;
+	
+	/* payment_secret, if specified by invoice. */
+	struct secret *payment_secret;
+	
+	/* Payment metadata, if specified by invoice. */
+	const u8 *payment_metadata;
+	
+	/* To know if the last attempt failed, succeeded or is it pending. */
+	enum payment_status status;	
+
+	u32 final_cltv;
+
+	/* Inside pay_plugin->payments list */
+	struct list_node list;
+
+	/* Description and labels, if any. */
+	const char *description, *label;
+
+	
+	/* Penalty for CLTV delays */
+	double delay_feefactor;
+	
+	/* Penalty for base fee */
+	double base_fee_penalty;
+	
+	/* With these the effective linear fee cost is computed as
+	 * 
+	 * linear fee cost = 
+	 * 	millionths 
+	 * 	+ base_fee* base_fee_penalty
+	 * 	+delay*delay_feefactor;
+	 * */
+
+	/* The minimum acceptable prob. of success */
+	double min_prob_success;
+	
+	/* Conversion from prob. cost to millionths */
+	double prob_cost_factor;
+	/* linear prob. cost = 
+	 * 	- prob_cost_factor * log prob. */
+
+
+	/* If this is paying a local offer, this is the one (sendpay ensures we
+	 * don't pay twice for single-use offers) */
+	// TODO(eduardo): this is not being used!
+	struct sha256 *local_offer_id;
+
+	/* DEVELOPER allows disabling shadow route */
+	bool use_shadow;
+	
+	/* Groupid, so listpays() can group them back together */
+	u64 groupid;
+	
+	struct command_result * result;
+};
+
+/* Data only kept while the payment is being processed. */
+struct renepay
+{
+	/* The command, and our owner (needed for timer func) */
+	struct command *cmd;
+	
+	/* Payment information that will eventually outlive renepay and be
+	 * registered. */
+	struct payment * payment;
+
+	/* Localmods to apply to gossip_map for our own use. */
+	bool localmods_applied;
+	struct gossmap_localmods *local_gossmods;
+	
+	/* Channels we decided to disable for various reasons. */
+	struct short_channel_id *disabled;
+
+	/* Timers. */
+	struct plugin_timer *rexmit_timer;
+	
+	/* Keep track of the number of attempts. */
+	int next_attempt;
+	/* Used in get_payflows to set ids to each pay_flow. */
+	u64 next_partid;
+	
+	/* Root to destroy pending flows */
+	tal_t *all_flows;
+};
+
+struct payment * payment_new(struct renepay *renepay);
+struct renepay * renepay_new(struct command *cmd);
+void renepay_cleanup(
+		struct renepay * renepay,
+		struct gossmap * gossmap);
+
+void payment_fail(struct payment * p);
+void payment_success(struct payment * p);
+struct amount_msat payment_sent(struct payment const * p);
+struct amount_msat payment_delivered(struct payment const * p);
+struct amount_msat payment_amount(struct payment const * p);
+struct amount_msat payment_fees(struct payment const*p);
+
+void payment_note(struct payment *p, const char *fmt, ...);
+void payment_assert_delivering_incomplete(struct payment const * p);
+void payment_assert_delivering_all(struct payment const * p);
+
+
+int renepay_current_attempt(const struct renepay *renepay);
+int renepay_attempt_count(const struct renepay *renepay);
+void renepay_new_attempt(struct renepay *renepay);
+
+struct command_result *renepay_success(struct renepay *renepay);
+
+struct command_result *renepay_fail(
+	struct renepay * renepay,
+	enum jsonrpc_errcode code,
+	const char *fmt, ...);
+	
+u64 renepay_parts(struct renepay const * renepay);
+
+#endif // LIGHTNING_PLUGINS_RENEPAY_PAYMENT_H
diff --git a/plugins/renepay/test/run-mcf-diamond.c b/plugins/renepay/test/run-mcf-diamond.c
index 7112c004aa91..23777916e25b 100644
--- a/plugins/renepay/test/run-mcf-diamond.c
+++ b/plugins/renepay/test/run-mcf-diamond.c
@@ -1,7 +1,9 @@
 #include "config.h"
 
-#define FLOW_UNITTEST // logs are written in /tmp/debug.txt
+#define RENEPAY_UNITTEST // logs are written in /tmp/debug.txt
+#include "../payment.c"
 #include "../flow.c"
+#include "../uncertainty_network.c"
 #include "../mcf.c"
 
 #include <ccan/array_size/array_size.h>
diff --git a/plugins/renepay/test/run-mcf.c b/plugins/renepay/test/run-mcf.c
index 92a0ac5adf61..c2c994817e44 100644
--- a/plugins/renepay/test/run-mcf.c
+++ b/plugins/renepay/test/run-mcf.c
@@ -1,7 +1,9 @@
 #include "config.h"
 
-#define FLOW_UNITTEST // logs are written in /tmp/debug.txt
+#define RENEPAY_UNITTEST // logs are written in /tmp/debug.txt
+#include "../payment.c"
 #include "../flow.c"
+#include "../uncertainty_network.c"
 #include "../mcf.c"
 
 #include <ccan/array_size/array_size.h>
diff --git a/plugins/renepay/test/run-testflow.c b/plugins/renepay/test/run-testflow.c
index 4c1a9a7ebf3b..32931db02124 100644
--- a/plugins/renepay/test/run-testflow.c
+++ b/plugins/renepay/test/run-testflow.c
@@ -10,9 +10,11 @@
 #include <ccan/read_write_all/read_write_all.h>
 
 #define MYLOG "/tmp/debug.txt"
-#define FLOW_UNITTEST // logs are written in MYLOG
-#include <plugins/renepay/flow.h>
+#define RENEPAY_UNITTEST // logs are written in MYLOG
+#include "../payment.c"
 #include "../flow.c"
+#include "../uncertainty_network.c"
+#include "../mcf.c"
 
 static const u8 canned_map[] = {
 0x0c, 0x80, 0x00, 0x01, 0xbc, 0x86, 0xe4, 0xbf, 0x95, 0x00, 0x00, 0x00, 0x00, 0x10, 0x08, 0x00, 
diff --git a/plugins/renepay/uncertainty_network.c b/plugins/renepay/uncertainty_network.c
new file mode 100644
index 000000000000..465bc485a680
--- /dev/null
+++ b/plugins/renepay/uncertainty_network.c
@@ -0,0 +1,346 @@
+#include <plugins/renepay/uncertainty_network.h>
+#include <plugins/renepay/payment.h>
+#include <plugins/renepay/debug.h>
+#include <plugins/renepay/flow.h>
+
+static bool chan_extra_check_invariants(struct chan_extra *ce)
+{
+	bool all_ok = true;
+	for(int i=0;i<2;++i)
+	{
+		all_ok &= amount_msat_less_eq(ce->half[i].known_min,
+					      ce->half[i].known_max);
+		all_ok &= amount_msat_less_eq(ce->half[i].known_max,
+					      ce->capacity);
+	}
+	struct amount_msat diff_cb,diff_ca;
+	
+	all_ok &= amount_msat_sub(&diff_cb,ce->capacity,ce->half[1].known_max);
+	all_ok &= amount_msat_sub(&diff_ca,ce->capacity,ce->half[1].known_min);
+	
+	all_ok &= amount_msat_eq(ce->half[0].known_min,diff_cb);
+	all_ok &= amount_msat_eq(ce->half[0].known_max,diff_ca);
+	return all_ok;
+}
+
+
+/* Checks the entire uncertainty network for invariant violations. */
+bool uncertainty_network_check_invariants(struct chan_extra_map *chan_extra_map)
+{
+	bool all_ok = true;
+	
+	struct chan_extra_map_iter it;
+	for(struct chan_extra *ce = chan_extra_map_first(chan_extra_map,&it);
+	    ce && all_ok;
+	    ce=chan_extra_map_next(chan_extra_map,&it))
+	{
+		all_ok &= chan_extra_check_invariants(ce);
+	}
+		
+	return all_ok;
+}
+
+static void add_hintchan(
+		struct chan_extra_map *chan_extra_map,
+		struct renepay * renepay,
+		const struct node_id *src,
+		const struct node_id *dst,
+		u16 cltv_expiry_delta,
+		const struct short_channel_id scid,
+		u32 fee_base_msat,
+		u32 fee_proportional_millionths)
+{
+	int dir = node_id_cmp(src, dst) < 0 ? 0 : 1;
+	
+	struct chan_extra *ce = chan_extra_map_get(chan_extra_map,
+						   scid);
+	if(!ce)
+	{
+		/* this channel is not public, we don't know his capacity */
+		// TODO(eduardo): one possible solution is set the capacity to
+		// MAX_CAP and the state to [0,MAX_CAP]. Alternatively we set
+		// the capacity to amoung and state to [amount,amount].
+		ce = new_chan_extra(chan_extra_map,
+				    scid,
+				    MAX_CAP);
+		/* FIXME: features? */
+		gossmap_local_addchan(renepay->local_gossmods, 
+				      src, dst, &scid, NULL);
+		gossmap_local_updatechan(renepay->local_gossmods,
+					 &scid,
+					 /* We assume any HTLC is allowed */
+					 AMOUNT_MSAT(0), MAX_CAP,
+					 fee_base_msat, fee_proportional_millionths,
+					 cltv_expiry_delta,
+					 true,
+					 dir);
+	}
+	
+	/* It is wrong to assume that this channel has sufficient capacity! 
+	 * Doing so leads to knowledge updates in which the known min liquidity
+	 * is greater than the channel's capacity. */
+	// chan_extra_can_send(chan_extra_map,scid,dir,amount);
+}
+
+/* Add routehints provided by bolt11 */
+void uncertainty_network_add_routehints(
+		struct chan_extra_map *chan_extra_map,
+		struct renepay *renepay)
+{
+	struct payment const * const p = renepay->payment;
+	struct bolt11 *b11;
+	char *fail;
+
+	b11 =
+	    bolt11_decode(tmpctx, p->invstr, 
+	    		  plugin_feature_set(renepay->cmd->plugin),
+			  p->description, chainparams, &fail);
+	if (b11 == NULL)
+		debug_err("add_routehints: Invalid bolt11: %s", fail);
+		
+	for (size_t i = 0; i < tal_count(b11->routes); i++) {
+		/* Each one, presumably, leads to the destination */
+		const struct route_info *r = b11->routes[i];
+		const struct node_id *end = & p->destination;
+		for (int j = tal_count(r)-1; j >= 0; j--) {
+			add_hintchan(
+				chan_extra_map,
+				renepay, &r[j].pubkey, end,
+				r[j].cltv_expiry_delta,
+				r[j].short_channel_id,
+				r[j].fee_base_msat,
+				r[j].fee_proportional_millionths);
+			end = &r[j].pubkey;
+		}
+	}
+}
+
+/* Mirror the gossmap in the public uncertainty network.
+ * result: Every channel in gossmap must have associated data in chan_extra_map,
+ * while every channel in chan_extra_map is also registered in gossmap.
+ * */
+void uncertainty_network_update(
+		const struct gossmap *gossmap,
+		struct chan_extra_map *chan_extra_map)
+{
+	const tal_t* this_ctx = tal(tmpctx,tal_t);
+	
+	// For each chan in chan_extra_map remove if not in the gossmap
+	struct short_channel_id *del_list
+		= tal_arr(this_ctx,struct short_channel_id,0);
+		
+	struct chan_extra_map_iter it;
+	for(struct chan_extra *ce = chan_extra_map_first(chan_extra_map,&it);
+	    ce;
+	    ce=chan_extra_map_next(chan_extra_map,&it))
+	{
+		struct gossmap_chan * chan = gossmap_find_chan(gossmap,&ce->scid);
+		/* Only if the channel is not in the gossmap and there are not
+		 * HTLCs pending we can remove it. */
+		if(!chan && !chan_extra_is_busy(ce))
+		{
+			// TODO(eduardo): is this efficiently implemented?
+			// otherwise i'll use a ccan list
+			tal_arr_expand(&del_list, ce->scid);
+		}
+	}
+	
+	for(size_t i=0;i<tal_count(del_list);++i)
+	{
+ 		struct chan_extra *ce = chan_extra_map_get(chan_extra_map,del_list[i]);
+		if(!ce)
+		{
+			debug_err("%s (line %d) unexpected chan_extra ce is NULL",
+				__PRETTY_FUNCTION__,
+				__LINE__);
+		}
+		chan_extra_map_del(chan_extra_map, ce);
+		tal_free(ce);
+	}
+	
+	// For each channel in the gossmap, create a extra data in
+	// chan_extra_map
+	for(struct gossmap_chan *chan = gossmap_first_chan(gossmap);
+	    chan;
+	    chan=gossmap_next_chan(gossmap,chan))
+	{
+		struct short_channel_id scid =
+			gossmap_chan_scid(gossmap,chan);
+		struct chan_extra *ce = chan_extra_map_get(chan_extra_map,
+							   gossmap_chan_scid(gossmap,chan));	
+		if(!ce)
+		{
+			struct amount_sat cap;
+			struct amount_msat cap_msat;
+			
+			if(!gossmap_chan_get_capacity(gossmap,chan,&cap))
+			{
+				debug_err("%s (line %d) unable to fetch channel capacity",
+					__PRETTY_FUNCTION__,
+					__LINE__);
+			}
+			if(!amount_sat_to_msat(&cap_msat,cap))
+			{
+				debug_err("%s (line %d) unable convert sat to msat",
+					__PRETTY_FUNCTION__,
+					__LINE__);
+			}
+			new_chan_extra(chan_extra_map,scid,cap_msat);
+		}
+	}
+	tal_free(this_ctx);
+}
+
+void uncertainty_network_flow_success(
+		struct chan_extra_map *chan_extra_map,
+		struct pay_flow *flow)
+{
+	for (size_t i = 0; i < tal_count(flow->path_scids); i++)
+	{
+		chan_extra_sent_success(
+			chan_extra_map,
+			flow->path_scids[i],
+			flow->path_dirs[i],
+			flow->amounts[i]);
+	}
+}
+/* All parts up to erridx succeeded, so we know something about min
+ * capacity! */
+void uncertainty_network_channel_can_send(
+		struct chan_extra_map * chan_extra_map,
+		struct pay_flow *flow,
+		u32 erridx)
+{
+	for (size_t i = 0; i < erridx; i++)
+	{
+		chan_extra_can_send(chan_extra_map,
+				    flow->path_scids[i],
+				    flow->path_dirs[i],
+				    
+				    /* This channel can send all that was
+				     * commited in HTLCs.
+				     * Had we removed the commited amount then
+				     * we would have to put here flow->amounts[i]. */
+				    AMOUNT_MSAT(0));
+	}
+}
+
+/* listpeerchannels gives us the certainty on local channels' capacity.  Of course,
+ * this is racy and transient, but better than nothing! */
+bool uncertainty_network_update_from_listpeerchannels(
+		struct chan_extra_map * chan_extra_map,
+		struct node_id my_id,
+		struct renepay * renepay,
+		const char *buf,
+		const jsmntok_t *toks)
+{
+	struct payment * const p = renepay->payment;
+	const jsmntok_t *channels, *channel;
+	size_t i;
+
+	if (json_get_member(buf, toks, "error"))
+		goto malformed;
+
+	channels = json_get_member(buf, toks, "channels");
+	if (!channels)
+		goto malformed;
+
+	json_for_each_arr(i, channel, channels) {
+		struct short_channel_id scid;
+		const jsmntok_t *scidtok = json_get_member(buf, channel, "short_channel_id");
+		/* If channel is still opening, this won't be there. 
+		 * Also it won't be in the gossmap, so there is 
+		 * no need to mark it as disabled. */
+		if (!scidtok)
+			continue;
+		if (!json_to_short_channel_id(buf, scidtok, &scid))
+			goto malformed;
+	
+		bool connected;
+		if(!json_to_bool(buf,
+				 json_get_member(buf,channel,"peer_connected"),
+				 &connected))
+			goto malformed;
+	
+		if (!connected) {
+			debug_paynote(p, "local channel %s disabled:"
+				" peer disconnected",
+				type_to_string(tmpctx,
+					       struct short_channel_id,
+					       &scid));
+			tal_arr_expand(&renepay->disabled, scid);
+			continue;
+		}
+		
+		const jsmntok_t *spendabletok, *dirtok,*statetok, *totaltok, 
+			*peeridtok;
+		struct amount_msat spendable,capacity;
+		int dir;
+		
+		const struct node_id src=my_id;
+		struct node_id dst;
+		
+		spendabletok = json_get_member(buf, channel, "spendable_msat");
+		dirtok = json_get_member(buf, channel, "direction");
+		statetok = json_get_member(buf, channel, "state");
+		totaltok = json_get_member(buf, channel, "total_msat");
+		peeridtok = json_get_member(buf,channel,"peer_id");
+		
+		if(spendabletok==NULL || dirtok==NULL || statetok==NULL || 
+		   totaltok==NULL || peeridtok==NULL)
+			goto malformed;
+		if (!json_to_msat(buf, spendabletok, &spendable))
+			goto malformed;
+		if (!json_to_msat(buf, totaltok, &capacity))
+			goto malformed;
+		if (!json_to_int(buf, dirtok,&dir))
+			goto malformed;
+		if(!json_to_node_id(buf,peeridtok,&dst))
+			goto malformed;
+			
+		/* Don't report opening/closing channels */
+		if (!json_tok_streq(buf, statetok, "CHANNELD_NORMAL")) {
+			tal_arr_expand(&renepay->disabled, scid);
+			continue;
+		}
+		
+		struct chan_extra *ce = chan_extra_map_get(chan_extra_map,
+							   scid);
+			  
+		if(!ce)
+		{
+			/* this channel is not public, but it belongs to us */
+			ce = new_chan_extra(chan_extra_map,
+					    scid,
+					    capacity);
+			/* FIXME: features? */
+			gossmap_local_addchan(renepay->local_gossmods, 
+					      &src, &dst, &scid, NULL);
+			gossmap_local_updatechan(renepay->local_gossmods,
+						 &scid,
+						 
+						 /* TODO(eduardo): does it
+						  * matter to consider HTLC
+						  * limits in our own channel? */
+						 AMOUNT_MSAT(0),capacity,
+						 
+						 /* fees = */0,0,
+						 
+						 /* TODO(eduardo): does it
+						  * matter to set this delay? */
+						 /*delay=*/0,
+						 true,
+						 dir);
+		}
+		
+		// TODO(eduardo): this includes pending HTLC of previous
+		// payments!
+		/* We know min and max liquidity exactly now! */
+		chan_extra_set_liquidity(chan_extra_map,
+					 scid,dir,spendable);
+	}
+	return true;
+
+malformed:
+	return false;
+}
diff --git a/plugins/renepay/uncertainty_network.h b/plugins/renepay/uncertainty_network.h
new file mode 100644
index 000000000000..24bd5a05c6b6
--- /dev/null
+++ b/plugins/renepay/uncertainty_network.h
@@ -0,0 +1,46 @@
+#ifndef LIGHTNING_PLUGINS_RENEPAY_UNCERTAINTY_NETWORK_H
+#define LIGHTNING_PLUGINS_RENEPAY_UNCERTAINTY_NETWORK_H
+
+#include <common/gossmap.h>
+#include <plugins/renepay/flow.h>
+#include <plugins/renepay/pay_flow.h>
+#include <plugins/renepay/payment.h>
+
+
+/* Checks the entire uncertainty network for invariant violations. */
+bool uncertainty_network_check_invariants(struct chan_extra_map *chan_extra_map);
+
+/* Add routehints provided by bolt11 */
+void uncertainty_network_add_routehints(
+		struct chan_extra_map *chan_extra_map,
+		struct renepay *renepay);
+
+/* Mirror the gossmap in the public uncertainty network.
+ * result: Every channel in gossmap must have associated data in chan_extra_map,
+ * while every channel in chan_extra_map is also registered in gossmap.
+ * */
+void uncertainty_network_update(
+		const struct gossmap *gossmap,
+		struct chan_extra_map *chan_extra_map);
+
+void uncertainty_network_flow_success(
+		struct chan_extra_map *chan_extra_map,
+		struct pay_flow *flow);
+
+/* All parts up to erridx succeeded, so we know something about min
+ * capacity! */
+void uncertainty_network_channel_can_send(
+		struct chan_extra_map * chan_extra_map,
+		struct pay_flow *flow,
+		u32 erridx);
+
+/* listpeerchannels gives us the certainty on local channels' capacity.  Of course,
+ * this is racy and transient, but better than nothing! */
+bool uncertainty_network_update_from_listpeerchannels(
+		struct chan_extra_map * chan_extra_map,
+		struct node_id my_id,
+		struct renepay * renepay,
+		const char *buf,
+		const jsmntok_t *toks);
+
+#endif // LIGHTNING_PLUGINS_RENEPAY_UNCERTAINTY_NETWORK_H
diff --git a/tests/test_renepay.py b/tests/test_renepay.py
index 14ac7a0b9dff..a1c3ccd0c027 100644
--- a/tests/test_renepay.py
+++ b/tests/test_renepay.py
@@ -82,9 +82,9 @@ def test_pay(node_factory):
     l1.rpc.call('renepay',{'invstring':inv,'use_shadow':False})
     # This won't work: can't provide an amount (even if correct!)
     with pytest.raises(RpcError):
-        l1.rpc.call('renepay',{'invstring':inv,'amount_msat':123000000})
+        l1.rpc.call('renepay',{'invstring':inv,'amount_msat':123000})
     with pytest.raises(RpcError):
-        l1.rpc.call('renepay',{'invstring':inv,'amount_msat':122000000})
+        l1.rpc.call('renepay',{'invstring':inv,'amount_msat':122000})
 
     # Check pay_index is not null
     outputs = l2.db_query('SELECT pay_index IS NOT NULL AS q FROM invoices WHERE label="label";')
@@ -98,8 +98,6 @@ def test_pay(node_factory):
         with pytest.raises(RpcError):
             l1.rpc.call('renepay',{'invstring':inv2,'use_shadow':False})
         
-        # TODO(eduardo): renepay can't handle a msat precision, fix this and add
-        # a test to check it
         l1.rpc.call('renepay',{'invstring':inv2,'use_shadow':False,'amount_msat':random.randint(1000,999999)})
 
     # Should see 6 completed payments
@@ -188,11 +186,11 @@ def test_limits(node_factory):
         l1.rpc.call('renepay', {'invstring': inv2['bolt11'],'min_prob_success': '0.5'})
     assert err.value.error['code'] == PAY_ROUTE_NOT_FOUND
     
-    # if we try again we can finish this payment
-    l1.rpc.call('renepay', {'invstring': inv2['bolt11']})
-    invoice = only_one(l6.rpc.listinvoices('inv2')['invoices'])
-    assert isinstance(invoice['amount_received_msat'], Millisatoshi)
-    assert invoice['amount_received_msat'] >= Millisatoshi('800000sat')
+    # # if we try again we can finish this payment
+    # l1.rpc.call('renepay', {'invstring': inv2['bolt11']})
+    # invoice = only_one(l6.rpc.listinvoices('inv2')['invoices'])
+    # assert isinstance(invoice['amount_received_msat'], Millisatoshi)
+    # assert invoice['amount_received_msat'] >= Millisatoshi('800000sat')
 
 # @pytest.mark.developer("Gossip is too slow without developer")
 # def test_pay_exclude_node(node_factory, bitcoind):

From 00c203571c15dc66c22409e506387a8bd9a3d924 Mon Sep 17 00:00:00 2001
From: Lagrang3 <eduardo.quintana@pm.me>
Date: Mon, 3 Jul 2023 10:47:49 +0100
Subject: [PATCH 64/64] renepay: use sendpay notifications

Instead of waitsendpay
---
 plugins/renepay/flow.c                 |   27 -
 plugins/renepay/pay.c                  |  850 ++--
 plugins/renepay/pay.h                  |    5 +-
 plugins/renepay/pay_flow.c             |   30 +-
 plugins/renepay/pay_flow.h             |   67 +-
 plugins/renepay/payment.c              |    4 +
 plugins/renepay/test/run-payflow_map.c |   96 +
 plugins/renepay/uncertainty_network.c  |    5 +-
 tests/test_renepay.py                  | 5289 +-----------------------
 9 files changed, 745 insertions(+), 5628 deletions(-)
 create mode 100644 plugins/renepay/test/run-payflow_map.c

diff --git a/plugins/renepay/flow.c b/plugins/renepay/flow.c
index b1679195afbe..6cf4f19b6cfe 100644
--- a/plugins/renepay/flow.c
+++ b/plugins/renepay/flow.c
@@ -364,33 +364,6 @@ get_chan_extra_half_by_chan_verify(
 	return h;
 }
 
-// // TODO(eduardo): not sure we should have this function
-// struct chan_extra_half *new_chan_extra_half(struct chan_extra_map *chan_extra_map,
-// 					    const struct short_channel_id scid,
-// 					    int dir,
-// 					    struct amount_msat capacity)
-// {
-// 	struct chan_extra *ce = tal(chan_extra_map, struct chan_extra);
-// 
-// 	ce->scid = scid;
-// 	for (size_t i = 0; i <= 1; i++) {
-// 		ce->half[i].num_htlcs = 0;
-// 		ce->half[i].htlc_total = AMOUNT_MSAT(0);
-// 		ce->half[i].known_min = AMOUNT_MSAT(0);
-// 		ce->half[i].known_max = capacity;
-// 	}
-// 	/* Remove self from map when done */
-// 	chan_extra_map_add(chan_extra_map, ce);
-// 	
-// 	// TODO(eduardo):
-// 	// Is this desctructor really necessary? the chan_extra will deallocated
-// 	// when the chan_extra_map is freed. Anyways valgrind complains that the
-// 	// hash table is removing the element with a freed pointer.
-// 	// tal_add_destructor2(ce, destroy_chan_extra, chan_extra_map);
-// 	return &ce->half[dir];
-// }
-
-
 /* Assuming a uniform distribution, what is the chance this f gets through?
  * Here we compute the conditional probability of success for a flow f, given
  * the knowledge that the liquidity is in the range [a,b) and some amount
diff --git a/plugins/renepay/pay.c b/plugins/renepay/pay.c
index 4216f8acc967..49b21de13111 100644
--- a/plugins/renepay/pay.c
+++ b/plugins/renepay/pay.c
@@ -19,8 +19,6 @@
 #include <plugins/renepay/payment.h>
 #include <plugins/renepay/uncertainty_network.h>
 
-// TODO(eduardo): understand the use of struct command_result as a return.
-
 // TODO(eduardo): maybe there are too many debug_err and plugin_err and
 // plugin_log(...,LOG_BROKEN,...) that could be resolved with a command_fail
 
@@ -83,6 +81,12 @@ static void memleak_mark(struct plugin *p, struct htable *memtable)
 }
 #endif
 
+static void destroy_payflow(struct pay_flow *flow)
+{
+	remove_htlc_payflow(pay_plugin->chan_extra_map,flow);
+	payflow_map_del(pay_plugin->payflow_map, flow);
+}
+
 static const char *init(struct plugin *p,
 			const char *buf UNUSED, const jsmntok_t *config UNUSED)
 {
@@ -106,6 +110,9 @@ static const char *init(struct plugin *p,
 	
 	pay_plugin->chan_extra_map = tal(pay_plugin->ctx,struct chan_extra_map);
 	chan_extra_map_init(pay_plugin->chan_extra_map);
+	
+	pay_plugin->payflow_map = tal(pay_plugin->ctx,struct payflow_map);
+	payflow_map_init(pay_plugin->payflow_map);
 
 	pay_plugin->gossmap = gossmap_load(pay_plugin->ctx,
 					   GOSSIP_STORE_FILENAME,
@@ -211,36 +218,6 @@ static void timer_kick(struct renepay * renepay)
 	}
 }
 
-/* Once this function is called we know that the payment succeeded and we
- * terminate with payment_success. */
-static struct command_result *waitsendpay_succeeded(struct command *cmd,
-						    const char *buf,
-						    const jsmntok_t *result,
-						    struct pay_flow *flow)
-{
-	plugin_log(pay_plugin->plugin,LOG_DBG,"calling %s",__PRETTY_FUNCTION__);
-	struct payment * const p = flow->payment;
-	payment_success(p);
-	
-	const jsmntok_t *preimagetok
-		= json_get_member(buf, result, "payment_preimage");
-	
-	struct preimage preimage;
-	
-	if (!preimagetok || !json_to_preimage(buf, preimagetok,&preimage))
-		plugin_err(cmd->plugin,
-			   "Strange return from waitsendpay: %.*s",
-			   json_tok_full_len(result),
-			   json_tok_full(buf, result));
-	
-	p->preimage = tal_dup_or_null(p,struct preimage,&preimage);
-
-	uncertainty_network_flow_success(pay_plugin->chan_extra_map,flow);
-	tal_free(flow);
-	return command_still_pending(cmd);
-}
-
-
 /* Sometimes we don't know exactly who to blame... */
 static struct command_result *handle_unhandleable_error(struct renepay * renepay,
 							struct pay_flow *flow,
@@ -412,275 +389,6 @@ static u8 *channel_update_from_onion_error(const tal_t *ctx,
 	return patch_channel_update(ctx, take(channel_update));
 }
 
-/* This is a waitsendpay_fail branch that happens in the background (ie. after
- * renepay command returned). */
-static struct command_result *waitsendpay_failed_background(
-		const char *buf,
-		const jsmntok_t *err,
-		struct pay_flow *flow)
-{
-	plugin_log(pay_plugin->plugin,LOG_DBG,"calling %s",__PRETTY_FUNCTION__);
-	struct payment * const p = flow->payment;
-	
-	payment_fail(p);
-	
-	u64 errcode;
-	if (!json_to_u64(buf, json_get_member(buf, err, "code"), &errcode))
-		goto done;
-	
-	if(errcode!=PAY_TRY_OTHER_ROUTE)
-		goto done;
-
-	const jsmntok_t* datatok = json_get_member(buf, err, "data");
-
-	/* OK, we expect an onion error reply. */
-	const jsmntok_t * erridxtok = json_get_member(buf, datatok, "erring_index");
-	if (!erridxtok)
-		goto done;
-	
-	u32 erridx;
-	json_to_u32(buf, erridxtok, &erridx);
-	const jsmntok_t *errchantok = json_get_member(buf, datatok, "erring_channel");
-	struct short_channel_id errscid;
-	json_to_short_channel_id(buf, errchantok, &errscid);
-	
-	if (erridx<tal_count(flow->path_scids) 
-	    && !short_channel_id_eq(&errscid, &flow->path_scids[erridx]))
-	    	goto done;
-
-	const jsmntok_t *failcodetok = json_get_member(buf, datatok, "failcode");
-	u32 onionerr;
-	json_to_u32(buf, failcodetok, &onionerr);
-
-	uncertainty_network_channel_can_send(
-			pay_plugin->chan_extra_map,
-			flow,
-			erridx);
-	
-	/* Insufficient funds! */
-	if((enum onion_wire)onionerr == WIRE_TEMPORARY_CHANNEL_FAILURE){	
-		plugin_log(pay_plugin->plugin,LOG_DBG,"waitsendpay_failed: Insufficient funds!");
-		
-		chan_extra_cannot_send(p,pay_plugin->chan_extra_map,
-				       flow->path_scids[erridx],
-				       flow->path_dirs[erridx],
-				    /* This channel can't send all that was
-				     * commited in HTLCs.
-				     * Had we removed the commited amount then
-				     * we would have to put here flow->amounts[erridx]. */
-				       AMOUNT_MSAT(0));
-	}
-done:
-	payflow_fail(flow);
-	return command_still_pending(NULL);
-}
-
-static struct command_result *waitsendpay_failed(struct command *cmd,
-						 const char *buf,
-						 const jsmntok_t *err,
-						 struct pay_flow *flow)
-{
-	plugin_log(pay_plugin->plugin,LOG_DBG,"calling %s",__PRETTY_FUNCTION__);
-	plugin_log(pay_plugin->plugin,LOG_DBG,"waitsendpay failed with reply %.*s",
-		   json_tok_full_len(err),
-		   json_tok_full(buf, err));
-	
-	struct payment * const p = flow->payment;
-	struct renepay * const renepay = p->renepay;
-	
-	payment_fail(p);
-	if(!renepay)
-		return waitsendpay_failed_background(buf,err,flow);
-	
-	u64 errcode;
-	struct command_result *ret;
-	const jsmntok_t *datatok, *msgtok, *errchantok, *erridxtok, *failcodetok;
-	u32 onionerr, erridx;
-	struct short_channel_id errscid;
-	
-	const jsmntok_t *rawoniontok;
-
-	if (!json_to_u64(buf, json_get_member(buf, err, "code"), &errcode))
-		plugin_err(cmd->plugin, "Bad errcode from waitsendpay: %.*s",
-			   json_tok_full_len(err), json_tok_full(buf, err));
-
-	switch (errcode) {
-	case PAY_UNPARSEABLE_ONION:
-		plugin_log(pay_plugin->plugin,LOG_DBG,"waitsendpay_failed: PAY_UNPARSEABLE_ONION");
-		ret = handle_unhandleable_error(renepay, flow, "unparsable onion reply");
-		if (ret)
-			return ret;
-		goto done;
-	case PAY_DESTINATION_PERM_FAIL:
-		plugin_log(pay_plugin->plugin,LOG_DBG,
-			"waitsendpay_failed: PAY_DESTINATION_PERM_FAIL");
-		
-		/* Inmediate failure. No information. */
-		payflow_fail(flow);
-		return renepay_fail(renepay,PAY_DESTINATION_PERM_FAIL,
-				    "Got an final failure from destination");
-	case PAY_TRY_OTHER_ROUTE:
-		plugin_log(pay_plugin->plugin,LOG_DBG,"waitsendpay_failed: PAY_TRY_OTHER_ROUTE");
-		break;
-	default:
-		plugin_log(pay_plugin->plugin,LOG_DBG,
-			   "Unexpected errcode from waitsendpay: %.*s",
-			   json_tok_full_len(err), json_tok_full(buf, err));
-		/* Inmediate failure. No information. */
-		payflow_fail(flow);
-		return renepay_fail(renepay,errcode, "Unexpected errcode from waitsendpay.");
-	}
-
-	datatok = json_get_member(buf, err, "data");
-
-	/* OK, we expect an onion error reply. */
-	erridxtok = json_get_member(buf, datatok, "erring_index");
-	if (!erridxtok) {
-		ret = handle_unhandleable_error(renepay, flow, "Missing erring_index");
-		if (ret)
-			return ret;
-		goto done;
-	}
-	json_to_u32(buf, erridxtok, &erridx);
-	errchantok = json_get_member(buf, datatok, "erring_channel");
-	json_to_short_channel_id(buf, errchantok, &errscid);
-	
-	if (erridx<tal_count(flow->path_scids) 
-	    && !short_channel_id_eq(&errscid, &flow->path_scids[erridx]))
-		plugin_err(pay_plugin->plugin, "Erring channel %u/%zu was %s not %s (path %s)",
-			   erridx, tal_count(flow->path_scids),
-			   type_to_string(tmpctx, struct short_channel_id, &errscid),
-			   type_to_string(tmpctx, struct short_channel_id, &flow->path_scids[erridx]),
-			   flow_path_to_str(tmpctx, flow));
-
-	failcodetok = json_get_member(buf, datatok, "failcode");
-	json_to_u32(buf, failcodetok, &onionerr);
-
-	msgtok = json_get_member(buf, err, "message");
-	rawoniontok = json_get_member(buf, datatok, "raw_message");
-	
-	debug_paynote(p, "onion error %s from node #%u %s: %.*s",
-		onion_wire_name(onionerr),
-		erridx,
-		type_to_string(tmpctx, struct short_channel_id, &errscid),
-		msgtok->end - msgtok->start, buf + msgtok->start);
-		
-	plugin_log(pay_plugin->plugin,LOG_DBG,
-		"onion error %s from node #%u %s: %.*s",
-		onion_wire_name(onionerr),
-		erridx,
-		type_to_string(tmpctx, struct short_channel_id, &errscid),
-		msgtok->end - msgtok->start, buf + msgtok->start);
-	
-	uncertainty_network_channel_can_send(
-			pay_plugin->chan_extra_map,
-			flow,
-			erridx);
-	
-	switch ((enum onion_wire)onionerr) {
-	/* These definitely mean eliminate channel */
-	case WIRE_PERMANENT_CHANNEL_FAILURE:
-	case WIRE_REQUIRED_CHANNEL_FEATURE_MISSING:
-	/* FIXME: lnd returns this for disconnected peer, so don't disable perm! */
-	case WIRE_UNKNOWN_NEXT_PEER:
-	case WIRE_CHANNEL_DISABLED:
-	/* These mean node is weird, but we eliminate channel here too */
-	case WIRE_INVALID_REALM:
-	case WIRE_TEMPORARY_NODE_FAILURE:
-	case WIRE_PERMANENT_NODE_FAILURE:
-	case WIRE_REQUIRED_NODE_FEATURE_MISSING:
-	/* These shouldn't happen, but eliminate channel */
-	case WIRE_INVALID_ONION_VERSION:
-	case WIRE_INVALID_ONION_HMAC:
-	case WIRE_INVALID_ONION_KEY:
-	case WIRE_INVALID_ONION_PAYLOAD:
-	case WIRE_INVALID_ONION_BLINDING:
-	case WIRE_EXPIRY_TOO_FAR:
-		plugin_log(pay_plugin->plugin,LOG_DBG,"waitsendpay_failed: removing scid");
-		debug_paynote(p, "... so we're removing scid");
-		tal_arr_expand(&renepay->disabled, errscid);
-		goto done;
-
-	/* These can be fixed (maybe) by applying the included channel_update */
-	case WIRE_AMOUNT_BELOW_MINIMUM:
-	case WIRE_FEE_INSUFFICIENT:
-	case WIRE_INCORRECT_CLTV_EXPIRY:
-	case WIRE_EXPIRY_TOO_SOON:
-		plugin_log(pay_plugin->plugin,LOG_DBG,"waitsendpay_failed: apply channel_update");
-		/* FIXME: Check scid! */
-		// TODO(eduardo): check
-		const u8 *update = channel_update_from_onion_error(tmpctx, buf, rawoniontok);
-		if (update)
-			return submit_update(cmd, flow, update, errscid);
-
-		debug_paynote(p, "... missing an update, so we're removing scid");
-		tal_arr_expand(&renepay->disabled, errscid);
-		goto done;
-
-	/* Insufficient funds! */
-	case WIRE_TEMPORARY_CHANNEL_FAILURE: {
-		plugin_log(pay_plugin->plugin,LOG_DBG,"waitsendpay_failed: Insufficient funds!");
-		
-		chan_extra_cannot_send(p,pay_plugin->chan_extra_map,
-				       flow->path_scids[erridx],
-				       flow->path_dirs[erridx],
-				    /* This channel can't send all that was
-				     * commited in HTLCs.
-				     * Had we removed the commited amount then
-				     * we would have to put here flow->amounts[erridx]. */
-				       AMOUNT_MSAT(0));
-		goto done;
-	}
-
-	/* FIXME: We should probably pause until all parts returned, or at least
-	 * extend rexmit timer! */
-	case WIRE_MPP_TIMEOUT:
-		plugin_log(pay_plugin->plugin,LOG_DBG,"waitsendpay_failed: WIRE_MPP_TIMEOUT");
-		debug_paynote(p, "... will continue");
-		goto done;
-
-	/* These are from the final distination: fail */
-	case WIRE_INCORRECT_OR_UNKNOWN_PAYMENT_DETAILS:
-	case WIRE_FINAL_INCORRECT_CLTV_EXPIRY:
-	case WIRE_FINAL_INCORRECT_HTLC_AMOUNT:
-		plugin_log(pay_plugin->plugin,LOG_DBG,"waitsendpay_failed: final destination fail");
-		debug_paynote(p, "... fatal");
-		/* Inmediate failure. No information. */
-		payflow_fail(flow);
-		return renepay_fail(renepay,PAY_DESTINATION_PERM_FAIL,
-				    "Destination said %s: %.*s",
-				    onion_wire_name(onionerr),
-				    msgtok->end - msgtok->start,
-				    buf + msgtok->start);
-	}
-	/* Unknown response? */
-	if (erridx == tal_count(flow->path_nodes)) {
-		plugin_log(pay_plugin->plugin,LOG_DBG,"waitsendpay_failed: unknown error code %u: %.*s",
-				    onionerr,
-				    msgtok->end - msgtok->start,
-				    buf + msgtok->start);
-		debug_paynote(p, "... fatal");
-		payflow_fail(flow);
-		return renepay_fail(renepay,PAY_DESTINATION_PERM_FAIL,
-				    "Destination gave unknown error code %u: %.*s",
-				    onionerr,
-				    msgtok->end - msgtok->start,
-				    buf + msgtok->start);
-	}
-	debug_paynote(p, "... eliminating and continuing");
-	plugin_log(pay_plugin->plugin, LOG_BROKEN,
-		   "Node %s (%u/%zu) gave unknown error code %u",
-		   type_to_string(tmpctx, struct node_id,
-				  &flow->path_nodes[erridx]),
-		   erridx, tal_count(flow->path_nodes),
-		   onionerr);
-	tal_arr_expand(&renepay->disabled, errscid);
-
-done:
-	payflow_fail(flow);
-	return command_still_pending(cmd);
-}
-
 /* Once we've sent it, we immediate wait for reply. */
 static struct command_result *flow_sent(struct command *cmd,
 					const char *buf,
@@ -688,27 +396,7 @@ static struct command_result *flow_sent(struct command *cmd,
 					struct pay_flow *flow)
 {
 	plugin_log(pay_plugin->plugin,LOG_DBG,"calling %s",__PRETTY_FUNCTION__);
-	
-	struct payment *p = flow->payment;
-	struct out_req *req;
-	
-	// TODO(eduardo): instead of calling this waitsendpay rpc request, we
-	// could subscribe to sendpay_success and sendpay_failure. We should
-	// investigate if the received data contains payment_hash+groupid+partid
-	// so that we can connect it to a corresponding pay_flow.
-	req = jsonrpc_request_start(cmd->plugin,
-				    /* cmd = */ NULL /* allow us to receive
-				    responses after command finishes. */, 
-				    "waitsendpay",
-				    waitsendpay_succeeded,
-				    waitsendpay_failed,
-				    cast_const(struct pay_flow *, flow));
-	json_add_sha256(req->js, "payment_hash", &p->payment_hash);
-	json_add_u64(req->js, "groupid", p->groupid);
-	json_add_u64(req->js, "partid", flow->partid);
-	/* FIXME: We don't set timeout... */
-
-	return send_outreq(cmd->plugin, req);
+	return command_still_pending(cmd);
 }
 
 /* sendpay really only fails immediately in two ways:
@@ -723,6 +411,7 @@ static struct command_result *flow_sendpay_failed(struct command *cmd,
 	plugin_log(pay_plugin->plugin,LOG_DBG,"calling %s",__PRETTY_FUNCTION__);
 	
 	struct payment *p = flow->payment;
+	debug_assert(p);
 	struct renepay * renepay = p->renepay;
 	debug_assert(renepay);
 	
@@ -792,7 +481,7 @@ sendpay_flows(struct command *cmd,
 		
 		json_add_amount_msat_only(req->js, "amount_msat", p->amount);
 			
-		json_add_u64(req->js, "partid", flows[i]->partid);
+		json_add_u64(req->js, "partid", flows[i]->key.partid);
 		
 		json_add_u64(req->js, "groupid", p->groupid);
 		if (p->payment_metadata)
@@ -815,16 +504,14 @@ sendpay_flows(struct command *cmd,
 		 * channels from chan_extra_map. */
 		tal_steal(pay_plugin->ctx,flows[i]);
 		
+		/* Let's keep record of this flow. */
+		payflow_map_add(pay_plugin->payflow_map,flows[i]); 
+		
 		/* record these HTLC along the flow path */
 		commit_htlc_payflow(pay_plugin->chan_extra_map,flows[i]);
 		
 		/* Remove the HTLC from the chan_extra_map after finish. */
-		// TODO(eduardo): I think flows should be contained in a data
-		// structure, so this constructor should also remove them from
-		// that data structure.
-		tal_add_destructor2(flows[i],
-				    remove_htlc_payflow,
-				    pay_plugin->chan_extra_map);
+		tal_add_destructor(flows[i], destroy_payflow);
 		
 		send_outreq(cmd->plugin, req);
 	}
@@ -1541,7 +1228,347 @@ static struct command_result *json_pay(struct command *cmd,
 	// everything use TIMER_FORGET_SEC.
 }
 
-static struct command_result *json_shutdown(struct command *cmd,
+static void handle_sendpay_failure_renepay(
+		struct command *cmd,
+		const char *buf,
+		const jsmntok_t *result,
+		struct renepay *renepay,
+		struct pay_flow *flow)
+{
+	debug_assert(renepay);
+	debug_assert(flow);
+	struct payment *p = renepay->payment;
+	debug_assert(p);
+	
+	u64 errcode;
+	if (!json_to_u64(buf, json_get_member(buf, result, "code"), &errcode))
+	{
+		plugin_log(pay_plugin->plugin,LOG_BROKEN,
+			  "Failed to get code from sendpay_failure notification"
+			  ", received json: %.*s",
+			  json_tok_full_len(result),
+			  json_tok_full(buf,result));
+		return;
+	}
+	const jsmntok_t *msgtok = json_get_member(buf, result, "message");
+	const char *message;
+	if(msgtok)
+		message = tal_fmt(tmpctx,"%.*s",
+				  msgtok->end - msgtok->start, 
+				  buf + msgtok->start);
+	else
+		message = "[message missing from sendpay_failure notification]";
+	
+	switch(errcode)
+	{
+		case PAY_UNPARSEABLE_ONION:
+			debug_paynote(p, "Unparsable onion reply on route %s",
+				      flow_path_to_str(tmpctx, flow));
+			goto unhandleable;
+		case PAY_TRY_OTHER_ROUTE:
+			break;
+		case PAY_DESTINATION_PERM_FAIL:
+			renepay_fail(renepay,errcode,
+				     "Got a final failure from destination");
+			return;
+		default:
+			renepay_fail(renepay,errcode,
+				     "Unexpected errocode from sendpay_failure: %.*s",
+				     json_tok_full_len(result),
+				     json_tok_full(buf,result));
+			return;
+	}
+	
+	const jsmntok_t* datatok = json_get_member(buf, result, "data");
+	
+	if(!datatok)
+	{
+		plugin_err(pay_plugin->plugin,
+			  "Failed to get data from sendpay_failure notification"
+			  ", received json: %.*s",
+			  json_tok_full_len(result),
+			  json_tok_full(buf,result));
+	}
+	
+	
+	/* OK, we expect an onion error reply. */
+	u32 erridx;
+	const jsmntok_t * erridxtok = json_get_member(buf, datatok, "erring_index");
+	if (!erridxtok || !json_to_u32(buf, erridxtok, &erridx))
+	{
+		debug_paynote(p, "Missing erring_index reply on route %s",
+			      flow_path_to_str(tmpctx, flow));
+		plugin_log(pay_plugin->plugin,LOG_DBG,
+			   "%s (line %d) missing erring_index "
+			   "(erridxtok %p, erridx %d) on request %.*s",
+			   __PRETTY_FUNCTION__,__LINE__,
+			   erridxtok, erridx,
+			   json_tok_full_len(result),
+			   json_tok_full(buf,result));
+		goto unhandleable;
+	}
+	
+	struct short_channel_id errscid;	
+	const jsmntok_t *errchantok = json_get_member(buf, datatok, "erring_channel");
+	if(!errchantok || !json_to_short_channel_id(buf, errchantok, &errscid))
+	{
+		debug_paynote(p, "Missing erring_channel reply on route %s",
+			      flow_path_to_str(tmpctx, flow));
+		goto unhandleable;
+	}
+	
+	if (erridx<tal_count(flow->path_scids) 
+	    && !short_channel_id_eq(&errscid, &flow->path_scids[erridx]))
+	{
+		debug_paynote(p,
+			      "erring_index (%d) does not correspond"
+			      "to erring_channel (%s) on route %s",
+			      erridx,
+			      type_to_string(tmpctx,struct short_channel_id,&errscid),
+			      flow_path_to_str(tmpctx,flow));
+		goto unhandleable;
+	}
+	
+	u32 onionerr;
+	const jsmntok_t *failcodetok = json_get_member(buf, datatok, "failcode");
+	if(!failcodetok || !json_to_u32(buf, failcodetok, &onionerr))
+	{
+		// TODO(eduardo): I wonder which error code should I show the
+		// user in this case?
+		renepay_fail(renepay,LIGHTNINGD,
+			  "Failed to get failcode from sendpay_failure notification"
+			  ", received json: %.*s",
+			  json_tok_full_len(result),
+			  json_tok_full(buf,result));
+		return;
+	}
+	
+	debug_paynote(p,
+		"onion error %s from node #%u %s: %s",
+		onion_wire_name(onionerr),
+		erridx,
+		type_to_string(tmpctx, struct short_channel_id, &errscid),
+		message);
+	
+	const jsmntok_t *rawoniontok = json_get_member(buf, datatok, "raw_message");
+	if(!rawoniontok)
+		goto unhandleable;
+	
+	switch ((enum onion_wire)onionerr) {
+	/* These definitely mean eliminate channel */
+	case WIRE_PERMANENT_CHANNEL_FAILURE:
+	case WIRE_REQUIRED_CHANNEL_FEATURE_MISSING:
+	/* FIXME: lnd returns this for disconnected peer, so don't disable perm! */
+	case WIRE_UNKNOWN_NEXT_PEER:
+	case WIRE_CHANNEL_DISABLED:
+	/* These mean node is weird, but we eliminate channel here too */
+	case WIRE_INVALID_REALM:
+	case WIRE_TEMPORARY_NODE_FAILURE:
+	case WIRE_PERMANENT_NODE_FAILURE:
+	case WIRE_REQUIRED_NODE_FEATURE_MISSING:
+	/* These shouldn't happen, but eliminate channel */
+	case WIRE_INVALID_ONION_VERSION:
+	case WIRE_INVALID_ONION_HMAC:
+	case WIRE_INVALID_ONION_KEY:
+	case WIRE_INVALID_ONION_PAYLOAD:
+	case WIRE_INVALID_ONION_BLINDING:
+	case WIRE_EXPIRY_TOO_FAR:
+		debug_paynote(p, "we're removing scid %s",
+			      type_to_string(tmpctx,struct short_channel_id,&errscid));
+		tal_arr_expand(&renepay->disabled, errscid);
+		return;
+
+	/* These can be fixed (maybe) by applying the included channel_update */
+	case WIRE_AMOUNT_BELOW_MINIMUM:
+	case WIRE_FEE_INSUFFICIENT:
+	case WIRE_INCORRECT_CLTV_EXPIRY:
+	case WIRE_EXPIRY_TOO_SOON:
+		plugin_log(pay_plugin->plugin,LOG_DBG,"sendpay_failure, apply channel_update");
+		/* FIXME: Check scid! */
+		// TODO(eduardo): check
+		const u8 *update = channel_update_from_onion_error(tmpctx, buf, rawoniontok);
+		if (update)
+		{
+			submit_update(cmd, flow, update, errscid);
+			return;
+		}
+
+		debug_paynote(p, "missing an update, so we're removing scid %s",
+			      type_to_string(tmpctx,struct short_channel_id,&errscid));
+		tal_arr_expand(&renepay->disabled, errscid);
+		return;
+
+	case WIRE_TEMPORARY_CHANNEL_FAILURE:
+	case WIRE_MPP_TIMEOUT:
+		return;
+	
+	/* These are from the final distination: fail */
+	case WIRE_INCORRECT_OR_UNKNOWN_PAYMENT_DETAILS:
+	case WIRE_FINAL_INCORRECT_CLTV_EXPIRY:
+	case WIRE_FINAL_INCORRECT_HTLC_AMOUNT:
+		debug_paynote(p,"final destination failure");
+		renepay_fail(renepay,errcode,
+				    "Destination said %s: %s",
+				    onion_wire_name(onionerr),
+				    message);
+		return;
+	}
+	
+	debug_assert(erridx<=tal_count(flow->path_nodes));
+	
+	if(erridx == tal_count(flow->path_nodes))
+	{
+		debug_paynote(p,"unkown onion error code %u, fatal",
+			      onionerr);
+		renepay_fail(renepay,errcode,
+			     "Destination gave unknown error code %u: %s",
+			     onionerr,message);
+		return;
+	}else
+	{
+		debug_paynote(p,"unkown onion error code %u, removing scid %s",
+			      onionerr,
+			      type_to_string(tmpctx,struct short_channel_id,&errscid));
+		tal_arr_expand(&renepay->disabled, errscid);
+		return;
+	}
+	unhandleable:
+	// TODO(eduardo): check
+	handle_unhandleable_error(renepay, flow, "");
+}
+
+static void handle_sendpay_failure_flow(
+		struct command *cmd,
+		const char *buf,
+		const jsmntok_t *result,
+		struct pay_flow *flow)
+{
+	// TODO(eduardo): review with Rusty the level of severity of the
+	// different cases of error below.
+	debug_assert(flow);
+	
+	struct payment * const p = flow->payment;
+	payment_fail(p);
+	
+	u64 errcode;
+	if (!json_to_u64(buf, json_get_member(buf, result, "code"), &errcode))
+	{
+		plugin_log(pay_plugin->plugin,LOG_BROKEN,
+			  "Failed to get code from sendpay_failure notification"
+			  ", received json: %.*s",
+			  json_tok_full_len(result),
+			  json_tok_full(buf,result));
+		return;
+	}
+	const jsmntok_t *msgtok = json_get_member(buf, result, "message");
+	const char *message;
+	if(msgtok)
+		message = tal_fmt(tmpctx,"%.*s",
+				  msgtok->end - msgtok->start, 
+				  buf + msgtok->start);
+	else
+		message = "[message missing from sendpay_failure notification]";
+		
+	if(errcode!=PAY_TRY_OTHER_ROUTE)
+		return;
+
+	const jsmntok_t* datatok = json_get_member(buf, result, "data");
+	if(!datatok)
+	{
+		plugin_err(pay_plugin->plugin,
+			  "Failed to get data from sendpay_failure notification"
+			  ", received json: %.*s",
+			  json_tok_full_len(result),
+			  json_tok_full(buf,result));
+	}
+
+	/* OK, we expect an onion error reply. */
+	u32 erridx;
+	const jsmntok_t * erridxtok = json_get_member(buf, datatok, "erring_index");
+	if (!erridxtok || !json_to_u32(buf, erridxtok, &erridx))
+	{
+		plugin_log(pay_plugin->plugin,LOG_BROKEN,
+			  "Failed to get erring_index from sendpay_failure notification"
+			  ", received json: %.*s",
+			  json_tok_full_len(result),
+			  json_tok_full(buf,result));
+		return;
+	}
+	
+	struct short_channel_id errscid;
+	const jsmntok_t *errchantok = json_get_member(buf, datatok, "erring_channel");
+	if(!errchantok || !json_to_short_channel_id(buf, errchantok, &errscid))
+	{
+		plugin_log(pay_plugin->plugin,LOG_BROKEN,
+			  "Failed to get erring_channel from sendpay_failure notification"
+			  ", received json: %.*s",
+			  json_tok_full_len(result),
+			  json_tok_full(buf,result));
+		return;
+	}
+	
+	if (erridx<tal_count(flow->path_scids) 
+	    && !short_channel_id_eq(&errscid, &flow->path_scids[erridx]))
+	{
+		plugin_err(pay_plugin->plugin, 
+			   "Erring channel %u/%zu was %s not %s (path %s)",
+			   erridx, tal_count(flow->path_scids),
+			   type_to_string(tmpctx, 
+			   		  struct short_channel_id, 
+					  &errscid),
+			   type_to_string(tmpctx, 
+			   		  struct short_channel_id, 
+			   		  &flow->path_scids[erridx]),
+			   flow_path_to_str(tmpctx, flow));
+		return;
+	}
+
+
+	u32 onionerr;
+	const jsmntok_t *failcodetok = json_get_member(buf, datatok, "failcode");
+	if(!failcodetok || !json_to_u32(buf, failcodetok, &onionerr))
+	{
+		plugin_log(pay_plugin->plugin,LOG_BROKEN,
+			  "Failed to get failcode from sendpay_failure notification"
+			  ", received json: %.*s",
+			  json_tok_full_len(result),
+			  json_tok_full(buf,result));
+		return;
+	
+	}
+	
+	plugin_log(pay_plugin->plugin,LOG_UNUSUAL,
+		"onion error %s from node #%u %s: "
+		"%s",
+		onion_wire_name(onionerr),
+		erridx,
+		type_to_string(tmpctx, struct short_channel_id, &errscid),
+		message);
+	
+	/* we know that all channels before erridx where able to commit to this payment */
+	uncertainty_network_channel_can_send(
+			pay_plugin->chan_extra_map,
+			flow,
+			erridx);
+	
+	/* Insufficient funds! */
+	if((enum onion_wire)onionerr == WIRE_TEMPORARY_CHANNEL_FAILURE)
+	{	
+		plugin_log(pay_plugin->plugin,LOG_DBG,
+			   "sendpay_failure says insufficient funds!");
+		
+		chan_extra_cannot_send(p,pay_plugin->chan_extra_map,
+				       flow->path_scids[erridx],
+				       flow->path_dirs[erridx],
+				    /* This channel can't send all that was
+				     * commited in HTLCs.
+				     * Had we removed the commited amount then
+				     * we would have to put here flow->amounts[erridx]. */
+				       AMOUNT_MSAT(0));
+	}
+}
+
+static struct command_result *notification_shutdown(struct command *cmd,
 					         const char *buf,
 					         const jsmntok_t *params)
 {
@@ -1553,6 +1580,159 @@ static struct command_result *json_shutdown(struct command *cmd,
 	pay_plugin->ctx = tal_free(pay_plugin->ctx);
 	return notification_handled(cmd);
 }
+static struct command_result *notification_sendpay_success(
+		struct command *cmd,
+		const char *buf,
+		const jsmntok_t *params)
+{
+	struct pay_flow *flow = NULL;
+	const jsmntok_t *resulttok = json_get_member(buf,params,"sendpay_success");
+	if(!resulttok)
+		debug_err("Failed to get result from sendpay_success notification"
+			  ", received json: %.*s",
+			  json_tok_full_len(params),
+			  json_tok_full(buf,params));
+	
+	// 1. generate the key of this payflow
+	struct payflow_key key;
+	key.payment_hash = tal(tmpctx,struct sha256);
+	
+	const jsmntok_t *parttok = json_get_member(buf,resulttok,"partid");
+	if(!parttok || !json_to_u64(buf,parttok,&key.partid))
+	{
+		// No partid, is this a single-path payment?
+		key.partid = 0;
+		// debug_err("Failed to get partid from sendpay_success notification"
+		// 	  ", received json: %.*s",
+		// 	  json_tok_full_len(params),
+		// 	  json_tok_full(buf,params));
+	}
+	const jsmntok_t *grouptok = json_get_member(buf,resulttok,"groupid");
+	if(!grouptok || !json_to_u64(buf,grouptok,&key.groupid))
+		debug_err("Failed to get groupid from sendpay_success notification"
+			  ", received json: %.*s",
+			  json_tok_full_len(params),
+			  json_tok_full(buf,params));
+	
+	const jsmntok_t *hashtok = json_get_member(buf,resulttok,"payment_hash");
+	if(!hashtok || !json_to_sha256(buf,hashtok,key.payment_hash))
+		debug_err("Failed to get payment_hash from sendpay_success notification"
+			  ", received json: %.*s",
+			  json_tok_full_len(params),
+			  json_tok_full(buf,params));
+	
+	plugin_log(pay_plugin->plugin,LOG_DBG,
+		"I received a sendpay_success with key %s",
+		fmt_payflow_key(tmpctx,&key));		
+	
+	// 2. is this payflow recorded in renepay?
+	flow = payflow_map_get(pay_plugin->payflow_map,key);
+	if(!flow)
+	{
+		plugin_log(pay_plugin->plugin,LOG_DBG,
+			"sendpay_success does not correspond to a renepay attempt, %s",
+			fmt_payflow_key(tmpctx,&key));		
+		goto done;
+	}
+	
+	// 3. mark as success
+	struct payment * const p = flow->payment;
+	debug_assert(p);
+	
+	payment_success(p);
+	
+	const jsmntok_t *preimagetok
+		= json_get_member(buf, resulttok, "payment_preimage");
+	struct preimage preimage;
+	
+	if (!preimagetok || !json_to_preimage(buf, preimagetok,&preimage))
+		debug_err("Failed to get payment_preimage from sendpay_success notification"
+			  ", received json: %.*s",
+			  json_tok_full_len(params),
+			  json_tok_full(buf,params));
+	
+	p->preimage = tal_dup_or_null(p,struct preimage,&preimage);
+	
+	// 4. update information and release pending HTLCs
+	uncertainty_network_flow_success(pay_plugin->chan_extra_map,flow);
+	
+	done:
+	tal_free(flow); 
+	return notification_handled(cmd);
+}
+static struct command_result *notification_sendpay_failure(
+		struct command *cmd,
+		const char *buf,
+		const jsmntok_t *params)
+{
+	struct pay_flow *flow = NULL;
+	
+	const jsmntok_t *resulttok = json_get_member(buf,params,"sendpay_failure");
+	if(!resulttok)
+		debug_err("Failed to get result from sendpay_failure notification"
+			  ", received json: %.*s",
+			  json_tok_full_len(params),
+			  json_tok_full(buf,params));
+	
+	const jsmntok_t *datatok = json_get_member(buf,resulttok,"data");
+	if(!datatok)
+		debug_err("Failed to get data from sendpay_failure notification"
+			  ", received json: %.*s",
+			  json_tok_full_len(params),
+			  json_tok_full(buf,params));
+	
+	
+	// 1. generate the key of this payflow
+	struct payflow_key key;
+	key.payment_hash = tal(tmpctx,struct sha256);
+	
+	const jsmntok_t *parttok = json_get_member(buf,datatok,"partid");
+	if(!parttok || !json_to_u64(buf,parttok,&key.partid))
+	{
+		// No partid, is this a single-path payment?
+		key.partid = 0;
+	}
+	const jsmntok_t *grouptok = json_get_member(buf,datatok,"groupid");
+	if(!grouptok || !json_to_u64(buf,grouptok,&key.groupid))
+		debug_err("Failed to get groupid from sendpay_failure notification"
+			  ", received json: %.*s",
+			  json_tok_full_len(params),
+			  json_tok_full(buf,params));
+	
+	const jsmntok_t *hashtok = json_get_member(buf,datatok,"payment_hash");
+	if(!hashtok || !json_to_sha256(buf,hashtok,key.payment_hash))
+		debug_err("Failed to get payment_hash from sendpay_failure notification"
+			  ", received json: %.*s",
+			  json_tok_full_len(params),
+			  json_tok_full(buf,params));
+	
+	plugin_log(pay_plugin->plugin,LOG_DBG,
+		"I received a sendpay_failure with key %s",
+		fmt_payflow_key(tmpctx,&key));		
+	
+	// 2. is this payflow recorded in renepay?
+	flow = payflow_map_get(pay_plugin->payflow_map,key);
+	if(!flow)
+	{
+		plugin_log(pay_plugin->plugin,LOG_DBG,
+			"sendpay_failure does not correspond to a renepay attempt, %s",
+			fmt_payflow_key(tmpctx,&key));		
+		goto done;
+	}
+	
+	// 3. process failure
+	handle_sendpay_failure_flow(cmd,buf,resulttok,flow);
+	
+	// there is possibly a pending renepay command for this flow
+	struct renepay * const renepay = flow->payment->renepay;
+	
+	if(renepay)
+		handle_sendpay_failure_renepay(cmd,buf,resulttok,renepay,flow);
+	
+	done:
+	if(flow) payflow_fail(flow);
+	return notification_handled(cmd);
+}
 
 static const struct plugin_command commands[] = {
 	{
@@ -1574,7 +1754,15 @@ static const struct plugin_command commands[] = {
 static const struct plugin_notification notifications[] = {
 	{
 		"shutdown",
-		json_shutdown,
+		notification_shutdown,
+	},
+	{
+		"sendpay_success",
+		notification_sendpay_success,
+	},
+	{
+		"sendpay_failure",
+		notification_sendpay_failure,
 	}
 };
 
diff --git a/plugins/renepay/pay.h b/plugins/renepay/pay.h
index 1a2b79fa5a29..82b48bcde90d 100644
--- a/plugins/renepay/pay.h
+++ b/plugins/renepay/pay.h
@@ -10,8 +10,6 @@
 
 // TODO(eduardo): renepaystatus should be similar to paystatus
 
-// TODO(eduardo): notice that payment must outlive the command!
-
 // TODO(eduardo): MCF should consider pending HTLCs occupy some capacity in the
 // routing channels.
 
@@ -74,6 +72,9 @@ struct pay_plugin {
 
 	/* Per-channel metadata: some persists between payments */
 	struct chan_extra_map *chan_extra_map;
+	
+	/* Pending senpays. */
+	struct payflow_map * payflow_map;
 
 	bool debug_mcf; 
 	bool debug_payflow; 
diff --git a/plugins/renepay/pay_flow.c b/plugins/renepay/pay_flow.c
index c8d3de7dc888..5bce761da538 100644
--- a/plugins/renepay/pay_flow.c
+++ b/plugins/renepay/pay_flow.c
@@ -217,8 +217,12 @@ static struct pay_flow **flows_to_pay_flows(struct payment *payment,
 
 		plen = tal_count(f->path);
 		pay_flows[i] = pf;
+		
 		pf->payment = payment;
-		pf->partid = (*next_partid)++;
+		pf->key.partid = (*next_partid)++;
+		pf->key.groupid = payment->groupid;
+		pf->key.payment_hash = &payment->payment_hash;
+		
 		/* Convert gossmap_chan into scids and nodes */
 		pf->path_scids = tal_arr(pf, struct short_channel_id, plen);
 		pf->path_nodes = tal_arr(pf, struct node_id, plen);
@@ -339,6 +343,8 @@ struct pay_flow **get_payflows(struct renepay * renepay,
 			       bool is_entire_payment,
 			       char const ** err_msg)
 {
+	*err_msg = tal_fmt(tmpctx,"[no error]");
+	
 	struct payment * p = renepay->payment;
 	bitmap *disabled;
 	struct pay_flow **pay_flows;
@@ -352,7 +358,7 @@ struct pay_flow **get_payflows(struct renepay * renepay,
 		goto fail;
 	}
 	dst = gossmap_find_node(pay_plugin->gossmap, &p->destination);
-	if (!src) {
+	if (!dst) {
 		debug_paynote(p, "No trace of destination in network gossip");
 		*err_msg = tal_fmt(tmpctx,"Destination is unreacheable in the network gossip.");
 		goto fail;
@@ -375,10 +381,13 @@ struct pay_flow **get_payflows(struct renepay * renepay,
 				p->base_fee_penalty,
 				p->prob_cost_factor);
 		if (!flows) {
-			debug_paynote(p, "Failed to find any paths for %s",
-				type_to_string(tmpctx,
-					       struct amount_msat,
-					       &amount));
+			debug_paynote(p,
+				      "minflow couldn't find a feasible flow for %s",
+				      type_to_string(tmpctx,struct amount_msat,&amount));
+				      
+			*err_msg = tal_fmt(tmpctx,
+					   "minflow couldn't find a feasible flow for %s",
+					   type_to_string(tmpctx,struct amount_msat,&amount));
 			goto fail;
 		}
 
@@ -539,8 +548,8 @@ const char* fmt_payflows(const tal_t *ctx,
 }
 
 void remove_htlc_payflow(
-		struct pay_flow *flow,
-		struct chan_extra_map *chan_extra_map)
+		struct chan_extra_map *chan_extra_map,
+		struct pay_flow *flow)
 {
 	for (size_t i = 0; i < tal_count(flow->path_scids); i++) {
 		struct chan_extra_half *h = get_chan_extra_half_by_scid(
@@ -607,17 +616,18 @@ struct amount_msat payflow_delivered(const struct pay_flow *flow)
 	return flow->amounts[tal_count(flow->amounts)-1];
 }
 
-void payflow_fail(struct pay_flow *flow)
+struct pay_flow* payflow_fail(struct pay_flow *flow)
 {
 	debug_assert(flow);
 	struct payment * p = flow->payment;
+	debug_assert(p);
 	
 	payment_fail(p);
 	amount_msat_reduce(&p->total_delivering, payflow_delivered(flow));
 	amount_msat_reduce(&p->total_sent, flow->amounts[0]);
 
 	/* Release the HTLCs in the uncertainty_network. */
-	tal_free(flow);
+	return tal_free(flow);
 }
 
 
diff --git a/plugins/renepay/pay_flow.h b/plugins/renepay/pay_flow.h
index 26db4180a34c..0cf58e0bd712 100644
--- a/plugins/renepay/pay_flow.h
+++ b/plugins/renepay/pay_flow.h
@@ -1,20 +1,30 @@
 #ifndef LIGHTNING_PLUGINS_RENEPAY_PAY_FLOW_H
 #define LIGHTNING_PLUGINS_RENEPAY_PAY_FLOW_H
 #include <ccan/short_types/short_types.h>
+#include <ccan/ccan/tal/str/str.h>
 #include <plugins/renepay/payment.h>
 #include <plugins/renepay/debug.h>
+#include <plugins/renepay/flow.h>
+#include <common/utils.h>
 
 /* This is like a struct flow, but independent of gossmap, and contains
  * all we need to actually send the part payment. */
 struct pay_flow {
 	/* So we can be an independent object for callbacks. */
 	struct payment * payment;
-
-	/* This flow belongs to some attempt. */
+	
+	// TODO(eduardo): remove this, unnecessary
 	int attempt;
+
+	/* Information to link this flow to a unique sendpay. */
+	struct payflow_key
+	{
+		// TODO(eduardo): pointer or value?
+		struct sha256 *payment_hash;
+		u64 groupid;
+		u64 partid;
+	} key;
 	
-	/* Part id for RPC interface */
-	u64 partid;
 	/* The series of channels and nodes to traverse. */
 	struct short_channel_id *path_scids;
 	struct node_id *path_nodes;
@@ -27,6 +37,49 @@ struct pay_flow {
 	double success_prob;
 };
 
+static inline struct payflow_key 
+payflow_key(struct sha256 *hash, u64 groupid, u64 partid)
+{
+	struct payflow_key k= {hash,groupid,partid};
+	return k;
+}
+
+static inline const char* fmt_payflow_key(
+		const tal_t *ctx,
+		const struct payflow_key * k)
+{
+	char *str = tal_fmt(
+		ctx,
+		"key: groupid=%ld, partid=%ld, payment_hash=%s",
+		k->groupid,k->partid,
+		type_to_string(ctx,struct sha256,k->payment_hash));
+	return str;
+}
+
+
+static inline const struct payflow_key 
+payflow_get_key(const struct pay_flow * pf)
+{
+	return pf->key;
+}
+
+static inline size_t payflow_key_hash(const struct payflow_key k)
+{
+	return k.payment_hash->u.u32[0] ^ (k.groupid << 32) ^ k.partid;
+}
+
+static inline bool payflow_key_equal(struct pay_flow const *pf,
+				      const struct payflow_key k)
+{
+	return pf->key.partid==k.partid && pf->key.groupid==k.groupid 
+		&& sha256_eq(pf->key.payment_hash,k.payment_hash); 
+}
+
+HTABLE_DEFINE_TYPE(struct pay_flow,
+		   payflow_get_key, payflow_key_hash, payflow_key_equal,
+		   payflow_map);
+
+
 struct pay_flow **get_payflows(struct renepay * renepay,
 			       struct amount_msat amount,
 			       struct amount_msat feebudget,
@@ -39,8 +92,8 @@ void commit_htlc_payflow(
 		const struct pay_flow *flow);
 
 void remove_htlc_payflow(
-		struct pay_flow *flow,
-		struct chan_extra_map *chan_extra_map);
+		struct chan_extra_map *chan_extra_map,
+		struct pay_flow *flow);
 
 const char *flow_path_to_str(const tal_t *ctx, const struct pay_flow *flow);
 
@@ -53,6 +106,6 @@ struct amount_msat payflow_delivered(const struct pay_flow *flow);
 /* Removes amounts from payment and frees flow pointer. 
  * A possible destructor for flow would remove HTLCs from the
  * uncertainty_network and remove the flow from any data structure. */
-void payflow_fail(struct pay_flow *flow);
+struct pay_flow* payflow_fail(struct pay_flow *flow);
 
 #endif /* LIGHTNING_PLUGINS_RENEPAY_PAY_FLOW_H */
diff --git a/plugins/renepay/payment.c b/plugins/renepay/payment.c
index 3528366cc810..832d1015ef04 100644
--- a/plugins/renepay/payment.c
+++ b/plugins/renepay/payment.c
@@ -207,6 +207,10 @@ void renepay_cleanup(
 	debug_info("calling %s",__PRETTY_FUNCTION__);
 	/* Always remove our local mods (routehints) so others can use
 	 * gossmap. We do this only after the payment completes. */
+	// TODO(eduardo): it can happen that local_gossmods removed below
+	// contained a set of channels for which there is information in the
+	// uncertainty network (chan_extra_map) and that are part of some pending
+	// payflow (payflow_map). Handle this situation.
 	if(renepay->localmods_applied)
 		gossmap_remove_localmods(gossmap,
 					 renepay->local_gossmods);
diff --git a/plugins/renepay/test/run-payflow_map.c b/plugins/renepay/test/run-payflow_map.c
new file mode 100644
index 000000000000..c0a65904d55a
--- /dev/null
+++ b/plugins/renepay/test/run-payflow_map.c
@@ -0,0 +1,96 @@
+/* Eduardo: testing payflow_map.
+ * */
+
+#include "config.h"
+#include <ccan/array_size/array_size.h>
+#include <ccan/read_write_all/read_write_all.h>
+#include <common/bigsize.h>
+#include <common/channel_id.h>
+#include <common/gossip_store.h>
+#include <common/node_id.h>
+#include <common/setup.h>
+#include <common/type_to_string.h>
+#include <common/wireaddr.h>
+#include <stdio.h>
+#include <assert.h>
+
+#include <bitcoin/short_channel_id.h>
+#include <ccan/htable/htable_type.h>
+
+#define RENEPAY_UNITTEST
+#include <plugins/renepay/pay_flow.h>
+
+static void destroy_payflow(
+		struct pay_flow *p,
+		struct payflow_map * map)
+{
+	printf("calling %s with  %s\n",
+		__PRETTY_FUNCTION__,
+		fmt_payflow_key(tmpctx,&p->key));
+	payflow_map_del(map, p);
+}
+static struct pay_flow* new_payflow(
+		const tal_t *ctx,
+		struct sha256 * payment_hash,
+		u64 gid,
+		u64 pid)
+{
+	struct pay_flow *p = tal(ctx,struct pay_flow);
+	
+	p->payment=NULL;
+	p->key.payment_hash=payment_hash;
+	p->key.groupid = gid;
+	p->key.partid = pid;
+	
+	return p;
+}
+
+static void valgrind_ok1(void)
+{
+	const char seed[] = "seed";
+	struct sha256 hash;
+	
+	sha256(&hash,seed,sizeof(seed));
+	
+	tal_t *this_ctx = tal(tmpctx,tal_t);
+	
+	struct payflow_map *map
+		= tal(this_ctx, struct payflow_map);
+	
+	payflow_map_init(map);
+	
+	{
+		tal_t *local_ctx = tal(this_ctx,tal_t);
+		
+		struct pay_flow *p1 = new_payflow(local_ctx,
+						  &hash,1,1);
+		struct pay_flow *p2 = new_payflow(local_ctx,
+						  &hash,2,3);
+		
+		printf("key1 = %s\n",fmt_payflow_key(local_ctx,&p1->key));
+		printf("key1 = %s\n",fmt_payflow_key(local_ctx,&p2->key));
+		printf("key hash 1 = %ld\n",payflow_key_hash(p1->key));
+		printf("key hash 2 = %ld\n",payflow_key_hash(p2->key));
+		
+		payflow_map_add(map,p1); tal_add_destructor2(p1,destroy_payflow,map);
+		payflow_map_add(map,p2); tal_add_destructor2(p2,destroy_payflow,map);
+		
+		struct pay_flow *q1 = payflow_map_get(map,payflow_key(&hash,1,1));
+		struct pay_flow *q2 = payflow_map_get(map,payflow_key(&hash,2,3));
+	
+		assert(payflow_key_hash(q1->key)==payflow_key_hash(p1->key));
+		assert(payflow_key_hash(q2->key)==payflow_key_hash(p2->key));
+		
+		tal_free(local_ctx);
+	}
+	
+	tal_free(this_ctx);
+
+}
+int main(int argc, char *argv[])
+{
+	common_setup(argv[0]);
+	valgrind_ok1();
+	common_shutdown();
+}
+
diff --git a/plugins/renepay/uncertainty_network.c b/plugins/renepay/uncertainty_network.c
index 465bc485a680..0c1cf3be4cb7 100644
--- a/plugins/renepay/uncertainty_network.c
+++ b/plugins/renepay/uncertainty_network.c
@@ -155,7 +155,10 @@ void uncertainty_network_update(
 				__LINE__);
 		}
 		chan_extra_map_del(chan_extra_map, ce);
-		tal_free(ce);
+		tal_free(ce); 
+		// TODO(eduardo): if you had added a destructor to ce, you could have removed
+		// the ce from the map automatically.
+		
 	}
 	
 	// For each channel in the gossmap, create a extra data in
diff --git a/tests/test_renepay.py b/tests/test_renepay.py
index a1c3ccd0c027..138841a4d45b 100644
--- a/tests/test_renepay.py
+++ b/tests/test_renepay.py
@@ -54,6 +54,37 @@ def test_mpp(node_factory):
     assert details['amount_msat'] == send_amount
     assert details['destination'] == l6.info['id']
 
+def test_errors(node_factory):
+    opts = [
+        {'disable-mpp': None, 'fee-base':0, 'fee-per-satoshi':0},
+    ]
+    l1, l2, l3, l4, l5,l6 = node_factory.get_nodes(6, opts=opts*6)
+    send_amount = Millisatoshi('21sat')
+    inv = l6.rpc.invoice(send_amount, 'test_renepay', 'description')['bolt11']
+    
+    failmsg = r'We don\'t have any channels'
+    with pytest.raises(RpcError, match=failmsg) as err:
+        l1.rpc.call('renepay',{'invstring':inv})
+    node_factory.join_nodes([l1, l2, l4],
+        wait_for_announce=True,fundamount=1000000)
+    node_factory.join_nodes([l1, l3, l5], 
+        wait_for_announce=True,fundamount=1000000)
+    
+    failmsg = r'Destination is unreacheable in the network gossip.'
+    with pytest.raises(RpcError, match=failmsg) as err:
+        l1.rpc.call('renepay',{'invstring':inv})
+    
+    node_factory.join_nodes([l4,l6],
+        wait_for_announce=True,fundamount=1000000)
+    node_factory.join_nodes([l5,l6], 
+        wait_for_announce=True,fundamount=1000000)
+    
+    details = l1.rpc.call('renepay',{'invstring':inv, 'use_shadow':False})
+    assert details['status'] == 'complete'
+    assert details['amount_msat'] == send_amount
+    assert details['destination'] == l6.info['id']
+		
+
 ### SAME TESTS USED IN THE STANDARD PAY COMMAND ###
 
 @pytest.mark.developer("needs to deactivate shadow routing")
@@ -106,6 +137,9 @@ def test_pay(node_factory):
     # Test listsendpays indexed by bolt11.
     payments = l1.rpc.listsendpays(inv)['payments']
     assert len(payments) == 1 and payments[0]['payment_preimage'] == preimage
+    
+    # let's buy some time to let bkpr receive coin_movement notification
+    ret = l1.rpc.call('renepaystatus')
 
     # Check channels apy summary view of channel activity
     apys_1 = l1.rpc.bkpr_channelsapy()['channels_apy']
@@ -181,5258 +215,13 @@ def test_limits(node_factory):
     
     inv2 = l6.rpc.invoice("800000sat", "inv2", 'description')
     failmsg = r'Probability is too small'
-    # Delay too high.
     with pytest.raises(RpcError, match=failmsg) as err:
         l1.rpc.call('renepay', {'invstring': inv2['bolt11'],'min_prob_success': '0.5'})
     assert err.value.error['code'] == PAY_ROUTE_NOT_FOUND
     
-    # # if we try again we can finish this payment
-    # l1.rpc.call('renepay', {'invstring': inv2['bolt11']})
-    # invoice = only_one(l6.rpc.listinvoices('inv2')['invoices'])
-    # assert isinstance(invoice['amount_received_msat'], Millisatoshi)
-    # assert invoice['amount_received_msat'] >= Millisatoshi('800000sat')
+    # if we try again we can finish this payment
+    l1.rpc.call('renepay', {'invstring': inv2['bolt11'],'min_prob_success':0})
+    invoice = only_one(l6.rpc.listinvoices('inv2')['invoices'])
+    assert isinstance(invoice['amount_received_msat'], Millisatoshi)
+    assert invoice['amount_received_msat'] >= Millisatoshi('800000sat')
 
-# @pytest.mark.developer("Gossip is too slow without developer")
-# def test_pay_exclude_node(node_factory, bitcoind):
-#     """Test excluding the node if there's the NODE-level error in the failure_code
-#     """
-#     # FIXME: Remove our reliance on HTLCs failing on startup and the need for
-#     #        this plugin
-#     opts = [
-#         {'disable-mpp': None},
-#         {'plugin': os.path.join(os.getcwd(), 'tests/plugins/fail_htlcs.py')},
-#         {},
-#         {'fee-base': 100, 'fee-per-satoshi': 1000},
-#         {}
-#     ]
-#     l1, l2, l3, l4, l5 = node_factory.get_nodes(5, opts=opts)
-#     node_factory.join_nodes([l1, l2, l3], wait_for_announce=True)
-#     amount = 10**8
-# 
-#     inv = l3.rpc.invoice(amount, "test1", 'description')['bolt11']
-#     with pytest.raises(RpcError):
-#         l1.rpc.pay(inv)
-# 
-#     # It should have retried (once without routehint, too)
-#     status = l1.rpc.call('paystatus', {'bolt11': inv})['pay'][0]['attempts']
-# 
-#     # Excludes channel, then ignores routehint which includes that, then
-#     # it excludes other channel.
-#     assert len(status) == 2
-#     assert status[0]['strategy'] == "Initial attempt"
-#     assert status[0]['failure']['data']['failcodename'] == 'WIRE_TEMPORARY_NODE_FAILURE'
-#     assert 'failure' in status[1]
-# 
-#     # Get a fresh invoice, but do it before other routes exist, so routehint
-#     # will be via l2.
-#     inv = l3.rpc.invoice(amount, "test2", 'description')['bolt11']
-#     assert only_one(l1.rpc.decodepay(inv)['routes'])[0]['pubkey'] == l2.info['id']
-# 
-#     # l1->l4->l5->l3 is the longer route. This makes sure this route won't be
-#     # tried for the first pay attempt. Just to be sure we also raise the fees
-#     # that l4 leverages.
-#     l1.rpc.connect(l4.info['id'], 'localhost', l4.port)
-#     l4.rpc.connect(l5.info['id'], 'localhost', l5.port)
-#     l5.rpc.connect(l3.info['id'], 'localhost', l3.port)
-#     scid14, _ = l1.fundchannel(l4, 10**6, wait_for_active=False)
-#     scid45, _ = l4.fundchannel(l5, 10**6, wait_for_active=False)
-#     scid53, _ = l5.fundchannel(l3, 10**6, wait_for_active=False)
-#     mine_funding_to_announce(bitcoind, [l1, l2, l3, l4, l5])
-# 
-#     l1.daemon.wait_for_logs([r'update for channel {}/0 now ACTIVE'
-#                              .format(scid14),
-#                              r'update for channel {}/1 now ACTIVE'
-#                              .format(scid14),
-#                              r'update for channel {}/0 now ACTIVE'
-#                              .format(scid45),
-#                              r'update for channel {}/1 now ACTIVE'
-#                              .format(scid45),
-#                              r'update for channel {}/0 now ACTIVE'
-#                              .format(scid53),
-#                              r'update for channel {}/1 now ACTIVE'
-#                              .format(scid53)])
-# 
-#     # This `pay` will work
-#     l1.rpc.pay(inv)
-# 
-#     # It should have retried (once without routehint, too)
-#     status = l1.rpc.call('paystatus', {'bolt11': inv})['pay'][0]['attempts']
-# 
-#     # Excludes channel, then ignores routehint which includes that, then
-#     # it excludes other channel.
-#     assert len(status) == 2
-#     assert status[0]['strategy'] == "Initial attempt"
-#     assert status[0]['failure']['data']['failcodename'] == 'WIRE_TEMPORARY_NODE_FAILURE'
-#     assert 'success' in status[1]
-# 
-# 
-# def test_pay0(node_factory):
-#     """Test paying 0 amount
-#     """
-#     l1, l2 = node_factory.line_graph(2)
-#     chanid = l1.get_channel_scid(l2)
-# 
-#     # Get any-amount invoice
-#     inv = l2.rpc.invoice("any", "any", 'description')
-#     rhash = inv['payment_hash']
-# 
-#     routestep = {
-#         'amount_msat': 0,
-#         'id': l2.info['id'],
-#         'delay': 10,
-#         'channel': chanid
-#     }
-# 
-#     # Amount must be nonzero!
-#     l1.rpc.sendpay([routestep], rhash, payment_secret=inv['payment_secret'])
-#     with pytest.raises(RpcError, match=r'WIRE_AMOUNT_BELOW_MINIMUM'):
-#         l1.rpc.waitsendpay(rhash)
-# 
-# 
-# @pytest.mark.developer("needs DEVELOPER=1")
-# def test_pay_disconnect(node_factory, bitcoind):
-#     """If the remote node has disconnected, we fail payment, but can try again when it reconnects"""
-#     l1, l2 = node_factory.line_graph(2, opts={'dev-max-fee-multiplier': 5,
-#                                               'may_reconnect': True,
-#                                               'allow_warning': True})
-# 
-#     # Dummy payment to kick off update_fee messages
-#     l1.pay(l2, 1000)
-# 
-#     inv = l2.rpc.invoice(123000, 'test_pay_disconnect', 'description')
-#     rhash = inv['payment_hash']
-# 
-#     wait_for(lambda: [c['active'] for c in l1.rpc.listchannels()['channels']] == [True, True])
-# 
-#     # Can't use `pay` since that'd notice that we can't route, due to disabling channel_update
-#     route = l1.rpc.getroute(l2.info['id'], 123000, 1)["route"]
-# 
-#     l2.stop()
-#     # Make sure channeld has exited!
-#     wait_for(lambda: 'owner' not in only_one(l1.rpc.listpeerchannels(l2.info['id'])['channels']))
-# 
-#     # Can't pay while its offline.
-#     with pytest.raises(RpcError, match=r'failed: WIRE_TEMPORARY_CHANNEL_FAILURE \(First peer not ready\)'):
-#         l1.rpc.sendpay(route, rhash, payment_secret=inv['payment_secret'])
-# 
-#     l2.start()
-#     l1.daemon.wait_for_log('peer_out WIRE_CHANNEL_REESTABLISH')
-# 
-#     # Make l2 upset by asking for crazy fee.
-#     l1.set_feerates((10**6, 10**6, 10**6, 10**6), False)
-# 
-#     # Wait for l1 notice
-#     l1.daemon.wait_for_log(r'Peer transient failure in CHANNELD_NORMAL: channeld WARNING: .*: update_fee \d+ outside range 1875-75000')
-# 
-#     # Make l2 fail hard.
-#     l2.rpc.close(l1.info['id'], unilateraltimeout=1)
-#     l2.daemon.wait_for_log('sendrawtx exit')
-#     bitcoind.generate_block(1, wait_for_mempool=1)
-#     sync_blockheight(bitcoind, [l1, l2])
-# 
-#     # Should fail due to permenant channel fail
-#     with pytest.raises(RpcError, match=r'WIRE_UNKNOWN_NEXT_PEER'):
-#         l1.rpc.sendpay(route, rhash, payment_secret=inv['payment_secret'])
-# 
-#     assert not l1.daemon.is_in_log('Payment is still in progress')
-# 
-#     # After it sees block, someone should close channel.
-#     l1.daemon.wait_for_log('ONCHAIN')
-# 
-# 
-# @pytest.mark.developer("needs DEVELOPER=1 for dev_suppress_gossip")
-# def test_pay_get_error_with_update(node_factory):
-#     """We should process an update inside a temporary_channel_failure"""
-#     l1, l2, l3 = node_factory.line_graph(3, opts={'log-level': 'io'}, fundchannel=True, wait_for_announce=True)
-#     chanid2 = l2.get_channel_scid(l3)
-# 
-#     inv = l3.rpc.invoice(123000, 'test_pay_get_error_with_update', 'description')
-# 
-#     # Make sure l2 doesn't tell l1 directly that channel is disabled.
-#     l2.rpc.dev_suppress_gossip()
-#     l3.stop()
-# 
-#     # Make sure that l2 has seen disconnect, considers channel disabled.
-#     wait_for(lambda: [c['active'] for c in l2.rpc.listchannels(chanid2)['channels']] == [False, False])
-# 
-#     assert(l1.is_channel_active(chanid2))
-#     with pytest.raises(RpcError, match=r'WIRE_TEMPORARY_CHANNEL_FAILURE'):
-#         l1.rpc.pay(inv['bolt11'])
-# 
-#     # Make sure we get an onionreply, without the type prefix of the nested
-#     # channel_update, and it should patch it to include a type prefix. The
-#     # prefix 0x0102 should be in the channel_update, but not in the
-#     # onionreply (negation of 0x0102 in the RE)
-#     l1.daemon.wait_for_log(r'Extracted channel_update 0102.*from onionreply 1007008a[0-9a-fA-F]{276}$')
-# 
-#     # And now monitor for l1 to apply the channel_update we just extracted
-#     wait_for(lambda: not l1.is_channel_active(chanid2))
-# 
-# 
-# @pytest.mark.developer("needs DEVELOPER=1 for dev_suppress_gossip, dev-routes")
-# def test_pay_error_update_fees(node_factory):
-#     """We should process an update inside a temporary_channel_failure"""
-#     l1, l2, l3 = node_factory.line_graph(3, fundchannel=True, wait_for_announce=True)
-# 
-#     # Don't include any routehints in first invoice.
-#     inv1 = l3.dev_invoice(amount_msat=123000,
-#                           label='test_pay_error_update_fees',
-#                           description='description',
-#                           dev_routes=[])
-# 
-#     inv2 = l3.rpc.invoice(123000, 'test_pay_error_update_fees2', 'desc')  # noqa: F841
-# 
-#     # Make sure l2 doesn't tell l1 directly that channel fee is changed.
-#     l2.rpc.dev_suppress_gossip()
-#     l2.rpc.setchannel(l3.info['id'], 1337, 137, enforcedelay=0)
-# 
-#     # Should bounce off and retry...
-#     l1.rpc.pay(inv1['bolt11'])
-#     attempts = only_one(l1.rpc.paystatus(inv1['bolt11'])['pay'])['attempts']
-#     assert len(attempts) == 2
-#     # WIRE_FEE_INSUFFICIENT = UPDATE|12
-#     assert attempts[0]['failure']['data']['failcode'] == 4108
-# 
-#     # FIXME: We *DO NOT* handle misleading routehints!
-#     # # Should ignore old routehint and do the same...
-#     # l1.rpc.pay(inv2['bolt11'])
-#     # attempts = only_one(l1.rpc.paystatus(inv2['bolt11'])['pay'])['attempts']
-#     # assert len(attempts) == 2
-#     # # WIRE_FEE_INSUFFICIENT = UPDATE|12
-#     # assert attempts[0]['failure']['data']['failcode'] == 4108
-# 
-# 
-# @pytest.mark.developer("needs to deactivate shadow routing")
-# def test_pay_optional_args(node_factory):
-#     l1, l2 = node_factory.line_graph(2)
-# 
-#     inv1 = l2.rpc.invoice(123000, 'test_pay', 'desc')['bolt11']
-#     l1.dev_pay(inv1, label='desc', use_shadow=False)
-#     payment1 = l1.rpc.listsendpays(inv1)['payments']
-#     assert len(payment1) and payment1[0]['amount_sent_msat'] == 123000
-#     assert payment1[0]['label'] == 'desc'
-# 
-#     inv2 = l2.rpc.invoice(321000, 'test_pay2', 'description')['bolt11']
-#     l1.dev_pay(inv2, riskfactor=5.0, use_shadow=False)
-#     payment2 = l1.rpc.listsendpays(inv2)['payments']
-#     assert(len(payment2) == 1)
-#     # The pay plugin uses `sendonion` since 0.9.0 and `lightningd` doesn't
-#     # learn about the amount we intended to send (that's why we annotate the
-#     # root of a payment tree with the bolt11 invoice).
-# 
-#     anyinv = l2.rpc.invoice('any', 'any_pay', 'desc')['bolt11']
-#     l1.dev_pay(anyinv, label='desc', amount_msat=500, use_shadow=False)
-#     payment3 = l1.rpc.listsendpays(anyinv)['payments']
-#     assert len(payment3) == 1
-#     assert payment3[0]['label'] == 'desc'
-# 
-#     # Should see 3 completed transactions
-#     assert len(l1.rpc.listsendpays()['payments']) == 3
-# 
-# 
-# @pytest.mark.developer("needs to deactivate shadow routing")
-# @pytest.mark.openchannel('v1')
-# @pytest.mark.openchannel('v2')
-# def test_payment_success_persistence(node_factory, bitcoind, executor):
-#     # Start two nodes and open a channel.. die during payment.
-#     # Feerates identical so we don't get gratuitous commit to update them
-#     disconnect = ['+WIRE_COMMITMENT_SIGNED']
-#     if EXPERIMENTAL_DUAL_FUND:
-#         # We have to add an extra 'wire-commitment-signed' because
-#         # dual funding uses this for channel establishment also
-#         disconnect = ['=WIRE_COMMITMENT_SIGNED'] + disconnect
-# 
-#     l1 = node_factory.get_node(disconnect=disconnect,
-#                                options={'dev-no-reconnect': None},
-#                                may_reconnect=True,
-#                                feerates=(7500, 7500, 7500, 7500))
-#     l2 = node_factory.get_node(may_reconnect=True)
-#     l1.rpc.connect(l2.info['id'], 'localhost', l2.port)
-# 
-#     chanid, _ = l1.fundchannel(l2, 100000)
-# 
-#     inv1 = l2.rpc.invoice(1000, 'inv1', 'inv1')
-# 
-#     # Fire off a pay request, it'll get interrupted by a restart
-#     executor.submit(l1.dev_pay, inv1['bolt11'], use_shadow=False)
-# 
-#     l1.daemon.wait_for_log(r'dev_disconnect: \+WIRE_COMMITMENT_SIGNED')
-# 
-#     print("Killing l1 in mid HTLC")
-#     l1.daemon.kill()
-# 
-#     # Restart l1, without disconnect stuff.
-#     del l1.daemon.opts['dev-no-reconnect']
-#     del l1.daemon.opts['dev-disconnect']
-# 
-#     # Should reconnect, and sort the payment out.
-#     l1.start()
-# 
-#     wait_for(lambda: l1.rpc.listsendpays()['payments'][0]['status'] != 'pending')
-# 
-#     payments = l1.rpc.listsendpays()['payments']
-#     invoices = l2.rpc.listinvoices('inv1')['invoices']
-#     assert len(payments) == 1 and payments[0]['status'] == 'complete'
-#     assert len(invoices) == 1 and invoices[0]['status'] == 'paid'
-# 
-#     l1.wait_channel_active(chanid)
-# 
-#     # A duplicate should succeed immediately (nop) and return correct preimage.
-#     preimage = l1.dev_pay(
-#         inv1['bolt11'],
-#         use_shadow=False
-#     )['payment_preimage']
-#     assert l1.rpc.dev_rhash(preimage)['rhash'] == inv1['payment_hash']
-# 
-# 
-# @pytest.mark.developer("needs DEVELOPER=1")
-# @pytest.mark.openchannel('v1')
-# @pytest.mark.openchannel('v2')
-# def test_payment_failed_persistence(node_factory, executor):
-#     # Start two nodes and open a channel.. die during payment.
-#     # Feerates identical so we don't get gratuitous commit to update them
-#     disconnect = ['+WIRE_COMMITMENT_SIGNED']
-#     if EXPERIMENTAL_DUAL_FUND:
-#         # We have to add an extra 'wire-commitment-signed' because
-#         # dual funding uses this for channel establishment also
-#         disconnect = ['=WIRE_COMMITMENT_SIGNED'] + disconnect
-#     l1 = node_factory.get_node(disconnect=disconnect,
-#                                options={'dev-no-reconnect': None},
-#                                may_reconnect=True,
-#                                feerates=(7500, 7500, 7500, 7500))
-#     l2 = node_factory.get_node(may_reconnect=True)
-#     l1.rpc.connect(l2.info['id'], 'localhost', l2.port)
-# 
-#     l1.fundchannel(l2, 100000)
-# 
-#     # Expires almost immediately, so it will fail.
-#     inv1 = l2.rpc.invoice(1000, 'inv1', 'inv1', 5)
-# 
-#     # Fire off a pay request, it'll get interrupted by a restart
-#     executor.submit(l1.rpc.pay, inv1['bolt11'])
-# 
-#     l1.daemon.wait_for_log(r'dev_disconnect: \+WIRE_COMMITMENT_SIGNED')
-# 
-#     print("Killing l1 in mid HTLC")
-#     l1.daemon.kill()
-# 
-#     # Restart l1, without disconnect stuff.
-#     del l1.daemon.opts['dev-no-reconnect']
-#     del l1.daemon.opts['dev-disconnect']
-# 
-#     # Make sure invoice has expired.
-#     time.sleep(5 + 1)
-# 
-#     # Should reconnect, and fail the payment
-#     l1.start()
-# 
-#     wait_for(lambda: l1.rpc.listsendpays()['payments'][0]['status'] != 'pending')
-# 
-#     payments = l1.rpc.listsendpays()['payments']
-#     invoices = l2.rpc.listinvoices('inv1')['invoices']
-#     assert len(invoices) == 1 and invoices[0]['status'] == 'expired'
-#     assert len(payments) == 1 and payments[0]['status'] == 'failed'
-# 
-#     # Another attempt should also fail.
-#     with pytest.raises(RpcError):
-#         l1.rpc.pay(inv1['bolt11'])
-# 
-# 
-# @pytest.mark.developer("needs DEVELOPER=1")
-# def test_payment_duplicate_uncommitted(node_factory, executor):
-#     # We want to test two payments at the same time, before we send commit
-#     l1 = node_factory.get_node(options={'dev-disable-commit-after': 0})
-#     l2 = node_factory.get_node()
-# 
-#     l1.rpc.connect(l2.info['id'], 'localhost', l2.port)
-# 
-#     l1.fundchannel(l2, 100000)
-# 
-#     inv1 = l2.rpc.invoice(1000, 'inv1', 'inv1')
-# 
-#     # Start first payment, but not yet in db.
-#     fut = executor.submit(l1.rpc.pay, inv1['bolt11'])
-# 
-#     # Make sure that's started...
-#     l1.daemon.wait_for_log('peer_out WIRE_UPDATE_ADD_HTLC')
-# 
-#     # We should see it in listsendpays
-#     payments = l1.rpc.listsendpays()['payments']
-#     assert len(payments) == 1
-#     assert payments[0]['status'] == 'pending' and payments[0]['payment_hash'] == inv1['payment_hash']
-# 
-#     # Second one will succeed eventually.
-#     fut2 = executor.submit(l1.rpc.pay, inv1['bolt11'])
-# 
-#     # Now, let it commit.
-#     l1.rpc.dev_reenable_commit(l2.info['id'])
-# 
-#     # These should succeed.
-#     fut.result(TIMEOUT)
-#     fut2.result(TIMEOUT)
-# 
-# 
-# @pytest.mark.developer("Too slow without --dev-fast-gossip")
-# def test_pay_maxfee_shadow(node_factory):
-#     """Test that we respect maxfeepercent for shadow routing."""
-#     l1, l2, l3 = node_factory.line_graph(3, fundchannel=True,
-#                                          wait_for_announce=True)
-#     # We use this to search for shadow routes
-#     wait_for(
-#         lambda: len(l1.rpc.listchannels(source=l2.info["id"])["channels"]) > 1
-#     )
-# 
-#     # shadow routes are random, so run multiple times.
-#     for i in range(5):
-#         # A tiny amount, we must not add the base_fee between l2 and l3
-#         amount = 2
-#         bolt11 = l2.rpc.invoice(amount, "tiny.{}".format(i), "tiny")["bolt11"]
-#         pay_status = l1.rpc.pay(bolt11)
-#         assert pay_status["amount_msat"] == Millisatoshi(amount)
-# 
-#     # shadow routes are random, so run multiple times.
-#     for i in range(5):
-#         # A bigger amount, shadow routing could have been used but we set a low
-#         # maxfeepercent.
-#         amount = 20000
-#         bolt11 = l2.rpc.invoice(amount, "big.{}".format(i), "bigger")["bolt11"]
-#         pay_status = l1.rpc.pay(bolt11, maxfeepercent=0.001)
-#         assert pay_status["amount_msat"] == Millisatoshi(amount)
-# 
-# 
-# def test_sendpay(node_factory):
-#     l1, l2 = node_factory.line_graph(2, fundamount=10**6)
-# 
-#     amt = 200000000
-#     inv = l2.rpc.invoice(amt, 'testpayment2', 'desc')
-#     rhash = inv['payment_hash']
-# 
-#     def invoice_unpaid(dst, label):
-#         invoices = dst.rpc.listinvoices(label)['invoices']
-#         return len(invoices) == 1 and invoices[0]['status'] == 'unpaid'
-# 
-#     routestep = {
-#         'amount_msat': amt,
-#         'id': l2.info['id'],
-#         'delay': 5,
-#         'channel': first_scid(l1, l2)
-#     }
-# 
-#     # Insufficient funds.
-#     with pytest.raises(RpcError):
-#         rs = copy.deepcopy(routestep)
-#         rs['amount_msat'] = rs['amount_msat'] - 1
-#         l1.rpc.sendpay([rs], rhash, payment_secret=inv['payment_secret'])
-#         l1.rpc.waitsendpay(rhash)
-#     assert invoice_unpaid(l2, 'testpayment2')
-# 
-#     # Gross overpayment (more than factor of 2)
-#     with pytest.raises(RpcError):
-#         rs = copy.deepcopy(routestep)
-#         rs['amount_msat'] = rs['amount_msat'] * 2 + 1
-#         l1.rpc.sendpay([rs], rhash, payment_secret=inv['payment_secret'])
-#         l1.rpc.waitsendpay(rhash)
-#     assert invoice_unpaid(l2, 'testpayment2')
-# 
-#     # Insufficient delay.
-#     with pytest.raises(RpcError):
-#         rs = copy.deepcopy(routestep)
-#         rs['delay'] = rs['delay'] - 2
-#         l1.rpc.sendpay([rs], rhash, payment_secret=inv['payment_secret'])
-#         l1.rpc.waitsendpay(rhash)
-#     assert invoice_unpaid(l2, 'testpayment2')
-# 
-#     # Bad ID.
-#     l1.rpc.check_request_schemas = False
-#     with pytest.raises(RpcError):
-#         rs = copy.deepcopy(routestep)
-#         rs['id'] = '00000000000000000000000000000000'
-#         l1.rpc.sendpay([rs], rhash, payment_secret=inv['payment_secret'])
-#     assert invoice_unpaid(l2, 'testpayment2')
-#     l1.rpc.check_request_schemas = True
-# 
-#     # Bad payment_secret
-#     l1.rpc.sendpay([routestep], rhash, payment_secret="00" * 32)
-#     with pytest.raises(RpcError):
-#         l1.rpc.waitsendpay(rhash)
-#     assert invoice_unpaid(l2, 'testpayment2')
-# 
-#     # Missing payment_secret
-#     l1.rpc.sendpay([routestep], rhash)
-#     with pytest.raises(RpcError):
-#         l1.rpc.waitsendpay(rhash)
-#     assert invoice_unpaid(l2, 'testpayment2')
-# 
-#     # FIXME: test paying via another node, should fail to pay twice.
-#     c1 = only_one(l1.rpc.listpeerchannels(l2.info['id'])['channels'])
-#     c2 = only_one(l2.rpc.listpeerchannels(l1.info['id'])['channels'])
-#     assert c1['to_us_msat'] == 10**6 * 1000
-#     assert c1['total_msat'] == 10**6 * 1000
-#     assert c2['to_us_msat'] == 0
-#     assert c2['total_msat'] == 10**6 * 1000
-# 
-#     # This works.
-#     before = int(time.time())
-#     details = l1.rpc.sendpay([routestep], rhash, payment_secret=inv['payment_secret'])
-#     after = int(time.time())
-#     preimage = l1.rpc.waitsendpay(rhash)['payment_preimage']
-#     # Check details
-#     assert details['payment_hash'] == rhash
-#     assert details['destination'] == l2.info['id']
-#     assert details['amount_msat'] == amt
-#     assert details['created_at'] >= before
-#     assert details['created_at'] <= after
-#     # Check receiver
-#     assert only_one(l2.rpc.listinvoices('testpayment2')['invoices'])['status'] == 'paid'
-#     assert only_one(l2.rpc.listinvoices('testpayment2')['invoices'])['pay_index'] == 1
-#     assert only_one(l2.rpc.listinvoices('testpayment2')['invoices'])['amount_received_msat'] == rs['amount_msat']
-#     assert only_one(l2.rpc.listinvoices('testpayment2')['invoices'])['payment_preimage'] == preimage
-# 
-#     # Balances should reflect it.
-#     def check_balances():
-#         c1 = only_one(l1.rpc.listpeerchannels(l2.info['id'])['channels'])
-#         c2 = only_one(l2.rpc.listpeerchannels(l1.info['id'])['channels'])
-#         return (
-#             c1['to_us_msat'] == 10**6 * 1000 - amt
-#             and c1['total_msat'] == 10**6 * 1000
-#             and c2['to_us_msat'] == amt
-#             and c2['total_msat'] == 10**6 * 1000
-#         )
-#     wait_for(check_balances)
-# 
-#     # Repeat will "succeed", but won't actually send anything (duplicate)
-#     assert not l1.daemon.is_in_log('Payment ./.: .* COMPLETE')
-#     details = l1.rpc.sendpay([routestep], rhash, payment_secret=inv['payment_secret'])
-#     assert details['status'] == "complete"
-#     preimage2 = details['payment_preimage']
-#     assert preimage == preimage2
-#     l1.daemon.wait_for_log('Payment ./.: .* COMPLETE')
-#     assert only_one(l2.rpc.listinvoices('testpayment2')['invoices'])['status'] == 'paid'
-#     assert only_one(l2.rpc.listinvoices('testpayment2')['invoices'])['amount_received_msat'] == rs['amount_msat']
-# 
-#     # Overpaying by "only" a factor of 2 succeeds.
-#     inv = l2.rpc.invoice(amt, 'testpayment3', 'desc')
-#     rhash = inv['payment_hash']
-#     assert only_one(l2.rpc.listinvoices('testpayment3')['invoices'])['status'] == 'unpaid'
-#     routestep = {'amount_msat': amt * 2, 'id': l2.info['id'], 'delay': 5, 'channel': first_scid(l1, l2)}
-#     l1.rpc.sendpay([routestep], rhash, payment_secret=inv['payment_secret'])
-#     preimage3 = l1.rpc.waitsendpay(rhash)['payment_preimage']
-#     assert only_one(l2.rpc.listinvoices('testpayment3')['invoices'])['status'] == 'paid'
-#     assert only_one(l2.rpc.listinvoices('testpayment3')['invoices'])['amount_received_msat'] == amt * 2
-# 
-#     # Test listsendpays
-#     payments = l1.rpc.listsendpays()['payments']
-#     assert len(payments) == 7  # Failed attempts also create entries, but with a different groupid
-# 
-#     invoice2 = only_one(l2.rpc.listinvoices('testpayment2')['invoices'])
-#     payments = l1.rpc.listsendpays(payment_hash=invoice2['payment_hash'])['payments']
-#     assert len(payments) == 6  # Failed attempts also create entries, but with a different groupid
-# 
-#     assert payments[-1]['status'] == 'complete'
-#     assert payments[-1]['payment_preimage'] == preimage2
-# 
-#     invoice3 = only_one(l2.rpc.listinvoices('testpayment3')['invoices'])
-#     payments = l1.rpc.listsendpays(payment_hash=invoice3['payment_hash'])['payments']
-#     assert len(payments) == 1
-# 
-#     assert payments[-1]['status'] == 'complete'
-#     assert payments[-1]['payment_preimage'] == preimage3
-# 
-# 
-# @unittest.skipIf(TEST_NETWORK != 'regtest', "The reserve computation is bitcoin specific")
-# def test_sendpay_cant_afford(node_factory):
-#     # Set feerates the same so we don't have to wait for update.
-#     l1, l2 = node_factory.line_graph(2, fundamount=10**6,
-#                                      opts={'feerates': (15000, 15000, 15000, 15000)})
-# 
-#     # Can't pay more than channel capacity.
-#     with pytest.raises(RpcError):
-#         l1.pay(l2, 10**9 + 1)
-# 
-#     # Reserve is 1%.
-#     reserve = 10**7
-# 
-#     # # This is how we recalc constants (v. v. slow!)
-#     # minimum = 1
-#     # maximum = 10**9
-#     # while maximum - minimum > 1:
-#     #     l1, l2 = node_factory.line_graph(2, fundamount=10**6,
-#     #                                      opts={'feerates': (15000, 15000, 15000, 15000)})
-#     #     try:
-#     #         l1.pay(l2, (minimum + maximum) // 2)
-#     #         minimum = (minimum + maximum) // 2
-#     #     except RpcError:
-#     #         maximum = (minimum + maximum) // 2
-#     #     print("{} - {}".format(minimum, maximum))
-#     # assert False
-# 
-#     # This is the fee, which needs to be taken into account for l1.
-#     if EXPERIMENTAL_FEATURES:
-#         # option_anchor_outputs
-#         available = 10**9 - 44700000
-#     else:
-#         available = 10**9 - 32040000
-# 
-#     # Can't pay past reserve.
-#     with pytest.raises(RpcError):
-#         l1.pay(l2, available)
-#     with pytest.raises(RpcError):
-#         l1.pay(l2, available - reserve + 1)
-# 
-#     # Can pay up to reserve (1%)
-#     l1.pay(l2, available - reserve)
-# 
-#     # And now it can't pay back, due to its own reserve.
-#     with pytest.raises(RpcError):
-#         l2.pay(l1, available - reserve)
-# 
-#     # But this should work.
-#     l2.pay(l1, available - reserve * 2)
-# 
-# 
-# def test_decodepay(node_factory):
-#     l1 = node_factory.get_node()
-# 
-#     # BOLT #11:
-#     # > ### Please make a donation of any amount using payment_hash 0001020304050607080900010203040506070809000102030405060708090102 to me @03e7156ae33b0a208d0744199163177e909e80176e55d97a2f221ede0f934dd9ad
-#     # > lnbc1pvjluezpp5qqqsyqcyq5rqwzqfqqqsyqcyq5rqwzqfqqqsyqcyq5rqwzqfqypqdpl2pkx2ctnv5sxxmmwwd5kgetjypeh2ursdae8g6twvus8g6rfwvs8qun0dfjkxaq8rkx3yf5tcsyz3d73gafnh3cax9rn449d9p5uxz9ezhhypd0elx87sjle52x86fux2ypatgddc6k63n7erqz25le42c4u4ecky03ylcqca784w
-#     #
-#     # Breakdown:
-#     #
-#     # * `lnbc`: prefix, lightning on bitcoin mainnet
-#     # * `1`: Bech32 separator
-#     # * `pvjluez`: timestamp (1496314658)
-#     # * `p`: payment hash
-#     #   * `p5`: `data_length` (`p` = 1, `5` = 20. 1 * 32 + 20 == 52)
-#     #   * `qqqsyqcyq5rqwzqfqqqsyqcyq5rqwzqfqqqsyqcyq5rqwzqfqypq`: payment hash 0001020304050607080900010203040506070809000102030405060708090102
-#     # * `d`: short description
-#     #   * `pl`: `data_length` (`p` = 1, `l` = 31. 1 * 32 + 31 == 63)
-#     #   * `2pkx2ctnv5sxxmmwwd5kgetjypeh2ursdae8g6twvus8g6rfwvs8qun0dfjkxaq`: 'Please consider supporting this project'
-#     # * `32vjcgqxyuj7nqphl3xmmhls2rkl3t97uan4j0xa87gj5779czc8p0z58zf5wpt9ggem6adl64cvawcxlef9djqwp2jzzfvs272504sp`: signature
-#     # * `0lkg3c`: Bech32 checksum
-#     b11 = l1.rpc.decodepay(
-#         'lnbc1pvjluezpp5qqqsyqcyq5rqwzqfqqqsyqcyq5rqwzqfqqqsyqcyq5rqwzqfqypqd'
-#         'pl2pkx2ctnv5sxxmmwwd5kgetjypeh2ursdae8g6twvus8g6rfwvs8qun0dfjkxaq8rk'
-#         'x3yf5tcsyz3d73gafnh3cax9rn449d9p5uxz9ezhhypd0elx87sjle52x86fux2ypatg'
-#         'ddc6k63n7erqz25le42c4u4ecky03ylcqca784w'
-#     )
-#     assert b11['currency'] == 'bc'
-#     assert b11['created_at'] == 1496314658
-#     assert b11['payment_hash'] == '0001020304050607080900010203040506070809000102030405060708090102'
-#     assert b11['description'] == 'Please consider supporting this project'
-#     assert b11['expiry'] == 3600
-#     assert b11['payee'] == '03e7156ae33b0a208d0744199163177e909e80176e55d97a2f221ede0f934dd9ad'
-# 
-#     # BOLT #11:
-#     # > ### Please send $3 for a cup of coffee to the same peer, within 1 minute
-#     # > lnbc2500u1pvjluezpp5qqqsyqcyq5rqwzqfqqqsyqcyq5rqwzqfqqqsyqcyq5rqwzqfqypqdq5xysxxatsyp3k7enxv4jsxqzpuaztrnwngzn3kdzw5hydlzf03qdgm2hdq27cqv3agm2awhz5se903vruatfhq77w3ls4evs3ch9zw97j25emudupq63nyw24cg27h2rspfj9srp
-#     #
-#     # Breakdown:
-#     #
-#     # * `lnbc`: prefix, lightning on bitcoin mainnet
-#     # * `2500u`: amount (2500 micro-bitcoin)
-#     # * `1`: Bech32 separator
-#     # * `pvjluez`: timestamp (1496314658)
-#     # * `p`: payment hash...
-#     # * `d`: short description
-#     #   * `q5`: `data_length` (`q` = 0, `5` = 20. 0 * 32 + 20 == 20)
-#     #   * `xysxxatsyp3k7enxv4js`: '1 cup coffee'
-#     # * `x`: expiry time
-#     #   * `qz`: `data_length` (`q` = 0, `z` = 2. 0 * 32 + 2 == 2)
-#     #   * `pu`: 60 seconds (`p` = 1, `u` = 28.  1 * 32 + 28 == 60)
-#     # * `azh8qt5w7qeewkmxtv55khqxvdfs9zzradsvj7rcej9knpzdwjykcq8gv4v2dl705pjadhpsc967zhzdpuwn5qzjm0s4hqm2u0vuhhqq`: signature
-#     # * `7vc09u`: Bech32 checksum
-#     b11 = l1.rpc.decodepay(
-#         'lnbc2500u1pvjluezpp5qqqsyqcyq5rqwzqfqqqsyqcyq5rqwzqfqqqsyqcyq5rqwzqf'
-#         'qypqdq5xysxxatsyp3k7enxv4jsxqzpuaztrnwngzn3kdzw5hydlzf03qdgm2hdq27cq'
-#         'v3agm2awhz5se903vruatfhq77w3ls4evs3ch9zw97j25emudupq63nyw24cg27h2rsp'
-#         'fj9srp'
-#     )
-#     assert b11['currency'] == 'bc'
-#     assert b11['amount_msat'] == Millisatoshi(2500 * 10**11 // 1000000)
-#     assert b11['created_at'] == 1496314658
-#     assert b11['payment_hash'] == '0001020304050607080900010203040506070809000102030405060708090102'
-#     assert b11['description'] == '1 cup coffee'
-#     assert b11['expiry'] == 60
-#     assert b11['payee'] == '03e7156ae33b0a208d0744199163177e909e80176e55d97a2f221ede0f934dd9ad'
-# 
-#     # BOLT #11:
-#     # > ### Now send $24 for an entire list of things (hashed)
-#     # > lnbc20m1pvjluezpp5qqqsyqcyq5rqwzqfqqqsyqcyq5rqwzqfqqqsyqcyq5rqwzqfqypqhp58yjmdan79s6qqdhdzgynm4zwqd5d7xmw5fk98klysy043l2ahrqscc6gd6ql3jrc5yzme8v4ntcewwz5cnw92tz0pc8qcuufvq7khhr8wpald05e92xw006sq94mg8v2ndf4sefvf9sygkshp5zfem29trqq2yxxz7
-#     #
-#     # Breakdown:
-#     #
-#     # * `lnbc`: prefix, lightning on bitcoin mainnet
-#     # * `20m`: amount (20 milli-bitcoin)
-#     # * `1`: Bech32 separator
-#     # * `pvjluez`: timestamp (1496314658)
-#     # * `p`: payment hash...
-#     # * `h`: tagged field: hash of description
-#     # * `p5`: `data_length` (`p` = 1, `5` = 20. 1 * 32 + 20 == 52)
-#     # * `8yjmdan79s6qqdhdzgynm4zwqd5d7xmw5fk98klysy043l2ahrqs`: SHA256 of 'One piece of chocolate cake, one icecream cone, one pickle, one slice of swiss cheese, one slice of salami, one lollypop, one piece of cherry pie, one sausage, one cupcake, and one slice of watermelon'
-#     # * `vjfls3ljx9e93jkw0kw40yxn4pevgzflf83qh2852esjddv4xk4z70nehrdcxa4fk0t6hlcc6vrxywke6njenk7yzkzw0quqcwxphkcp`: signature
-#     # * `vam37w`: Bech32 checksum
-#     b11 = l1.rpc.decodepay(
-#         'lnbc20m1pvjluezpp5qqqsyqcyq5rqwzqfqqqsyqcyq5rqwzqfqqqsyqcyq5rqwzqfqy'
-#         'pqhp58yjmdan79s6qqdhdzgynm4zwqd5d7xmw5fk98klysy043l2ahrqscc6gd6ql3jr'
-#         'c5yzme8v4ntcewwz5cnw92tz0pc8qcuufvq7khhr8wpald05e92xw006sq94mg8v2ndf'
-#         '4sefvf9sygkshp5zfem29trqq2yxxz7',
-#         'One piece of chocolate cake, one icecream cone, one pickle, one slic'
-#         'e of swiss cheese, one slice of salami, one lollypop, one piece of c'
-#         'herry pie, one sausage, one cupcake, and one slice of watermelon'
-#     )
-#     assert b11['currency'] == 'bc'
-#     assert b11['amount_msat'] == Millisatoshi(str(20 * 10**11 // 1000) + 'msat')
-#     assert b11['created_at'] == 1496314658
-#     assert b11['payment_hash'] == '0001020304050607080900010203040506070809000102030405060708090102'
-#     assert b11['expiry'] == 3600
-#     assert b11['payee'] == '03e7156ae33b0a208d0744199163177e909e80176e55d97a2f221ede0f934dd9ad'
-# 
-#     # > ### The same, on testnet, with a fallback address mk2QpYatsKicvFVuTAQLBryyccRXMUaGHP
-#     # > lntb20m1pvjluezhp58yjmdan79s6qqdhdzgynm4zwqd5d7xmw5fk98klysy043l2ahrqspp5qqqsyqcyq5rqwzqfqqqsyqcyq5rqwzqfqqqsyqcyq5rqwzqfqypqfpp3x9et2e20v6pu37c5d9vax37wxq72un98kmzzhznpurw9sgl2v0nklu2g4d0keph5t7tj9tcqd8rexnd07ux4uv2cjvcqwaxgj7v4uwn5wmypjd5n69z2xm3xgksg28nwht7f6zspwp3f9t
-#     #
-#     # Breakdown:
-#     #
-#     # * `lntb`: prefix, lightning on bitcoin testnet
-#     # * `20m`: amount (20 milli-bitcoin)
-#     # * `1`: Bech32 separator
-#     # * `pvjluez`: timestamp (1496314658)
-#     # * `p`: payment hash...
-#     # * `f`: tagged field: fallback address
-#     # * `pp`: `data_length` (`p` = 1. 1 * 32 + 1 == 33)
-#     # * `3x9et2e20v6pu37c5d9vax37wxq72un98`: `3` = 17, so P2PKH address
-#     # * `h`: tagged field: hash of description...
-#     # * `qh84fmvn2klvglsjxfy0vq2mz6t9kjfzlxfwgljj35w2kwa60qv49k7jlsgx43yhs9nuutllkhhnt090mmenuhp8ue33pv4klmrzlcqp`: signature
-#     # * `us2s2r`: Bech32 checksum
-#     b11 = l1.rpc.decodepay(
-#         'lntb20m1pvjluezhp58yjmdan79s6qqdhdzgynm4zwqd5d7xmw5fk98klysy043l2ahr'
-#         'qspp5qqqsyqcyq5rqwzqfqqqsyqcyq5rqwzqfqqqsyqcyq5rqwzqfqypqfpp3x9et2e2'
-#         '0v6pu37c5d9vax37wxq72un98kmzzhznpurw9sgl2v0nklu2g4d0keph5t7tj9tcqd8r'
-#         'exnd07ux4uv2cjvcqwaxgj7v4uwn5wmypjd5n69z2xm3xgksg28nwht7f6zspwp3f9t',
-#         'One piece of chocolate cake, one icecream cone, one pickle, one slic'
-#         'e of swiss cheese, one slice of salami, one lollypop, one piece of c'
-#         'herry pie, one sausage, one cupcake, and one slice of watermelon'
-#     )
-#     assert b11['currency'] == 'tb'
-#     assert b11['amount_msat'] == Millisatoshi(20 * 10**11 // 1000)
-#     assert b11['created_at'] == 1496314658
-#     assert b11['payment_hash'] == '0001020304050607080900010203040506070809000102030405060708090102'
-#     assert b11['expiry'] == 3600
-#     assert b11['payee'] == '03e7156ae33b0a208d0744199163177e909e80176e55d97a2f221ede0f934dd9ad'
-#     assert len(b11['fallbacks']) == 1
-#     assert b11['fallbacks'][0]['type'] == 'P2PKH'
-#     assert b11['fallbacks'][0]['addr'] == 'mk2QpYatsKicvFVuTAQLBryyccRXMUaGHP'
-# 
-#     # > ### On mainnet, with fallback address 1RustyRX2oai4EYYDpQGWvEL62BBGqN9T with extra routing info to go via nodes 029e03a901b85534ff1e92c43c74431f7ce72046060fcf7a95c37e148f78c77255 then 039e03a901b85534ff1e92c43c74431f7ce72046060fcf7a95c37e148f78c77255
-#     # > lnbc20m1pvjluezpp5qqqsyqcyq5rqwzqfqqqsyqcyq5rqwzqfqqqsyqcyq5rqwzqfqypqhp58yjmdan79s6qqdhdzgynm4zwqd5d7xmw5fk98klysy043l2ahrqsfpp3qjmp7lwpagxun9pygexvgpjdc4jdj85fr9yq20q82gphp2nflc7jtzrcazrra7wwgzxqc8u7754cdlpfrmccae92qgzqvzq2ps8pqqqqqqpqqqqq9qqqvpeuqafqxu92d8lr6fvg0r5gv0heeeqgcrqlnm6jhphu9y00rrhy4grqszsvpcgpy9qqqqqqgqqqqq7qqzqj9n4evl6mr5aj9f58zp6fyjzup6ywn3x6sk8akg5v4tgn2q8g4fhx05wf6juaxu9760yp46454gpg5mtzgerlzezqcqvjnhjh8z3g2qqdhhwkj
-#     #
-#     # Breakdown:
-#     #
-#     # * `lnbc`: prefix, lightning on bitcoin mainnet
-#     # * `20m`: amount (20 milli-bitcoin)
-#     # * `1`: Bech32 separator
-#     # * `pvjluez`: timestamp (1496314658)
-#     # * `p`: payment hash...
-#     # * `h`: tagged field: hash of description...
-#     # * `f`: tagged field: fallback address
-#     #   * `pp`: `data_length` (`p` = 1. 1 * 32 + 1 == 33)
-#     #   * `3` = 17, so P2PKH address
-#     #   * `qjmp7lwpagxun9pygexvgpjdc4jdj85f`: 160 bit P2PKH address
-#     # * `r`: tagged field: route information
-#     #   * `9y`: `data_length` (`9` = 5, `y` = 4.  5 * 32 + 4 = 164)
-#     #     `q20q82gphp2nflc7jtzrcazrra7wwgzxqc8u7754cdlpfrmccae92qgzqvzq2ps8pqqqqqqqqqqqq9qqqvpeuqafqxu92d8lr6fvg0r5gv0heeeqgcrqlnm6jhphu9y00rrhy4grqszsvpcgpy9qqqqqqqqqqqq7qqzq`: pubkey `029e03a901b85534ff1e92c43c74431f7ce72046060fcf7a95c37e148f78c77255`, `short_channel_id` 0102030405060708, `fee_base_msat` 1 millisatoshi, `fee_proportional_millionths` 20, `cltv_expiry_delta` 3.  pubkey `039e03a901b85534ff1e92c43c74431f7ce72046060fcf7a95c37e148f78c77255`, `short_channel_id` 030405060708090a, `fee_base_msat` 2 millisatoshi, `fee_proportional_millionths` 30, `cltv_expiry_delta` 4.
-#     # * `j9n4evl6mr5aj9f58zp6fyjzup6ywn3x6sk8akg5v4tgn2q8g4fhx05wf6juaxu9760yp46454gpg5mtzgerlzezqcqvjnhjh8z3g2qq`: signature
-#     # * `dhhwkj`: Bech32 checksum
-#     b11 = l1.rpc.decodepay('lnbc20m1pvjluezpp5qqqsyqcyq5rqwzqfqqqsyqcyq5rqwzqfqqqsyqcyq5rqwzqfqypqhp58yjmdan79s6qqdhdzgynm4zwqd5d7xmw5fk98klysy043l2ahrqsfpp3qjmp7lwpagxun9pygexvgpjdc4jdj85fr9yq20q82gphp2nflc7jtzrcazrra7wwgzxqc8u7754cdlpfrmccae92qgzqvzq2ps8pqqqqqqpqqqqq9qqqvpeuqafqxu92d8lr6fvg0r5gv0heeeqgcrqlnm6jhphu9y00rrhy4grqszsvpcgpy9qqqqqqgqqqqq7qqzqj9n4evl6mr5aj9f58zp6fyjzup6ywn3x6sk8akg5v4tgn2q8g4fhx05wf6juaxu9760yp46454gpg5mtzgerlzezqcqvjnhjh8z3g2qqdhhwkj', 'One piece of chocolate cake, one icecream cone, one pickle, one slice of swiss cheese, one slice of salami, one lollypop, one piece of cherry pie, one sausage, one cupcake, and one slice of watermelon')
-#     assert b11['currency'] == 'bc'
-#     assert b11['amount_msat'] == Millisatoshi(20 * 10**11 // 1000)
-#     assert b11['created_at'] == 1496314658
-#     assert b11['payment_hash'] == '0001020304050607080900010203040506070809000102030405060708090102'
-#     assert b11['expiry'] == 3600
-#     assert b11['payee'] == '03e7156ae33b0a208d0744199163177e909e80176e55d97a2f221ede0f934dd9ad'
-#     assert len(b11['fallbacks']) == 1
-#     assert b11['fallbacks'][0]['type'] == 'P2PKH'
-#     assert b11['fallbacks'][0]['addr'] == '1RustyRX2oai4EYYDpQGWvEL62BBGqN9T'
-#     assert len(b11['routes']) == 1
-#     assert len(b11['routes'][0]) == 2
-#     assert b11['routes'][0][0]['pubkey'] == '029e03a901b85534ff1e92c43c74431f7ce72046060fcf7a95c37e148f78c77255'
-#     # 0x010203:0x040506:0x0708
-#     assert b11['routes'][0][0]['short_channel_id'] == '66051x263430x1800'
-#     assert b11['routes'][0][0]['fee_base_msat'] == 1
-#     assert b11['routes'][0][0]['fee_proportional_millionths'] == 20
-#     assert b11['routes'][0][0]['cltv_expiry_delta'] == 3
-# 
-#     assert b11['routes'][0][1]['pubkey'] == '039e03a901b85534ff1e92c43c74431f7ce72046060fcf7a95c37e148f78c77255'
-#     # 0x030405:0x060708:0x090a
-#     assert b11['routes'][0][1]['short_channel_id'] == '197637x395016x2314'
-#     assert b11['routes'][0][1]['fee_base_msat'] == 2
-#     assert b11['routes'][0][1]['fee_proportional_millionths'] == 30
-#     assert b11['routes'][0][1]['cltv_expiry_delta'] == 4
-# 
-#     # > ### On mainnet, with fallback (P2SH) address 3EktnHQD7RiAE6uzMj2ZifT9YgRrkSgzQX
-#     # > lnbc20m1pvjluezhp58yjmdan79s6qqdhdzgynm4zwqd5d7xmw5fk98klysy043l2ahrqspp5qqqsyqcyq5rqwzqfqqqsyqcyq5rqwzqfqqqsyqcyq5rqwzqfqypqfppj3a24vwu6r8ejrss3axul8rxldph2q7z9kmrgvr7xlaqm47apw3d48zm203kzcq357a4ls9al2ea73r8jcceyjtya6fu5wzzpe50zrge6ulk4nvjcpxlekvmxl6qcs9j3tz0469gq5g658y
-#     #
-#     # Breakdown:
-#     #
-#     # * `lnbc`: prefix, lightning on bitcoin mainnet
-#     # * `20m`: amount (20 milli-bitcoin)
-#     # * `1`: Bech32 separator
-#     # * `pvjluez`: timestamp (1496314658)
-#     # * `p`: payment hash...
-#     # * `f`: tagged field: fallback address.
-#     # * `pp`: `data_length` (`p` = 1. 1 * 32 + 1 == 33)
-#     # * `j3a24vwu6r8ejrss3axul8rxldph2q7z9`: `j` = 18, so P2SH address
-#     # * `h`: tagged field: hash of description...
-#     # * `2jhz8j78lv2jynuzmz6g8ve53he7pheeype33zlja5azae957585uu7x59w0f2l3rugyva6zpu394y4rh093j6wxze0ldsvk757a9msq`: signature
-#     # * `mf9swh`: Bech32 checksum
-#     b11 = l1.rpc.decodepay('lnbc20m1pvjluezhp58yjmdan79s6qqdhdzgynm4zwqd5d7xmw5fk98klysy043l2ahrqspp5qqqsyqcyq5rqwzqfqqqsyqcyq5rqwzqfqqqsyqcyq5rqwzqfqypqfppj3a24vwu6r8ejrss3axul8rxldph2q7z9kmrgvr7xlaqm47apw3d48zm203kzcq357a4ls9al2ea73r8jcceyjtya6fu5wzzpe50zrge6ulk4nvjcpxlekvmxl6qcs9j3tz0469gq5g658y', 'One piece of chocolate cake, one icecream cone, one pickle, one slice of swiss cheese, one slice of salami, one lollypop, one piece of cherry pie, one sausage, one cupcake, and one slice of watermelon')
-#     assert b11['currency'] == 'bc'
-#     assert b11['amount_msat'] == Millisatoshi(20 * 10**11 // 1000)
-#     assert b11['created_at'] == 1496314658
-#     assert b11['payment_hash'] == '0001020304050607080900010203040506070809000102030405060708090102'
-#     assert b11['expiry'] == 3600
-#     assert b11['payee'] == '03e7156ae33b0a208d0744199163177e909e80176e55d97a2f221ede0f934dd9ad'
-#     assert len(b11['fallbacks']) == 1
-#     assert b11['fallbacks'][0]['type'] == 'P2SH'
-#     assert b11['fallbacks'][0]['addr'] == '3EktnHQD7RiAE6uzMj2ZifT9YgRrkSgzQX'
-# 
-#     # > ### On mainnet, with fallback (P2WPKH) address bc1qw508d6qejxtdg4y5r3zarvary0c5xw7kv8f3t4
-#     # > lnbc20m1pvjluezhp58yjmdan79s6qqdhdzgynm4zwqd5d7xmw5fk98klysy043l2ahrqspp5qqqsyqcyq5rqwzqfqqqsyqcyq5rqwzqfqqqsyqcyq5rqwzqfqypqfppqw508d6qejxtdg4y5r3zarvary0c5xw7kepvrhrm9s57hejg0p662ur5j5cr03890fa7k2pypgttmh4897d3raaq85a293e9jpuqwl0rnfuwzam7yr8e690nd2ypcq9hlkdwdvycqa0qza8
-#     #
-#     # * `lnbc`: prefix, lightning on bitcoin mainnet
-#     # * `20m`: amount (20 milli-bitcoin)
-#     # * `1`: Bech32 separator
-#     # * `pvjluez`: timestamp (1496314658)
-#     # * `p`: payment hash...
-#     # * `f`: tagged field: fallback address.
-#     # * `pp`: `data_length` (`p` = 1. 1 * 32 + 1 == 33)
-#     # * `q`: 0, so witness version 0.
-#     # * `qw508d6qejxtdg4y5r3zarvary0c5xw7k`: 160 bits = P2WPKH.
-#     # * `h`: tagged field: hash of description...
-#     # * `gw6tk8z0p0qdy9ulggx65lvfsg3nxxhqjxuf2fvmkhl9f4jc74gy44d5ua9us509prqz3e7vjxrftn3jnk7nrglvahxf7arye5llphgq`: signature
-#     # * `qdtpa4`: Bech32 checksum
-#     b11 = l1.rpc.decodepay('lnbc20m1pvjluezhp58yjmdan79s6qqdhdzgynm4zwqd5d7xmw5fk98klysy043l2ahrqspp5qqqsyqcyq5rqwzqfqqqsyqcyq5rqwzqfqqqsyqcyq5rqwzqfqypqfppqw508d6qejxtdg4y5r3zarvary0c5xw7kepvrhrm9s57hejg0p662ur5j5cr03890fa7k2pypgttmh4897d3raaq85a293e9jpuqwl0rnfuwzam7yr8e690nd2ypcq9hlkdwdvycqa0qza8', 'One piece of chocolate cake, one icecream cone, one pickle, one slice of swiss cheese, one slice of salami, one lollypop, one piece of cherry pie, one sausage, one cupcake, and one slice of watermelon')
-#     assert b11['currency'] == 'bc'
-#     assert b11['amount_msat'] == Millisatoshi(20 * 10**11 // 1000)
-#     assert b11['created_at'] == 1496314658
-#     assert b11['payment_hash'] == '0001020304050607080900010203040506070809000102030405060708090102'
-#     assert b11['expiry'] == 3600
-#     assert b11['payee'] == '03e7156ae33b0a208d0744199163177e909e80176e55d97a2f221ede0f934dd9ad'
-#     assert len(b11['fallbacks']) == 1
-#     assert b11['fallbacks'][0]['type'] == 'P2WPKH'
-#     assert b11['fallbacks'][0]['addr'] == 'bc1qw508d6qejxtdg4y5r3zarvary0c5xw7kv8f3t4'
-# 
-#     # > ### On mainnet, with fallback (P2WSH) address bc1qrp33g0q5c5txsp9arysrx4k6zdkfs4nce4xj0gdcccefvpysxf3qccfmv3
-#     # > lnbc20m1pvjluezhp58yjmdan79s6qqdhdzgynm4zwqd5d7xmw5fk98klysy043l2ahrqspp5qqqsyqcyq5rqwzqfqqqsyqcyq5rqwzqfqqqsyqcyq5rqwzqfqypqfp4qrp33g0q5c5txsp9arysrx4k6zdkfs4nce4xj0gdcccefvpysxf3q28j0v3rwgy9pvjnd48ee2pl8xrpxysd5g44td63g6xcjcu003j3qe8878hluqlvl3km8rm92f5stamd3jw763n3hck0ct7p8wwj463cql26ava
-#     #
-#     # * `lnbc`: prefix, lightning on bitcoin mainnet
-#     # * `20m`: amount (20 milli-bitcoin)
-#     # * `1`: Bech32 separator
-#     # * `pvjluez`: timestamp (1496314658)
-#     # * `p`: payment hash...
-#     # * `f`: tagged field: fallback address.
-#     # * `p4`: `data_length` (`p` = 1, `4` = 21. 1 * 32 + 21 == 53)
-#     # * `q`: 0, so witness version 0.
-#     # * `rp33g0q5c5txsp9arysrx4k6zdkfs4nce4xj0gdcccefvpysxf3q`: 260 bits = P2WSH.
-#     # * `h`: tagged field: hash of description...
-#     # * `5yps56lmsvgcrf476flet6js02m93kgasews8q3jhtp7d6cqckmh70650maq4u65tk53ypszy77v9ng9h2z3q3eqhtc3ewgmmv2grasp`: signature
-#     # * `akvd7y`: Bech32 checksum
-#     b11 = l1.rpc.decodepay('lnbc20m1pvjluezhp58yjmdan79s6qqdhdzgynm4zwqd5d7xmw5fk98klysy043l2ahrqspp5qqqsyqcyq5rqwzqfqqqsyqcyq5rqwzqfqqqsyqcyq5rqwzqfqypqfp4qrp33g0q5c5txsp9arysrx4k6zdkfs4nce4xj0gdcccefvpysxf3q28j0v3rwgy9pvjnd48ee2pl8xrpxysd5g44td63g6xcjcu003j3qe8878hluqlvl3km8rm92f5stamd3jw763n3hck0ct7p8wwj463cql26ava', 'One piece of chocolate cake, one icecream cone, one pickle, one slice of swiss cheese, one slice of salami, one lollypop, one piece of cherry pie, one sausage, one cupcake, and one slice of watermelon')
-#     assert b11['currency'] == 'bc'
-#     assert b11['amount_msat'] == Millisatoshi(20 * 10**11 // 1000)
-#     assert b11['created_at'] == 1496314658
-#     assert b11['payment_hash'] == '0001020304050607080900010203040506070809000102030405060708090102'
-#     assert b11['expiry'] == 3600
-#     assert b11['payee'] == '03e7156ae33b0a208d0744199163177e909e80176e55d97a2f221ede0f934dd9ad'
-#     assert len(b11['fallbacks']) == 1
-#     assert b11['fallbacks'][0]['type'] == 'P2WSH'
-#     assert b11['fallbacks'][0]['addr'] == 'bc1qrp33g0q5c5txsp9arysrx4k6zdkfs4nce4xj0gdcccefvpysxf3qccfmv3'
-# 
-#     # > ### Please send $30 for coffee beans to the same peer, which supports features 1 and 9
-#     # > lnbc25m1pvjluezpp5qqqsyqcyq5rqwzqfqqqsyqcyq5rqwzqfqqqsyqcyq5rqwzqfqypqdq5vdhkven9v5sxyetpdees9qzsze992adudgku8p05pstl6zh7av6rx2f297pv89gu5q93a0hf3g7lynl3xq56t23dpvah6u7y9qey9lccrdml3gaqwc6nxsl5ktzm464sq73t7cl
-#     #
-#     # Breakdown:
-#     #
-#     # * `lnbc`: prefix, Lightning on Bitcoin mainnet
-#     # * `25m`: amount (25 milli-bitcoin)
-#     # * `1`: Bech32 separator
-#     # * `pvjluez`: timestamp (1496314658)
-#     # * `p`: payment hash...
-#     # * `d`: short description
-#     #   * `q5`: `data_length` (`q` = 0, `5` = 20; 0 * 32 + 20 == 20)
-#     #   * `vdhkven9v5sxyetpdees`: 'coffee beans'
-#     # * `9`: features
-#     #   * `qz`: `data_length` (`q` = 0, `z` = 2; 0 * 32 + 2 == 2)
-#     #   * `sz`: b1000000010
-#     # * `e992adudgku8p05pstl6zh7av6rx2f297pv89gu5q93a0hf3g7lynl3xq56t23dpvah6u7y9qey9lccrdml3gaqwc6nxsl5ktzm464sq`: signature
-#     # * `73t7cl`: Bech32 checksum
-#     b11 = l1.rpc.decodepay('lnbc25m1pvjluezpp5qqqsyqcyq5rqwzqfqqqsyqcyq5rqwzqfqqqsyqcyq5rqwzqfqypqdq5vdhkven9v5sxyetpdees9qzsze992adudgku8p05pstl6zh7av6rx2f297pv89gu5q93a0hf3g7lynl3xq56t23dpvah6u7y9qey9lccrdml3gaqwc6nxsl5ktzm464sq73t7cl')
-#     assert b11['currency'] == 'bc'
-#     assert b11['amount_msat'] == Millisatoshi(25 * 10**11 // 1000)
-#     assert b11['created_at'] == 1496314658
-#     assert b11['payment_hash'] == '0001020304050607080900010203040506070809000102030405060708090102'
-#     assert b11['description'] == 'coffee beans'
-#     assert b11['expiry'] == 3600
-#     assert b11['payee'] == '03e7156ae33b0a208d0744199163177e909e80176e55d97a2f221ede0f934dd9ad'
-#     assert b11['features'] == '0202'
-# 
-#     # > # Same, but using invalid unknown feature 100
-#     # > lnbc25m1pvjluezpp5qqqsyqcyq5rqwzqfqqqsyqcyq5rqwzqfqqqsyqcyq5rqwzqfqypqdq5vdhkven9v5sxyetpdees9q4pqqqqqqqqqqqqqqqqqqszk3ed62snp73037h4py4gry05eltlp0uezm2w9ajnerhmxzhzhsu40g9mgyx5v3ad4aqwkmvyftzk4k9zenz90mhjcy9hcevc7r3lx2sphzfxz7
-#     #
-#     # Breakdown:
-#     #
-#     # * `lnbc`: prefix, Lightning on Bitcoin mainnet
-#     # * `25m`: amount (25 milli-bitcoin)
-#     # * `1`: Bech32 separator
-#     # * `pvjluez`: timestamp (1496314658)
-#     # * `p`: payment hash...
-#     # * `d`: short description
-#     #   * `q5`: `data_length` (`q` = 0, `5` = 20; 0 * 32 + 20 == 20)
-#     #   * `vdhkven9v5sxyetpdees`: 'coffee beans'
-#     # * `9`: features
-#     #   * `q4`: `data_length` (`q` = 0, `4` = 21; 0 * 32 + 21 == 21)
-#     #   * `pqqqqqqqqqqqqqqqqqqsz`: b00001...(90 zeroes)...1000000010
-#     # * `k3ed62snp73037h4py4gry05eltlp0uezm2w9ajnerhmxzhzhsu40g9mgyx5v3ad4aqwkmvyftzk4k9zenz90mhjcy9hcevc7r3lx2sp`: signature
-#     # * `hzfxz7`: Bech32 checksum
-#     with pytest.raises(RpcError, match='unknown feature.*100'):
-#         l1.rpc.decodepay('lnbc25m1pvjluezpp5qqqsyqcyq5rqwzqfqqqsyqcyq5rqwzqfqqqsyqcyq5rqwzqfqypqdq5vdhkven9v5sxyetpdees9q4pqqqqqqqqqqqqqqqqqqszk3ed62snp73037h4py4gry05eltlp0uezm2w9ajnerhmxzhzhsu40g9mgyx5v3ad4aqwkmvyftzk4k9zenz90mhjcy9hcevc7r3lx2sphzfxz7')
-# 
-#     # Example of an invoice without a multiplier suffix to the amount. This
-#     # should then be interpreted as 7 BTC according to the spec:
-#     #
-#     #   `amount`: optional number in that currency, followed by an optional
-#     #   `multiplier` letter. The unit encoded here is the 'social' convention of
-#     #   a payment unit -- in the case of Bitcoin the unit is 'bitcoin' NOT
-#     #   satoshis.
-#     b11 = "lnbcrt71p0g4u8upp5xn4k45tsp05akmn65s5k2063d5fyadhjse9770xz5sk7u4x6vcmqdqqcqzynxqrrssx94cf4p727jamncsvcd8m99n88k423ruzq4dxwevfatpp5gx2mksj2swshjlx4pe3j5w9yed5xjktrktzd3nc2a04kq8yu84l7twhwgpxjn3pw"
-#     b11 = l1.rpc.decodepay(b11)
-#     sat_per_btc = 10**8
-#     assert(b11['amount_msat'] == 7 * sat_per_btc * 1000)
-# 
-#     with pytest.raises(RpcError):
-#         l1.rpc.decodepay('1111111')
-# 
-# 
-# @pytest.mark.developer("Too slow without --dev-fast-gossip")
-# def test_forward(node_factory, bitcoind):
-#     # Connect 1 -> 2 -> 3.
-#     l1, l2, l3 = node_factory.line_graph(3, wait_for_announce=True)
-# 
-#     # If they're at different block heights we can get spurious errors.
-#     sync_blockheight(bitcoind, [l1, l2, l3])
-# 
-#     chanid1 = only_one(l1.rpc.listpeerchannels(l2.info['id'])['channels'])['short_channel_id']
-#     chanid2 = only_one(l2.rpc.listpeerchannels(l3.info['id'])['channels'])['short_channel_id']
-#     assert only_one(l2.rpc.listpeerchannels(l1.info['id'])['channels'])['short_channel_id'] == chanid1
-#     assert only_one(l3.rpc.listpeerchannels(l2.info['id'])['channels'])['short_channel_id'] == chanid2
-# 
-#     inv = l3.rpc.invoice(100000000, 'testpayment1', 'desc')
-#     rhash = inv['payment_hash']
-#     assert only_one(l3.rpc.listinvoices('testpayment1')['invoices'])['status'] == 'unpaid'
-# 
-#     # Fee for node2 is 10 millionths, plus 1.
-#     amt = 100000000
-#     fee = amt * 10 // 1000000 + 1
-# 
-#     baseroute = [{'amount_msat': amt + fee,
-#                   'id': l2.info['id'],
-#                   'delay': 12,
-#                   'channel': chanid1},
-#                  {'amount_msat': amt,
-#                   'id': l3.info['id'],
-#                   'delay': 6,
-#                   'channel': chanid2}]
-# 
-#     # Unknown other peer
-#     route = copy.deepcopy(baseroute)
-#     route[1]['id'] = '031a8dc444e41bb989653a4501e11175a488a57439b0c4947704fd6e3de5dca607'
-#     l1.rpc.sendpay(route, rhash, payment_secret=inv['payment_secret'])
-#     with pytest.raises(RpcError):
-#         l1.rpc.waitsendpay(rhash)
-# 
-#     # Delay too short (we always add one internally anyway, so subtract 2 here).
-#     route = copy.deepcopy(baseroute)
-#     route[0]['delay'] = 8
-#     l1.rpc.sendpay(route, rhash, payment_secret=inv['payment_secret'])
-#     with pytest.raises(RpcError):
-#         l1.rpc.waitsendpay(rhash)
-# 
-#     # Final delay too short
-#     route = copy.deepcopy(baseroute)
-#     route[1]['delay'] = 3
-#     l1.rpc.sendpay(route, rhash, payment_secret=inv['payment_secret'])
-#     with pytest.raises(RpcError):
-#         l1.rpc.waitsendpay(rhash)
-# 
-#     # This one works
-#     route = copy.deepcopy(baseroute)
-#     l1.rpc.sendpay(route, rhash, payment_secret=inv['payment_secret'])
-#     l1.rpc.waitsendpay(rhash)
-# 
-#     # Check that invoice payment and fee are tracked appropriately
-#     l1.daemon.wait_for_log('coin_move .* [(]invoice[)]')
-#     l1.rpc.bkpr_dumpincomecsv('koinly', 'koinly.csv')
-# 
-#     koinly_path = os.path.join(l1.daemon.lightning_dir, TEST_NETWORK, 'koinly.csv')
-#     koinly_csv = open(koinly_path, 'rb').read()
-#     expected_line = r'0.00100000000,.*,,,0.00000001001,.*,invoice'
-#     assert only_one(re.findall(expected_line, str(koinly_csv)))
-# 
-# 
-# @pytest.mark.developer("needs --dev-fast-gossip")
-# def test_forward_different_fees_and_cltv(node_factory, bitcoind):
-#     # FIXME: Check BOLT quotes here too
-#     # BOLT #7:
-#     # ```
-#     #    B
-#     #   / \
-#     #  /   \
-#     # A     C
-#     #  \   /
-#     #   \ /
-#     #    D
-#     # ```
-#     #
-#     # Each advertises the following `cltv_expiry_delta` on its end of every
-#     # channel:
-#     #
-#     # 1. A: 10 blocks
-#     # 2. B: 20 blocks
-#     # 3. C: 30 blocks
-#     # 4. D: 40 blocks
-#     #
-#     # C also uses a minimum `cltv_expiry` of 9 (the default) when requesting
-#     # payments.
-#     #
-#     # Also, each node has the same fee scheme which it uses for each of its
-#     # channels:
-#     #
-#     # 1. A: 100 base + 1000 millionths
-#     # 1. B: 200 base + 2000 millionths
-#     # 1. C: 300 base + 3000 millionths
-#     # 1. D: 400 base + 4000 millionths
-# 
-#     # We don't do D yet.
-#     l1, l2, l3 = node_factory.get_nodes(3, opts=[{'cltv-delta': 10, 'fee-base': 100, 'fee-per-satoshi': 1000},
-#                                                  {'cltv-delta': 20, 'fee-base': 200, 'fee-per-satoshi': 2000},
-#                                                  {'cltv-delta': 30, 'cltv-final': 9, 'fee-base': 300, 'fee-per-satoshi': 3000}])
-# 
-#     ret = l1.rpc.connect(l2.info['id'], 'localhost', l2.port)
-#     assert ret['id'] == l2.info['id']
-# 
-#     l1.daemon.wait_for_log('Handed peer, entering loop')
-#     l2.daemon.wait_for_log('Handed peer, entering loop')
-# 
-#     ret = l2.rpc.connect(l3.info['id'], 'localhost', l3.port)
-#     assert ret['id'] == l3.info['id']
-# 
-#     l2.daemon.wait_for_log('Handed peer, entering loop')
-#     l3.daemon.wait_for_log('Handed peer, entering loop')
-# 
-#     c1, _ = l1.fundchannel(l2, 10**6)
-#     c2, _ = l2.fundchannel(l3, 10**6)
-#     mine_funding_to_announce(bitcoind, [l1, l2, l3])
-# 
-#     # Make sure l1 has seen announce for all channels.
-#     l1.wait_channel_active(c1)
-#     l1.wait_channel_active(c2)
-# 
-#     # BOLT #7:
-#     #
-#     # If B were to send 4,999,999 millisatoshi directly to C, it wouldn't
-#     # charge itself a fee nor add its own `cltv_expiry_delta`, so it would
-#     # use C's requested `cltv_expiry` of 9.  We also assume it adds a
-#     # "shadow route" to give an extra CLTV of 42.  It could also add extra
-#     # cltv deltas at other hops, as these values are a minimum, but we don't
-#     # here for simplicity:
-# 
-#     # FIXME: Add shadow route
-#     shadow_route = 0
-#     route = l2.rpc.getroute(l3.info['id'], 4999999, 1)["route"]
-#     assert len(route) == 1
-# 
-#     # BOLT #7:
-#     #
-#     #    * `amount_msat`: 4999999
-#     #    * `cltv_expiry`: current-block-height + 9 + 42
-#     #    * `onion_routing_packet`:
-#     #      * `amt_to_forward` = 4999999
-#     #      * `outgoing_cltv_value` = current-block-height + 9 + 42
-#     #
-#     assert route[0]['amount_msat'] == 4999999
-#     assert route[0]['delay'] == 9 + shadow_route
-# 
-#     # BOLT #7:
-#     # If A were to send 4,999,999 millisatoshi to C via B, it needs to
-#     # pay B the fee it specified in the B->C `channel_update`, calculated as
-#     # per [HTLC Fees](#htlc_fees):
-#     #
-#     # 200 + 4999999 * 2000 / 1000000 = 10199
-#     #
-#     # Similarly, it would need to add the `cltv_expiry` from B->C's
-#     # `channel_update` (20), plus C's requested minimum (9), plus 42 for the
-#     # "shadow route".  Thus the `update_add_htlc` message from A to B would
-#     # be:
-#     #
-#     #    * `amount_msat`: 5010198
-#     #    * `cltv_expiry`: current-block-height + 20 + 9 + 42
-#     #    * `onion_routing_packet`:
-#     #      * `amt_to_forward` = 4999999
-#     #      * `outgoing_cltv_value` = current-block-height + 9 + 42
-#     route = l1.rpc.getroute(l3.info['id'], 4999999, 1)["route"]
-#     assert len(route) == 2
-# 
-#     assert route[0]['amount_msat'] == 5010198
-#     assert route[0]['delay'] == 20 + 9 + shadow_route
-#     assert route[1]['amount_msat'] == 4999999
-#     assert route[1]['delay'] == 9 + shadow_route
-# 
-#     inv = l3.rpc.invoice(4999999, 'test_forward_different_fees_and_cltv', 'desc')
-#     rhash = inv['payment_hash']
-#     assert only_one(l3.rpc.listinvoices('test_forward_different_fees_and_cltv')['invoices'])['status'] == 'unpaid'
-# 
-#     # This should work.
-#     l1.rpc.sendpay(route, rhash, payment_secret=inv['payment_secret'])
-#     l1.rpc.waitsendpay(rhash)
-# 
-#     # We add one to the blockcount for a bit of fuzz (FIXME: Shadowroute would fix this!)
-#     shadow_route = 1
-#     l1.daemon.wait_for_log("Adding HTLC 0 amount=5010198msat cltv={} gave CHANNEL_ERR_ADD_OK"
-#                            .format(bitcoind.rpc.getblockcount() + 20 + 9 + shadow_route))
-#     l2.daemon.wait_for_log("Adding HTLC 0 amount=4999999msat cltv={} gave CHANNEL_ERR_ADD_OK"
-#                            .format(bitcoind.rpc.getblockcount() + 9 + shadow_route))
-#     l3.daemon.wait_for_log("Resolved invoice 'test_forward_different_fees_and_cltv' with amount 4999999msat")
-#     assert only_one(l3.rpc.listinvoices('test_forward_different_fees_and_cltv')['invoices'])['status'] == 'paid'
-# 
-#     # Check that we see all the channels
-#     shortids = set(c['short_channel_id'] for c in l2.rpc.listchannels()['channels'])
-#     for scid in shortids:
-#         c = l1.rpc.listchannels(scid)['channels']
-#         # We get one entry for each direction.
-#         assert len(c) == 2
-#         assert c[0]['short_channel_id'] == scid
-#         assert c[1]['short_channel_id'] == scid
-#         assert c[0]['source'] == c[1]['destination']
-#         assert c[1]['source'] == c[0]['destination']
-# 
-# 
-# @pytest.mark.developer("too slow without --dev-fast-gossip")
-# def test_forward_pad_fees_and_cltv(node_factory, bitcoind):
-#     """Test that we are allowed extra locktime delta, and fees"""
-# 
-#     l1, l2, l3 = node_factory.get_nodes(3, opts=[{'cltv-delta': 10, 'fee-base': 100, 'fee-per-satoshi': 1000},
-#                                                  {'cltv-delta': 20, 'fee-base': 200, 'fee-per-satoshi': 2000},
-#                                                  {'cltv-delta': 30, 'cltv-final': 9, 'fee-base': 300, 'fee-per-satoshi': 3000}])
-# 
-#     ret = l1.rpc.connect(l2.info['id'], 'localhost', l2.port)
-#     assert ret['id'] == l2.info['id']
-# 
-#     l1.daemon.wait_for_log('Handed peer, entering loop')
-#     l2.daemon.wait_for_log('Handed peer, entering loop')
-# 
-#     ret = l2.rpc.connect(l3.info['id'], 'localhost', l3.port)
-#     assert ret['id'] == l3.info['id']
-# 
-#     l2.daemon.wait_for_log('Handed peer, entering loop')
-#     l3.daemon.wait_for_log('Handed peer, entering loop')
-# 
-#     c1, _ = l1.fundchannel(l2, 10**6)
-#     c2, _ = l2.fundchannel(l3, 10**6)
-#     mine_funding_to_announce(bitcoind, [l1, l2, l3])
-# 
-#     # Make sure l1 has seen announce for all channels.
-#     l1.wait_channel_active(c1)
-#     l1.wait_channel_active(c2)
-# 
-#     route = l1.rpc.getroute(l3.info['id'], 4999999, 1)["route"]
-#     assert len(route) == 2
-# 
-#     assert route[0]['amount_msat'] == 5010198
-#     assert route[0]['delay'] == 20 + 9
-#     assert route[1]['amount_msat'] == 4999999
-#     assert route[1]['delay'] == 9
-# 
-#     # Modify so we overpay, overdo the cltv.
-#     route[0]['amount_msat'] += 2000
-#     route[0]['delay'] += 20
-#     route[1]['amount_msat'] += 1000
-#     route[1]['delay'] += 10
-# 
-#     # This should work.
-#     inv = l3.rpc.invoice(4999999, 'test_forward_pad_fees_and_cltv', 'desc')
-#     rhash = inv['payment_hash']
-#     l1.rpc.sendpay(route, rhash, payment_secret=inv['payment_secret'])
-#     l1.rpc.waitsendpay(rhash)
-#     assert only_one(l3.rpc.listinvoices('test_forward_pad_fees_and_cltv')['invoices'])['status'] == 'paid'
-# 
-#     # Do some checks of the bookkeeper's records
-#     def _income_tagset(node, tagset):
-#         incomes = node.rpc.bkpr_listincome()['income_events']
-#         return [e for e in incomes if e['tag'] in tagset]
-# 
-#     tags = ['invoice', 'invoice_fee']
-#     wait_for(lambda: len(_income_tagset(l1, tags)) == 2)
-#     incomes = _income_tagset(l1, tags)
-#     # the balance on l3 should equal the invoice
-#     accts = l3.rpc.bkpr_listbalances()['accounts']
-#     assert len(accts) == 2
-#     wallet = accts[0]
-#     chan_acct = accts[1]
-#     assert wallet['account'] == 'wallet'
-#     assert only_one(wallet['balances'])['balance_msat'] == Millisatoshi(0)
-#     assert incomes[0]['tag'] == 'invoice'
-#     assert only_one(chan_acct['balances'])['balance_msat'] == incomes[0]['debit_msat']
-#     inve = only_one([e for e in l1.rpc.bkpr_listaccountevents()['events'] if e['tag'] == 'invoice'])
-#     assert inve['debit_msat'] == incomes[0]['debit_msat'] + incomes[1]['debit_msat']
-# 
-# 
-# @pytest.mark.developer("needs DEVELOPER=1 for dev_ignore_htlcs")
-# def test_forward_stats(node_factory, bitcoind):
-#     """Check that we track forwarded payments correctly.
-# 
-#     We wire up the network to have l1 as payment initiator, l2 as
-#     forwarded (the one we check) and l3-l5 as payment recipients. l3
-#     accepts correctly, l4 rejects (because it doesn't know the payment
-#     hash) and l5 will keep the HTLC dangling by disconnecting.
-# 
-#     """
-#     amount = 10**5
-#     l1, l2, l3, l4, l5 = node_factory.get_nodes(5, opts=[{}] * 4 + [{'may_fail': True}])
-#     node_factory.join_nodes([l1, l2, l3], wait_for_announce=False)
-#     l2.openchannel(l4, 10**6, wait_for_announce=False)
-#     l2.openchannel(l5, 10**6, wait_for_announce=False)
-# 
-#     mine_funding_to_announce(bitcoind, [l1, l2, l3, l4, l5])
-# 
-#     wait_for(lambda: len(l1.rpc.listchannels()['channels']) == 8)
-# 
-#     inv = l3.rpc.invoice(amount, "first", "desc")
-#     payment_hash = inv['payment_hash']
-#     route = l1.rpc.getroute(l3.info['id'], amount, 1)['route']
-# 
-#     l1.rpc.sendpay(route, payment_hash, payment_secret=inv['payment_secret'])
-#     l1.rpc.waitsendpay(payment_hash)
-# 
-#     # l4 rejects since it doesn't know the payment_hash
-#     route = l1.rpc.getroute(l4.info['id'], amount, 1)['route']
-#     payment_hash = "F" * 64
-#     with pytest.raises(RpcError):
-#         l1.rpc.sendpay(route, payment_hash, payment_secret=inv['payment_secret'])
-#         l1.rpc.waitsendpay(payment_hash)
-# 
-#     # l5 will hold the HTLC hostage.
-#     l5.rpc.dev_ignore_htlcs(id=l2.info['id'], ignore=True)
-#     route = l1.rpc.getroute(l5.info['id'], amount, 1)['route']
-#     inv = l5.rpc.invoice(amount, "first", "desc")
-#     payment_hash = inv['payment_hash']
-#     l1.rpc.sendpay(route, payment_hash, payment_secret=inv['payment_secret'])
-# 
-#     l5.daemon.wait_for_log(r'their htlc .* dev_ignore_htlcs')
-# 
-#     # Select all forwardings, ordered by htlc_id to ensure the order
-#     # matches below
-#     forwardings = l2.db_query("SELECT *, in_msatoshi - out_msatoshi as fee "
-#                               "FROM forwards "
-#                               "ORDER BY in_htlc_id;")
-#     assert(len(forwardings) == 3)
-#     states = [f['state'] for f in forwardings]
-#     assert(states == [1, 2, 0])  # settled, failed, offered
-# 
-#     inchan = l2.rpc.listpeerchannels(l1.info['id'])['channels'][0]
-#     outchan = l2.rpc.listpeerchannels(l3.info['id'])['channels'][0]
-# 
-#     # Check that we correctly account channel changes
-#     assert inchan['in_payments_offered'] == 3
-#     assert inchan['in_payments_fulfilled'] == 1
-#     assert inchan['in_offered_msat'] >= Millisatoshi(3 * amount)
-#     assert inchan['in_fulfilled_msat'] >= Millisatoshi(amount)
-# 
-#     assert outchan['out_payments_offered'] == 1
-#     assert outchan['out_payments_fulfilled'] == 1
-#     assert outchan['out_offered_msat'] >= Millisatoshi(amount)
-#     assert outchan['out_offered_msat'] == outchan['out_fulfilled_msat']
-# 
-#     assert outchan['out_fulfilled_msat'] < inchan['in_fulfilled_msat']
-# 
-#     stats = l2.rpc.listforwards()
-# 
-#     assert [f['status'] for f in stats['forwards']] == ['settled', 'failed', 'offered']
-#     assert l2.rpc.getinfo()['fees_collected_msat'] == 1 + amount // 100000
-#     assert l1.rpc.getinfo()['fees_collected_msat'] == 0
-#     assert l3.rpc.getinfo()['fees_collected_msat'] == 0
-#     assert stats['forwards'][0]['received_time'] <= stats['forwards'][0]['resolved_time']
-#     assert stats['forwards'][1]['received_time'] <= stats['forwards'][1]['resolved_time']
-#     assert 'received_time' in stats['forwards'][2] and 'resolved_time' not in stats['forwards'][2]
-# 
-# 
-# @pytest.mark.developer("too slow without --dev-fast-gossip")
-# @pytest.mark.slow_test
-# def test_forward_local_failed_stats(node_factory, bitcoind, executor):
-#     """Check that we track forwarded payments correctly.
-# 
-#     We wire up the network to have l1 and l6 as payment initiator, l2 as
-#     forwarded (the one we check) and l3-l5 as payment recipients.
-# 
-#     There 5 cases for FORWARD_LOCAL_FAILED status:
-#     1. When Msater resolves the reply about the next peer infor(sent
-#        by Gossipd), and need handle unknown next peer failure in
-#        channel_resolve_reply(). For this case, we ask l1 pay to l3
-#        through l2 but close the channel between l2 and l3 after
-#        getroute(), the payment will fail in l2 because of
-#        WIRE_UNKNOWN_NEXT_PEER;
-#     2. When Master handle the forward process with the htlc_in and
-#        the id of next hop, it tries to drive a new htlc_out but fails
-#        in forward_htlc(). For this case, we ask l1 pay to 14 through
-#        with no fee, so the payment will fail in l2 becase of
-#        WIRE_FEE_INSUFFICIENT;
-#     3. When we send htlc_out, Master asks Channeld to add a new htlc
-#        into the outgoing channel but Channeld fails. Master need
-#        handle and store this failure in rcvd_htlc_reply(). For this
-#        case, we ask l1 pay to l5 with 10**8 sat though the channel
-#        (l2-->l5) with the max capacity of 10**4 msat , the payment
-#        will fail in l2 because of CHANNEL_ERR_MAX_HTLC_VALUE_EXCEEDED;
-#     4. When Channeld receives a new revoke message, if the state of
-#        corresponding htlc is RCVD_ADD_ACK_REVOCATION, Master will tries
-#        to resolve onionpacket and handle the failure before resolving
-#        the next hop in peer_got_revoke(). For this case, we ask l6 pay
-#        to l4 though l1 and l2, but we replace the second node_id in route
-#        with the wrong one, so the payment will fail in l2 because of
-#        WIRE_INVALID_ONION_KEY;
-#     5. When Onchaind finds the htlc time out or missing htlc, Master
-#        need handle these failure as FORWARD_LOCAL_FAILED in if it's forward
-#        payment case. For this case, we ask l1 pay to l4 though l2 with the
-#        amount less than the invoice(the payment must fail in l4), and we
-#        also ask l5 disconnected before sending update_fail_htlc, so the
-#        htlc will be holding until l2 meets timeout and handle it as local_fail.
-#     """
-# 
-#     amount = 10**8
-# 
-#     disconnects = ['-WIRE_UPDATE_FAIL_HTLC', 'permfail']
-# 
-#     l1, l2, l3, l4, l5, l6 = node_factory.get_nodes(6, opts=[{},
-#                                                              {},
-#                                                              {},
-#                                                              {'disconnect': disconnects},
-#                                                              {},
-#                                                              {}])
-# 
-#     l1.rpc.connect(l2.info['id'], 'localhost', l2.port)
-#     l2.rpc.connect(l3.info['id'], 'localhost', l3.port)
-#     l2.rpc.connect(l4.info['id'], 'localhost', l4.port)
-#     l2.rpc.connect(l5.info['id'], 'localhost', l5.port)
-#     l6.rpc.connect(l1.info['id'], 'localhost', l1.port)
-#     c12, _ = l1.fundchannel(l2, 10**6)
-#     c23, _ = l2.fundchannel(l3, 10**6)
-#     c24, _ = l2.fundchannel(l4, 10**6)
-#     c25, _ = l2.fundchannel(l5, 10**4 * 3)
-#     l6.fundchannel(l1, 10**6)
-# 
-#     # Make sure routes finalized.
-#     mine_funding_to_announce(bitcoind, [l1, l2, l3, l4, l5, l6])
-#     l1.wait_channel_active(c23)
-#     l1.wait_channel_active(c24)
-#     l1.wait_channel_active(c25)
-#     l6.wait_channel_active(c24)
-# 
-#     wait_for(lambda: len(l1.rpc.listchannels()['channels']) == 10)
-# 
-#     """1. When Msater resolves the reply about the next peer infor(sent
-#        by Gossipd), and need handle unknown next peer failure in
-#        channel_resolve_reply();
-# 
-#        For this case, we ask l1 pay to l3 through l2 but close the channel
-#        between l2 and l3 after getroute(), the payment will fail in l2
-#        because of WIRE_UNKNOWN_NEXT_PEER;
-#     """
-# 
-#     inv = l3.rpc.invoice(amount, "first", "desc")
-#     payment_hash = inv['payment_hash']
-#     route = l1.rpc.getroute(l3.info['id'], amount, 1)['route']
-# 
-#     l2.rpc.close(c23, 1)
-# 
-#     with pytest.raises(RpcError):
-#         l1.rpc.sendpay(route, payment_hash, payment_secret=inv['payment_secret'])
-#         l1.rpc.waitsendpay(payment_hash)
-# 
-#     """2. When Master handle the forward process with the htlc_in and
-#        the id of next hop, it tries to drive a new htlc_out but fails
-#        in forward_htlc();
-# 
-#        For this case, we ask l1 pay to 14 through with no fee, so the
-#        payment will fail in l2 becase of WIRE_FEE_INSUFFICIENT;
-#     """
-# 
-#     inv = l4.rpc.invoice(amount, "third", "desc")
-#     payment_hash = inv['payment_hash']
-#     fee = amount * 10 // 1000000 + 1
-# 
-#     route = [{'amount_msat': amount,
-#               'id': l2.info['id'],
-#               'delay': 12,
-#               'channel': c12},
-#              {'amount_msat': amount,
-#               'id': l4.info['id'],
-#               'delay': 6,
-#               'channel': c24}]
-# 
-#     with pytest.raises(RpcError):
-#         l1.rpc.sendpay(route, payment_hash, payment_secret=inv['payment_secret'])
-#         l1.rpc.waitsendpay(payment_hash)
-# 
-#     """3. When we send htlc_out, Master asks Channeld to add a new htlc
-#        into the outgoing channel but Channeld fails. Master need
-#        handle and store this failure in rcvd_htlc_reply();
-# 
-#        For this case, we ask l1 pay to l5 with 10**8 sat though the channel
-#        (l2-->l5) with the max capacity of 10**4 msat , the payment will
-#        fail in l2 because of CHANNEL_ERR_MAX_HTLC_VALUE_EXCEEDED;
-#     """
-# 
-#     inv = l5.rpc.invoice(amount, "second", "desc")
-#     payment_hash = inv['payment_hash']
-#     fee = amount * 10 // 1000000 + 1
-# 
-#     route = [{'amount_msat': amount + fee,
-#               'id': l2.info['id'],
-#               'delay': 12,
-#               'channel': c12},
-#              {'amount_msat': amount,
-#               'id': l5.info['id'],
-#               'delay': 6,
-#               'channel': c25}]
-# 
-#     with pytest.raises(RpcError):
-#         l1.rpc.sendpay(route, payment_hash, payment_secret=inv['payment_secret'])
-#         l1.rpc.waitsendpay(payment_hash)
-# 
-#     """4. When Channeld receives a new revoke message, if the state of
-#        corresponding htlc is RCVD_ADD_ACK_REVOCATION, Master will tries
-#        to resolve onionpacket and handle the failure before resolving
-#        the next hop in peer_got_revoke();
-# 
-#        For this case, we ask l6 pay to l4 though l1 and l2, but we replace
-#        the second node_id in route with the wrong one, so the payment will
-#        fail in l2 because of WIRE_INVALID_ONION_KEY;
-#     """
-# 
-#     inv = l4.rpc.invoice(amount, 'fourth', 'desc')
-#     payment_hash = inv['payment_hash']
-#     route = l6.rpc.getroute(l4.info['id'], amount, 1)['route']
-# 
-#     mangled_nodeid = '0265b6ab5ec860cd257865d61ef0bbf5b3339c36cbda8b26b74e7f1dca490b6510'
-# 
-#     # Replace id with a different pubkey, so onion encoded badly at l2 hop.
-#     route[1]['id'] = mangled_nodeid
-# 
-#     with pytest.raises(RpcError):
-#         l6.rpc.sendpay(route, payment_hash, payment_secret=inv['payment_secret'])
-#         l6.rpc.waitsendpay(payment_hash)
-# 
-#     """5. When Onchaind finds the htlc time out or missing htlc, Master
-#        need handle these failure as FORWARD_LOCAL_FAILED in if it's forward
-#        payment case.
-# 
-#        For this case, we ask l1 pay to l4 though l2 with the amount less than
-#        the invoice(the payment must fail in l4), and we also ask l5 disconnected
-#        before sending update_fail_htlc, so the htlc will be holding until l2
-#        meets timeout and handle it as local_fail.
-#     """
-#     inv = l4.rpc.invoice(amount, 'onchain_timeout', 'desc')
-#     payment_hash = inv['payment_hash']
-#     fee = amount * 10 // 1000000 + 1
-# 
-#     # We underpay, so it fails.
-#     route = [{'amount_msat': amount + fee - 1,
-#               'id': l2.info['id'],
-#               'delay': 12,
-#               'channel': c12},
-#              {'amount_msat': amount - 1,
-#               'id': l4.info['id'],
-#               'delay': 5,
-#               'channel': c24}]
-# 
-#     executor.submit(l1.rpc.sendpay, route, payment_hash, payment_secret=inv['payment_secret'])
-# 
-#     l4.daemon.wait_for_log('permfail')
-#     l4.wait_for_channel_onchain(l2.info['id'])
-#     l2.bitcoin.generate_block(1)
-#     l2.daemon.wait_for_log(' to ONCHAIN')
-#     l4.daemon.wait_for_log(' to ONCHAIN')
-# 
-#     # Wait for timeout.
-#     l2.daemon.wait_for_log('Propose handling THEIR_UNILATERAL/OUR_HTLC by OUR_HTLC_TIMEOUT_TO_US .* after 6 blocks')
-#     bitcoind.generate_block(6)
-# 
-#     l2.wait_for_onchaind_broadcast('OUR_HTLC_TIMEOUT_TO_US',
-#                                    'THEIR_UNILATERAL/OUR_HTLC')
-# 
-#     bitcoind.generate_block(1)
-#     l2.daemon.wait_for_log('Resolved THEIR_UNILATERAL/OUR_HTLC by our proposal OUR_HTLC_TIMEOUT_TO_US')
-#     l4.daemon.wait_for_log('Ignoring output.*: OUR_UNILATERAL/THEIR_HTLC')
-# 
-#     bitcoind.generate_block(100)
-#     sync_blockheight(bitcoind, [l2])
-# 
-#     # give time to let l2 store the local_failed stats
-#     time.sleep(5)
-# 
-#     # Select all forwardings, and check the status
-#     stats = l2.rpc.listforwards()
-# 
-#     assert [f['status'] for f in stats['forwards']] == ['local_failed', 'local_failed', 'local_failed', 'local_failed', 'local_failed']
-#     assert l2.rpc.getinfo()['fees_collected_msat'] == 0
-# 
-#     assert 'received_time' in stats['forwards'][0] and 'resolved_time' not in stats['forwards'][0]
-#     assert 'received_time' in stats['forwards'][1] and 'resolved_time' not in stats['forwards'][1]
-#     assert 'received_time' in stats['forwards'][2] and 'resolved_time' not in stats['forwards'][2]
-#     assert 'received_time' in stats['forwards'][3] and 'resolved_time' not in stats['forwards'][3]
-#     assert 'received_time' in stats['forwards'][3] and 'resolved_time' not in stats['forwards'][4]
-# 
-#     # Correct in and out channels
-#     assert [s['in_channel'] for s in stats['forwards']] == [c12] * 5
-#     assert [s.get('out_channel') for s in stats['forwards']] == [c23, c24, c25, None, c24]
-# 
-# 
-# @pytest.mark.developer("too slow without --dev-fast-gossip")
-# @pytest.mark.slow_test
-# def test_htlcs_cltv_only_difference(node_factory, bitcoind):
-#     # l1 -> l2 -> l3 -> l4
-#     # l4 ignores htlcs, so they stay.
-#     # l3 will see a reconnect from l4 when l4 restarts.
-#     l1, l2, l3, l4 = node_factory.line_graph(4, wait_for_announce=True, opts=[{}] * 2 + [{'dev-no-reconnect': None, 'may_reconnect': True}] * 2)
-# 
-#     inv = l4.rpc.invoice(amount_msat=10**8, label='x', description='desc')
-#     h = inv['payment_hash']
-#     l4.rpc.dev_ignore_htlcs(id=l3.info['id'], ignore=True)
-# 
-#     # L2 tries to pay
-#     r = l2.rpc.getroute(l4.info['id'], 10**8, 1)["route"]
-#     l2.rpc.sendpay(r, h, payment_secret=inv['payment_secret'])
-# 
-#     # Now increment CLTV
-#     bitcoind.generate_block(1)
-#     sync_blockheight(bitcoind, [l1, l2, l3, l4])
-# 
-#     # L1 tries to pay
-#     r = l1.rpc.getroute(l4.info['id'], 10**8, 1)["route"]
-#     l1.rpc.sendpay(r, h, payment_secret=inv['payment_secret'])
-# 
-#     # Now increment CLTV
-#     bitcoind.generate_block(1)
-#     sync_blockheight(bitcoind, [l1, l2, l3, l4])
-# 
-#     # L3 tries to pay
-#     r = l3.rpc.getroute(l4.info['id'], 10**8, 1)["route"]
-#     l3.rpc.sendpay(r, h, payment_secret=inv['payment_secret'])
-# 
-#     # Give them time to go through.
-#     time.sleep(5)
-# 
-#     # Will all be connected OK.
-#     assert only_one(l1.rpc.listpeers(l2.info['id'])['peers'])['connected']
-#     assert only_one(l2.rpc.listpeers(l3.info['id'])['peers'])['connected']
-#     assert only_one(l3.rpc.listpeers(l4.info['id'])['peers'])['connected']
-# 
-#     # TODO Remove our reliance on HTLCs failing on startup and the need for
-#     #      this plugin
-#     l4.daemon.opts['plugin'] = os.path.join(os.getcwd(), 'tests/plugins/fail_htlcs.py')
-# 
-#     # Restarting tail node will stop it ignoring HTLCs (it will actually
-#     # fail them immediately).
-#     l4.restart()
-#     l3.rpc.connect(l4.info['id'], 'localhost', l4.port)
-# 
-#     wait_for(lambda: only_one(l1.rpc.listsendpays()['payments'])['status'] == 'failed')
-#     wait_for(lambda: only_one(l2.rpc.listsendpays()['payments'])['status'] == 'failed')
-#     wait_for(lambda: only_one(l3.rpc.listsendpays()['payments'])['status'] == 'failed')
-# 
-#     # Should all still be connected.
-#     assert only_one(l1.rpc.listpeers(l2.info['id'])['peers'])['connected']
-#     assert only_one(l2.rpc.listpeers(l3.info['id'])['peers'])['connected']
-#     assert only_one(l3.rpc.listpeers(l4.info['id'])['peers'])['connected']
-# 
-# 
-# def test_pay_variants(node_factory):
-#     l1, l2 = node_factory.line_graph(2)
-# 
-#     # Upper case is allowed
-#     b11 = l2.rpc.invoice(123000, 'test_pay_variants upper', 'description')['bolt11'].upper()
-#     l1.rpc.decodepay(b11)
-#     l1.rpc.pay(b11)
-# 
-#     # lightning: prefix is allowed
-#     b11 = 'lightning:' + l2.rpc.invoice(123000, 'test_pay_variants with prefix', 'description')['bolt11']
-#     l1.rpc.decodepay(b11)
-#     l1.rpc.pay(b11)
-# 
-#     # BOTH is allowed.
-#     b11 = 'LIGHTNING:' + l2.rpc.invoice(123000, 'test_pay_variants upper with prefix', 'description')['bolt11'].upper()
-#     l1.rpc.decodepay(b11)
-#     l1.rpc.pay(b11)
-# 
-# 
-# @pytest.mark.developer("gossip without DEVELOPER=1 is slow")
-# @pytest.mark.slow_test
-# def test_pay_retry(node_factory, bitcoind, executor, chainparams):
-#     """Make sure pay command retries properly. """
-# 
-#     def exhaust_channel(opener, peer, scid, already_spent=0):
-#         """Spend all available capacity (10^6 - 1%) of channel
-#         """
-#         chan = only_one(opener.rpc.listpeerchannels(peer.info['id'])["channels"])
-#         maxpay = chan['spendable_msat']
-#         lbl = ''.join(random.choice(string.ascii_letters) for _ in range(20))
-#         inv = peer.rpc.invoice(maxpay, lbl, "exhaust_channel")
-#         routestep = {
-#             'amount_msat': maxpay,
-#             'id': peer.info['id'],
-#             'delay': 10,
-#             'channel': scid
-#         }
-#         opener.rpc.sendpay([routestep], inv['payment_hash'], payment_secret=inv['payment_secret'])
-#         opener.rpc.waitsendpay(inv['payment_hash'])
-# 
-#     # We connect every node to l5; in a line and individually.
-#     # Keep fixed fees so we can easily calculate exhaustion
-#     l1, l2, l3, l4, l5 = node_factory.line_graph(5, fundchannel=False,
-#                                                  opts={'feerates': (7500, 7500, 7500, 7500), 'disable-mpp': None})
-# 
-#     # scid12
-#     l1.fundchannel(l2, 10**6, wait_for_active=False)
-#     # scid23
-#     l2.fundchannel(l3, 10**6, wait_for_active=False)
-#     # scid34
-#     l3.fundchannel(l4, 10**6, wait_for_active=False)
-#     scid45, _ = l4.fundchannel(l5, 10**6, wait_for_active=False)
-# 
-#     l1.rpc.connect(l5.info['id'], 'localhost', l5.port)
-#     scid15, _ = l1.fundchannel(l5, 10**6, wait_for_active=False)
-#     l2.rpc.connect(l5.info['id'], 'localhost', l5.port)
-#     scid25, _ = l2.fundchannel(l5, 10**6, wait_for_active=False)
-#     l3.rpc.connect(l5.info['id'], 'localhost', l5.port)
-#     scid35, _ = l3.fundchannel(l5, 10**6, wait_for_active=False)
-# 
-#     # Make sure l1 sees all 7 channels
-#     mine_funding_to_announce(bitcoind, [l1, l2, l3, l4, l5])
-#     wait_for(lambda: len(l1.rpc.listchannels()['channels']) == 14)
-# 
-#     # Exhaust shortcut channels one at a time, to force retries.
-#     exhaust_channel(l1, l5, scid15)
-#     exhaust_channel(l2, l5, scid25)
-#     exhaust_channel(l3, l5, scid35)
-# 
-#     def listpays_nofail(b11):
-#         while True:
-#             pays = l1.rpc.listpays(b11)['pays']
-#             if len(pays) != 0:
-#                 if only_one(pays)['status'] == 'complete':
-#                     return
-#                 assert only_one(pays)['status'] != 'failed'
-# 
-#     inv = l5.rpc.invoice(10**8, 'test_retry', 'test_retry')
-# 
-#     # Make sure listpays doesn't transiently show failure while pay
-#     # is retrying.
-#     fut = executor.submit(listpays_nofail, inv['bolt11'])
-# 
-#     # Pay l1->l5 should succeed via straight line (eventually)
-#     l1.dev_pay(inv['bolt11'], use_shadow=False)
-# 
-#     # This should be OK.
-#     fut.result()
-# 
-#     # This should make it fail.
-#     exhaust_channel(l4, l5, scid45, 10**8)
-# 
-#     # It won't try l1->l5, since it knows that's under capacity.
-#     # It will try l1->l2->l5, which fails.
-#     # It will try l1->l2->l3->l5, which fails.
-#     # It will try l1->l2->l3->l4->l5, which fails.
-#     # Finally, fails to find a route.
-#     inv = l5.rpc.invoice(10**8, 'test_retry2', 'test_retry2')['bolt11']
-#     with pytest.raises(RpcError, match=r'4 attempts'):
-#         l1.dev_pay(inv, use_shadow=False)
-# 
-# 
-# @pytest.mark.developer("needs DEVELOPER=1 otherwise gossip takes 5 minutes!")
-# @pytest.mark.slow_test
-# def test_pay_routeboost(node_factory, bitcoind):
-#     """Make sure we can use routeboost information. """
-#     # l1->l2->l3--private-->l4
-#     l1, l2 = node_factory.line_graph(2, announce_channels=True, wait_for_announce=True)
-#     l3, l4, l5 = node_factory.line_graph(3, announce_channels=False, wait_for_announce=False)
-# 
-#     # This should a "could not find a route" because that's true.
-#     error = r'Destination [a-f0-9]{66} is not reachable directly and all routehints were unusable'
-# 
-#     with pytest.raises(RpcError, match=error):
-#         l1.rpc.pay(l5.rpc.invoice(10**8, 'test_retry', 'test_retry')['bolt11'])
-# 
-#     l2.rpc.connect(l3.info['id'], 'localhost', l3.port)
-#     scidl2l3, _ = l2.fundchannel(l3, 10**6)
-# 
-#     # Make sure l1 knows about the 2->3 channel.
-#     mine_funding_to_announce(bitcoind, [l1, l2, l3, l4, l5])
-#     l1.daemon.wait_for_logs([r'update for channel {}/0 now ACTIVE'
-#                              .format(scidl2l3),
-#                              r'update for channel {}/1 now ACTIVE'
-#                              .format(scidl2l3)])
-#     # Make sure l4 knows about 2->3 channel too so it's not a dead-end.
-#     l4.daemon.wait_for_logs([r'update for channel {}/0 now ACTIVE'
-#                              .format(scidl2l3),
-#                              r'update for channel {}/1 now ACTIVE'
-#                              .format(scidl2l3)])
-# 
-#     # Get an l4 invoice; it should put the private channel in routeboost.
-#     inv = l4.rpc.invoice(10**5, 'test_pay_routeboost', 'test_pay_routeboost',
-#                          exposeprivatechannels=True)
-#     assert 'warning_capacity' not in inv
-#     assert 'warning_offline' not in inv
-#     assert only_one(only_one(l1.rpc.decodepay(inv['bolt11'])['routes']))
-# 
-#     # Now we should be able to pay it.
-#     l1.dev_pay(inv['bolt11'], use_shadow=False)
-# 
-#     # Status should show all the gory details.
-#     status = l1.rpc.call('paystatus', [inv['bolt11']])
-#     assert only_one(status['pay'])['bolt11'] == inv['bolt11']
-#     assert only_one(status['pay'])['amount_msat'] == Millisatoshi(10**5)
-#     assert only_one(status['pay'])['destination'] == l4.info['id']
-#     assert 'label' not in only_one(status['pay'])
-#     assert 'routehint_modifications' not in only_one(status['pay'])
-#     assert 'local_exclusions' not in only_one(status['pay'])
-#     attempts = only_one(status['pay'])['attempts']
-#     scid34 = l3.rpc.listpeerchannels(l4.info['id'])['channels'][0]['alias']['local']
-#     assert(len(attempts) == 1)
-#     a = attempts[0]
-#     assert(a['strategy'] == "Initial attempt")
-#     assert('success' in a)
-#     assert('payment_preimage' in a['success'])
-# 
-#     # With dev-route option we can test longer routehints.
-#     if DEVELOPER:
-#         scid45 = l4.rpc.listpeerchannels(l5.info['id'])['channels'][0]['alias']['local']
-#         routel3l4l5 = [{'id': l3.info['id'],
-#                         'short_channel_id': scid34,
-#                         'fee_base_msat': 1000,
-#                         'fee_proportional_millionths': 10,
-#                         'cltv_expiry_delta': 6},
-#                        {'id': l4.info['id'],
-#                         'short_channel_id': scid45,
-#                         'fee_base_msat': 1000,
-#                         'fee_proportional_millionths': 10,
-#                         'cltv_expiry_delta': 6}]
-#         inv = l5.dev_invoice(amount_msat=10**5,
-#                              label='test_pay_routeboost2',
-#                              description='test_pay_routeboost2',
-#                              dev_routes=[routel3l4l5])
-#         l1.dev_pay(inv['bolt11'], use_shadow=False)
-#         status = l1.rpc.call('paystatus', [inv['bolt11']])
-#         pay = only_one(status['pay'])
-#         attempts = pay['attempts']
-#         assert(len(attempts) == 1)
-#         assert 'failure' not in attempts[0]
-#         assert 'success' in attempts[0]
-# 
-#         # Finally, it should fall back to second routehint if first fails.
-#         # (Note, this is not public because it's not 6 deep)
-#         l3.rpc.connect(l5.info['id'], 'localhost', l5.port)
-#         scid35, _ = l3.fundchannel(l5, 10**6)
-#         l4.stop()
-#         routel3l5 = [{'id': l3.info['id'],
-#                       'short_channel_id': scid35,
-#                       'fee_base_msat': 1000,
-#                       'fee_proportional_millionths': 10,
-#                       'cltv_expiry_delta': 6}]
-#         inv = l5.dev_invoice(amount_msat=10**5,
-#                              label='test_pay_routeboost5',
-#                              description='test_pay_routeboost5',
-#                              dev_routes=[routel3l4l5, routel3l5])
-#         l1.dev_pay(inv['bolt11'], label="paying test_pay_routeboost5",
-#                    use_shadow=False)
-# 
-#         status = l1.rpc.call('paystatus', [inv['bolt11']])
-#         assert only_one(status['pay'])['bolt11'] == inv['bolt11']
-#         assert only_one(status['pay'])['destination'] == l5.info['id']
-#         assert only_one(status['pay'])['label'] == "paying test_pay_routeboost5"
-#         assert 'routehint_modifications' not in only_one(status['pay'])
-#         assert 'local_exclusions' not in only_one(status['pay'])
-#         attempts = only_one(status['pay'])['attempts']
-# 
-#         # First one fails, second one succeeds, no routehint would come last.
-#         assert len(attempts) == 2
-#         assert 'success' not in attempts[0]
-#         assert 'success' in attempts[1]
-#         # TODO Add assertion on the routehint once we add them to the pay
-#         # output
-# 
-# 
-# @pytest.mark.developer("updates are delayed without --dev-fast-gossip")
-# def test_setchannel_usage(node_factory, bitcoind):
-#     # TEST SETUP
-#     #
-#     # [l1] ---> [l2]  (channel funded)
-#     #   |
-#     #   o - - > [l3]  (only connected)
-#     #
-#     # - check initial SQL values
-#     # - check setchannel can be used
-#     # - checks command's return object format
-#     # - check custom SQL fee values
-#     # - check values in local nodes listchannels output
-#     # - json throws exception on negative values
-#     # - checks if peer id can be used instead of scid
-#     # - checks fee_base_msat and fee_proportional_millionths in `listpeers` out
-#     DEF_BASE = 10
-#     DEF_BASE_MSAT = Millisatoshi(DEF_BASE)
-#     DEF_PPM = 100
-#     # Minus reserve
-#     MAX_HTLC = Millisatoshi(int(FUNDAMOUNT * 1000 * 0.99))
-# 
-#     l1, l2, l3 = node_factory.get_nodes(3,
-#                                         opts={'fee-base': DEF_BASE, 'fee-per-satoshi': DEF_PPM})
-#     node_factory.join_nodes([l1, l2])
-#     l1.rpc.connect(l3.info['id'], 'localhost', l3.port)
-# 
-#     def channel_get_config(scid):
-#         return l1.db.query(
-#             'SELECT feerate_base, feerate_ppm, htlc_minimum_msat, htlc_maximum_msat FROM channels '
-#             'WHERE scid={};'.format(scid_to_int(scid)))
-# 
-#     # get short channel id
-#     scid = l1.get_channel_scid(l2)
-# 
-#     # feerates should be init with global config
-#     db_fees = l1.db_query('SELECT feerate_base, feerate_ppm, htlc_maximum_msat FROM channels;')
-#     assert(db_fees[0]['feerate_base'] == DEF_BASE)
-#     assert(db_fees[0]['feerate_ppm'] == DEF_PPM)
-#     # This will be the capacity - reserves:
-#     assert(db_fees[0]['htlc_maximum_msat'] == MAX_HTLC)
-#     # this is also what listpeers should return
-#     channel = only_one(l1.rpc.listpeerchannels()['channels'])
-#     assert channel['fee_base_msat'] == DEF_BASE_MSAT
-#     assert channel['fee_proportional_millionths'] == DEF_PPM
-#     assert channel['maximum_htlc_out_msat'] == MAX_HTLC
-# 
-#     # custom setchannel scid <feebase> <feeppm> <htlcmin> <htlcmax>
-#     result = l1.rpc.setchannel(scid, 1337, 137, 17, 133337)
-# 
-#     # check result format
-#     assert(len(result['channels']) == 1)
-#     assert(re.match('^[0-9a-f]{64}$', result['channels'][0]['channel_id']))
-#     assert(result['channels'][0]['peer_id'] == l2.info['id'])
-#     assert(result['channels'][0]['short_channel_id'] == scid)
-#     assert(result['channels'][0]['fee_base_msat'] == 1337)
-#     assert(result['channels'][0]['fee_proportional_millionths'] == 137)
-#     assert(result['channels'][0]['minimum_htlc_out_msat'] == 17)
-#     assert(result['channels'][0]['maximum_htlc_out_msat'] == 133337)
-# 
-#     # check if custom values made it into the database
-#     db_fees = channel_get_config(scid)
-#     assert(db_fees[0]['feerate_base'] == 1337)
-#     assert(db_fees[0]['feerate_ppm'] == 137)
-#     assert(db_fees[0]['htlc_minimum_msat'] == 17)
-#     assert(db_fees[0]['htlc_maximum_msat'] == 133337)
-#     # also check for updated values in `listpeers`
-#     channel = only_one(l1.rpc.listpeerchannels()['channels'])
-#     assert channel['fee_base_msat'] == Millisatoshi(1337)
-#     assert channel['fee_proportional_millionths'] == 137
-#     assert channel['minimum_htlc_out_msat'] == 17
-#     assert channel['maximum_htlc_out_msat'] == 133337
-# 
-#     # wait for gossip and check if l1 sees new fees in listchannels
-#     wait_for(lambda: [c['base_fee_millisatoshi'] for c in l1.rpc.listchannels(scid)['channels']] == [DEF_BASE, 1337])
-#     wait_for(lambda: [c['fee_per_millionth'] for c in l1.rpc.listchannels(scid)['channels']] == [DEF_PPM, 137])
-#     wait_for(lambda: [c['htlc_minimum_msat'] for c in l1.rpc.listchannels(scid)['channels']] == [0, 17])
-#     wait_for(lambda: [c['htlc_maximum_msat'] for c in l1.rpc.listchannels(scid)['channels']] == [MAX_HTLC, 133337])
-# 
-#     # also test with named and missing parameters
-#     result = l1.rpc.setchannel(feeppm=42, id=scid)
-#     assert(len(result['channels']) == 1)
-#     assert(re.match('^[0-9a-f]{64}$', result['channels'][0]['channel_id']))
-#     assert(result['channels'][0]['short_channel_id'] == scid)
-#     assert(result['channels'][0]['fee_base_msat'] == 1337)
-#     assert(result['channels'][0]['fee_proportional_millionths'] == 42)
-#     assert result['channels'][0]['minimum_htlc_out_msat'] == 17
-#     assert(result['channels'][0]['maximum_htlc_out_msat'] == 133337)
-# 
-#     result = l1.rpc.setchannel(feebase=43, id=scid)
-#     assert(len(result['channels']) == 1)
-#     assert(re.match('^[0-9a-f]{64}$', result['channels'][0]['channel_id']))
-#     assert(result['channels'][0]['short_channel_id'] == scid)
-#     assert(result['channels'][0]['fee_base_msat'] == 43)
-#     assert(result['channels'][0]['fee_proportional_millionths'] == 42)
-#     assert result['channels'][0]['minimum_htlc_out_msat'] == 17
-#     assert(result['channels'][0]['maximum_htlc_out_msat'] == 133337)
-# 
-#     result = l1.rpc.setchannel(htlcmin=45, id=scid)
-#     assert(len(result['channels']) == 1)
-#     assert(re.match('^[0-9a-f]{64}$', result['channels'][0]['channel_id']))
-#     assert(result['channels'][0]['short_channel_id'] == scid)
-#     assert(result['channels'][0]['fee_base_msat'] == 43)
-#     assert(result['channels'][0]['fee_proportional_millionths'] == 42)
-#     assert result['channels'][0]['minimum_htlc_out_msat'] == 45
-#     assert(result['channels'][0]['maximum_htlc_out_msat'] == 133337)
-# 
-#     result = l1.rpc.setchannel(htlcmax=43333, id=scid)
-#     assert(len(result['channels']) == 1)
-#     assert(re.match('^[0-9a-f]{64}$', result['channels'][0]['channel_id']))
-#     assert(result['channels'][0]['short_channel_id'] == scid)
-#     assert(result['channels'][0]['fee_base_msat'] == 43)
-#     assert(result['channels'][0]['fee_proportional_millionths'] == 42)
-#     assert result['channels'][0]['minimum_htlc_out_msat'] == 45
-#     assert(result['channels'][0]['maximum_htlc_out_msat'] == 43333)
-# 
-#     # check if negative fees raise error and DB keeps values
-#     # JSONRPC2_INVALID_PARAMS := -32602
-#     from pyln.client import LightningRpc
-#     with pytest.raises(RpcError, match=r'-32602'):
-#         # Need to bypass pyln since it'd check args locally. We also
-#         # have to sidestep the schema validation, it attempts to
-#         # instantiate Millisatoshis and fails due to the non-negative
-#         # constraint.
-#         LightningRpc.call(l1.rpc, 'setchannel', {
-#             "id": scid,
-#             "feebase": -1,
-#             "feeppm": -1
-#         })
-# 
-#     # test if zero fees is possible
-#     result = l1.rpc.setchannel(scid, 0, 0)
-#     assert(result['channels'][0]['short_channel_id'] == scid)
-#     assert(result['channels'][0]['fee_base_msat'] == 0)
-#     assert(result['channels'][0]['fee_proportional_millionths'] == 0)
-# 
-#     db_fees = channel_get_config(scid)
-#     assert(db_fees[0]['feerate_base'] == 0)
-#     assert(db_fees[0]['feerate_ppm'] == 0)
-#     # also check for updated values in `listpeers`
-#     channel = only_one(l1.rpc.listpeerchannels()['channels'])
-#     assert channel['fee_base_msat'] == Millisatoshi(0)
-#     assert channel['fee_proportional_millionths'] == 0
-# 
-#     # check also peer id can be used
-#     result = l1.rpc.setchannel(l2.info['id'], 142, 143)
-#     assert(len(result['channels']) == 1)
-#     assert(result['channels'][0]['peer_id'] == l2.info['id'])
-#     assert(result['channels'][0]['short_channel_id'] == scid)
-#     assert(result['channels'][0]['fee_base_msat'] == 142)
-#     assert(result['channels'][0]['fee_proportional_millionths'] == 143)
-# 
-#     db_fees = channel_get_config(scid)
-#     assert(db_fees[0]['feerate_base'] == 142)
-#     assert(db_fees[0]['feerate_ppm'] == 143)
-# 
-#     # check if invalid scid raises proper error
-#     with pytest.raises(RpcError, match=r'-1.*Could not find any active channels of peer with that id'):
-#         result = l1.rpc.setchannel(l3.info['id'], 42, 43)
-#     with pytest.raises(RpcError, match=r'-32602.*id: should be a channel ID or short channel ID: invalid token'):
-#         result = l1.rpc.setchannel('f42' + scid[3:], 42, 43)
-# 
-#     # check if 'base' unit can be modified to satoshi
-#     result = l1.rpc.setchannel(scid, '1sat')
-#     assert(len(result['channels']) == 1)
-#     assert(result['channels'][0]['peer_id'] == l2.info['id'])
-#     assert(result['channels'][0]['short_channel_id'] == scid)
-#     assert(result['channels'][0]['fee_base_msat'] == 1000)
-#     db_fees = channel_get_config(scid)
-#     assert(db_fees[0]['feerate_base'] == 1000)
-# 
-#     # check if 'ppm' values greater than u32_max fail
-#     with pytest.raises(RpcError, match=r'-32602.*ppm: should be an integer: invalid token'):
-#         LightningRpc.call(l1.rpc, 'setchannel', payload={
-#             "id": scid,
-#             'feebase': 0,
-#             'feeppm': 2**32,
-#         })
-# 
-#     # check if 'base' values greater than u32_max fail
-#     with pytest.raises(RpcError, match=r'-32602.*base: exceeds u32 max: invalid token'):
-#         LightningRpc.call(l1.rpc, 'setchannel', payload={
-#             "id": scid,
-#             "feebase": 2**32,
-#         })
-# 
-# 
-# @pytest.mark.developer("gossip without DEVELOPER=1 is slow")
-# def test_setchannel_state(node_factory, bitcoind):
-#     # TEST SETUP
-#     #
-#     # [l0] --> [l1] --> [l2]
-#     #
-#     # Initiate channel [l1,l2] and try to set feerates other states than
-#     # CHANNELD_NORMAL or CHANNELD_AWAITING_LOCKIN. Should raise error.
-#     # Use l0 to make a forward through l1/l2 for testing.
-#     DEF_BASE = 0
-#     DEF_PPM = 0
-# 
-#     l0, l1, l2 = node_factory.get_nodes(3, opts={
-#         'fee-base': DEF_BASE,
-#         'fee-per-satoshi': DEF_PPM
-#     })
-# 
-#     # connection and funding
-#     l0.rpc.connect(l1.info['id'], 'localhost', l1.port)
-#     l0.fundchannel(l1, 1000000, wait_for_active=True)
-#     l1.rpc.connect(l2.info['id'], 'localhost', l2.port)
-#     scid, _ = l1.fundchannel(l2, 1000000, wait_for_active=False)
-# 
-#     # try setting the fee in state AWAITING_LOCKIN should be possible
-#     # assert(l1.channel_state(l2) == "CHANNELD_AWAITING_LOCKIN")
-#     result = l1.rpc.setchannel(l2.info['id'], 42, 0)
-#     assert(result['channels'][0]['peer_id'] == l2.info['id'])
-#     # cid = result['channels'][0]['channel_id']
-# 
-#     # test routing correct new fees once routing is established
-#     mine_funding_to_announce(bitcoind, [l0, l1, l2])
-# 
-#     l0.wait_for_route(l2)
-#     inv = l2.rpc.invoice(100000, 'test_setchannel_state', 'desc')['bolt11']
-#     result = l0.dev_pay(inv, use_shadow=False)
-#     assert result['status'] == 'complete'
-#     assert result['amount_sent_msat'] == 100042
-# 
-#     # Disconnect and unilaterally close from l2 to l1
-#     l2.rpc.disconnect(l1.info['id'], force=True)
-#     result = l2.rpc.close(scid, 1)
-#     assert result['type'] == 'unilateral'
-# 
-#     # wait for l1 to see unilateral close via bitcoin network
-#     while l1.channel_state(l2) == "CHANNELD_NORMAL":
-#         bitcoind.generate_block(1)
-#     # assert l1.channel_state(l2) == "FUNDING_SPEND_SEEN"
-# 
-#     # Try to setchannel in order to raise expected error.
-#     # To reduce false positive flakes, only test if state is not NORMAL anymore.
-#     with pytest.raises(RpcError, match=r'-1.*'):
-#         # l1.rpc.setchannel(l2.info['id'], 10, 1)
-#         l1.rpc.setchannel(l2.info['id'], 10, 1)
-# 
-# 
-# @pytest.mark.developer("gossip without DEVELOPER=1 is slow")
-# def test_setchannel_routing(node_factory, bitcoind):
-#     # TEST SETUP
-#     #
-#     # [l1] <--default_fees--> [l2] <--specific_fees--> [l3]
-#     #
-#     # - json listchannels is able to see the new values in foreign node
-#     # - routing calculates fees correctly
-#     # - payment can be done using specific fees
-#     # - channel specific fees can be disabled again
-#     # - payment can be done using global fees
-#     # - htlc max is honored
-#     DEF_BASE = 1
-#     DEF_PPM = 10
-#     MAX_HTLC = Millisatoshi(int(FUNDAMOUNT * 1000 * 0.99))
-#     MIN_HTLC = Millisatoshi(0)
-# 
-#     l1, l2, l3 = node_factory.line_graph(
-#         3, announce_channels=True, wait_for_announce=True,
-#         opts={'fee-base': DEF_BASE, 'fee-per-satoshi': DEF_PPM,
-#               'disable-mpp': None})
-# 
-#     # get short channel id for 2->3
-#     scid = l2.get_channel_scid(l3)
-# 
-#     # TEST CUSTOM VALUES
-#     l2.rpc.setchannel(scid, 1337, 137, 17, 4000000, enforcedelay=0)
-# 
-#     # wait for l1 to see updated channel via gossip
-#     wait_for(lambda: [c['base_fee_millisatoshi'] for c in l1.rpc.listchannels(scid)['channels']] == [1337, DEF_BASE])
-#     wait_for(lambda: [c['fee_per_millionth'] for c in l1.rpc.listchannels(scid)['channels']] == [137, DEF_PPM])
-#     wait_for(lambda: [c['htlc_minimum_msat'] for c in l1.rpc.listchannels(scid)['channels']] == [17, MIN_HTLC])
-#     wait_for(lambda: [c['htlc_maximum_msat'] for c in l1.rpc.listchannels(scid)['channels']] == [4000000, MAX_HTLC])
-# 
-#     # test fees are applied to HTLC forwards
-#     #
-#     # BOLT #7:
-#     # If l1 were to send 4,999,999 millisatoshi to l3 via l2, it needs to
-#     # pay l2 the fee it specified in the l2->l3 `channel_update`, calculated as
-#     # per [HTLC Fees](#htlc_fees):  base + amt * pm / 10**6
-# 
-#     # Note: we use fp16 internally for channel max, so we overestimate:
-#     # from devtools/fp16 4000000: fp16:5fa1 min 3999744 max 4001791
-#     # Since it rounds up, it will use 4001792 as max capacity.
-# 
-#     # Should refuse to route this!
-#     with pytest.raises(RpcError, match=r'Could not find a route'):
-#         l1.rpc.getroute(l3.info['id'], 4001793, 1, fuzzpercent=0)["route"]
-# 
-#     # We should consider this unroutable!  (MPP is disabled!)
-#     inv = l3.dev_invoice(amount_msat=4001793,
-#                          label='test_setchannel_1',
-#                          description='desc',
-#                          dev_routes=[])
-#     with pytest.raises(RpcError) as routefail:
-#         l1.dev_pay(inv['bolt11'], use_shadow=False)
-#     assert routefail.value.error['attempts'][0]['failreason'] == 'No path found'
-# 
-#     # 1337 + 4000000 * 137 / 1000000 = 1885
-#     route_ok = l1.rpc.getroute(l3.info['id'], 4000000, 1)["route"]
-#     assert len(route_ok) == 2
-#     assert route_ok[0]['amount_msat'] == 4001885
-#     assert route_ok[1]['amount_msat'] == 4000000
-# 
-#     # Make variant that tries to pay more than allowed htlc!
-#     route_bad = copy.deepcopy(route_ok)
-#     route_bad[0]['amount_msat'] = Millisatoshi(4001887)
-#     route_bad[1]['amount_msat'] = Millisatoshi(4000001)
-#     assert route_bad != route_ok
-# 
-#     # In case l3 includes a routehint, we need to make sure they also know
-#     # about the new fees, otherwise we may end up with the old feerate
-#     wait_for(lambda: [(c['base_fee_millisatoshi'], c['fee_per_millionth'], c['htlc_minimum_msat'], c['htlc_maximum_msat'], c['active']) for c in l3.rpc.listchannels(scid)['channels']] == [(1337, 137, 17, 4000000, True), (DEF_BASE, DEF_PPM, MIN_HTLC, MAX_HTLC, True)])
-# 
-#     # do and check actual payment
-#     inv = l3.rpc.invoice(4000000, 'test_setchannel_2', 'desc')
-#     # Check that routehint from l3 incorporated new feerate!
-#     decoded = l1.rpc.decodepay(inv['bolt11'])
-#     assert decoded['routes'] == [[{'pubkey': l2.info['id'], 'short_channel_id': scid, 'fee_base_msat': 1337, 'fee_proportional_millionths': 137, 'cltv_expiry_delta': 6}]]
-# 
-#     # This will fail.
-#     l1.rpc.sendpay(route_bad, inv['payment_hash'], payment_secret=inv['payment_secret'])
-#     with pytest.raises(RpcError, match='WIRE_TEMPORARY_CHANNEL_FAILURE'):
-#         l1.rpc.waitsendpay(inv['payment_hash'])
-# 
-#     # This will succeed
-#     l1.rpc.sendpay(route_ok, inv['payment_hash'], payment_secret=inv['payment_secret'])
-#     l1.rpc.waitsendpay(inv['payment_hash'])
-# 
-#     # Now try below minimum
-#     route_ok = l1.rpc.getroute(l3.info['id'], 17, 1)["route"]
-#     assert len(route_ok) == 2
-#     assert route_ok[0]['amount_msat'] == 1337 + 17
-#     assert route_ok[1]['amount_msat'] == 17
-# 
-#     route_bad = copy.deepcopy(route_ok)
-#     route_bad[0]['amount_msat'] = Millisatoshi(1337 + 16)
-#     route_bad[1]['amount_msat'] = Millisatoshi(16)
-#     assert route_bad != route_ok
-# 
-#     inv = l3.rpc.invoice(17, 'test_setchannel_3', 'desc')
-# 
-#     # This will fail.
-#     l1.rpc.sendpay(route_bad, inv['payment_hash'], payment_secret=inv['payment_secret'])
-#     with pytest.raises(RpcError, match='WIRE_TEMPORARY_CHANNEL_FAILURE'):
-#         l1.rpc.waitsendpay(inv['payment_hash'])
-# 
-#     # This will succeed
-#     l1.rpc.sendpay(route_ok, inv['payment_hash'], payment_secret=inv['payment_secret'])
-#     l1.rpc.waitsendpay(inv['payment_hash'])
-# 
-#     # Check that this one warns about capacity!
-#     inv = l3.rpc.call('invoice', {'amount_msat': 4001793,
-#                                   'label': 'test_setchannel_4',
-#                                   'description': 'desc'})
-#     assert 'warning_capacity' in inv
-# 
-# 
-# @pytest.mark.developer("gossip without DEVELOPER=1 is slow")
-# def test_setchannel_zero(node_factory, bitcoind):
-#     # TEST SETUP
-#     #
-#     # [l1] <--default_fees--> [l2] <--specific_fees--> [l3]
-#     #
-#     # - json listchannels is able to see the new values in foreign node
-#     # - routing calculates fees correctly
-#     # - payment can be done using zero fees
-#     DEF_BASE = 1
-#     DEF_PPM = 10
-#     MAX_HTLC = Millisatoshi(int(FUNDAMOUNT * 1000 * 0.99))
-# 
-#     l1, l2, l3 = node_factory.line_graph(
-#         3, announce_channels=True, wait_for_announce=True,
-#         opts={'fee-base': DEF_BASE, 'fee-per-satoshi': DEF_PPM})
-# 
-#     # get short channel id for 2->3
-#     scid = l2.get_channel_scid(l3)
-# 
-#     # TEST ZERO fees possible
-#     l2.rpc.setchannel(scid, 0, 0)
-#     wait_for(lambda: [c['base_fee_millisatoshi'] for c in l1.rpc.listchannels(scid)['channels']] == [0, DEF_BASE])
-#     wait_for(lambda: [c['fee_per_millionth'] for c in l1.rpc.listchannels(scid)['channels']] == [0, DEF_PPM])
-# 
-#     # test if zero fees are applied
-#     route = l1.rpc.getroute(l3.info['id'], 4999999, 1)["route"]
-#     assert len(route) == 2
-#     assert route[0]['amount_msat'] == 4999999
-#     assert route[1]['amount_msat'] == 4999999
-# 
-#     # Wait for l3 to know about our low-balling, otherwise they'll add a wrong
-#     # routehint to the invoice.
-#     wait_for(lambda: [(c['base_fee_millisatoshi'], c['fee_per_millionth'], c['active']) for c in l3.rpc.listchannels(scid)['channels']] == [(0, 0, True), (DEF_BASE, DEF_PPM, True)])
-# 
-#     # do and check actual payment
-#     inv = l3.rpc.invoice(4999999, 'test_setchannel_3', 'desc')['bolt11']
-#     result = l1.dev_pay(inv, use_shadow=False)
-#     assert result['status'] == 'complete'
-#     assert result['amount_sent_msat'] == 4999999
-# 
-#     # FIXME: hack something up to advertize min_htlc > 0, then test mintoolow.
-#     with pytest.raises(RpcError, match="htlcmax cannot be less than htlcmin"):
-#         l2.rpc.setchannel(scid, htlcmin=100000, htlcmax=99999)
-# 
-#     ret = l2.rpc.setchannel(scid, htlcmax=FUNDAMOUNT * 1000)
-#     assert 'warning_htlcmax_too_high' in only_one(ret['channels'])
-#     assert only_one(ret['channels'])['maximum_htlc_out_msat'] == MAX_HTLC
-# 
-# 
-# @pytest.mark.developer("gossip without DEVELOPER=1 is slow")
-# def test_setchannel_restart(node_factory, bitcoind):
-#     # TEST SETUP
-#     #
-#     # [l1] <--default_fees--> [l2] <--specific_fees--> [l3]
-#     #
-#     # - l2 sets fees to custom values and restarts
-#     # - l1 routing can be made with the custom fees
-#     # - l2 sets fees to UIN32_MAX (db update default) and restarts
-#     # - l1 routing can be made to l3 and global (1 10) fees are applied
-#     DEF_BASE = 1
-#     DEF_PPM = 10
-#     MIN_HTLC = Millisatoshi(0)
-#     MAX_HTLC = Millisatoshi(int(FUNDAMOUNT * 1000 * 0.99))
-#     OPTS = {'may_reconnect': True, 'fee-base': DEF_BASE, 'fee-per-satoshi': DEF_PPM}
-# 
-#     l1, l2, l3 = node_factory.line_graph(3, announce_channels=True, wait_for_announce=True, opts=OPTS)
-# 
-#     # get short channel idS
-#     scid12 = l1.get_channel_scid(l2)
-#     scid23 = l2.get_channel_scid(l3)
-# 
-#     # l2 set custom fees
-#     l2.rpc.setchannel(scid23, 1337, 137, 17, 500001)
-# 
-#     # restart l2 and reconnect
-#     l2.restart()
-#     l1.rpc.connect(l2.info['id'], 'localhost', l2.port)
-#     l2.rpc.connect(l3.info['id'], 'localhost', l3.port)
-# 
-#     # Make sure l1's gossipd registered channeld activating channel.
-#     wait_for(lambda: [c['active'] for c in l1.rpc.listchannels(scid12)['channels']] == [True, True])
-# 
-#     # l1 wait for channel update from l2
-#     wait_for(lambda: [(c['base_fee_millisatoshi'], c['fee_per_millionth'], c['htlc_minimum_msat'], c['htlc_maximum_msat'], c['active']) for c in l1.rpc.listchannels(scid23)['channels']] == [(1337, 137, 17, 500001, True), (DEF_BASE, DEF_PPM, MIN_HTLC, MAX_HTLC, True)])
-# 
-#     # In case l3 includes a routehint, we need to make sure they also know
-#     # about the new fees, otherwise we may end up with the old feerate
-#     wait_for(lambda: [(c['base_fee_millisatoshi'], c['fee_per_millionth'], c['htlc_minimum_msat'], c['htlc_maximum_msat'], c['active']) for c in l3.rpc.listchannels(scid23)['channels']] == [(1337, 137, 17, 500001, True), (DEF_BASE, DEF_PPM, MIN_HTLC, MAX_HTLC, True)])
-# 
-#     # l1 can make payment to l3 with custom fees being applied
-#     # Note: BOLT #7 math works out to 1405 msat fees
-#     inv = l3.rpc.invoice(499999, 'test_setchannel_1', 'desc')['bolt11']
-#     result = l1.dev_pay(inv, use_shadow=False)
-#     assert result['status'] == 'complete'
-#     assert result['amount_sent_msat'] == 501404
-# 
-# 
-# @pytest.mark.developer("updates are delayed without --dev-fast-gossip")
-# def test_setchannel_all(node_factory, bitcoind):
-#     # TEST SETUP
-#     #
-#     # [l1]----> [l2]
-#     #   |
-#     #   o-----> [l3]
-#     DEF_BASE = 10
-#     DEF_PPM = 100
-# 
-#     l1, l2, l3 = node_factory.get_nodes(3, opts={'fee-base': DEF_BASE, 'fee-per-satoshi': DEF_PPM})
-#     l1.rpc.connect(l2.info['id'], 'localhost', l2.port)
-#     l1.rpc.connect(l3.info['id'], 'localhost', l3.port)
-#     l1.fundchannel(l2, 1000000)
-#     l1.fundchannel(l3, 1000000)
-# 
-#     # get short channel id
-#     scid2 = l1.get_channel_scid(l2)
-#     scid3 = l1.get_channel_scid(l3)
-# 
-#     # now try to set all (two) channels using wildcard syntax
-#     result = l1.rpc.setchannel("all", 0xDEAD, 0xBEEF, 0xBAD, 0xCAFE)
-# 
-#     wait_for(lambda: [c['base_fee_millisatoshi'] for c in l1.rpc.listchannels(scid2)['channels']] == [DEF_BASE, 0xDEAD])
-#     wait_for(lambda: [c['fee_per_millionth'] for c in l1.rpc.listchannels(scid2)['channels']] == [DEF_PPM, 0xBEEF])
-#     wait_for(lambda: [c['base_fee_millisatoshi'] for c in l1.rpc.listchannels(scid3)['channels']] == [0xDEAD, DEF_BASE])
-#     wait_for(lambda: [c['fee_per_millionth'] for c in l1.rpc.listchannels(scid3)['channels']] == [0xBEEF, DEF_PPM])
-# 
-#     # Don't assume order!
-#     assert len(result['channels']) == 2
-#     if result['channels'][0]['peer_id'] == l3.info['id']:
-#         result['channels'] = [result['channels'][1], result['channels'][0]]
-#     assert result['channels'][0]['peer_id'] == l2.info['id']
-#     assert result['channels'][0]['short_channel_id'] == scid2
-#     assert result['channels'][0]['fee_base_msat'] == 0xDEAD
-#     assert result['channels'][0]['fee_proportional_millionths'] == 0xBEEF
-#     assert result['channels'][0]['minimum_htlc_out_msat'] == 0xBAD
-#     assert result['channels'][0]['maximum_htlc_out_msat'] == 0xCAFE
-#     assert result['channels'][1]['peer_id'] == l3.info['id']
-#     assert result['channels'][1]['short_channel_id'] == scid3
-#     assert result['channels'][1]['fee_base_msat'] == 0xDEAD
-#     assert result['channels'][1]['fee_proportional_millionths'] == 0xBEEF
-#     assert result['channels'][1]['minimum_htlc_out_msat'] == 0xBAD
-#     assert result['channels'][1]['maximum_htlc_out_msat'] == 0xCAFE
-# 
-# 
-# @pytest.mark.developer("updates are delayed without --dev-fast-gossip")
-# def test_setchannel_startup_opts(node_factory, bitcoind):
-#     """Tests that custom config/cmdline options are applied correctly when set
-#     """
-#     opts = {
-#         'fee-base': 2,
-#         'fee-per-satoshi': 3,
-#         'htlc-minimum-msat': '4msat',
-#         'htlc-maximum-msat': '5msat'
-#     }
-#     l1, l2 = node_factory.line_graph(2, opts=opts, wait_for_announce=True)
-# 
-#     result = l2.rpc.listchannels()['channels']
-#     assert result[0]['base_fee_millisatoshi'] == 2
-#     assert result[0]['fee_per_millionth'] == 3
-#     assert result[0]['htlc_minimum_msat'] == Millisatoshi(4)
-#     assert result[0]['htlc_maximum_msat'] == Millisatoshi(5)
-#     assert result[1]['base_fee_millisatoshi'] == 2
-#     assert result[1]['fee_per_millionth'] == 3
-#     assert result[1]['htlc_minimum_msat'] == Millisatoshi(4)
-#     assert result[1]['htlc_maximum_msat'] == Millisatoshi(5)
-# 
-# 
-# @pytest.mark.developer("gossip without DEVELOPER=1 is slow")
-# def test_channel_spendable(node_factory, bitcoind):
-#     """Test that spendable_msat is accurate"""
-#     sats = 10**6
-#     l1, l2 = node_factory.line_graph(2, fundamount=sats, wait_for_announce=True,
-#                                      opts={'plugin': os.path.join(os.getcwd(), 'tests/plugins/hold_invoice.py'), 'holdtime': '30'})
-# 
-#     inv = l2.rpc.invoice('any', 'inv', 'for testing')
-#     payment_hash = inv['payment_hash']
-# 
-#     # We should be able to spend this much, and not one msat more!
-#     amount = l1.rpc.listpeerchannels()['channels'][0]['spendable_msat']
-#     route = l1.rpc.getroute(l2.info['id'], amount + 1, riskfactor=1, fuzzpercent=0)['route']
-#     l1.rpc.sendpay(route, payment_hash, payment_secret=inv['payment_secret'])
-# 
-#     # This should fail locally with "capacity exceeded"
-#     with pytest.raises(RpcError, match=r"Capacity exceeded.*'erring_index': 0"):
-#         l1.rpc.waitsendpay(payment_hash, TIMEOUT)
-# 
-#     # Exact amount should succeed.
-#     route = l1.rpc.getroute(l2.info['id'], amount, riskfactor=1, fuzzpercent=0)['route']
-#     l1.rpc.sendpay(route, payment_hash, payment_secret=inv['payment_secret'])
-# 
-#     # Amount should drop to 0 once HTLC is sent; we have time, thanks to
-#     # hold_invoice.py plugin.
-#     wait_for(lambda: len(l1.rpc.listpeerchannels()['channels'][0]['htlcs']) == 1)
-#     assert l1.rpc.listpeerchannels()['channels'][0]['spendable_msat'] == Millisatoshi(0)
-#     l1.rpc.waitsendpay(payment_hash, TIMEOUT)
-# 
-#     # Make sure l2 thinks it's all over.
-#     wait_for(lambda: len(l2.rpc.listpeerchannels()['channels'][0]['htlcs']) == 0)
-#     # Now, reverse should work similarly.
-#     inv = l1.rpc.invoice('any', 'inv', 'for testing')
-#     payment_hash = inv['payment_hash']
-#     amount = l2.rpc.listpeerchannels()['channels'][0]['spendable_msat']
-# 
-#     # Turns out we won't route this, as it's over max - reserve:
-#     route = l2.rpc.getroute(l1.info['id'], amount + 1, riskfactor=1, fuzzpercent=0)['route']
-#     l2.rpc.sendpay(route, payment_hash, payment_secret=inv['payment_secret'])
-# 
-#     # This should fail locally with "capacity exceeded"
-#     with pytest.raises(RpcError, match=r"Capacity exceeded.*'erring_index': 0"):
-#         l2.rpc.waitsendpay(payment_hash, TIMEOUT)
-# 
-#     # Exact amount should succeed.
-#     route = l2.rpc.getroute(l1.info['id'], amount, riskfactor=1, fuzzpercent=0)['route']
-#     l2.rpc.sendpay(route, payment_hash, payment_secret=inv['payment_secret'])
-# 
-#     # Amount should drop to 0 once HTLC is sent; we have time, thanks to
-#     # hold_invoice.py plugin.
-#     wait_for(lambda: len(l2.rpc.listpeerchannels()['channels'][0]['htlcs']) == 1)
-#     assert l2.rpc.listpeerchannels()['channels'][0]['spendable_msat'] == Millisatoshi(0)
-#     l2.rpc.waitsendpay(payment_hash, TIMEOUT)
-# 
-# 
-# @pytest.mark.developer("gossip without DEVELOPER=1 is slow")
-# def test_channel_receivable(node_factory, bitcoind):
-#     """Test that receivable_msat is accurate"""
-#     sats = 10**6
-#     l1, l2 = node_factory.line_graph(2, fundamount=sats, wait_for_announce=True,
-#                                      opts={'plugin': os.path.join(os.getcwd(), 'tests/plugins/hold_invoice.py'), 'holdtime': '30'})
-# 
-#     inv = l2.rpc.invoice('any', 'inv', 'for testing')
-#     payment_hash = inv['payment_hash']
-# 
-#     # We should be able to receive this much, and not one msat more!
-#     amount = l2.rpc.listpeerchannels()['channels'][0]['receivable_msat']
-#     route = l1.rpc.getroute(l2.info['id'], amount + 1, riskfactor=1, fuzzpercent=0)['route']
-#     l1.rpc.sendpay(route, payment_hash, payment_secret=inv['payment_secret'])
-# 
-#     # This should fail locally with "capacity exceeded"
-#     with pytest.raises(RpcError, match=r"Capacity exceeded.*'erring_index': 0"):
-#         l1.rpc.waitsendpay(payment_hash, TIMEOUT)
-# 
-#     # Exact amount should succeed.
-#     route = l1.rpc.getroute(l2.info['id'], amount, riskfactor=1, fuzzpercent=0)['route']
-#     l1.rpc.sendpay(route, payment_hash, payment_secret=inv['payment_secret'])
-# 
-#     # Amount should drop to 0 once HTLC is sent; we have time, thanks to
-#     # hold_invoice.py plugin.
-#     wait_for(lambda: len(l2.rpc.listpeerchannels()['channels'][0]['htlcs']) == 1)
-#     assert l2.rpc.listpeerchannels()['channels'][0]['receivable_msat'] == Millisatoshi(0)
-#     l1.rpc.waitsendpay(payment_hash, TIMEOUT)
-# 
-#     # Make sure both think it's all over.
-#     wait_for(lambda: len(l1.rpc.listpeerchannels()['channels'][0]['htlcs']) == 0)
-#     wait_for(lambda: len(l2.rpc.listpeerchannels()['channels'][0]['htlcs']) == 0)
-#     # Now, reverse should work similarly.
-#     inv = l1.rpc.invoice('any', 'inv', 'for testing')
-#     payment_hash = inv['payment_hash']
-#     amount = l1.rpc.listpeerchannels()['channels'][0]['receivable_msat']
-# 
-#     # Turns out we won't route this, as it's over max - reserve:
-#     route = l2.rpc.getroute(l1.info['id'], amount + 1, riskfactor=1, fuzzpercent=0)['route']
-#     l2.rpc.sendpay(route, payment_hash, payment_secret=inv['payment_secret'])
-# 
-#     # This should fail locally with "capacity exceeded"
-#     with pytest.raises(RpcError, match=r"Capacity exceeded.*'erring_index': 0"):
-#         l2.rpc.waitsendpay(payment_hash, TIMEOUT)
-# 
-#     # Exact amount should succeed.
-#     route = l2.rpc.getroute(l1.info['id'], amount, riskfactor=1, fuzzpercent=0)['route']
-#     l2.rpc.sendpay(route, payment_hash, payment_secret=inv['payment_secret'])
-# 
-#     # Amount should drop to 0 once HTLC is sent; we have time, thanks to
-#     # hold_invoice.py plugin.
-#     wait_for(lambda: len(l1.rpc.listpeerchannels()['channels'][0]['htlcs']) == 1)
-#     assert l1.rpc.listpeerchannels()['channels'][0]['receivable_msat'] == Millisatoshi(0)
-#     l2.rpc.waitsendpay(payment_hash, TIMEOUT)
-# 
-# 
-# @pytest.mark.developer("gossip without DEVELOPER=1 is slow")
-# def test_channel_spendable_large(node_factory, bitcoind):
-#     """Test that spendable_msat is accurate for large channels"""
-#     # This is almost the max allowable spend.
-#     sats = 4294967
-#     l1, l2 = node_factory.line_graph(
-#         2,
-#         fundamount=sats,
-#         wait_for_announce=True,
-#         opts={
-#             'plugin': os.path.join(os.getcwd(), 'tests/plugins/hold_invoice.py'),
-#             'holdtime': '30'
-#         }
-#     )
-# 
-#     inv = l2.rpc.invoice('any', 'inv', 'for testing')
-#     payment_hash = inv['payment_hash']
-# 
-#     # We should be able to spend this much, and not one msat more!
-#     spendable = l1.rpc.listpeerchannels()['channels'][0]['spendable_msat']
-# 
-#     # receivable from the other side should calculate to the exact same amount
-#     receivable = l2.rpc.listpeerchannels()['channels'][0]['receivable_msat']
-#     assert spendable == receivable
-# 
-#     # route or waitsendpay fill fail.
-#     with pytest.raises(RpcError):
-#         route = l1.rpc.getroute(l2.info['id'], spendable + 1, riskfactor=1, fuzzpercent=0)['route']
-#         l1.rpc.sendpay(route, payment_hash, payment_secret=inv['payment_secret'])
-#         l1.rpc.waitsendpay(payment_hash, TIMEOUT)
-# 
-#     # Exact amount should succeed.
-#     route = l1.rpc.getroute(l2.info['id'], spendable, riskfactor=1, fuzzpercent=0)['route']
-#     l1.rpc.sendpay(route, payment_hash, payment_secret=inv['payment_secret'])
-#     l1.rpc.waitsendpay(payment_hash, TIMEOUT)
-# 
-# 
-# def test_channel_spendable_receivable_capped(node_factory, bitcoind):
-#     """Test that spendable_msat and receivable_msat is capped at 2^32-1"""
-#     sats = 16777215
-#     l1, l2 = node_factory.line_graph(2, fundamount=sats, wait_for_announce=False)
-#     assert l1.rpc.listpeerchannels()['channels'][0]['spendable_msat'] == Millisatoshi(0xFFFFFFFF)
-#     assert l2.rpc.listpeerchannels()['channels'][0]['receivable_msat'] == Millisatoshi(0xFFFFFFFF)
-# 
-# 
-# @unittest.skipIf(True, "Test is extremely flaky")
-# @unittest.skipIf(not DEVELOPER and VALGRIND, "Doesn't raise exception, needs better sync")
-# def test_lockup_drain(node_factory, bitcoind):
-#     """Try to get channel into a state where opener can't afford fees on additional HTLC, so peer can't add HTLC"""
-#     l1, l2 = node_factory.line_graph(2, opts={'may_reconnect': True})
-# 
-#     # l1 sends all the money to l2 until even 1 msat can't get through.
-#     total = l1.drain(l2)
-# 
-#     # Even if feerate now increases 2x (30000), l2 should be able to send
-#     # non-dust HTLC to l1.
-#     l1.force_feerates(30000)
-#     l2.pay(l1, total // 2)
-# 
-#     # reset fees and send all back again
-#     l1.force_feerates(15000)
-#     l1.drain(l2)
-# 
-#     # But if feerate increase just a little more, l2 should not be able to send
-#     # non-fust HTLC to l1
-#     l1.force_feerates(30002)  # TODO: Why does 30001 fail? off by one in C code?
-#     wait_for(lambda: l1.rpc.listpeers()['peers'][0]['connected'])
-#     with pytest.raises(RpcError, match=r".*Capacity exceeded.*"):
-#         l2.pay(l1, total // 2)
-# 
-# 
-# @pytest.mark.developer("needs DEVELOPER=1 for dev_ignore_htlcs")
-# def test_htlc_too_dusty_outgoing(node_factory, bitcoind, chainparams):
-#     """ Try to hit the 'too much dust' limit, should fail the HTLC """
-#     feerate = 10000
-# 
-#     # elements txs are bigger so they become dusty faster
-#     max_dust_limit_sat = 100000 if chainparams['elements'] else 50000
-#     non_dust_htlc_val_sat = 20000 if chainparams['elements'] else 10000
-#     htlc_val_sat = 10000 if chainparams['elements'] else 5000
-# 
-#     l1, l2 = node_factory.line_graph(2, opts={'may_reconnect': True,
-#                                               'feerates': (feerate, feerate, feerate, feerate),
-#                                               'max-dust-htlc-exposure-msat': '{}sat'.format(max_dust_limit_sat),
-#                                               'allow_warning': True})
-# 
-#     # l2 holds all of l1's htlcs hostage
-#     l2.rpc.dev_ignore_htlcs(id=l1.info['id'], ignore=True)
-# 
-#     # l2's max dust limit is set to 100k
-#     htlc_val_msat = htlc_val_sat * 1000
-#     num_dusty_htlcs = max_dust_limit_sat // htlc_val_sat
-# 
-#     # add a some non-dusty htlcs, these will fail when we raise the dust limit
-#     route = l1.rpc.getroute(l2.info['id'], non_dust_htlc_val_sat * 1000, 1)['route']
-#     for i in range(0, 3):
-#         inv = l2.rpc.invoice((non_dust_htlc_val_sat * 1000), str(i + 100), str(i + 100))
-#         l1.rpc.sendpay(route, inv['payment_hash'], payment_secret=inv['payment_secret'])
-#         l2.daemon.wait_for_log(r'their htlc .* dev_ignore_htlcs')
-#         res = only_one(l1.rpc.listsendpays(payment_hash=inv['payment_hash'])['payments'])
-#         assert res['status'] == 'pending'
-# 
-#     # add some dusty-htlcs
-#     route = l1.rpc.getroute(l2.info['id'], htlc_val_msat, 1)['route']
-#     for i in range(0, num_dusty_htlcs):
-#         inv = l2.rpc.invoice(htlc_val_msat, str(i), str(i))
-#         l1.rpc.sendpay(route, inv['payment_hash'], payment_secret=inv['payment_secret'])
-#         l2.daemon.wait_for_log(r'their htlc .* dev_ignore_htlcs')
-#         res = only_one(l1.rpc.listsendpays(payment_hash=inv['payment_hash'])['payments'])
-#         assert res['status'] == 'pending'
-# 
-#     # one more should tip it over, and return a payment failure
-#     inv = l2.rpc.invoice(htlc_val_msat, str(num_dusty_htlcs), str(num_dusty_htlcs))
-#     l1.rpc.sendpay(route, inv['payment_hash'], payment_secret=inv['payment_secret'])
-#     l1.daemon.wait_for_log('CHANNEL_ERR_DUST_FAILURE')
-#     wait_for(lambda: only_one(l1.rpc.listsendpays(payment_hash=inv['payment_hash'])['payments'])['status'] == 'failed')
-# 
-#     # but we can still add a non dust htlc
-#     route = l1.rpc.getroute(l2.info['id'], non_dust_htlc_val_sat * 1000, 1)['route']
-#     inv = l2.rpc.invoice((10000 * 1000), str(120), str(120))
-#     l1.rpc.sendpay(route, inv['payment_hash'], payment_secret=inv['payment_secret'])
-#     l2.daemon.wait_for_log(r'their htlc .* dev_ignore_htlcs')
-#     res = only_one(l1.rpc.listsendpays(payment_hash=inv['payment_hash'])['payments'])
-#     assert res['status'] == 'pending'
-# 
-#     # Ok, adjust our feerate upward, so the non-dust htlcs are now dust
-#     # note that this is above the buffer we've been keeping, so the channel
-#     # should automatically fail
-#     l1.set_feerates([feerate * 2] * 4, False)
-#     l1.restart()
-# 
-#     # Make sure fails before we try sending htlc!
-#     l1.daemon.wait_for_log('Too much dust to update fee')
-# 
-#     # the channel should start warning -- too much dust
-#     inv = l2.rpc.invoice(htlc_val_msat, str(num_dusty_htlcs + 1), str(num_dusty_htlcs + 1))
-#     with pytest.raises(RpcError, match=r'WIRE_TEMPORARY_CHANNEL_FAILURE'):
-#         l1.rpc.sendpay(route, inv['payment_hash'], payment_secret=inv['payment_secret'])
-# 
-# 
-# @pytest.mark.developer("needs DEVELOPER=1 for dev_ignore_htlcs")
-# def test_htlc_too_dusty_incoming(node_factory, bitcoind):
-#     """ Try to hit the 'too much dust' limit, should fail the HTLC """
-#     feerate = 30000
-#     l1, l2, l3 = node_factory.line_graph(3, opts=[{'may_reconnect': True,
-#                                                    'feerates': (feerate, feerate, feerate, feerate),
-#                                                    'max-dust-htlc-exposure-msat': '200000sat'},
-#                                                   {'may_reconnect': True,
-#                                                    'feerates': (feerate, feerate, feerate, feerate),
-#                                                    'max-dust-htlc-exposure-msat': '100000sat',
-#                                                    'fee-base': 0,
-#                                                    'fee-per-satoshi': 0},
-#                                                   {'max-dust-htlc-exposure-msat': '500000sat'}],
-#                                          wait_for_announce=True)
-# 
-#     # on the l2->l3, and l3 holds all the htlcs hostage
-#     # have l3 hold onto all the htlcs and not fulfill them
-#     l3.rpc.dev_ignore_htlcs(id=l2.info['id'], ignore=True)
-# 
-#     # l2's max dust limit is set to 100k
-#     max_dust_limit_sat = 100000
-#     htlc_val_sat = 10000
-#     htlc_val_msat = htlc_val_sat * 1000
-#     num_dusty_htlcs = max_dust_limit_sat // htlc_val_sat
-#     route = l1.rpc.getroute(l3.info['id'], htlc_val_msat, 1)['route']
-# 
-#     # l1 sends as much money as it can
-#     for i in range(0, num_dusty_htlcs):
-#         inv = l3.rpc.invoice(htlc_val_msat, str(i), str(i))
-#         l1.rpc.sendpay(route, inv['payment_hash'], payment_secret=inv['payment_secret'])
-#         l3.daemon.wait_for_log(r'their htlc .* dev_ignore_htlcs')
-#         res = only_one(l1.rpc.listsendpays(payment_hash=inv['payment_hash'])['payments'])
-#         assert res['status'] == 'pending'
-# 
-#     # one more should tip it over, and return a payment failure
-#     inv = l3.rpc.invoice(htlc_val_msat, str(num_dusty_htlcs), str(num_dusty_htlcs))
-#     l1.rpc.sendpay(route, inv['payment_hash'], payment_secret=inv['payment_secret'])
-#     l2.daemon.wait_for_log('failing immediately, as requested')
-#     wait_for(lambda: only_one(l1.rpc.listsendpays(payment_hash=inv['payment_hash'])['payments'])['status'] == 'failed')
-# 
-# 
-# def test_error_returns_blockheight(node_factory, bitcoind):
-#     """Test that incorrect_or_unknown_payment_details returns block height"""
-#     l1, l2 = node_factory.line_graph(2)
-# 
-#     l1.rpc.sendpay([{'amount_msat': 100,
-#                      'id': l2.info['id'],
-#                      'delay': 10,
-#                      'channel': l1.get_channel_scid(l2)}],
-#                    '00' * 32, payment_secret='00' * 32)
-# 
-#     with pytest.raises(RpcError, match=r"INCORRECT_OR_UNKNOWN_PAYMENT_DETAILS.*'erring_index': 1") as err:
-#         l1.rpc.waitsendpay('00' * 32, TIMEOUT)
-# 
-#     # BOLT #4:
-#     # 1. type: PERM|15 (`incorrect_or_unknown_payment_details`)
-#     # 2. data:
-#     #    * [`u64`:`htlc_msat`]
-#     #    * [`u32`:`height`]
-#     assert (err.value.error['data']['raw_message']
-#             == '400f{:016x}{:08x}'.format(100, bitcoind.rpc.getblockcount()))
-# 
-# 
-# @pytest.mark.developer('Needs dev-routes')
-# def test_tlv_or_legacy(node_factory, bitcoind):
-#     # Ideally we'd test with l2 NOT var-onion, but then it can no longer connect
-#     # to us!
-#     l1, l2, l3 = node_factory.line_graph(3,
-#                                          opts={'plugin': os.path.join(os.getcwd(), 'tests/plugins/print_htlc_onion.py')})
-# 
-#     scid12 = l1.get_channel_scid(l2)
-#     scid23 = l2.get_channel_scid(l3)
-# 
-#     # We need to force l3 to provide route hint from l2 (it won't normally,
-#     # since it sees l2 as a dead end).
-#     inv = l3.dev_invoice(amount_msat=10000,
-#                          label="test_tlv1",
-#                          description="test_tlv1",
-#                          dev_routes=[[{'id': l2.info['id'],
-#                                        'short_channel_id': scid23,
-#                                        'fee_base_msat': 1,
-#                                        'fee_proportional_millionths': 10,
-#                                        'cltv_expiry_delta': 6}]])['bolt11']
-#     l1.rpc.pay(inv)
-# 
-#     # Since L1 hasn't seen broadcast, it doesn't know L2 isn't TLV, but invoice tells it about L3
-#     l2.daemon.wait_for_log("Got onion.*'type': 'tlv'")
-#     l3.daemon.wait_for_log("Got onion.*'type': 'tlv'")
-# 
-#     # We need 5 more blocks to announce l1->l2 channel.
-#     mine_funding_to_announce(bitcoind, [l1, l2, l3])
-# 
-#     # Make sure l1 knows about l2
-#     wait_for(lambda: 'alias' in l1.rpc.listnodes(l2.info['id'])['nodes'][0])
-# 
-#     # Make sure l3 knows about l1->l2, so it will add route hint now.
-#     wait_for(lambda: len(l3.rpc.listchannels(scid12)['channels']) > 0)
-# 
-#     # Now it should send TLV to l2.
-#     inv = l3.rpc.invoice(10000, "test_tlv2", "test_tlv2")['bolt11']
-# 
-#     l1.rpc.pay(inv)
-#     l2.daemon.wait_for_log("Got onion.*'type': 'tlv'")
-#     l3.daemon.wait_for_log("Got onion.*'type': 'tlv'")
-# 
-# 
-# @unittest.skipIf(TEST_NETWORK != 'regtest', "Invoice is network specific")
-# def test_pay_no_secret(node_factory, bitcoind):
-#     l1, l2 = node_factory.line_graph(2, wait_for_announce=True)
-# 
-#     l2.rpc.invoice(100000, "test_pay_no_secret", "test_pay_no_secret",
-#                    preimage='00' * 32, expiry=2000000000)
-# 
-#     # Produced from modified version (different secret!).
-#     inv_badsecret = 'lnbcrt1u1pwuedm6pp5ve584t0cv27hwmy0cx9ca8uwyqyfw9y9dm3r8vus9fv36r2l9yjsdqaw3jhxazlwpshjhmwda0hxetrwfjhgxq8pmnt9qqcqp9sp52au0npwmw4xxv2rfrat04kh9p3jlmklgavhfxqukx0l05pw5tccs9qypqsqa286dmt2xh3jy8cd8ndeyr845q8a7nhgjkerdqjns76jraux6j25ddx9f5k5r2ey0kk942x3uhaff66794kyjxxcd48uevf7p6ja53gqjj5ur7'
-#     with pytest.raises(RpcError, match=r"INCORRECT_OR_UNKNOWN_PAYMENT_DETAILS.*'erring_index': 1"):
-#         l1.rpc.pay(inv_badsecret)
-# 
-#     # Produced from old version (no secret!)
-#     inv_nosecret = 'lnbcrt1u1pwue4vapp5ve584t0cv27hwmy0cx9ca8uwyqyfw9y9dm3r8vus9fv36r2l9yjsdqaw3jhxazlwpshjhmwda0hxetrwfjhgxq8pmnt9qqcqp9570xsjyykvssa6ty8fjth6f2y8h09myngad9utesttwjwclv95fz3lgd402f9e5yzpnxmkypg55rkvpg522gcz4ymsjl2w3m4jhw4jsp55m7tl'
-#     with pytest.raises(RpcError, match=r"INCORRECT_OR_UNKNOWN_PAYMENT_DETAILS.*'erring_index': 1"):
-#         l1.rpc.pay(inv_nosecret)
-# 
-# 
-# def test_shadow_routing(node_factory):
-#     """
-#     Test the value randomization through shadow routing
-# 
-#     Note there is a very low (0.5**10) probability that it fails.
-#     """
-#     # We need l3 for random walk
-#     l1, l2, l3 = node_factory.line_graph(3, wait_for_announce=True)
-# 
-#     amount = 10000
-#     total_amount = 0
-#     n_payments = 10
-#     for i in range(n_payments):
-#         inv = l3.rpc.invoice(amount, "{}".format(i), "test")["bolt11"]
-#         total_amount += l1.rpc.pay(inv)["amount_sent_msat"]
-# 
-#     assert total_amount.millisatoshis > n_payments * amount
-#     # Test that the added amount isn't absurd
-#     assert total_amount.millisatoshis < (n_payments * amount) * (1 + 0.01)
-# 
-#     # FIXME: Test cltv delta too ?
-# 
-# 
-# def test_createonion_rpc(node_factory):
-#     l1 = node_factory.get_node()
-# 
-#     # From bolt04/onion-test.json:
-#     hops = [{
-#         "pubkey": "02eec7245d6b7d2ccb30380bfbe2a3648cd7a942653f5aa340edcea1f283686619",
-#         "payload": "1202023a98040205dc06080000000000000001"
-#     }, {
-#         "pubkey": "0324653eac434488002cc06bbfb7f10fe18991e35f9fe4302dbea6d2353dc0ab1c",
-#         "payload": "52020236b00402057806080000000000000002fd02013c0102030405060708090a0b0c0d0e0f0102030405060708090a0b0c0d0e0f0102030405060708090a0b0c0d0e0f0102030405060708090a0b0c0d0e0f"
-#     }, {
-#         "pubkey": "027f31ebc5462c1fdce1b737ecff52d37d75dea43ce11c74d25aa297165faa2007",
-#         "payload": "12020230d4040204e206080000000000000003"
-#     }, {
-#         "pubkey": "032c0b7cf95324a07d05398b240174dc0c2be444d96b159aa6c7f7b1e668680991",
-#         "payload": "1202022710040203e806080000000000000004"
-#     }, {
-#         "pubkey": "02edabbd16b41c8371b92ef2f04c1185b4f03b6dcd52ba9b78d9d7c89c8f221145",
-#         "payload": "fd011002022710040203e8082224a33562c54507a9334e79f0dc4f17d407e6d7c61f0e2f3d0d38599502f617042710fd012de02a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a"
-#     }]
-# 
-#     res = l1.rpc.createonion(hops=hops, assocdata="BB" * 32)
-#     assert(len(res['onion']) == 2 * 1366)
-#     assert(len(res['shared_secrets']) == len(hops))
-# 
-#     res = l1.rpc.createonion(hops=hops, assocdata="42" * 32,
-#                              session_key="41" * 32)
-#     # The trailer is generated using the filler and can be ued as a
-#     # checksum. This trailer is from the test-vector in the specs.
-#     assert(res['onion'].endswith('9126aaefb627719f421e20'))
-# 
-# 
-# @pytest.mark.developer("gossip propagation is slow without DEVELOPER=1")
-# def test_sendonion_rpc(node_factory):
-#     l1, l2, l3, l4 = node_factory.line_graph(4, wait_for_announce=True)
-#     amt = 10**3
-#     route = l1.rpc.getroute(l4.info['id'], 10**3, 10)['route']
-#     inv = l4.rpc.invoice(amt, "lbl", "desc")
-# 
-#     first_hop = route[0]
-#     blockheight = l1.rpc.getinfo()['blockheight']
-# 
-#     def truncate_encode(i: int):
-#         """Encode a tu64 (or tu32 etc) value"""
-#         ret = struct.pack("!Q", i)
-#         while ret.startswith(b'\0'):
-#             ret = ret[1:]
-#         return ret
-# 
-#     def serialize_payload_tlv(n):
-#         block, tx, out = n['channel'].split('x')
-# 
-#         payload = TlvPayload()
-#         # BOLT #4:
-#         #     1. type: 2 (`amt_to_forward`)
-#         #     2. data:
-#         #         * [`tu64`:`amt_to_forward`]
-#         b = BytesIO()
-#         b.write(truncate_encode(int(n['amount_msat'])))
-#         payload.add_field(2, b.getvalue())
-#         # BOLT #4:
-#         #    1. type: 4 (`outgoing_cltv_value`)
-#         #    2. data:
-#         #        * [`tu32`:`outgoing_cltv_value`]
-#         b = BytesIO()
-#         b.write(truncate_encode(blockheight + n['delay']))
-#         payload.add_field(4, b.getvalue())
-#         # BOLT #4:
-#         #    1. type: 6 (`short_channel_id`)
-#         #    2. data:
-#         #        * [`short_channel_id`:`short_channel_id`]
-#         b = BytesIO()
-#         b.write(struct.pack("!Q", int(block) << 40 | int(tx) << 16 | int(out)))
-#         payload.add_field(6, b.getvalue())
-#         return payload.to_bytes().hex()
-# 
-#     def serialize_payload_final_tlv(n, payment_secret: str):
-#         payload = TlvPayload()
-#         # BOLT #4:
-#         #     1. type: 2 (`amt_to_forward`)
-#         #     2. data:
-#         #         * [`tu64`:`amt_to_forward`]
-#         b = BytesIO()
-#         b.write(truncate_encode(int(n['amount_msat'])))
-#         payload.add_field(2, b.getvalue())
-#         # BOLT #4:
-#         #    1. type: 4 (`outgoing_cltv_value`)
-#         #    2. data:
-#         #        * [`tu32`:`outgoing_cltv_value`]
-#         b = BytesIO()
-#         b.write(truncate_encode(blockheight + n['delay']))
-#         payload.add_field(4, b.getvalue())
-#         # BOLT #4:
-#         #    1. type: 8 (`payment_data`)
-#         #    2. data:
-#         #        * [`32*byte`:`payment_secret`]
-#         #        * [`tu64`:`total_msat`]
-#         b = BytesIO()
-#         b.write(bytes.fromhex(payment_secret))
-#         b.write(truncate_encode(int(n['amount_msat'])))
-#         payload.add_field(8, b.getvalue())
-#         return payload.to_bytes().hex()
-# 
-#     # Need to shift the parameters by one hop
-#     hops = []
-#     for h, n in zip(route[:-1], route[1:]):
-#         # We tell the node h about the parameters to use for n (a.k.a. h + 1)
-#         hops.append({
-#             "pubkey": h['id'],
-#             "payload": serialize_payload_tlv(n)
-#         })
-# 
-#     # The last hop has a special payload:
-#     hops.append({
-#         "pubkey": route[-1]['id'],
-#         "payload": serialize_payload_final_tlv(route[-1], inv['payment_secret'])
-#     })
-# 
-#     onion = l1.rpc.createonion(hops=hops, assocdata=inv['payment_hash'])
-# 
-#     l1.rpc.sendonion(onion=onion['onion'], first_hop=first_hop,
-#                      payment_hash=inv['payment_hash'])
-# 
-#     l1.rpc.waitsendpay(payment_hash=inv['payment_hash'])
-#     invs = l4.rpc.listinvoices(label="lbl")['invoices']
-#     assert(len(invs) == 1 and invs[0]['status'] == 'paid')
-# 
-#     pays = l1.rpc.listsendpays()['payments']
-#     assert(len(pays) == 1 and pays[0]['status'] == 'complete'
-#            and pays[0]['payment_hash'] == inv['payment_hash'])
-# 
-#     # And now for a failing payment, using a payment_hash that doesn't match an
-#     # invoice
-#     payment_hash = "00" * 32
-#     onion = l1.rpc.createonion(hops=hops, assocdata=payment_hash)
-#     l1.rpc.sendonion(onion=onion['onion'], first_hop=first_hop,
-#                      payment_hash=payment_hash)
-# 
-#     try:
-#         l1.rpc.waitsendpay(payment_hash=payment_hash)
-#         raise ValueError()
-#     except RpcError as e:
-#         assert(e.error['code'] == 202)
-#         assert(e.error['message'] == "Malformed error reply")
-# 
-#     pays = l1.rpc.listsendpays(payment_hash=payment_hash)['payments']
-#     assert(len(pays) == 1 and pays[0]['status'] == 'failed'
-#            and pays[0]['payment_hash'] == payment_hash)
-#     assert('erroronion' in pays[0])
-# 
-#     # Fail onion is msg + padding = 256 + 2*2 byte lengths + 32 byte HMAC
-#     assert(len(pays[0]['erroronion']) == (256 + 32 + 2 + 2) * 2)
-# 
-#     # Let's try that again, this time we give it the shared_secrets so it
-#     # should be able to decode the error.
-#     payment_hash = "01" * 32
-#     onion = l1.rpc.createonion(hops=hops, assocdata=payment_hash)
-#     l1.rpc.sendonion(onion=onion['onion'], first_hop=first_hop,
-#                      payment_hash=payment_hash,
-#                      shared_secrets=onion['shared_secrets'])
-# 
-#     try:
-#         l1.rpc.waitsendpay(payment_hash=payment_hash)
-#     except RpcError as e:
-#         assert(e.error['code'] == 204)
-#         assert(e.error['data']['raw_message'] == "400f00000000000003e80000006c")
-# 
-# 
-# @pytest.mark.developer("needs dev-disable-commit-after, dev-no-htlc-timeout")
-# @pytest.mark.openchannel('v1')
-# @pytest.mark.openchannel('v2')
-# def test_partial_payment(node_factory, bitcoind, executor):
-#     # We want to test two payments at the same time, before we send commit
-#     l1, l2, l3, l4 = node_factory.get_nodes(4, [{}] + [{'dev-disable-commit-after': 0, 'dev-no-htlc-timeout': None}] * 2 + [{'plugin': os.path.join(os.getcwd(), 'tests/plugins/print_htlc_onion.py')}])
-# 
-#     # Two routes to l4: one via l2, and one via l3.
-#     l1.rpc.connect(l2.info['id'], 'localhost', l2.port)
-#     l1.fundchannel(l2, 100000)
-#     l1.rpc.connect(l3.info['id'], 'localhost', l3.port)
-#     l1.fundchannel(l3, 100000)
-#     l2.rpc.connect(l4.info['id'], 'localhost', l4.port)
-#     scid24, _ = l2.fundchannel(l4, 100000)
-#     l3.rpc.connect(l4.info['id'], 'localhost', l4.port)
-#     scid34, _ = l3.fundchannel(l4, 100000)
-#     mine_funding_to_announce(bitcoind, [l1, l2, l3, l4])
-# 
-#     # Wait until l1 knows about all channels.
-#     wait_for(lambda: len(l1.rpc.listchannels()['channels']) == 8)
-# 
-#     inv = l4.rpc.invoice(1000, 'inv', 'inv')
-#     paysecret = l4.rpc.decodepay(inv['bolt11'])['payment_secret']
-# 
-#     # Separate routes for each part of the payment.
-#     r134 = l1.rpc.getroute(l4.info['id'], 501, 1, exclude=[scid24 + '/0', scid24 + '/1'])['route']
-#     r124 = l1.rpc.getroute(l4.info['id'], 499, 1, exclude=[scid34 + '/0', scid34 + '/1'])['route']
-# 
-#     # These can happen in parallel.
-#     l1.rpc.sendpay(
-#         route=r134,
-#         payment_hash=inv['payment_hash'],
-#         amount_msat=1000,
-#         bolt11=inv['bolt11'],
-#         payment_secret=paysecret,
-#         partid=1,
-#         groupid=1
-#     )
-# 
-#     # Can't mix non-parallel payment!
-#     with pytest.raises(RpcError, match=r'Already have parallel payment in progress'):
-#         l1.rpc.sendpay(
-#             route=r124,
-#             payment_hash=inv['payment_hash'],
-#             amount_msat=499,
-#             payment_secret=paysecret,
-#             groupid=1,
-#         )
-# 
-#     # It will not allow a parallel with different msatoshi!
-#     with pytest.raises(RpcError, match=r'msatoshi was previously 1000msat, now 999msat'):
-#         l1.rpc.sendpay(
-#             route=r124,
-#             payment_hash=inv['payment_hash'],
-#             amount_msat=999,
-#             bolt11=inv['bolt11'],
-#             payment_secret=paysecret,
-#             partid=2,
-#             groupid=1,
-#         )
-# 
-#     # This will work fine.
-#     l1.rpc.sendpay(
-#         route=r124,
-#         payment_hash=inv['payment_hash'],
-#         amount_msat=1000,
-#         bolt11=inv['bolt11'],
-#         payment_secret=paysecret,
-#         partid=2,
-#         groupid=1,
-#     )
-# 
-#     # Any more would exceed total payment
-#     with pytest.raises(RpcError, match=r'Already have 1000msat of 1000msat payments in progress'):
-#         l1.rpc.sendpay(
-#             route=r124,
-#             payment_hash=inv['payment_hash'],
-#             amount_msat=1000,
-#             bolt11=inv['bolt11'],
-#             payment_secret=paysecret,
-#             partid=3,
-#             groupid=1,
-#         )
-# 
-#     # But repeat is a NOOP, as long as they're exactly the same!
-#     with pytest.raises(RpcError, match=r'Already pending with amount 501msat \(not 499msat\)'):
-#         l1.rpc.sendpay(
-#             route=r124,
-#             payment_hash=inv['payment_hash'],
-#             amount_msat=1000,
-#             bolt11=inv['bolt11'],
-#             payment_secret=paysecret,
-#             partid=1,
-#             groupid=1,
-#         )
-# 
-#     l1.rpc.sendpay(
-#         route=r134,
-#         payment_hash=inv['payment_hash'],
-#         amount_msat=1000,
-#         bolt11=inv['bolt11'],
-#         payment_secret=paysecret,
-#         partid=1,
-#         groupid=1,
-#     )
-# 
-#     l1.rpc.sendpay(
-#         route=r124,
-#         payment_hash=inv['payment_hash'],
-#         amount_msat=1000,
-#         bolt11=inv['bolt11'],
-#         payment_secret=paysecret,
-#         partid=2,
-#         groupid=1,
-#     )
-# 
-#     # Make sure they've got the HTLCs before we unsuppress
-#     l2.daemon.wait_for_logs('peer_in WIRE_UPDATE_ADD_HTLC')
-#     l3.daemon.wait_for_log('peer_in WIRE_UPDATE_ADD_HTLC')
-# 
-#     # Now continue, payments will succeed due to MPP.
-#     l2.rpc.dev_reenable_commit(l4.info['id'])
-#     l3.rpc.dev_reenable_commit(l4.info['id'])
-#     l2.rpc.dev_reenable_commit(l1.info['id'])
-#     l3.rpc.dev_reenable_commit(l1.info['id'])
-# 
-#     res = l1.rpc.waitsendpay(payment_hash=inv['payment_hash'], partid=1)
-#     assert res['partid'] == 1
-#     res = l1.rpc.waitsendpay(payment_hash=inv['payment_hash'], partid=2)
-#     assert res['partid'] == 2
-# 
-#     for i in range(2):
-#         line = l4.daemon.wait_for_log('print_htlc_onion.py: Got onion')
-#         assert "'type': 'tlv'" in line
-#         assert "'forward_msat': 499" in line or "'forward_msat': 501" in line
-#         assert "'total_msat': 1000" in line
-#         assert "'payment_secret': '{}'".format(paysecret) in line
-# 
-#     pay = only_one(l1.rpc.listpays()['pays'])
-#     assert pay['bolt11'] == inv['bolt11']
-#     assert pay['status'] == 'complete'
-#     assert pay['number_of_parts'] == 2
-#     assert pay['amount_sent_msat'] == Millisatoshi(1002)
-# 
-#     # It will immediately succeed if we pay again.
-#     pay = l1.rpc.sendpay(
-#         route=r124,
-#         payment_hash=inv['payment_hash'],
-#         amount_msat=1000,
-#         bolt11=inv['bolt11'],
-#         payment_secret=paysecret,
-#         partid=2,
-#         groupid=1,
-#     )
-#     assert pay['status'] == 'complete'
-# 
-#     # If we try with an unknown partid, it will refuse.
-#     with pytest.raises(RpcError, match=r'Already succeeded'):
-#         l1.rpc.sendpay(
-#             route=r124,
-#             payment_hash=inv['payment_hash'],
-#             amount_msat=1000,
-#             bolt11=inv['bolt11'],
-#             payment_secret=paysecret,
-#             partid=3,
-#             groupid=1)
-# 
-# 
-# def test_partial_payment_timeout(node_factory, bitcoind):
-#     l1, l2 = node_factory.line_graph(2)
-# 
-#     inv = l2.rpc.invoice(1000, 'inv', 'inv')
-#     paysecret = l2.rpc.decodepay(inv['bolt11'])['payment_secret']
-# 
-#     route = l1.rpc.getroute(l2.info['id'], 500, 1)['route']
-#     l1.rpc.sendpay(
-#         route=route,
-#         payment_hash=inv['payment_hash'],
-#         amount_msat=1000,
-#         bolt11=inv['bolt11'],
-#         payment_secret=paysecret,
-#         partid=1,
-#         groupid=1,
-#     )
-# 
-#     with pytest.raises(RpcError, match=r'WIRE_MPP_TIMEOUT'):
-#         l1.rpc.waitsendpay(
-#             payment_hash=inv['payment_hash'],
-#             timeout=70 + TIMEOUT // 4,
-#             partid=1,
-#             groupid=1,
-#         )
-#     l2.daemon.wait_for_log(r'HTLC set contains 1 HTLCs, for a total of 500msat out of 1000msat \(payment_secret\)')
-# 
-#     # We can still pay it normally.
-#     l1.rpc.sendpay(
-#         route=route,
-#         payment_hash=inv['payment_hash'],
-#         amount_msat=1000,
-#         bolt11=inv['bolt11'],
-#         payment_secret=paysecret,
-#         partid=1,
-#         groupid=2
-#     )
-#     l1.rpc.sendpay(
-#         route=route,
-#         payment_hash=inv['payment_hash'],
-#         amount_msat=1000,
-#         bolt11=inv['bolt11'],
-#         payment_secret=paysecret,
-#         partid=2,
-#         groupid=2
-#     )
-#     l1.rpc.waitsendpay(payment_hash=inv['payment_hash'], timeout=TIMEOUT, partid=1, groupid=2)
-#     l1.rpc.waitsendpay(payment_hash=inv['payment_hash'], timeout=TIMEOUT, partid=2, groupid=2)
-#     l2.daemon.wait_for_log(r'HTLC set contains 1 HTLCs, for a total of 500msat out of 1000msat \(payment_secret\)')
-#     l2.daemon.wait_for_log(r'HTLC set contains 2 HTLCs, for a total of 1000msat out of 1000msat \(payment_secret\)')
-# 
-# 
-# def test_partial_payment_restart(node_factory, bitcoind):
-#     """Test that we recover a set when we restart"""
-#     l1, l2, l3 = node_factory.line_graph(3, wait_for_announce=True,
-#                                          opts=[{}]
-#                                          + [{'may_reconnect': True}] * 2)
-# 
-#     inv = l3.rpc.invoice(1000, 'inv', 'inv')
-#     paysecret = l3.rpc.decodepay(inv['bolt11'])['payment_secret']
-# 
-#     route = l1.rpc.getroute(l3.info['id'], 500, 1)['route']
-# 
-#     l1.rpc.sendpay(
-#         route=route,
-#         payment_hash=inv['payment_hash'],
-#         amount_msat=1000,
-#         bolt11=inv['bolt11'],
-#         payment_secret=paysecret,
-#         partid=1,
-#         groupid=1,
-#     )
-# 
-#     wait_for(lambda: [f['status'] for f in l2.rpc.listforwards()['forwards']] == ['offered'])
-# 
-#     # Restart, and make sure it's reconnected to l2.
-#     l3.restart()
-#     print(l2.rpc.listpeers())
-#     wait_for(lambda: [p['connected'] for p in l2.rpc.listpeers()['peers']] == [True, True])
-# 
-#     # Pay second part.
-#     l1.rpc.sendpay(
-#         route=route,
-#         payment_hash=inv['payment_hash'],
-#         amount_msat=1000,
-#         bolt11=inv['bolt11'],
-#         payment_secret=paysecret,
-#         partid=2,
-#         groupid=1,
-#     )
-# 
-#     l1.rpc.waitsendpay(payment_hash=inv['payment_hash'], timeout=TIMEOUT, partid=1)
-#     l1.rpc.waitsendpay(payment_hash=inv['payment_hash'], timeout=TIMEOUT, partid=2)
-# 
-# 
-# @pytest.mark.developer("needs dev-disconnect")
-# def test_partial_payment_htlc_loss(node_factory, bitcoind):
-#     """Test that we discard a set when the HTLC is lost"""
-#     # We want l2 to fail once it has completed first htlc.
-#     l1, l2, l3 = node_factory.line_graph(3, wait_for_announce=True,
-#                                          opts=[{},
-#                                                {'disconnect': ['=WIRE_UPDATE_ADD_HTLC', '+WIRE_REVOKE_AND_ACK']},
-#                                                {}])
-# 
-#     inv = l3.rpc.invoice(1000, 'inv', 'inv')
-#     paysecret = l3.rpc.decodepay(inv['bolt11'])['payment_secret']
-# 
-#     route = l1.rpc.getroute(l3.info['id'], 500, 1)['route']
-# 
-#     l1.rpc.sendpay(route=route, payment_hash=inv['payment_hash'], amount_msat=1000, bolt11=inv['bolt11'], payment_secret=paysecret, partid=1)
-# 
-#     wait_for(lambda: not only_one(l2.rpc.listpeers(l3.info['id'])['peers'])['connected'])
-#     l2.rpc.dev_fail(l3.info['id'])
-# 
-#     # Since HTLC is missing from commit (dust), it's closed as soon as l2 sees
-#     # it onchain.  l3 shouldn't crash though.
-#     bitcoind.generate_block(1, wait_for_mempool=1)
-# 
-#     with pytest.raises(RpcError,
-#                        match=r'WIRE_PERMANENT_CHANNEL_FAILURE \(reply from remote\)'):
-#         l1.rpc.waitsendpay(payment_hash=inv['payment_hash'], timeout=TIMEOUT, partid=1)
-# 
-# 
-# def test_createonion_limits(node_factory):
-#     l1, = node_factory.get_nodes(1)
-#     hops = [{
-#         # privkey: 41bfd2660762506c9933ade59f1debf7e6495b10c14a92dbcd2d623da2507d3d
-#         "pubkey": "0266e4598d1d3c415f572a8488830b60f7e744ed9235eb0b1ba93283b315c03518",
-#         "payload": bytes([227] + [0] * 227).hex(),
-#     }, {
-#         "pubkey": "0324653eac434488002cc06bbfb7f10fe18991e35f9fe4302dbea6d2353dc0ab1c",
-#         "payload": bytes([227] + [0] * 227).hex(),
-#     }, {
-#         "pubkey": "027f31ebc5462c1fdce1b737ecff52d37d75dea43ce11c74d25aa297165faa2007",
-#         "payload": bytes([227] + [0] * 227).hex(),
-#     }, {
-#         "pubkey": "032c0b7cf95324a07d05398b240174dc0c2be444d96b159aa6c7f7b1e668680991",
-#         "payload": bytes([227] + [0] * 227).hex(),
-#     }, {
-#         "pubkey": "02edabbd16b41c8371b92ef2f04c1185b4f03b6dcd52ba9b78d9d7c89c8f221145",
-#         "payload": bytes([227] + [0] * 227).hex(),
-#     }]
-# 
-#     # This should success since it's right at the edge
-#     l1.rpc.createonion(hops=hops, assocdata="BB" * 32)
-# 
-#     # This one should fail however
-#     with pytest.raises(RpcError, match=r'Payloads exceed maximum onion packet size.'):
-#         hops[0]['payload'] = bytes([228] + [0] * 228).hex()
-#         l1.rpc.createonion(hops=hops, assocdata="BB" * 32)
-# 
-#     # But with a larger onion, it will work!
-#     oniontool = os.path.join(os.path.dirname(__file__), "..", "devtools", "onion")
-#     onion = l1.rpc.createonion(hops=hops, assocdata="BB" * 32, onion_size=1301)['onion']
-# 
-#     # Oniontool wants a filename :(
-#     onionfile = os.path.join(l1.daemon.lightning_dir, 'onion')
-#     with open(onionfile, "w") as f:
-#         f.write(onion)
-# 
-#     subprocess.check_output(
-#         [oniontool, '--assoc-data', "BB" * 32,
-#          'decode', onionfile, "41bfd2660762506c9933ade59f1debf7e6495b10c14a92dbcd2d623da2507d3d"]
-#     )
-# 
-# 
-# @pytest.mark.developer("needs use_shadow")
-# def test_blockheight_disagreement(node_factory, bitcoind, executor):
-#     """
-#     While a payment is in-transit from payer to payee, a block
-#     might be mined, so that the blockheight the payer used to
-#     initiate the payment is no longer the blockheight when the
-#     payee receives it.
-#     This leads to a failure which *used* to be
-#     `final_expiry_too_soon`, a non-permanent failure, but
-#     which is *now* `incorrect_or_unknown_payment_details`,
-#     a permanent failure.
-#     `pay` treats permanent failures as, well, permanent, and
-#     gives up on receiving such failure from the payee, but
-#     this particular subcase of blockheight disagreement is
-#     actually a non-permanent failure (the payer only needs
-#     to synchronize to the same blockheight as the payee).
-#     """
-#     l1, l2 = node_factory.line_graph(2)
-# 
-#     sync_blockheight(bitcoind, [l1, l2])
-# 
-#     # Arrange l1 to stop getting new blocks.
-#     def no_more_blocks(req):
-#         return {"result": None,
-#                 "error": {"code": -8, "message": "Block height out of range"}, "id": req['id']}
-#     l1.daemon.rpcproxy.mock_rpc('getblockhash', no_more_blocks)
-# 
-#     # Increase blockheight and make sure l2 knows it.
-#     # Why 2? Because `pay` uses min_final_cltv_expiry + 1.
-#     # But 2 blocks coming in close succession, plus slow
-#     # forwarding nodes and block propagation, are still
-#     # possible on the mainnet, thus this test.
-#     bitcoind.generate_block(2)
-#     sync_blockheight(bitcoind, [l2])
-# 
-#     # Have l2 make an invoice.
-#     inv = l2.rpc.invoice(1000, 'l', 'd')['bolt11']
-# 
-#     # Have l1 pay l2
-#     def pay(l1, inv):
-#         l1.dev_pay(inv, use_shadow=False)
-#     fut = executor.submit(pay, l1, inv)
-# 
-#     # Make sure l1 sends out the HTLC.
-#     l1.daemon.wait_for_logs([r'NEW:: HTLC LOCAL'])
-# 
-#     height = bitcoind.rpc.getblockchaininfo()['blocks']
-#     l1.daemon.wait_for_log('Remote node appears to be on a longer chain.*catch up to block {}'.format(height))
-# 
-#     # Unblock l1 from new blocks.
-#     l1.daemon.rpcproxy.mock_rpc('getblockhash', None)
-# 
-#     # pay command should complete without error
-#     fut.result()
-# 
-# 
-# def test_sendpay_msatoshi_arg(node_factory):
-#     """sendpay msatoshi arg was used for non-MPP to indicate the amount
-# they asked for.  But using it with anything other than the final amount
-# caused a crash in 0.8.0, so we then disallowed it.
-#     """
-#     l1, l2 = node_factory.line_graph(2)
-# 
-#     inv = l2.rpc.invoice(1000, 'inv', 'inv')
-# 
-#     # Can't send non-MPP payment which specifies msatoshi != final.
-#     with pytest.raises(RpcError, match=r'Do not specify msatoshi \(1001msat\) without'
-#                        ' partid: if you do, it must be exactly'
-#                        r' the final amount \(1000msat\)'):
-#         l1.rpc.sendpay(route=l1.rpc.getroute(l2.info['id'], 1000, 1)['route'], payment_hash=inv['payment_hash'], amount_msat=1001, bolt11=inv['bolt11'], payment_secret=inv['payment_secret'])
-#     with pytest.raises(RpcError, match=r'Do not specify msatoshi \(999msat\) without'
-#                        ' partid: if you do, it must be exactly'
-#                        r' the final amount \(1000msat\)'):
-#         l1.rpc.sendpay(route=l1.rpc.getroute(l2.info['id'], 1000, 1)['route'], payment_hash=inv['payment_hash'], amount_msat=999, bolt11=inv['bolt11'], payment_secret=inv['payment_secret'])
-# 
-#     # Can't send MPP payment which pays any more than amount.
-#     with pytest.raises(RpcError, match=r'Final amount 1001msat is greater than 1000msat, despite MPP'):
-#         l1.rpc.sendpay(route=l1.rpc.getroute(l2.info['id'], 1001, 1)['route'], payment_hash=inv['payment_hash'], amount_msat=1000, bolt11=inv['bolt11'], partid=1, payment_secret=inv['payment_secret'])
-# 
-#     # But this works
-#     l1.rpc.sendpay(route=l1.rpc.getroute(l2.info['id'], 1001, 1)['route'], payment_hash=inv['payment_hash'], amount_msat=1001, bolt11=inv['bolt11'], payment_secret=inv['payment_secret'])
-#     l1.rpc.waitsendpay(inv['payment_hash'])
-# 
-#     inv = only_one(l2.rpc.listinvoices('inv')['invoices'])
-#     assert inv['status'] == 'paid'
-#     assert inv['amount_received_msat'] == Millisatoshi(1001)
-# 
-# 
-# def test_reject_invalid_payload(node_factory):
-#     """Send an onion payload with an unknown even type.
-# 
-#     Recipient l2 should reject it the incoming HTLC with an invalid onion
-#     payload error.
-# 
-#     """
-# 
-#     l1, l2 = node_factory.line_graph(2)
-#     amt = 10**3
-#     route = l1.rpc.getroute(l2.info['id'], amt, 10)['route']
-#     inv = l2.rpc.invoice(amt, "lbl", "desc")
-# 
-#     first_hop = route[0]
-# 
-#     # A TLV payload with an unknown even type:
-#     payload = TlvPayload()
-#     payload.add_field(0xB000, b'Hi there')
-#     hops = [{"pubkey": l2.info['id'], "payload": payload.to_hex()}]
-#     onion = l1.rpc.createonion(hops=hops, assocdata=inv['payment_hash'])
-#     l1.rpc.sendonion(onion=onion['onion'],
-#                      first_hop=first_hop,
-#                      payment_hash=inv['payment_hash'],
-#                      shared_secrets=onion['shared_secrets'])
-# 
-#     l2.daemon.wait_for_log(r'Failing HTLC because of an invalid payload')
-# 
-#     with pytest.raises(RpcError, match=r'WIRE_INVALID_ONION_PAYLOAD'):
-#         l1.rpc.waitsendpay(inv['payment_hash'])
-# 
-# 
-# @pytest.mark.skip("Needs to be updated for modern onion")
-# @unittest.skipIf(not EXPERIMENTAL_FEATURES, "Needs blinding args to sendpay")
-# def test_sendpay_blinding(node_factory):
-#     l1, l2, l3, l4 = node_factory.line_graph(4)
-# 
-#     blindedpathtool = os.path.join(os.path.dirname(__file__), "..", "devtools", "blindedpath")
-# 
-#     # Create blinded path l2->l4
-#     output = subprocess.check_output(
-#         [blindedpathtool, '--simple-output', 'create',
-#          l2.info['id'] + "/" + l2.get_channel_scid(l3),
-#          l3.info['id'] + "/" + l3.get_channel_scid(l4),
-#          l4.info['id']]
-#     ).decode('ASCII').strip()
-# 
-#     # First line is blinding, then <peerid> then <encblob>.
-#     blinding, p1, p1enc, p2, p2enc, p3 = output.split('\n')
-#     # First hop can't be blinded!
-#     assert p1 == l2.info['id']
-# 
-#     amt = 10**3
-#     inv = l4.rpc.invoice(amt, "lbl", "desc")
-# 
-#     route = [{'id': l2.info['id'],
-#               'channel': l1.get_channel_scid(l2),
-#               'amount_msat': Millisatoshi(1002),
-#               'delay': 21,
-#               'blinding': blinding,
-#               'enctlv': p1enc},
-#              {'id': p2,
-#               'amount_msat': Millisatoshi(1001),
-#               'delay': 15,
-#               # FIXME: this is a dummy!
-#               'channel': '0x0x0',
-#               'enctlv': p2enc},
-#              {'id': p3,
-#               # FIXME: this is a dummy!
-#               'channel': '0x0x0',
-#               'amount_msat': Millisatoshi(1000),
-#               'delay': 9,
-#               'style': 'tlv'}]
-#     l1.rpc.sendpay(route=route,
-#                    payment_hash=inv['payment_hash'],
-#                    bolt11=inv['bolt11'], payment_secret=inv['payment_secret'])
-#     l1.rpc.waitsendpay(inv['payment_hash'])
-# 
-# 
-# def test_excluded_adjacent_routehint(node_factory, bitcoind):
-#     """Test case where we try have a routehint which leads to an adjacent
-#     node, but the result exceeds our maxfee; we crashed trying to find
-#     what part of the path was most expensive in that case
-# 
-#     """
-#     l1, l2, l3 = node_factory.line_graph(3)
-# 
-#     # We'll be forced to use routehint, since we don't know about l3.
-#     wait_for(lambda: len(l3.rpc.listchannels(source=l2.info['id'])['channels']) == 1)
-#     inv = l3.rpc.invoice(10**3, "lbl", "desc", exposeprivatechannels=l2.get_channel_scid(l3))
-# 
-#     # This will make it reject the routehint.
-#     err = r'Fee exceeds our fee budget: 1msat > 0msat, discarding route'
-#     with pytest.raises(RpcError, match=err):
-#         l1.rpc.pay(bolt11=inv['bolt11'], maxfeepercent=0, exemptfee=0)
-# 
-# 
-# def test_keysend(node_factory):
-#     amt = 10000
-#     l1, l2, l3, l4 = node_factory.line_graph(
-#         4,
-#         wait_for_announce=True,
-#         opts=[{}, {}, {}, {'disable-plugin': 'keysend'}]
-#     )
-# 
-#     # The keysend featurebit must be set in the announcement, i.e., l1 should
-#     # learn that l3 supports keysends.
-#     features = l1.rpc.listnodes(l3.info['id'])['nodes'][0]['features']
-#     assert(int(features, 16) >> 55 & 0x01 == 1)
-# 
-#     # If we disable keysend, then the featurebit must not be set,
-#     # i.e., l4 doesn't support it.
-#     features = l1.rpc.listnodes(l4.info['id'])['nodes'][0]['features']
-#     assert(int(features, 16) >> 55 & 0x01 == 0)
-# 
-#     # Self-sends are not allowed (see #4438)
-#     with pytest.raises(RpcError, match=r'We are the destination.'):
-#         l1.rpc.keysend(l1.info['id'], amt)
-# 
-#     # Send an indirect one from l1 to l3
-#     l1.rpc.keysend(l3.info['id'], amt)
-#     invs = l3.rpc.listinvoices()['invoices']
-#     assert(len(invs) == 1)
-# 
-#     inv = invs[0]
-#     print(inv)
-#     assert(inv['amount_received_msat'] >= Millisatoshi(amt))
-# 
-#     # Now send a direct one instead from l1 to l2
-#     l1.rpc.keysend(l2.info['id'], amt)
-#     invs = l2.rpc.listinvoices()['invoices']
-#     assert(len(invs) == 1)
-# 
-#     inv = invs[0]
-#     assert(inv['amount_received_msat'] >= Millisatoshi(amt))
-# 
-#     # And finally try to send a keysend payment to l4, which doesn't
-#     # support it. It MUST fail.
-#     with pytest.raises(RpcError, match=r"Recipient [0-9a-f]{66} reported an invalid payload"):
-#         l3.rpc.keysend(l4.info['id'], amt)
-# 
-# 
-# def test_keysend_strip_tlvs(node_factory):
-#     """Use the extratlvs option to deliver a message with sphinx' TLV type, which keysend strips.
-#     """
-#     amt = 10**7
-#     l1, l2 = node_factory.line_graph(
-#         2,
-#         wait_for_announce=True,
-#         opts=[
-#             {
-#                 # Not needed, just for listconfigs test.
-#                 'accept-htlc-tlv-types': '133773310,99990',
-#                 "plugin": os.path.join(os.path.dirname(__file__), "plugins/sphinx-receiver.py"),
-#             },
-#             {
-#                 "plugin": os.path.join(os.path.dirname(__file__), "plugins/sphinx-receiver.py"),
-#             },
-#         ]
-#     )
-# 
-#     # Make sure listconfigs works here
-#     assert l1.rpc.listconfigs()['accept-htlc-tlv-types'] == '133773310,99990'
-# 
-#     # l1 is configured to accept, so l2 should still filter them out
-#     l1.rpc.keysend(l2.info['id'], amt, extratlvs={133773310: 'FEEDC0DE'})
-#     inv = only_one(l2.rpc.listinvoices()['invoices'])
-#     assert not l2.daemon.is_in_log(r'plugin-sphinx-receiver.py.*extratlvs.*133773310.*feedc0de')
-# 
-#     assert(inv['amount_received_msat'] >= Millisatoshi(amt))
-#     assert inv['description'] == 'keysend'
-#     l2.rpc.delinvoice(inv['label'], 'paid')
-# 
-#     # Now try again with the TLV type in extra_tlvs as string:
-#     l1.rpc.keysend(l2.info['id'], amt, extratlvs={133773310: b'hello there'.hex()})
-#     inv = only_one(l2.rpc.listinvoices()['invoices'])
-#     assert inv['description'] == 'keysend: hello there'
-#     l2.daemon.wait_for_log('Keysend payment uses illegal even field 133773310: stripping')
-#     l2.rpc.delinvoice(inv['label'], 'paid')
-# 
-#     # We can (just!) fit a giant description in.
-#     l1.rpc.keysend(l2.info['id'], amt, extratlvs={133773310: (b'a' * 1100).hex()})
-#     inv = only_one(l2.rpc.listinvoices()['invoices'])
-#     assert inv['description'] == 'keysend: ' + 'a' * 1100
-#     l2.rpc.delinvoice(inv['label'], 'paid')
-#     l2.daemon.wait_for_log('Keysend payment uses illegal even field 133773310: stripping')
-# 
-#     # Now try with some special characters
-#     ksinfo = """💕 ₿"'
-# More info
-# """
-#     # Since we're at it, use this to test string-keyed TLVs
-#     l1.rpc.keysend(l2.info['id'], amt, extratlvs={"133773310": bytes(ksinfo, encoding='utf8').hex()})
-#     inv = only_one(l2.rpc.listinvoices()['invoices'])
-#     assert inv['description'] == 'keysend: ' + ksinfo
-#     l2.daemon.wait_for_log('Keysend payment uses illegal even field 133773310: stripping')
-# 
-#     # Now reverse the direction. l1 accepts 133773310, but filters out
-#     # other even unknown types (like 133773312).
-#     l2.rpc.keysend(l1.info['id'], amt, extratlvs={
-#         "133773310": b"helloworld".hex(),  # This one is allowlisted
-#         "133773312": b"filterme".hex(),  # This one will get stripped
-#     })
-# 
-#     # The invoice_payment hook must contain the allowlisted TLV type,
-#     # but not the stripped one.
-#     assert l1.daemon.wait_for_log(r'plugin-sphinx-receiver.py: invoice_payment.*extratlvs.*133773310')
-#     assert not l1.daemon.is_in_log(r'plugin-sphinx-receiver.py: invoice_payment.*extratlvs.*133773312')
-# 
-# 
-# def test_keysend_routehint(node_factory):
-#     """Test whether we can deliver a keysend by adding a routehint on the cli
-#     """
-#     amt = 10000
-#     l1, l2 = node_factory.line_graph(2, wait_for_announce=True)
-#     l3 = node_factory.get_node()
-#     l2.connect(l3)
-#     l2.fundchannel(l3, announce_channel=False)
-# 
-#     dest = l3.info['id']
-#     routehints = [
-#         [
-#             {
-#                 'scid': only_one(l3.rpc.listpeerchannels()['channels'])['alias']['remote'],
-#                 'id': l2.info['id'],
-#                 'feebase': '1msat',
-#                 'feeprop': 10,
-#                 'expirydelta': 9,
-#             }
-#         ],
-#         [  # Dummy
-#             {
-#                 'scid': '1x2x3',
-#                 'id': '02' * 33,
-#                 'feebase': 1,
-#                 'feeprop': 1,
-#                 'expirydelta': 9,
-#             },
-#         ],
-#     ]
-# 
-#     # Without any hints we should fail:
-#     with pytest.raises(RpcError):
-#         l1.rpc.call("keysend", payload={'destination': dest, 'amount_msat': amt})
-# 
-#     # We should also fail with only non-working hints:
-#     with pytest.raises(RpcError):
-#         l1.rpc.call("keysend", payload={'destination': dest, 'amount_msat': amt, 'routehints': routehints[1:]})
-# 
-#     l1.rpc.call("keysend", payload={'destination': dest, 'amount_msat': amt, 'routehints': routehints})
-#     invs = l3.rpc.listinvoices()['invoices']
-#     assert(len(invs) == 1)
-# 
-#     inv = invs[0]
-#     assert(inv['amount_received_msat'] >= Millisatoshi(amt))
-# 
-# 
-# def test_invalid_onion_channel_update(node_factory):
-#     '''
-#     Some onion failures "should" send a `channel_update`.
-# 
-#     This test checks to see if we handle things correctly
-#     even if some remote node does not send the required
-#     `channel_update`.
-#     '''
-#     plugin = os.path.join(os.getcwd(), 'tests/plugins/fail_htlcs_invalid.py')
-#     l1, l2, l3 = node_factory.line_graph(3,
-#                                          opts=[{},
-#                                                {'plugin': plugin},
-#                                                {}],
-#                                          wait_for_announce=True)
-# 
-#     l1id = l1.info['id']
-# 
-#     inv = l3.rpc.invoice(12345, 'inv', 'inv')['bolt11']
-#     # Should fail, since l2 will always fail to forward.
-#     with pytest.raises(RpcError):
-#         l1.rpc.pay(inv)
-# 
-#     # l1 should still be alive afterwards.
-#     assert l1.rpc.getinfo()['id'] == l1id
-# 
-# 
-# @pytest.mark.developer("Requires use_shadow")
-# def test_pay_exemptfee(node_factory):
-#     """Tiny payment, huge fee
-# 
-#     l1 -> l2 -> l3
-# 
-#     Create a tiny invoice for 1 msat, it'll be dominated by the base_fee on
-#     the l2->l3 channel. So it'll get rejected on the first attempt if we set
-#     the exemptfee way too low. The default fee exemption threshold is
-#     5000msat, so 5001msat is not exempted by default and a 5001msat fee on
-#     l2->l3 should trigger this.
-# 
-#     """
-#     l1, l2, l3 = node_factory.line_graph(
-#         3,
-#         opts=[{}, {'fee-base': 5001, 'fee-per-satoshi': 0}, {}],
-#         wait_for_announce=True
-#     )
-# 
-#     err = r'Ran out of routes to try'
-# 
-#     with pytest.raises(RpcError, match=err):
-#         l1.dev_pay(l3.rpc.invoice(1, "lbl1", "desc")['bolt11'], use_shadow=False)
-# 
-#     # If we tell our node that 5001msat is ok this should work
-#     l1.dev_pay(l3.rpc.invoice(1, "lbl2", "desc")['bolt11'], use_shadow=False, exemptfee=5001)
-# 
-#     # Given the above network this is the smallest amount that passes without
-#     # the fee-exemption (notice that we let it through on equality).
-#     threshold = int(5001 / 0.05)
-# 
-#     # This should be just below the fee-exemption and is the first value that is allowed through
-#     with pytest.raises(RpcError, match=err):
-#         l1.dev_pay(l3.rpc.invoice(threshold - 1, "lbl3", "desc")['bolt11'], use_shadow=False)
-# 
-#     # While this'll work just fine
-#     l1.dev_pay(l3.rpc.invoice(int(5001 * 200), "lbl4", "desc")['bolt11'], use_shadow=False)
-# 
-# 
-# @pytest.mark.developer("Requires use_shadow flag")
-# def test_pay_peer(node_factory, bitcoind):
-#     """If we have a direct channel to the destination we should use that.
-# 
-#     This is complicated a bit by not having sufficient capacity, but the
-#     channel_hints can help us there.
-# 
-#     l1 -> l2
-#      |   ^
-#      v  /
-#      l3
-#     """
-#     # Set the dust exposure higher, this gets triggered on liquid
-#     l1, l2, l3 = node_factory.get_nodes(3, opts={'max-dust-htlc-exposure-msat': '100000sat'})
-#     node_factory.join_nodes([l1, l2])
-#     node_factory.join_nodes([l1, l3])
-#     node_factory.join_nodes([l3, l2], wait_for_announce=True)
-# 
-#     wait_for(lambda: len(l1.rpc.listchannels()['channels']) == 6)
-# 
-#     def spendable(n1, n2):
-#         chan = n1.rpc.listpeerchannels(n2.info['id'])['channels'][0]
-#         avail = chan['spendable_msat']
-#         return avail
-# 
-#     amt = Millisatoshi(10**8)
-#     # How many payments do we expect to go through directly?
-#     direct = spendable(l1, l2).millisatoshis // amt.millisatoshis
-# 
-#     # Remember the l1 -> l3 capacity, it should not change until we run out of
-#     # direct capacity.
-#     l1l3cap = spendable(l1, l3)
-# 
-#     for i in range(0, direct):
-#         inv = l2.rpc.invoice(amt.millisatoshis, "lbl{}".format(i),
-#                              "desc{}".format(i))['bolt11']
-#         l1.dev_pay(inv, use_shadow=False)
-# 
-#     # We should not have more than amt in the direct channel anymore
-#     assert(spendable(l1, l2) < amt)
-#     assert(spendable(l1, l3) == l1l3cap)
-# 
-#     # Next one should take the alternative, but it should still work
-#     inv = l2.rpc.invoice(amt.millisatoshis, "final", "final")['bolt11']
-#     l1.dev_pay(inv, use_shadow=False)
-# 
-# 
-# def test_mpp_presplit(node_factory):
-#     """Make a rather large payment of 5*10ksat and see it being split.
-#     """
-#     MPP_TARGET_SIZE = 10**7  # Taken from libpluin-pay.c
-#     amt = 5 * MPP_TARGET_SIZE
-# 
-#     # Assert that the amount we're going to send is indeed larger than our
-#     # split size.
-#     assert(MPP_TARGET_SIZE < amt)
-# 
-#     l1, l2, l3 = node_factory.line_graph(
-#         3, fundamount=10**8, wait_for_announce=True,
-#         opts={'wumbo': None,
-#               'max-dust-htlc-exposure-msat': '500000sat'}
-#     )
-# 
-#     inv = l3.rpc.invoice(amt, 'lbl', 'desc')['bolt11']
-#     p = l1.rpc.pay(inv)
-# 
-#     assert(p['parts'] >= 5)
-#     inv = l3.rpc.listinvoices()['invoices'][0]
-# 
-#     assert(inv['amount_msat'] == inv['amount_received_msat'])
-# 
-#     # Make sure that bolt11 isn't duplicated for every part
-#     bolt11s = 0
-#     count = 0
-#     for p in l1.rpc.listsendpays()['payments']:
-#         if 'bolt11' in p:
-#             bolt11s += 1
-#         count += 1
-# 
-#     # You were supposed to mpp!
-#     assert count > 1
-#     # Not every one should have the bolt11 string
-#     assert bolt11s < count
-#     # In fact, only one should
-#     assert bolt11s == 1
-# 
-#     # But listpays() gathers it:
-#     assert only_one(l1.rpc.listpays()['pays'])['bolt11'] == inv['bolt11']
-# 
-# 
-# def test_mpp_adaptive(node_factory, bitcoind):
-#     """We have two paths, both too small on their own, let's combine them.
-# 
-#     ```dot
-#     digraph {
-#       l1 -> l2 [label="scid=103x1x1, cap=amt-1"];
-#       l2 -> l4 [label="scid=105x1x1, cap=max"];
-#       l1 -> l3 [label="scid=107x1x1, cap=max"];
-#       l3 -> l4 [label="scid=109x1x1, cap=amt-1"];
-#     }
-#     """
-#     amt = 10**7 - 1
-#     l1, l2, l3, l4 = node_factory.get_nodes(4)
-# 
-#     l1.connect(l2)
-#     l2.connect(l4)
-#     l1.connect(l3)
-#     l3.connect(l4)
-# 
-#     # First roadblock right away on an outgoing channel
-#     l2.fundchannel(l1, amt)
-#     l2.fundchannel(l4, amt, wait_for_active=True)
-#     l2.rpc.pay(l1.rpc.invoice(
-#         amt + 99999000 - 1,  # Slightly less than amt + reserve
-#         label="reb l1->l2",
-#         description="Rebalance l1 -> l2"
-#     )['bolt11'])
-# 
-#     # Second path fails only after the first hop
-#     l1.fundchannel(l3, amt)
-#     l4.fundchannel(l3, amt, wait_for_active=True)
-#     l4.rpc.pay(l3.rpc.invoice(
-#         amt + 99999000 - 1,  # Slightly less than amt + reserve
-#         label="reb l3->l4",
-#         description="Rebalance l3 -> l4"
-#     )['bolt11'])
-#     l1.rpc.listpeers()
-# 
-#     # Make sure neither channel can fit the payment by itself.
-#     c12 = l1.rpc.listpeerchannels(l2.info['id'])['channels'][0]
-#     c34 = l3.rpc.listpeerchannels(l4.info['id'])['channels'][0]
-#     assert(c12['spendable_msat'].millisatoshis < amt)
-#     assert(c34['spendable_msat'].millisatoshis < amt)
-# 
-#     # Make sure all HTLCs entirely resolved before we mine more blocks!
-#     def all_htlcs(n):
-#         htlcs = []
-#         for p in n.rpc.listpeers()['peers']:
-#             for c in n.rpc.listpeerchannels(p['id'])['channels']:
-#                 htlcs += c['htlcs']
-#         return htlcs
-# 
-#     wait_for(lambda: all([all_htlcs(n) == [] for n in [l1, l2, l3, l4]]))
-# 
-#     mine_funding_to_announce(bitcoind, [l1, l2, l3, l4])
-#     wait_for(lambda: len(l1.rpc.listchannels()['channels']) == 8)
-# 
-#     inv = l4.rpc.invoice(
-#         amt,
-#         label="splittest",
-#         description="Needs to be split into at least 2"
-#     )['bolt11']
-# 
-#     p = l1.rpc.pay(inv)
-#     from pprint import pprint
-#     pprint(p)
-#     pprint(l1.rpc.paystatus(inv))
-# 
-#     # Make sure that bolt11 isn't duplicated for every part
-#     bolt11s = 0
-#     count = 0
-#     for p in l1.rpc.listsendpays()['payments']:
-#         if 'bolt11' in p:
-#             bolt11s += 1
-#         count += 1
-# 
-#     # You were supposed to mpp!
-#     assert count > 1
-#     # Not every one should have the bolt11 string
-#     assert bolt11s < count
-# 
-#     # listpays() shows bolt11 string
-#     assert 'bolt11' in only_one(l1.rpc.listpays()['pays'])
-# 
-# 
-# def test_pay_fail_unconfirmed_channel(node_factory, bitcoind):
-#     '''
-#     Replicate #3855.
-#     `pay` crash when any direct channel is still
-#     unconfirmed.
-#     '''
-#     l1, l2 = node_factory.get_nodes(2)
-# 
-#     amount_sat = 10 ** 6
-# 
-#     # create l2->l1 channel.
-#     l2.fundwallet(amount_sat * 5)
-#     l1.rpc.connect(l2.info['id'], 'localhost', l2.port)
-#     l2.rpc.fundchannel(l1.info['id'], amount_sat * 3)
-#     # channel is still unconfirmed.
-# 
-#     # Attempt to pay from l1 to l2.
-#     # This should fail since the channel capacities are wrong.
-#     invl2 = l2.rpc.invoice(Millisatoshi(amount_sat * 1000), 'i', 'i')['bolt11']
-#     with pytest.raises(RpcError):
-#         l1.rpc.pay(invl2)
-# 
-#     # Let the channel confirm.
-#     bitcoind.generate_block(6)
-#     sync_blockheight(bitcoind, [l1, l2])
-# 
-#     # Now give enough capacity so l1 can pay.
-#     invl1 = l1.rpc.invoice(Millisatoshi(amount_sat * 2 * 1000), 'j', 'j')['bolt11']
-#     l2.rpc.pay(invl1)
-# 
-#     # Wait for us to recognize that the channel is available
-#     wait_for(lambda: l1.rpc.listpeerchannels()['channels'][0]['spendable_msat'].millisatoshis > amount_sat * 1000)
-# 
-#     # Now l1 can pay to l2.
-#     l1.rpc.pay(invl2)
-# 
-# 
-# def test_bolt11_null_after_pay(node_factory, bitcoind):
-#     l1, l2 = node_factory.get_nodes(2)
-# 
-#     amount_sat = 10 ** 6
-#     # pay a generic bolt11 and test if the label bol11 is null
-#     # inside the command listpays
-# 
-#     # create l2->l1 channel.
-#     l2.fundwallet(amount_sat * 5)
-#     l1.rpc.connect(l2.info['id'], 'localhost', l2.port)
-#     # Make sure l2 considers it fully connected too!
-#     wait_for(lambda: l2.rpc.listpeers(l1.info['id']) != {'peers': []})
-#     l2.rpc.fundchannel(l1.info['id'], amount_sat * 3)
-# 
-#     # Let the channel confirm.
-#     bitcoind.generate_block(6)
-#     sync_blockheight(bitcoind, [l1, l2])
-#     wait_for(lambda: only_one(l1.rpc.listpeerchannels()['channels'])['state'] == 'CHANNELD_NORMAL')
-# 
-#     amt = Millisatoshi(amount_sat * 2 * 1000)
-#     invl1 = l1.rpc.invoice(amt, 'j', 'j')['bolt11']
-#     l2.rpc.pay(invl1)
-# 
-#     pays = l2.rpc.listpays()["pays"]
-#     assert(pays[0]["bolt11"] == invl1)
-#     assert('amount_msat' in pays[0] and pays[0]['amount_msat'] == amt)
-#     assert('created_at' in pays[0])
-#     assert('completed_at' in pays[0])
-# 
-# 
-# def test_mpp_presplit_routehint_conflict(node_factory, bitcoind):
-#     '''
-#     We had a bug where pre-splitting the payment prevents *any*
-#     routehints from being taken.
-#     We tickle that bug here by building l1->l2->l3, but with
-#     l2->l3 as an unpublished channel.
-#     If the payment is large enough to trigger pre-splitting, the
-#     routehints are not applied in any of the splits.
-#     '''
-#     l1, l2, l3 = node_factory.get_nodes(3)
-# 
-#     l1.rpc.connect(l2.info['id'], 'localhost', l2.port)
-#     l1l2, _ = l1.fundchannel(l2, 10**7, announce_channel=True)
-#     l2.rpc.connect(l3.info['id'], 'localhost', l3.port)
-#     l2.fundchannel(l3, 10**7, announce_channel=False)
-# 
-#     mine_funding_to_announce(bitcoind, [l1, l2, l3])
-# 
-#     # Wait for l3 to learn about l1->l2, otherwise it will think
-#     # l2 is a deadend and not add it to the routehint.
-#     wait_for(lambda: len(l3.rpc.listchannels(l1l2)['channels']) >= 2)
-# 
-#     inv = l3.rpc.invoice(Millisatoshi(2 * 10000 * 1000), 'i', 'i', exposeprivatechannels=True)['bolt11']
-# 
-#     l1.rpc.pay(inv)
-# 
-# 
-# def test_delpay_argument_invalid(node_factory, bitcoind):
-#     """
-#     This test includes all possible combinations of input error inside the
-#     delpay command.
-#     """
-# 
-#     # Create the line graph l2 -> l1 with a channel of 10 ** 5 sat!
-#     l2, l1 = node_factory.line_graph(2, fundamount=10**5, wait_for_announce=True)
-# 
-#     l2.rpc.check_request_schemas = False
-#     with pytest.raises(RpcError):
-#         l2.rpc.delpay()
-#     l2.rpc.check_request_schemas = True
-# 
-#     # sanity check
-#     inv = l1.rpc.invoice(10 ** 5, 'inv', 'inv')
-#     payment_hash = "AA" * 32
-#     with pytest.raises(RpcError):
-#         l2.rpc.delpay(payment_hash, 'complete')
-# 
-#     l2.rpc.pay(inv['bolt11'])
-# 
-#     wait_for(lambda: l2.rpc.listpays(inv['bolt11'])['pays'][0]['status'] == 'complete')
-# 
-#     payment_hash = inv['payment_hash']
-# 
-#     # payment paid with wrong status (pending status is a illegal input)
-#     l2.rpc.check_request_schemas = False
-#     with pytest.raises(RpcError):
-#         l2.rpc.delpay(payment_hash, 'pending')
-# 
-#     with pytest.raises(RpcError):
-#         l2.rpc.delpay(payment_hash, 'invalid_status')
-#     l2.rpc.check_request_schemas = True
-# 
-#     with pytest.raises(RpcError):
-#         l2.rpc.delpay(payment_hash, 'failed')
-# 
-#     # test if the node is still ready
-#     payments = l2.rpc.delpay(payment_hash, 'complete')
-# 
-#     assert payments['payments'][0]['bolt11'] == inv['bolt11']
-#     assert len(payments['payments']) == 1
-#     assert len(l2.rpc.listpays()['pays']) == 0
-# 
-# 
-# def test_delpay_payment_split(node_factory, bitcoind):
-#     """
-#     Test behavior of delpay with an MPP
-#     """
-#     MPP_TARGET_SIZE = 10**7  # Taken from libpluin-pay.c
-#     amt = 4 * MPP_TARGET_SIZE
-# 
-#     l1, l2, l3 = node_factory.line_graph(3, fundamount=10**5,
-#                                          wait_for_announce=True)
-#     inv = l3.rpc.invoice(amt, 'lbl', 'desc')
-#     l1.rpc.pay(inv['bolt11'])
-# 
-#     assert len(l1.rpc.listpays()['pays']) == 1
-#     delpay_result = l1.rpc.delpay(inv['payment_hash'], 'complete')['payments']
-#     assert len(delpay_result) >= 4
-#     assert len(l1.rpc.listpays()['pays']) == 0
-# 
-# 
-# def test_listpay_result_with_paymod(node_factory, bitcoind):
-#     """
-#     The object of this test is to verify the correct behavior
-#     of the RPC command listpay e with two different type of
-#     payment, such as: keysend (without invoice) and pay (with invoice).
-#     l1 -> keysend -> l2
-#     l2 -> pay invoice -> l3
-#     """
-# 
-#     amount_sat = 10 ** 6
-# 
-#     l1, l2, l3 = node_factory.line_graph(3, wait_for_announce=True)
-# 
-#     invl2 = l2.rpc.invoice(amount_sat * 2, "inv_l2", "inv_l2")
-#     l1.rpc.pay(invl2['bolt11'])
-# 
-#     l2.rpc.keysend(l3.info['id'], amount_sat * 2, "keysend_l3")
-# 
-#     assert 'bolt11' in l1.rpc.listpays()['pays'][0]
-#     assert 'bolt11' not in l2.rpc.listpays()['pays'][0]
-#     assert 'payment_hash' in l2.rpc.listpays()['pays'][0]
-#     assert 'payment_hash' in l1.rpc.listpays()['pays'][0]
-#     assert 'destination' in l1.rpc.listpays()['pays'][0]
-#     assert 'destination' in l2.rpc.listpays()['pays'][0]
-# 
-# 
-# def test_listsendpays_and_listpays_order(node_factory):
-#     """listsendpays should be in increasing id order, listpays in created_at"""
-#     l1, l2 = node_factory.line_graph(2)
-#     for i in range(5):
-#         inv = l2.rpc.invoice(1000 - i, "test {}".format(i), "test")['bolt11']
-#         l1.rpc.pay(inv)
-# 
-#     ids = [p['id'] for p in l1.rpc.listsendpays()['payments']]
-#     assert ids == sorted(ids)
-# 
-#     created_at = [p['created_at'] for p in l1.rpc.listpays()['pays']]
-#     assert created_at == sorted(created_at)
-# 
-# 
-# @pytest.mark.developer("needs use_shadow")
-# def test_mpp_waitblockheight_routehint_conflict(node_factory, bitcoind, executor):
-#     '''
-#     We have a bug where a blockheight disagreement between us and
-#     the receiver causes us to advance through the routehints a bit
-#     too aggressively.
-#     '''
-#     l1, l2, l3 = node_factory.get_nodes(3)
-# 
-#     l1.rpc.connect(l2.info['id'], 'localhost', l2.port)
-#     l1l2, _ = l1.fundchannel(l2, 10**7, announce_channel=True)
-#     l2.rpc.connect(l3.info['id'], 'localhost', l3.port)
-#     l2.fundchannel(l3, 10**7, announce_channel=False)
-# 
-#     mine_funding_to_announce(bitcoind, [l1, l2, l3])
-# 
-#     # Wait for l3 to learn about l1->l2, otherwise it will think
-#     # l2 is a deadend and not add it to the routehint.
-#     wait_for(lambda: len(l3.rpc.listchannels(l1l2)['channels']) >= 2)
-# 
-#     # Now make the l1 payer stop receiving blocks.
-#     def no_more_blocks(req):
-#         return {"result": None,
-#                 "error": {"code": -8, "message": "Block height out of range"}, "id": req['id']}
-#     l1.daemon.rpcproxy.mock_rpc('getblockhash', no_more_blocks)
-# 
-#     # Increase blockheight by 2, like in test_blockheight_disagreement.
-#     bitcoind.generate_block(2)
-#     sync_blockheight(bitcoind, [l3])
-# 
-#     inv = l3.rpc.invoice(Millisatoshi(2 * 10000 * 1000), 'i', 'i', exposeprivatechannels=True)['bolt11']
-# 
-#     # Have l1 pay l3
-#     def pay(l1, inv):
-#         l1.dev_pay(inv, use_shadow=False)
-#     fut = executor.submit(pay, l1, inv)
-# 
-#     # Make sure l1 sends out the HTLC.
-#     l1.daemon.wait_for_logs([r'NEW:: HTLC LOCAL'])
-# 
-#     # Unblock l1 from new blocks.
-#     l1.daemon.rpcproxy.mock_rpc('getblockhash', None)
-# 
-#     # pay command should complete without error
-#     fut.result(TIMEOUT)
-# 
-# 
-# @pytest.mark.developer("channel setup very slow (~10 minutes) if not DEVELOPER")
-# @pytest.mark.slow_test
-# @pytest.mark.openchannel('v1')
-# @pytest.mark.openchannel('v2')
-# @unittest.skipIf(True, "Temporarily disabled while flake diagnosed: blame Rusty!")
-# def test_mpp_interference_2(node_factory, bitcoind, executor):
-#     '''
-#     We create a "public network" that looks like so.
-#     Each channel is perfectly balanced, with 7 * unit
-#     funds on each side.
-# 
-#         4 -- 5
-#         |   /|
-#         |  / |
-#         | /  |
-#         |/   |
-#         6 -- 7
-# 
-#     l1 is the payee, who will later issue some invoices.
-#     It arranges unpublished channels from the above public
-#     network:
-# 
-#         l5->l1: 7 * unit
-#         l6->l1: 5 * unit
-#         l4->l1: 3 * unit
-#         l7->l1: 2 * unit
-# 
-#     l2 and l3 are payers.
-#     They create some unpublished channels to the public network:
-# 
-#         l2->l4, l2->l6: 6 * unit each
-#         l3->l7, l3->l6: 6 * unit each
-# 
-#     Finally, l1 issues 6 * unit invoices, simultaneously, to l2 and l3.
-#     Both of them perform `pay` simultaneously, in order to test if
-#     they interfere with each other.
-# 
-#     This test then tries to check if both of them can pay, given
-#     that there is sufficient incoming capacity, and then some,
-#     to the payee, and the public network is perfectly balanced
-#     with more than sufficient capacity, as well.
-#     '''
-#     opts = {'feerates': (1000, 1000, 1000, 1000)}
-# 
-#     l1, l2, l3, l4, l5, l6, l7 = node_factory.get_nodes(7, opts=opts)
-# 
-#     # Unit
-#     unit = Millisatoshi(11000 * 1000)
-# 
-#     # Build the public network.
-#     public_network = [l4.fundbalancedchannel(l5, unit * 14),
-#                       l4.fundbalancedchannel(l6, unit * 14),
-#                       l5.fundbalancedchannel(l6, unit * 14),
-#                       l5.fundbalancedchannel(l7, unit * 14),
-#                       l6.fundbalancedchannel(l7, unit * 14)]
-# 
-#     # Build unpublished channels to the merchant l1.
-#     l4.rpc.connect(l1.info['id'], 'localhost', l1.port)
-#     l5.rpc.connect(l1.info['id'], 'localhost', l1.port)
-#     l6.rpc.connect(l1.info['id'], 'localhost', l1.port)
-#     l7.rpc.connect(l1.info['id'], 'localhost', l1.port)
-# 
-#     # If we're 'dual-funding', turn off the reciprocal funding
-#     # so that we can fund channels without making them balanced
-#     if EXPERIMENTAL_DUAL_FUND:
-#         for n in [l1, l2, l3, l4, l5, l6, l7]:
-#             n.rpc.call('funderupdate', {'fund_probability': 0})
-# 
-#     # The order in which the routes are built should not matter so
-#     # shuffle them.
-#     incoming_builders = [lambda: l5.fundchannel(l1, int((unit * 7).to_satoshi()), announce_channel=False),
-#                          lambda: l6.fundchannel(l1, int((unit * 5).to_satoshi()), announce_channel=False),
-#                          lambda: l4.fundchannel(l1, int((unit * 3).to_satoshi()), announce_channel=False),
-#                          lambda: l7.fundchannel(l1, int((unit * 2).to_satoshi()), announce_channel=False)]
-#     random.shuffle(incoming_builders)
-#     for b in incoming_builders:
-#         b()
-# 
-#     # Build unpublished channels from the buyers l2 and l3.
-#     l2.rpc.connect(l4.info['id'], 'localhost', l4.port)
-#     l2.rpc.connect(l6.info['id'], 'localhost', l6.port)
-#     l3.rpc.connect(l7.info['id'], 'localhost', l7.port)
-#     l3.rpc.connect(l6.info['id'], 'localhost', l6.port)
-#     l2.fundchannel(l4, int((unit * 6).to_satoshi()), announce_channel=False)
-#     l2.fundchannel(l6, int((unit * 6).to_satoshi()), announce_channel=False)
-#     l3.fundchannel(l7, int((unit * 6).to_satoshi()), announce_channel=False)
-#     l3.fundchannel(l6, int((unit * 6).to_satoshi()), announce_channel=False)
-# 
-#     # Now wait for the buyers to learn the entire public network.
-#     mine_funding_to_announce(bitcoind, [l1, l2, l3, l4, l5, l6, l7])
-#     for channel in public_network:
-#         wait_for(lambda: len(l2.rpc.listchannels(channel)['channels']) == 2)
-#         wait_for(lambda: len(l3.rpc.listchannels(channel)['channels']) == 2)
-# 
-#     # At this point, we have the following incoming channel capacities:
-#     # 74094000, 52314000, 30318000, 19318000
-# 
-#     # We *always* rotate through, since we have no published channels,
-#     # but we can select badly and get an overlap. e.g. first invoice
-#     # takes 30318000, 19318000 and 74094000.  Second will then take
-#     # 52314000, and have to reuse 30318000, which gets exhausted by the
-#     # first payer, thus leaving them unable to pay 66000000.
-# 
-#     # So we re-do this until we only have 4 or fewer routehints.
-#     while True:
-#         # Buyers check out some purchaseable stuff from the merchant.
-#         i2 = l1.rpc.invoice(unit * 6, ''.join(random.choice(string.ascii_letters + string.digits) for _ in range(20)), 'i2')['bolt11']
-#         i3 = l1.rpc.invoice(unit * 6, ''.join(random.choice(string.ascii_letters + string.digits) for _ in range(20)), 'i3')['bolt11']
-#         if len(l1.rpc.decodepay(i2)['routes'] + l1.rpc.decodepay(i3)['routes']) <= 4:
-#             break
-# 
-#     # Pay simultaneously!
-#     p2 = executor.submit(l2.rpc.pay, i2)
-#     p3 = executor.submit(l3.rpc.pay, i3)
-# 
-#     # Both payments should succeed.
-#     p2.result(TIMEOUT)
-#     p3.result(TIMEOUT)
-# 
-# 
-# def test_large_mpp_presplit(node_factory):
-#     """Make sure that ludicrous amounts don't saturate channels
-# 
-#     We aim to have at most PRESPLIT_MAX_SPLITS HTLCs created directly from the
-#     `presplit` modifier. The modifier will scale up its target size to
-#     guarantee this, while still bucketizing payments that are in the following
-#     range:
-# 
-#     ```
-#     target_size = PRESPLIT_MAX_SPLITS^{n} + MPP_TARGET_SIZE
-#     target_size < amount <= target_size * PRESPLIT_MAX_SPLITS
-#     ```
-# 
-#     """
-#     PRESPLIT_MAX_SPLITS = 16
-#     MPP_TARGET_SIZE = 10 ** 7
-#     amt = 400 * MPP_TARGET_SIZE
-# 
-#     l1, l2, l3 = node_factory.line_graph(
-#         3, fundamount=10**8, wait_for_announce=True,
-#         opts={'wumbo': None}
-#     )
-# 
-#     inv = l3.rpc.invoice(amt, 'lbl', 'desc')['bolt11']
-#     p = l1.rpc.pay(inv)
-# 
-#     assert(p['parts'] <= PRESPLIT_MAX_SPLITS)
-#     inv = l3.rpc.listinvoices()['invoices'][0]
-# 
-#     assert(inv['amount_msat'] == inv['amount_received_msat'])
-# 
-# 
-# @pytest.mark.developer("builds large network, which is slow if not DEVELOPER")
-# @pytest.mark.slow_test
-# def test_mpp_overload_payee(node_factory, bitcoind):
-#     """
-#     We had a bug where if the payer is unusually well-connected compared
-#     to the payee, the payee is unable to accept a large payment since the
-#     payer will split it into lots of tiny payments, which would choke the
-#     max-concurrent-htlcs limit of the payee.
-#     """
-#     # Default value as of this writing.
-#     # However, with anchor commitments we might be able to safely lift this
-#     # default limit in the future, so explicitly put this value here, since
-#     # that is what our test assumes.
-#     opts = {'max-concurrent-htlcs': 30}
-# 
-#     l1, l2, l3, l4, l5, l6 = node_factory.get_nodes(6, opts=opts)
-# 
-#     # Respect wumbo.
-#     # Using max-sized channels shows that the issue is not capacity
-#     # but rather max-concurrent-htlcs.
-#     # This is grade-school level.
-#     amt = 2**24 - 1
-# 
-#     # Build the public network.
-#     # l1 is the very well-connected payer.
-#     # l2 is the poorly-connected payee.
-#     # l3->l6 are well-connected hop nodes.
-#     public_network = [l1.fundbalancedchannel(l3, amt),
-#                       l1.fundbalancedchannel(l4, amt),
-#                       l1.fundbalancedchannel(l5, amt),
-#                       l1.fundbalancedchannel(l6, amt),
-#                       l2.fundbalancedchannel(l6, amt),
-#                       l3.fundbalancedchannel(l4, amt),
-#                       l3.fundbalancedchannel(l5, amt),
-#                       l3.fundbalancedchannel(l6, amt),
-#                       l4.fundbalancedchannel(l5, amt),
-#                       l5.fundbalancedchannel(l6, amt)]
-# 
-#     # Ensure l1 knows the entire public network.
-#     mine_funding_to_announce(bitcoind, [l1, l2, l3, l4, l5, l6])
-#     for c in public_network:
-#         wait_for(lambda: len(l1.rpc.listchannels(c)['channels']) >= 2)
-# 
-#     # Now create a 400,000-sat invoice.
-#     # This assumes the MPP presplitter strongly prefers to
-#     # create lot sizes of 10,000 sats each.
-#     # This leads the presplitter to prefer to split into
-#     # around 40 HTLCs of 10,000 sats each, but since
-#     # max-concurrent-htlcs is set to 30, l2 would be unable
-#     # to receive.
-#     inv = l2.rpc.invoice(Millisatoshi(400000 * 1000), 'i', 'i')['bolt11']
-# 
-#     # pay.
-#     l1.rpc.pay(inv)
-# 
-# 
-# @unittest.skipIf(EXPERIMENTAL_FEATURES, "this is always on with EXPERIMENTAL_FEATURES")
-# def test_offer_needs_option(node_factory):
-#     """Make sure we don't make offers without offer command"""
-#     l1 = node_factory.get_node()
-#     with pytest.raises(RpcError, match='experimental-offers not enabled'):
-#         l1.rpc.call('offer', {'amount': '1msat', 'description': 'test'})
-#     with pytest.raises(RpcError, match='experimental-offers not enabled'):
-#         l1.rpc.call('invoicerequest', {'amount': '2msat',
-#                                        'description': 'simple test'})
-#     with pytest.raises(RpcError, match='Unknown command'):
-#         l1.rpc.call('fetchinvoice', {'offer': 'aaaa'})
-# 
-#     # Decode still works though
-#     assert l1.rpc.decode('lno1qgsqvgnwgcg35z6ee2h3yczraddm72xrfua9uve2rlrm9deu7xyfzrcgqyqs5pr5v4ehg93pqfnwgkvdr57yzh6h92zg3qctvrm7w38djg67kzcm4yeg8vc4cq63s')['valid']
-# 
-# 
-# def test_offer(node_factory, bitcoind):
-#     plugin = os.path.join(os.path.dirname(__file__), 'plugins/currencyUSDAUD5000.py')
-#     l1 = node_factory.get_node(options={'plugin': plugin, 'experimental-offers': None})
-# 
-#     # Try empty description
-#     ret = l1.rpc.call('offer', [9, ''])
-#     l1.rpc.decode(ret['bolt12'])
-# 
-#     bolt12tool = os.path.join(os.path.dirname(__file__), "..", "devtools", "bolt12-cli")
-#     # Try different amount strings
-#     for amount in ['1msat', '0.1btc', 'any', '1USD', '1.10AUD']:
-#         ret = l1.rpc.call('offer', {'amount': amount,
-#                                     'description': 'test for ' + amount})
-#         offer = only_one(l1.rpc.call('listoffers', [ret['offer_id']])['offers'])
-# 
-#         assert offer['bolt12'] == ret['bolt12']
-#         assert offer['offer_id'] == ret['offer_id']
-# 
-#         output = subprocess.check_output([bolt12tool, 'decode',
-#                                           offer['bolt12']]).decode('ASCII')
-#         if amount == 'any':
-#             assert 'amount' not in output
-#         else:
-#             assert 'amount' in output
-# 
-#     # Try wrong amount precision:
-#     with pytest.raises(RpcError, match='Currency AUD requires 2 minor units'):
-#         l1.rpc.call('offer', {'amount': '1.100AUD',
-#                               'description': 'test for invalid amount'})
-# 
-#     with pytest.raises(RpcError, match='Currency AUD requires 2 minor units'):
-#         l1.rpc.call('offer', {'amount': '1.1AUD',
-#                               'description': 'test for invalid amount'})
-# 
-#     # Make sure it fails on unknown currencies.
-#     with pytest.raises(RpcError, match='No values available for currency EUR'):
-#         l1.rpc.call('offer', {'amount': '1.00EUR',
-#                               'description': 'test for unknown currency'})
-# 
-#     # Test label and description
-#     weird_label = 'label \\ " \t \n'
-#     weird_desc = 'description \\ " \t \n ナンセンス 1杯'
-#     ret = l1.rpc.call('offer', {'amount': '0.1btc',
-#                                 'description': weird_desc,
-#                                 'label': weird_label})
-#     offer = only_one(l1.rpc.call('listoffers', [ret['offer_id']])['offers'])
-#     assert offer['label'] == weird_label
-# 
-#     output = subprocess.check_output([bolt12tool, 'decode',
-#                                       offer['bolt12']]).decode('UTF-8')
-#     assert 'description: ' + weird_desc in output
-# 
-#     # Test issuer
-#     weird_issuer = 'description \\ " \t \n ナンセンス 1杯'
-#     ret = l1.rpc.call('offer', {'amount': '100000sat',
-#                                 'description': 'issuer test',
-#                                 'issuer': weird_issuer})
-#     offer = only_one(l1.rpc.call('listoffers', [ret['offer_id']])['offers'])
-# 
-#     output = subprocess.check_output([bolt12tool, 'decode',
-#                                       offer['bolt12']]).decode('UTF-8')
-#     assert 'issuer: ' + weird_issuer in output
-# 
-#     # Test quantity
-#     ret = l1.rpc.call('offer', {'amount': '100000sat',
-#                                 'description': 'quantity_max existence test',
-#                                 'quantity_max': 0})
-#     offer = only_one(l1.rpc.call('listoffers', [ret['offer_id']])['offers'])
-#     output = subprocess.check_output([bolt12tool, 'decode',
-#                                       offer['bolt12']]).decode('UTF-8')
-#     assert 'quantity_max: 0' in output
-# 
-#     ret = l1.rpc.call('offer', {'amount': '100000sat',
-#                                 'description': 'quantity_max test',
-#                                 'quantity_max': 2})
-#     offer = only_one(l1.rpc.call('listoffers', [ret['offer_id']])['offers'])
-#     output = subprocess.check_output([bolt12tool, 'decode',
-#                                       offer['bolt12']]).decode('UTF-8')
-#     assert 'quantity_max: 2' in output
-# 
-#     # Test absolute_expiry
-#     exp = int(time.time() + 2)
-#     ret = l1.rpc.call('offer', {'amount': '100000sat',
-#                                 'description': 'quantity_max test',
-#                                 'absolute_expiry': exp})
-#     offer = only_one(l1.rpc.call('listoffers', [ret['offer_id']])['offers'])
-#     output = subprocess.check_output([bolt12tool, 'decode',
-#                                       offer['bolt12']]).decode('UTF-8')
-#     assert 'absolute_expiry: {}'.format(exp) in output
-# 
-#     # Recurrence tests!
-#     for r in [['1second', 'seconds', 1],
-#               ['10seconds', 'seconds', 10],
-#               ['1minute', 'seconds', 60],
-#               ['10minutes', 'seconds', 600],
-#               ['1hour', 'seconds', 3600],
-#               ['10hours', 'seconds', 36000],
-#               ['1day', 'days', 1],
-#               ['10days', 'days', 10],
-#               ['1week', 'days', 7],
-#               ['10weeks', 'days', 70],
-#               ['1month', 'months', 1],
-#               ['10months', 'months', 10],
-#               ['1year', 'years', 1],
-#               ['10years', 'years', 10]]:
-#         ret = l1.rpc.call('offer', {'amount': '100000sat',
-#                                     'description': 'quantity_max test',
-#                                     'recurrence': r[0]})
-#         offer = only_one(l1.rpc.call('listoffers', [ret['offer_id']])['offers'])
-#         output = subprocess.check_output([bolt12tool, 'decode',
-#                                           offer['bolt12']]).decode('UTF-8')
-#         assert 'recurrence: every {} {}\n'.format(r[2], r[1]) in output
-# 
-#     # Test limit
-#     ret = l1.rpc.call('offer', {'amount': '100000sat',
-#                                 'description': 'quantity_max test',
-#                                 'recurrence': '10minutes',
-#                                 'recurrence_limit': 5})
-#     offer = only_one(l1.rpc.call('listoffers', [ret['offer_id']])['offers'])
-#     output = subprocess.check_output([bolt12tool, 'decode',
-#                                       offer['bolt12']]).decode('UTF-8')
-#     assert 'recurrence: every 600 seconds limit 5\n' in output
-# 
-#     # Test base
-#     # (1456740000 == 10:00:00 (am) UTC on 29 February, 2016)
-#     ret = l1.rpc.call('offer', {'amount': '100000sat',
-#                                 'description': 'quantity_max test',
-#                                 'recurrence': '10minutes',
-#                                 'recurrence_base': '@1456740000'})
-#     offer = only_one(l1.rpc.call('listoffers', [ret['offer_id']])['offers'])
-#     output = subprocess.check_output([bolt12tool, 'decode',
-#                                       offer['bolt12']]).decode('UTF-8')
-#     assert 'recurrence: every 600 seconds start 1456740000' in output
-#     assert '(can start any period)' not in output
-# 
-#     ret = l1.rpc.call('offer', {'amount': '100000sat',
-#                                 'description': 'quantity_max test',
-#                                 'recurrence': '10minutes',
-#                                 'recurrence_base': 1456740000})
-#     offer = only_one(l1.rpc.call('listoffers', [ret['offer_id']])['offers'])
-#     output = subprocess.check_output([bolt12tool, 'decode',
-#                                       offer['bolt12']]).decode('UTF-8')
-#     assert 'recurrence: every 600 seconds start 1456740000' in output
-#     assert '(can start any period)' in output
-# 
-#     # Test paywindow
-#     ret = l1.rpc.call('offer', {'amount': '100000sat',
-#                                 'description': 'quantity_max test',
-#                                 'recurrence': '10minutes',
-#                                 'recurrence_paywindow': '-10+20'})
-#     offer = only_one(l1.rpc.call('listoffers', [ret['offer_id']])['offers'])
-#     output = subprocess.check_output([bolt12tool, 'decode',
-#                                       offer['bolt12']]).decode('UTF-8')
-#     assert 'recurrence: every 600 seconds paywindow -10 to +20\n' in output
-# 
-#     ret = l1.rpc.call('offer', {'amount': '100000sat',
-#                                 'description': 'quantity_max test',
-#                                 'recurrence': '10minutes',
-#                                 'recurrence_paywindow': '-10+600%'})
-#     offer = only_one(l1.rpc.call('listoffers', [ret['offer_id']])['offers'])
-#     output = subprocess.check_output([bolt12tool, 'decode',
-#                                       offer['bolt12']]).decode('UTF-8')
-#     assert 'recurrence: every 600 seconds paywindow -10 to +600 (pay proportional)\n' in output
-# 
-# 
-# def test_offer_deprecated_api(node_factory, bitcoind):
-#     l1, l2 = node_factory.line_graph(2, opts={'experimental-offers': None,
-#                                               'allow-deprecated-apis': True})
-# 
-#     offer = l2.rpc.call('offer', {'amount': '2msat',
-#                                   'description': 'test_offer_deprecated_api'})
-#     inv = l1.rpc.call('fetchinvoice', {'offer': offer['bolt12']})
-# 
-#     # Deprecated fields make schema checker upset.
-#     l1.rpc.jsonschemas = {}
-#     l1.rpc.pay(inv['invoice'])
-# 
-# 
-# @pytest.mark.developer("dev-no-modern-onion is DEVELOPER-only")
-# def test_fetchinvoice_3hop(node_factory, bitcoind):
-#     l1, l2, l3, l4 = node_factory.line_graph(4, wait_for_announce=True,
-#                                              opts={'experimental-offers': None,
-#                                                    'may_reconnect': True,
-#                                                    'dev-no-reconnect': None})
-#     offer1 = l4.rpc.call('offer', {'amount': '2msat',
-#                                    'description': 'simple test'})
-#     assert offer1['created'] is True
-# 
-#     l1.rpc.call('fetchinvoice', {'offer': offer1['bolt12']})
-# 
-# 
-# def test_fetchinvoice(node_factory, bitcoind):
-#     # We remove the conversion plugin on l3, causing it to get upset.
-#     l1, l2, l3 = node_factory.line_graph(3, wait_for_announce=True,
-#                                          opts=[{'experimental-offers': None},
-#                                                {'experimental-offers': None},
-#                                                {'experimental-offers': None,
-#                                                 'allow_broken_log': True}])
-# 
-#     # Simple offer first.
-#     offer1 = l3.rpc.call('offer', {'amount': '2msat',
-#                                    'description': 'simple test'})
-#     assert offer1['created'] is True
-# 
-#     inv1 = l1.rpc.call('fetchinvoice', {'offer': offer1['bolt12']})
-#     inv2 = l1.rpc.call('fetchinvoice', {'offer': offer1['bolt12'],
-#                                         'payer_note': 'Thanks for the fish!'})
-#     assert inv1 != inv2
-#     assert 'next_period' not in inv1
-#     assert 'next_period' not in inv2
-#     assert only_one(l3.rpc.call('listoffers', [offer1['offer_id']])['offers'])['used'] is False
-#     l1.rpc.pay(inv1['invoice'])
-#     assert only_one(l3.rpc.call('listoffers', [offer1['offer_id']])['offers'])['used'] is True
-#     l1.rpc.pay(inv2['invoice'])
-#     assert only_one(l3.rpc.call('listoffers', [offer1['offer_id']])['offers'])['used'] is True
-# 
-#     # listinvoices will show these on l3
-#     assert [x['local_offer_id'] for x in l3.rpc.listinvoices()['invoices']] == [offer1['offer_id'], offer1['offer_id']]
-# 
-#     assert 'invreq_payer_note' not in only_one(l3.rpc.call('listinvoices', {'invstring': inv1['invoice']})['invoices'])
-#     assert only_one(l3.rpc.call('listinvoices', {'invstring': inv2['invoice']})['invoices'])['invreq_payer_note'] == 'Thanks for the fish!'
-# 
-#     # BTW, test listinvoices-by-offer_id:
-#     assert len(l3.rpc.listinvoices(offer_id=offer1['offer_id'])['invoices']) == 2
-# 
-#     # We can also set the amount explicitly, to tip.
-#     inv1 = l1.rpc.call('fetchinvoice', {'offer': offer1['bolt12'], 'amount_msat': 3})
-#     assert l1.rpc.call('decode', [inv1['invoice']])['invoice_amount_msat'] == 3
-#     l1.rpc.pay(inv1['invoice'])
-# 
-#     # More than ~5x expected is rejected as absurd (it's actually a divide test,
-#     # which means we need 15 here, not 11).
-#     with pytest.raises(RpcError, match="Remote node sent failure message.*Amount vastly exceeds 2msat"):
-#         l1.rpc.call('fetchinvoice', {'offer': offer1['bolt12'], 'amount_msat': 15})
-# 
-#     # Underpay is rejected.
-#     with pytest.raises(RpcError, match="Remote node sent failure message.*Amount must be at least 2msat"):
-#         l1.rpc.call('fetchinvoice', {'offer': offer1['bolt12'], 'amount_msat': 1})
-# 
-#     # If no amount is specified in offer, one must be in invoice.
-#     offer_noamount = l3.rpc.call('offer', {'amount': 'any',
-#                                            'description': 'any amount test'})
-#     with pytest.raises(RpcError, match="msatoshi parameter required"):
-#         l1.rpc.call('fetchinvoice', {'offer': offer_noamount['bolt12']})
-#     inv1 = l1.rpc.call('fetchinvoice', {'offer': offer_noamount['bolt12'], 'amount_msat': 100})
-#     # But amount won't appear in changes
-#     assert 'msat' not in inv1['changes']
-# 
-#     # Single-use invoice can be fetched multiple times, only paid once.
-#     offer2 = l3.rpc.call('offer', {'amount': '1msat',
-#                                    'description': 'single-use test',
-#                                    'single_use': True})['bolt12']
-# 
-#     inv1 = l1.rpc.call('fetchinvoice', {'offer': offer2})
-#     inv2 = l1.rpc.call('fetchinvoice', {'offer': offer2})
-#     assert inv1 != inv2
-#     assert 'next_period' not in inv1
-#     assert 'next_period' not in inv2
-# 
-#     l1.rpc.pay(inv1['invoice'])
-# 
-#     # We can't pay the other one now.
-#     # FIXME: Even dummy blinded paths always return WIRE_INVALID_ONION_BLINDING!
-#     with pytest.raises(RpcError, match="INVALID_ONION_BLINDING.*'erring_node': '{}'".format(l3.info['id'])):
-#         l1.rpc.pay(inv2['invoice'])
-# 
-#     # We can't reuse the offer, either.
-#     with pytest.raises(RpcError, match='Offer no longer available'):
-#         l1.rpc.call('fetchinvoice', {'offer': offer2})
-# 
-#     # Recurring offer.
-#     offer3 = l2.rpc.call('offer', {'amount': '1msat',
-#                                    'description': 'recurring test',
-#                                    'recurrence': '1minutes'})
-#     assert only_one(l2.rpc.call('listoffers', [offer3['offer_id']])['offers'])['used'] is False
-# 
-#     ret = l1.rpc.call('fetchinvoice', {'offer': offer3['bolt12'],
-#                                        'recurrence_counter': 0,
-#                                        'recurrence_label': 'test recurrence'})
-#     period1 = ret['next_period']
-#     assert period1['counter'] == 1
-#     assert period1['endtime'] == period1['starttime'] + 59
-#     assert period1['paywindow_start'] == period1['starttime'] - 60
-#     assert period1['paywindow_end'] == period1['endtime']
-#     assert only_one(l2.rpc.call('listoffers', [offer3['offer_id']])['offers'])['used'] is False
-# 
-#     l1.rpc.pay(ret['invoice'], label='test recurrence')
-#     assert only_one(l2.rpc.call('listoffers', [offer3['offer_id']])['offers'])['used'] is True
-# 
-#     ret = l1.rpc.call('fetchinvoice', {'offer': offer3['bolt12'],
-#                                        'recurrence_counter': 1,
-#                                        'recurrence_label': 'test recurrence'})
-#     period2 = ret['next_period']
-#     assert period2['counter'] == 2
-#     assert period2['starttime'] == period1['endtime'] + 1
-#     assert period2['endtime'] == period2['starttime'] + 59
-#     assert period2['paywindow_start'] == period2['starttime'] - 60
-#     assert period2['paywindow_end'] == period2['endtime']
-# 
-#     # Can't request 2 before paying 1.
-#     with pytest.raises(RpcError, match='previous invoice has not been paid'):
-#         l1.rpc.call('fetchinvoice', {'offer': offer3['bolt12'],
-#                                      'recurrence_counter': 2,
-#                                      'recurrence_label': 'test recurrence'})
-# 
-#     l1.rpc.pay(ret['invoice'], label='test recurrence')
-# 
-#     # Now we can, but it's too early:
-#     with pytest.raises(RpcError, match="Too early: can't send until time {}".format(period1['starttime'])):
-#         l1.rpc.call('fetchinvoice', {'offer': offer3['bolt12'],
-#                                      'recurrence_counter': 2,
-#                                      'recurrence_label': 'test recurrence'})
-# 
-#     # Wait until the correct moment.
-#     while time.time() < period1['starttime']:
-#         time.sleep(1)
-# 
-#     l1.rpc.call('fetchinvoice', {'offer': offer3['bolt12'],
-#                                  'recurrence_counter': 2,
-#                                  'recurrence_label': 'test recurrence'})
-# 
-#     # Check we can request invoice without a channel.
-#     l4 = node_factory.get_node(options={'experimental-offers': None})
-#     l4.rpc.connect(l2.info['id'], 'localhost', l2.port)
-#     # ... even if we can't find ourselves.
-#     l4.rpc.call('fetchinvoice', {'offer': offer3['bolt12'],
-#                                  'recurrence_counter': 0,
-#                                  'recurrence_label': 'test nochannel'})
-#     # ... even if we know it from gossmap
-#     wait_for(lambda: l4.rpc.listnodes(l3.info['id'])['nodes'] != [])
-#     l4.rpc.connect(l3.info['id'], 'localhost', l3.port)
-#     l4.rpc.call('fetchinvoice', {'offer': offer1['bolt12']})
-# 
-#     # Now, test amount in different currency!
-#     plugin = os.path.join(os.path.dirname(__file__), 'plugins/currencyUSDAUD5000.py')
-#     l3.rpc.plugin_start(plugin)
-# 
-#     offerusd = l3.rpc.call('offer', {'amount': '10.05USD',
-#                                      'description': 'USD test'})['bolt12']
-# 
-#     inv = l1.rpc.call('fetchinvoice', {'offer': offerusd})
-#     assert inv['changes']['amount_msat'] == Millisatoshi(int(10.05 * 5000))
-# 
-#     # If we remove plugin, it can no longer give us an invoice.
-#     l3.rpc.plugin_stop(plugin)
-# 
-#     with pytest.raises(RpcError, match="Internal error"):
-#         l1.rpc.call('fetchinvoice', {'offer': offerusd})
-#     l3.daemon.wait_for_log("Unknown command 'currencyconvert'")
-#     # But we can still pay the (already-converted) invoice.
-#     l1.rpc.pay(inv['invoice'])
-# 
-#     # Identical creation gives it again, just with created false.
-#     offer1 = l3.rpc.call('offer', {'amount': '2msat',
-#                                    'description': 'simple test'})
-#     assert offer1['created'] is False
-#     l3.rpc.call('disableoffer', {'offer_id': offer1['offer_id']})
-#     with pytest.raises(RpcError, match="1000.*Already exists, but isn't active"):
-#         l3.rpc.call('offer', {'amount': '2msat',
-#                               'description': 'simple test'})
-# 
-#     # Test timeout.
-#     l3.stop()
-#     with pytest.raises(RpcError, match='Timeout waiting for response'):
-#         l1.rpc.call('fetchinvoice', {'offer': offer1['bolt12'], 'timeout': 10})
-# 
-#     # Now try an offer with a more complex paywindow (only 10 seconds before)
-#     offer = l2.rpc.call('offer', {'amount': '1msat',
-#                                   'description': 'paywindow test',
-#                                   'recurrence': '20seconds',
-#                                   'recurrence_paywindow': '-10+0'})['bolt12']
-# 
-#     ret = l1.rpc.call('fetchinvoice', {'offer': offer,
-#                                        'recurrence_counter': 0,
-#                                        'recurrence_label': 'test paywindow'})
-#     period3 = ret['next_period']
-#     assert period3['counter'] == 1
-#     assert period3['endtime'] == period3['starttime'] + 19
-#     assert period3['paywindow_start'] == period3['starttime'] - 10
-#     assert period3['paywindow_end'] == period3['starttime']
-#     l1.rpc.pay(ret['invoice'], label='test paywindow')
-# 
-#     # We can get another invoice, as many times as we want.
-#     # (It may return the same one!).
-#     while int(time.time()) <= period3['paywindow_start']:
-#         time.sleep(1)
-# 
-#     l1.rpc.call('fetchinvoice', {'offer': offer,
-#                                  'recurrence_counter': 1,
-#                                  'recurrence_label': 'test paywindow'})
-#     l1.rpc.call('fetchinvoice', {'offer': offer,
-#                                  'recurrence_counter': 1,
-#                                  'recurrence_label': 'test paywindow'})
-# 
-#     # Wait until too late!
-#     while int(time.time()) <= period3['paywindow_end']:
-#         time.sleep(1)
-# 
-#     with pytest.raises(RpcError, match="Too late: expired time {}".format(period3['paywindow_end'])):
-#         l1.rpc.call('fetchinvoice', {'offer': offer,
-#                                      'recurrence_counter': 1,
-#                                      'recurrence_label': 'test paywindow'})
-# 
-# 
-# @pytest.mark.developer("Needs dev-allow-localhost for autoconnect, dev-force-features to avoid routing onionmsgs")
-# def test_fetchinvoice_autoconnect(node_factory, bitcoind):
-#     """We should autoconnect if we need to, to route."""
-# 
-#     if EXPERIMENTAL_FEATURES:
-#         # We have to force option_onion_messages off!
-#         opts1 = {'dev-force-features': '-39'}
-#     else:
-#         opts1 = {}
-#     l1, l2 = node_factory.line_graph(2, wait_for_announce=True,
-#                                      opts=[opts1,
-#                                            {'experimental-offers': None,
-#                                             'dev-allow-localhost': None}])
-# 
-#     l3 = node_factory.get_node(options={'experimental-offers': None})
-#     l3.rpc.connect(l1.info['id'], 'localhost', l1.port)
-#     wait_for(lambda: l3.rpc.listnodes(l2.info['id'])['nodes'] != [])
-# 
-#     offer = l2.rpc.call('offer', {'amount': '2msat',
-#                                   'description': 'simple test'})
-#     l3.rpc.call('fetchinvoice', {'offer': offer['bolt12']})
-#     assert l3.rpc.listpeers(l2.info['id'])['peers'] != []
-# 
-#     # Similarly for an invoice_request.
-#     l3.rpc.disconnect(l2.info['id'])
-#     invreq = l2.rpc.call('invoicerequest', {'amount': '2msat',
-#                                             'description': 'simple test'})
-#     # Ofc l2 can't actually pay it!
-#     with pytest.raises(RpcError, match='pay attempt failed: "Ran out of routes to try'):
-#         l3.rpc.call('sendinvoice', {'invreq': invreq['bolt12'], 'label': 'payme!'})
-# 
-#     assert l3.rpc.listpeers(l2.info['id'])['peers'] != []
-# 
-#     # But if we create a channel l3->l1->l2 (and balance!), l2 can!
-#     node_factory.join_nodes([l3, l1], wait_for_announce=True)
-#     # Make sure l2 knows about it
-#     wait_for(lambda: l2.rpc.listnodes(l3.info['id'])['nodes'] != [])
-# 
-#     l3.rpc.pay(l2.rpc.invoice(FUNDAMOUNT * 500, 'balancer', 'balancer')['bolt11'])
-#     # Make sure l2 has capacity (can be still resolving!).
-#     wait_for(lambda: only_one(l2.rpc.listpeerchannels(l1.info['id'])['channels'])['spendable_msat'] != Millisatoshi(0))
-# 
-#     l3.rpc.disconnect(l2.info['id'])
-#     l3.rpc.call('sendinvoice', {'invreq': invreq['bolt12'], 'label': 'payme for real!'})
-#     # It will have autoconnected, to send invoice (since l1 says it doesn't do onion messages!)
-#     assert l3.rpc.listpeers(l2.info['id'])['peers'] != []
-# 
-# 
-# def test_pay_waitblockheight_timeout(node_factory, bitcoind):
-#     plugin = os.path.join(os.path.dirname(__file__), 'plugins', 'endlesswaitblockheight.py')
-#     l1, l2 = node_factory.line_graph(2, opts=[{}, {'plugin': plugin}])
-# 
-#     sync_blockheight(bitcoind, [l1, l2])
-#     inv = l2.rpc.invoice(42, 'lbl', 'desc')['bolt11']
-# 
-#     with pytest.raises(RpcError, match=r'WIRE_INCORRECT_OR_UNKNOWN_PAYMENT_DETAILS'):
-#         l1.rpc.pay(inv)
-# 
-#     # Post mortem checks that we tried only once.
-#     status = l1.rpc.paystatus(inv)
-# 
-#     # Should have only one attempt that triggered the wait, which then failed.
-#     assert len(status['pay']) == 1
-#     assert len(status['pay'][0]['attempts']) == 1
-# 
-# 
-# @pytest.mark.developer("dev-rawrequest is DEVELOPER-only")
-# def test_dev_rawrequest(node_factory):
-#     l1, l2 = node_factory.line_graph(2, fundchannel=False,
-#                                      opts={'experimental-offers': None})
-# 
-#     offer = l2.rpc.call('offer', {'amount': '2msat',
-#                                   'description': 'simple test'})
-#     # Get fetchinvoice to make us an invoice_request
-#     l1.rpc.call('fetchinvoice', {'offer': offer['bolt12']})
-# 
-#     m = re.search(r'invoice_request: \\"([a-z0-9]*)\\"', l1.daemon.is_in_log('invoice_request:'))
-#     ret = l1.rpc.call('dev-rawrequest', {'invreq': m.group(1),
-#                                          'nodeid': l2.info['id'],
-#                                          'timeout': 10})
-#     assert 'invoice' in ret
-# 
-# 
-# def test_sendinvoice(node_factory, bitcoind):
-#     l2opts = {'experimental-offers': None}
-#     l1, l2 = node_factory.line_graph(2, wait_for_announce=True,
-#                                      opts=[{'experimental-offers': None},
-#                                            l2opts])
-# 
-#     # Simple offer to send money (balances channel a little)
-#     invreq = l1.rpc.call('invoicerequest', {'amount': '100000sat',
-#                                             'description': 'simple test'})
-# 
-#     # Fetchinvoice will refuse, since it's not an offer.
-#     with pytest.raises(RpcError, match='unexpected prefix lnr'):
-#         l2.rpc.call('fetchinvoice', {'offer': invreq['bolt12']})
-# 
-#     # Pay will refuse, since it's not an invoice.
-#     with pytest.raises(RpcError, match='unexpected prefix lnr'):
-#         l2.rpc.call('fetchinvoice', {'offer': invreq['bolt12']})
-# 
-#     # used will be false
-#     assert only_one(l1.rpc.call('listinvoicerequests', [invreq['invreq_id']])['invoicerequests'])['used'] is False
-# 
-#     # sendinvoice should work.
-#     out = l2.rpc.call('sendinvoice', {'invreq': invreq['bolt12'],
-#                                       'label': 'test sendinvoice 1'})
-#     assert out['label'] == 'test sendinvoice 1'
-#     assert out['description'] == 'simple test'
-#     assert 'bolt12' in out
-#     assert 'payment_hash' in out
-#     assert out['status'] == 'paid'
-#     assert 'payment_preimage' in out
-#     assert 'expires_at' in out
-#     assert out['amount_msat'] == Millisatoshi(100000000)
-#     assert 'pay_index' in out
-#     assert out['amount_received_msat'] == Millisatoshi(100000000)
-# 
-#     # Note, if we're slow, this fails with "Offer no longer available",
-#     # *but* if it hasn't heard about payment success yet, l2 will fail
-#     # simply because payments are already pending.
-#     with pytest.raises(RpcError, match='no longer available|pay attempt failed'):
-#         l2.rpc.call('sendinvoice', {'invreq': invreq['bolt12'],
-#                                     'label': 'test sendinvoice 2'})
-# 
-#     # Technically, l1 may not have gotten payment success, so we need to wait.
-#     wait_for(lambda: only_one(l1.rpc.call('listinvoicerequests', [invreq['invreq_id']])['invoicerequests'])['used'] is True)
-# 
-#     # Offer with issuer: we must copy issuer into our invoice!
-#     invreq = l1.rpc.call('invoicerequest', {'amount': '10000sat',
-#                                             'description': 'simple test',
-#                                             'issuer': "clightning test suite"})
-# 
-#     out = l2.rpc.call('sendinvoice', {'invreq': invreq['bolt12'],
-#                                       'label': 'test sendinvoice 3'})
-#     assert out['label'] == 'test sendinvoice 3'
-#     assert out['description'] == 'simple test'
-#     assert 'issuer' not in out
-#     assert 'bolt12' in out
-#     assert 'payment_hash' in out
-#     assert out['status'] == 'paid'
-#     assert 'payment_preimage' in out
-#     assert 'expires_at' in out
-#     assert out['amount_msat'] == Millisatoshi(10000000)
-#     assert 'pay_index' in out
-#     assert out['amount_received_msat'] == Millisatoshi(10000000)
-# 
-# 
-# def test_self_pay(node_factory):
-#     """Repro test for issue 4345: pay ourselves via the pay plugin.
-# 
-#     """
-#     l1, l2 = node_factory.line_graph(2, wait_for_announce=True)
-# 
-#     inv = l1.rpc.invoice(10000, 'test', 'test')['bolt11']
-# 
-#     with pytest.raises(RpcError):
-#         l1.rpc.pay(inv)
-# 
-# 
-# @unittest.skipIf(TEST_NETWORK != 'regtest', "Canned invoice is network specific")
-# def test_unreachable_routehint(node_factory, bitcoind):
-#     """Test that we discard routehints that we can't reach.
-# 
-#     Reachability is tested by checking whether we can reach the
-#     entrypoint of the routehint, i.e., the first node in the
-#     routehint. The network we create is partitioned on purpose for
-#     this: first we attempt with an unknown destination and an unknown
-#     routehint entrypoint, later we make them known, but still
-#     unreachable, by connecting them without a channel.
-# 
-#     """
-# 
-#     # Create a partitioned network, first without connecting it, then
-#     # connecting it without a channel so they can sync gossip. Notice
-#     # that l4 is there only to trick the deadend heuristic.
-#     l1, l2 = node_factory.line_graph(2, wait_for_announce=True)
-#     l3, l4, l5 = node_factory.line_graph(3, wait_for_announce=True)
-#     entrypoint = '0382ce59ebf18be7d84677c2e35f23294b9992ceca95491fcf8a56c6cb2d9de199'
-# 
-#     # Generate an invoice with exactly one routehint.
-#     for i in range(100):
-#         invoice = l5.rpc.invoice(10, 'attempt{}'.format(i), 'description')['bolt11']
-#         decoded = l1.rpc.decodepay(invoice)
-#         if 'routes' in decoded and len(decoded['routes']) == 1:
-#             break
-# 
-#     assert('routes' in decoded and len(decoded['routes']) == 1)
-# 
-#     with pytest.raises(RpcError, match=r'Destination [a-f0-9]{66} is not reachable'):
-#         l1.rpc.pay(invoice)
-# 
-#     l1.daemon.wait_for_log(
-#         r"Removed routehint 0 because entrypoint {entrypoint} is unknown.".format(
-#             entrypoint=entrypoint
-#         )
-#     )
-# 
-#     # Now connect l2 to l3 to create a bridge, but without a
-#     # channel. The entrypoint will become known, but still
-#     # unreachable, resulting in a slightly different error message,
-#     # but the routehint will still be removed.
-#     l2.connect(l3)
-#     wait_for(lambda: len(l1.rpc.listnodes(entrypoint)['nodes']) == 1)
-# 
-#     with pytest.raises(RpcError, match=r'Destination [a-f0-9]{66} is not reachable') as excinfo:
-#         l1.rpc.pay(invoice)
-# 
-#     # Verify that we failed for the correct reason.
-#     l1.daemon.wait_for_log(
-#         r"Removed routehint 0 because entrypoint {entrypoint} is unreachable.".format(
-#             entrypoint=entrypoint
-#         )
-#     )
-# 
-#     # Since we aborted once we realized the destination is unreachable
-#     # both directly, and via the routehints we should now just have a
-#     # single attempt.
-#     assert(len(excinfo.value.error['attempts']) == 1)
-# 
-# 
-# def test_routehint_tous(node_factory, bitcoind):
-#     """
-# Test bug where trying to pay an invoice from an *offline* node which
-# gives a routehint straight to us causes an issue
-# """
-# 
-#     # Existence of l1 makes l3 use l2 for routehint (otherwise it sees deadend)
-#     l1, l2 = node_factory.line_graph(2, wait_for_announce=True)
-#     l3 = node_factory.get_node()
-#     l3.rpc.connect(l2.info['id'], 'localhost', l2.port)
-#     scid23, _ = l2.fundchannel(l3, 1000000, announce_channel=False)
-#     # Make sure l3 sees l1->l2 channel.
-#     wait_for(lambda: l3.rpc.listnodes(l1.info['id'])['nodes'] != [])
-# 
-#     inv = l3.rpc.invoice(10, "test", "test")['bolt11']
-#     decoded = l3.rpc.decodepay(inv)
-#     assert(only_one(only_one(decoded['routes']))['short_channel_id']
-#            == only_one(l3.rpc.listpeerchannels()['channels'])['alias']['remote'])
-# 
-#     l3.stop()
-#     with pytest.raises(RpcError, match=r'Destination .* is not reachable directly and all routehints were unusable'):
-#         l2.rpc.pay(inv)
-# 
-# 
-# def test_pay_low_max_htlcs(node_factory):
-#     """Test we can pay if *any* HTLC slots are available"""
-# 
-#     l1, l2, l3 = node_factory.line_graph(3,
-#                                          opts={'max-concurrent-htlcs': 1},
-#                                          wait_for_announce=True)
-#     l1.rpc.pay(l3.rpc.invoice(FUNDAMOUNT * 50, "test", "test")['bolt11'])
-#     l1.daemon.wait_for_log(
-#         r'Number of pre-split HTLCs \([0-9]+\) exceeds our HTLC budget \([0-9]+\), skipping pre-splitter'
-#     )
-# 
-# 
-# def test_setchannel_enforcement_delay(node_factory, bitcoind):
-#     # Fees start at 1msat + 1%
-#     l1, l2, l3 = node_factory.line_graph(3, wait_for_announce=True,
-#                                          opts={'fee-base': 1,
-#                                                'fee-per-satoshi': 10000})
-# 
-#     chanid1 = only_one(l1.rpc.listpeerchannels(l2.info['id'])['channels'])['short_channel_id']
-#     chanid2 = only_one(l2.rpc.listpeerchannels(l3.info['id'])['channels'])['short_channel_id']
-# 
-#     route = [{'amount_msat': 1011,
-#               'id': l2.info['id'],
-#               'delay': 20,
-#               'channel': chanid1},
-#              {'amount_msat': 1000,
-#               'id': l3.info['id'],
-#               'delay': 10,
-#               'channel': chanid2}]
-# 
-#     # This works.
-#     inv = l3.rpc.invoice(1000, "test1", "test1")
-#     l1.rpc.sendpay(route,
-#                    payment_hash=inv['payment_hash'],
-#                    payment_secret=inv['payment_secret'])
-#     l1.rpc.waitsendpay(inv['payment_hash'])
-# 
-#     # Increase fee immediately; l1 payment rejected.
-#     l2.rpc.setchannel("all", 2, 10000, enforcedelay=0)
-# 
-#     inv = l3.rpc.invoice(1000, "test2", "test2")
-#     l1.rpc.sendpay(route,
-#                    payment_hash=inv['payment_hash'],
-#                    payment_secret=inv['payment_secret'])
-#     with pytest.raises(RpcError, match=r'WIRE_FEE_INSUFFICIENT'):
-#         l1.rpc.waitsendpay(inv['payment_hash'])
-# 
-#     # Test increased amount.
-#     route[0]['amount_msat'] += 1
-#     inv = l3.rpc.invoice(1000, "test3", "test3")
-#     l1.rpc.sendpay(route,
-#                    payment_hash=inv['payment_hash'],
-#                    payment_secret=inv['payment_secret'])
-#     l1.rpc.waitsendpay(inv['payment_hash'])
-# 
-#     # Now, give us 30 seconds please.
-#     l2.rpc.setchannel("all", 3, 10000, enforcedelay=30)
-#     inv = l3.rpc.invoice(1000, "test4", "test4")
-#     l1.rpc.sendpay(route,
-#                    payment_hash=inv['payment_hash'],
-#                    payment_secret=inv['payment_secret'])
-#     l1.rpc.waitsendpay(inv['payment_hash'])
-#     l2.daemon.wait_for_log("Allowing payment using older feerate")
-# 
-#     time.sleep(30)
-#     inv = l3.rpc.invoice(1000, "test5", "test5")
-#     l1.rpc.sendpay(route,
-#                    payment_hash=inv['payment_hash'],
-#                    payment_secret=inv['payment_secret'])
-#     with pytest.raises(RpcError, match=r'WIRE_FEE_INSUFFICIENT'):
-#         l1.rpc.waitsendpay(inv['payment_hash'])
-# 
-# 
-# def test_listpays_with_filter_by_status(node_factory, bitcoind):
-#     """
-#     This test check if the filtering by status of the command listpays
-#     has some mistakes.
-#     """
-# 
-#     # Create the line graph l2 -> l1 with a channel of 10 ** 5 sat!
-#     l2, l1 = node_factory.line_graph(2, fundamount=10**5, wait_for_announce=True)
-# 
-#     inv = l1.rpc.invoice(10 ** 5, 'inv', 'inv')
-#     l2.rpc.pay(inv['bolt11'])
-# 
-#     wait_for(lambda: l2.rpc.listpays(inv['bolt11'])['pays'][0]['status'] == 'complete')
-# 
-#     # test if the node is still ready
-#     payments = l2.rpc.listpays(status='failed')
-# 
-#     assert len(payments['pays']) == 0
-# 
-#     payments = l2.rpc.listpays()
-#     assert len(l2.rpc.listpays()['pays']) == 1
-# 
-# 
-# def test_sendpay_grouping(node_factory, bitcoind):
-#     """Paying an invoice multiple times, listpays should list them individually
-#     """
-#     l1, l2, l3 = node_factory.line_graph(
-#         3,
-#         wait_for_announce=True,
-#         opts=[
-#             {},
-#             {'may_reconnect': True},
-#             {'may_reconnect': True},
-#         ],
-#     )
-#     wait_for(lambda: len(l1.rpc.listnodes()['nodes']) == 3)
-# 
-#     inv = l3.rpc.invoice(amount_msat='any', label='lbl1', description='desc')['bolt11']
-#     l3.stop()  # Stop recipient so the first attempt fails
-# 
-#     assert(len(l1.db.query("SELECT * FROM payments")) == 0)
-#     assert(len(l1.rpc.listpays()['pays']) == 0)
-# 
-#     with pytest.raises(RpcError, match=r'Ran out of routes to try after [0-9]+ attempts'):
-#         l1.rpc.pay(inv, amount_msat='100000msat')
-# 
-#     # After this one invocation we have one entry in `listpays`
-#     assert(len(l1.rpc.listpays()['pays']) == 1)
-# 
-#     with pytest.raises(RpcError, match=r'Ran out of routes to try after [0-9]+ attempts'):
-#         l1.rpc.pay(inv, amount_msat='200000msat')
-# 
-#     # Surprise: we should have 2 entries after 2 invocations
-#     assert(len(l1.rpc.listpays()['pays']) == 2)
-# 
-#     l3.start()
-#     invoices = l3.rpc.listinvoices()['invoices']
-#     assert(len(invoices) == 1)
-#     assert(invoices[0]['status'] == 'unpaid')
-#     # Will reconnect automatically
-#     wait_for(lambda: only_one(l3.rpc.listpeers()['peers'])['connected'] is True)
-#     scid = l3.rpc.listpeerchannels()['channels'][0]['short_channel_id']
-#     wait_for(lambda: [c['active'] for c in l1.rpc.listchannels(scid)['channels']] == [True, True])
-#     l1.rpc.pay(inv, amount_msat='420000msat')
-# 
-#     # And finally we should have all 3 attempts to pay the invoice
-#     pays = l1.rpc.listpays()['pays']
-#     assert(len(pays) == 3)
-#     assert([p['status'] for p in pays] == ['failed', 'failed', 'complete'])
-# 
-# 
-# def test_pay_manual_exclude(node_factory, bitcoind):
-#     l1, l2, l3 = node_factory.line_graph(3, wait_for_announce=True)
-#     l1_id = l1.rpc.getinfo()['id']
-#     l2_id = l2.rpc.getinfo()['id']
-#     l3_id = l3.rpc.getinfo()['id']
-#     chan12 = l1.rpc.listpeerchannels(l2_id)['channels'][0]
-#     chan23 = l2.rpc.listpeerchannels(l3_id)['channels'][0]
-#     scid12 = chan12['short_channel_id'] + '/' + str(chan12['direction'])
-#     scid23 = chan23['short_channel_id'] + '/' + str(chan23['direction'])
-#     inv = l3.rpc.invoice(amount_msat=123000, label='label1', description='desc')['bolt11']
-#     # Exclude the payer node id
-#     with pytest.raises(RpcError, match=r'Payer is manually excluded'):
-#         l1.rpc.pay(inv, exclude=[l1_id])
-#     # Exclude the direct payee node id
-#     with pytest.raises(RpcError, match=r'Payee is manually excluded'):
-#         l2.rpc.pay(inv, exclude=[l3_id])
-#     # Exclude intermediate node id
-#     with pytest.raises(RpcError, match=r'is not reachable directly and all routehints were unusable.'):
-#         l1.rpc.pay(inv, exclude=[l2_id])
-#     # Exclude intermediate channel id
-#     with pytest.raises(RpcError, match=r'is not reachable directly and all routehints were unusable.'):
-#         l1.rpc.pay(inv, exclude=[scid12])
-#     # Exclude direct channel id
-#     with pytest.raises(RpcError, match=r'is not reachable directly and all routehints were unusable.'):
-#         l2.rpc.pay(inv, exclude=[scid23])
-# 
-# 
-# @unittest.skipIf(TEST_NETWORK != 'regtest', "Invoice is network specific")
-# def test_pay_bolt11_metadata(node_factory, bitcoind):
-#     l1, l2 = node_factory.line_graph(2)
-# 
-#     # BOLT #11:
-#     # > ### Please send 0.01 BTC with payment metadata 0x01fafaf0
-#     # > lnbc10m1pvjluezpp5qqqsyqcyq5rqwzqfqqqsyqcyq5rqwzqfqqqsyqcyq5rqwzqfqypqdp9wpshjmt9de6zqmt9w3skgct5vysxjmnnd9jx2mq8q8a04uqsp5zyg3zyg3zyg3zyg3zyg3zyg3zyg3zyg3zyg3zyg3zyg3zyg3zygs9q2gqqqqqqsgq7hf8he7ecf7n4ffphs6awl9t6676rrclv9ckg3d3ncn7fct63p6s365duk5wrk202cfy3aj5xnnp5gs3vrdvruverwwq7yzhkf5a3xqpd05wjc
-# 
-#     b11 = l1.rpc.decode('lnbc10m1pvjluezpp5qqqsyqcyq5rqwzqfqqqsyqcyq5rqwzqfqqqsyqcyq5rqwzqfqypqdp9wpshjmt9de6zqmt9w3skgct5vysxjmnnd9jx2mq8q8a04uqsp5zyg3zyg3zyg3zyg3zyg3zyg3zyg3zyg3zyg3zyg3zyg3zyg3zygs9q2gqqqqqqsgq7hf8he7ecf7n4ffphs6awl9t6676rrclv9ckg3d3ncn7fct63p6s365duk5wrk202cfy3aj5xnnp5gs3vrdvruverwwq7yzhkf5a3xqpd05wjc')
-#     assert b11['payment_metadata'] == '01fafaf0'
-# 
-#     # I previously hacked lightningd to add "this is metadata" to metadata.
-#     # After CI started failing, I *also* hacked it to set expiry to BIGNUM.
-#     inv = "lnbcrt1230n1p3yzgcxsp5q8g040f9rl9mu2unkjuj0vn262s6nyrhz5hythk3ueu2lfzahmzspp5ve584t0cv27hwmy0cx9ca8uwyqyfw9y9dm3r8vus9fv36r2l9yjsdq8v3jhxccmq6w35xjueqd9ejqmt9w3skgct5vyxqxra2q2qcqp99q2sqqqqqysgqfw6efxpzk5x5vfj8se46yg667x5cvhyttnmuqyk0q7rmhx3gs249qhtdggnek8c5adm2pztkjddlwyn2art2zg9xap2ckczzl3fzz4qqsej6mf"
-#     # Make l2 "know" about this invoice.
-#     l2.rpc.invoice(amount_msat=123000, label='label1', description='desc', preimage='00' * 32)
-# 
-#     with pytest.raises(RpcError, match=r'WIRE_INVALID_ONION_PAYLOAD'):
-#         l1.rpc.pay(inv)
-# 
-#     l2.daemon.wait_for_log("Unexpected payment_metadata {}".format(b'this is metadata'.hex()))
-# 
-# 
-# @pytest.mark.developer("needs to dev-disconnect")
-# def test_pay_middle_fail(node_factory, bitcoind, executor):
-#     """Test the case where a HTLC is failed, but not on peer's side, then
-#     we go onchain"""
-#     # Set feerates the same so we don't have update_fee interfering.
-#     # We want to disconnect on revoke-and-ack we send for failing htlc.
-#     l1, l2, l3 = node_factory.line_graph(3, wait_for_announce=True,
-#                                          opts=[{'feerates': (1500,) * 4},
-#                                                {'feerates': (1500,) * 4},
-#                                                {'feerates': (1500,) * 4,
-#                                                 'disconnect': ['-WIRE_REVOKE_AND_ACK*2']}])
-# 
-#     chanid12 = only_one(l1.rpc.listpeerchannels(l2.info['id'])['channels'])['short_channel_id']
-#     chanid23 = only_one(l2.rpc.listpeerchannels(l3.info['id'])['channels'])['short_channel_id']
-# 
-#     # Make a failing payment.
-#     route = [{'amount_msat': 1011,
-#               'id': l2.info['id'],
-#               'delay': 20,
-#               'channel': chanid12},
-#              {'amount_msat': 1000,
-#               'id': l3.info['id'],
-#               'delay': 10,
-#               'channel': chanid23}]
-# 
-#     # Start payment, it will fail.
-#     l1.rpc.sendpay(route, payment_hash='00' * 32)
-# 
-#     wait_for(lambda: only_one(l3.rpc.listpeers(l2.info['id'])['peers'])['connected'] is False)
-# 
-#     # After this (cltv is actually +11, and we give it 1 block grace)
-#     # l2 will go onchain since HTLC is not resolved.
-#     bitcoind.generate_block(12)
-#     sync_blockheight(bitcoind, [l1, l2, l3])
-#     wait_for(lambda: only_one(l2.rpc.listpeerchannels(l3.info['id'])['channels'])['state'] == 'AWAITING_UNILATERAL')
-# 
-#     # Three blocks and it will resolve the parent.
-#     bitcoind.generate_block(3, wait_for_mempool=1)
-# 
-#     # And that will fail upstream
-#     with pytest.raises(RpcError, match=r'WIRE_INCORRECT_OR_UNKNOWN_PAYMENT_DETAILS'):
-#         l1.rpc.waitsendpay('00' * 32)
-# 
-# 
-# def test_sendpay_dual_amounts(node_factory):
-#     """Test that handing *both* msatoshi and amount_msat to sendpay works"""
-#     l1 = node_factory.get_node(options={'allow-deprecated-apis': True})
-# 
-#     route = [{'amount_msat': 1011,
-#               'msatoshi': 1011,
-#               'id': '022d223620a359a47ff7f7ac447c85c46c923da53389221a0054c11c1e3ca31d59',
-#               'delay': 20,
-#               'channel': '1x1x1'},
-#              {'amount_msat': 1000,
-#               'msatoshi': 1000,
-#               'id': '035d2b1192dfba134e10e540875d366ebc8bc353d5aa766b80c090b39c3a5d885d',
-#               'delay': 10,
-#               'channel': '2x2x2'}]
-#     l1.rpc.check("sendpay", route=route, payment_hash="00" * 32)
-# 
-#     with pytest.raises(RpcError, match=r'No connection to first peer found'):
-#         l1.rpc.sendpay(route=route, payment_hash="00" * 32)
-# 
-# 
-# @unittest.skipIf(TEST_NETWORK != 'regtest', "Invoice is network specific")
-# @pytest.mark.developer("needs createinvoicerequest which allows unsigned invoice containing payerinfo")
-# @pytest.mark.slow_test
-# def test_payerkey(node_factory):
-#     """payerkey calculation should not change across releases!"""
-#     nodes = node_factory.get_nodes(7)
-# 
-#     expected_keys = ["02294ec1cd3f100947fe859d71a42cb87932e36e7771abf2d50b02a7a92be8e4d5",
-#                      "026a4a3b6b0c694da6f14629ca5140713fc703591a6d8aae5c79ba9b5556fc5723",
-#                      "03defd2b1f3004b0145351f469f34512c6fa4d02fe891a977bafdb34fe7b73ea48",
-#                      "03eccb00c0a3c760465bb69a6297d7cfa5bcbd989d5a88e435bd8d6e4c723013cd",
-#                      "021b4bfa652f0df7498d734b0ca888b4e3b07f59e1a974ec7d4a9d6046e8e5ab92",
-#                      "03fc91d60b061e517f9182e3e40ea14c27df520c51db204f1409ff50e5cf9a5e4d",
-#                      "03a3bbda0137722ba62207b9d3e5e6cc2a11e58480f801892093e01383aacb7fb2"]
-# 
-#     for n, k in zip(nodes, expected_keys):
-#         b12 = n.rpc.createinvoicerequest('lnr1qqgz2d7u2smys9dc5q2447e8thjlgq3qqc3xu3s3rg94nj40zfsy866mhu5vxne6tcej5878k2mneuvgjy8ssqvepgz5zsjrg3z3vggzvkm2khkgvrxj27r96c00pwl4kveecdktm29jdd6w0uwu5jgtv5v9qgqxyfhyvyg6pdvu4tcjvpp7kkal9rp57wj7xv4pl3ajku70rzy3pu', False)['bolt12']
-#         assert n.rpc.decode(b12)['invreq_payer_id'] == k
-# 
-# 
-# def test_pay_multichannel_use_zeroconf(bitcoind, node_factory):
-#     """Check that we use the zeroconf direct channel to pay when we need to"""
-#     # 0. Setup normal channel, 200k sats.
-#     zeroconf_plugin = Path(__file__).parent / "plugins" / "zeroconf-selective.py"
-#     l1, l2 = node_factory.line_graph(2, wait_for_announce=False,
-#                                      fundamount=200_000,
-#                                      opts=[{},
-#                                            {'plugin': zeroconf_plugin,
-#                                             'zeroconf-allow': 'any'}])
-# 
-#     # 1. Open a zeoconf channel l1 -> l2
-#     zeroconf_sats = 1_000_000
-# 
-#     # 1.1 Add funds to l1's wallet for the channel open
-#     l1.fundwallet(zeroconf_sats * 2)  # This will mine a block!
-#     sync_blockheight(bitcoind, [l1, l2])
-# 
-#     # 1.2 Open the zeroconf channel
-#     l1.rpc.fundchannel(l2.info['id'], zeroconf_sats, announce=False, mindepth=0)
-# 
-#     # 1.3 Wait until all channels active.
-#     wait_for(lambda: all([c['state'] == 'CHANNELD_NORMAL' for c in l1.rpc.listpeerchannels()['channels'] + l2.rpc.listpeerchannels()['channels']]))
-# 
-#     # 2. Have l2 generate an invoice to be paid
-#     invoice_sats = "500000sat"
-#     inv = l2.rpc.invoice(invoice_sats, "test", "test")
-# 
-#     # 3. Send a payment over the zeroconf channel
-#     riskfactor = 0
-#     l1.rpc.pay(inv['bolt11'], riskfactor=riskfactor)
-# 
-# 
-# @pytest.mark.developer("needs dev-no-reconnect, dev-routes to force failover")
-# def test_delpay_works(node_factory, bitcoind):
-#     """
-#     One failure, one success; deleting the success works (groupid=1, partid=2)
-#     """
-#     l1, l2, l3 = node_factory.line_graph(3, fundamount=10**5,
-#                                          wait_for_announce=True)
-#     # Expensive route!
-#     l4 = node_factory.get_node(options={'fee-per-satoshi': 1000,
-#                                         'fee-base': 2000})
-#     node_factory.join_nodes([l1, l4, l3], wait_for_announce=True)
-# 
-#     # Don't give a hint, so l1 chooses cheapest.
-#     inv = l3.dev_invoice(10**5, 'lbl', 'desc', dev_routes=[])
-#     l3.rpc.disconnect(l2.info['id'], force=True)
-#     l1.rpc.pay(inv['bolt11'])
-# 
-#     assert len(l1.rpc.listsendpays()['payments']) == 2
-#     failed = [p for p in l1.rpc.listsendpays()['payments'] if p['status'] == 'complete'][0]
-#     l1.rpc.delpay(payment_hash=failed['payment_hash'],
-#                   status=failed['status'],
-#                   groupid=failed['groupid'],
-#                   partid=failed['partid'])
-# 
-#     with pytest.raises(RpcError, match=r'No payment for that payment_hash'):
-#         l1.rpc.delpay(payment_hash=failed['payment_hash'],
-#                       status=failed['status'],
-#                       groupid=failed['groupid'],
-#                       partid=failed['partid'])