From 9992f5bca2f757a57fe1fd87799700a8bf11e080 Mon Sep 17 00:00:00 2001 From: Peter Neuroth Date: Wed, 30 Jul 2025 12:31:42 +0200 Subject: [PATCH 01/10] channeld: add extra_tlvs from update_add_htlc msg We currently only consider known tlv types in the internal representation of a htlc. This commit adds the remaining unknown tlv fields to the htlc as well. This is in prepareation to forward these to the htlc_accepted_hook. Signed-off-by: Peter Neuroth --- channeld/channeld.c | 30 +++++++++++++++++++++++------- channeld/channeld_htlc.h | 4 ++++ channeld/full_channel.c | 15 +++++++++++++-- channeld/full_channel.h | 3 ++- channeld/test/run-full_channel.c | 4 ++-- tests/plugins/channeld_fakenet.c | 2 +- 6 files changed, 45 insertions(+), 13 deletions(-) diff --git a/channeld/channeld.c b/channeld/channeld.c index a94cd887b3af..93c3067499c1 100644 --- a/channeld/channeld.c +++ b/channeld/channeld.c @@ -621,9 +621,14 @@ static void handle_peer_add_htlc(struct peer *peer, const u8 *msg) peer_failed_warn(peer->pps, &peer->channel_id, "Bad peer_add_htlc %s", tal_hex(msg, msg)); } + add_err = channel_add_htlc(peer->channel, REMOTE, id, amount, cltv_expiry, &payment_hash, - onion_routing_packet, tlvs->blinded_path, &htlc, NULL, + onion_routing_packet, + take(tlvs->blinded_path), &htlc, NULL, + /* NOTE: It might be better to remove the + * blinded_path from the extra_tlvs */ + tlvs->fields, /* We don't immediately fail incoming htlcs, * instead we wait and fail them after * they've been committed */ @@ -5113,13 +5118,24 @@ static void resend_commitment(struct peer *peer, struct changed_htlc *last) last[i].id); if (h->state == SENT_ADD_COMMIT) { - struct tlv_update_add_htlc_tlvs *tlvs; - if (h->path_key) { + struct tlv_update_add_htlc_tlvs *tlvs = NULL; + if (h->extra_tlvs || h->path_key) { tlvs = tlv_update_add_htlc_tlvs_new(tmpctx); - tlvs->blinded_path = tal_dup(tlvs, struct pubkey, + } + if (h->extra_tlvs) { + tlvs->fields = tal_dup_talarr(tmpctx, + struct tlv_field, + h->extra_tlvs); + } + if (h->path_key) { + /* It is fine to just set the binded_path + * independent of what is in tlv->fields as the + * towire logic will serialize unknown fields + * and known types seperately. */ + tlvs->blinded_path = tal_dup(tlvs, + struct pubkey, h->path_key); - } else - tlvs = NULL; + } msg = towire_update_add_htlc(NULL, &peer->channel_id, h->id, h->amount, &h->rhash, @@ -6078,7 +6094,7 @@ static void handle_offer_htlc(struct peer *peer, const u8 *inmsg) e = channel_add_htlc(peer->channel, LOCAL, peer->htlc_id, amount, cltv_expiry, &payment_hash, onion_routing_packet, take(path_key), NULL, - &htlc_fee, true); + &htlc_fee, NULL, true); status_debug("Adding HTLC %"PRIu64" amount=%s cltv=%u gave %s", peer->htlc_id, fmt_amount_msat(tmpctx, amount), diff --git a/channeld/channeld_htlc.h b/channeld/channeld_htlc.h index 61258b0af9d3..14d41f07ab71 100644 --- a/channeld/channeld_htlc.h +++ b/channeld/channeld_htlc.h @@ -5,6 +5,7 @@ #include #include #include +#include struct htlc { /* What's the status. */ @@ -29,6 +30,9 @@ struct htlc { /* Blinding (optional). */ struct pubkey *path_key; + /* Any extra tlvs attached to this hltc (optional). */ + struct tlv_field *extra_tlvs; + /* Should we immediately fail this htlc? */ bool fail_immediate; }; diff --git a/channeld/full_channel.c b/channeld/full_channel.c index e434bd4c8977..2bcc4d58097b 100644 --- a/channeld/full_channel.c +++ b/channeld/full_channel.c @@ -588,6 +588,7 @@ static enum channel_add_err add_htlc(struct channel *channel, struct htlc **htlcp, bool enforce_aggregate_limits, struct amount_sat *htlc_fee, + struct tlv_field *extra_tlvs, bool err_immediate_failures) { struct htlc *htlc, *old; @@ -613,6 +614,15 @@ static enum channel_add_err add_htlc(struct channel *channel, htlc->failed = NULL; htlc->r = NULL; htlc->routing = tal_dup_arr(htlc, u8, routing, TOTAL_PACKET_SIZE(ROUTING_INFO_SIZE), 0); + if (extra_tlvs && tal_count(extra_tlvs) > 0) { + htlc->extra_tlvs = tal_dup_talarr(htlc, struct tlv_field, extra_tlvs); + for (size_t i = 0; i < tal_count(extra_tlvs); i++) { + /* We need to attach the value to the correct parent */ + htlc->extra_tlvs[i].value = tal_dup_talarr(htlc, u8, htlc->extra_tlvs[i].value); + } + } else { + htlc->extra_tlvs = NULL; + } /* FIXME: Change expiry to simple u32 */ @@ -905,6 +915,7 @@ enum channel_add_err channel_add_htlc(struct channel *channel, const struct pubkey *path_key TAKES, struct htlc **htlcp, struct amount_sat *htlc_fee, + struct tlv_field *extra_tlvs, bool err_immediate_failures) { enum htlc_state state; @@ -923,7 +934,7 @@ enum channel_add_err channel_add_htlc(struct channel *channel, return add_htlc(channel, state, id, amount, cltv_expiry, payment_hash, routing, path_key, - htlcp, true, htlc_fee, err_immediate_failures); + htlcp, true, htlc_fee, extra_tlvs, err_immediate_failures); } struct htlc *channel_get_htlc(struct channel *channel, enum side sender, u64 id) @@ -1621,7 +1632,7 @@ bool channel_force_htlcs(struct channel *channel, &htlcs[i]->payment_hash, htlcs[i]->onion_routing_packet, htlcs[i]->path_key, - &htlc, false, NULL, false); + &htlc, false, NULL, NULL, false); if (e != CHANNEL_ERR_ADD_OK) { status_broken("%s HTLC %"PRIu64" failed error %u", htlc_state_owner(htlcs[i]->state) == LOCAL diff --git a/channeld/full_channel.h b/channeld/full_channel.h index 3eaba5086c07..33e546272e75 100644 --- a/channeld/full_channel.h +++ b/channeld/full_channel.h @@ -68,7 +68,6 @@ struct channel *new_full_channel(const tal_t *ctx, * @remote_splice_amnt: how much is being spliced in (or out, if -ve) of remote side. * @other_anchor_outnum: which output (-1 if none) is the !!side anchor * @funding_pubkeys: The funding pubkeys (specify NULL to use channel's value). - * * Returns the unsigned commitment transaction for the committed state * for @side, followed by the htlc transactions in output order and * fills in @htlc_map, or NULL on key derivation failure. @@ -115,6 +114,7 @@ u32 actual_feerate(const struct channel *channel, * @routing: routing information (copied) * @blinding: optional blinding information for this HTLC. * @htlcp: optional pointer for resulting htlc: filled in if and only if CHANNEL_ERR_NONE. + * @extra_tlvs: optinal tlvs attached to this HTLC. * @err_immediate_failures: in some cases (dusty htlcs) we want to immediately * fail the htlc; for peer incoming don't want to * error, but rather mark it as failed and fail after @@ -134,6 +134,7 @@ enum channel_add_err channel_add_htlc(struct channel *channel, const struct pubkey *blinding TAKES, struct htlc **htlcp, struct amount_sat *htlc_fee, + struct tlv_field *extra_tlvs, bool err_immediate_failures); /** diff --git a/channeld/test/run-full_channel.c b/channeld/test/run-full_channel.c index c0d41ab79a04..69128019418e 100644 --- a/channeld/test/run-full_channel.c +++ b/channeld/test/run-full_channel.c @@ -177,7 +177,7 @@ static const struct htlc **include_htlcs(struct channel *channel, enum side side memset(&preimage, i, sizeof(preimage)); sha256(&hash, &preimage, sizeof(preimage)); e = channel_add_htlc(channel, sender, i, msatoshi, 500+i, &hash, - dummy_routing, NULL, NULL, NULL, true); + dummy_routing, NULL, NULL, NULL, NULL, true); assert(e == CHANNEL_ERR_ADD_OK); htlcs[i] = channel_get_htlc(channel, sender, i); } @@ -269,7 +269,7 @@ static void send_and_fulfill_htlc(struct channel *channel, sha256(&rhash, &r, sizeof(r)); assert(channel_add_htlc(channel, sender, 1337, msatoshi, 900, &rhash, - dummy_routing, NULL, NULL, NULL, true) + dummy_routing, NULL, NULL, NULL, NULL, true) == CHANNEL_ERR_ADD_OK); htlc = channel_get_htlc(channel, sender, 1337); assert(htlc); diff --git a/tests/plugins/channeld_fakenet.c b/tests/plugins/channeld_fakenet.c index 7b44cf381712..5795af2382de 100644 --- a/tests/plugins/channeld_fakenet.c +++ b/tests/plugins/channeld_fakenet.c @@ -893,7 +893,7 @@ static void handle_offer_htlc(struct info *info, const u8 *inmsg) e = channel_add_htlc(info->channel, LOCAL, htlc->htlc_id, amount, cltv_expiry, &htlc->payment_hash, onion_routing_packet, take(blinding), NULL, - &htlc_fee, true); + &htlc_fee, NULL, true); status_debug("Adding HTLC %"PRIu64" amount=%s cltv=%u gave %s", htlc->htlc_id, fmt_amount_msat(tmpctx, amount), cltv_expiry, From ab8b83f8802529837204c98d56bcfeea57a7234b Mon Sep 17 00:00:00 2001 From: Peter Neuroth Date: Wed, 30 Jul 2025 13:52:31 +0200 Subject: [PATCH 02/10] channeld: Add extra_tlvs to wire htlcs This appends the extra_tlvs to the internal wire htlcs "added" and "existing" for the extra tlvs to be handed to lightningd. Signed-off-by: Peter Neuroth --- channeld/channeld.c | 1 + channeld/full_channel.c | 3 +- common/htlc_wire.c | 63 ++++++++++++++++++++++++++++++++++++++++- common/htlc_wire.h | 5 +++- 4 files changed, 69 insertions(+), 3 deletions(-) diff --git a/channeld/channeld.c b/channeld/channeld.c index 93c3067499c1..da09c2c70620 100644 --- a/channeld/channeld.c +++ b/channeld/channeld.c @@ -1524,6 +1524,7 @@ static void marshall_htlc_info(const tal_t *ctx, htlc->routing, sizeof(a.onion_routing_packet)); a.path_key = htlc->path_key; + a.extra_tlvs = htlc->extra_tlvs; a.fail_immediate = htlc->fail_immediate; tal_arr_expand(added, a); } else if (htlc->state == RCVD_REMOVE_COMMIT) { diff --git a/channeld/full_channel.c b/channeld/full_channel.c index 2bcc4d58097b..699ccdf40ca4 100644 --- a/channeld/full_channel.c +++ b/channeld/full_channel.c @@ -1632,7 +1632,8 @@ bool channel_force_htlcs(struct channel *channel, &htlcs[i]->payment_hash, htlcs[i]->onion_routing_packet, htlcs[i]->path_key, - &htlc, false, NULL, NULL, false); + &htlc, false, NULL, + htlcs[i]->extra_tlvs, false); if (e != CHANNEL_ERR_ADD_OK) { status_broken("%s HTLC %"PRIu64" failed error %u", htlc_state_owner(htlcs[i]->state) == LOCAL diff --git a/common/htlc_wire.c b/common/htlc_wire.c index aa3ef92f1d73..10893156c558 100644 --- a/common/htlc_wire.c +++ b/common/htlc_wire.c @@ -4,6 +4,7 @@ #include #include #include +#include static struct failed_htlc *failed_htlc_dup(const tal_t *ctx, const struct failed_htlc *f TAKES) @@ -33,7 +34,8 @@ struct existing_htlc *new_existing_htlc(const tal_t *ctx, const u8 onion_routing_packet[TOTAL_PACKET_SIZE(ROUTING_INFO_SIZE)], const struct pubkey *path_key TAKES, const struct preimage *preimage TAKES, - const struct failed_htlc *failed TAKES) + const struct failed_htlc *failed TAKES, + const struct tlv_field *extra_tlvs TAKES) { struct existing_htlc *existing = tal(ctx, struct existing_htlc); @@ -51,6 +53,17 @@ struct existing_htlc *new_existing_htlc(const tal_t *ctx, existing->failed = failed_htlc_dup(existing, failed); else existing->failed = NULL; + if (extra_tlvs) { + existing->extra_tlvs = tal_dup_talarr(existing, struct tlv_field, extra_tlvs); + for (size_t i = 0; i < tal_count(extra_tlvs); i++) { + /* We need to attach the value to the correct parent */ + existing->extra_tlvs[i].value + = tal_dup_talarr(existing, u8, + existing->extra_tlvs[i].value); + } + } else { + existing->extra_tlvs = NULL; + } return existing; } @@ -70,6 +83,16 @@ void towire_added_htlc(u8 **pptr, const struct added_htlc *added) towire_pubkey(pptr, added->path_key); } else towire_bool(pptr, false); + if (added->extra_tlvs) { + u8 *tmp_pptr = tal_arr(tmpctx, u8, 0); + towire_tlvstream_raw(&tmp_pptr, added->extra_tlvs); + + towire_bool(pptr, true); + towire_u16(pptr, tal_bytelen(tmp_pptr)); + towire_u8_array(pptr, tmp_pptr, + tal_bytelen(tmp_pptr)); + } else + towire_bool(pptr, false); towire_bool(pptr, added->fail_immediate); } @@ -97,6 +120,16 @@ void towire_existing_htlc(u8 **pptr, const struct existing_htlc *existing) towire_pubkey(pptr, existing->path_key); } else towire_bool(pptr, false); + if (existing->extra_tlvs) { + u8 *tmp_pptr = tal_arr(tmpctx, u8, 0); + towire_tlvstream_raw(&tmp_pptr, existing->extra_tlvs); + + towire_bool(pptr, true); + towire_u16(pptr, tal_bytelen(tmp_pptr)); + towire_u8_array(pptr, tmp_pptr, + tal_bytelen(tmp_pptr)); + } else + towire_bool(pptr, false); } void towire_fulfilled_htlc(u8 **pptr, const struct fulfilled_htlc *fulfilled) @@ -163,6 +196,20 @@ void fromwire_added_htlc(const u8 **cursor, size_t *max, fromwire_pubkey(cursor, max, added->path_key); } else added->path_key = NULL; + if (fromwire_bool(cursor, max)) { + size_t tlv_len = fromwire_u16(cursor, max); + /* NOTE: We might consider to be more strict and only allow for + * known tlv types from the tlvs_tlv_update_add_htlc_tlvs + * record. */ + const u64 *allowed = cast_const(u64 *, FROMWIRE_TLV_ANY_TYPE); + added->extra_tlvs = tal_arr(added, struct tlv_field, 0); + if (!fromwire_tlv(cursor, &tlv_len, NULL, 0, added, + &added->extra_tlvs, allowed, NULL, NULL)) { + tal_free(added->extra_tlvs); + added->extra_tlvs = NULL; + } + } else + added->extra_tlvs = NULL; added->fail_immediate = fromwire_bool(cursor, max); } @@ -192,6 +239,20 @@ struct existing_htlc *fromwire_existing_htlc(const tal_t *ctx, fromwire_pubkey(cursor, max, existing->path_key); } else existing->path_key = NULL; + if (fromwire_bool(cursor, max)) { + size_t tlv_len = fromwire_u16(cursor, max); + /* NOTE: We might consider to be more strict and only allow for + * known tlv types from the tlvs_tlv_update_add_htlc_tlvs + * record. */ + const u64 *allowed = cast_const(u64 *, FROMWIRE_TLV_ANY_TYPE); + existing->extra_tlvs = tal_arr(existing, struct tlv_field, 0); + if (!fromwire_tlv(cursor, &tlv_len, NULL, 0, existing, + &existing->extra_tlvs, allowed, NULL, NULL)) { + tal_free(existing->extra_tlvs); + existing->extra_tlvs = NULL; + } + } else + existing->extra_tlvs = NULL; return existing; } diff --git a/common/htlc_wire.h b/common/htlc_wire.h index 4d758a649028..c50fece3e141 100644 --- a/common/htlc_wire.h +++ b/common/htlc_wire.h @@ -17,6 +17,7 @@ struct added_htlc { u8 onion_routing_packet[TOTAL_PACKET_SIZE(ROUTING_INFO_SIZE)]; bool fail_immediate; struct pubkey *path_key; + struct tlv_field *extra_tlvs; }; /* This is how lightningd tells us about HTLCs which already exist at startup */ @@ -33,6 +34,7 @@ struct existing_htlc { struct preimage *payment_preimage; /* If failed, this is set */ const struct failed_htlc *failed; + struct tlv_field *extra_tlvs; }; struct fulfilled_htlc { @@ -69,7 +71,8 @@ struct existing_htlc *new_existing_htlc(const tal_t *ctx, const u8 onion_routing_packet[TOTAL_PACKET_SIZE(ROUTING_INFO_SIZE)], const struct pubkey *path_key TAKES, const struct preimage *preimage TAKES, - const struct failed_htlc *failed TAKES); + const struct failed_htlc *failed TAKES, + const struct tlv_field *extra_tlvs TAKES); void towire_added_htlc(u8 **pptr, const struct added_htlc *added); void towire_existing_htlc(u8 **pptr, const struct existing_htlc *existing); From 604af9c0e0343b74743deece381a57806888f4ee Mon Sep 17 00:00:00 2001 From: Peter Neuroth Date: Wed, 30 Jul 2025 13:55:20 +0200 Subject: [PATCH 03/10] channeld: Add extra_tlvs to incomming wire msg This appends the extra_tlvs to the internal channeld_offer_htlc wire msg. We also recombine the extra_tlvs with the blinded path key for forwarding htlcs. Signed-off-by: Peter Neuroth --- channeld/channeld.c | 33 +++++++++++++++++++++++++++----- channeld/channeld_wire.csv | 2 ++ lightningd/peer_htlcs.c | 9 +++++---- tests/plugins/channeld_fakenet.c | 4 ++-- wallet/test/Makefile | 4 +++- wallet/test/run-wallet.c | 16 +--------------- 6 files changed, 41 insertions(+), 27 deletions(-) diff --git a/channeld/channeld.c b/channeld/channeld.c index da09c2c70620..a4b134e44d65 100644 --- a/channeld/channeld.c +++ b/channeld/channeld.c @@ -49,6 +49,7 @@ #include #include #include +#include #include /* stdin == requests, 3 == peer, 4 = HSM */ @@ -6075,7 +6076,9 @@ static void handle_offer_htlc(struct peer *peer, const u8 *inmsg) const char *failstr; struct amount_sat htlc_fee; struct pubkey *path_key; + struct tlv_field *extra_tlvs; struct tlv_update_add_htlc_tlvs *tlvs; + u8 *extra_tlvs_raw; if (!peer->channel_ready[LOCAL] || !peer->channel_ready[REMOTE]) status_failed(STATUS_FAIL_MASTER_IO, @@ -6083,19 +6086,39 @@ static void handle_offer_htlc(struct peer *peer, const u8 *inmsg) if (!fromwire_channeld_offer_htlc(tmpctx, inmsg, &amount, &cltv_expiry, &payment_hash, - onion_routing_packet, &path_key)) + onion_routing_packet, &path_key, &extra_tlvs_raw)) master_badmsg(WIRE_CHANNELD_OFFER_HTLC, inmsg); - if (path_key) { + + if (extra_tlvs_raw || path_key) { tlvs = tlv_update_add_htlc_tlvs_new(tmpctx); - tlvs->blinded_path = tal_dup(tlvs, struct pubkey, path_key); - } else + } else { tlvs = NULL; + } + + if (extra_tlvs_raw) { + const u8 *cursor = extra_tlvs_raw; + size_t max = tal_bytelen(extra_tlvs_raw); + u64 failedtype; + const u64 *allowed = cast_const(u64 *, FROMWIRE_TLV_ANY_TYPE); + if (!fromwire_tlv(&cursor, &max, NULL, 0, + tlvs, &tlvs->fields, + allowed, NULL, &failedtype)) { + status_unusual("Malformed TLV type %"PRIu64": %s", + failedtype, tal_hex(tmpctx, extra_tlvs_raw)); + } + extra_tlvs = tlvs->fields; + } else { + extra_tlvs = NULL; + } + if (path_key) { + tlvs->blinded_path = tal_dup(tlvs, struct pubkey, path_key); + } e = channel_add_htlc(peer->channel, LOCAL, peer->htlc_id, amount, cltv_expiry, &payment_hash, onion_routing_packet, take(path_key), NULL, - &htlc_fee, NULL, true); + &htlc_fee, extra_tlvs, true); status_debug("Adding HTLC %"PRIu64" amount=%s cltv=%u gave %s", peer->htlc_id, fmt_amount_msat(tmpctx, amount), diff --git a/channeld/channeld_wire.csv b/channeld/channeld_wire.csv index 3abacb990fdf..be5a5c6dee36 100644 --- a/channeld/channeld_wire.csv +++ b/channeld/channeld_wire.csv @@ -94,6 +94,8 @@ msgdata,channeld_offer_htlc,cltv_expiry,u32, msgdata,channeld_offer_htlc,payment_hash,sha256, msgdata,channeld_offer_htlc,onion_routing_packet,u8,1366 msgdata,channeld_offer_htlc,path_key,?pubkey, +msgdata,channeld_offer_htlc,extra_tlvs_len,u16, +msgdata,channeld_offer_htlc,extra_tlvs,u8,extra_tlvs_len # Reply; synchronous since IDs have to increment. msgtype,channeld_offer_htlc_reply,1104 diff --git a/lightningd/peer_htlcs.c b/lightningd/peer_htlcs.c index fc4a7063dcde..f21d796bb2bf 100644 --- a/lightningd/peer_htlcs.c +++ b/lightningd/peer_htlcs.c @@ -701,7 +701,7 @@ const u8 *send_htlc_out(const tal_t *ctx, struct htlc_in *in, struct htlc_out **houtp) { - u8 *msg; + u8 *msg, *raw_tlvs = NULL; *houtp = NULL; @@ -743,7 +743,8 @@ const u8 *send_htlc_out(const tal_t *ctx, } msg = towire_channeld_offer_htlc(out, amount, cltv, payment_hash, - onion_routing_packet, path_key); + onion_routing_packet, path_key, + raw_tlvs); subd_req(out->peer->ld, out->owner, take(msg), -1, 0, rcvd_htlc_reply, *houtp); @@ -2646,7 +2647,7 @@ const struct existing_htlc **peer_htlcs(const tal_t *ctx, hin->onion_routing_packet, hin->path_key, hin->preimage, - f); + f, NULL); tal_arr_expand(&htlcs, existing); } @@ -2678,7 +2679,7 @@ const struct existing_htlc **peer_htlcs(const tal_t *ctx, hout->onion_routing_packet, hout->path_key, hout->preimage, - f); + f, NULL); tal_arr_expand(&htlcs, existing); } diff --git a/tests/plugins/channeld_fakenet.c b/tests/plugins/channeld_fakenet.c index 5795af2382de..fc08d2bf1352 100644 --- a/tests/plugins/channeld_fakenet.c +++ b/tests/plugins/channeld_fakenet.c @@ -871,7 +871,7 @@ static void delayed_forward(struct delayed_forward *dfwd) static void handle_offer_htlc(struct info *info, const u8 *inmsg) { - u8 *msg; + u8 *msg, *extratlvs; u32 cltv_expiry; struct amount_msat amount; u8 onion_routing_packet[TOTAL_PACKET_SIZE(ROUTING_INFO_SIZE)]; @@ -887,7 +887,7 @@ static void handle_offer_htlc(struct info *info, const u8 *inmsg) htlc->htlc_id = htlc_id; if (!fromwire_channeld_offer_htlc(tmpctx, inmsg, &amount, &cltv_expiry, &htlc->payment_hash, - onion_routing_packet, &blinding)) + onion_routing_packet, &blinding, &extratlvs)) master_badmsg(WIRE_CHANNELD_OFFER_HTLC, inmsg); e = channel_add_htlc(info->channel, LOCAL, htlc->htlc_id, diff --git a/wallet/test/Makefile b/wallet/test/Makefile index 1418608727fb..61789189963f 100644 --- a/wallet/test/Makefile +++ b/wallet/test/Makefile @@ -30,9 +30,11 @@ WALLET_TEST_COMMON_OBJS := \ common/utxo.o \ common/wireaddr.o \ common/version.o \ + common/bigsize.o \ wallet/db_sqlite3_sqlgen.o \ wire/towire.o \ - wire/fromwire.o + wire/fromwire.o \ + wire/tlvstream.o $(WALLET_TEST_PROGRAMS): $(BITCOIN_OBJS) $(WALLET_TEST_COMMON_OBJS) $(WALLET_TEST_OBJS): $(WALLET_HDRS) $(WALLET_SRC) diff --git a/wallet/test/run-wallet.c b/wallet/test/run-wallet.c index 3084da4dd427..3bc0b79a2d65 100644 --- a/wallet/test/run-wallet.c +++ b/wallet/test/run-wallet.c @@ -46,9 +46,6 @@ static void test_error(struct lightningd *ld, bool fatal, const char *fmt, va_li void add_node_announcement_sig(u8 *nannounce UNNEEDED, const secp256k1_ecdsa_signature *sig UNNEEDED) { fprintf(stderr, "add_node_announcement_sig called!\n"); abort(); } -/* Generated stub for bigsize_put */ -size_t bigsize_put(u8 buf[BIGSIZE_MAX_LEN] UNNEEDED, bigsize_t v UNNEEDED) -{ fprintf(stderr, "bigsize_put called!\n"); abort(); } /* Generated stub for bitcoind_getrawblockbyheight_ */ void bitcoind_getrawblockbyheight_(const tal_t *ctx UNNEEDED, struct bitcoind *bitcoind UNNEEDED, @@ -368,12 +365,6 @@ bool fromwire_onchaind_dev_memleak_reply(const void *p UNNEEDED, bool *leak UNNE /* Generated stub for fromwire_openingd_dev_memleak_reply */ bool fromwire_openingd_dev_memleak_reply(const void *p UNNEEDED, bool *leak UNNEEDED) { fprintf(stderr, "fromwire_openingd_dev_memleak_reply called!\n"); abort(); } -/* Generated stub for fromwire_tlv */ -bool fromwire_tlv(const u8 **cursor UNNEEDED, size_t *max UNNEEDED, - const struct tlv_record_type *types UNNEEDED, size_t num_types UNNEEDED, - void *record UNNEEDED, struct tlv_field **fields UNNEEDED, - const u64 *extra_types UNNEEDED, size_t *err_off UNNEEDED, u64 *err_type UNNEEDED) -{ fprintf(stderr, "fromwire_tlv called!\n"); abort(); } /* Generated stub for get_network_blockheight */ u32 get_network_blockheight(const struct chain_topology *topo UNNEEDED) { fprintf(stderr, "get_network_blockheight called!\n"); abort(); } @@ -1079,7 +1070,7 @@ u8 *towire_channeld_got_commitsig_reply(const tal_t *ctx UNNEEDED) u8 *towire_channeld_got_revoke_reply(const tal_t *ctx UNNEEDED) { fprintf(stderr, "towire_channeld_got_revoke_reply called!\n"); abort(); } /* Generated stub for towire_channeld_offer_htlc */ -u8 *towire_channeld_offer_htlc(const tal_t *ctx UNNEEDED, struct amount_msat amount_msat UNNEEDED, u32 cltv_expiry UNNEEDED, const struct sha256 *payment_hash UNNEEDED, const u8 onion_routing_packet[1366] UNNEEDED, const struct pubkey *path_key UNNEEDED) +u8 *towire_channeld_offer_htlc(const tal_t *ctx UNNEEDED, struct amount_msat amount_msat UNNEEDED, u32 cltv_expiry UNNEEDED, const struct sha256 *payment_hash UNNEEDED, const u8 onion_routing_packet[1366] UNNEEDED, const struct pubkey *path_key UNNEEDED, const u8 *extra_tlvs UNNEEDED) { fprintf(stderr, "towire_channeld_offer_htlc called!\n"); abort(); } /* Generated stub for towire_channeld_sending_commitsig_reply */ u8 *towire_channeld_sending_commitsig_reply(const tal_t *ctx UNNEEDED) @@ -1185,11 +1176,6 @@ u8 *towire_temporary_channel_failure(const tal_t *ctx UNNEEDED, const u8 *channe /* Generated stub for towire_temporary_node_failure */ u8 *towire_temporary_node_failure(const tal_t *ctx UNNEEDED) { fprintf(stderr, "towire_temporary_node_failure called!\n"); abort(); } -/* Generated stub for towire_tlv */ -void towire_tlv(u8 **pptr UNNEEDED, - const struct tlv_record_type *types UNNEEDED, size_t num_types UNNEEDED, - const void *record UNNEEDED) -{ fprintf(stderr, "towire_tlv called!\n"); abort(); } /* Generated stub for towire_unknown_next_peer */ u8 *towire_unknown_next_peer(const tal_t *ctx UNNEEDED) { fprintf(stderr, "towire_unknown_next_peer called!\n"); abort(); } From 9b578a02d8f95e51048f1508ad853ed197cdb1c8 Mon Sep 17 00:00:00 2001 From: Peter Neuroth Date: Wed, 30 Jul 2025 15:42:22 +0200 Subject: [PATCH 04/10] lightningd: Add extra_tlvs to htlc_accepted_hook Add serializing and deserializing of the extra tlvs to to the htlc_accepted_hook to allow plugin users to replace the tlv stream that is attached to the update_add_htlc message on forwards. Signed-off-by: Peter Neuroth --- lightningd/htlc_end.c | 23 +++++++++ lightningd/htlc_end.h | 8 +++ lightningd/pay.c | 6 +-- lightningd/peer_htlcs.c | 102 ++++++++++++++++++++++++++++++++++++--- lightningd/peer_htlcs.h | 1 + wallet/test/run-wallet.c | 4 ++ wallet/wallet.c | 14 ++++++ 7 files changed, 147 insertions(+), 11 deletions(-) diff --git a/lightningd/htlc_end.c b/lightningd/htlc_end.c index 46083353877c..b040f3b62176 100644 --- a/lightningd/htlc_end.c +++ b/lightningd/htlc_end.c @@ -6,6 +6,7 @@ #include #include #include +#include size_t hash_htlc_key(const struct htlc_key *k) { @@ -130,6 +131,7 @@ struct htlc_in *new_htlc_in(const tal_t *ctx, const struct secret *shared_secret TAKES, const struct pubkey *path_key TAKES, const u8 *onion_routing_packet, + const struct tlv_field *extra_tlvs, bool fail_immediate) { struct htlc_in *hin = tal(ctx, struct htlc_in); @@ -146,6 +148,15 @@ struct htlc_in *new_htlc_in(const tal_t *ctx, hin->path_key = tal_dup_or_null(hin, struct pubkey, path_key); memcpy(hin->onion_routing_packet, onion_routing_packet, sizeof(hin->onion_routing_packet)); + if (extra_tlvs) { + hin->extra_tlvs = tal_dup_talarr(hin, struct tlv_field, extra_tlvs); + for (size_t i = 0; i < tal_count(extra_tlvs); i++) { + /* We need to attach the value to the correct parent */ + hin->extra_tlvs[i].value = tal_dup_talarr(hin, u8, hin->extra_tlvs[i].value); + } + } else { + hin->extra_tlvs = NULL; + } hin->hstate = RCVD_ADD_COMMIT; hin->badonion = 0; @@ -265,6 +276,7 @@ struct htlc_out *new_htlc_out(const tal_t *ctx, const struct sha256 *payment_hash, const u8 *onion_routing_packet, const struct pubkey *path_key, + const struct tlv_field* extra_tlvs, bool am_origin, struct amount_msat final_msat, u64 partid, @@ -291,6 +303,17 @@ struct htlc_out *new_htlc_out(const tal_t *ctx, hout->timeout = NULL; hout->path_key = tal_dup_or_null(hout, struct pubkey, path_key); + + if (extra_tlvs) { + hout->extra_tlvs = tal_dup_talarr(hout, struct tlv_field, extra_tlvs); + for (size_t i = 0; i < tal_count(extra_tlvs); i++) { + /* We need to attach the value to the correct parent */ + hout->extra_tlvs[i].value = tal_dup_talarr(hout, u8, hout->extra_tlvs[i].value); + } + } else { + hout->extra_tlvs = NULL; + } + hout->am_origin = am_origin; if (am_origin) { hout->partid = partid; diff --git a/lightningd/htlc_end.h b/lightningd/htlc_end.h index 6b106caca9bf..5d1932516bf7 100644 --- a/lightningd/htlc_end.h +++ b/lightningd/htlc_end.h @@ -58,6 +58,9 @@ struct htlc_in { /* The decoded onion payload after hooks processed it. */ struct onion_payload *payload; + + /* Incommimg extra update_add_htlc_tlv tlvs */ + struct tlv_field *extra_tlvs; }; struct htlc_out { @@ -106,6 +109,9 @@ struct htlc_out { /* Timer we use in case they don't add an HTLC in a timely manner. */ struct oneshot *timeout; + + /* Extra tlvs that are extended to the update_add_htlc_tlvs */ + struct tlv_field *extra_tlvs; }; static inline const struct htlc_key *keyof_htlc_in(const struct htlc_in *in) @@ -158,6 +164,7 @@ struct htlc_in *new_htlc_in(const tal_t *ctx, const struct secret *shared_secret TAKES, const struct pubkey *path_key TAKES, const u8 *onion_routing_packet, + const struct tlv_field *extra_tlvs TAKES, bool fail_immediate); /* You need to set the ID, then connect_htlc_out this! */ @@ -168,6 +175,7 @@ struct htlc_out *new_htlc_out(const tal_t *ctx, const struct sha256 *payment_hash, const u8 *onion_routing_packet, const struct pubkey *path_key, + const struct tlv_field *extra_tlvs, bool am_origin, struct amount_msat final_msat, u64 partid, diff --git a/lightningd/pay.c b/lightningd/pay.c index 5b17f92a0735..f7a267813c6e 100644 --- a/lightningd/pay.c +++ b/lightningd/pay.c @@ -783,8 +783,8 @@ static const u8 *send_onion(const tal_t *ctx, struct lightningd *ld, return send_htlc_out(ctx, channel, first_hop->amount, base_expiry + first_hop->delay, final_amount, payment_hash, - path_key, partid, groupid, onion, NULL, hout); -} + path_key, NULL, partid, groupid, onion, NULL, hout); + } static struct command_result *check_invoice_request_usage(struct command *cmd, const struct sha256 *local_invreq_id) @@ -2093,7 +2093,7 @@ static struct command_result *json_injectpaymentonion(struct command *cmd, failmsg = send_htlc_out(tmpctx, next, *msat, *cltv, *destination_msat, payment_hash, - next_path_key, *partid, *groupid, + next_path_key, NULL, *partid, *groupid, serialize_onionpacket(tmpctx, rs->next), NULL, &hout); if (failmsg) { diff --git a/lightningd/peer_htlcs.c b/lightningd/peer_htlcs.c index f21d796bb2bf..6c946f0b44c2 100644 --- a/lightningd/peer_htlcs.c +++ b/lightningd/peer_htlcs.c @@ -3,6 +3,7 @@ #include #include #include +#include #include #include #include @@ -23,6 +24,10 @@ #include #include #include +#include +#include +#include +#include #ifndef SUPERVERBOSE #define SUPERVERBOSE(...) @@ -695,6 +700,7 @@ const u8 *send_htlc_out(const tal_t *ctx, struct amount_msat final_msat, const struct sha256 *payment_hash, const struct pubkey *path_key, + const struct tlv_field *extra_tlvs, u64 partid, u64 groupid, const u8 *onion_routing_packet, @@ -729,7 +735,8 @@ const u8 *send_htlc_out(const tal_t *ctx, /* Make peer's daemon own it, catch if it dies. */ *houtp = new_htlc_out(out->owner, out, amount, cltv, payment_hash, onion_routing_packet, - path_key, in == NULL, + path_key, extra_tlvs, + in == NULL, final_msat, partid, groupid, in); tal_add_destructor(*houtp, destroy_hout_subd_died); @@ -742,6 +749,13 @@ const u8 *send_htlc_out(const tal_t *ctx, *houtp); } + if (extra_tlvs) { + raw_tlvs = tal_arr(tmpctx, u8, 0); + towire_tlvstream_raw(&raw_tlvs, + tal_dup_talarr(tmpctx, struct tlv_field, + extra_tlvs)); + } + msg = towire_channeld_offer_htlc(out, amount, cltv, payment_hash, onion_routing_packet, path_key, raw_tlvs); @@ -797,7 +811,8 @@ static void forward_htlc(struct htlc_in *hin, const struct short_channel_id *forward_scid, const struct channel_id *forward_to, const u8 next_onion[TOTAL_PACKET_SIZE(ROUTING_INFO_SIZE)], - const struct pubkey *next_path_key) + const struct pubkey *next_path_key, + const struct tlv_field *extra_tlvs) { const u8 *failmsg; struct lightningd *ld = hin->key.channel->peer->ld; @@ -912,7 +927,7 @@ static void forward_htlc(struct htlc_in *hin, failmsg = send_htlc_out(tmpctx, next, amt_to_forward, outgoing_cltv_value, AMOUNT_MSAT(0), &hin->payment_hash, - next_path_key, 0 /* partid */, 0 /* groupid */, + next_path_key, extra_tlvs, 0 /* partid */, 0 /* groupid */, next_onion, hin, &hout); if (!failmsg) return; @@ -942,6 +957,7 @@ struct htlc_accepted_hook_payload { u64 failtlvtype; size_t failtlvpos; const char *failexplanation; + u8 *extra_tlvs_raw; }; static void @@ -998,8 +1014,8 @@ static bool htlc_accepted_hook_deserialize(struct htlc_accepted_hook_payload *re struct htlc_in *hin = request->hin; struct lightningd *ld = request->ld; struct preimage payment_preimage; - const jsmntok_t *resulttok, *paykeytok, *payloadtok, *fwdtok; - u8 *failonion; + const jsmntok_t *resulttok, *paykeytok, *payloadtok, *fwdtok, *extra_tlvs_tok; + u8 *failonion, *raw_tlvs; if (!toks || !buffer) return true; @@ -1013,6 +1029,49 @@ static bool htlc_accepted_hook_deserialize(struct htlc_accepted_hook_payload *re json_strdup(tmpctx, buffer, toks)); } + extra_tlvs_tok = json_get_member(buffer, toks, "extra_tlvs"); + if (extra_tlvs_tok) { + size_t max; + struct tlv_update_add_htlc_tlvs *check_extra_tlvs; + + raw_tlvs = json_tok_bin_from_hex(tmpctx, buffer, + extra_tlvs_tok); + if (!raw_tlvs) + fatal("Bad custom tlvs for htlc_accepted" + " hook: %.*s", + extra_tlvs_tok->end - extra_tlvs_tok->start, + buffer + extra_tlvs_tok->start); + + max = tal_bytelen(raw_tlvs); + + /* We check if the custom tlvs are still valid BOLT#1 tlvs. + * As these are appended to forwarded htlcs we check for valid + * update_add_htlc_tlvs (restricts to known even types). + * NOTE: We may be less strict and allow unknown evens .*/ + const u8 *cursor = raw_tlvs; + check_extra_tlvs = fromwire_tlv_update_add_htlc_tlvs(tmpctx, + &cursor, + &max); + if (!check_extra_tlvs) { + fatal("htlc_accepted_hook returned bad extra_tlvs %s", + tal_hex(tmpctx, raw_tlvs)); + } + + /* If we got a blinded path key we replace the next path key + * with it. */ + if (check_extra_tlvs->blinded_path) { + tal_free(request->next_path_key); + request->next_path_key + = tal_steal(request, + check_extra_tlvs->blinded_path); + } + + /* We made it and got a valid extra_tlvs: Replace the current + * extra_tlvs with it. */ + tal_free(request->extra_tlvs_raw); + request->extra_tlvs_raw = tal_steal(request, raw_tlvs); + } + payloadtok = json_get_member(buffer, toks, "payload"); if (payloadtok) { u8 *payload = json_tok_bin_from_hex(rs, buffer, payloadtok); @@ -1170,6 +1229,9 @@ static void htlc_accepted_hook_serialize(struct htlc_accepted_hook_payload *p, json_add_u32(s, "cltv_expiry", expiry); json_add_s32(s, "cltv_expiry_relative", expiry - blockheight); json_add_sha256(s, "payment_hash", &hin->payment_hash); + if (p->extra_tlvs_raw) { + json_add_hex_talarr(s, "extra_tlvs", p->extra_tlvs_raw); + } json_object_end(s); } @@ -1200,13 +1262,25 @@ htlc_accepted_hook_final(struct htlc_accepted_hook_payload *request STEALS) NULL, request->failtlvtype, request->failtlvpos))); } else if (rs->nextcase == ONION_FORWARD) { + struct tlv_field *extra_tlvs; + + if (request->extra_tlvs_raw) { + const u8 *cursor = request->extra_tlvs_raw; + size_t max = tal_bytelen(cursor); + extra_tlvs = tal_arr(request, struct tlv_field, 0); + fromwire_tlv(&cursor, &max, NULL, 0, request, + &extra_tlvs, NULL, NULL, NULL); + } else { + extra_tlvs = NULL; + } + forward_htlc(hin, hin->cltv_expiry, request->payload->amt_to_forward, request->payload->outgoing_cltv, request->payload->forward_channel, request->fwd_channel_id, serialize_onionpacket(tmpctx, rs->next), - request->next_path_key); + request->next_path_key, extra_tlvs); } else handle_localpay(hin, request->payload->amt_to_forward, @@ -1484,6 +1558,14 @@ static bool peer_accepted_htlc(const tal_t *ctx, hook_payload->fwd_channel_id = calc_forwarding_channel(ld, hook_payload); + if(hin->extra_tlvs) { + hook_payload->extra_tlvs_raw = tal_arr(hook_payload, u8, 0); + towire_tlvstream_raw(&hook_payload->extra_tlvs_raw, + hin->extra_tlvs); + } else { + hook_payload->extra_tlvs_raw = NULL; + } + plugin_hook_call_htlc_accepted(ld, NULL, hook_payload); /* Falling through here is ok, after all the HTLC locked */ @@ -2210,6 +2292,7 @@ static bool channel_added_their_htlc(struct channel *channel, op ? &shared_secret : NULL, added->path_key, added->onion_routing_packet, + added->extra_tlvs, added->fail_immediate); /* Save an incoming htlc to the wallet */ @@ -2641,13 +2724,15 @@ const struct existing_htlc **peer_htlcs(const tal_t *ctx, else f = NULL; + existing = new_existing_htlc(htlcs, hin->key.id, hin->hstate, hin->msat, &hin->payment_hash, hin->cltv_expiry, hin->onion_routing_packet, hin->path_key, hin->preimage, - f, NULL); + f, + hin->extra_tlvs); tal_arr_expand(&htlcs, existing); } @@ -2679,7 +2764,8 @@ const struct existing_htlc **peer_htlcs(const tal_t *ctx, hout->onion_routing_packet, hout->path_key, hout->preimage, - f, NULL); + f, + hout->extra_tlvs); tal_arr_expand(&htlcs, existing); } diff --git a/lightningd/peer_htlcs.h b/lightningd/peer_htlcs.h index 0ac26821d0a9..a9c73e47dcb1 100644 --- a/lightningd/peer_htlcs.h +++ b/lightningd/peer_htlcs.h @@ -33,6 +33,7 @@ const u8 *send_htlc_out(const tal_t *ctx, struct amount_msat final_msat, const struct sha256 *payment_hash, const struct pubkey *path_key, + const struct tlv_field *extra_tlvs, u64 partid, u64 groupid, const u8 *onion_routing_packet, diff --git a/wallet/test/run-wallet.c b/wallet/test/run-wallet.c index 3bc0b79a2d65..2021b2577199 100644 --- a/wallet/test/run-wallet.c +++ b/wallet/test/run-wallet.c @@ -365,6 +365,10 @@ bool fromwire_onchaind_dev_memleak_reply(const void *p UNNEEDED, bool *leak UNNE /* Generated stub for fromwire_openingd_dev_memleak_reply */ bool fromwire_openingd_dev_memleak_reply(const void *p UNNEEDED, bool *leak UNNEEDED) { fprintf(stderr, "fromwire_openingd_dev_memleak_reply called!\n"); abort(); } +/* Generated stub for fromwire_tlv_update_add_htlc_tlvs */ +struct tlv_update_add_htlc_tlvs *fromwire_tlv_update_add_htlc_tlvs(const tal_t *ctx UNNEEDED, + const u8 **cursor UNNEEDED, size_t *max UNNEEDED) +{ fprintf(stderr, "fromwire_tlv_update_add_htlc_tlvs called!\n"); abort(); } /* Generated stub for get_network_blockheight */ u32 get_network_blockheight(const struct chain_topology *topo UNNEEDED) { fprintf(stderr, "get_network_blockheight called!\n"); abort(); } diff --git a/wallet/wallet.c b/wallet/wallet.c index 090ba7fbc3c4..66e4a3c2bf8c 100644 --- a/wallet/wallet.c +++ b/wallet/wallet.c @@ -3413,6 +3413,13 @@ static bool wallet_stmt2htlc_in(struct channel *channel, /* FIXME: save path_key in db !*/ in->path_key = NULL; in->payload = NULL; + /* FIXME: save extra_tlvs in db! But: check the implications that a + * spammy peer - giving us big extra tlvs - would have on our database. + * Right now, not saving the extra tlvs in the db seems OK as it is + * only relevant in the case that I forward but restart in the middle + * of a payment. + */ + in->extra_tlvs = NULL; db_col_sha256(stmt, "payment_hash", &in->payment_hash); @@ -3485,6 +3492,13 @@ static bool wallet_stmt2htlc_out(struct wallet *wallet, db_col_sha256(stmt, "payment_hash", &out->payment_hash); /* FIXME: save path_key in db !*/ out->path_key = NULL; + /* FIXME: save extra_tlvs in db! But: check the implications that a + * spammy peer - giving us big extra tlvs - would have on our database. + * Right now, not saving the extra tlvs in the db seems OK as it is + * only relevant in the case that I forward but restart in the middle + * of a payment. + */ + out->extra_tlvs = NULL; out->preimage = db_col_optional(out, stmt, "payment_key", preimage); From 6805bb103e6a8d00050fb15d53e4e66d40197220 Mon Sep 17 00:00:00 2001 From: Peter Neuroth Date: Wed, 30 Jul 2025 16:29:26 +0200 Subject: [PATCH 05/10] tests: Add tests for extra_tlvs in hook Adds some testcases for custom tlvs, set by a htlc_accepted_hook. We check that the custom tlvs replace the update_add_htlc_tlvs and get forwarded to the peer. We also check that a malformed tlv will result in a **BROKEN** behaviour. Signed-off-by: Peter Neuroth --- tests/plugins/htlc_accepted-customtlv.py | 38 +++++++++++++++++++++ tests/test_plugin.py | 42 ++++++++++++++++++++++++ 2 files changed, 80 insertions(+) create mode 100755 tests/plugins/htlc_accepted-customtlv.py diff --git a/tests/plugins/htlc_accepted-customtlv.py b/tests/plugins/htlc_accepted-customtlv.py new file mode 100755 index 000000000000..8358ad2bd0f1 --- /dev/null +++ b/tests/plugins/htlc_accepted-customtlv.py @@ -0,0 +1,38 @@ +#!/usr/bin/env python3 +"""A simply plugin that returns a custom tlv stream (byte encoded) to be +attached to a forwarding HTLC. +""" + +from pyln.client import Plugin + + +plugin = Plugin() +custom_tlvs = None + + +@plugin.hook("htlc_accepted") +def on_htlc_accepted(htlc, onion, plugin, **kwargs): + if 'extra_tlvs' in htlc: + print(f"called htlc accepted hook with extra_tlvs: {htlc['extra_tlvs']}") + print(f'returning continue with custom extra_tlvs: {custom_tlvs}') + if custom_tlvs: + return {"result": "continue", "extra_tlvs": custom_tlvs} + return {"result": "continue"} + + +@plugin.method("setcustomtlvs") +def setcustomtlvs(plugin, tlvs): + """Sets the custom tlv to return when receiving an incoming HTLC. + """ + global custom_tlvs + print(f'setting custom tlv to {tlvs}') + custom_tlvs = tlvs + + +@plugin.init() +def on_init(**kwargs): + global custom_tlvs + custom_tlvs = None + + +plugin.run() diff --git a/tests/test_plugin.py b/tests/test_plugin.py index ece9a020dc66..d187ac61efa7 100644 --- a/tests/test_plugin.py +++ b/tests/test_plugin.py @@ -2419,6 +2419,48 @@ def test_htlc_accepted_hook_failmsg(node_factory): l1.rpc.pay(inv) +def test_htlc_accepted_hook_customtlvs(node_factory): + """ Passes an custom extra tlv field to the hooks return that should be set + as the `update_add_htlc_tlvs` in the `update_add_htlc` message on + forwards. + """ + plugin = os.path.join(os.path.dirname(__file__), 'plugins/htlc_accepted-customtlv.py') + l1, l2, l3 = node_factory.line_graph(3, opts=[{}, {'plugin': plugin}, {'plugin': plugin}], wait_for_announce=True) + + # Single tlv - Check that we receive the extra tlv at l3 attached by l2. + single_tlv = "fe00010001012a" # represents type: 65537, lenght: 1, value: 42 + l2.rpc.setcustomtlvs(tlvs=single_tlv) + inv = l3.rpc.invoice(1000, 'customtlvs-singletlv', '')['bolt11'] + l1.rpc.pay(inv) + l3.daemon.wait_for_log(f"called htlc accepted hook with extra_tlvs: {single_tlv}") + + # Mutliple tlvs - Check that we recieve multiple extra tlvs at l3 attached by l2. + multi_tlv = "fdffff012afe00010001020539" # represents type: 65535, length: 1, value: 42 and type: 65537, length: 2, value: 1337 + l2.rpc.setcustomtlvs(tlvs=multi_tlv) + inv = l3.rpc.invoice(1000, 'customtlvs-multitlvs', '')['bolt11'] + l1.rpc.pay(inv) + l3.daemon.wait_for_log(f"called htlc accepted hook with extra_tlvs: {multi_tlv}") + + +def test_htlc_accepted_hook_malformedtlvs(node_factory): + """ Passes an custom extra tlv field to the hooks return that is malformed + and should cause a broken log. + l1 -- l2 -- l3 + """ + plugin = os.path.join(os.path.dirname(__file__), 'plugins/htlc_accepted-customtlv.py') + l1, l2, l3 = node_factory.line_graph(3, opts=[{}, {'plugin': plugin, 'broken_log': "lightningd: ", 'may_fail': True}, {}], wait_for_announce=True) + + mal_tlv = "fe00010001020539fdffff012a" # is malformed, types are 65537 and 65535 not in asc order. + l2.rpc.setcustomtlvs(tlvs=mal_tlv) + inv = l3.rpc.invoice(1000, 'customtlvs-maltlvs', '') + phash = inv['payment_hash'] + route = l1.rpc.getroute(l3.info['id'], 1000, 1)['route'] + + # Here shouldn't use `pay` command because l2 should fail with a broken log. + l1.rpc.sendpay(route, phash, payment_secret=inv['payment_secret']) + assert l2.daemon.wait_for_log("BROKEN.*htlc_accepted_hook returned bad extra_tlvs") + + def test_hook_dep(node_factory): dep_a = os.path.join(os.path.dirname(__file__), 'plugins/dep_a.py') dep_b = os.path.join(os.path.dirname(__file__), 'plugins/dep_b.py') From 2f9904d910f26e9d5a19a154b5761be6a3543d1a Mon Sep 17 00:00:00 2001 From: Peter Neuroth Date: Wed, 30 Jul 2025 16:32:36 +0200 Subject: [PATCH 06/10] docs: Add extra_tlvs to the htlc_accepted_hook doc Changelog-Added: The `htlc_accepted_hook` now gets the TLV-stream attached to the HTLC passed through as `extra_tlvs` and can replace it. Signed-off-by: Peter Neuroth --- doc/developers-guide/plugin-development/hooks.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/doc/developers-guide/plugin-development/hooks.md b/doc/developers-guide/plugin-development/hooks.md index 1e1cfdbdf41d..4dd75fe9ebce 100644 --- a/doc/developers-guide/plugin-development/hooks.md +++ b/doc/developers-guide/plugin-development/hooks.md @@ -413,7 +413,8 @@ The payload of the hook call has the following format: "amount_msat": 43, "cltv_expiry": 500028, "cltv_expiry_relative": 10, - "payment_hash": "0000000000000000000000000000000000000000000000000000000000000000" + "payment_hash": "0000000000000000000000000000000000000000000000000000000000000000", + "extra_tlvs": "fdffff012afe00010001020539" }, "forward_to": "0000000000000000000000000000000000000000000000000000000000000000" } @@ -439,6 +440,7 @@ For detailed information about each field please refer to [BOLT 04 of the specif - `cltv_expiry` determines when the HTLC reverts back to the sender. `cltv_expiry` minus `outgoing_cltv_expiry` should be equal or larger than our `cltv_delta` setting. - `cltv_expiry_relative` hints how much time we still have to claim the HTLC. It is the `cltv_expiry` minus the current `blockheight` and is passed along mainly to avoid the plugin having to look up the current blockheight. - `payment_hash` is the hash whose `payment_preimage` will unlock the funds and allow us to claim the HTLC. + - `extra_tlvs` is an optional TLV-stream attached to the HTLC. - `forward_to`: if set, the channel_id we intend to forward this to (will not be present if the short_channel_id was invalid or we were the final destination). The hook response must have one of the following formats: @@ -457,6 +459,8 @@ It can also replace the `onion.payload` by specifying a `payload` in the respons It can also specify `forward_to` in the response, replacing the destination. This usually only makes sense if it wants to choose an alternate channel to the same next peer, but is useful if the `payload` is also replaced. +Also, it can specify `extra_tlvs` in the response. This will replace the TLV-stream `update_add_htlc_tlvs` in the `update_add_htlc` message for forwarded htlcs. + ```json { "result": "fail", From e93f1d0223c3dbfebd659e265872d1d5a6ab83ae Mon Sep 17 00:00:00 2001 From: Peter Neuroth Date: Thu, 31 Jul 2025 17:09:37 +0200 Subject: [PATCH 07/10] tools: Filter "highlight" case insensitive MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit There was a problem with a ‘highlight’ that was misunderstood as a spelling mistake in lib-wally. Since ‘hightlight’ is already filtered out, we simply instruct grep to ignore upper/lower case when filtering. Signed-off-by: Peter Neuroth --- tools/check-spelling.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/check-spelling.sh b/tools/check-spelling.sh index 68cd202398c5..a66d017854b6 100755 --- a/tools/check-spelling.sh +++ b/tools/check-spelling.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash -if git --no-pager grep -nHiE 'l[ightn]{6}g|l[ightn]{8}g|ilghtning|lgihtning|lihgtning|ligthning|lighnting|lightinng|lightnnig|lightnign' -- . ':!tools/check-spelling.sh' ':!tests/data/routing_gossip_store' | grep -vE "highlighting|LightningGrpc"; then +if git --no-pager grep -nHiE 'l[ightn]{6}g|l[ightn]{8}g|ilghtning|lgihtning|lihgtning|ligthning|lighnting|lightinng|lightnnig|lightnign' -- . ':!tools/check-spelling.sh' ':!tests/data/routing_gossip_store' | grep -viE "highlighting|LightningGrpc"; then echo "Identified a likely misspelling of the word \"lightning\" (see above). Please fix." echo "Is this warning incorrect? Please teach tools/check-spelling.sh about the exciting new word." exit 1 From 3a3ece0c98fcc84e64d7cff2927761ae143cfa3f Mon Sep 17 00:00:00 2001 From: Peter Neuroth Date: Thu, 31 Jul 2025 17:14:51 +0200 Subject: [PATCH 08/10] tools: Remove lockfiles from spell-checking The rare case happened where a lockfile sha-sum contained a "Ctlv" which spell-check complained about. Stupid lockfiles that don't know it is actually "cltv"! Signed-off-by: Peter Neuroth --- tools/check-spelling.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/check-spelling.sh b/tools/check-spelling.sh index a66d017854b6..6fce914a82f5 100755 --- a/tools/check-spelling.sh +++ b/tools/check-spelling.sh @@ -6,7 +6,7 @@ if git --no-pager grep -nHiE 'l[ightn]{6}g|l[ightn]{8}g|ilghtning|lgihtning|lihg exit 1 fi -if git --no-pager grep -nHiEP '(?&2 exit 1 fi From 1a076ce8d0218e482e5e4b07b3533b552efe4b84 Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Thu, 14 Aug 2025 11:22:41 +0930 Subject: [PATCH 09/10] common: handle taken() extra_tlvs in new_existing_htlc properly. Reported-by: Christian Decker Signed-off-by: Rusty Russell --- common/htlc_wire.c | 30 ++++++++++++++++++++---------- common/htlc_wire.h | 4 ++++ lightningd/htlc_end.c | 23 +++++++---------------- 3 files changed, 31 insertions(+), 26 deletions(-) diff --git a/common/htlc_wire.c b/common/htlc_wire.c index 10893156c558..b50e7c291959 100644 --- a/common/htlc_wire.c +++ b/common/htlc_wire.c @@ -25,6 +25,23 @@ static struct failed_htlc *failed_htlc_dup(const tal_t *ctx, return newf; } +/* Helper to duplicate an array of tlv_field (vs an array of tlv_field *) */ +struct tlv_field *tlv_field_arr_dup(const tal_t *ctx, + const struct tlv_field *arr TAKES) +{ + struct tlv_field *ret; + bool needs_copy = !is_taken(arr); + + ret = tal_dup_talarr(ctx, struct tlv_field, arr); + if (needs_copy) { + for (size_t i = 0; i < tal_count(ret); i++) { + /* We need to attach the value to the correct parent */ + ret[i].value = tal_dup_talarr(ret, u8, ret[i].value); + } + } + return ret; +} + struct existing_htlc *new_existing_htlc(const tal_t *ctx, u64 id, enum htlc_state state, @@ -53,17 +70,10 @@ struct existing_htlc *new_existing_htlc(const tal_t *ctx, existing->failed = failed_htlc_dup(existing, failed); else existing->failed = NULL; - if (extra_tlvs) { - existing->extra_tlvs = tal_dup_talarr(existing, struct tlv_field, extra_tlvs); - for (size_t i = 0; i < tal_count(extra_tlvs); i++) { - /* We need to attach the value to the correct parent */ - existing->extra_tlvs[i].value - = tal_dup_talarr(existing, u8, - existing->extra_tlvs[i].value); - } - } else { + if (extra_tlvs) + existing->extra_tlvs = tlv_field_arr_dup(existing, extra_tlvs); + else existing->extra_tlvs = NULL; - } return existing; } diff --git a/common/htlc_wire.h b/common/htlc_wire.h index c50fece3e141..5cc94a9859c3 100644 --- a/common/htlc_wire.h +++ b/common/htlc_wire.h @@ -62,6 +62,10 @@ struct changed_htlc { u64 id; }; +/* Helper to duplicate an array of tlv_field (vs an array of tlv_field *) */ +struct tlv_field *tlv_field_arr_dup(const tal_t *ctx, + const struct tlv_field *arr TAKES); + struct existing_htlc *new_existing_htlc(const tal_t *ctx, u64 id, enum htlc_state state, diff --git a/lightningd/htlc_end.c b/lightningd/htlc_end.c index b040f3b62176..91d66b0c3357 100644 --- a/lightningd/htlc_end.c +++ b/lightningd/htlc_end.c @@ -3,6 +3,7 @@ #include #include #include +#include #include #include #include @@ -148,15 +149,10 @@ struct htlc_in *new_htlc_in(const tal_t *ctx, hin->path_key = tal_dup_or_null(hin, struct pubkey, path_key); memcpy(hin->onion_routing_packet, onion_routing_packet, sizeof(hin->onion_routing_packet)); - if (extra_tlvs) { - hin->extra_tlvs = tal_dup_talarr(hin, struct tlv_field, extra_tlvs); - for (size_t i = 0; i < tal_count(extra_tlvs); i++) { - /* We need to attach the value to the correct parent */ - hin->extra_tlvs[i].value = tal_dup_talarr(hin, u8, hin->extra_tlvs[i].value); - } - } else { + if (extra_tlvs) + hin->extra_tlvs = tlv_field_arr_dup(hin, extra_tlvs); + else hin->extra_tlvs = NULL; - } hin->hstate = RCVD_ADD_COMMIT; hin->badonion = 0; @@ -304,15 +300,10 @@ struct htlc_out *new_htlc_out(const tal_t *ctx, hout->path_key = tal_dup_or_null(hout, struct pubkey, path_key); - if (extra_tlvs) { - hout->extra_tlvs = tal_dup_talarr(hout, struct tlv_field, extra_tlvs); - for (size_t i = 0; i < tal_count(extra_tlvs); i++) { - /* We need to attach the value to the correct parent */ - hout->extra_tlvs[i].value = tal_dup_talarr(hout, u8, hout->extra_tlvs[i].value); - } - } else { + if (extra_tlvs) + hout->extra_tlvs = tlv_field_arr_dup(hout, extra_tlvs); + else hout->extra_tlvs = NULL; - } hout->am_origin = am_origin; if (am_origin) { From 7c929d720d93bfc1fc1b931f3a82046104168e0d Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Thu, 14 Aug 2025 11:40:46 +0930 Subject: [PATCH 10/10] common/htlc_wire: add towire/fromwire helpers for wrapped tlv streams. And make sure we check the length properly in fromwire! Signed-off-by: Rusty Russell --- common/htlc_wire.c | 72 ++++++++++++++++++++++++---------------------- 1 file changed, 38 insertions(+), 34 deletions(-) diff --git a/common/htlc_wire.c b/common/htlc_wire.c index b50e7c291959..49c8211de84e 100644 --- a/common/htlc_wire.c +++ b/common/htlc_wire.c @@ -78,6 +78,18 @@ struct existing_htlc *new_existing_htlc(const tal_t *ctx, return existing; } +static void towire_len_and_tlvstream(u8 **pptr, struct tlv_field *extra_tlvs) +{ + /* Making a copy is a bit awful, but it's the easiest way to + * get the length */ + u8 *tmp_pptr = tal_arr(tmpctx, u8, 0); + towire_tlvstream_raw(&tmp_pptr, extra_tlvs); + + assert(tal_bytelen(tmp_pptr) == (u16)tal_bytelen(tmp_pptr)); + towire_u16(pptr, tal_bytelen(tmp_pptr)); + towire_u8_array(pptr, tmp_pptr, tal_bytelen(tmp_pptr)); +} + /* FIXME: We could adapt tools/generate-wire.py to generate structures * and code like this. */ void towire_added_htlc(u8 **pptr, const struct added_htlc *added) @@ -94,13 +106,8 @@ void towire_added_htlc(u8 **pptr, const struct added_htlc *added) } else towire_bool(pptr, false); if (added->extra_tlvs) { - u8 *tmp_pptr = tal_arr(tmpctx, u8, 0); - towire_tlvstream_raw(&tmp_pptr, added->extra_tlvs); - towire_bool(pptr, true); - towire_u16(pptr, tal_bytelen(tmp_pptr)); - towire_u8_array(pptr, tmp_pptr, - tal_bytelen(tmp_pptr)); + towire_len_and_tlvstream(pptr, added->extra_tlvs); } else towire_bool(pptr, false); towire_bool(pptr, added->fail_immediate); @@ -131,13 +138,8 @@ void towire_existing_htlc(u8 **pptr, const struct existing_htlc *existing) } else towire_bool(pptr, false); if (existing->extra_tlvs) { - u8 *tmp_pptr = tal_arr(tmpctx, u8, 0); - towire_tlvstream_raw(&tmp_pptr, existing->extra_tlvs); - towire_bool(pptr, true); - towire_u16(pptr, tal_bytelen(tmp_pptr)); - towire_u8_array(pptr, tmp_pptr, - tal_bytelen(tmp_pptr)); + towire_len_and_tlvstream(pptr, existing->extra_tlvs); } else towire_bool(pptr, false); } @@ -192,6 +194,28 @@ void towire_shachain(u8 **pptr, const struct shachain *shachain) } } +static struct tlv_field *fromwire_len_and_tlvstream(const tal_t *ctx, + const u8 **cursor, size_t *max) +{ + struct tlv_field *tlvs = tal_arr(ctx, struct tlv_field, 0); + size_t len = fromwire_u16(cursor, max); + + /* Subtle: we are not using fromwire_tal_arrn here, which + * would do this. */ + if (len > *max) { + fromwire_fail(cursor, max); + return NULL; + } + + /* NOTE: We might consider to be more strict and only allow for + * known tlv types from the tlvs_tlv_update_add_htlc_tlvs + * record. */ + if (!fromwire_tlv(cursor, &len, NULL, 0, cast_const(void *, ctx), + &tlvs, FROMWIRE_TLV_ANY_TYPE, NULL, NULL)) + return tal_free(tlvs); + return tlvs; +} + void fromwire_added_htlc(const u8 **cursor, size_t *max, struct added_htlc *added) { @@ -207,17 +231,7 @@ void fromwire_added_htlc(const u8 **cursor, size_t *max, } else added->path_key = NULL; if (fromwire_bool(cursor, max)) { - size_t tlv_len = fromwire_u16(cursor, max); - /* NOTE: We might consider to be more strict and only allow for - * known tlv types from the tlvs_tlv_update_add_htlc_tlvs - * record. */ - const u64 *allowed = cast_const(u64 *, FROMWIRE_TLV_ANY_TYPE); - added->extra_tlvs = tal_arr(added, struct tlv_field, 0); - if (!fromwire_tlv(cursor, &tlv_len, NULL, 0, added, - &added->extra_tlvs, allowed, NULL, NULL)) { - tal_free(added->extra_tlvs); - added->extra_tlvs = NULL; - } + added->extra_tlvs = fromwire_len_and_tlvstream(added, cursor, max); } else added->extra_tlvs = NULL; added->fail_immediate = fromwire_bool(cursor, max); @@ -250,17 +264,7 @@ struct existing_htlc *fromwire_existing_htlc(const tal_t *ctx, } else existing->path_key = NULL; if (fromwire_bool(cursor, max)) { - size_t tlv_len = fromwire_u16(cursor, max); - /* NOTE: We might consider to be more strict and only allow for - * known tlv types from the tlvs_tlv_update_add_htlc_tlvs - * record. */ - const u64 *allowed = cast_const(u64 *, FROMWIRE_TLV_ANY_TYPE); - existing->extra_tlvs = tal_arr(existing, struct tlv_field, 0); - if (!fromwire_tlv(cursor, &tlv_len, NULL, 0, existing, - &existing->extra_tlvs, allowed, NULL, NULL)) { - tal_free(existing->extra_tlvs); - existing->extra_tlvs = NULL; - } + existing->extra_tlvs = fromwire_len_and_tlvstream(existing, cursor, max); } else existing->extra_tlvs = NULL; return existing;