From 02e552ee16cab4e9c5b2e275943d6a76770b9598 Mon Sep 17 00:00:00 2001 From: Tyler Longwell Date: Sun, 5 Apr 2026 15:28:43 -0400 Subject: [PATCH 1/6] feat: accept kind:1 text notes and kind:3 follow lists (NIP-01/NIP-02/NIP-16) - Add KIND_TEXT_NOTE constant and is_replaceable() helper to sprout-core - Allow kind:1 (MessagesWrite) and kind:3 (UsersWrite) in ingest allowlist - Route all NIP-16 replaceable kinds through replace_addressable_event() - Rewrite replace_addressable_event() for atomicity, stale-write protection, and NULL channel_id safety (pg_advisory_xact_lock + IS NOT DISTINCT FROM) - Include global events (channel_id=NULL) in NIP-50 and /api/search results - Add NIPs 2 and 16 to NIP-11 supported_nips - Update UsersWrite scope comment for kind:3 --- crates/sprout-auth/src/scope.rs | 5 +- crates/sprout-core/src/kind.rs | 8 ++ crates/sprout-db/src/lib.rs | 128 ++++++++++++++++-- crates/sprout-relay/src/api/search.rs | 20 ++- crates/sprout-relay/src/handlers/ingest.rs | 35 +++-- crates/sprout-relay/src/handlers/req.rs | 23 ++-- .../sprout-relay/src/handlers/side_effects.rs | 8 +- crates/sprout-relay/src/nip11.rs | 2 +- crates/sprout-search/src/index.rs | 20 ++- crates/sprout-search/src/query.rs | 2 +- desktop/src-tauri/src/models.rs | 4 +- desktop/src/app/AppShell.tsx | 6 +- .../src/features/search/ui/SearchDialog.tsx | 4 +- desktop/src/shared/api/tauri.ts | 4 +- desktop/src/shared/api/types.ts | 4 +- desktop/src/testing/e2eBridge.ts | 6 +- 16 files changed, 213 insertions(+), 66 deletions(-) diff --git a/crates/sprout-auth/src/scope.rs b/crates/sprout-auth/src/scope.rs index 904bd9386..e3197da21 100644 --- a/crates/sprout-auth/src/scope.rs +++ b/crates/sprout-auth/src/scope.rs @@ -154,8 +154,9 @@ impl FromStr for Scope { /// `SubscriptionsRead`, `SubscriptionsWrite`) are intentionally excluded — they require /// `sprout-admin mint-token`. /// -/// `UsersWrite` is included because it only guards self-profile endpoints -/// (`PUT /api/users/me/profile`, `PUT /api/users/me/channel-add-policy`). +/// `UsersWrite` is included because it guards self-profile endpoints +/// (`PUT /api/users/me/profile`, `PUT /api/users/me/channel-add-policy`) +/// and contact list (kind:3) publishing. pub const SELF_MINTABLE_SCOPES: &[Scope] = &[ Scope::MessagesRead, Scope::MessagesWrite, diff --git a/crates/sprout-core/src/kind.rs b/crates/sprout-core/src/kind.rs index 25bb78423..d64b99fa3 100644 --- a/crates/sprout-core/src/kind.rs +++ b/crates/sprout-core/src/kind.rs @@ -7,6 +7,8 @@ // Standard NIP kinds /// NIP-01: User profile metadata. pub const KIND_PROFILE: u32 = 0; +/// NIP-01: Short text note. +pub const KIND_TEXT_NOTE: u32 = 1; /// NIP-02: Contact list / follow list. pub const KIND_CONTACT_LIST: u32 = 3; /// NIP-09: Event deletion request. @@ -214,6 +216,7 @@ pub const KIND_MEDIA_UPLOAD: u32 = 49001; /// All registered kind constants — used for duplicate detection and iteration. pub const ALL_KINDS: &[u32] = &[ KIND_PROFILE, + KIND_TEXT_NOTE, KIND_CONTACT_LIST, KIND_DELETION, KIND_REACTION, @@ -300,6 +303,11 @@ pub const fn is_ephemeral(kind: u32) -> bool { kind >= EPHEMERAL_KIND_MIN && kind <= EPHEMERAL_KIND_MAX } +/// Returns `true` if `kind` is replaceable (NIP-01: kinds 0, 3, 41, 10000–19999). +pub const fn is_replaceable(kind: u32) -> bool { + matches!(kind, 0 | 3 | 41 | 10000..=19999) +} + /// Returns `true` if `kind` is a workflow execution event (46001–46012). /// These must not trigger workflows (prevents infinite loops). pub const fn is_workflow_execution_kind(kind: u32) -> bool { diff --git a/crates/sprout-db/src/lib.rs b/crates/sprout-db/src/lib.rs index da1eaa206..07cc9d4d7 100644 --- a/crates/sprout-db/src/lib.rs +++ b/crates/sprout-db/src/lib.rs @@ -1267,10 +1267,15 @@ impl Db { Ok(result.rows_affected()) } - // ── Addressable events ────────────────────────────────────────────────── + // ── Replaceable events ───────────────────────────────────────────────── - /// Replace an addressable event (NIP-33-like): soft-delete any existing - /// event with the same (kind, pubkey, channel_id) and insert the new one. + /// Atomically replace a replaceable event: NIP-16 kinds (0, 3, 41, 10000–19999) + /// and NIP-29 discovery state (39000–39002, called from side_effects.rs). + /// + /// Keeps only the event with the highest `created_at` per (kind, pubkey, channel_id). + /// Same-second ties are broken by lowest event `id` (NIP-16 deterministic ordering). + /// Returns `(event, false)` for stale writes and duplicate IDs — callers should + /// skip fan-out/dispatch when `was_inserted` is false. pub async fn replace_addressable_event( &self, event: &nostr::Event, @@ -1278,13 +1283,78 @@ impl Db { ) -> Result<(StoredEvent, bool)> { let kind_i32 = sprout_core::kind::event_kind_i32(event); let pubkey_bytes = event.pubkey.to_bytes(); + let created_at_secs = event.created_at.as_u64() as i64; + let created_at = chrono::DateTime::from_timestamp(created_at_secs, 0) + .ok_or(DbError::InvalidTimestamp(created_at_secs))?; + + // Stable advisory-lock key: hash (kind, pubkey, channel_id) to i64. + // Uses FNV-1a for determinism — Rust's DefaultHasher is NOT stable across processes. + let lock_key = { + let mut h: u64 = 0xcbf29ce484222325; // FNV offset basis + for b in kind_i32.to_le_bytes() { + h ^= b as u64; + h = h.wrapping_mul(0x100000001b3); // FNV prime + } + for b in pubkey_bytes.as_slice() { + h ^= *b as u64; + h = h.wrapping_mul(0x100000001b3); + } + if let Some(ch) = channel_id { + for b in ch.as_bytes() { + h ^= *b as u64; + h = h.wrapping_mul(0x100000001b3); + } + } + h as i64 + }; + let mut tx = self.pool.begin().await?; - // Soft-delete existing events with the same (kind, pubkey, channel_id). - // The idx_events_addressable index supports this lookup efficiently. + // Serialize all writers for the same (kind, pubkey, channel_id) tuple. + // Advisory lock is transaction-scoped — released on commit/rollback. + sqlx::query("SELECT pg_advisory_xact_lock($1)") + .bind(lock_key) + .execute(&mut *tx) + .await?; + + // Check for the newest existing event. ORDER BY + LIMIT 1 is defensive against + // historical data where prior bugs may have left multiple live rows. + let existing: Option<(chrono::DateTime, Vec)> = sqlx::query_as( + "SELECT created_at, id FROM events \ + WHERE kind = $1 AND pubkey = $2 \ + AND channel_id IS NOT DISTINCT FROM $3 \ + AND deleted_at IS NULL \ + ORDER BY created_at DESC, id ASC LIMIT 1", + ) + .bind(kind_i32) + .bind(pubkey_bytes.as_slice()) + .bind(channel_id) + .fetch_optional(&mut *tx) + .await?; + + // Stale-write protection: reject if incoming is not newer. + // NIP-16: created_at is second-resolution. On same-second tie, lowest + // event id (lexicographic) wins — deterministic across relays. + let incoming_id = event.id.as_bytes().as_slice(); + if let Some((existing_ts, existing_id)) = existing { + let dominated = created_at < existing_ts + || (created_at == existing_ts && incoming_id >= existing_id.as_slice()); + if dominated { + tx.rollback().await?; + let received_at = chrono::Utc::now(); + return Ok(( + StoredEvent::with_received_at(event.clone(), received_at, channel_id, false), + false, + )); + } + } + + // Soft-delete the old event (if any). IS NOT DISTINCT FROM for NULL safety. sqlx::query( "UPDATE events SET deleted_at = NOW() \ - WHERE kind = $1 AND pubkey = $2 AND channel_id = $3 AND deleted_at IS NULL", + WHERE kind = $1 AND pubkey = $2 \ + AND channel_id IS NOT DISTINCT FROM $3 \ + AND deleted_at IS NULL", ) .bind(kind_i32) .bind(pubkey_bytes.as_slice()) @@ -1292,11 +1362,51 @@ impl Db { .execute(&mut *tx) .await?; + // Insert the new event inside the same transaction. + let sig_bytes = event.sig.serialize(); + let tags_json = serde_json::to_value(&event.tags)?; + let received_at = chrono::Utc::now(); + + let insert_result = sqlx::query( + "INSERT INTO events (id, pubkey, created_at, kind, tags, content, sig, received_at, channel_id) \ + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9) \ + ON CONFLICT DO NOTHING", + ) + .bind(event.id.as_bytes().as_slice()) + .bind(pubkey_bytes.as_slice()) + .bind(created_at) + .bind(kind_i32) + .bind(&tags_json) + .bind(&event.content) + .bind(sig_bytes.as_slice()) + .bind(received_at) + .bind(channel_id) + .execute(&mut *tx) + .await?; + + let was_inserted = insert_result.rows_affected() > 0; + if !was_inserted { + // ON CONFLICT fired — the event ID already exists. Rollback the + // soft-delete so we don't lose the previous replaceable event. + tx.rollback().await?; + return Ok(( + StoredEvent::with_received_at(event.clone(), received_at, channel_id, false), + false, + )); + } + tx.commit().await?; - // Insert the new event (outside the tx — uses the standard path with - // dedup via ON CONFLICT DO NOTHING). - self.insert_event(event, channel_id).await + // Mentions are a denormalized index — safe outside the transaction. + // insert_event() normally handles this, but we inlined the INSERT above. + if let Err(e) = crate::insert_mentions(&self.pool, event, channel_id).await { + tracing::warn!(event_id = %event.id, "Failed to insert mentions: {e}"); + } + + Ok(( + StoredEvent::with_received_at(event.clone(), received_at, channel_id, true), + true, + )) } } diff --git a/crates/sprout-relay/src/api/search.rs b/crates/sprout-relay/src/api/search.rs index 9f4db51c1..63a83a6b6 100644 --- a/crates/sprout-relay/src/api/search.rs +++ b/crates/sprout-relay/src/api/search.rs @@ -48,13 +48,15 @@ pub async fn search_handler( .await .unwrap_or_default(); - // Build Typesense filter_by: channel_id:=[id1,id2,...] + // Build Typesense filter_by: channel_id:=[id1,id2,...] || global events let filter_by = if channel_ids.is_empty() { - // No accessible channels — return empty results immediately. - return Ok(Json(serde_json::json!({ "hits": [], "found": 0 }))); + Some("channel_id:=__global__".to_string()) } else { let ids: Vec = channel_ids.iter().map(|id| id.to_string()).collect(); - Some(format!("channel_id:=[{}]", ids.join(","))) + Some(format!( + "(channel_id:=[{}] || channel_id:=__global__)", + ids.join(",") + )) }; let search_query = SearchQuery { @@ -82,19 +84,15 @@ pub async fn search_handler( .map(|c| (c.id.to_string(), c.name)) .collect(); - // Filter out hits with no channel_id (spec requirement: "Exclude hits with channel_id: None"). - // This also prevents a deserialization mismatch — the desktop expects channel_id: String. + // Global events have channel_id: null — include them in results. let hits: Vec = search_result .hits .into_iter() - .filter(|hit| hit.channel_id.is_some()) .map(|hit| { - let channel_name = hit + let channel_name: Option<&String> = hit .channel_id .as_deref() - .and_then(|id| channel_name_map.get(id)) - .cloned() - .unwrap_or_default(); + .and_then(|id| channel_name_map.get(id)); serde_json::json!({ "event_id": hit.event_id, "content": hit.content, diff --git a/crates/sprout-relay/src/handlers/ingest.rs b/crates/sprout-relay/src/handlers/ingest.rs index 28630c6f1..c5e0ce9d0 100644 --- a/crates/sprout-relay/src/handlers/ingest.rs +++ b/crates/sprout-relay/src/handlers/ingest.rs @@ -12,14 +12,14 @@ use uuid::Uuid; use nostr::Event; use sprout_auth::Scope; use sprout_core::kind::{ - event_kind_u32, KIND_AUTH, KIND_CANVAS, KIND_DELETION, KIND_FORUM_COMMENT, KIND_FORUM_POST, - KIND_FORUM_VOTE, KIND_GIFT_WRAP, KIND_MEMBER_ADDED_NOTIFICATION, + event_kind_u32, KIND_AUTH, KIND_CANVAS, KIND_CONTACT_LIST, KIND_DELETION, KIND_FORUM_COMMENT, + KIND_FORUM_POST, KIND_FORUM_VOTE, KIND_GIFT_WRAP, KIND_MEMBER_ADDED_NOTIFICATION, KIND_MEMBER_REMOVED_NOTIFICATION, KIND_NIP29_CREATE_GROUP, KIND_NIP29_DELETE_EVENT, KIND_NIP29_DELETE_GROUP, KIND_NIP29_EDIT_METADATA, KIND_NIP29_JOIN_REQUEST, KIND_NIP29_LEAVE_REQUEST, KIND_NIP29_PUT_USER, KIND_NIP29_REMOVE_USER, KIND_PRESENCE_UPDATE, KIND_PROFILE, KIND_REACTION, KIND_STREAM_MESSAGE, KIND_STREAM_MESSAGE_BOOKMARKED, KIND_STREAM_MESSAGE_DIFF, KIND_STREAM_MESSAGE_EDIT, KIND_STREAM_MESSAGE_PINNED, - KIND_STREAM_MESSAGE_SCHEDULED, KIND_STREAM_MESSAGE_V2, KIND_STREAM_REMINDER, + KIND_STREAM_MESSAGE_SCHEDULED, KIND_STREAM_MESSAGE_V2, KIND_STREAM_REMINDER, KIND_TEXT_NOTE, }; use sprout_core::verification::verify_event; @@ -141,6 +141,8 @@ pub enum IngestError { fn required_scope_for_kind(kind: u32, event: &Event) -> Result { match kind { KIND_PROFILE => Ok(Scope::UsersWrite), + KIND_TEXT_NOTE => Ok(Scope::MessagesWrite), + KIND_CONTACT_LIST => Ok(Scope::UsersWrite), KIND_DELETION | KIND_REACTION | KIND_GIFT_WRAP @@ -779,7 +781,7 @@ pub async fn ingest_event( } // ── 5. Channel resolution ──────────────────────────────────────────── - let channel_id = if kind_u32 == KIND_REACTION { + let mut channel_id = if kind_u32 == KIND_REACTION { match derive_reaction_channel(&state.db, &event).await { ReactionChannelResult::Channel(ch_id) => Some(ch_id), ReactionChannelResult::NoChannel => None, @@ -840,6 +842,13 @@ pub async fn ingest_event( extract_channel_id(&event) }; + // ── 5b. Global-only kinds ignore h-tags ───────────────────────────── + // kind:0 (profile), kind:1 (text note), kind:3 (contact list) are always global. + // If a client includes a stray h-tag, ignore it — these kinds are never channel-scoped. + if matches!(kind_u32, KIND_PROFILE | KIND_TEXT_NOTE | KIND_CONTACT_LIST) { + channel_id = None; + } + // ── 6. h-tag requirement ───────────────────────────────────────────── if requires_h_channel_scope(kind_u32) && channel_id.is_none() { return Err(IngestError::Rejected( @@ -850,12 +859,13 @@ pub async fn ingest_event( // ── 7. Token channel access ────────────────────────────────────────── if let Some(ch_id) = channel_id { check_token_channel_access(&auth, ch_id).map_err(IngestError::AuthFailed)?; - } else if kind_u32 == KIND_NIP29_CREATE_GROUP && auth.channel_ids().is_some() { - // Channel-scoped tokens cannot create channels outside their scope. - // kind:9007 without an h-tag would auto-create a server-assigned UUID, - // bypassing the token's channel restriction. + } else if auth.channel_ids().is_some() { + // Channel-scoped tokens cannot publish global events — that would bypass + // the token's channel restriction. This covers kind:1 (global text notes), + // kind:3 (contact lists), kind:0 (profiles), and kind:9007 (create-group + // without an h-tag, which would auto-assign a server UUID). return Err(IngestError::AuthFailed( - "restricted: channel-scoped tokens must include an h tag for create-group".into(), + "restricted: channel-scoped tokens cannot publish global events".into(), )); } @@ -1220,11 +1230,12 @@ pub async fn ingest_event( }); } - let (stored_event, was_inserted) = if kind_u32 == KIND_PROFILE { - // kind:0 is replaceable — use addressable event storage. + let (stored_event, was_inserted) = if sprout_core::kind::is_replaceable(kind_u32) { + // NIP-16 replaceable event — atomic replace with stale-write protection. + // channel_id is None for global kinds (0, 1, 3) due to step 5b above. state .db - .replace_addressable_event(&event, None) + .replace_addressable_event(&event, channel_id) .await .map_err(|e| IngestError::Internal(format!("error: {e}")))? } else { diff --git a/crates/sprout-relay/src/handlers/req.rs b/crates/sprout-relay/src/handlers/req.rs index d08c42237..8dc6e61d6 100644 --- a/crates/sprout-relay/src/handlers/req.rs +++ b/crates/sprout-relay/src/handlers/req.rs @@ -242,17 +242,19 @@ async fn handle_search_req( conn: &ConnectionState, state: &AppState, ) { - if accessible_channels.is_empty() { - conn.send(RelayMessage::eose(sub_id)); - return; - } - let all_channels_filter = { - let ids: Vec = accessible_channels - .iter() - .map(|id| id.to_string()) - .collect(); - format!("channel_id:=[{}]", ids.join(",")) + if accessible_channels.is_empty() { + "channel_id:=__global__".to_string() + } else { + let ids: Vec = accessible_channels + .iter() + .map(|id| id.to_string()) + .collect(); + format!( + "(channel_id:=[{}] || channel_id:=__global__)", + ids.join(",") + ) + } }; let mut seen_ids: HashSet = HashSet::new(); @@ -354,7 +356,6 @@ async fn handle_search_req( let hit_ids: Vec> = search_result .hits .into_iter() - .filter(|h| h.channel_id.is_some()) .filter_map(|h| hex::decode(&h.event_id).ok()) .filter(|bytes| bytes.len() == 32) .collect(); diff --git a/crates/sprout-relay/src/handlers/side_effects.rs b/crates/sprout-relay/src/handlers/side_effects.rs index 338b0c727..f84907852 100644 --- a/crates/sprout-relay/src/handlers/side_effects.rs +++ b/crates/sprout-relay/src/handlers/side_effects.rs @@ -461,12 +461,14 @@ async fn emit_addressable_discovery_event( .sign_with_keys(&state.relay_keypair) .map_err(|e| anyhow::anyhow!("failed to sign kind:{kind}: {e}"))?; - let (stored, _) = state + let (stored, was_inserted) = state .db .replace_addressable_event(&event, Some(channel_id)) .await?; - let kind_u32 = event_kind_u32(&stored.event); - dispatch_persistent_event(state, &stored, kind_u32, relay_pubkey_hex).await; + if was_inserted { + let kind_u32 = event_kind_u32(&stored.event); + dispatch_persistent_event(state, &stored, kind_u32, relay_pubkey_hex).await; + } Ok(()) } diff --git a/crates/sprout-relay/src/nip11.rs b/crates/sprout-relay/src/nip11.rs index 179e37ab7..79a4a5a7f 100644 --- a/crates/sprout-relay/src/nip11.rs +++ b/crates/sprout-relay/src/nip11.rs @@ -56,7 +56,7 @@ impl RelayInfo { description: "Sprout — private team communication relay".to_string(), pubkey: None, contact: None, - supported_nips: vec![1, 10, 11, 17, 25, 29, 42, 50], + supported_nips: vec![1, 2, 10, 11, 17, 25, 29, 42, 50], software: "https://github.com/sprout-rs/sprout".to_string(), version: env!("CARGO_PKG_VERSION").to_string(), limitation: Some(RelayLimitation { diff --git a/crates/sprout-search/src/index.rs b/crates/sprout-search/src/index.rs index 6eba8884a..ffc010cb8 100644 --- a/crates/sprout-search/src/index.rs +++ b/crates/sprout-search/src/index.rs @@ -29,7 +29,19 @@ pub fn event_to_document(event: &StoredEvent) -> Result { }) .collect(); - let channel_id = event.channel_id.as_ref().map(|id| id.to_string()); + // Global events use a sentinel value instead of NULL/absent. Typesense 27.1's + // `__missing__` filter does not reliably match absent optional fields, so we use + // an explicit `__global__` sentinel that can be matched with `channel_id:=__global__`. + // NOTE: Historical docs indexed before this change have channel_id absent/null and + // won't match the sentinel filter. A full reindex (`just reindex-search`) is needed + // after deploy to backfill. Pre-existing global events (kind:0 only) were already + // excluded from search results by the old `.filter(|h| h.channel_id.is_some())`, so + // this is not a regression — those docs were never returned. + let channel_id_val = event + .channel_id + .as_ref() + .map(|id| id.to_string()) + .unwrap_or_else(|| "__global__".to_string()); let doc = json!({ "id": nostr_event.id.to_string(), @@ -37,7 +49,7 @@ pub fn event_to_document(event: &StoredEvent) -> Result { // Cast to i32 for Typesense schema (int32 field). nostr Kind is u16; all Sprout kinds fit in i32. "kind": event_kind_i32(nostr_event), "pubkey": nostr_event.pubkey.to_string(), - "channel_id": channel_id, + "channel_id": channel_id_val, "created_at": nostr_event.created_at.as_u64() as i64, "tags_flat": tags_flat, }); @@ -251,10 +263,10 @@ mod tests { } #[test] - fn document_no_channel_id_is_null() { + fn document_no_channel_id_uses_global_sentinel() { let stored = make_stored_event("no channel", Kind::TextNote, None); let doc = event_to_document(&stored).unwrap(); - assert!(doc["channel_id"].is_null()); + assert_eq!(doc["channel_id"].as_str().unwrap(), "__global__"); } #[test] diff --git a/crates/sprout-search/src/query.rs b/crates/sprout-search/src/query.rs index 21dbe94e9..b34cc0502 100644 --- a/crates/sprout-search/src/query.rs +++ b/crates/sprout-search/src/query.rs @@ -182,7 +182,7 @@ fn parse_response(ts_resp: TypesenseSearchResponse) -> Result, + pub channel_name: Option, pub created_at: u64, pub score: f64, } diff --git a/desktop/src/app/AppShell.tsx b/desktop/src/app/AppShell.tsx index eab4a3da8..836420559 100644 --- a/desktop/src/app/AppShell.tsx +++ b/desktop/src/app/AppShell.tsx @@ -393,11 +393,13 @@ export function AppShell() { pubkey: hit.pubkey, created_at: hit.createdAt, kind: hit.kind, - tags: [["h", hit.channelId]], + tags: hit.channelId ? [["h", hit.channelId]] : [], content: hit.content, sig: "", }); - void handleOpenChannel(hit.channelId); + if (hit.channelId) { + void handleOpenChannel(hit.channelId); + } void getEventById(hit.eventId) .then((event) => { diff --git a/desktop/src/features/search/ui/SearchDialog.tsx b/desktop/src/features/search/ui/SearchDialog.tsx index ad7316862..bda0d534c 100644 --- a/desktop/src/features/search/ui/SearchDialog.tsx +++ b/desktop/src/features/search/ui/SearchDialog.tsx @@ -317,7 +317,9 @@ export function SearchDialog({
{results.map((hit, index) => { - const channel = channelLookup.get(hit.channelId); + const channel = hit.channelId + ? channelLookup.get(hit.channelId) + : undefined; const authorLabel = resolveUserLabel({ pubkey: hit.pubkey, currentPubkey, diff --git a/desktop/src/shared/api/tauri.ts b/desktop/src/shared/api/tauri.ts index 434c659ac..27a8f8efb 100644 --- a/desktop/src/shared/api/tauri.ts +++ b/desktop/src/shared/api/tauri.ts @@ -169,8 +169,8 @@ type RawSearchHit = { content: string; kind: number; pubkey: string; - channel_id: string; - channel_name: string; + channel_id: string | null; + channel_name: string | null; created_at: number; score: number; }; diff --git a/desktop/src/shared/api/types.ts b/desktop/src/shared/api/types.ts index 034ea0ade..f85c8ebe7 100644 --- a/desktop/src/shared/api/types.ts +++ b/desktop/src/shared/api/types.ts @@ -213,8 +213,8 @@ export type SearchHit = { content: string; kind: number; pubkey: string; - channelId: string; - channelName: string; + channelId: string | null; + channelName: string | null; createdAt: number; score: number; }; diff --git a/desktop/src/testing/e2eBridge.ts b/desktop/src/testing/e2eBridge.ts index d73b11f2f..f3a01c391 100644 --- a/desktop/src/testing/e2eBridge.ts +++ b/desktop/src/testing/e2eBridge.ts @@ -165,8 +165,8 @@ type RawSearchHit = { content: string; kind: number; pubkey: string; - channel_id: string; - channel_name: string; + channel_id: string | null; + channel_name: string | null; created_at: number; score: number; }; @@ -2955,7 +2955,7 @@ async function handleSearchMessages( return ( hit.content.toLowerCase().includes(query) || - hit.channel_name.toLowerCase().includes(query) + (hit.channel_name?.toLowerCase().includes(query) ?? false) ); }) .slice(0, limit); From 870eb960e0a6e6b3e2999ddfa215b2f74f61429b Mon Sep 17 00:00:00 2001 From: Tyler Longwell Date: Sun, 5 Apr 2026 16:43:13 -0400 Subject: [PATCH 2/6] =?UTF-8?q?chore:=20crossfire=20review=20nits=20?= =?UTF-8?q?=E2=80=94=20NIP-16=20in=20supported=5Fnips,=20kind:41=20constan?= =?UTF-8?q?t,=20doc=20comments?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Add NIP 16 to supported_nips (was missing despite commit message claiming it) - Add KIND_CHANNEL_METADATA constant for kind:41 (was a bare literal in is_replaceable) - Add NIP-33 exclusion comment on is_replaceable() to prevent future confusion - Add advisory lock collision semantics comment (extra serialization, not incorrectness) --- crates/sprout-core/src/kind.rs | 7 ++++++- crates/sprout-db/src/lib.rs | 1 + crates/sprout-relay/src/nip11.rs | 2 +- 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/crates/sprout-core/src/kind.rs b/crates/sprout-core/src/kind.rs index d64b99fa3..3e3175864 100644 --- a/crates/sprout-core/src/kind.rs +++ b/crates/sprout-core/src/kind.rs @@ -11,6 +11,8 @@ pub const KIND_PROFILE: u32 = 0; pub const KIND_TEXT_NOTE: u32 = 1; /// NIP-02: Contact list / follow list. pub const KIND_CONTACT_LIST: u32 = 3; +/// NIP-01: Channel metadata (replaceable). Not used by Sprout today. +pub const KIND_CHANNEL_METADATA: u32 = 41; /// NIP-09: Event deletion request. pub const KIND_DELETION: u32 = 5; /// NIP-25: Content is emoji char or `+`/`-`. @@ -218,6 +220,7 @@ pub const ALL_KINDS: &[u32] = &[ KIND_PROFILE, KIND_TEXT_NOTE, KIND_CONTACT_LIST, + KIND_CHANNEL_METADATA, KIND_DELETION, KIND_REACTION, KIND_GIFT_WRAP, @@ -304,8 +307,10 @@ pub const fn is_ephemeral(kind: u32) -> bool { } /// Returns `true` if `kind` is replaceable (NIP-01: kinds 0, 3, 41, 10000–19999). +/// NIP-33 parameterized-replaceable kinds (30000–39999) use a different replacement +/// key (includes `d`-tag) and are handled separately via `replace_addressable_event`. pub const fn is_replaceable(kind: u32) -> bool { - matches!(kind, 0 | 3 | 41 | 10000..=19999) + matches!(kind, 0 | 3 | KIND_CHANNEL_METADATA | 10000..=19999) } /// Returns `true` if `kind` is a workflow execution event (46001–46012). diff --git a/crates/sprout-db/src/lib.rs b/crates/sprout-db/src/lib.rs index 07cc9d4d7..a3b64000b 100644 --- a/crates/sprout-db/src/lib.rs +++ b/crates/sprout-db/src/lib.rs @@ -1289,6 +1289,7 @@ impl Db { // Stable advisory-lock key: hash (kind, pubkey, channel_id) to i64. // Uses FNV-1a for determinism — Rust's DefaultHasher is NOT stable across processes. + // Collisions cause extra serialization, not incorrect behavior. let lock_key = { let mut h: u64 = 0xcbf29ce484222325; // FNV offset basis for b in kind_i32.to_le_bytes() { diff --git a/crates/sprout-relay/src/nip11.rs b/crates/sprout-relay/src/nip11.rs index 79a4a5a7f..f7ab0e9ec 100644 --- a/crates/sprout-relay/src/nip11.rs +++ b/crates/sprout-relay/src/nip11.rs @@ -56,7 +56,7 @@ impl RelayInfo { description: "Sprout — private team communication relay".to_string(), pubkey: None, contact: None, - supported_nips: vec![1, 2, 10, 11, 17, 25, 29, 42, 50], + supported_nips: vec![1, 2, 10, 11, 16, 17, 25, 29, 42, 50], software: "https://github.com/sprout-rs/sprout".to_string(), version: env!("CARGO_PKG_VERSION").to_string(), limitation: Some(RelayLimitation { From 8b780bccaf40a78e6cd8cbda5fb4a7c5d4b812f9 Mon Sep 17 00:00:00 2001 From: Tyler Longwell Date: Sun, 5 Apr 2026 16:43:13 -0400 Subject: [PATCH 3/6] feat: NIP-33 parameterized replaceable events (d-tag keyed replacement) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add correct NIP-33 support for parameterized replaceable events (kind 30000-39999), where the latest event per (author, kind, d_tag) tuple wins. Storage plumbing that unblocks long-form articles (kind:30023) and other user-owned addressable content. Schema: - d_tag TEXT column on events table (nullable) - idx_events_parameterized partial index on (kind, pubkey, d_tag, deleted_at) WHERE d_tag IS NOT NULL Storage: - extract_d_tag() at the shared insert layer — NIP-33 kinds get the d tag value (or empty string per spec), all others get NULL - replace_parameterized_event() with pg_advisory_xact_lock, stale-write protection, and transactional insert (prevents delete-without-reinsert) - d_tag wired through all INSERT paths including replace_addressable_event Ingest: - Route parameterized replaceable kinds through the new replacement function between the existing replaceable and default paths Query: - Push #d tag filter into SQL for NIP-33-only kind filters, preventing under-fetch on authors + kinds + #d lookups under LIMIT pressure - Gated to NIP-33 kinds only so non-NIP-33 events (d_tag=NULL) are not silently excluded Backfill: - Idempotent startup backfill in relay main (no-ops when populated) - scripts/backfill-d-tag.sql for manual use - Integrated into dev-setup.sh after pgschema apply NIP-29 group metadata (kind 39000-39002) continues using replace_addressable_event() via side_effects.rs — unchanged. Those events get d_tag populated via the shared insert layer for consistency. --- crates/sprout-core/src/kind.rs | 39 ++++- crates/sprout-db/src/event.rs | 140 +++++++++++++++++- crates/sprout-db/src/lib.rs | 164 ++++++++++++++++++++- crates/sprout-relay/src/handlers/ingest.rs | 25 +++- crates/sprout-relay/src/handlers/req.rs | 59 ++++++++ crates/sprout-relay/src/main.rs | 8 + crates/sprout-relay/src/nip11.rs | 2 +- schema/schema.sql | 2 + scripts/backfill-d-tag.sql | 17 +++ scripts/dev-setup.sh | 11 ++ 10 files changed, 450 insertions(+), 17 deletions(-) create mode 100644 scripts/backfill-d-tag.sql diff --git a/crates/sprout-core/src/kind.rs b/crates/sprout-core/src/kind.rs index d64b99fa3..7b62432a4 100644 --- a/crates/sprout-core/src/kind.rs +++ b/crates/sprout-core/src/kind.rs @@ -11,6 +11,8 @@ pub const KIND_PROFILE: u32 = 0; pub const KIND_TEXT_NOTE: u32 = 1; /// NIP-02: Contact list / follow list. pub const KIND_CONTACT_LIST: u32 = 3; +/// NIP-01: Channel metadata (replaceable). Not used by Sprout today. +pub const KIND_CHANNEL_METADATA: u32 = 41; /// NIP-09: Event deletion request. pub const KIND_DELETION: u32 = 5; /// NIP-25: Content is emoji char or `+`/`-`. @@ -60,6 +62,11 @@ pub const KIND_NIP29_GROUP_MEMBERS: u32 = 39002; /// NIP-29: Addressable group roles definition. pub const KIND_NIP29_GROUP_ROLES: u32 = 39003; +/// Lower bound of the NIP-33 parameterized replaceable range (30000–39999). +pub const PARAM_REPLACEABLE_KIND_MIN: u32 = 30000; +/// Upper bound of the NIP-33 parameterized replaceable range (30000–39999). +pub const PARAM_REPLACEABLE_KIND_MAX: u32 = 39999; + /// Lower bound of the ephemeral event range (20000–29999). Never stored. pub const EPHEMERAL_KIND_MIN: u32 = 20000; /// Upper bound of the ephemeral event range (20000–29999). Never stored. @@ -218,6 +225,7 @@ pub const ALL_KINDS: &[u32] = &[ KIND_PROFILE, KIND_TEXT_NOTE, KIND_CONTACT_LIST, + KIND_CHANNEL_METADATA, KIND_DELETION, KIND_REACTION, KIND_GIFT_WRAP, @@ -304,8 +312,17 @@ pub const fn is_ephemeral(kind: u32) -> bool { } /// Returns `true` if `kind` is replaceable (NIP-01: kinds 0, 3, 41, 10000–19999). +/// NIP-33 parameterized-replaceable kinds (30000–39999) use a different replacement +/// key (includes `d`-tag) and are handled separately via `replace_parameterized_event`. pub const fn is_replaceable(kind: u32) -> bool { - matches!(kind, 0 | 3 | 41 | 10000..=19999) + matches!(kind, 0 | 3 | KIND_CHANNEL_METADATA | 10000..=19999) +} + +/// Returns `true` if `kind` is in the NIP-33 parameterized replaceable range (30000–39999). +/// +/// These events are keyed by `(pubkey, kind, d_tag)` — the latest `created_at` wins. +pub const fn is_parameterized_replaceable(kind: u32) -> bool { + kind >= PARAM_REPLACEABLE_KIND_MIN && kind <= PARAM_REPLACEABLE_KIND_MAX } /// Returns `true` if `kind` is a workflow execution event (46001–46012). @@ -343,4 +360,24 @@ mod tests { assert!(seen.insert(k), "duplicate kind value: {k}"); } } + + #[test] + fn parameterized_replaceable_range() { + assert!(!is_parameterized_replaceable(29999)); + assert!(is_parameterized_replaceable(30000)); + assert!(is_parameterized_replaceable(30023)); // NIP-23 long-form + assert!(is_parameterized_replaceable(39000)); // NIP-29 group metadata + assert!(is_parameterized_replaceable(39999)); + assert!(!is_parameterized_replaceable(40000)); + } + + #[test] + fn replaceable_and_parameterized_are_disjoint() { + for kind in 0..=65535u32 { + assert!( + !(is_replaceable(kind) && is_parameterized_replaceable(kind)), + "kind {kind} is both replaceable and parameterized replaceable" + ); + } + } } diff --git a/crates/sprout-db/src/event.rs b/crates/sprout-db/src/event.rs index b7cf9b12c..356899a10 100644 --- a/crates/sprout-db/src/event.rs +++ b/crates/sprout-db/src/event.rs @@ -9,7 +9,7 @@ use nostr::Event; use sqlx::{PgPool, QueryBuilder, Row}; use uuid::Uuid; -use sprout_core::kind::{event_kind_i32, is_ephemeral, KIND_AUTH}; +use sprout_core::kind::{event_kind_i32, is_ephemeral, is_parameterized_replaceable, KIND_AUTH}; use sprout_core::StoredEvent; use crate::error::{DbError, Result}; @@ -34,6 +34,34 @@ pub struct EventQuery { /// Restrict to events with a `p` tag mentioning this hex pubkey. /// Joins against `event_mentions` table (indexed). pub p_tag_hex: Option, + /// Restrict to events with this exact `d_tag` value (NIP-33). + /// Pushed into SQL via the `idx_events_parameterized` index. + pub d_tag: Option, +} + +/// Extract the `d_tag` value for storage. +/// +/// For NIP-33 parameterized replaceable events (kind 30000–39999): returns the first +/// `d` tag's value, or `""` if no `d` tag is present (per NIP-33 spec). +/// For all other events: returns `None` (column stays NULL). +pub fn extract_d_tag(event: &Event) -> Option { + let kind_u32 = event.kind.as_u16() as u32; + if !is_parameterized_replaceable(kind_u32) { + return None; + } + let val = event + .tags + .iter() + .find_map(|tag| { + let parts = tag.as_slice(); + if parts.len() >= 2 && parts[0] == "d" { + Some(parts[1].to_string()) + } else { + None + } + }) + .unwrap_or_default(); // Missing d tag → empty string per NIP-33 + Some(val) } /// Insert a Nostr event. Rejects AUTH and ephemeral kinds. @@ -64,10 +92,11 @@ pub async fn insert_event( let created_at = DateTime::from_timestamp(created_at_secs, 0) .ok_or(DbError::InvalidTimestamp(created_at_secs))?; let received_at = Utc::now(); + let d_tag = extract_d_tag(event); let result = sqlx::query( r#" - INSERT INTO events (id, pubkey, created_at, kind, tags, content, sig, received_at, channel_id) - VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9) + INSERT INTO events (id, pubkey, created_at, kind, tags, content, sig, received_at, channel_id, d_tag) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10) ON CONFLICT DO NOTHING "#, ) @@ -80,6 +109,7 @@ pub async fn insert_event( .bind(sig_bytes.as_slice()) .bind(received_at) .bind(channel_id) + .bind(d_tag.as_deref()) .execute(pool) .await?; @@ -152,6 +182,11 @@ pub async fn query_events(pool: &PgPool, q: &EventQuery) -> Result) -> nostr::Event { + let keys = Keys::generate(); + EventBuilder::new(Kind::Custom(kind), "test", tags) + .sign_with_keys(&keys) + .expect("sign") + } + + #[test] + fn extract_d_tag_from_nip33_event() { + let event = make_event_with_kind_and_tags( + 30023, + vec![Tag::parse(&["d", "my-article-slug"]).unwrap()], + ); + assert_eq!(extract_d_tag(&event), Some("my-article-slug".to_string())); + } + + #[test] + fn extract_d_tag_first_d_wins() { + let event = make_event_with_kind_and_tags( + 30023, + vec![ + Tag::parse(&["d", "first"]).unwrap(), + Tag::parse(&["d", "second"]).unwrap(), + ], + ); + assert_eq!(extract_d_tag(&event), Some("first".to_string())); + } + + #[test] + fn extract_d_tag_missing_becomes_empty_string() { + // NIP-33: "if there is no d tag, the d tag is considered to be ''" + let event = + make_event_with_kind_and_tags(30023, vec![Tag::parse(&["p", "abc123"]).unwrap()]); + assert_eq!(extract_d_tag(&event), Some(String::new())); + } + + #[test] + fn extract_d_tag_empty_value_preserved() { + let event = make_event_with_kind_and_tags(30023, vec![Tag::parse(&["d", ""]).unwrap()]); + assert_eq!(extract_d_tag(&event), Some(String::new())); + } + + #[test] + fn extract_d_tag_non_nip33_returns_none() { + // kind:1 (text note) — not parameterized replaceable + let event = make_event_with_kind_and_tags( + 1, + vec![Tag::parse(&["d", "should-be-ignored"]).unwrap()], + ); + assert_eq!(extract_d_tag(&event), None); + } + + #[test] + fn extract_d_tag_nip29_group_metadata() { + // kind:39000 is in the 30000–39999 range — d_tag should be extracted + let event = + make_event_with_kind_and_tags(39000, vec![Tag::parse(&["d", "group-id"]).unwrap()]); + assert_eq!(extract_d_tag(&event), Some("group-id".to_string())); + } + + #[test] + fn extract_d_tag_boundary_kinds() { + // kind:29999 — just below range + let below = make_event_with_kind_and_tags(29999, vec![Tag::parse(&["d", "val"]).unwrap()]); + assert_eq!(extract_d_tag(&below), None); + + // kind:30000 — lower bound + let lower = make_event_with_kind_and_tags(30000, vec![Tag::parse(&["d", "val"]).unwrap()]); + assert_eq!(extract_d_tag(&lower), Some("val".to_string())); + + // kind:39999 — upper bound + let upper = make_event_with_kind_and_tags(39999, vec![Tag::parse(&["d", "val"]).unwrap()]); + assert_eq!(extract_d_tag(&upper), Some("val".to_string())); + + // kind:40000 — just above range + let above = make_event_with_kind_and_tags(40000, vec![Tag::parse(&["d", "val"]).unwrap()]); + assert_eq!(extract_d_tag(&above), None); + } + + #[test] + fn extract_d_tag_single_element_d_tag_ignored() { + // A d tag with only one element (no value) should not match — parts.len() < 2 + let event = make_event_with_kind_and_tags(30023, vec![Tag::parse(&["d"]).unwrap()]); + // No d tag with a value → empty string per NIP-33 + assert_eq!(extract_d_tag(&event), Some(String::new())); + } +} diff --git a/crates/sprout-db/src/lib.rs b/crates/sprout-db/src/lib.rs index 07cc9d4d7..359a2bdb4 100644 --- a/crates/sprout-db/src/lib.rs +++ b/crates/sprout-db/src/lib.rs @@ -1179,6 +1179,25 @@ impl Db { partition::ensure_future_partitions(&self.pool, months_ahead).await } + /// Backfill `d_tag` for existing NIP-33 events (kind 30000–39999) that have `d_tag IS NULL`. + /// + /// Idempotent — safe to call on every startup. No-ops when all rows are already populated. + /// Runs a single UPDATE touching only NIP-33 rows with NULL d_tag. + pub async fn backfill_d_tags(&self) -> Result { + let result = sqlx::query( + "UPDATE events \ + SET d_tag = COALESCE( \ + (SELECT elem->>1 FROM jsonb_array_elements(tags) AS elem \ + WHERE elem->>0 = 'd' LIMIT 1), \ + '' \ + ) \ + WHERE kind BETWEEN 30000 AND 39999 AND d_tag IS NULL", + ) + .execute(&self.pool) + .await?; + Ok(result.rows_affected()) + } + // ── Pubkey Allowlist ───────────────────────────────────────────────────── /// Check if a pubkey is in the allowlist. @@ -1289,6 +1308,7 @@ impl Db { // Stable advisory-lock key: hash (kind, pubkey, channel_id) to i64. // Uses FNV-1a for determinism — Rust's DefaultHasher is NOT stable across processes. + // Collisions cause extra serialization, not incorrect behavior. let lock_key = { let mut h: u64 = 0xcbf29ce484222325; // FNV offset basis for b in kind_i32.to_le_bytes() { @@ -1366,10 +1386,11 @@ impl Db { let sig_bytes = event.sig.serialize(); let tags_json = serde_json::to_value(&event.tags)?; let received_at = chrono::Utc::now(); + let d_tag = crate::event::extract_d_tag(event); let insert_result = sqlx::query( - "INSERT INTO events (id, pubkey, created_at, kind, tags, content, sig, received_at, channel_id) \ - VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9) \ + "INSERT INTO events (id, pubkey, created_at, kind, tags, content, sig, received_at, channel_id, d_tag) \ + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10) \ ON CONFLICT DO NOTHING", ) .bind(event.id.as_bytes().as_slice()) @@ -1381,6 +1402,7 @@ impl Db { .bind(sig_bytes.as_slice()) .bind(received_at) .bind(channel_id) + .bind(d_tag.as_deref()) .execute(&mut *tx) .await?; @@ -1408,6 +1430,144 @@ impl Db { true, )) } + + /// Atomically replace a NIP-33 parameterized replaceable event (kind 30000–39999). + /// + /// Keeps only the event with the highest `created_at` per `(kind, pubkey, d_tag)`. + /// Same-second ties are broken by lowest event `id` (deterministic ordering). + /// The entire check → soft-delete → insert runs in a single transaction with + /// an advisory lock to prevent concurrent-insert races. + /// + /// **Channel policy:** NIP-33 replacement keys on `(kind, pubkey, d_tag)` globally — + /// `channel_id` is NOT part of the replacement key. This matches the Nostr spec: + /// an author's parameterized replaceable event is a single global resource identified + /// by its d-tag, regardless of which channel it was submitted to. The `channel_id` + /// parameter is stored on the new row for query scoping but does not affect replacement. + /// + /// Note: `replace_addressable_event()` keys on `channel_id` because it serves + /// relay-signed NIP-29 group metadata (kind 39000–39002) where the relay is the + /// author and channel_id distinguishes groups. User-submitted NIP-33 events use + /// this function instead, where the author's pubkey + d-tag is the natural key. + pub async fn replace_parameterized_event( + &self, + event: &nostr::Event, + d_tag: &str, + channel_id: Option, + ) -> Result<(StoredEvent, bool)> { + let kind_i32 = sprout_core::kind::event_kind_i32(event); + let pubkey_bytes = event.pubkey.to_bytes(); + let created_at_secs = event.created_at.as_u64() as i64; + let created_at = chrono::DateTime::from_timestamp(created_at_secs, 0) + .ok_or(DbError::InvalidTimestamp(created_at_secs))?; + + // Stable advisory-lock key: FNV-1a over (kind, pubkey, d_tag). + // Same algorithm as replace_addressable_event — deterministic across processes. + let lock_key = { + let mut h: u64 = 0xcbf29ce484222325; // FNV offset basis + for b in kind_i32.to_le_bytes() { + h ^= b as u64; + h = h.wrapping_mul(0x100000001b3); + } + for b in pubkey_bytes.as_slice() { + h ^= *b as u64; + h = h.wrapping_mul(0x100000001b3); + } + for b in d_tag.as_bytes() { + h ^= *b as u64; + h = h.wrapping_mul(0x100000001b3); + } + h as i64 + }; + + let mut tx = self.pool.begin().await?; + + sqlx::query("SELECT pg_advisory_xact_lock($1)") + .bind(lock_key) + .execute(&mut *tx) + .await?; + + // Check for existing event with same (kind, pubkey, d_tag). + let existing: Option<(chrono::DateTime, Vec)> = sqlx::query_as( + "SELECT created_at, id FROM events \ + WHERE kind = $1 AND pubkey = $2 AND d_tag = $3 AND deleted_at IS NULL \ + ORDER BY created_at DESC, id ASC LIMIT 1", + ) + .bind(kind_i32) + .bind(pubkey_bytes.as_slice()) + .bind(d_tag) + .fetch_optional(&mut *tx) + .await?; + + // Stale-write protection: reject if incoming is not newer. + let incoming_id = event.id.as_bytes().as_slice(); + if let Some((existing_ts, existing_id)) = existing { + let dominated = created_at < existing_ts + || (created_at == existing_ts && incoming_id >= existing_id.as_slice()); + if dominated { + tx.rollback().await?; + let received_at = chrono::Utc::now(); + return Ok(( + StoredEvent::with_received_at(event.clone(), received_at, channel_id, false), + false, + )); + } + + // Soft-delete the older event(s). + sqlx::query( + "UPDATE events SET deleted_at = NOW() \ + WHERE kind = $1 AND pubkey = $2 AND d_tag = $3 AND deleted_at IS NULL", + ) + .bind(kind_i32) + .bind(pubkey_bytes.as_slice()) + .bind(d_tag) + .execute(&mut *tx) + .await?; + } + + // Insert the new event inside the transaction. + let sig_bytes = event.sig.serialize(); + let tags_json = serde_json::to_value(&event.tags)?; + let received_at = chrono::Utc::now(); + + let insert_result = sqlx::query( + "INSERT INTO events (id, pubkey, created_at, kind, tags, content, sig, received_at, channel_id, d_tag) \ + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10) \ + ON CONFLICT DO NOTHING", + ) + .bind(event.id.as_bytes().as_slice()) + .bind(pubkey_bytes.as_slice()) + .bind(created_at) + .bind(kind_i32) + .bind(&tags_json) + .bind(&event.content) + .bind(sig_bytes.as_slice()) + .bind(received_at) + .bind(channel_id) + .bind(d_tag) + .execute(&mut *tx) + .await?; + + let was_inserted = insert_result.rows_affected() > 0; + if !was_inserted { + tx.rollback().await?; + return Ok(( + StoredEvent::with_received_at(event.clone(), received_at, channel_id, false), + false, + )); + } + + tx.commit().await?; + + // Mentions are a denormalized index — safe outside the transaction. + if let Err(e) = crate::insert_mentions(&self.pool, event, channel_id).await { + tracing::warn!(event_id = %event.id, "Failed to insert mentions: {e}"); + } + + Ok(( + StoredEvent::with_received_at(event.clone(), received_at, channel_id, true), + true, + )) + } } /// A full API token record. diff --git a/crates/sprout-relay/src/handlers/ingest.rs b/crates/sprout-relay/src/handlers/ingest.rs index c5e0ce9d0..d7733b389 100644 --- a/crates/sprout-relay/src/handlers/ingest.rs +++ b/crates/sprout-relay/src/handlers/ingest.rs @@ -12,14 +12,15 @@ use uuid::Uuid; use nostr::Event; use sprout_auth::Scope; use sprout_core::kind::{ - event_kind_u32, KIND_AUTH, KIND_CANVAS, KIND_CONTACT_LIST, KIND_DELETION, KIND_FORUM_COMMENT, - KIND_FORUM_POST, KIND_FORUM_VOTE, KIND_GIFT_WRAP, KIND_MEMBER_ADDED_NOTIFICATION, - KIND_MEMBER_REMOVED_NOTIFICATION, KIND_NIP29_CREATE_GROUP, KIND_NIP29_DELETE_EVENT, - KIND_NIP29_DELETE_GROUP, KIND_NIP29_EDIT_METADATA, KIND_NIP29_JOIN_REQUEST, - KIND_NIP29_LEAVE_REQUEST, KIND_NIP29_PUT_USER, KIND_NIP29_REMOVE_USER, KIND_PRESENCE_UPDATE, - KIND_PROFILE, KIND_REACTION, KIND_STREAM_MESSAGE, KIND_STREAM_MESSAGE_BOOKMARKED, - KIND_STREAM_MESSAGE_DIFF, KIND_STREAM_MESSAGE_EDIT, KIND_STREAM_MESSAGE_PINNED, - KIND_STREAM_MESSAGE_SCHEDULED, KIND_STREAM_MESSAGE_V2, KIND_STREAM_REMINDER, KIND_TEXT_NOTE, + event_kind_u32, is_parameterized_replaceable, KIND_AUTH, KIND_CANVAS, KIND_CONTACT_LIST, + KIND_DELETION, KIND_FORUM_COMMENT, KIND_FORUM_POST, KIND_FORUM_VOTE, KIND_GIFT_WRAP, + KIND_MEMBER_ADDED_NOTIFICATION, KIND_MEMBER_REMOVED_NOTIFICATION, KIND_NIP29_CREATE_GROUP, + KIND_NIP29_DELETE_EVENT, KIND_NIP29_DELETE_GROUP, KIND_NIP29_EDIT_METADATA, + KIND_NIP29_JOIN_REQUEST, KIND_NIP29_LEAVE_REQUEST, KIND_NIP29_PUT_USER, KIND_NIP29_REMOVE_USER, + KIND_PRESENCE_UPDATE, KIND_PROFILE, KIND_REACTION, KIND_STREAM_MESSAGE, + KIND_STREAM_MESSAGE_BOOKMARKED, KIND_STREAM_MESSAGE_DIFF, KIND_STREAM_MESSAGE_EDIT, + KIND_STREAM_MESSAGE_PINNED, KIND_STREAM_MESSAGE_SCHEDULED, KIND_STREAM_MESSAGE_V2, + KIND_STREAM_REMINDER, KIND_TEXT_NOTE, }; use sprout_core::verification::verify_event; @@ -1238,6 +1239,14 @@ pub async fn ingest_event( .replace_addressable_event(&event, channel_id) .await .map_err(|e| IngestError::Internal(format!("error: {e}")))? + } else if is_parameterized_replaceable(kind_u32) { + // NIP-33 parameterized replaceable — keyed by (kind, pubkey, d_tag). + let d_tag = sprout_db::event::extract_d_tag(&event).unwrap_or_default(); + state + .db + .replace_parameterized_event(&event, &d_tag, channel_id) + .await + .map_err(|e| IngestError::Internal(format!("error: {e}")))? } else { let thread_params = thread_meta.as_ref().map(|m| m.as_params()); match state diff --git a/crates/sprout-relay/src/handlers/req.rs b/crates/sprout-relay/src/handlers/req.rs index 8dc6e61d6..06b4a620a 100644 --- a/crates/sprout-relay/src/handlers/req.rs +++ b/crates/sprout-relay/src/handlers/req.rs @@ -469,6 +469,33 @@ fn filter_to_query_params(filter: &Filter, channel_id: Option) -> Ev } }); + // Push single-value #d tag into SQL via the d_tag column (NIP-33). + // Critical for parameterized replaceable lookups (authors + kinds + #d) + // where many events from the same author would push the target past LIMIT. + // + // Only push when the filter exclusively targets NIP-33 kinds (30000–39999), + // because `d_tag` is only populated for those kinds. Non-NIP-33 events have + // `d_tag = NULL`, so pushing `AND d_tag = $N` for a mixed-kind or kindless + // filter would silently exclude non-NIP-33 rows that match via their tags. + let filter_is_nip33_only = kinds.as_ref().is_some_and(|ks| { + !ks.is_empty() + && ks + .iter() + .all(|&k| sprout_core::kind::is_parameterized_replaceable(k as u32)) + }); + let d_tag_key = nostr::SingleLetterTag::lowercase(nostr::Alphabet::D); + let d_tag = if filter_is_nip33_only { + filter.generic_tags.get(&d_tag_key).and_then(|values| { + if values.len() == 1 { + values.iter().next().map(|v| v.to_string()) + } else { + None + } + }) + } else { + None + }; + EventQuery { channel_id, kinds, @@ -477,6 +504,7 @@ fn filter_to_query_params(filter: &Filter, channel_id: Option) -> Ev until, limit: Some(limit), p_tag_hex, + d_tag, ..Default::default() } } @@ -601,4 +629,35 @@ mod tests { assert!(has_search); assert!(!has_non_search, "all-search filters should not be mixed"); } + + #[test] + fn d_tag_pushdown_only_for_nip33_kinds() { + let d_tag = SingleLetterTag::lowercase(Alphabet::D); + + // NIP-33 kind with #d → pushdown active + let nip33_filter = Filter::new() + .kind(nostr::Kind::Custom(30023)) + .custom_tag(d_tag, ["my-slug"]); + let q = filter_to_query_params(&nip33_filter, None); + assert_eq!(q.d_tag, Some("my-slug".to_string())); + + // Non-NIP-33 kind with #d → pushdown NOT active (would miss rows with d_tag=NULL) + let non_nip33_filter = Filter::new() + .kind(nostr::Kind::Custom(1)) + .custom_tag(d_tag, ["some-value"]); + let q2 = filter_to_query_params(&non_nip33_filter, None); + assert_eq!(q2.d_tag, None); + + // Mixed kinds (one NIP-33, one not) → pushdown NOT active + let mixed_filter = Filter::new() + .kinds([nostr::Kind::Custom(30023), nostr::Kind::Custom(1)]) + .custom_tag(d_tag, ["slug"]); + let q3 = filter_to_query_params(&mixed_filter, None); + assert_eq!(q3.d_tag, None); + + // No kinds specified → pushdown NOT active + let no_kinds_filter = Filter::new().custom_tag(d_tag, ["slug"]); + let q4 = filter_to_query_params(&no_kinds_filter, None); + assert_eq!(q4.d_tag, None); + } } diff --git a/crates/sprout-relay/src/main.rs b/crates/sprout-relay/src/main.rs index 8a9af7693..4f9c8489b 100644 --- a/crates/sprout-relay/src/main.rs +++ b/crates/sprout-relay/src/main.rs @@ -59,6 +59,14 @@ async fn main() -> anyhow::Result<()> { error!("Failed to ensure partitions: {e}"); } + // NIP-33: backfill d_tag for any existing parameterized replaceable events + // that predate the column addition. Idempotent — no-ops when fully populated. + match db.backfill_d_tags().await { + Ok(0) => {} + Ok(n) => info!("Backfilled d_tag for {n} NIP-33 events"), + Err(e) => error!("Failed to backfill d_tags: {e}"), + } + let audit_pool = sqlx::PgPool::connect(&config.database_url) .await .map_err(|e| anyhow::anyhow!("Audit DB connection failed: {e}"))?; diff --git a/crates/sprout-relay/src/nip11.rs b/crates/sprout-relay/src/nip11.rs index 79a4a5a7f..f7ab0e9ec 100644 --- a/crates/sprout-relay/src/nip11.rs +++ b/crates/sprout-relay/src/nip11.rs @@ -56,7 +56,7 @@ impl RelayInfo { description: "Sprout — private team communication relay".to_string(), pubkey: None, contact: None, - supported_nips: vec![1, 2, 10, 11, 17, 25, 29, 42, 50], + supported_nips: vec![1, 2, 10, 11, 16, 17, 25, 29, 42, 50], software: "https://github.com/sprout-rs/sprout".to_string(), version: env!("CARGO_PKG_VERSION").to_string(), limitation: Some(RelayLimitation { diff --git a/schema/schema.sql b/schema/schema.sql index e5e82ae60..01c11f11d 100644 --- a/schema/schema.sql +++ b/schema/schema.sql @@ -94,6 +94,7 @@ CREATE TABLE events ( received_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), channel_id UUID, deleted_at TIMESTAMPTZ, + d_tag TEXT, PRIMARY KEY (created_at, id) ) PARTITION BY RANGE (created_at); @@ -120,6 +121,7 @@ CREATE INDEX idx_events_kind_created ON events (kind, created_at); CREATE INDEX idx_events_id ON events (id); CREATE INDEX idx_events_deleted ON events (deleted_at); CREATE INDEX idx_events_addressable ON events (kind, pubkey, channel_id, deleted_at); +CREATE INDEX idx_events_parameterized ON events (kind, pubkey, d_tag, deleted_at) WHERE d_tag IS NOT NULL; -- ── Event mentions ──────────────────────────────────────────────────────────── diff --git a/scripts/backfill-d-tag.sql b/scripts/backfill-d-tag.sql new file mode 100644 index 000000000..8684accbf --- /dev/null +++ b/scripts/backfill-d-tag.sql @@ -0,0 +1,17 @@ +-- Backfill d_tag for existing NIP-33 range events (kind 30000–39999). +-- Idempotent: only updates rows where d_tag is still NULL. +-- Includes soft-deleted rows so the column is fully populated. +-- Run once after adding the d_tag column to the events table. +-- +-- Usage: psql $DATABASE_URL -f scripts/backfill-d-tag.sql + +UPDATE events +SET d_tag = COALESCE( + (SELECT elem->>1 + FROM jsonb_array_elements(tags) AS elem + WHERE elem->>0 = 'd' + LIMIT 1), + '' +) +WHERE kind BETWEEN 30000 AND 39999 + AND d_tag IS NULL; diff --git a/scripts/dev-setup.sh b/scripts/dev-setup.sh index f7991cfea..005308ba9 100755 --- a/scripts/dev-setup.sh +++ b/scripts/dev-setup.sh @@ -151,6 +151,17 @@ else sleep 2 done success "Migrations applied via pgschema" + + # Run data backfills (idempotent — safe to re-run). + BACKFILL_DIR="${REPO_ROOT}/scripts" + if [[ -f "${BACKFILL_DIR}/backfill-d-tag.sql" ]]; then + log "Running d_tag backfill for NIP-33 events..." + if psql "${DATABASE_URL}" -f "${BACKFILL_DIR}/backfill-d-tag.sql" 2>/dev/null; then + success "d_tag backfill complete" + else + warn "d_tag backfill failed (relay startup will retry automatically)" + fi + fi else error "pgschema not found at ${PGSCHEMA}. Run: ./bin/hermit install pgschema" exit 1 From d4912ac2d5850adefe2f3c11bf94783e2a011976 Mon Sep 17 00:00:00 2001 From: Tyler Longwell Date: Sun, 5 Apr 2026 20:44:20 -0400 Subject: [PATCH 4/6] =?UTF-8?q?chore:=20crossfire=20review=20nits=20?= =?UTF-8?q?=E2=80=94=20NIP-33=20in=20supported=5Fnips,=20d=5Ftag=20length?= =?UTF-8?q?=20bound,=20multi-value=20#d=20test?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Crossfire review findings (3 reviewers: codex GPT-5.4, 2× Claude 4.6 Opus): - Add NIP-33 to NIP-11 supported_nips (was inconsistently adding NIP-16 only) - Cap d_tag at 1024 bytes in extract_d_tag() to bound index/storage cost - Add test for multi-value #d pushdown (verifies it correctly falls back) - Add test for oversized d_tag truncation --- crates/sprout-db/src/event.rs | 27 ++++++++++++++++++++++++- crates/sprout-relay/src/handlers/req.rs | 7 +++++++ crates/sprout-relay/src/nip11.rs | 2 +- 3 files changed, 34 insertions(+), 2 deletions(-) diff --git a/crates/sprout-db/src/event.rs b/crates/sprout-db/src/event.rs index 356899a10..7ddd78ba6 100644 --- a/crates/sprout-db/src/event.rs +++ b/crates/sprout-db/src/event.rs @@ -39,11 +39,17 @@ pub struct EventQuery { pub d_tag: Option, } +/// Maximum length for a `d_tag` value (bytes). NIP-33 d-tags are short identifiers; +/// anything beyond this is either a bug or abuse. Truncated, not rejected — the event +/// is still valid, we just cap the storage/index key. +const D_TAG_MAX_LEN: usize = 1024; + /// Extract the `d_tag` value for storage. /// /// For NIP-33 parameterized replaceable events (kind 30000–39999): returns the first /// `d` tag's value, or `""` if no `d` tag is present (per NIP-33 spec). /// For all other events: returns `None` (column stays NULL). +/// Values longer than [`D_TAG_MAX_LEN`] are truncated to bound index/storage cost. pub fn extract_d_tag(event: &Event) -> Option { let kind_u32 = event.kind.as_u16() as u32; if !is_parameterized_replaceable(kind_u32) { @@ -61,7 +67,16 @@ pub fn extract_d_tag(event: &Event) -> Option { } }) .unwrap_or_default(); // Missing d tag → empty string per NIP-33 - Some(val) + if val.len() > D_TAG_MAX_LEN { + // Truncate on a char boundary to avoid splitting a multi-byte sequence. + let mut end = D_TAG_MAX_LEN; + while end > 0 && !val.is_char_boundary(end) { + end -= 1; + } + Some(val[..end].to_string()) + } else { + Some(val) + } } /// Insert a Nostr event. Rejects AUTH and ephemeral kinds. @@ -723,4 +738,14 @@ mod tests { // No d tag with a value → empty string per NIP-33 assert_eq!(extract_d_tag(&event), Some(String::new())); } + + #[test] + fn extract_d_tag_truncates_oversized_value() { + let long_val = "x".repeat(2048); + let event = + make_event_with_kind_and_tags(30023, vec![Tag::parse(&["d", &long_val]).unwrap()]); + let result = extract_d_tag(&event).unwrap(); + assert!(result.len() <= super::D_TAG_MAX_LEN); + assert_eq!(result.len(), super::D_TAG_MAX_LEN); + } } diff --git a/crates/sprout-relay/src/handlers/req.rs b/crates/sprout-relay/src/handlers/req.rs index 06b4a620a..6cdf21b93 100644 --- a/crates/sprout-relay/src/handlers/req.rs +++ b/crates/sprout-relay/src/handlers/req.rs @@ -659,5 +659,12 @@ mod tests { let no_kinds_filter = Filter::new().custom_tag(d_tag, ["slug"]); let q4 = filter_to_query_params(&no_kinds_filter, None); assert_eq!(q4.d_tag, None); + + // Multi-value #d → pushdown NOT active (can't push OR into single column match) + let multi_d_filter = Filter::new() + .kind(nostr::Kind::Custom(30023)) + .custom_tag(d_tag, ["slug-a", "slug-b"]); + let q5 = filter_to_query_params(&multi_d_filter, None); + assert_eq!(q5.d_tag, None); } } diff --git a/crates/sprout-relay/src/nip11.rs b/crates/sprout-relay/src/nip11.rs index f7ab0e9ec..1bd8cb9b9 100644 --- a/crates/sprout-relay/src/nip11.rs +++ b/crates/sprout-relay/src/nip11.rs @@ -56,7 +56,7 @@ impl RelayInfo { description: "Sprout — private team communication relay".to_string(), pubkey: None, contact: None, - supported_nips: vec![1, 2, 10, 11, 16, 17, 25, 29, 42, 50], + supported_nips: vec![1, 2, 10, 11, 16, 17, 25, 29, 33, 42, 50], software: "https://github.com/sprout-rs/sprout".to_string(), version: env!("CARGO_PKG_VERSION").to_string(), limitation: Some(RelayLimitation { From 91b1e749dc6f96406b4fd15e89652c579ac6c4ff Mon Sep 17 00:00:00 2001 From: Tyler Longwell Date: Sun, 5 Apr 2026 21:29:10 -0400 Subject: [PATCH 5/6] fix: reject oversized d_tag at ingest instead of silent truncation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Codex review correctly identified that truncating d_tag values mutates the NIP-33 identity key — two events sharing the first 1024 bytes of their d-tag would incorrectly replace each other, and REQ queries by the full #d value would miss the stored row. Fix: extract_d_tag() now returns the full value (pure extraction), and the ingest handler rejects events with d_tag > 1024 bytes via IngestError::Rejected. D_TAG_MAX_LEN is pub so the ingest layer can reference it. --- crates/sprout-db/src/event.rs | 24 +++++++--------------- crates/sprout-relay/src/handlers/ingest.rs | 7 +++++++ 2 files changed, 14 insertions(+), 17 deletions(-) diff --git a/crates/sprout-db/src/event.rs b/crates/sprout-db/src/event.rs index 7ddd78ba6..0559d7b0f 100644 --- a/crates/sprout-db/src/event.rs +++ b/crates/sprout-db/src/event.rs @@ -40,16 +40,14 @@ pub struct EventQuery { } /// Maximum length for a `d_tag` value (bytes). NIP-33 d-tags are short identifiers; -/// anything beyond this is either a bug or abuse. Truncated, not rejected — the event -/// is still valid, we just cap the storage/index key. -const D_TAG_MAX_LEN: usize = 1024; +/// anything beyond this is either a bug or abuse. +pub const D_TAG_MAX_LEN: usize = 1024; /// Extract the `d_tag` value for storage. /// /// For NIP-33 parameterized replaceable events (kind 30000–39999): returns the first /// `d` tag's value, or `""` if no `d` tag is present (per NIP-33 spec). /// For all other events: returns `None` (column stays NULL). -/// Values longer than [`D_TAG_MAX_LEN`] are truncated to bound index/storage cost. pub fn extract_d_tag(event: &Event) -> Option { let kind_u32 = event.kind.as_u16() as u32; if !is_parameterized_replaceable(kind_u32) { @@ -67,16 +65,7 @@ pub fn extract_d_tag(event: &Event) -> Option { } }) .unwrap_or_default(); // Missing d tag → empty string per NIP-33 - if val.len() > D_TAG_MAX_LEN { - // Truncate on a char boundary to avoid splitting a multi-byte sequence. - let mut end = D_TAG_MAX_LEN; - while end > 0 && !val.is_char_boundary(end) { - end -= 1; - } - Some(val[..end].to_string()) - } else { - Some(val) - } + Some(val) } /// Insert a Nostr event. Rejects AUTH and ephemeral kinds. @@ -740,12 +729,13 @@ mod tests { } #[test] - fn extract_d_tag_truncates_oversized_value() { + fn extract_d_tag_preserves_full_value() { + // extract_d_tag returns the full value — length enforcement is at the ingest layer. let long_val = "x".repeat(2048); let event = make_event_with_kind_and_tags(30023, vec![Tag::parse(&["d", &long_val]).unwrap()]); let result = extract_d_tag(&event).unwrap(); - assert!(result.len() <= super::D_TAG_MAX_LEN); - assert_eq!(result.len(), super::D_TAG_MAX_LEN); + assert_eq!(result.len(), 2048); + assert_eq!(result, long_val); } } diff --git a/crates/sprout-relay/src/handlers/ingest.rs b/crates/sprout-relay/src/handlers/ingest.rs index d7733b389..29329618c 100644 --- a/crates/sprout-relay/src/handlers/ingest.rs +++ b/crates/sprout-relay/src/handlers/ingest.rs @@ -1242,6 +1242,13 @@ pub async fn ingest_event( } else if is_parameterized_replaceable(kind_u32) { // NIP-33 parameterized replaceable — keyed by (kind, pubkey, d_tag). let d_tag = sprout_db::event::extract_d_tag(&event).unwrap_or_default(); + if d_tag.len() > sprout_db::event::D_TAG_MAX_LEN { + return Err(IngestError::Rejected(format!( + "invalid: d tag too long ({} bytes, max {})", + d_tag.len(), + sprout_db::event::D_TAG_MAX_LEN, + ))); + } state .db .replace_parameterized_event(&event, &d_tag, channel_id) From 2fcaeddc2b74761a739ba5b136e66c9875994180 Mon Sep 17 00:00:00 2001 From: tlongwell-block <109685178+tlongwell-block@users.noreply.github.com> Date: Mon, 6 Apr 2026 21:09:08 -0400 Subject: [PATCH 6/6] feat: NIP-23 long-form articles (kind:30023) (#249) --- crates/sprout-core/src/kind.rs | 5 + crates/sprout-relay/src/handlers/ingest.rs | 86 ++++- crates/sprout-relay/src/nip11.rs | 37 +- .../sprout-test-client/tests/e2e_long_form.rs | 319 ++++++++++++++++++ 4 files changed, 432 insertions(+), 15 deletions(-) create mode 100644 crates/sprout-test-client/tests/e2e_long_form.rs diff --git a/crates/sprout-core/src/kind.rs b/crates/sprout-core/src/kind.rs index 7b62432a4..164b48ae2 100644 --- a/crates/sprout-core/src/kind.rs +++ b/crates/sprout-core/src/kind.rs @@ -21,6 +21,10 @@ pub const KIND_REACTION: u32 = 7; pub const KIND_GIFT_WRAP: u32 = 1059; /// NIP-94: File metadata attachment. pub const KIND_FILE_METADATA: u32 = 1063; +/// NIP-23: Long-form content (articles, blog posts, RFCs). +/// Parameterized replaceable (NIP-33, 30000–39999 range) — keyed by `(pubkey, kind, d_tag)`. +/// Stored globally (channel_id = NULL); author-owned, not channel-scoped. +pub const KIND_LONG_FORM: u32 = 30023; /// NIP-42 auth event — never stored (carries bearer tokens). pub const KIND_AUTH: u32 = 22242; @@ -276,6 +280,7 @@ pub const ALL_KINDS: &[u32] = &[ KIND_SUBSCRIPTION_RESUMED, KIND_MEMBER_ADDED_NOTIFICATION, KIND_MEMBER_REMOVED_NOTIFICATION, + KIND_LONG_FORM, KIND_FORUM_POST, KIND_FORUM_VOTE, KIND_FORUM_COMMENT, diff --git a/crates/sprout-relay/src/handlers/ingest.rs b/crates/sprout-relay/src/handlers/ingest.rs index 29329618c..6b00a7982 100644 --- a/crates/sprout-relay/src/handlers/ingest.rs +++ b/crates/sprout-relay/src/handlers/ingest.rs @@ -14,13 +14,13 @@ use sprout_auth::Scope; use sprout_core::kind::{ event_kind_u32, is_parameterized_replaceable, KIND_AUTH, KIND_CANVAS, KIND_CONTACT_LIST, KIND_DELETION, KIND_FORUM_COMMENT, KIND_FORUM_POST, KIND_FORUM_VOTE, KIND_GIFT_WRAP, - KIND_MEMBER_ADDED_NOTIFICATION, KIND_MEMBER_REMOVED_NOTIFICATION, KIND_NIP29_CREATE_GROUP, - KIND_NIP29_DELETE_EVENT, KIND_NIP29_DELETE_GROUP, KIND_NIP29_EDIT_METADATA, - KIND_NIP29_JOIN_REQUEST, KIND_NIP29_LEAVE_REQUEST, KIND_NIP29_PUT_USER, KIND_NIP29_REMOVE_USER, - KIND_PRESENCE_UPDATE, KIND_PROFILE, KIND_REACTION, KIND_STREAM_MESSAGE, - KIND_STREAM_MESSAGE_BOOKMARKED, KIND_STREAM_MESSAGE_DIFF, KIND_STREAM_MESSAGE_EDIT, - KIND_STREAM_MESSAGE_PINNED, KIND_STREAM_MESSAGE_SCHEDULED, KIND_STREAM_MESSAGE_V2, - KIND_STREAM_REMINDER, KIND_TEXT_NOTE, + KIND_LONG_FORM, KIND_MEMBER_ADDED_NOTIFICATION, KIND_MEMBER_REMOVED_NOTIFICATION, + KIND_NIP29_CREATE_GROUP, KIND_NIP29_DELETE_EVENT, KIND_NIP29_DELETE_GROUP, + KIND_NIP29_EDIT_METADATA, KIND_NIP29_JOIN_REQUEST, KIND_NIP29_LEAVE_REQUEST, + KIND_NIP29_PUT_USER, KIND_NIP29_REMOVE_USER, KIND_PRESENCE_UPDATE, KIND_PROFILE, KIND_REACTION, + KIND_STREAM_MESSAGE, KIND_STREAM_MESSAGE_BOOKMARKED, KIND_STREAM_MESSAGE_DIFF, + KIND_STREAM_MESSAGE_EDIT, KIND_STREAM_MESSAGE_PINNED, KIND_STREAM_MESSAGE_SCHEDULED, + KIND_STREAM_MESSAGE_V2, KIND_STREAM_REMINDER, KIND_TEXT_NOTE, }; use sprout_core::verification::verify_event; @@ -142,7 +142,7 @@ pub enum IngestError { fn required_scope_for_kind(kind: u32, event: &Event) -> Result { match kind { KIND_PROFILE => Ok(Scope::UsersWrite), - KIND_TEXT_NOTE => Ok(Scope::MessagesWrite), + KIND_TEXT_NOTE | KIND_LONG_FORM => Ok(Scope::MessagesWrite), KIND_CONTACT_LIST => Ok(Scope::UsersWrite), KIND_DELETION | KIND_REACTION @@ -242,6 +242,24 @@ pub(crate) async fn derive_reaction_channel( } } +/// Kinds that are always global (`channel_id = NULL`). +/// +/// If a client includes a stray `h` tag on these kinds, the ingest pipeline +/// sets `channel_id = None` — these events are never channel-scoped. +/// +/// Note: the raw `h` tag remains on the stored event (Nostr events are signed, +/// so tags cannot be stripped without invalidating the signature). The read-path +/// filter matching in `filter.rs` treats explicit `h` tags as authoritative, +/// which means a stray `h` tag can still match `#h` queries. This is a known +/// limitation affecting all global-only kinds and should be addressed in the +/// filter layer as a follow-up. +pub(crate) fn is_global_only_kind(kind: u32) -> bool { + matches!( + kind, + KIND_PROFILE | KIND_TEXT_NOTE | KIND_CONTACT_LIST | KIND_LONG_FORM + ) +} + /// Kinds that require an `h` tag for channel scoping. pub(crate) fn requires_h_channel_scope(kind: u32) -> bool { matches!( @@ -844,9 +862,7 @@ pub async fn ingest_event( }; // ── 5b. Global-only kinds ignore h-tags ───────────────────────────── - // kind:0 (profile), kind:1 (text note), kind:3 (contact list) are always global. - // If a client includes a stray h-tag, ignore it — these kinds are never channel-scoped. - if matches!(kind_u32, KIND_PROFILE | KIND_TEXT_NOTE | KIND_CONTACT_LIST) { + if is_global_only_kind(kind_u32) { channel_id = None; } @@ -1314,8 +1330,8 @@ pub async fn ingest_event( mod tests { use super::*; use sprout_core::kind::{ - KIND_CANVAS, KIND_FORUM_COMMENT, KIND_FORUM_POST, KIND_FORUM_VOTE, KIND_PRESENCE_UPDATE, - KIND_STREAM_MESSAGE, KIND_STREAM_MESSAGE_DIFF, + KIND_CANVAS, KIND_FORUM_COMMENT, KIND_FORUM_POST, KIND_FORUM_VOTE, KIND_LONG_FORM, + KIND_PRESENCE_UPDATE, KIND_STREAM_MESSAGE, KIND_STREAM_MESSAGE_DIFF, }; #[test] @@ -1371,13 +1387,54 @@ mod tests { assert!(!requires_h_channel_scope(KIND_REACTION)); } + #[test] + fn long_form_is_in_scope_allowlist() { + let dummy = make_dummy_event(); + assert!( + required_scope_for_kind(KIND_LONG_FORM, &dummy).is_ok(), + "KIND_LONG_FORM (30023) should be accepted" + ); + } + + #[test] + fn long_form_requires_messages_write_scope() { + let dummy = make_dummy_event(); + assert_eq!( + required_scope_for_kind(KIND_LONG_FORM, &dummy).unwrap(), + Scope::MessagesWrite, + ); + } + + #[test] + fn long_form_does_not_require_h_tag() { + // kind:30023 is global (author-owned, not channel-scoped) + assert!(!requires_h_channel_scope(KIND_LONG_FORM)); + } + + #[test] + fn long_form_is_global_only() { + // kind:30023 is always global — ingest nulls channel_id even if an h-tag is present + assert!(is_global_only_kind(KIND_LONG_FORM)); + } + + #[test] + fn global_only_and_channel_scoped_are_disjoint() { + // A kind cannot be both global-only and channel-scoped + for kind in 0..=65535u32 { + assert!( + !(is_global_only_kind(kind) && requires_h_channel_scope(kind)), + "kind {kind} is both global-only and channel-scoped" + ); + } + } + #[test] fn ephemeral_kinds_not_in_scope_allowlist() { assert!(required_scope_for_kind(KIND_PRESENCE_UPDATE, &make_dummy_event()).is_err()); } #[test] - fn per_kind_scope_allowlist_covers_all_18_migrated_kinds() { + fn per_kind_scope_allowlist_covers_all_migrated_kinds() { let dummy = make_dummy_event(); let migrated = [ KIND_PROFILE, @@ -1398,6 +1455,7 @@ mod tests { KIND_FORUM_POST, KIND_FORUM_VOTE, KIND_FORUM_COMMENT, + KIND_LONG_FORM, ]; for kind in migrated { assert!( diff --git a/crates/sprout-relay/src/nip11.rs b/crates/sprout-relay/src/nip11.rs index 1bd8cb9b9..c9cf2e62e 100644 --- a/crates/sprout-relay/src/nip11.rs +++ b/crates/sprout-relay/src/nip11.rs @@ -4,6 +4,11 @@ use serde::{Deserialize, Serialize}; use crate::connection::MAX_FRAME_BYTES; +/// NIPs supported by this relay, advertised in the NIP-11 document. +/// Kept as a module-level constant so tests can verify it without constructing +/// a full `Config` (which reads env vars and races with config.rs tests). +pub(crate) const SUPPORTED_NIPS: &[u32] = &[1, 2, 10, 11, 16, 17, 23, 25, 29, 33, 42, 50]; + /// Relay information document served at `GET /` with `Accept: application/nostr+json`. #[derive(Debug, Clone, Serialize, Deserialize)] pub struct RelayInfo { @@ -56,7 +61,7 @@ impl RelayInfo { description: "Sprout — private team communication relay".to_string(), pubkey: None, contact: None, - supported_nips: vec![1, 2, 10, 11, 16, 17, 25, 29, 33, 42, 50], + supported_nips: SUPPORTED_NIPS.to_vec(), software: "https://github.com/sprout-rs/sprout".to_string(), version: env!("CARGO_PKG_VERSION").to_string(), limitation: Some(RelayLimitation { @@ -80,3 +85,33 @@ pub async fn relay_info_handler( ) -> axum::response::Json { axum::response::Json(RelayInfo::from_config(&state.config)) } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn supported_nips_includes_nip23_and_nip33() { + // Tests the production SUPPORTED_NIPS constant directly — no Config::from_env() + // needed, avoiding the env-var race with config.rs tests. + assert!( + SUPPORTED_NIPS.contains(&23), + "NIP-23 (long-form content) must be advertised" + ); + assert!( + SUPPORTED_NIPS.contains(&33), + "NIP-33 (parameterized replaceable) must be advertised" + ); + } + + #[test] + fn supported_nips_are_sorted() { + let mut sorted = SUPPORTED_NIPS.to_vec(); + sorted.sort(); + assert_eq!( + SUPPORTED_NIPS, + &sorted[..], + "supported_nips should be sorted" + ); + } +} diff --git a/crates/sprout-test-client/tests/e2e_long_form.rs b/crates/sprout-test-client/tests/e2e_long_form.rs new file mode 100644 index 000000000..ff2c4934b --- /dev/null +++ b/crates/sprout-test-client/tests/e2e_long_form.rs @@ -0,0 +1,319 @@ +//! End-to-end tests for NIP-23 long-form content (kind:30023). +//! +//! These tests require a running relay instance. By default they are marked +//! `#[ignore]` so that `cargo test` does not fail in CI when the relay is not +//! available. +//! +//! # Running +//! +//! Start the relay, then run: +//! +//! ```text +//! cargo test --test e2e_long_form -- --ignored +//! ``` +//! +//! Override the relay URL with the `RELAY_URL` environment variable: +//! +//! ```text +//! RELAY_URL=ws://relay.example.com cargo test --test e2e_long_form -- --ignored +//! ``` + +use std::time::Duration; + +use nostr::{Alphabet, EventBuilder, Filter, Keys, Kind, SingleLetterTag, Tag, Timestamp}; +use sprout_test_client::SproutTestClient; + +const KIND_LONG_FORM: u16 = 30023; + +fn relay_url() -> String { + std::env::var("RELAY_URL").unwrap_or_else(|_| "ws://localhost:3000".to_string()) +} + +fn sub_id(name: &str) -> String { + format!("e2e-{name}-{}", uuid::Uuid::new_v4()) +} + +/// Build a kind:30023 event with standard NIP-23 tags. +fn build_long_form_event( + keys: &Keys, + d_tag: &str, + title: &str, + content: &str, + extra_tags: Vec, +) -> nostr::Event { + let mut tags = vec![ + Tag::parse(&["d", d_tag]).unwrap(), + Tag::parse(&["title", title]).unwrap(), + ]; + tags.extend(extra_tags); + EventBuilder::new(Kind::Custom(KIND_LONG_FORM), content, tags) + .sign_with_keys(keys) + .unwrap() +} + +// ── Tests ───────────────────────────────────────────────────────────────────── + +/// kind:30023 events are accepted by the relay. +#[tokio::test] +#[ignore] +async fn test_long_form_accepted() { + let url = relay_url(); + let keys = Keys::generate(); + let mut client = SproutTestClient::connect(&url, &keys) + .await + .expect("connect"); + + let event = build_long_form_event( + &keys, + "test-article-accept", + "Test Article", + "# Hello\n\nThis is a test article.", + vec![], + ); + + let ok = client.send_event(event).await.expect("send event"); + assert!( + ok.accepted, + "relay should accept kind:30023: {}", + ok.message + ); + + client.disconnect().await.expect("disconnect"); +} + +/// kind:30023 events are retrievable via REQ with kinds filter. +#[tokio::test] +#[ignore] +async fn test_long_form_retrievable() { + let url = relay_url(); + let keys = Keys::generate(); + let mut client = SproutTestClient::connect(&url, &keys) + .await + .expect("connect"); + + let d_tag = format!("retrieve-{}", uuid::Uuid::new_v4().simple()); + let event = build_long_form_event( + &keys, + &d_tag, + "Retrievable Article", + "# Retrievable\n\nBody text.", + vec![], + ); + let event_id = event.id; + + let ok = client.send_event(event).await.expect("send event"); + assert!(ok.accepted, "relay should accept: {}", ok.message); + + // Query back by kind + author + let sid = sub_id("retrieve"); + let filter = Filter::new() + .kind(Kind::Custom(KIND_LONG_FORM)) + .author(keys.public_key()); + client + .subscribe(&sid, vec![filter]) + .await + .expect("subscribe"); + + let events = client + .collect_until_eose(&sid, Duration::from_secs(5)) + .await + .expect("collect"); + + assert!( + events.iter().any(|e| e.id == event_id), + "should find the published article in query results" + ); + + client.disconnect().await.expect("disconnect"); +} + +/// kind:30023 is stored globally (channel_id = NULL) — stray h-tags are ignored. +/// An event with a stray h-tag should still be retrievable via a global query +/// (no h-tag filter), proving it was stored as global. +#[tokio::test] +#[ignore] +async fn test_long_form_stray_h_tag_ignored() { + let url = relay_url(); + let keys = Keys::generate(); + let mut client = SproutTestClient::connect(&url, &keys) + .await + .expect("connect"); + + // Publish with a stray h-tag (a UUID that doesn't correspond to any channel). + let fake_channel = uuid::Uuid::new_v4().to_string(); + let d_tag = format!("stray-h-{}", uuid::Uuid::new_v4().simple()); + let event = build_long_form_event( + &keys, + &d_tag, + "Stray H-Tag Article", + "Should be stored globally despite h-tag.", + vec![Tag::parse(&["h", &fake_channel]).unwrap()], + ); + let event_id = event.id; + + let ok = client.send_event(event).await.expect("send event"); + assert!(ok.accepted, "relay should accept: {}", ok.message); + + // Query globally (no h-tag filter) — should find the article. + let sid = sub_id("stray-h"); + let filter = Filter::new() + .kind(Kind::Custom(KIND_LONG_FORM)) + .author(keys.public_key()); + client + .subscribe(&sid, vec![filter]) + .await + .expect("subscribe"); + + let events = client + .collect_until_eose(&sid, Duration::from_secs(5)) + .await + .expect("collect"); + + assert!( + events.iter().any(|e| e.id == event_id), + "article with stray h-tag should be retrievable via global query" + ); + + // NOTE: Ideally, querying with #h= should NOT return the + // article since it's global. However, the raw h-tag remains on the stored + // event (Nostr events are signed — tags can't be stripped without breaking + // the signature), and the read-path filter matching in filter.rs treats + // explicit h-tags as authoritative. This is a pre-existing limitation + // affecting all global-only kinds (0, 1, 3, 30023) and should be fixed + // in the filter layer as a follow-up. + + client.disconnect().await.expect("disconnect"); +} + +/// NIP-33 replacement: publishing a newer kind:30023 with the same d-tag replaces the old one. +#[tokio::test] +#[ignore] +async fn test_long_form_nip33_replacement() { + let url = relay_url(); + let keys = Keys::generate(); + let mut client = SproutTestClient::connect(&url, &keys) + .await + .expect("connect"); + + let d_tag = format!("replace-{}", uuid::Uuid::new_v4().simple()); + + // Publish v1 + let v1 = build_long_form_event(&keys, &d_tag, "Article v1", "Version 1 content.", vec![]); + let ok1 = client.send_event(v1).await.expect("send v1"); + assert!(ok1.accepted, "v1 should be accepted: {}", ok1.message); + + // Small delay to ensure different created_at timestamps + tokio::time::sleep(Duration::from_secs(1)).await; + + // Publish v2 with the same d-tag + let v2 = build_long_form_event( + &keys, + &d_tag, + "Article v2", + "Version 2 content — updated.", + vec![], + ); + let v2_id = v2.id; + let ok2 = client.send_event(v2).await.expect("send v2"); + assert!(ok2.accepted, "v2 should be accepted: {}", ok2.message); + + // Query — should only get v2 (v1 replaced) + let sid = sub_id("replace"); + let filter = Filter::new() + .kind(Kind::Custom(KIND_LONG_FORM)) + .author(keys.public_key()) + .custom_tag(SingleLetterTag::lowercase(Alphabet::D), [d_tag.as_str()]); + client + .subscribe(&sid, vec![filter]) + .await + .expect("subscribe"); + + let events = client + .collect_until_eose(&sid, Duration::from_secs(5)) + .await + .expect("collect"); + + assert_eq!( + events.len(), + 1, + "should have exactly one event after replacement" + ); + assert_eq!(events[0].id, v2_id, "surviving event should be v2"); + assert!( + events[0].content.contains("Version 2"), + "content should be v2" + ); + + client.disconnect().await.expect("disconnect"); +} + +/// NIP-33 stale-write protection: an older event cannot replace a newer one. +#[tokio::test] +#[ignore] +async fn test_long_form_stale_write_rejected() { + let url = relay_url(); + let keys = Keys::generate(); + let mut client = SproutTestClient::connect(&url, &keys) + .await + .expect("connect"); + + let d_tag = format!("stale-{}", uuid::Uuid::new_v4().simple()); + + // Publish the "newer" event first (with a future-ish timestamp) + let newer = { + let tags = vec![ + Tag::parse(&["d", &d_tag]).unwrap(), + Tag::parse(&["title", "Newer Article"]).unwrap(), + ]; + EventBuilder::new(Kind::Custom(KIND_LONG_FORM), "Newer content.", tags) + .custom_created_at(Timestamp::from(nostr::Timestamp::now().as_u64() + 100)) + .sign_with_keys(&keys) + .unwrap() + }; + let newer_id = newer.id; + let ok1 = client.send_event(newer).await.expect("send newer"); + assert!(ok1.accepted, "newer should be accepted: {}", ok1.message); + + // Now try to publish an "older" event with the same d-tag but earlier timestamp + let older = { + let tags = vec![ + Tag::parse(&["d", &d_tag]).unwrap(), + Tag::parse(&["title", "Older Article"]).unwrap(), + ]; + EventBuilder::new(Kind::Custom(KIND_LONG_FORM), "Older content.", tags) + .custom_created_at(Timestamp::from(nostr::Timestamp::now().as_u64() - 100)) + .sign_with_keys(&keys) + .unwrap() + }; + let _ok2 = client.send_event(older).await.expect("send older"); + // Stale write may be rejected or accepted-as-duplicate — either way, + // the older event must NOT replace the newer one. + + // Query — should still have the newer event + let sid = sub_id("stale"); + let filter = Filter::new() + .kind(Kind::Custom(KIND_LONG_FORM)) + .author(keys.public_key()) + .custom_tag(SingleLetterTag::lowercase(Alphabet::D), [d_tag.as_str()]); + client + .subscribe(&sid, vec![filter]) + .await + .expect("subscribe"); + + let events = client + .collect_until_eose(&sid, Duration::from_secs(5)) + .await + .expect("collect"); + + assert_eq!(events.len(), 1, "should have exactly one event"); + assert_eq!( + events[0].id, newer_id, + "surviving event should be the newer one" + ); + assert!( + events[0].content.contains("Newer"), + "content should be from the newer event" + ); + + client.disconnect().await.expect("disconnect"); +}