From f8ee869e36142e89c3ab0559ab56cc0bf62e3726 Mon Sep 17 00:00:00 2001 From: Valter Balegas Date: Wed, 20 Aug 2025 23:10:20 +0100 Subject: [PATCH 1/8] Add electric-collection reference page --- docs/collections/electric-collection.md | 217 ++++++++++++++++++++++++ docs/config.json | 4 + 2 files changed, 221 insertions(+) create mode 100644 docs/collections/electric-collection.md diff --git a/docs/collections/electric-collection.md b/docs/collections/electric-collection.md new file mode 100644 index 000000000..5930e9c9b --- /dev/null +++ b/docs/collections/electric-collection.md @@ -0,0 +1,217 @@ +--- +title: Electric Collection +--- + +# Electric Collection + +Electric collections provide seamless integration between TanStack DB and ElectricSQL, enabling real-time data synchronization with your Postgres database through Electric's sync engine. + +## Overview + +The `@tanstack/electric-db-collection` package allows you to create collections that: +- Automatically sync data from Postgres via Electric shapes +- Support optimistic updates with transaction confirmation and automatic rollback on errors +- Handle persistence through customizable mutation handlers + +## Installation + +```bash +npm install @tanstack/electric-db-collection @tanstack/db +``` + +## Basic Usage + +```typescript +import { createCollection } from '@tanstack/db' +import { electricCollectionOptions } from '@tanstack/electric-db-collection' + +const todosCollection = createCollection( + electricCollectionOptions({ + shapeOptions: { + url: 'https://example.com/v1/shape', + params: { + table: 'todos', + }, + }, + getKey: (item) => item.id, + }) +) +``` + +## Configuration Options + +The `electricCollectionOptions` function accepts the following options: + +### Required Options + +- `shapeOptions`: Configuration for the ElectricSQL ShapeStream +- `getKey`: Function to extract the unique key from an item + +### Shape Options + +- `url`: The URL to your Electric sync service +- `params`: Shape parameters including: + - `table`: The database table to sync + - `where`: Optional WHERE clause for filtering (e.g., `"status = 'active'"`) + - `columns`: Optional array of columns to sync + +### Collection Options + +- `id`: Unique identifier for the collection +- `schema`: Schema for validating items (any Standard Schema compatible schema) +- `sync`: Custom sync configuration + +### Persistence Handlers + +- `onInsert`: Handler called before insert operations +- `onUpdate`: Handler called before update operations +- `onDelete`: Handler called before delete operations + +## Persistence Handlers + +You can define handlers that are called when mutations occur, for instance to persist changes to your backend. Such handlers **must return a transaction ID** (`txid`) to track when the mutation has been synchronized back from Electric: + +```typescript +const todosCollection = createCollection( + electricCollectionOptions({ + id: 'todos', + schema: todoSchema, + getKey: (item) => item.id, + + shapeOptions: { + url: 'https://api.electric-sql.cloud/v1/shape', + params: { table: 'todos' }, + }, + + onInsert: async ({ transaction }) => { + const newItem = transaction.mutations[0].modified + const response = await api.todos.create(newItem) + + return { txid: response.txid } + }, + + // you can also implement onUpdate and onDelete handlers + }) +) +``` + +### Understanding Transaction IDs + +When you commit a transaction in Postgres, it gets assigned a transaction ID. Your API should return this transaction ID in the HTTP response so that the collection can automatically wait for this transaction ID to confirm the optimistic update. + +Here is an example of a function to extract the transaction Id from Postgres: + +```ts +async function generateTxId(tx) { + // The ::xid cast strips off the epoch, giving you the raw 32-bit value + // that matches what PostgreSQL sends in logical replication streams + // (and then exposed through Electric which we'll match against + // in the client). + const result = await tx.execute( + sql`SELECT pg_current_xact_id()::xid::text as txid` + ) + const txid = result.rows[0]?.txid + + if (txid === undefined) { + throw new Error(`Failed to get transaction ID`) + } + + return parseInt(txid as string, 10) +} +``` + +## Utility Methods + +The collection provides these utility methods via `collection.utils`: + +- `awaitTxId(txid, timeout?)`: Manually wait for a specific transaction ID to be synchronized + +```typescript +todosCollection.utils.awaitTxId(12345) +``` + +This is useful when you need to ensure a mutation has been synchronized before proceeding with other operations. + +## Optimistic Updates with Explicit Transactions + +For more advanced use cases, you can create custom actions that can do multiple mutations across collections transactionally. In this case, you need to explicitly await for the transaction ID using `utils.awaitTxId()`. + +```typescript +const addTodoAction = createOptimisticAction({ + onMutate: ({ text }) => { + // optimistically insert with a temporary ID + const tempId = crypto.randomUUID() + todosCollection.insert({ + id: tempId, + text, + completed: false, + created_at: new Date(), + }) + + // ... mutate other collections + }, + + mutationFn: async ({ text }) => { + const response = await api.todos.create({ + data: { text, completed: false } + }) + + await todosCollection.utils.awaitTxId(response.txid) + + } +}) +``` + +## Shape Configuration + +Electric shapes allow you to filter and control what data syncs to your collection: + +### Basic Table Sync + +```typescript +const todosCollection = createCollection( + electricCollectionOptions({ + shapeOptions: { + url: 'https://example.com/v1/shape', + params: { + table: 'todos', + }, + }, + getKey: (item) => item.id, + }) +) +``` + +### Filtered Sync with WHERE Clause + +```typescript +const activeTodosCollection = createCollection( + electricCollectionOptions({ + shapeOptions: { + url: 'https://example.com/v1/shape', + params: { + table: 'todos', + where: "status = 'active' AND deleted_at IS NULL", + }, + }, + getKey: (item) => item.id, + }) +) +``` + +### Column Selection + +```typescript +const todoSummaryCollection = createCollection( + electricCollectionOptions({ + shapeOptions: { + url: 'https://example.com/v1/shape', + params: { + table: 'todos', + columns: ['id', 'title', 'status'], // Only sync these columns + }, + }, + getKey: (item) => item.id, + }) +) +``` diff --git a/docs/config.json b/docs/config.json index 546b1b44f..90a7225d3 100644 --- a/docs/config.json +++ b/docs/config.json @@ -84,6 +84,10 @@ { "label": "Query Collection", "to": "collections/query-collection" + }, + { + "label": "Electric Collection", + "to": "collections/electric-collection" } ] }, From e192746172df24b5de28d386310ab963c4223f88 Mon Sep 17 00:00:00 2001 From: Valter Balegas Date: Wed, 20 Aug 2025 23:21:33 +0100 Subject: [PATCH 2/8] changeset --- .changeset/old-trams-check.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/old-trams-check.md diff --git a/.changeset/old-trams-check.md b/.changeset/old-trams-check.md new file mode 100644 index 000000000..b9ff8f549 --- /dev/null +++ b/.changeset/old-trams-check.md @@ -0,0 +1,5 @@ +--- +"@tanstack/db": patch +--- + +docs: electric-collection reference page From 8fd94d78d6269ab6b84b1303ddac468e6e8ae78d Mon Sep 17 00:00:00 2001 From: Valter Balegas Date: Thu, 21 Aug 2025 16:05:56 +0100 Subject: [PATCH 3/8] Address review --- docs/collections/electric-collection.md | 167 +++++++++++++----------- 1 file changed, 88 insertions(+), 79 deletions(-) diff --git a/docs/collections/electric-collection.md b/docs/collections/electric-collection.md index 5930e9c9b..f7b32f076 100644 --- a/docs/collections/electric-collection.md +++ b/docs/collections/electric-collection.md @@ -10,28 +10,25 @@ Electric collections provide seamless integration between TanStack DB and Electr The `@tanstack/electric-db-collection` package allows you to create collections that: - Automatically sync data from Postgres via Electric shapes -- Support optimistic updates with transaction confirmation and automatic rollback on errors +- Support optimistic updates with transaction matching and automatic rollback on errors - Handle persistence through customizable mutation handlers ## Installation ```bash -npm install @tanstack/electric-db-collection @tanstack/db +npm install @tanstack/electric-db-collection @tanstack/react-db ``` ## Basic Usage ```typescript -import { createCollection } from '@tanstack/db' +import { createCollection } from '@tanstack/react-db' import { electricCollectionOptions } from '@tanstack/electric-db-collection' const todosCollection = createCollection( electricCollectionOptions({ shapeOptions: { - url: 'https://example.com/v1/shape', - params: { - table: 'todos', - }, + url: '/api/todos', }, getKey: (item) => item.id, }) @@ -45,21 +42,15 @@ The `electricCollectionOptions` function accepts the following options: ### Required Options - `shapeOptions`: Configuration for the ElectricSQL ShapeStream -- `getKey`: Function to extract the unique key from an item - -### Shape Options + - `url`: The URL of your proxy to Electric -- `url`: The URL to your Electric sync service -- `params`: Shape parameters including: - - `table`: The database table to sync - - `where`: Optional WHERE clause for filtering (e.g., `"status = 'active'"`) - - `columns`: Optional array of columns to sync +- `getKey`: Function to extract the unique key from an item ### Collection Options -- `id`: Unique identifier for the collection -- `schema`: Schema for validating items (any Standard Schema compatible schema) -- `sync`: Custom sync configuration +- `id`: Unique identifier for the collection (optional) +- `schema`: Schema for validating items. Any Standard Schema compatible schema (optional) +- `sync`: Custom sync configuration (optional) ### Persistence Handlers @@ -69,7 +60,9 @@ The `electricCollectionOptions` function accepts the following options: ## Persistence Handlers -You can define handlers that are called when mutations occur, for instance to persist changes to your backend. Such handlers **must return a transaction ID** (`txid`) to track when the mutation has been synchronized back from Electric: +Handlers can be defined to run on mutations. They are useful to send mutations to the backend and confirming them once Electric delivers the corresponding transactions. Until confirmation, TanStack DB blocks sync data for the collection to prevent race conditions. To avoid any delays, it’s important to use a matching strategy. + +The most reliable strategy is for the backend to include the transaction ID (txid) in its response, allowing the client to match each mutation with Electric’s transaction identifiers for precise confirmation. If no strategy is provided, client mutations are automatically confirmed after three seconds. ```typescript const todosCollection = createCollection( @@ -79,7 +72,7 @@ const todosCollection = createCollection( getKey: (item) => item.id, shapeOptions: { - url: 'https://api.electric-sql.cloud/v1/shape', + url: '/api/todos', params: { table: 'todos' }, }, @@ -95,11 +88,7 @@ const todosCollection = createCollection( ) ``` -### Understanding Transaction IDs - -When you commit a transaction in Postgres, it gets assigned a transaction ID. Your API should return this transaction ID in the HTTP response so that the collection can automatically wait for this transaction ID to confirm the optimistic update. - -Here is an example of a function to extract the transaction Id from Postgres: +On the backend, you can extract the `txid` for a transaction by querying Postgres directly. ```ts async function generateTxId(tx) { @@ -120,6 +109,80 @@ async function generateTxId(tx) { } ``` +It might not always be feasible to return transactions identifiers from your API. In that case, you can use a custom match function. + +```js +onInsert: async ({ transaction }) => { + const item = transaction.mutations[0].modified + await api.todos.create(item) // API doesn't return txid + + return { + matchStream: (stream) => matchStream( + stream, + ['insert'], + (msg) => msg.value.id === item.id, + 3000 // timeout fallback + ) + } +} +``` + +## Using a proxy with Electric + +Electric is typically deployed behind a proxy server that handles shape configuration, authentication and authorization. This provides better security and allows you to control what data users can access without exposing Electric to the client. + + +Here is an example proxy implementation using TanStack Starter: + +```js +import { createServerFileRoute } from "@tanstack/react-start/server" + +// Electric URL +const baseUrl = 'http://.../v1/shape' + +const serve = async ({ request }: { request: Request }) => { + // ...check user authorization + + const url = new URL(request.url) + const originUrl = new URL(baseUrl) + + // passthrough parameters from electric client + url.searchParams.forEach((value, key) => { + if ( + [ + "live", + "handle", + "offset", + "cursor", + ].includes(key) + ) { + originUrl.searchParams.set(key, value) + } + }) + + // the shape parameters + originUrl.searchParams.set("table", "todos") + // originUrl.searchParams.set("where", "completed = true") + // originUrl.searchParams.set("columns", "id,text,completed") + + const response = await fetch(originUrl) + const headers = new Headers(response.headers) + headers.delete("content-encoding") + headers.delete("content-length") + + return new Response(response.body, { + status: response.status, + statusText: response.statusText, + headers, + }) +} + +export const ServerRoute = createServerFileRoute("/api/todos").methods({ + GET: serve, +}) +``` + + ## Utility Methods The collection provides these utility methods via `collection.utils`: @@ -161,57 +224,3 @@ const addTodoAction = createOptimisticAction({ } }) ``` - -## Shape Configuration - -Electric shapes allow you to filter and control what data syncs to your collection: - -### Basic Table Sync - -```typescript -const todosCollection = createCollection( - electricCollectionOptions({ - shapeOptions: { - url: 'https://example.com/v1/shape', - params: { - table: 'todos', - }, - }, - getKey: (item) => item.id, - }) -) -``` - -### Filtered Sync with WHERE Clause - -```typescript -const activeTodosCollection = createCollection( - electricCollectionOptions({ - shapeOptions: { - url: 'https://example.com/v1/shape', - params: { - table: 'todos', - where: "status = 'active' AND deleted_at IS NULL", - }, - }, - getKey: (item) => item.id, - }) -) -``` - -### Column Selection - -```typescript -const todoSummaryCollection = createCollection( - electricCollectionOptions({ - shapeOptions: { - url: 'https://example.com/v1/shape', - params: { - table: 'todos', - columns: ['id', 'title', 'status'], // Only sync these columns - }, - }, - getKey: (item) => item.id, - }) -) -``` From 5efc44e3ae3e869e82c8d826d542d5bc68cc73f8 Mon Sep 17 00:00:00 2001 From: Valter Balegas Date: Thu, 21 Aug 2025 23:38:42 +0100 Subject: [PATCH 4/8] changed header --- docs/collections/electric-collection.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/collections/electric-collection.md b/docs/collections/electric-collection.md index f7b32f076..d1a4014db 100644 --- a/docs/collections/electric-collection.md +++ b/docs/collections/electric-collection.md @@ -46,11 +46,11 @@ The `electricCollectionOptions` function accepts the following options: - `getKey`: Function to extract the unique key from an item -### Collection Options +### Optional -- `id`: Unique identifier for the collection (optional) -- `schema`: Schema for validating items. Any Standard Schema compatible schema (optional) -- `sync`: Custom sync configuration (optional) +- `id`: Unique identifier for the collection +- `schema`: Schema for validating items. Any Standard Schema compatible schema +- `sync`: Custom sync configuration ### Persistence Handlers From 19dea74d55de725dd769672879bf77dc6fd6a1da Mon Sep 17 00:00:00 2001 From: Valter Balegas Date: Thu, 21 Aug 2025 23:43:00 +0100 Subject: [PATCH 5/8] reordered sections --- docs/collections/electric-collection.md | 27 ++++++++++++------------- 1 file changed, 13 insertions(+), 14 deletions(-) diff --git a/docs/collections/electric-collection.md b/docs/collections/electric-collection.md index d1a4014db..9a930eeee 100644 --- a/docs/collections/electric-collection.md +++ b/docs/collections/electric-collection.md @@ -127,7 +127,7 @@ onInsert: async ({ transaction }) => { } ``` -## Using a proxy with Electric +### Electric Proxy Example Electric is typically deployed behind a proxy server that handles shape configuration, authentication and authorization. This provides better security and allows you to control what data users can access without exposing Electric to the client. @@ -182,19 +182,6 @@ export const ServerRoute = createServerFileRoute("/api/todos").methods({ }) ``` - -## Utility Methods - -The collection provides these utility methods via `collection.utils`: - -- `awaitTxId(txid, timeout?)`: Manually wait for a specific transaction ID to be synchronized - -```typescript -todosCollection.utils.awaitTxId(12345) -``` - -This is useful when you need to ensure a mutation has been synchronized before proceeding with other operations. - ## Optimistic Updates with Explicit Transactions For more advanced use cases, you can create custom actions that can do multiple mutations across collections transactionally. In this case, you need to explicitly await for the transaction ID using `utils.awaitTxId()`. @@ -224,3 +211,15 @@ const addTodoAction = createOptimisticAction({ } }) ``` + +## Utility Methods + +The collection provides these utility methods via `collection.utils`: + +- `awaitTxId(txid, timeout?)`: Manually wait for a specific transaction ID to be synchronized + +```typescript +todosCollection.utils.awaitTxId(12345) +``` + +This is useful when you need to ensure a mutation has been synchronized before proceeding with other operations. From 86e1471bd9aaee9bd570ab6846b795c5b646805e Mon Sep 17 00:00:00 2001 From: Valter Balegas Date: Fri, 22 Aug 2025 23:03:20 +0100 Subject: [PATCH 6/8] Addressed last comment --- docs/collections/electric-collection.md | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/docs/collections/electric-collection.md b/docs/collections/electric-collection.md index 9a930eeee..7bc69272b 100644 --- a/docs/collections/electric-collection.md +++ b/docs/collections/electric-collection.md @@ -109,24 +109,6 @@ async function generateTxId(tx) { } ``` -It might not always be feasible to return transactions identifiers from your API. In that case, you can use a custom match function. - -```js -onInsert: async ({ transaction }) => { - const item = transaction.mutations[0].modified - await api.todos.create(item) // API doesn't return txid - - return { - matchStream: (stream) => matchStream( - stream, - ['insert'], - (msg) => msg.value.id === item.id, - 3000 // timeout fallback - ) - } -} -``` - ### Electric Proxy Example Electric is typically deployed behind a proxy server that handles shape configuration, authentication and authorization. This provides better security and allows you to control what data users can access without exposing Electric to the client. From b482df707f6e0aa81c23ad7af5221296f9d95d18 Mon Sep 17 00:00:00 2001 From: Valter Balegas Date: Tue, 26 Aug 2025 00:55:25 +0100 Subject: [PATCH 7/8] addressed more feedback --- docs/collections/electric-collection.md | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/docs/collections/electric-collection.md b/docs/collections/electric-collection.md index 7bc69272b..195154f09 100644 --- a/docs/collections/electric-collection.md +++ b/docs/collections/electric-collection.md @@ -118,6 +118,7 @@ Here is an example proxy implementation using TanStack Starter: ```js import { createServerFileRoute } from "@tanstack/react-start/server" +import { ELECTRIC_PROTOCOL_QUERY_PARAMS } from "@electric-sql/client" // Electric URL const baseUrl = 'http://.../v1/shape' @@ -130,21 +131,18 @@ const serve = async ({ request }: { request: Request }) => { // passthrough parameters from electric client url.searchParams.forEach((value, key) => { - if ( - [ - "live", - "handle", - "offset", - "cursor", - ].includes(key) - ) { + if (ELECTRIC_PROTOCOL_QUERY_PARAMS.includes(key)) { originUrl.searchParams.set(key, value) } }) - // the shape parameters + // set shape parameters + // full spec: https://github.com/electric-sql/electric/blob/main/website/electric-api.yaml originUrl.searchParams.set("table", "todos") + // Where clause to filter rows in the table (optional). // originUrl.searchParams.set("where", "completed = true") + + // Select the columns to sync (optional) // originUrl.searchParams.set("columns", "id,text,completed") const response = await fetch(originUrl) From f88cf4d8dbe1edb03877e134d179ce5d510acd5b Mon Sep 17 00:00:00 2001 From: Valter Balegas Date: Tue, 26 Aug 2025 01:02:32 +0100 Subject: [PATCH 8/8] indentation --- docs/collections/electric-collection.md | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/docs/collections/electric-collection.md b/docs/collections/electric-collection.md index 195154f09..c57336df3 100644 --- a/docs/collections/electric-collection.md +++ b/docs/collections/electric-collection.md @@ -22,7 +22,7 @@ npm install @tanstack/electric-db-collection @tanstack/react-db ## Basic Usage ```typescript -import { createCollection } from '@tanstack/react-db' +import { createCollection } from '@tanstack/react-db' import { electricCollectionOptions } from '@tanstack/electric-db-collection' const todosCollection = createCollection( @@ -70,8 +70,7 @@ const todosCollection = createCollection( id: 'todos', schema: todoSchema, getKey: (item) => item.id, - - shapeOptions: { + shapeOptions: { url: '/api/todos', params: { table: 'todos' }, }, @@ -124,9 +123,8 @@ import { ELECTRIC_PROTOCOL_QUERY_PARAMS } from "@electric-sql/client" const baseUrl = 'http://.../v1/shape' const serve = async ({ request }: { request: Request }) => { - // ...check user authorization - - const url = new URL(request.url) + // ...check user authorization + const url = new URL(request.url) const originUrl = new URL(baseUrl) // passthrough parameters from electric client @@ -186,8 +184,7 @@ const addTodoAction = createOptimisticAction({ data: { text, completed: false } }) - await todosCollection.utils.awaitTxId(response.txid) - + await todosCollection.utils.awaitTxId(response.txid) } }) ```