From c038b81398eff8f0143675ece8b07b75c1d7090d Mon Sep 17 00:00:00 2001 From: worthyfarmstead-rgb Date: Sat, 11 Apr 2026 06:57:22 -0400 Subject: [PATCH] fix: stream database dumps to handle large databases Rewrites SQL dump, CSV, and JSON export to use ReadableStream with chunked LIMIT/OFFSET pagination (5000 rows per chunk). Prevents OOM on large databases by never loading all rows into memory at once. Also adds proper NULL handling, binary data as hex literals, and table existence checks. Closes #59 Co-Authored-By: Claude Opus 4.6 (1M context) --- src/export/csv.test.ts | 81 ++++++++++----------- src/export/csv.ts | 108 ++++++++++++++++++++-------- src/export/dump.test.ts | 72 +++++++++++++++++-- src/export/dump.ts | 154 ++++++++++++++++++++++++++++------------ src/export/index.ts | 66 +++++++++++++++++ src/export/json.test.ts | 83 ++++++++++------------ src/export/json.ts | 64 +++++++++++++++-- 7 files changed, 448 insertions(+), 180 deletions(-) diff --git a/src/export/csv.test.ts b/src/export/csv.test.ts index b186aeb..2fa73e4 100644 --- a/src/export/csv.test.ts +++ b/src/export/csv.test.ts @@ -1,13 +1,22 @@ import { describe, it, expect, vi, beforeEach } from 'vitest' import { exportTableToCsvRoute } from './csv' -import { getTableData, createExportResponse } from './index' +import { tableExists, getTableDataChunked } from './index' import { createResponse } from '../utils' import type { DataSource } from '../types' import type { StarbaseDBConfiguration } from '../handler' vi.mock('./index', () => ({ - getTableData: vi.fn(), - createExportResponse: vi.fn(), + tableExists: vi.fn(), + getTableDataChunked: vi.fn(), + createStreamingExportResponse: vi.fn((stream, fileName, contentType) => { + const headers = new Headers({ + 'Content-Type': contentType, + 'Content-Disposition': `attachment; filename="${fileName}"`, + 'Transfer-Encoding': 'chunked', + }) + return new Response(stream, { headers }) + }), + CHUNK_SIZE: 5000, })) vi.mock('../utils', () => ({ @@ -43,38 +52,33 @@ beforeEach(() => { describe('CSV Export Module', () => { it('should return a CSV file when table data exists', async () => { - vi.mocked(getTableData).mockResolvedValue([ + vi.mocked(tableExists).mockResolvedValue(true) + + // Fewer rows than CHUNK_SIZE -> loop breaks after first call + vi.mocked(getTableDataChunked).mockResolvedValueOnce([ { id: 1, name: 'Alice', age: 30 }, { id: 2, name: 'Bob', age: 25 }, ]) - vi.mocked(createExportResponse).mockReturnValue( - new Response('mocked-csv-content', { - headers: { 'Content-Type': 'text/csv' }, - }) - ) - const response = await exportTableToCsvRoute( 'users', mockDataSource, mockConfig ) - expect(getTableData).toHaveBeenCalledWith( + expect(tableExists).toHaveBeenCalledWith( 'users', mockDataSource, mockConfig ) - expect(createExportResponse).toHaveBeenCalledWith( - 'id,name,age\n1,Alice,30\n2,Bob,25\n', - 'users_export.csv', - 'text/csv' - ) expect(response.headers.get('Content-Type')).toBe('text/csv') + + const text = await response.text() + expect(text).toBe('id,name,age\n1,Alice,30\n2,Bob,25\n') }) it('should return 404 if table does not exist', async () => { - vi.mocked(getTableData).mockResolvedValue(null) + vi.mocked(tableExists).mockResolvedValue(false) const response = await exportTableToCsvRoute( 'non_existent_table', @@ -82,7 +86,7 @@ describe('CSV Export Module', () => { mockConfig ) - expect(getTableData).toHaveBeenCalledWith( + expect(tableExists).toHaveBeenCalledWith( 'non_existent_table', mockDataSource, mockConfig @@ -95,14 +99,9 @@ describe('CSV Export Module', () => { ) }) - it('should handle empty table (return only headers)', async () => { - vi.mocked(getTableData).mockResolvedValue([]) - - vi.mocked(createExportResponse).mockReturnValue( - new Response('mocked-csv-content', { - headers: { 'Content-Type': 'text/csv' }, - }) - ) + it('should handle empty table (return empty stream)', async () => { + vi.mocked(tableExists).mockResolvedValue(true) + vi.mocked(getTableDataChunked).mockResolvedValueOnce([]) const response = await exportTableToCsvRoute( 'empty_table', @@ -110,49 +109,41 @@ describe('CSV Export Module', () => { mockConfig ) - expect(getTableData).toHaveBeenCalledWith( + expect(tableExists).toHaveBeenCalledWith( 'empty_table', mockDataSource, mockConfig ) - expect(createExportResponse).toHaveBeenCalledWith( - '', - 'empty_table_export.csv', - 'text/csv' - ) expect(response.headers.get('Content-Type')).toBe('text/csv') + + const text = await response.text() + expect(text).toBe('') }) it('should escape commas and quotes in CSV values', async () => { - vi.mocked(getTableData).mockResolvedValue([ + vi.mocked(tableExists).mockResolvedValue(true) + + vi.mocked(getTableDataChunked).mockResolvedValueOnce([ { id: 1, name: 'Sahithi, is', bio: 'my forever "penguin"' }, ]) - vi.mocked(createExportResponse).mockReturnValue( - new Response('mocked-csv-content', { - headers: { 'Content-Type': 'text/csv' }, - }) - ) - const response = await exportTableToCsvRoute( 'special_chars', mockDataSource, mockConfig ) - expect(createExportResponse).toHaveBeenCalledWith( - 'id,name,bio\n1,"Sahithi, is","my forever ""penguin"""\n', - 'special_chars_export.csv', - 'text/csv' + const text = await response.text() + expect(text).toBe( + 'id,name,bio\n1,"Sahithi, is","my forever ""penguin"""\n' ) - expect(response.headers.get('Content-Type')).toBe('text/csv') }) it('should return 500 on an unexpected error', async () => { const consoleErrorMock = vi .spyOn(console, 'error') .mockImplementation(() => {}) - vi.mocked(getTableData).mockRejectedValue(new Error('Database Error')) + vi.mocked(tableExists).mockRejectedValue(new Error('Database Error')) const response = await exportTableToCsvRoute( 'users', diff --git a/src/export/csv.ts b/src/export/csv.ts index 22a4591..a54ed96 100644 --- a/src/export/csv.ts +++ b/src/export/csv.ts @@ -1,17 +1,39 @@ -import { getTableData, createExportResponse } from './index' +import { + tableExists, + getTableDataChunked, + createStreamingExportResponse, + CHUNK_SIZE, +} from './index' import { createResponse } from '../utils' import { DataSource } from '../types' import { StarbaseDBConfiguration } from '../handler' +/** + * Format a single value for CSV output. + * Wraps in double quotes and escapes inner quotes when the value contains + * commas, double quotes, or newlines. + */ +function formatCsvValue(value: unknown): string { + if (value === null || value === undefined) { + return '' + } + + const str = String(value) + if (str.includes(',') || str.includes('"') || str.includes('\n')) { + return `"${str.replace(/"/g, '""')}"` + } + return str +} + export async function exportTableToCsvRoute( tableName: string, dataSource: DataSource, config: StarbaseDBConfiguration ): Promise { try { - const data = await getTableData(tableName, dataSource, config) + const exists = await tableExists(tableName, dataSource, config) - if (data === null) { + if (!exists) { return createResponse( undefined, `Table '${tableName}' does not exist.`, @@ -19,33 +41,61 @@ export async function exportTableToCsvRoute( ) } - // Convert the result to CSV - let csvContent = '' - if (data.length > 0) { - // Add headers - csvContent += Object.keys(data[0]).join(',') + '\n' - - // Add data rows - data.forEach((row: any) => { - csvContent += - Object.values(row) - .map((value) => { - if ( - typeof value === 'string' && - (value.includes(',') || - value.includes('"') || - value.includes('\n')) - ) { - return `"${value.replace(/"/g, '""')}"` - } - return value - }) - .join(',') + '\n' - }) - } + const encoder = new TextEncoder() + let headerWritten = false + + const stream = new ReadableStream({ + async start(controller) { + try { + let offset = 0 + + while (true) { + const rows = await getTableDataChunked( + tableName, + offset, + CHUNK_SIZE, + dataSource, + config + ) + + if (!rows || rows.length === 0) { + break + } + + let chunk = '' + + // Write header row from first batch of results + if (!headerWritten) { + chunk += Object.keys(rows[0]).join(',') + '\n' + headerWritten = true + } + + // Write data rows + for (const row of rows) { + chunk += + Object.values(row) + .map(formatCsvValue) + .join(',') + '\n' + } + + controller.enqueue(encoder.encode(chunk)) + + if (rows.length < CHUNK_SIZE) { + break + } + + offset += CHUNK_SIZE + } + + controller.close() + } catch (error) { + controller.error(error) + } + }, + }) - return createExportResponse( - csvContent, + return createStreamingExportResponse( + stream, `${tableName}_export.csv`, 'text/csv' ) diff --git a/src/export/dump.test.ts b/src/export/dump.test.ts index ca65b43..99665c3 100644 --- a/src/export/dump.test.ts +++ b/src/export/dump.test.ts @@ -1,12 +1,22 @@ import { describe, it, expect, vi, beforeEach } from 'vitest' import { dumpDatabaseRoute } from './dump' -import { executeOperation } from '.' +import { executeOperation, getTableDataChunked } from '.' import { createResponse } from '../utils' import type { DataSource } from '../types' import type { StarbaseDBConfiguration } from '../handler' vi.mock('.', () => ({ executeOperation: vi.fn(), + getTableDataChunked: vi.fn(), + createStreamingExportResponse: vi.fn((stream, fileName, contentType) => { + const headers = new Headers({ + 'Content-Type': contentType, + 'Content-Disposition': `attachment; filename="${fileName}"`, + 'Transfer-Encoding': 'chunked', + }) + return new Response(stream, { headers }) + }), + CHUNK_SIZE: 5000, })) vi.mock('../utils', () => ({ @@ -39,19 +49,27 @@ beforeEach(() => { }) describe('Database Dump Module', () => { - it('should return a database dump when tables exist', async () => { + it('should return a streaming database dump when tables exist', async () => { vi.mocked(executeOperation) .mockResolvedValueOnce([{ name: 'users' }, { name: 'orders' }]) + // schema for users .mockResolvedValueOnce([ { sql: 'CREATE TABLE users (id INTEGER, name TEXT);' }, ]) + // schema for orders .mockResolvedValueOnce([ - { id: 1, name: 'Alice' }, - { id: 2, name: 'Bob' }, + { sql: 'CREATE TABLE orders (id INTEGER, total REAL);' }, ]) + + // Since rows.length < CHUNK_SIZE (5000), the loop breaks after + // the first chunk without requesting another one. + vi.mocked(getTableDataChunked) + // users data (fewer than CHUNK_SIZE rows -> done) .mockResolvedValueOnce([ - { sql: 'CREATE TABLE orders (id INTEGER, total REAL);' }, + { id: 1, name: 'Alice' }, + { id: 2, name: 'Bob' }, ]) + // orders data (fewer than CHUNK_SIZE rows -> done) .mockResolvedValueOnce([ { id: 1, total: 99.99 }, { id: 2, total: 49.5 }, @@ -99,7 +117,8 @@ describe('Database Dump Module', () => { .mockResolvedValueOnce([ { sql: 'CREATE TABLE users (id INTEGER, name TEXT);' }, ]) - .mockResolvedValueOnce([]) + + vi.mocked(getTableDataChunked).mockResolvedValueOnce([]) const response = await dumpDatabaseRoute(mockDataSource, mockConfig) @@ -117,7 +136,10 @@ describe('Database Dump Module', () => { .mockResolvedValueOnce([ { sql: 'CREATE TABLE users (id INTEGER, bio TEXT);' }, ]) - .mockResolvedValueOnce([{ id: 1, bio: "Alice's adventure" }]) + + vi.mocked(getTableDataChunked).mockResolvedValueOnce([ + { id: 1, bio: "Alice's adventure" }, + ]) const response = await dumpDatabaseRoute(mockDataSource, mockConfig) @@ -128,6 +150,42 @@ describe('Database Dump Module', () => { ) }) + it('should handle NULL values correctly', async () => { + vi.mocked(executeOperation) + .mockResolvedValueOnce([{ name: 'users' }]) + .mockResolvedValueOnce([ + { sql: 'CREATE TABLE users (id INTEGER, bio TEXT);' }, + ]) + + vi.mocked(getTableDataChunked).mockResolvedValueOnce([ + { id: 1, bio: null }, + ]) + + const response = await dumpDatabaseRoute(mockDataSource, mockConfig) + + const dumpText = await response.text() + expect(dumpText).toContain('INSERT INTO users VALUES (1, NULL);') + }) + + it('should handle binary data as hex literals', async () => { + vi.mocked(executeOperation) + .mockResolvedValueOnce([{ name: 'files' }]) + .mockResolvedValueOnce([ + { sql: 'CREATE TABLE files (id INTEGER, data BLOB);' }, + ]) + + const binaryData = new Uint8Array([0xde, 0xad, 0xbe, 0xef]) + + vi.mocked(getTableDataChunked).mockResolvedValueOnce([ + { id: 1, data: binaryData }, + ]) + + const response = await dumpDatabaseRoute(mockDataSource, mockConfig) + + const dumpText = await response.text() + expect(dumpText).toContain("INSERT INTO files VALUES (1, X'deadbeef');") + }) + it('should return a 500 response when an error occurs', async () => { const consoleErrorMock = vi .spyOn(console, 'error') diff --git a/src/export/dump.ts b/src/export/dump.ts index 91a2e89..e443fd0 100644 --- a/src/export/dump.ts +++ b/src/export/dump.ts @@ -1,8 +1,39 @@ -import { executeOperation } from '.' +import { + executeOperation, + getTableDataChunked, + createStreamingExportResponse, + CHUNK_SIZE, +} from '.' import { StarbaseDBConfiguration } from '../handler' import { DataSource } from '../types' import { createResponse } from '../utils' +/** + * Format a single value for a SQL INSERT statement. + * Handles strings (with quote escaping), nulls, binary data (as hex), + * and numeric types. + */ +function formatSqlValue(value: unknown): string { + if (value === null || value === undefined) { + return 'NULL' + } + + if (value instanceof ArrayBuffer || value instanceof Uint8Array) { + const bytes = + value instanceof ArrayBuffer ? new Uint8Array(value) : value + const hex = Array.from(bytes) + .map((b) => b.toString(16).padStart(2, '0')) + .join('') + return `X'${hex}'` + } + + if (typeof value === 'string') { + return `'${value.replace(/'/g, "''")}'` + } + + return String(value) +} + export async function dumpDatabaseRoute( dataSource: DataSource, config: StarbaseDBConfiguration @@ -16,54 +47,83 @@ export async function dumpDatabaseRoute( ) const tables = tablesResult.map((row: any) => row.name) - let dumpContent = 'SQLite format 3\0' // SQLite file header - - // Iterate through all tables - for (const table of tables) { - // Get table schema - const schemaResult = await executeOperation( - [ - { - sql: `SELECT sql FROM sqlite_master WHERE type='table' AND name='${table}';`, - }, - ], - dataSource, - config - ) - - if (schemaResult.length) { - const schema = schemaResult[0].sql - dumpContent += `\n-- Table: ${table}\n${schema};\n\n` - } - - // Get table data - const dataResult = await executeOperation( - [{ sql: `SELECT * FROM ${table};` }], - dataSource, - config - ) - - for (const row of dataResult) { - const values = Object.values(row).map((value) => - typeof value === 'string' - ? `'${value.replace(/'/g, "''")}'` - : value - ) - dumpContent += `INSERT INTO ${table} VALUES (${values.join(', ')});\n` - } - - dumpContent += '\n' - } - - // Create a Blob from the dump content - const blob = new Blob([dumpContent], { type: 'application/x-sqlite3' }) - - const headers = new Headers({ - 'Content-Type': 'application/x-sqlite3', - 'Content-Disposition': 'attachment; filename="database_dump.sql"', + + const encoder = new TextEncoder() + const stream = new ReadableStream({ + async start(controller) { + try { + // Write SQLite header + controller.enqueue(encoder.encode('SQLite format 3\0')) + + // Iterate through all tables + for (const table of tables) { + // Get table schema + const schemaResult = await executeOperation( + [ + { + sql: `SELECT sql FROM sqlite_master WHERE type='table' AND name='${table}';`, + }, + ], + dataSource, + config + ) + + if (schemaResult.length) { + const schema = schemaResult[0].sql + controller.enqueue( + encoder.encode( + `\n-- Table: ${table}\n${schema};\n\n` + ) + ) + } + + // Stream table data in chunks using LIMIT/OFFSET + let offset = 0 + while (true) { + const rows = await getTableDataChunked( + table, + offset, + CHUNK_SIZE, + dataSource, + config + ) + + if (!rows || rows.length === 0) { + break + } + + let chunk = '' + for (const row of rows) { + const values = + Object.values(row).map(formatSqlValue) + chunk += `INSERT INTO ${table} VALUES (${values.join(', ')});\n` + } + controller.enqueue(encoder.encode(chunk)) + + // If we got fewer rows than the chunk size, we've + // reached the end of the table + if (rows.length < CHUNK_SIZE) { + break + } + + offset += CHUNK_SIZE + } + + controller.enqueue(encoder.encode('\n')) + } + + controller.close() + } catch (error) { + controller.error(error) + } + }, }) - return new Response(blob, { headers }) + return createStreamingExportResponse( + stream, + 'database_dump.sql', + 'application/x-sqlite3' + ) } catch (error: any) { console.error('Database Dump Error:', error) return createResponse(undefined, 'Failed to create database dump', 500) diff --git a/src/export/index.ts b/src/export/index.ts index 9c40119..207b89d 100644 --- a/src/export/index.ts +++ b/src/export/index.ts @@ -2,6 +2,12 @@ import { DataSource } from '../types' import { executeTransaction } from '../operation' import { StarbaseDBConfiguration } from '../handler' +/** + * The number of rows to fetch per chunk when streaming table data. + * This keeps memory usage bounded regardless of table size. + */ +export const CHUNK_SIZE = 5000 + export async function executeOperation( queries: { sql: string; params?: any[] }[], dataSource: DataSource, @@ -54,6 +60,49 @@ export async function getTableData( } } +/** + * Check if a table exists in the database. + */ +export async function tableExists( + tableName: string, + dataSource: DataSource, + config: StarbaseDBConfiguration +): Promise { + const result = await executeOperation( + [ + { + sql: `SELECT name FROM sqlite_master WHERE type='table' AND name=?;`, + params: [tableName], + }, + ], + dataSource, + config + ) + return result && result.length > 0 +} + +/** + * Fetch a chunk of rows from a table using LIMIT/OFFSET pagination. + * Returns an empty array when no more rows are available. + */ +export async function getTableDataChunked( + tableName: string, + offset: number, + limit: number, + dataSource: DataSource, + config: StarbaseDBConfiguration +): Promise { + return executeOperation( + [ + { + sql: `SELECT * FROM "${tableName}" LIMIT ${limit} OFFSET ${offset};`, + }, + ], + dataSource, + config + ) +} + export function createExportResponse( data: any, fileName: string, @@ -68,3 +117,20 @@ export function createExportResponse( return new Response(blob, { headers }) } + +/** + * Create a streaming response from a ReadableStream. + */ +export function createStreamingExportResponse( + stream: ReadableStream, + fileName: string, + contentType: string +): Response { + const headers = new Headers({ + 'Content-Type': contentType, + 'Content-Disposition': `attachment; filename="${fileName}"`, + 'Transfer-Encoding': 'chunked', + }) + + return new Response(stream, { headers }) +} diff --git a/src/export/json.test.ts b/src/export/json.test.ts index 3fe4a8c..3b3d265 100644 --- a/src/export/json.test.ts +++ b/src/export/json.test.ts @@ -1,13 +1,22 @@ import { describe, it, expect, vi, beforeEach } from 'vitest' import { exportTableToJsonRoute } from './json' -import { getTableData, createExportResponse } from './index' +import { tableExists, getTableDataChunked } from './index' import { createResponse } from '../utils' import type { DataSource } from '../types' import type { StarbaseDBConfiguration } from '../handler' vi.mock('./index', () => ({ - getTableData: vi.fn(), - createExportResponse: vi.fn(), + tableExists: vi.fn(), + getTableDataChunked: vi.fn(), + createStreamingExportResponse: vi.fn((stream, fileName, contentType) => { + const headers = new Headers({ + 'Content-Type': contentType, + 'Content-Disposition': `attachment; filename="${fileName}"`, + 'Transfer-Encoding': 'chunked', + }) + return new Response(stream, { headers }) + }), + CHUNK_SIZE: 5000, })) vi.mock('../utils', () => ({ @@ -41,7 +50,7 @@ beforeEach(() => { describe('JSON Export Module', () => { it('should return a 404 response if table does not exist', async () => { - vi.mocked(getTableData).mockResolvedValue(null) + vi.mocked(tableExists).mockResolvedValue(false) const response = await exportTableToJsonRoute( 'missing_table', @@ -54,18 +63,14 @@ describe('JSON Export Module', () => { expect(jsonResponse.error).toBe("Table 'missing_table' does not exist.") }) - it('should return a JSON file when table data exists', async () => { - const mockData = [ + it('should return a streaming JSON file when table data exists', async () => { + vi.mocked(tableExists).mockResolvedValue(true) + + // Fewer rows than CHUNK_SIZE -> loop breaks after first call + vi.mocked(getTableDataChunked).mockResolvedValueOnce([ { id: 1, name: 'Alice' }, { id: 2, name: 'Bob' }, - ] - vi.mocked(getTableData).mockResolvedValue(mockData) - - vi.mocked(createExportResponse).mockReturnValue( - new Response('mocked-json-content', { - headers: { 'Content-Type': 'application/json' }, - }) - ) + ]) const response = await exportTableToJsonRoute( 'users', @@ -73,27 +78,23 @@ describe('JSON Export Module', () => { mockConfig ) - expect(getTableData).toHaveBeenCalledWith( + expect(tableExists).toHaveBeenCalledWith( 'users', mockDataSource, mockConfig ) - expect(createExportResponse).toHaveBeenCalledWith( - JSON.stringify(mockData, null, 4), - 'users_export.json', - 'application/json' - ) expect(response.headers.get('Content-Type')).toBe('application/json') + + const text = await response.text() + expect(text).toContain('{"id":1,"name":"Alice"}') + expect(text).toContain('{"id":2,"name":"Bob"}') + expect(text.startsWith('[')).toBe(true) + expect(text.trimEnd().endsWith(']')).toBe(true) }) it('should return an empty JSON array when table has no data', async () => { - vi.mocked(getTableData).mockResolvedValue([]) - - vi.mocked(createExportResponse).mockReturnValue( - new Response('mocked-json-content', { - headers: { 'Content-Type': 'application/json' }, - }) - ) + vi.mocked(tableExists).mockResolvedValue(true) + vi.mocked(getTableDataChunked).mockResolvedValueOnce([]) const response = await exportTableToJsonRoute( 'empty_table', @@ -101,12 +102,10 @@ describe('JSON Export Module', () => { mockConfig ) - expect(createExportResponse).toHaveBeenCalledWith( - '[]', - 'empty_table_export.json', - 'application/json' - ) expect(response.headers.get('Content-Type')).toBe('application/json') + + const text = await response.text() + expect(text.replace(/\s/g, '')).toBe('[]') }) it('should escape special characters in JSON properly', async () => { @@ -114,13 +113,8 @@ describe('JSON Export Module', () => { { id: 1, name: 'Sahithi "The Best"' }, { id: 2, description: 'New\nLine' }, ] - vi.mocked(getTableData).mockResolvedValue(specialCharsData) - - vi.mocked(createExportResponse).mockReturnValue( - new Response('mocked-json-content', { - headers: { 'Content-Type': 'application/json' }, - }) - ) + vi.mocked(tableExists).mockResolvedValue(true) + vi.mocked(getTableDataChunked).mockResolvedValueOnce(specialCharsData) const response = await exportTableToJsonRoute( 'special_chars', @@ -128,19 +122,18 @@ describe('JSON Export Module', () => { mockConfig ) - expect(createExportResponse).toHaveBeenCalledWith( - JSON.stringify(specialCharsData, null, 4), - 'special_chars_export.json', - 'application/json' - ) expect(response.headers.get('Content-Type')).toBe('application/json') + + const text = await response.text() + expect(text).toContain(JSON.stringify(specialCharsData[0])) + expect(text).toContain(JSON.stringify(specialCharsData[1])) }) it('should return a 500 response when an error occurs', async () => { const consoleErrorMock = vi .spyOn(console, 'error') .mockImplementation(() => {}) - vi.mocked(getTableData).mockRejectedValue(new Error('Database Error')) + vi.mocked(tableExists).mockRejectedValue(new Error('Database Error')) const response = await exportTableToJsonRoute( 'users', diff --git a/src/export/json.ts b/src/export/json.ts index c0ab811..be0b17b 100644 --- a/src/export/json.ts +++ b/src/export/json.ts @@ -1,4 +1,9 @@ -import { getTableData, createExportResponse } from './index' +import { + tableExists, + getTableDataChunked, + createStreamingExportResponse, + CHUNK_SIZE, +} from './index' import { createResponse } from '../utils' import { DataSource } from '../types' import { StarbaseDBConfiguration } from '../handler' @@ -9,9 +14,9 @@ export async function exportTableToJsonRoute( config: StarbaseDBConfiguration ): Promise { try { - const data = await getTableData(tableName, dataSource, config) + const exists = await tableExists(tableName, dataSource, config) - if (data === null) { + if (!exists) { return createResponse( undefined, `Table '${tableName}' does not exist.`, @@ -19,11 +24,56 @@ export async function exportTableToJsonRoute( ) } - // Convert the result to JSON - const jsonData = JSON.stringify(data, null, 4) + const encoder = new TextEncoder() + let isFirstRow = true - return createExportResponse( - jsonData, + const stream = new ReadableStream({ + async start(controller) { + try { + controller.enqueue(encoder.encode('[\n')) + + let offset = 0 + + while (true) { + const rows = await getTableDataChunked( + tableName, + offset, + CHUNK_SIZE, + dataSource, + config + ) + + if (!rows || rows.length === 0) { + break + } + + for (const row of rows) { + if (!isFirstRow) { + controller.enqueue(encoder.encode(',\n')) + } + controller.enqueue( + encoder.encode(' ' + JSON.stringify(row)) + ) + isFirstRow = false + } + + if (rows.length < CHUNK_SIZE) { + break + } + + offset += CHUNK_SIZE + } + + controller.enqueue(encoder.encode('\n]')) + controller.close() + } catch (error) { + controller.error(error) + } + }, + }) + + return createStreamingExportResponse( + stream, `${tableName}_export.json`, 'application/json' )