diff --git a/src/export/csv.test.ts b/src/export/csv.test.ts index b186aeb..2fa73e4 100644 --- a/src/export/csv.test.ts +++ b/src/export/csv.test.ts @@ -1,13 +1,22 @@ import { describe, it, expect, vi, beforeEach } from 'vitest' import { exportTableToCsvRoute } from './csv' -import { getTableData, createExportResponse } from './index' +import { tableExists, getTableDataChunked } from './index' import { createResponse } from '../utils' import type { DataSource } from '../types' import type { StarbaseDBConfiguration } from '../handler' vi.mock('./index', () => ({ - getTableData: vi.fn(), - createExportResponse: vi.fn(), + tableExists: vi.fn(), + getTableDataChunked: vi.fn(), + createStreamingExportResponse: vi.fn((stream, fileName, contentType) => { + const headers = new Headers({ + 'Content-Type': contentType, + 'Content-Disposition': `attachment; filename="${fileName}"`, + 'Transfer-Encoding': 'chunked', + }) + return new Response(stream, { headers }) + }), + CHUNK_SIZE: 5000, })) vi.mock('../utils', () => ({ @@ -43,38 +52,33 @@ beforeEach(() => { describe('CSV Export Module', () => { it('should return a CSV file when table data exists', async () => { - vi.mocked(getTableData).mockResolvedValue([ + vi.mocked(tableExists).mockResolvedValue(true) + + // Fewer rows than CHUNK_SIZE -> loop breaks after first call + vi.mocked(getTableDataChunked).mockResolvedValueOnce([ { id: 1, name: 'Alice', age: 30 }, { id: 2, name: 'Bob', age: 25 }, ]) - vi.mocked(createExportResponse).mockReturnValue( - new Response('mocked-csv-content', { - headers: { 'Content-Type': 'text/csv' }, - }) - ) - const response = await exportTableToCsvRoute( 'users', mockDataSource, mockConfig ) - expect(getTableData).toHaveBeenCalledWith( + expect(tableExists).toHaveBeenCalledWith( 'users', mockDataSource, mockConfig ) - expect(createExportResponse).toHaveBeenCalledWith( - 'id,name,age\n1,Alice,30\n2,Bob,25\n', - 'users_export.csv', - 'text/csv' - ) expect(response.headers.get('Content-Type')).toBe('text/csv') + + const text = await response.text() + expect(text).toBe('id,name,age\n1,Alice,30\n2,Bob,25\n') }) it('should return 404 if table does not exist', async () => { - vi.mocked(getTableData).mockResolvedValue(null) + vi.mocked(tableExists).mockResolvedValue(false) const response = await exportTableToCsvRoute( 'non_existent_table', @@ -82,7 +86,7 @@ describe('CSV Export Module', () => { mockConfig ) - expect(getTableData).toHaveBeenCalledWith( + expect(tableExists).toHaveBeenCalledWith( 'non_existent_table', mockDataSource, mockConfig @@ -95,14 +99,9 @@ describe('CSV Export Module', () => { ) }) - it('should handle empty table (return only headers)', async () => { - vi.mocked(getTableData).mockResolvedValue([]) - - vi.mocked(createExportResponse).mockReturnValue( - new Response('mocked-csv-content', { - headers: { 'Content-Type': 'text/csv' }, - }) - ) + it('should handle empty table (return empty stream)', async () => { + vi.mocked(tableExists).mockResolvedValue(true) + vi.mocked(getTableDataChunked).mockResolvedValueOnce([]) const response = await exportTableToCsvRoute( 'empty_table', @@ -110,49 +109,41 @@ describe('CSV Export Module', () => { mockConfig ) - expect(getTableData).toHaveBeenCalledWith( + expect(tableExists).toHaveBeenCalledWith( 'empty_table', mockDataSource, mockConfig ) - expect(createExportResponse).toHaveBeenCalledWith( - '', - 'empty_table_export.csv', - 'text/csv' - ) expect(response.headers.get('Content-Type')).toBe('text/csv') + + const text = await response.text() + expect(text).toBe('') }) it('should escape commas and quotes in CSV values', async () => { - vi.mocked(getTableData).mockResolvedValue([ + vi.mocked(tableExists).mockResolvedValue(true) + + vi.mocked(getTableDataChunked).mockResolvedValueOnce([ { id: 1, name: 'Sahithi, is', bio: 'my forever "penguin"' }, ]) - vi.mocked(createExportResponse).mockReturnValue( - new Response('mocked-csv-content', { - headers: { 'Content-Type': 'text/csv' }, - }) - ) - const response = await exportTableToCsvRoute( 'special_chars', mockDataSource, mockConfig ) - expect(createExportResponse).toHaveBeenCalledWith( - 'id,name,bio\n1,"Sahithi, is","my forever ""penguin"""\n', - 'special_chars_export.csv', - 'text/csv' + const text = await response.text() + expect(text).toBe( + 'id,name,bio\n1,"Sahithi, is","my forever ""penguin"""\n' ) - expect(response.headers.get('Content-Type')).toBe('text/csv') }) it('should return 500 on an unexpected error', async () => { const consoleErrorMock = vi .spyOn(console, 'error') .mockImplementation(() => {}) - vi.mocked(getTableData).mockRejectedValue(new Error('Database Error')) + vi.mocked(tableExists).mockRejectedValue(new Error('Database Error')) const response = await exportTableToCsvRoute( 'users', diff --git a/src/export/csv.ts b/src/export/csv.ts index 22a4591..a54ed96 100644 --- a/src/export/csv.ts +++ b/src/export/csv.ts @@ -1,17 +1,39 @@ -import { getTableData, createExportResponse } from './index' +import { + tableExists, + getTableDataChunked, + createStreamingExportResponse, + CHUNK_SIZE, +} from './index' import { createResponse } from '../utils' import { DataSource } from '../types' import { StarbaseDBConfiguration } from '../handler' +/** + * Format a single value for CSV output. + * Wraps in double quotes and escapes inner quotes when the value contains + * commas, double quotes, or newlines. + */ +function formatCsvValue(value: unknown): string { + if (value === null || value === undefined) { + return '' + } + + const str = String(value) + if (str.includes(',') || str.includes('"') || str.includes('\n')) { + return `"${str.replace(/"/g, '""')}"` + } + return str +} + export async function exportTableToCsvRoute( tableName: string, dataSource: DataSource, config: StarbaseDBConfiguration ): Promise { try { - const data = await getTableData(tableName, dataSource, config) + const exists = await tableExists(tableName, dataSource, config) - if (data === null) { + if (!exists) { return createResponse( undefined, `Table '${tableName}' does not exist.`, @@ -19,33 +41,61 @@ export async function exportTableToCsvRoute( ) } - // Convert the result to CSV - let csvContent = '' - if (data.length > 0) { - // Add headers - csvContent += Object.keys(data[0]).join(',') + '\n' - - // Add data rows - data.forEach((row: any) => { - csvContent += - Object.values(row) - .map((value) => { - if ( - typeof value === 'string' && - (value.includes(',') || - value.includes('"') || - value.includes('\n')) - ) { - return `"${value.replace(/"/g, '""')}"` - } - return value - }) - .join(',') + '\n' - }) - } + const encoder = new TextEncoder() + let headerWritten = false + + const stream = new ReadableStream({ + async start(controller) { + try { + let offset = 0 + + while (true) { + const rows = await getTableDataChunked( + tableName, + offset, + CHUNK_SIZE, + dataSource, + config + ) + + if (!rows || rows.length === 0) { + break + } + + let chunk = '' + + // Write header row from first batch of results + if (!headerWritten) { + chunk += Object.keys(rows[0]).join(',') + '\n' + headerWritten = true + } + + // Write data rows + for (const row of rows) { + chunk += + Object.values(row) + .map(formatCsvValue) + .join(',') + '\n' + } + + controller.enqueue(encoder.encode(chunk)) + + if (rows.length < CHUNK_SIZE) { + break + } + + offset += CHUNK_SIZE + } + + controller.close() + } catch (error) { + controller.error(error) + } + }, + }) - return createExportResponse( - csvContent, + return createStreamingExportResponse( + stream, `${tableName}_export.csv`, 'text/csv' ) diff --git a/src/export/dump.test.ts b/src/export/dump.test.ts index ca65b43..99665c3 100644 --- a/src/export/dump.test.ts +++ b/src/export/dump.test.ts @@ -1,12 +1,22 @@ import { describe, it, expect, vi, beforeEach } from 'vitest' import { dumpDatabaseRoute } from './dump' -import { executeOperation } from '.' +import { executeOperation, getTableDataChunked } from '.' import { createResponse } from '../utils' import type { DataSource } from '../types' import type { StarbaseDBConfiguration } from '../handler' vi.mock('.', () => ({ executeOperation: vi.fn(), + getTableDataChunked: vi.fn(), + createStreamingExportResponse: vi.fn((stream, fileName, contentType) => { + const headers = new Headers({ + 'Content-Type': contentType, + 'Content-Disposition': `attachment; filename="${fileName}"`, + 'Transfer-Encoding': 'chunked', + }) + return new Response(stream, { headers }) + }), + CHUNK_SIZE: 5000, })) vi.mock('../utils', () => ({ @@ -39,19 +49,27 @@ beforeEach(() => { }) describe('Database Dump Module', () => { - it('should return a database dump when tables exist', async () => { + it('should return a streaming database dump when tables exist', async () => { vi.mocked(executeOperation) .mockResolvedValueOnce([{ name: 'users' }, { name: 'orders' }]) + // schema for users .mockResolvedValueOnce([ { sql: 'CREATE TABLE users (id INTEGER, name TEXT);' }, ]) + // schema for orders .mockResolvedValueOnce([ - { id: 1, name: 'Alice' }, - { id: 2, name: 'Bob' }, + { sql: 'CREATE TABLE orders (id INTEGER, total REAL);' }, ]) + + // Since rows.length < CHUNK_SIZE (5000), the loop breaks after + // the first chunk without requesting another one. + vi.mocked(getTableDataChunked) + // users data (fewer than CHUNK_SIZE rows -> done) .mockResolvedValueOnce([ - { sql: 'CREATE TABLE orders (id INTEGER, total REAL);' }, + { id: 1, name: 'Alice' }, + { id: 2, name: 'Bob' }, ]) + // orders data (fewer than CHUNK_SIZE rows -> done) .mockResolvedValueOnce([ { id: 1, total: 99.99 }, { id: 2, total: 49.5 }, @@ -99,7 +117,8 @@ describe('Database Dump Module', () => { .mockResolvedValueOnce([ { sql: 'CREATE TABLE users (id INTEGER, name TEXT);' }, ]) - .mockResolvedValueOnce([]) + + vi.mocked(getTableDataChunked).mockResolvedValueOnce([]) const response = await dumpDatabaseRoute(mockDataSource, mockConfig) @@ -117,7 +136,10 @@ describe('Database Dump Module', () => { .mockResolvedValueOnce([ { sql: 'CREATE TABLE users (id INTEGER, bio TEXT);' }, ]) - .mockResolvedValueOnce([{ id: 1, bio: "Alice's adventure" }]) + + vi.mocked(getTableDataChunked).mockResolvedValueOnce([ + { id: 1, bio: "Alice's adventure" }, + ]) const response = await dumpDatabaseRoute(mockDataSource, mockConfig) @@ -128,6 +150,42 @@ describe('Database Dump Module', () => { ) }) + it('should handle NULL values correctly', async () => { + vi.mocked(executeOperation) + .mockResolvedValueOnce([{ name: 'users' }]) + .mockResolvedValueOnce([ + { sql: 'CREATE TABLE users (id INTEGER, bio TEXT);' }, + ]) + + vi.mocked(getTableDataChunked).mockResolvedValueOnce([ + { id: 1, bio: null }, + ]) + + const response = await dumpDatabaseRoute(mockDataSource, mockConfig) + + const dumpText = await response.text() + expect(dumpText).toContain('INSERT INTO users VALUES (1, NULL);') + }) + + it('should handle binary data as hex literals', async () => { + vi.mocked(executeOperation) + .mockResolvedValueOnce([{ name: 'files' }]) + .mockResolvedValueOnce([ + { sql: 'CREATE TABLE files (id INTEGER, data BLOB);' }, + ]) + + const binaryData = new Uint8Array([0xde, 0xad, 0xbe, 0xef]) + + vi.mocked(getTableDataChunked).mockResolvedValueOnce([ + { id: 1, data: binaryData }, + ]) + + const response = await dumpDatabaseRoute(mockDataSource, mockConfig) + + const dumpText = await response.text() + expect(dumpText).toContain("INSERT INTO files VALUES (1, X'deadbeef');") + }) + it('should return a 500 response when an error occurs', async () => { const consoleErrorMock = vi .spyOn(console, 'error') diff --git a/src/export/dump.ts b/src/export/dump.ts index 91a2e89..e443fd0 100644 --- a/src/export/dump.ts +++ b/src/export/dump.ts @@ -1,8 +1,39 @@ -import { executeOperation } from '.' +import { + executeOperation, + getTableDataChunked, + createStreamingExportResponse, + CHUNK_SIZE, +} from '.' import { StarbaseDBConfiguration } from '../handler' import { DataSource } from '../types' import { createResponse } from '../utils' +/** + * Format a single value for a SQL INSERT statement. + * Handles strings (with quote escaping), nulls, binary data (as hex), + * and numeric types. + */ +function formatSqlValue(value: unknown): string { + if (value === null || value === undefined) { + return 'NULL' + } + + if (value instanceof ArrayBuffer || value instanceof Uint8Array) { + const bytes = + value instanceof ArrayBuffer ? new Uint8Array(value) : value + const hex = Array.from(bytes) + .map((b) => b.toString(16).padStart(2, '0')) + .join('') + return `X'${hex}'` + } + + if (typeof value === 'string') { + return `'${value.replace(/'/g, "''")}'` + } + + return String(value) +} + export async function dumpDatabaseRoute( dataSource: DataSource, config: StarbaseDBConfiguration @@ -16,54 +47,83 @@ export async function dumpDatabaseRoute( ) const tables = tablesResult.map((row: any) => row.name) - let dumpContent = 'SQLite format 3\0' // SQLite file header - - // Iterate through all tables - for (const table of tables) { - // Get table schema - const schemaResult = await executeOperation( - [ - { - sql: `SELECT sql FROM sqlite_master WHERE type='table' AND name='${table}';`, - }, - ], - dataSource, - config - ) - - if (schemaResult.length) { - const schema = schemaResult[0].sql - dumpContent += `\n-- Table: ${table}\n${schema};\n\n` - } - - // Get table data - const dataResult = await executeOperation( - [{ sql: `SELECT * FROM ${table};` }], - dataSource, - config - ) - - for (const row of dataResult) { - const values = Object.values(row).map((value) => - typeof value === 'string' - ? `'${value.replace(/'/g, "''")}'` - : value - ) - dumpContent += `INSERT INTO ${table} VALUES (${values.join(', ')});\n` - } - - dumpContent += '\n' - } - - // Create a Blob from the dump content - const blob = new Blob([dumpContent], { type: 'application/x-sqlite3' }) - - const headers = new Headers({ - 'Content-Type': 'application/x-sqlite3', - 'Content-Disposition': 'attachment; filename="database_dump.sql"', + + const encoder = new TextEncoder() + const stream = new ReadableStream({ + async start(controller) { + try { + // Write SQLite header + controller.enqueue(encoder.encode('SQLite format 3\0')) + + // Iterate through all tables + for (const table of tables) { + // Get table schema + const schemaResult = await executeOperation( + [ + { + sql: `SELECT sql FROM sqlite_master WHERE type='table' AND name='${table}';`, + }, + ], + dataSource, + config + ) + + if (schemaResult.length) { + const schema = schemaResult[0].sql + controller.enqueue( + encoder.encode( + `\n-- Table: ${table}\n${schema};\n\n` + ) + ) + } + + // Stream table data in chunks using LIMIT/OFFSET + let offset = 0 + while (true) { + const rows = await getTableDataChunked( + table, + offset, + CHUNK_SIZE, + dataSource, + config + ) + + if (!rows || rows.length === 0) { + break + } + + let chunk = '' + for (const row of rows) { + const values = + Object.values(row).map(formatSqlValue) + chunk += `INSERT INTO ${table} VALUES (${values.join(', ')});\n` + } + controller.enqueue(encoder.encode(chunk)) + + // If we got fewer rows than the chunk size, we've + // reached the end of the table + if (rows.length < CHUNK_SIZE) { + break + } + + offset += CHUNK_SIZE + } + + controller.enqueue(encoder.encode('\n')) + } + + controller.close() + } catch (error) { + controller.error(error) + } + }, }) - return new Response(blob, { headers }) + return createStreamingExportResponse( + stream, + 'database_dump.sql', + 'application/x-sqlite3' + ) } catch (error: any) { console.error('Database Dump Error:', error) return createResponse(undefined, 'Failed to create database dump', 500) diff --git a/src/export/index.ts b/src/export/index.ts index 9c40119..207b89d 100644 --- a/src/export/index.ts +++ b/src/export/index.ts @@ -2,6 +2,12 @@ import { DataSource } from '../types' import { executeTransaction } from '../operation' import { StarbaseDBConfiguration } from '../handler' +/** + * The number of rows to fetch per chunk when streaming table data. + * This keeps memory usage bounded regardless of table size. + */ +export const CHUNK_SIZE = 5000 + export async function executeOperation( queries: { sql: string; params?: any[] }[], dataSource: DataSource, @@ -54,6 +60,49 @@ export async function getTableData( } } +/** + * Check if a table exists in the database. + */ +export async function tableExists( + tableName: string, + dataSource: DataSource, + config: StarbaseDBConfiguration +): Promise { + const result = await executeOperation( + [ + { + sql: `SELECT name FROM sqlite_master WHERE type='table' AND name=?;`, + params: [tableName], + }, + ], + dataSource, + config + ) + return result && result.length > 0 +} + +/** + * Fetch a chunk of rows from a table using LIMIT/OFFSET pagination. + * Returns an empty array when no more rows are available. + */ +export async function getTableDataChunked( + tableName: string, + offset: number, + limit: number, + dataSource: DataSource, + config: StarbaseDBConfiguration +): Promise { + return executeOperation( + [ + { + sql: `SELECT * FROM "${tableName}" LIMIT ${limit} OFFSET ${offset};`, + }, + ], + dataSource, + config + ) +} + export function createExportResponse( data: any, fileName: string, @@ -68,3 +117,20 @@ export function createExportResponse( return new Response(blob, { headers }) } + +/** + * Create a streaming response from a ReadableStream. + */ +export function createStreamingExportResponse( + stream: ReadableStream, + fileName: string, + contentType: string +): Response { + const headers = new Headers({ + 'Content-Type': contentType, + 'Content-Disposition': `attachment; filename="${fileName}"`, + 'Transfer-Encoding': 'chunked', + }) + + return new Response(stream, { headers }) +} diff --git a/src/export/json.test.ts b/src/export/json.test.ts index 3fe4a8c..3b3d265 100644 --- a/src/export/json.test.ts +++ b/src/export/json.test.ts @@ -1,13 +1,22 @@ import { describe, it, expect, vi, beforeEach } from 'vitest' import { exportTableToJsonRoute } from './json' -import { getTableData, createExportResponse } from './index' +import { tableExists, getTableDataChunked } from './index' import { createResponse } from '../utils' import type { DataSource } from '../types' import type { StarbaseDBConfiguration } from '../handler' vi.mock('./index', () => ({ - getTableData: vi.fn(), - createExportResponse: vi.fn(), + tableExists: vi.fn(), + getTableDataChunked: vi.fn(), + createStreamingExportResponse: vi.fn((stream, fileName, contentType) => { + const headers = new Headers({ + 'Content-Type': contentType, + 'Content-Disposition': `attachment; filename="${fileName}"`, + 'Transfer-Encoding': 'chunked', + }) + return new Response(stream, { headers }) + }), + CHUNK_SIZE: 5000, })) vi.mock('../utils', () => ({ @@ -41,7 +50,7 @@ beforeEach(() => { describe('JSON Export Module', () => { it('should return a 404 response if table does not exist', async () => { - vi.mocked(getTableData).mockResolvedValue(null) + vi.mocked(tableExists).mockResolvedValue(false) const response = await exportTableToJsonRoute( 'missing_table', @@ -54,18 +63,14 @@ describe('JSON Export Module', () => { expect(jsonResponse.error).toBe("Table 'missing_table' does not exist.") }) - it('should return a JSON file when table data exists', async () => { - const mockData = [ + it('should return a streaming JSON file when table data exists', async () => { + vi.mocked(tableExists).mockResolvedValue(true) + + // Fewer rows than CHUNK_SIZE -> loop breaks after first call + vi.mocked(getTableDataChunked).mockResolvedValueOnce([ { id: 1, name: 'Alice' }, { id: 2, name: 'Bob' }, - ] - vi.mocked(getTableData).mockResolvedValue(mockData) - - vi.mocked(createExportResponse).mockReturnValue( - new Response('mocked-json-content', { - headers: { 'Content-Type': 'application/json' }, - }) - ) + ]) const response = await exportTableToJsonRoute( 'users', @@ -73,27 +78,23 @@ describe('JSON Export Module', () => { mockConfig ) - expect(getTableData).toHaveBeenCalledWith( + expect(tableExists).toHaveBeenCalledWith( 'users', mockDataSource, mockConfig ) - expect(createExportResponse).toHaveBeenCalledWith( - JSON.stringify(mockData, null, 4), - 'users_export.json', - 'application/json' - ) expect(response.headers.get('Content-Type')).toBe('application/json') + + const text = await response.text() + expect(text).toContain('{"id":1,"name":"Alice"}') + expect(text).toContain('{"id":2,"name":"Bob"}') + expect(text.startsWith('[')).toBe(true) + expect(text.trimEnd().endsWith(']')).toBe(true) }) it('should return an empty JSON array when table has no data', async () => { - vi.mocked(getTableData).mockResolvedValue([]) - - vi.mocked(createExportResponse).mockReturnValue( - new Response('mocked-json-content', { - headers: { 'Content-Type': 'application/json' }, - }) - ) + vi.mocked(tableExists).mockResolvedValue(true) + vi.mocked(getTableDataChunked).mockResolvedValueOnce([]) const response = await exportTableToJsonRoute( 'empty_table', @@ -101,12 +102,10 @@ describe('JSON Export Module', () => { mockConfig ) - expect(createExportResponse).toHaveBeenCalledWith( - '[]', - 'empty_table_export.json', - 'application/json' - ) expect(response.headers.get('Content-Type')).toBe('application/json') + + const text = await response.text() + expect(text.replace(/\s/g, '')).toBe('[]') }) it('should escape special characters in JSON properly', async () => { @@ -114,13 +113,8 @@ describe('JSON Export Module', () => { { id: 1, name: 'Sahithi "The Best"' }, { id: 2, description: 'New\nLine' }, ] - vi.mocked(getTableData).mockResolvedValue(specialCharsData) - - vi.mocked(createExportResponse).mockReturnValue( - new Response('mocked-json-content', { - headers: { 'Content-Type': 'application/json' }, - }) - ) + vi.mocked(tableExists).mockResolvedValue(true) + vi.mocked(getTableDataChunked).mockResolvedValueOnce(specialCharsData) const response = await exportTableToJsonRoute( 'special_chars', @@ -128,19 +122,18 @@ describe('JSON Export Module', () => { mockConfig ) - expect(createExportResponse).toHaveBeenCalledWith( - JSON.stringify(specialCharsData, null, 4), - 'special_chars_export.json', - 'application/json' - ) expect(response.headers.get('Content-Type')).toBe('application/json') + + const text = await response.text() + expect(text).toContain(JSON.stringify(specialCharsData[0])) + expect(text).toContain(JSON.stringify(specialCharsData[1])) }) it('should return a 500 response when an error occurs', async () => { const consoleErrorMock = vi .spyOn(console, 'error') .mockImplementation(() => {}) - vi.mocked(getTableData).mockRejectedValue(new Error('Database Error')) + vi.mocked(tableExists).mockRejectedValue(new Error('Database Error')) const response = await exportTableToJsonRoute( 'users', diff --git a/src/export/json.ts b/src/export/json.ts index c0ab811..be0b17b 100644 --- a/src/export/json.ts +++ b/src/export/json.ts @@ -1,4 +1,9 @@ -import { getTableData, createExportResponse } from './index' +import { + tableExists, + getTableDataChunked, + createStreamingExportResponse, + CHUNK_SIZE, +} from './index' import { createResponse } from '../utils' import { DataSource } from '../types' import { StarbaseDBConfiguration } from '../handler' @@ -9,9 +14,9 @@ export async function exportTableToJsonRoute( config: StarbaseDBConfiguration ): Promise { try { - const data = await getTableData(tableName, dataSource, config) + const exists = await tableExists(tableName, dataSource, config) - if (data === null) { + if (!exists) { return createResponse( undefined, `Table '${tableName}' does not exist.`, @@ -19,11 +24,56 @@ export async function exportTableToJsonRoute( ) } - // Convert the result to JSON - const jsonData = JSON.stringify(data, null, 4) + const encoder = new TextEncoder() + let isFirstRow = true - return createExportResponse( - jsonData, + const stream = new ReadableStream({ + async start(controller) { + try { + controller.enqueue(encoder.encode('[\n')) + + let offset = 0 + + while (true) { + const rows = await getTableDataChunked( + tableName, + offset, + CHUNK_SIZE, + dataSource, + config + ) + + if (!rows || rows.length === 0) { + break + } + + for (const row of rows) { + if (!isFirstRow) { + controller.enqueue(encoder.encode(',\n')) + } + controller.enqueue( + encoder.encode(' ' + JSON.stringify(row)) + ) + isFirstRow = false + } + + if (rows.length < CHUNK_SIZE) { + break + } + + offset += CHUNK_SIZE + } + + controller.enqueue(encoder.encode('\n]')) + controller.close() + } catch (error) { + controller.error(error) + } + }, + }) + + return createStreamingExportResponse( + stream, `${tableName}_export.json`, 'application/json' )