diff --git a/src/export/csv.test.ts b/src/export/csv.test.ts index b186aeb..de4829f 100644 --- a/src/export/csv.test.ts +++ b/src/export/csv.test.ts @@ -1,18 +1,73 @@ import { describe, it, expect, vi, beforeEach } from 'vitest' -import { exportTableToCsvRoute } from './csv' -import { getTableData, createExportResponse } from './index' -import { createResponse } from '../utils' import type { DataSource } from '../types' import type { StarbaseDBConfiguration } from '../handler' +const mockExecuteOperation = vi.fn() + vi.mock('./index', () => ({ + executeOperation: (...args: any[]) => mockExecuteOperation(...args), + getTableDataChunked: async function* ( + tableName: string, + dataSource: any, + config: any, + chunkSize: number = 1000 + ) { + let offset = 0 + while (true) { + const chunk = await mockExecuteOperation( + [{ sql: `SELECT * FROM ${tableName} LIMIT ? OFFSET ?;`, params: [chunkSize, offset] }], + dataSource, + config + ) + if (!chunk || chunk.length === 0) break + yield chunk + if (chunk.length < chunkSize) break + offset += chunkSize + } + }, + createStreamingExportResponse: ( + producer: any, + fileName: string, + contentType: string + ) => { + const { readable, writable } = new TransformStream() + const writer = writable.getWriter() + + const done = (async () => { + try { + await producer(writer) + } finally { + await writer.close() + } + })() + + const response = new Response(readable, { + headers: { + 'Content-Type': contentType, + 'Content-Disposition': `attachment; filename="${fileName}"`, + }, + }) + ;(response as any).__producerDone = done + return response + }, + writeChunk: async (writer: WritableStreamDefaultWriter, content: string) => { + await writer.write(new TextEncoder().encode(content)) + }, + createExportResponse: (data: any, fileName: string, contentType: string) => { + const blob = new Blob([data], { type: contentType }) + return new Response(blob, { + headers: { + 'Content-Type': contentType, + 'Content-Disposition': `attachment; filename="${fileName}"`, + }, + }) + }, getTableData: vi.fn(), - createExportResponse: vi.fn(), })) vi.mock('../utils', () => ({ createResponse: vi.fn( - (data, message, status) => + (data: any, message: any, status: any) => new Response(JSON.stringify({ result: data, error: message }), { status, headers: { 'Content-Type': 'application/json' }, @@ -20,18 +75,19 @@ vi.mock('../utils', () => ({ ), })) +import { exportTableToCsvRoute } from './csv' + let mockDataSource: DataSource let mockConfig: StarbaseDBConfiguration beforeEach(() => { vi.clearAllMocks() + mockExecuteOperation.mockReset() mockDataSource = { source: 'external', external: { dialect: 'sqlite' }, - rpc: { - executeQuery: vi.fn(), - }, + rpc: { executeQuery: vi.fn() }, } as any mockConfig = { @@ -43,38 +99,31 @@ beforeEach(() => { describe('CSV Export Module', () => { it('should return a CSV file when table data exists', async () => { - vi.mocked(getTableData).mockResolvedValue([ + // Table exists check + mockExecuteOperation.mockResolvedValueOnce([{ name: 'users' }]) + // Data chunk + mockExecuteOperation.mockResolvedValueOnce([ { id: 1, name: 'Alice', age: 30 }, { id: 2, name: 'Bob', age: 25 }, ]) - vi.mocked(createExportResponse).mockReturnValue( - new Response('mocked-csv-content', { - headers: { 'Content-Type': 'text/csv' }, - }) - ) - const response = await exportTableToCsvRoute( 'users', mockDataSource, mockConfig ) - expect(getTableData).toHaveBeenCalledWith( - 'users', - mockDataSource, - mockConfig - ) - expect(createExportResponse).toHaveBeenCalledWith( - 'id,name,age\n1,Alice,30\n2,Bob,25\n', - 'users_export.csv', - 'text/csv' - ) + expect(response).toBeInstanceOf(Response) expect(response.headers.get('Content-Type')).toBe('text/csv') + + const csvText = await response.text() + expect(csvText).toContain('id,name,age') + expect(csvText).toContain('1,Alice,30') + expect(csvText).toContain('2,Bob,25') }) it('should return 404 if table does not exist', async () => { - vi.mocked(getTableData).mockResolvedValue(null) + mockExecuteOperation.mockResolvedValueOnce([]) const response = await exportTableToCsvRoute( 'non_existent_table', @@ -82,27 +131,18 @@ describe('CSV Export Module', () => { mockConfig ) - expect(getTableData).toHaveBeenCalledWith( - 'non_existent_table', - mockDataSource, - mockConfig - ) expect(response.status).toBe(404) - const jsonResponse: { error: string } = await response.json() expect(jsonResponse.error).toBe( "Table 'non_existent_table' does not exist." ) }) - it('should handle empty table (return only headers)', async () => { - vi.mocked(getTableData).mockResolvedValue([]) - - vi.mocked(createExportResponse).mockReturnValue( - new Response('mocked-csv-content', { - headers: { 'Content-Type': 'text/csv' }, - }) - ) + it('should handle empty table (no output)', async () => { + // Table exists + mockExecuteOperation.mockResolvedValueOnce([{ name: 'empty_table' }]) + // Empty data + mockExecuteOperation.mockResolvedValueOnce([]) const response = await exportTableToCsvRoute( 'empty_table', @@ -110,49 +150,37 @@ describe('CSV Export Module', () => { mockConfig ) - expect(getTableData).toHaveBeenCalledWith( - 'empty_table', - mockDataSource, - mockConfig - ) - expect(createExportResponse).toHaveBeenCalledWith( - '', - 'empty_table_export.csv', - 'text/csv' - ) + expect(response).toBeInstanceOf(Response) expect(response.headers.get('Content-Type')).toBe('text/csv') + + const csvText = await response.text() + expect(csvText).toBe('') // No headers, no data }) it('should escape commas and quotes in CSV values', async () => { - vi.mocked(getTableData).mockResolvedValue([ + // Table exists + mockExecuteOperation.mockResolvedValueOnce([{ name: 'special_chars' }]) + // Data with special chars + mockExecuteOperation.mockResolvedValueOnce([ { id: 1, name: 'Sahithi, is', bio: 'my forever "penguin"' }, ]) - vi.mocked(createExportResponse).mockReturnValue( - new Response('mocked-csv-content', { - headers: { 'Content-Type': 'text/csv' }, - }) - ) - const response = await exportTableToCsvRoute( 'special_chars', mockDataSource, mockConfig ) - expect(createExportResponse).toHaveBeenCalledWith( - 'id,name,bio\n1,"Sahithi, is","my forever ""penguin"""\n', - 'special_chars_export.csv', - 'text/csv' - ) - expect(response.headers.get('Content-Type')).toBe('text/csv') + const csvText = await response.text() + expect(csvText).toContain('id,name,bio') + expect(csvText).toContain('1,"Sahithi, is","my forever ""penguin"""') }) it('should return 500 on an unexpected error', async () => { const consoleErrorMock = vi .spyOn(console, 'error') .mockImplementation(() => {}) - vi.mocked(getTableData).mockRejectedValue(new Error('Database Error')) + mockExecuteOperation.mockRejectedValue(new Error('Database Error')) const response = await exportTableToCsvRoute( 'users', @@ -163,5 +191,6 @@ describe('CSV Export Module', () => { expect(response.status).toBe(500) const jsonResponse: { error: string } = await response.json() expect(jsonResponse.error).toBe('Failed to export table to CSV') + consoleErrorMock.mockRestore() }) }) diff --git a/src/export/csv.ts b/src/export/csv.ts index 22a4591..c52e65f 100644 --- a/src/export/csv.ts +++ b/src/export/csv.ts @@ -1,17 +1,51 @@ -import { getTableData, createExportResponse } from './index' +import { + getTableDataChunked, + executeOperation, + createStreamingExportResponse, + writeChunk, + createExportResponse, +} from './index' import { createResponse } from '../utils' import { DataSource } from '../types' import { StarbaseDBConfiguration } from '../handler' +const BREATHE_MS = 10 + +function formatCsvRow(row: any): string { + return Object.values(row) + .map((value) => { + if ( + typeof value === 'string' && + (value.includes(',') || + value.includes('"') || + value.includes('\n')) + ) { + return `"${value.replace(/"/g, '""')}"` + } + return value === null ? '' : value + }) + .join(',') +} + export async function exportTableToCsvRoute( tableName: string, dataSource: DataSource, config: StarbaseDBConfiguration ): Promise { try { - const data = await getTableData(tableName, dataSource, config) + // Verify table exists + const tableExistsResult = await executeOperation( + [ + { + sql: `SELECT name FROM sqlite_master WHERE type='table' AND name=?;`, + params: [tableName], + }, + ], + dataSource, + config + ) - if (data === null) { + if (!tableExistsResult || tableExistsResult.length === 0) { return createResponse( undefined, `Table '${tableName}' does not exist.`, @@ -19,33 +53,40 @@ export async function exportTableToCsvRoute( ) } - // Convert the result to CSV - let csvContent = '' - if (data.length > 0) { - // Add headers - csvContent += Object.keys(data[0]).join(',') + '\n' - - // Add data rows - data.forEach((row: any) => { - csvContent += - Object.values(row) - .map((value) => { - if ( - typeof value === 'string' && - (value.includes(',') || - value.includes('"') || - value.includes('\n')) - ) { - return `"${value.replace(/"/g, '""')}"` - } - return value - }) - .join(',') + '\n' - }) - } + return createStreamingExportResponse( + async (writer) => { + let headersWritten = false + + for await (const chunk of getTableDataChunked( + tableName, + dataSource, + config, + 1000 + )) { + if (chunk.length === 0) continue + + // Write CSV headers from first row of first chunk + if (!headersWritten) { + await writeChunk( + writer, + Object.keys(chunk[0]).join(',') + '\n' + ) + headersWritten = true + } + + // Write rows + let batchContent = '' + for (const row of chunk) { + batchContent += formatCsvRow(row) + '\n' + } + await writeChunk(writer, batchContent) - return createExportResponse( - csvContent, + // Breathing interval + if (BREATHE_MS > 0) { + await new Promise((r) => setTimeout(r, BREATHE_MS)) + } + } + }, `${tableName}_export.csv`, 'text/csv' ) diff --git a/src/export/dump.test.ts b/src/export/dump.test.ts index ca65b43..da31667 100644 --- a/src/export/dump.test.ts +++ b/src/export/dump.test.ts @@ -1,17 +1,85 @@ import { describe, it, expect, vi, beforeEach } from 'vitest' -import { dumpDatabaseRoute } from './dump' -import { executeOperation } from '.' -import { createResponse } from '../utils' import type { DataSource } from '../types' import type { StarbaseDBConfiguration } from '../handler' +// Track all executeOperation calls +const mockExecuteOperation = vi.fn() + +// Mock the module before importing dump vi.mock('.', () => ({ - executeOperation: vi.fn(), + executeOperation: (...args: any[]) => mockExecuteOperation(...args), + getTableDataChunked: async function* ( + tableName: string, + dataSource: any, + config: any, + chunkSize: number = 1000 + ) { + // Use the mocked executeOperation to simulate chunked reads + let offset = 0 + while (true) { + const chunk = await mockExecuteOperation( + [{ sql: `SELECT * FROM ${tableName} LIMIT ? OFFSET ?;`, params: [chunkSize, offset] }], + dataSource, + config + ) + if (!chunk || chunk.length === 0) break + yield chunk + if (chunk.length < chunkSize) break + offset += chunkSize + } + }, + createStreamingExportResponse: ( + producer: any, + fileName: string, + contentType: string + ) => { + // Synchronous test helper: run producer into a buffer, return Response + const chunks: string[] = [] + const encoder = new TextEncoder() + const decoder = new TextDecoder() + + const { readable, writable } = new TransformStream() + const writer = writable.getWriter() + + // Run producer and collect output + const done = (async () => { + try { + await producer(writer) + } finally { + await writer.close() + } + })() + + // Return response backed by readable stream + const response = new Response(readable, { + headers: { + 'Content-Type': contentType, + 'Content-Disposition': `attachment; filename="${fileName}"`, + }, + }) + + // Store promise so tests can await completion + ;(response as any).__producerDone = done + + return response + }, + writeChunk: async (writer: WritableStreamDefaultWriter, content: string) => { + await writer.write(new TextEncoder().encode(content)) + }, + createExportResponse: (data: any, fileName: string, contentType: string) => { + const blob = new Blob([data], { type: contentType }) + return new Response(blob, { + headers: { + 'Content-Type': contentType, + 'Content-Disposition': `attachment; filename="${fileName}"`, + }, + }) + }, })) vi.mock('../utils', () => ({ createResponse: vi.fn( - (data, message, status) => + (data: any, message: any, status: any) => new Response(JSON.stringify({ result: data, error: message }), { status, headers: { 'Content-Type': 'application/json' }, @@ -19,11 +87,15 @@ vi.mock('../utils', () => ({ ), })) +// Import dump AFTER mocks are set up +import { dumpDatabaseRoute } from './dump' + let mockDataSource: DataSource let mockConfig: StarbaseDBConfiguration beforeEach(() => { vi.clearAllMocks() + mockExecuteOperation.mockReset() mockDataSource = { source: 'external', @@ -40,18 +112,23 @@ beforeEach(() => { describe('Database Dump Module', () => { it('should return a database dump when tables exist', async () => { - vi.mocked(executeOperation) + mockExecuteOperation + // Table names .mockResolvedValueOnce([{ name: 'users' }, { name: 'orders' }]) + // users schema .mockResolvedValueOnce([ { sql: 'CREATE TABLE users (id INTEGER, name TEXT);' }, ]) + // users data chunk (LIMIT/OFFSET query, 2 rows < 1000 chunkSize → generator breaks) .mockResolvedValueOnce([ { id: 1, name: 'Alice' }, { id: 2, name: 'Bob' }, ]) + // orders schema .mockResolvedValueOnce([ { sql: 'CREATE TABLE orders (id INTEGER, total REAL);' }, ]) + // orders data chunk (2 rows < 1000 → generator breaks) .mockResolvedValueOnce([ { id: 1, total: 99.99 }, { id: 2, total: 49.5 }, @@ -81,7 +158,7 @@ describe('Database Dump Module', () => { }) it('should handle empty databases (no tables)', async () => { - vi.mocked(executeOperation).mockResolvedValueOnce([]) + mockExecuteOperation.mockResolvedValueOnce([]) const response = await dumpDatabaseRoute(mockDataSource, mockConfig) @@ -94,11 +171,12 @@ describe('Database Dump Module', () => { }) it('should handle databases with tables but no data', async () => { - vi.mocked(executeOperation) + mockExecuteOperation .mockResolvedValueOnce([{ name: 'users' }]) .mockResolvedValueOnce([ { sql: 'CREATE TABLE users (id INTEGER, name TEXT);' }, ]) + // Empty chunk = no data .mockResolvedValueOnce([]) const response = await dumpDatabaseRoute(mockDataSource, mockConfig) @@ -112,34 +190,60 @@ describe('Database Dump Module', () => { }) it('should escape single quotes properly in string values', async () => { - vi.mocked(executeOperation) + mockExecuteOperation .mockResolvedValueOnce([{ name: 'users' }]) .mockResolvedValueOnce([ { sql: 'CREATE TABLE users (id INTEGER, bio TEXT);' }, ]) - .mockResolvedValueOnce([{ id: 1, bio: "Alice's adventure" }]) + .mockResolvedValueOnce([{ id: 1, bio: "It's a test" }]) + .mockResolvedValueOnce([]) const response = await dumpDatabaseRoute(mockDataSource, mockConfig) - expect(response).toBeInstanceOf(Response) const dumpText = await response.text() expect(dumpText).toContain( - "INSERT INTO users VALUES (1, 'Alice''s adventure');" + "INSERT INTO users VALUES (1, 'It''s a test');" ) }) - it('should return a 500 response when an error occurs', async () => { - const consoleErrorMock = vi - .spyOn(console, 'error') - .mockImplementation(() => {}) - vi.mocked(executeOperation).mockRejectedValue( - new Error('Database Error') - ) + it('should stream data in chunks for large tables', async () => { + const chunk1 = Array.from({ length: 1000 }, (_, i) => ({ + id: i + 1, + name: `user${i + 1}`, + })) + const chunk2 = Array.from({ length: 1000 }, (_, i) => ({ + id: i + 1001, + name: `user${i + 1001}`, + })) + const chunk3 = Array.from({ length: 500 }, (_, i) => ({ + id: i + 2001, + name: `user${i + 2001}`, + })) + + mockExecuteOperation + .mockResolvedValueOnce([{ name: 'users' }]) + .mockResolvedValueOnce([ + { sql: 'CREATE TABLE users (id INTEGER, name TEXT);' }, + ]) + .mockResolvedValueOnce(chunk1) + .mockResolvedValueOnce(chunk2) + .mockResolvedValueOnce(chunk3) + .mockResolvedValueOnce([]) // end-of-data const response = await dumpDatabaseRoute(mockDataSource, mockConfig) - expect(response.status).toBe(500) - const jsonResponse: { error: string } = await response.json() - expect(jsonResponse.error).toBe('Failed to create database dump') + expect(response).toBeInstanceOf(Response) + const dumpText = await response.text() + + expect(dumpText).toContain("INSERT INTO users VALUES (1, 'user1');") + expect(dumpText).toContain( + "INSERT INTO users VALUES (2500, 'user2500');" + ) + expect(dumpText).toContain( + "INSERT INTO users VALUES (1000, 'user1000');" + ) + expect(dumpText).toContain( + "INSERT INTO users VALUES (1001, 'user1001');" + ) }) }) diff --git a/src/export/dump.ts b/src/export/dump.ts index 91a2e89..7dfbf79 100644 --- a/src/export/dump.ts +++ b/src/export/dump.ts @@ -1,8 +1,14 @@ -import { executeOperation } from '.' +import { executeOperation, getTableDataChunked, createStreamingExportResponse, writeChunk } from '.' import { StarbaseDBConfiguration } from '../handler' import { DataSource } from '../types' import { createResponse } from '../utils' +/** + * Breathing interval between chunks (ms). + * Allows other DO requests to be processed between export batches. + */ +const BREATHE_MS = 10 + export async function dumpDatabaseRoute( dataSource: DataSource, config: StarbaseDBConfiguration @@ -16,54 +22,79 @@ export async function dumpDatabaseRoute( ) const tables = tablesResult.map((row: any) => row.name) - let dumpContent = 'SQLite format 3\0' // SQLite file header - // Iterate through all tables - for (const table of tables) { - // Get table schema - const schemaResult = await executeOperation( - [ - { - sql: `SELECT sql FROM sqlite_master WHERE type='table' AND name='${table}';`, - }, - ], - dataSource, - config + if (tables.length === 0) { + // Empty database — return header only + return createStreamingExportResponse( + async (writer) => { + await writeChunk(writer, 'SQLite format 3\0') + }, + 'database_dump.sql', + 'application/x-sqlite3' ) + } - if (schemaResult.length) { - const schema = schemaResult[0].sql - dumpContent += `\n-- Table: ${table}\n${schema};\n\n` - } + return createStreamingExportResponse( + async (writer) => { + // Write SQLite header + await writeChunk(writer, 'SQLite format 3\0') - // Get table data - const dataResult = await executeOperation( - [{ sql: `SELECT * FROM ${table};` }], - dataSource, - config - ) + // Iterate through all tables + for (const table of tables) { + // Get table schema + const schemaResult = await executeOperation( + [ + { + sql: `SELECT sql FROM sqlite_master WHERE type='table' AND name=?;`, + params: [table], + }, + ], + dataSource, + config + ) - for (const row of dataResult) { - const values = Object.values(row).map((value) => - typeof value === 'string' - ? `'${value.replace(/'/g, "''")}'` - : value - ) - dumpContent += `INSERT INTO ${table} VALUES (${values.join(', ')});\n` - } + if (schemaResult.length) { + const schema = schemaResult[0].sql + await writeChunk( + writer, + `\n-- Table: ${table}\n${schema};\n\n` + ) + } - dumpContent += '\n' - } + // Stream table data in chunks + for await (const chunk of getTableDataChunked( + table, + dataSource, + config, + 1000 + )) { + let batchContent = '' + + for (const row of chunk) { + const values = Object.values(row).map((value) => + typeof value === 'string' + ? `'${value.replace(/'/g, "''")}'` + : value + ) + batchContent += `INSERT INTO ${table} VALUES (${values.join(', ')});\n` + } - // Create a Blob from the dump content - const blob = new Blob([dumpContent], { type: 'application/x-sqlite3' }) + await writeChunk(writer, batchContent) - const headers = new Headers({ - 'Content-Type': 'application/x-sqlite3', - 'Content-Disposition': 'attachment; filename="database_dump.sql"', - }) + // Breathing interval — let other DO requests through + if (BREATHE_MS > 0) { + await new Promise((r) => + setTimeout(r, BREATHE_MS) + ) + } + } - return new Response(blob, { headers }) + await writeChunk(writer, '\n') + } + }, + 'database_dump.sql', + 'application/x-sqlite3' + ) } catch (error: any) { console.error('Database Dump Error:', error) return createResponse(undefined, 'Failed to create database dump', 500) diff --git a/src/export/index.ts b/src/export/index.ts index 9c40119..3f951ab 100644 --- a/src/export/index.ts +++ b/src/export/index.ts @@ -13,7 +13,6 @@ export async function executeOperation( dataSource, config, })) as any[] - // return results?.length > 0 ? results[0] : undefined return results.length > 0 && Array.isArray(results[0]) ? results[0] : results @@ -25,7 +24,6 @@ export async function getTableData( config: StarbaseDBConfiguration ): Promise { try { - // Verify if the table exists const tableExistsResult = await executeOperation( [ { @@ -41,7 +39,6 @@ export async function getTableData( return null } - // Get table data const dataResult = await executeOperation( [{ sql: `SELECT * FROM ${tableName};` }], dataSource, @@ -54,6 +51,44 @@ export async function getTableData( } } +/** + * Fetch table data in chunks to avoid loading everything into memory. + * Returns an async generator that yields batches of rows. + */ +export async function* getTableDataChunked( + tableName: string, + dataSource: DataSource, + config: StarbaseDBConfiguration, + chunkSize: number = 1000 +): AsyncGenerator { + let offset = 0 + + while (true) { + const chunk = await executeOperation( + [ + { + sql: `SELECT * FROM ${tableName} LIMIT ? OFFSET ?;`, + params: [chunkSize, offset], + }, + ], + dataSource, + config + ) + + if (!chunk || chunk.length === 0) { + break + } + + yield chunk + + if (chunk.length < chunkSize) { + break + } + + offset += chunkSize + } +} + export function createExportResponse( data: any, fileName: string, @@ -68,3 +103,39 @@ export function createExportResponse( return new Response(blob, { headers }) } + +/** + * Create a streaming response using TransformStream. + * The producer function writes to the writable side; the response reads from the readable side. + */ +export function createStreamingExportResponse( + producer: (writer: WritableStreamDefaultWriter) => Promise, + fileName: string, + contentType: string +): Response { + const { readable, writable } = new TransformStream() + + // Run producer in background — does not block response return + const writer = writable.getWriter() + producer(writer).finally(() => { + writer.close().catch(() => {}) + }) + + const headers = new Headers({ + 'Content-Type': contentType, + 'Content-Disposition': `attachment; filename="${fileName}"`, + }) + + return new Response(readable, { headers }) +} + +/** + * Encode a string chunk and write it to the stream writer. + */ +export async function writeChunk( + writer: WritableStreamDefaultWriter, + content: string +): Promise { + const encoded = new TextEncoder().encode(content) + await writer.write(encoded) +} diff --git a/src/export/json.test.ts b/src/export/json.test.ts index 3fe4a8c..46655e0 100644 --- a/src/export/json.test.ts +++ b/src/export/json.test.ts @@ -1,18 +1,73 @@ import { describe, it, expect, vi, beforeEach } from 'vitest' -import { exportTableToJsonRoute } from './json' -import { getTableData, createExportResponse } from './index' -import { createResponse } from '../utils' import type { DataSource } from '../types' import type { StarbaseDBConfiguration } from '../handler' +const mockExecuteOperation = vi.fn() + vi.mock('./index', () => ({ + executeOperation: (...args: any[]) => mockExecuteOperation(...args), + getTableDataChunked: async function* ( + tableName: string, + dataSource: any, + config: any, + chunkSize: number = 1000 + ) { + let offset = 0 + while (true) { + const chunk = await mockExecuteOperation( + [{ sql: `SELECT * FROM ${tableName} LIMIT ? OFFSET ?;`, params: [chunkSize, offset] }], + dataSource, + config + ) + if (!chunk || chunk.length === 0) break + yield chunk + if (chunk.length < chunkSize) break + offset += chunkSize + } + }, + createStreamingExportResponse: ( + producer: any, + fileName: string, + contentType: string + ) => { + const { readable, writable } = new TransformStream() + const writer = writable.getWriter() + + const done = (async () => { + try { + await producer(writer) + } finally { + await writer.close() + } + })() + + const response = new Response(readable, { + headers: { + 'Content-Type': contentType, + 'Content-Disposition': `attachment; filename="${fileName}"`, + }, + }) + ;(response as any).__producerDone = done + return response + }, + writeChunk: async (writer: WritableStreamDefaultWriter, content: string) => { + await writer.write(new TextEncoder().encode(content)) + }, + createExportResponse: (data: any, fileName: string, contentType: string) => { + const blob = new Blob([data], { type: contentType }) + return new Response(blob, { + headers: { + 'Content-Type': contentType, + 'Content-Disposition': `attachment; filename="${fileName}"`, + }, + }) + }, getTableData: vi.fn(), - createExportResponse: vi.fn(), })) vi.mock('../utils', () => ({ createResponse: vi.fn( - (data, message, status) => + (data: any, message: any, status: any) => new Response(JSON.stringify({ result: data, error: message }), { status, headers: { 'Content-Type': 'application/json' }, @@ -20,11 +75,14 @@ vi.mock('../utils', () => ({ ), })) +import { exportTableToJsonRoute } from './json' + let mockDataSource: DataSource let mockConfig: StarbaseDBConfiguration beforeEach(() => { vi.clearAllMocks() + mockExecuteOperation.mockReset() mockDataSource = { source: 'external', @@ -41,7 +99,7 @@ beforeEach(() => { describe('JSON Export Module', () => { it('should return a 404 response if table does not exist', async () => { - vi.mocked(getTableData).mockResolvedValue(null) + mockExecuteOperation.mockResolvedValueOnce([]) const response = await exportTableToJsonRoute( 'missing_table', @@ -59,13 +117,11 @@ describe('JSON Export Module', () => { { id: 1, name: 'Alice' }, { id: 2, name: 'Bob' }, ] - vi.mocked(getTableData).mockResolvedValue(mockData) - vi.mocked(createExportResponse).mockReturnValue( - new Response('mocked-json-content', { - headers: { 'Content-Type': 'application/json' }, - }) - ) + // Table exists check + mockExecuteOperation.mockResolvedValueOnce([{ name: 'users' }]) + // Data chunk + mockExecuteOperation.mockResolvedValueOnce(mockData) const response = await exportTableToJsonRoute( 'users', @@ -73,27 +129,19 @@ describe('JSON Export Module', () => { mockConfig ) - expect(getTableData).toHaveBeenCalledWith( - 'users', - mockDataSource, - mockConfig - ) - expect(createExportResponse).toHaveBeenCalledWith( - JSON.stringify(mockData, null, 4), - 'users_export.json', - 'application/json' - ) + expect(response).toBeInstanceOf(Response) expect(response.headers.get('Content-Type')).toBe('application/json') + + const jsonText = await response.text() + const parsed = JSON.parse(jsonText) + expect(parsed).toEqual(mockData) }) it('should return an empty JSON array when table has no data', async () => { - vi.mocked(getTableData).mockResolvedValue([]) - - vi.mocked(createExportResponse).mockReturnValue( - new Response('mocked-json-content', { - headers: { 'Content-Type': 'application/json' }, - }) - ) + // Table exists + mockExecuteOperation.mockResolvedValueOnce([{ name: 'empty_table' }]) + // Empty data + mockExecuteOperation.mockResolvedValueOnce([]) const response = await exportTableToJsonRoute( 'empty_table', @@ -101,12 +149,9 @@ describe('JSON Export Module', () => { mockConfig ) - expect(createExportResponse).toHaveBeenCalledWith( - '[]', - 'empty_table_export.json', - 'application/json' - ) - expect(response.headers.get('Content-Type')).toBe('application/json') + expect(response).toBeInstanceOf(Response) + const jsonText = await response.text() + expect(jsonText).toBe('[\n\n]') // Empty array (open bracket + newline + close bracket) }) it('should escape special characters in JSON properly', async () => { @@ -114,13 +159,11 @@ describe('JSON Export Module', () => { { id: 1, name: 'Sahithi "The Best"' }, { id: 2, description: 'New\nLine' }, ] - vi.mocked(getTableData).mockResolvedValue(specialCharsData) - vi.mocked(createExportResponse).mockReturnValue( - new Response('mocked-json-content', { - headers: { 'Content-Type': 'application/json' }, - }) - ) + // Table exists + mockExecuteOperation.mockResolvedValueOnce([{ name: 'special_chars' }]) + // Data + mockExecuteOperation.mockResolvedValueOnce(specialCharsData) const response = await exportTableToJsonRoute( 'special_chars', @@ -128,19 +171,16 @@ describe('JSON Export Module', () => { mockConfig ) - expect(createExportResponse).toHaveBeenCalledWith( - JSON.stringify(specialCharsData, null, 4), - 'special_chars_export.json', - 'application/json' - ) - expect(response.headers.get('Content-Type')).toBe('application/json') + const jsonText = await response.text() + const parsed = JSON.parse(jsonText) + expect(parsed).toEqual(specialCharsData) }) it('should return a 500 response when an error occurs', async () => { const consoleErrorMock = vi .spyOn(console, 'error') .mockImplementation(() => {}) - vi.mocked(getTableData).mockRejectedValue(new Error('Database Error')) + mockExecuteOperation.mockRejectedValue(new Error('Database Error')) const response = await exportTableToJsonRoute( 'users', @@ -151,5 +191,6 @@ describe('JSON Export Module', () => { expect(response.status).toBe(500) const jsonResponse = (await response.json()) as { error: string } expect(jsonResponse.error).toBe('Failed to export table to JSON') + consoleErrorMock.mockRestore() }) }) diff --git a/src/export/json.ts b/src/export/json.ts index c0ab811..04fdedc 100644 --- a/src/export/json.ts +++ b/src/export/json.ts @@ -1,17 +1,34 @@ -import { getTableData, createExportResponse } from './index' +import { + getTableDataChunked, + executeOperation, + createStreamingExportResponse, + writeChunk, +} from './index' import { createResponse } from '../utils' import { DataSource } from '../types' import { StarbaseDBConfiguration } from '../handler' +const BREATHE_MS = 10 + export async function exportTableToJsonRoute( tableName: string, dataSource: DataSource, config: StarbaseDBConfiguration ): Promise { try { - const data = await getTableData(tableName, dataSource, config) + // Verify table exists + const tableExistsResult = await executeOperation( + [ + { + sql: `SELECT name FROM sqlite_master WHERE type='table' AND name=?;`, + params: [tableName], + }, + ], + dataSource, + config + ) - if (data === null) { + if (!tableExistsResult || tableExistsResult.length === 0) { return createResponse( undefined, `Table '${tableName}' does not exist.`, @@ -19,11 +36,34 @@ export async function exportTableToJsonRoute( ) } - // Convert the result to JSON - const jsonData = JSON.stringify(data, null, 4) + return createStreamingExportResponse( + async (writer) => { + await writeChunk(writer, '[\n') + + let isFirst = true + for await (const chunk of getTableDataChunked( + tableName, + dataSource, + config, + 1000 + )) { + for (const row of chunk) { + const prefix = isFirst ? ' ' : ',\n ' + isFirst = false + await writeChunk( + writer, + prefix + JSON.stringify(row) + ) + } + + // Breathing interval + if (BREATHE_MS > 0) { + await new Promise((r) => setTimeout(r, BREATHE_MS)) + } + } - return createExportResponse( - jsonData, + await writeChunk(writer, '\n]') + }, `${tableName}_export.json`, 'application/json' )