From 6cb804a3e508c04c7439ac368bca56b10e251aa4 Mon Sep 17 00:00:00 2001 From: arkw Date: Fri, 24 Jan 2025 16:44:31 +0900 Subject: [PATCH 01/13] update async_bindings.ts update opfs tests change dropFiles args --- lib/CMakeLists.txt | 3 + lib/src/webdb_api.cc | 20 +- .../duckdb-wasm/src/bindings/bindings_base.ts | 57 ++++- .../src/bindings/bindings_interface.ts | 3 +- packages/duckdb-wasm/src/bindings/config.ts | 4 + .../duckdb-wasm/src/bindings/duckdb_module.ts | 2 + .../src/parallel/async_bindings.ts | 64 ++++- .../src/parallel/async_bindings_interface.ts | 3 + .../src/parallel/worker_dispatcher.ts | 4 +- .../src/parallel/worker_request.ts | 4 +- packages/duckdb-wasm/test/opfs.test.ts | 230 +++++++++++++----- 11 files changed, 317 insertions(+), 77 deletions(-) diff --git a/lib/CMakeLists.txt b/lib/CMakeLists.txt index 8cfc7a59e..a0a7a88da 100644 --- a/lib/CMakeLists.txt +++ b/lib/CMakeLists.txt @@ -295,6 +295,9 @@ if(EMSCRIPTEN) _malloc, \ _calloc, \ _free, \ + stringToUTF8, \ + lengthBytesUTF8, \ + stackAlloc, \ _duckdb_web_clear_response, \ _duckdb_web_collect_file_stats, \ _duckdb_web_connect, \ diff --git a/lib/src/webdb_api.cc b/lib/src/webdb_api.cc index bad9c7f9e..dbe5b721b 100644 --- a/lib/src/webdb_api.cc +++ b/lib/src/webdb_api.cc @@ -1,3 +1,4 @@ +#include #include #include @@ -94,9 +95,24 @@ void duckdb_web_fs_drop_file(WASMResponse* packed, const char* file_name) { WASMResponseBuffer::Get().Store(*packed, webdb.DropFile(file_name)); } /// Drop a file -void duckdb_web_fs_drop_files(WASMResponse* packed) { +void duckdb_web_fs_drop_files(WASMResponse* packed, const char** names, int name_count) { GET_WEBDB(*packed); - WASMResponseBuffer::Get().Store(*packed, webdb.DropFiles()); + if (name_count == 0) { + WASMResponseBuffer::Get().Store(*packed, webdb.DropFiles()); + } else { + for (int i = 0; i < name_count; i++) { + const char* name = names[i]; + if (name == nullptr) { + std::cerr << "Error: NULL pointer detected at index " << i << std::endl; + continue; + } + if (std::strlen(name) == 0) { + std::cerr << "Error: Empty string detected at index " << i << std::endl; + continue; + } + WASMResponseBuffer::Get().Store(*packed, webdb.DropFile(name)); + } + } } /// Glob file infos void duckdb_web_fs_glob_file_infos(WASMResponse* packed, const char* file_name) { diff --git a/packages/duckdb-wasm/src/bindings/bindings_base.ts b/packages/duckdb-wasm/src/bindings/bindings_base.ts index f395bdb10..34f201d3d 100644 --- a/packages/duckdb-wasm/src/bindings/bindings_base.ts +++ b/packages/duckdb-wasm/src/bindings/bindings_base.ts @@ -583,12 +583,52 @@ export abstract class DuckDBBindingsBase implements DuckDBBindings { dropResponseBuffers(this.mod); } /** Drop files */ - public dropFiles(): void { - const [s, d, n] = callSRet(this.mod, 'duckdb_web_fs_drop_files', [], []); - if (s !== StatusCode.SUCCESS) { - throw new Error(readString(this.mod, d, n)); + public dropFiles(names?:string[]): void { + const pointers:number[] = []; + let pointerOfArray:number = -1; + try { + for (const str of (names ?? [])) { + if (str !== null && str !== undefined && str.length > 0) { + const size = this.mod.lengthBytesUTF8(str) + 1; + const ret = this.mod._malloc(size); + if (!ret) { + throw new Error(`Failed to allocate memory for string: ${str}`); + } + this.mod.stringToUTF8(str, ret, size); + pointers.push(ret); + } + } + pointerOfArray = this.mod._malloc(pointers.length * 4); + if (!pointerOfArray) { + throw new Error(`Failed to allocate memory for pointers array`); + } + for (let i = 0; i < pointers.length; i++) { + this.mod.HEAP32[(pointerOfArray >> 2) + i] = pointers[i]; + } + const [s, d, n] = callSRet( + this.mod, + 'duckdb_web_fs_drop_files', + [ + 'number', + 'number' + ], + [ + pointerOfArray, + pointers.length + ] + ); + if (s !== StatusCode.SUCCESS) { + throw new Error(readString(this.mod, d, n)); + } + dropResponseBuffers(this.mod); + } finally { + for (const pointer of pointers) { + this.mod._free(pointer); + } + if( pointerOfArray > 0 ){ + this.mod._free(pointerOfArray); + } } - dropResponseBuffers(this.mod); } /** Flush all files */ public flushFiles(): void { @@ -622,6 +662,13 @@ export abstract class DuckDBBindingsBase implements DuckDBBindings { throw new Error("Not an OPFS file name: " + file); } } + public async registerOPFSFileNameAsync(file: string): Promise { + if (file.startsWith("opfs://")) { + return await this.prepareFileHandle(file, DuckDBDataProtocol.BROWSER_FSACCESS); + } else { + throw new Error("Not an OPFS file name: " + file); + } + } public collectFileStatistics(file: string, enable: boolean): void { const [s, d, n] = callSRet(this.mod, 'duckdb_web_collect_file_stats', ['string', 'boolean'], [file, enable]); if (s !== StatusCode.SUCCESS) { diff --git a/packages/duckdb-wasm/src/bindings/bindings_interface.ts b/packages/duckdb-wasm/src/bindings/bindings_interface.ts index 271a42ef9..004d2d46c 100644 --- a/packages/duckdb-wasm/src/bindings/bindings_interface.ts +++ b/packages/duckdb-wasm/src/bindings/bindings_interface.ts @@ -58,11 +58,12 @@ export interface DuckDBBindings { prepareDBFileHandle(path: string, protocol: DuckDBDataProtocol): Promise; globFiles(path: string): WebFile[]; dropFile(name: string): void; - dropFiles(): void; + dropFiles(names?: string[]): void; flushFiles(): void; copyFileToPath(name: string, path: string): void; copyFileToBuffer(name: string): Uint8Array; registerOPFSFileName(file: string): void; + registerOPFSFileNameAsync(file: string): Promise; collectFileStatistics(file: string, enable: boolean): void; exportFileStatistics(file: string): FileStatistics; } diff --git a/packages/duckdb-wasm/src/bindings/config.ts b/packages/duckdb-wasm/src/bindings/config.ts index ce29ca0f5..27389d1eb 100644 --- a/packages/duckdb-wasm/src/bindings/config.ts +++ b/packages/duckdb-wasm/src/bindings/config.ts @@ -70,4 +70,8 @@ export interface DuckDBConfig { * Custom user agent string */ customUserAgent?: string; + /** + * Auto Opfs File Registration + */ + autoFileRegistration?: boolean; } diff --git a/packages/duckdb-wasm/src/bindings/duckdb_module.ts b/packages/duckdb-wasm/src/bindings/duckdb_module.ts index c75c3e2ed..aafbfb6cb 100644 --- a/packages/duckdb-wasm/src/bindings/duckdb_module.ts +++ b/packages/duckdb-wasm/src/bindings/duckdb_module.ts @@ -7,6 +7,8 @@ export interface DuckDBModule extends EmscriptenModule { stackSave: typeof stackSave; stackAlloc: typeof stackAlloc; stackRestore: typeof stackRestore; + lengthBytesUTF8: typeof lengthBytesUTF8; + stringToUTF8: typeof stringToUTF8; ccall: typeof ccall; PThread: PThread; diff --git a/packages/duckdb-wasm/src/parallel/async_bindings.ts b/packages/duckdb-wasm/src/parallel/async_bindings.ts index 4aaf3a6fa..cd95549fa 100644 --- a/packages/duckdb-wasm/src/parallel/async_bindings.ts +++ b/packages/duckdb-wasm/src/parallel/async_bindings.ts @@ -20,6 +20,7 @@ import { WebFile } from '../bindings/web_file'; import { DuckDBDataProtocol } from '../bindings'; const TEXT_ENCODER = new TextEncoder(); +const OPFS_PROTOCOL_REGEX = /'(opfs:\/\/\S*?)'/g; export class AsyncDuckDB implements AsyncDuckDBBindings { /** The message handler */ @@ -45,6 +46,8 @@ export class AsyncDuckDB implements AsyncDuckDBBindings { protected _nextMessageId = 0; /** The pending requests */ protected _pendingRequests: Map = new Map(); + /** The DuckDBConfig */ + protected _config: DuckDBConfig = {}; constructor(logger: Logger, worker: Worker | null = null) { this._logger = logger; @@ -59,6 +62,11 @@ export class AsyncDuckDB implements AsyncDuckDBBindings { return this._logger; } + /** Get the logger */ + public get config(): DuckDBConfig { + return this._config; + } + /** Attach to worker */ protected attach(worker: Worker): void { this._worker = worker; @@ -100,7 +108,7 @@ export class AsyncDuckDB implements AsyncDuckDBBindings { transfer: ArrayBuffer[] = [], ): Promise> { if (!this._worker) { - console.error('cannot send a message since the worker is not set!'); + console.error('cannot send a message since the worker is not set!:' + task.type+"," + task.data); return undefined as any; } const mid = this._nextMessageId++; @@ -317,8 +325,8 @@ export class AsyncDuckDB implements AsyncDuckDBBindings { return await this.postTask(task); } /** Try to drop files */ - public async dropFiles(): Promise { - const task = new WorkerTask(WorkerRequestType.DROP_FILES, null); + public async dropFiles(names?: string[]): Promise { + const task = new WorkerTask(WorkerRequestType.DROP_FILES, names); return await this.postTask(task); } /** Flush all files */ @@ -360,6 +368,8 @@ export class AsyncDuckDB implements AsyncDuckDBBindings { /** Open a new database */ public async open(config: DuckDBConfig): Promise { + config.autoFileRegistration = config.autoFileRegistration ?? false; + this._config = config; const task = new WorkerTask(WorkerRequestType.OPEN, config); await this.postTask(task); } @@ -394,18 +404,49 @@ export class AsyncDuckDB implements AsyncDuckDBBindings { /** Run a query */ public async runQuery(conn: ConnectionID, text: string): Promise { + if( this._config.autoFileRegistration ){ + const files = await this._preFileRegistration(text); + try { + return await this._runQueryAsync(conn, text); + } finally { + if( files.length > 0 ){ + await this.dropFiles(files); + } + } + } else { + return await this._runQueryAsync(conn, text); + } + } + private async _runQueryAsync(conn: ConnectionID, text: string): Promise { const task = new WorkerTask( WorkerRequestType.RUN_QUERY, [conn, text], ); return await this.postTask(task); } - /** Start a pending query */ public async startPendingQuery( conn: ConnectionID, text: string, allowStreamResult: boolean = false, + ): Promise { + if( this._config.autoFileRegistration ){ + const files = await this._preFileRegistration(text); + try { + return await this._startPendingQueryAsync(conn, text, allowStreamResult); + } finally { + if( files.length > 0 ){ + await this.dropFiles(files); + } + } + } else { + return await this._startPendingQueryAsync(conn, text, allowStreamResult); + } + } + private async _startPendingQueryAsync( + conn: ConnectionID, + text: string, + allowStreamResult: boolean = false, ): Promise { const task = new WorkerTask< WorkerRequestType.START_PENDING_QUERY, @@ -647,4 +688,19 @@ export class AsyncDuckDB implements AsyncDuckDBBindings { ); await this.postTask(task); } + + private async _preFileRegistration(text: string) { + const files = [...text.matchAll(OPFS_PROTOCOL_REGEX)].map(match => match[1]); + const result: string[] = []; + for (const file of files) { + try { + await this.registerOPFSFileName(file); + result.push(file); + } catch (e) { + console.error(e); + throw new Error("file Not found:" + file); + } + } + return result; + } } diff --git a/packages/duckdb-wasm/src/parallel/async_bindings_interface.ts b/packages/duckdb-wasm/src/parallel/async_bindings_interface.ts index 97ba2b191..a7845f2bd 100644 --- a/packages/duckdb-wasm/src/parallel/async_bindings_interface.ts +++ b/packages/duckdb-wasm/src/parallel/async_bindings_interface.ts @@ -32,4 +32,7 @@ export interface AsyncDuckDBBindings { insertArrowFromIPCStream(conn: number, buffer: Uint8Array, options?: CSVInsertOptions): Promise; insertCSVFromPath(conn: number, path: string, options: CSVInsertOptions): Promise; insertJSONFromPath(conn: number, path: string, options: JSONInsertOptions): Promise; + + dropFile(name: string):Promise; + dropFiles(names?: string[]):Promise; } diff --git a/packages/duckdb-wasm/src/parallel/worker_dispatcher.ts b/packages/duckdb-wasm/src/parallel/worker_dispatcher.ts index 3a5a8f295..d3b666ba7 100644 --- a/packages/duckdb-wasm/src/parallel/worker_dispatcher.ts +++ b/packages/duckdb-wasm/src/parallel/worker_dispatcher.ts @@ -149,7 +149,7 @@ export abstract class AsyncDuckDBDispatcher implements Logger { this.sendOK(request); break; case WorkerRequestType.DROP_FILES: - this._bindings.dropFiles(); + this._bindings.dropFiles(request.data); this.sendOK(request); break; case WorkerRequestType.FLUSH_FILES: @@ -361,7 +361,7 @@ export abstract class AsyncDuckDBDispatcher implements Logger { break; case WorkerRequestType.REGISTER_OPFS_FILE_NAME: - this._bindings.registerOPFSFileName(request.data[0]); + await this._bindings.registerOPFSFileNameAsync(request.data[0]); this.sendOK(request); break; diff --git a/packages/duckdb-wasm/src/parallel/worker_request.ts b/packages/duckdb-wasm/src/parallel/worker_request.ts index 9b9df0634..d92d13dab 100644 --- a/packages/duckdb-wasm/src/parallel/worker_request.ts +++ b/packages/duckdb-wasm/src/parallel/worker_request.ts @@ -116,7 +116,7 @@ export type WorkerRequestVariant = | WorkerRequest | WorkerRequest | WorkerRequest - | WorkerRequest + | WorkerRequest | WorkerRequest | WorkerRequest | WorkerRequest @@ -176,7 +176,7 @@ export type WorkerTaskVariant = | WorkerTask | WorkerTask | WorkerTask - | WorkerTask + | WorkerTask | WorkerTask | WorkerTask | WorkerTask diff --git a/packages/duckdb-wasm/test/opfs.test.ts b/packages/duckdb-wasm/test/opfs.test.ts index eaf1a0fcc..6dbb231e4 100644 --- a/packages/duckdb-wasm/test/opfs.test.ts +++ b/packages/duckdb-wasm/test/opfs.test.ts @@ -1,8 +1,8 @@ import * as duckdb from '../src/'; -import {LogLevel} from '../src/'; import * as arrow from 'apache-arrow'; export function testOPFS(baseDir: string, bundle: () => duckdb.DuckDBBundle): void { + const logger = new duckdb.ConsoleLogger(duckdb.LogLevel.ERROR); let db: duckdb.AsyncDuckDB; let conn: duckdb.AsyncDuckDBConnection; @@ -11,19 +11,10 @@ export function testOPFS(baseDir: string, bundle: () => duckdb.DuckDBBundle): vo }); afterAll(async () => { - if (conn) { - await conn.close(); - } - if (db) { - await db.terminate(); - } removeFiles(); }); beforeEach(async () => { - removeFiles(); - // - const logger = new duckdb.ConsoleLogger(LogLevel.ERROR); const worker = new Worker(bundle().mainWorker!); db = new duckdb.AsyncDuckDB(logger, worker); await db.instantiate(bundle().mainModule, bundle().pthreadWorker); @@ -36,18 +27,26 @@ export function testOPFS(baseDir: string, bundle: () => duckdb.DuckDBBundle): vo afterEach(async () => { if (conn) { - await conn.close(); + await conn.close().catch(() => { + }); } if (db) { - await db.terminate(); + await db.reset().catch(() => { + }); + await db.terminate().catch(() => { + }); + await db.dropFiles().catch(() => { + }); } - removeFiles(); + await removeFiles(); }); describe('Load Data in OPFS', () => { it('Import Small Parquet file', async () => { - await conn.send(`CREATE TABLE stu AS SELECT * FROM "${baseDir}/uni/studenten.parquet"`); + //1. data preparation + await conn.send(`CREATE TABLE stu AS SELECT * FROM "${ baseDir }/uni/studenten.parquet"`); await conn.send(`CHECKPOINT;`); + const result = await conn.send(`SELECT matrnr FROM stu;`); const batches = []; for await (const batch of result) { @@ -58,10 +57,11 @@ export function testOPFS(baseDir: string, bundle: () => duckdb.DuckDBBundle): vo new Int32Array([24002, 25403, 26120, 26830, 27550, 28106, 29120, 29555]), ); }); - it('Import Larget Parquet file', async () => { - await conn.send(`CREATE TABLE lineitem AS SELECT * FROM "${baseDir}/tpch/0_01/parquet/lineitem.parquet"`); + //1. data preparation + await conn.send(`CREATE TABLE lineitem AS SELECT * FROM "${ baseDir }/tpch/0_01/parquet/lineitem.parquet"`); await conn.send(`CHECKPOINT;`); + const result = await conn.send(`SELECT count(*)::INTEGER as cnt FROM lineitem;`); const batches = []; for await (const batch of result) { @@ -72,12 +72,15 @@ export function testOPFS(baseDir: string, bundle: () => duckdb.DuckDBBundle): vo }); it('Load Existing DB File', async () => { - await conn.send(`CREATE TABLE tmp AS SELECT * FROM "${baseDir}/tpch/0_01/parquet/lineitem.parquet"`); + //1. data preparation + await conn.send(`CREATE TABLE tmp AS SELECT * FROM "${ baseDir }/tpch/0_01/parquet/lineitem.parquet"`); await conn.send(`CHECKPOINT;`); + await conn.close(); + await db.reset(); + await db.dropFiles(); await db.terminate(); - const logger = new duckdb.ConsoleLogger(LogLevel.ERROR); const worker = new Worker(bundle().mainWorker!); db = new duckdb.AsyncDuckDB(logger, worker); await db.instantiate(bundle().mainModule, bundle().pthreadWorker); @@ -98,16 +101,13 @@ export function testOPFS(baseDir: string, bundle: () => duckdb.DuckDBBundle): vo it('Load Parquet file that are already with empty handler', async () => { //1. write to opfs - const parquetBuffer = await fetch(`${baseDir}/tpch/0_01/parquet/lineitem.parquet`).then(res => - res.arrayBuffer(), - ); - const opfsRoot = await navigator.storage.getDirectory(); - const fileHandle = await opfsRoot.getFileHandle('test.parquet', {create: true}); - const writable = await fileHandle.createWritable(); - await writable.write(parquetBuffer); - await writable.close(); + const fileHandler = await getOpfsFileHandlerFromUrl({ + url: `${ baseDir }/tpch/0_01/parquet/lineitem.parquet`, + path: 'test.parquet' + }); //2. handle is empty object, because worker gets a File Handle using the file name. - await db.registerFileHandle('test.parquet', null, duckdb.DuckDBDataProtocol.BROWSER_FSACCESS, true); + await db.registerFileHandle('test.parquet', fileHandler, duckdb.DuckDBDataProtocol.BROWSER_FSACCESS, true); + //3. data preparation await conn.send(`CREATE TABLE lineitem1 AS SELECT * FROM read_parquet('test.parquet')`); await conn.send(`CHECKPOINT;`); @@ -122,18 +122,14 @@ export function testOPFS(baseDir: string, bundle: () => duckdb.DuckDBBundle): vo it('Load Parquet file that are already with opfs file handler in datadir', async () => { //1. write to opfs - const parquetBuffer = await fetch(`${baseDir}/tpch/0_01/parquet/lineitem.parquet`).then(res => - res.arrayBuffer(), - ); - const opfsRoot = await navigator.storage.getDirectory(); - const datadir = await opfsRoot.getDirectoryHandle("datadir", {create: true}); - const fileHandle = await datadir.getFileHandle('test.parquet', {create: true}); - const writable = await fileHandle.createWritable(); - await writable.write(parquetBuffer); - await writable.close(); + const fileHandler = await getOpfsFileHandlerFromUrl({ + url: `${ baseDir }/tpch/0_01/parquet/lineitem.parquet`, + path: 'datadir/test.parquet' + }); //2. handle is opfs file handler - await db.registerFileHandle('test.parquet', fileHandle, duckdb.DuckDBDataProtocol.BROWSER_FSACCESS, true); - await conn.send(`CREATE TABLE lineitem1 AS SELECT * FROM read_parquet('test.parquet')`); + await db.registerFileHandle('datadir/test.parquet', fileHandler, duckdb.DuckDBDataProtocol.BROWSER_FSACCESS, true); + //3. data preparation + await conn.send(`CREATE TABLE lineitem1 AS SELECT * FROM read_parquet('datadir/test.parquet')`); await conn.send(`CHECKPOINT;`); const result1 = await conn.send(`SELECT count(*)::INTEGER as cnt FROM lineitem1;`); @@ -146,16 +142,14 @@ export function testOPFS(baseDir: string, bundle: () => duckdb.DuckDBBundle): vo }); it('Load Parquet file that are already', async () => { - const parquetBuffer = await fetch(`${baseDir}/tpch/0_01/parquet/lineitem.parquet`).then(res => - res.arrayBuffer(), - ); - const opfsRoot = await navigator.storage.getDirectory(); - const fileHandle = await opfsRoot.getFileHandle('test.parquet', {create: true}); - const writable = await fileHandle.createWritable(); - await writable.write(parquetBuffer); - await writable.close(); - + //1. write to opfs + const fileHandle = await getOpfsFileHandlerFromUrl({ + url: `${ baseDir }/tpch/0_01/parquet/lineitem.parquet`, + path: 'test.parquet' + }); + //2. handle is opfs file handler await db.registerFileHandle('test.parquet', fileHandle, duckdb.DuckDBDataProtocol.BROWSER_FSACCESS, true); + //3. data preparation await conn.send(`CREATE TABLE lineitem1 AS SELECT * FROM read_parquet('test.parquet')`); await conn.send(`CHECKPOINT;`); await conn.send(`CREATE TABLE lineitem2 AS SELECT * FROM read_parquet('test.parquet')`); @@ -196,19 +190,21 @@ export function testOPFS(baseDir: string, bundle: () => duckdb.DuckDBBundle): vo }); it('Drop File + Export as CSV to OPFS + Load CSV', async () => { + //1. write to opfs const opfsRoot = await navigator.storage.getDirectory(); - const testHandle = await opfsRoot.getFileHandle('test.csv', {create: true}); - await db.registerFileHandle('test.csv', testHandle, duckdb.DuckDBDataProtocol.BROWSER_FSACCESS, true); - await conn.send(`CREATE TABLE zzz AS SELECT * FROM "${baseDir}/tpch/0_01/parquet/lineitem.parquet"`); + const fileHandler = await opfsRoot.getFileHandle('test.csv', { create: true }); + //2. handle is opfs file handler + await db.registerFileHandle('test.csv', fileHandler, duckdb.DuckDBDataProtocol.BROWSER_FSACCESS, true); + //3. data preparation + await conn.send(`CREATE TABLE zzz AS SELECT * FROM '${ baseDir }/tpch/0_01/parquet/lineitem.parquet'`); await conn.send(`COPY (SELECT * FROM zzz) TO 'test.csv'`); await conn.send(`COPY (SELECT * FROM zzz) TO 'non_existing.csv'`); await conn.close(); await db.dropFile('test.csv'); await db.reset(); - await db.open({}); conn = await db.connect(); - await db.registerFileHandle('test.csv', testHandle, duckdb.DuckDBDataProtocol.BROWSER_FSACCESS, true); + await db.registerFileHandle('test.csv', fileHandler, duckdb.DuckDBDataProtocol.BROWSER_FSACCESS, true); const result = await conn.send(`SELECT count(*)::INTEGER as cnt FROM 'test.csv';`); const batches = []; @@ -221,26 +217,29 @@ export function testOPFS(baseDir: string, bundle: () => duckdb.DuckDBBundle): vo await db.dropFile('test.csv'); }); - it('Drop Files + Export as CSV to OPFS + Load CSV', async () => { + //1. write to opfs const opfsRoot = await navigator.storage.getDirectory(); - const testHandle1 = await opfsRoot.getFileHandle('test1.csv', {create: true}); - const testHandle2 = await opfsRoot.getFileHandle('test2.csv', {create: true}); - const testHandle3 = await opfsRoot.getFileHandle('test3.csv', {create: true}); + const testHandle1 = await opfsRoot.getFileHandle('test1.csv', { create: true }); + const testHandle2 = await opfsRoot.getFileHandle('test2.csv', { create: true }); + const testHandle3 = await opfsRoot.getFileHandle('test3.csv', { create: true }); + //2. handle is opfs file handler await db.registerFileHandle('test1.csv', testHandle1, duckdb.DuckDBDataProtocol.BROWSER_FSACCESS, true); await db.registerFileHandle('test2.csv', testHandle2, duckdb.DuckDBDataProtocol.BROWSER_FSACCESS, true); await db.registerFileHandle('test3.csv', testHandle3, duckdb.DuckDBDataProtocol.BROWSER_FSACCESS, true); - - await conn.send(`CREATE TABLE zzz AS SELECT * FROM "${baseDir}/tpch/0_01/parquet/lineitem.parquet"`); + //3. data preparation + await conn.send(`CREATE TABLE zzz AS SELECT * FROM "${ baseDir }/tpch/0_01/parquet/lineitem.parquet"`); await conn.send(`COPY (SELECT * FROM zzz) TO 'test1.csv'`); await conn.send(`COPY (SELECT * FROM zzz) TO 'test2.csv'`); await conn.send(`COPY (SELECT * FROM zzz) TO 'test3.csv'`); await conn.close(); - await db.dropFiles(); + //4. dropFiles + await db.dropFiles(['test1.csv', 'test2.csv', 'test3.csv']); + + //5. reset await db.reset(); - await db.open({}); conn = await db.connect(); await db.registerFileHandle('test1.csv', testHandle1, duckdb.DuckDBDataProtocol.BROWSER_FSACCESS, true); await db.registerFileHandle('test2.csv', testHandle2, duckdb.DuckDBDataProtocol.BROWSER_FSACCESS, true); @@ -273,8 +272,88 @@ export function testOPFS(baseDir: string, bundle: () => duckdb.DuckDBBundle): vo const table3 = await new arrow.Table<{ cnt: arrow.Int }>(batches3); expect(table3.getChildAt(0)?.get(0)).toBeGreaterThan(60_000); } + }); - await db.dropFiles(); + it('Load Parquet file when FROM clause', async () => { + //1. write to opfs + await getOpfsFileHandlerFromUrl({ + url: `${ baseDir }/tpch/0_01/parquet/lineitem.parquet`, + path: 'test.parquet' + }); + await conn.close(); + await db.reset(); + await db.dropFile('test.parquet'); + db.config.autoFileRegistration = true; + conn = await db.connect(); + //2. send query + const result1 = await conn.send(`SELECT count(*)::INTEGER as cnt FROM 'opfs://test.parquet'`); + const batches1 = []; + for await (const batch of result1) { + batches1.push(batch); + } + const table1 = await new arrow.Table<{ cnt: arrow.Int }>(batches1); + expect(table1.getChildAt(0)?.get(0)).toBeGreaterThan(60_000); + }); + + it('Load Parquet file when FROM clause + read_parquet', async () => { + //1. write to opfs + await getOpfsFileHandlerFromUrl({ + url: `${ baseDir }/uni/studenten.parquet`, + path: 'test.parquet' + }); + await conn.close(); + await db.reset(); + await db.dropFile('test.parquet'); + db.config.autoFileRegistration = true; + conn = await db.connect(); + //2. send query + const result = await conn.send(`SELECT * FROM read_parquet('opfs://test.parquet');`); + const batches = []; + for await (const batch of result) { + batches.push(batch); + } + const table = await new arrow.Table<{ cnt: arrow.Int }>(batches); + expect(table.getChildAt(0)?.toArray()).toEqual( + new Int32Array([24002, 25403, 26120, 26830, 27550, 28106, 29120, 29555]), + ); + }); + + it('Load Parquet file with dir when FROM clause', async () => { + //1. write to opfs + await getOpfsFileHandlerFromUrl({ + url: `${ baseDir }/tpch/0_01/parquet/lineitem.parquet`, + path: 'datadir/test.parquet' + }); + await conn.close(); + await db.reset(); + await db.dropFile('datadir/test.parquet'); + db.config.autoFileRegistration = true; + conn = await db.connect(); + //2. send query + const result1 = await conn.send(`SELECT count(*)::INTEGER as cnt FROM 'opfs://datadir/test.parquet'`); + const batches1 = []; + for await (const batch of result1) { + batches1.push(batch); + } + const table1 = await new arrow.Table<{ cnt: arrow.Int }>(batches1); + expect(table1.getChildAt(0)?.get(0)).toBeGreaterThan(60_000); + }); + + it('Load Parquet file with dir when FROM clause with IO Error', async () => { + //1. write to opfs + await getOpfsFileHandlerFromUrl({ + url: `${ baseDir }/tpch/0_01/parquet/lineitem.parquet`, + path: 'datadir/test.parquet' + }); + try { + //2. send query + await expectAsync( + conn.send(`SELECT count(*)::INTEGER as cnt FROM 'opfs://datadir/test.parquet'`) + ).toBeRejectedWithError("IO Error: No files found that match the pattern \"opfs://datadir/test.parquet\""); + } finally { + await db.reset(); + await db.dropFiles(); + } }); }); @@ -304,4 +383,33 @@ export function testOPFS(baseDir: string, bundle: () => duckdb.DuckDBBundle): vo await opfsRoot.removeEntry('datadir').catch(() => { }); } + + async function getOpfsFileHandlerFromUrl(params: { + url: string; + path: string; + }): Promise { + const PATH_SEP_REGEX = /\/|\\/; + const parquetBuffer = await fetch(params.url).then(res => + res.arrayBuffer(), + ); + const opfsRoot = await navigator.storage.getDirectory(); + let dirHandle: FileSystemDirectoryHandle = opfsRoot; + let fileName = params.path; + if (PATH_SEP_REGEX.test(params.path)) { + const folders = params.path.split(PATH_SEP_REGEX); + fileName = folders.pop()!; + if (!fileName) { + throw new Error(`Invalid path ${ params.path }`); + } + for (const folder of folders) { + dirHandle = await dirHandle.getDirectoryHandle(folder, { create: true }); + } + } + const fileHandle = await dirHandle.getFileHandle(fileName, { create: true }); + const writable = await fileHandle.createWritable(); + await writable.write(parquetBuffer); + await writable.close(); + + return fileHandle; + } } From add8fc817e807d9d04ed5f40d4bae3a8558caf0a Mon Sep 17 00:00:00 2001 From: arkw Date: Mon, 27 Jan 2025 04:01:34 +0900 Subject: [PATCH 02/13] add test for copy csv --- packages/duckdb-wasm/test/opfs.test.ts | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/packages/duckdb-wasm/test/opfs.test.ts b/packages/duckdb-wasm/test/opfs.test.ts index 6dbb231e4..c3cc5b6b8 100644 --- a/packages/duckdb-wasm/test/opfs.test.ts +++ b/packages/duckdb-wasm/test/opfs.test.ts @@ -7,11 +7,11 @@ export function testOPFS(baseDir: string, bundle: () => duckdb.DuckDBBundle): vo let conn: duckdb.AsyncDuckDBConnection; beforeAll(async () => { - removeFiles(); + await removeFiles(); }); afterAll(async () => { - removeFiles(); + await removeFiles(); }); beforeEach(async () => { @@ -355,6 +355,22 @@ export function testOPFS(baseDir: string, bundle: () => duckdb.DuckDBBundle): vo await db.dropFiles(); } }); + + it('Copy CSV to OPFS + Load CSV', async () => { + //1. data preparation + db.config.autoFileRegistration = true; + await conn.query(`COPY ( SELECT 32 AS value ) TO 'opfs://file.csv'`); + await conn.query(`COPY ( SELECT 42 AS value ) TO 'opfs://file.csv'`); + const result = await conn.send(`SELECT * FROM 'opfs://file.csv';`); + const batches = []; + for await (const batch of result) { + batches.push(batch); + } + const table = await new arrow.Table<{ cnt: arrow.Int }>(batches); + expect(table.getChildAt(0)?.toArray()).toEqual( + new BigInt64Array([42n]), + ); + }); }); async function removeFiles() { From 64c728824d625eace93d189f6a7b2693a8c9db64 Mon Sep 17 00:00:00 2001 From: arkw Date: Wed, 19 Feb 2025 15:10:02 +0900 Subject: [PATCH 03/13] Merge branch 'develop' into feature/from_opfs_path --- packages/duckdb-wasm/src/bindings/config.ts | 11 +++++++-- .../src/parallel/async_bindings.ts | 24 ++++++++++++++----- packages/duckdb-wasm/test/opfs.test.ts | 16 +++++++++---- submodules/duckdb | 2 +- 4 files changed, 40 insertions(+), 13 deletions(-) diff --git a/packages/duckdb-wasm/src/bindings/config.ts b/packages/duckdb-wasm/src/bindings/config.ts index 27389d1eb..6d7659a86 100644 --- a/packages/duckdb-wasm/src/bindings/config.ts +++ b/packages/duckdb-wasm/src/bindings/config.ts @@ -29,6 +29,13 @@ export interface DuckDBFilesystemConfig { allowFullHTTPReads?: boolean; } +export interface DuckDBOPFSConfig { + /** + * Auto Opfs File Registration + */ + autoFileRegistration?: boolean; +} + export enum DuckDBAccessMode { UNDEFINED = 0, AUTOMATIC = 1, @@ -71,7 +78,7 @@ export interface DuckDBConfig { */ customUserAgent?: string; /** - * Auto Opfs File Registration + * opfs string */ - autoFileRegistration?: boolean; + opfs?: DuckDBOPFSConfig; } diff --git a/packages/duckdb-wasm/src/parallel/async_bindings.ts b/packages/duckdb-wasm/src/parallel/async_bindings.ts index cd95549fa..7f7ebd0a4 100644 --- a/packages/duckdb-wasm/src/parallel/async_bindings.ts +++ b/packages/duckdb-wasm/src/parallel/async_bindings.ts @@ -20,7 +20,8 @@ import { WebFile } from '../bindings/web_file'; import { DuckDBDataProtocol } from '../bindings'; const TEXT_ENCODER = new TextEncoder(); -const OPFS_PROTOCOL_REGEX = /'(opfs:\/\/\S*?)'/g; +const REGEX_OPFS_FILE = /'(opfs:\/\/\S*?)'/g; +const REGEX_OPFS_PROTOCOL = /(opfs:\/\/\S*?)/g; export class AsyncDuckDB implements AsyncDuckDBBindings { /** The message handler */ @@ -368,7 +369,6 @@ export class AsyncDuckDB implements AsyncDuckDBBindings { /** Open a new database */ public async open(config: DuckDBConfig): Promise { - config.autoFileRegistration = config.autoFileRegistration ?? false; this._config = config; const task = new WorkerTask(WorkerRequestType.OPEN, config); await this.postTask(task); @@ -404,7 +404,7 @@ export class AsyncDuckDB implements AsyncDuckDBBindings { /** Run a query */ public async runQuery(conn: ConnectionID, text: string): Promise { - if( this._config.autoFileRegistration ){ + if( this.isOpenedOPFSAutoFileRegistration() ){ const files = await this._preFileRegistration(text); try { return await this._runQueryAsync(conn, text); @@ -417,6 +417,7 @@ export class AsyncDuckDB implements AsyncDuckDBBindings { return await this._runQueryAsync(conn, text); } } + private async _runQueryAsync(conn: ConnectionID, text: string): Promise { const task = new WorkerTask( WorkerRequestType.RUN_QUERY, @@ -424,13 +425,14 @@ export class AsyncDuckDB implements AsyncDuckDBBindings { ); return await this.postTask(task); } + /** Start a pending query */ public async startPendingQuery( conn: ConnectionID, text: string, allowStreamResult: boolean = false, ): Promise { - if( this._config.autoFileRegistration ){ + if( this.isOpenedOPFSAutoFileRegistration() ){ const files = await this._preFileRegistration(text); try { return await this._startPendingQueryAsync(conn, text, allowStreamResult); @@ -443,6 +445,7 @@ export class AsyncDuckDB implements AsyncDuckDBBindings { return await this._startPendingQueryAsync(conn, text, allowStreamResult); } } + private async _startPendingQueryAsync( conn: ConnectionID, text: string, @@ -455,6 +458,7 @@ export class AsyncDuckDB implements AsyncDuckDBBindings { >(WorkerRequestType.START_PENDING_QUERY, [conn, text, allowStreamResult]); return await this.postTask(task); } + /** Poll a pending query */ public async pollPendingQuery(conn: ConnectionID): Promise { const task = new WorkerTask( @@ -689,8 +693,16 @@ export class AsyncDuckDB implements AsyncDuckDBBindings { await this.postTask(task); } + private isOpenedOPFSAutoFileRegistration():boolean { + let path = this.config.path ?? ""; + if( path.search(REGEX_OPFS_PROTOCOL) > -1){ + return this.config.opfs?.autoFileRegistration ?? false; + } + return false; + } + private async _preFileRegistration(text: string) { - const files = [...text.matchAll(OPFS_PROTOCOL_REGEX)].map(match => match[1]); + const files = [...text.matchAll(REGEX_OPFS_FILE)].map(match => match[1]); const result: string[] = []; for (const file of files) { try { @@ -698,7 +710,7 @@ export class AsyncDuckDB implements AsyncDuckDBBindings { result.push(file); } catch (e) { console.error(e); - throw new Error("file Not found:" + file); + throw new Error("File Not found:" + file); } } return result; diff --git a/packages/duckdb-wasm/test/opfs.test.ts b/packages/duckdb-wasm/test/opfs.test.ts index c3cc5b6b8..e13709d58 100644 --- a/packages/duckdb-wasm/test/opfs.test.ts +++ b/packages/duckdb-wasm/test/opfs.test.ts @@ -283,7 +283,9 @@ export function testOPFS(baseDir: string, bundle: () => duckdb.DuckDBBundle): vo await conn.close(); await db.reset(); await db.dropFile('test.parquet'); - db.config.autoFileRegistration = true; + db.config.opfs = { + autoFileRegistration: true + }; conn = await db.connect(); //2. send query const result1 = await conn.send(`SELECT count(*)::INTEGER as cnt FROM 'opfs://test.parquet'`); @@ -304,7 +306,9 @@ export function testOPFS(baseDir: string, bundle: () => duckdb.DuckDBBundle): vo await conn.close(); await db.reset(); await db.dropFile('test.parquet'); - db.config.autoFileRegistration = true; + db.config.opfs = { + autoFileRegistration: true + }; conn = await db.connect(); //2. send query const result = await conn.send(`SELECT * FROM read_parquet('opfs://test.parquet');`); @@ -327,7 +331,9 @@ export function testOPFS(baseDir: string, bundle: () => duckdb.DuckDBBundle): vo await conn.close(); await db.reset(); await db.dropFile('datadir/test.parquet'); - db.config.autoFileRegistration = true; + db.config.opfs = { + autoFileRegistration: true + }; conn = await db.connect(); //2. send query const result1 = await conn.send(`SELECT count(*)::INTEGER as cnt FROM 'opfs://datadir/test.parquet'`); @@ -358,7 +364,9 @@ export function testOPFS(baseDir: string, bundle: () => duckdb.DuckDBBundle): vo it('Copy CSV to OPFS + Load CSV', async () => { //1. data preparation - db.config.autoFileRegistration = true; + db.config.opfs = { + autoFileRegistration: true + }; await conn.query(`COPY ( SELECT 32 AS value ) TO 'opfs://file.csv'`); await conn.query(`COPY ( SELECT 42 AS value ) TO 'opfs://file.csv'`); const result = await conn.send(`SELECT * FROM 'opfs://file.csv';`); diff --git a/submodules/duckdb b/submodules/duckdb index 19864453f..5f5512b82 160000 --- a/submodules/duckdb +++ b/submodules/duckdb @@ -1 +1 @@ -Subproject commit 19864453f7d0ed095256d848b46e7b8630989bac +Subproject commit 5f5512b827df6397afd31daedb4bbdee76520019 From e532a2a067c181b69584f163ce84ead964f2e2c7 Mon Sep 17 00:00:00 2001 From: arkw Date: Thu, 20 Feb 2025 08:24:19 +0900 Subject: [PATCH 04/13] add opfs_util and fix --- packages/duckdb-wasm/src/bindings/config.ts | 6 +++-- .../src/parallel/async_bindings.ts | 22 +++++++++---------- packages/duckdb-wasm/src/utils/opfs_util.ts | 10 +++++++++ packages/duckdb-wasm/test/opfs.test.ts | 8 +++---- 4 files changed, 28 insertions(+), 18 deletions(-) create mode 100644 packages/duckdb-wasm/src/utils/opfs_util.ts diff --git a/packages/duckdb-wasm/src/bindings/config.ts b/packages/duckdb-wasm/src/bindings/config.ts index 6d7659a86..a3013d19e 100644 --- a/packages/duckdb-wasm/src/bindings/config.ts +++ b/packages/duckdb-wasm/src/bindings/config.ts @@ -31,9 +31,11 @@ export interface DuckDBFilesystemConfig { export interface DuckDBOPFSConfig { /** - * Auto Opfs File Registration + * Defines how `opfs://` files are handled during SQL execution. + * - "auto": Automatically register `opfs://` files and drop them after execution. + * - "manual": Files must be manually registered and dropped. */ - autoFileRegistration?: boolean; + fileHandling?: "auto" | "manual"; } export enum DuckDBAccessMode { diff --git a/packages/duckdb-wasm/src/parallel/async_bindings.ts b/packages/duckdb-wasm/src/parallel/async_bindings.ts index 7f7ebd0a4..e2c9f3056 100644 --- a/packages/duckdb-wasm/src/parallel/async_bindings.ts +++ b/packages/duckdb-wasm/src/parallel/async_bindings.ts @@ -18,10 +18,9 @@ import { InstantiationProgress } from '../bindings/progress'; import { arrowToSQLField } from '../json_typedef'; import { WebFile } from '../bindings/web_file'; import { DuckDBDataProtocol } from '../bindings'; +import { searchOPFSFiles, isOPFSProtocol } from "../utils/opfs_util"; const TEXT_ENCODER = new TextEncoder(); -const REGEX_OPFS_FILE = /'(opfs:\/\/\S*?)'/g; -const REGEX_OPFS_PROTOCOL = /(opfs:\/\/\S*?)/g; export class AsyncDuckDB implements AsyncDuckDBBindings { /** The message handler */ @@ -404,8 +403,8 @@ export class AsyncDuckDB implements AsyncDuckDBBindings { /** Run a query */ public async runQuery(conn: ConnectionID, text: string): Promise { - if( this.isOpenedOPFSAutoFileRegistration() ){ - const files = await this._preFileRegistration(text); + if( this.shouldOPFSFileHandling() ){ + const files = await this.registerOPFSFileFromSQL(text); try { return await this._runQueryAsync(conn, text); } finally { @@ -432,8 +431,8 @@ export class AsyncDuckDB implements AsyncDuckDBBindings { text: string, allowStreamResult: boolean = false, ): Promise { - if( this.isOpenedOPFSAutoFileRegistration() ){ - const files = await this._preFileRegistration(text); + if( this.shouldOPFSFileHandling() ){ + const files = await this.registerOPFSFileFromSQL(text); try { return await this._startPendingQueryAsync(conn, text, allowStreamResult); } finally { @@ -693,16 +692,15 @@ export class AsyncDuckDB implements AsyncDuckDBBindings { await this.postTask(task); } - private isOpenedOPFSAutoFileRegistration():boolean { - let path = this.config.path ?? ""; - if( path.search(REGEX_OPFS_PROTOCOL) > -1){ - return this.config.opfs?.autoFileRegistration ?? false; + private shouldOPFSFileHandling():boolean { + if( isOPFSProtocol(this.config.path ?? "")){ + return this.config.opfs?.fileHandling == "auto"; } return false; } - private async _preFileRegistration(text: string) { - const files = [...text.matchAll(REGEX_OPFS_FILE)].map(match => match[1]); + private async registerOPFSFileFromSQL(text: string) { + const files = searchOPFSFiles(text); const result: string[] = []; for (const file of files) { try { diff --git a/packages/duckdb-wasm/src/utils/opfs_util.ts b/packages/duckdb-wasm/src/utils/opfs_util.ts new file mode 100644 index 000000000..822eb4a7c --- /dev/null +++ b/packages/duckdb-wasm/src/utils/opfs_util.ts @@ -0,0 +1,10 @@ +export const REGEX_OPFS_FILE = /'(opfs:\/\/\S*?)'/g; +export const REGEX_OPFS_PROTOCOL = /(opfs:\/\/\S*?)/g; + +export function isOPFSProtocol(path: string): boolean { + return path.search(REGEX_OPFS_PROTOCOL) > -1; +} + +export function searchOPFSFiles(text: string) { + return [...text.matchAll(REGEX_OPFS_FILE)].map(match => match[1]); +} \ No newline at end of file diff --git a/packages/duckdb-wasm/test/opfs.test.ts b/packages/duckdb-wasm/test/opfs.test.ts index e13709d58..63686c813 100644 --- a/packages/duckdb-wasm/test/opfs.test.ts +++ b/packages/duckdb-wasm/test/opfs.test.ts @@ -284,7 +284,7 @@ export function testOPFS(baseDir: string, bundle: () => duckdb.DuckDBBundle): vo await db.reset(); await db.dropFile('test.parquet'); db.config.opfs = { - autoFileRegistration: true + fileHandling: "auto" }; conn = await db.connect(); //2. send query @@ -307,7 +307,7 @@ export function testOPFS(baseDir: string, bundle: () => duckdb.DuckDBBundle): vo await db.reset(); await db.dropFile('test.parquet'); db.config.opfs = { - autoFileRegistration: true + fileHandling: "auto" }; conn = await db.connect(); //2. send query @@ -332,7 +332,7 @@ export function testOPFS(baseDir: string, bundle: () => duckdb.DuckDBBundle): vo await db.reset(); await db.dropFile('datadir/test.parquet'); db.config.opfs = { - autoFileRegistration: true + fileHandling: "auto" }; conn = await db.connect(); //2. send query @@ -365,7 +365,7 @@ export function testOPFS(baseDir: string, bundle: () => duckdb.DuckDBBundle): vo it('Copy CSV to OPFS + Load CSV', async () => { //1. data preparation db.config.opfs = { - autoFileRegistration: true + fileHandling: "auto" }; await conn.query(`COPY ( SELECT 32 AS value ) TO 'opfs://file.csv'`); await conn.query(`COPY ( SELECT 42 AS value ) TO 'opfs://file.csv'`); From c743a7497435888e59130c89dec77fe84c3a29ca Mon Sep 17 00:00:00 2001 From: arkw Date: Thu, 20 Feb 2025 19:58:11 +0900 Subject: [PATCH 05/13] change space indents --- packages/duckdb-wasm/src/bindings/bindings_base.ts | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/packages/duckdb-wasm/src/bindings/bindings_base.ts b/packages/duckdb-wasm/src/bindings/bindings_base.ts index 34f201d3d..68975162c 100644 --- a/packages/duckdb-wasm/src/bindings/bindings_base.ts +++ b/packages/duckdb-wasm/src/bindings/bindings_base.ts @@ -525,7 +525,7 @@ export abstract class DuckDBBindingsBase implements DuckDBBindings { } } } - return handle; + return handle; } /** Register a file object URL async */ public async registerFileHandleAsync( @@ -656,10 +656,10 @@ export abstract class DuckDBBindingsBase implements DuckDBBindings { } /** Enable tracking of file statistics */ public registerOPFSFileName(file: string): Promise { - if (file.startsWith("opfs://")) { - return this.prepareFileHandle(file, DuckDBDataProtocol.BROWSER_FSACCESS); - } else { - throw new Error("Not an OPFS file name: " + file); + if (file.startsWith("opfs://")) { + return this.prepareFileHandle(file, DuckDBDataProtocol.BROWSER_FSACCESS); + } else { + throw new Error("Not an OPFS file name: " + file); } } public async registerOPFSFileNameAsync(file: string): Promise { From d42ff6754eec96e751cf07ab3c4264d504325145 Mon Sep 17 00:00:00 2001 From: arkw Date: Thu, 20 Feb 2025 20:03:09 +0900 Subject: [PATCH 06/13] fix registerOPFSFileName --- packages/duckdb-wasm/src/bindings/bindings_base.ts | 9 +-------- packages/duckdb-wasm/src/bindings/bindings_interface.ts | 3 +-- packages/duckdb-wasm/src/parallel/worker_dispatcher.ts | 2 +- 3 files changed, 3 insertions(+), 11 deletions(-) diff --git a/packages/duckdb-wasm/src/bindings/bindings_base.ts b/packages/duckdb-wasm/src/bindings/bindings_base.ts index 68975162c..cf729a2ff 100644 --- a/packages/duckdb-wasm/src/bindings/bindings_base.ts +++ b/packages/duckdb-wasm/src/bindings/bindings_base.ts @@ -655,14 +655,7 @@ export abstract class DuckDBBindingsBase implements DuckDBBindings { return copy; } /** Enable tracking of file statistics */ - public registerOPFSFileName(file: string): Promise { - if (file.startsWith("opfs://")) { - return this.prepareFileHandle(file, DuckDBDataProtocol.BROWSER_FSACCESS); - } else { - throw new Error("Not an OPFS file name: " + file); - } - } - public async registerOPFSFileNameAsync(file: string): Promise { + public async registerOPFSFileName(file: string): Promise { if (file.startsWith("opfs://")) { return await this.prepareFileHandle(file, DuckDBDataProtocol.BROWSER_FSACCESS); } else { diff --git a/packages/duckdb-wasm/src/bindings/bindings_interface.ts b/packages/duckdb-wasm/src/bindings/bindings_interface.ts index 004d2d46c..08f8fee47 100644 --- a/packages/duckdb-wasm/src/bindings/bindings_interface.ts +++ b/packages/duckdb-wasm/src/bindings/bindings_interface.ts @@ -62,8 +62,7 @@ export interface DuckDBBindings { flushFiles(): void; copyFileToPath(name: string, path: string): void; copyFileToBuffer(name: string): Uint8Array; - registerOPFSFileName(file: string): void; - registerOPFSFileNameAsync(file: string): Promise; + registerOPFSFileName(file: string): Promise; collectFileStatistics(file: string, enable: boolean): void; exportFileStatistics(file: string): FileStatistics; } diff --git a/packages/duckdb-wasm/src/parallel/worker_dispatcher.ts b/packages/duckdb-wasm/src/parallel/worker_dispatcher.ts index d3b666ba7..db05cb50f 100644 --- a/packages/duckdb-wasm/src/parallel/worker_dispatcher.ts +++ b/packages/duckdb-wasm/src/parallel/worker_dispatcher.ts @@ -361,7 +361,7 @@ export abstract class AsyncDuckDBDispatcher implements Logger { break; case WorkerRequestType.REGISTER_OPFS_FILE_NAME: - await this._bindings.registerOPFSFileNameAsync(request.data[0]); + await this._bindings.registerOPFSFileName(request.data[0]); this.sendOK(request); break; From 2c03955e6142ef33f607ffc4a2d34e97fdb16f7d Mon Sep 17 00:00:00 2001 From: arkw Date: Thu, 20 Feb 2025 20:51:49 +0900 Subject: [PATCH 07/13] fix --- packages/duckdb-wasm/test/opfs.test.ts | 120 ++++++++++++------------- 1 file changed, 60 insertions(+), 60 deletions(-) diff --git a/packages/duckdb-wasm/test/opfs.test.ts b/packages/duckdb-wasm/test/opfs.test.ts index eaf1a0fcc..051a7ebf6 100644 --- a/packages/duckdb-wasm/test/opfs.test.ts +++ b/packages/duckdb-wasm/test/opfs.test.ts @@ -1,10 +1,21 @@ -import * as duckdb from '../src/'; -import {LogLevel} from '../src/'; +import { + AsyncDuckDB, + AsyncDuckDBConnection, + ConsoleLogger, + DuckDBAccessMode, + DuckDBBundle, + DuckDBDataProtocol, + LogLevel +} from '../src/'; import * as arrow from 'apache-arrow'; -export function testOPFS(baseDir: string, bundle: () => duckdb.DuckDBBundle): void { - let db: duckdb.AsyncDuckDB; - let conn: duckdb.AsyncDuckDBConnection; +export function testOPFS(baseDir: string, bundle: () => DuckDBBundle): void { + const logger = new ConsoleLogger(LogLevel.ERROR); + + let db: AsyncDuckDB; + let conn: AsyncDuckDBConnection; + const _ignore: () => void = () => { + }; beforeAll(async () => { removeFiles(); @@ -17,19 +28,18 @@ export function testOPFS(baseDir: string, bundle: () => duckdb.DuckDBBundle): vo if (db) { await db.terminate(); } - removeFiles(); + await removeFiles(); }); beforeEach(async () => { - removeFiles(); + await removeFiles(); // - const logger = new duckdb.ConsoleLogger(LogLevel.ERROR); const worker = new Worker(bundle().mainWorker!); - db = new duckdb.AsyncDuckDB(logger, worker); + db = new AsyncDuckDB(logger, worker); await db.instantiate(bundle().mainModule, bundle().pthreadWorker); await db.open({ path: 'opfs://test.db', - accessMode: duckdb.DuckDBAccessMode.READ_WRITE + accessMode: DuckDBAccessMode.READ_WRITE }); conn = await db.connect(); }); @@ -41,12 +51,12 @@ export function testOPFS(baseDir: string, bundle: () => duckdb.DuckDBBundle): vo if (db) { await db.terminate(); } - removeFiles(); + await removeFiles(); }); describe('Load Data in OPFS', () => { it('Import Small Parquet file', async () => { - await conn.send(`CREATE TABLE stu AS SELECT * FROM "${baseDir}/uni/studenten.parquet"`); + await conn.send(`CREATE TABLE stu AS SELECT * FROM "${ baseDir }/uni/studenten.parquet"`); await conn.send(`CHECKPOINT;`); const result = await conn.send(`SELECT matrnr FROM stu;`); const batches = []; @@ -60,7 +70,7 @@ export function testOPFS(baseDir: string, bundle: () => duckdb.DuckDBBundle): vo }); it('Import Larget Parquet file', async () => { - await conn.send(`CREATE TABLE lineitem AS SELECT * FROM "${baseDir}/tpch/0_01/parquet/lineitem.parquet"`); + await conn.send(`CREATE TABLE lineitem AS SELECT * FROM "${ baseDir }/tpch/0_01/parquet/lineitem.parquet"`); await conn.send(`CHECKPOINT;`); const result = await conn.send(`SELECT count(*)::INTEGER as cnt FROM lineitem;`); const batches = []; @@ -72,18 +82,17 @@ export function testOPFS(baseDir: string, bundle: () => duckdb.DuckDBBundle): vo }); it('Load Existing DB File', async () => { - await conn.send(`CREATE TABLE tmp AS SELECT * FROM "${baseDir}/tpch/0_01/parquet/lineitem.parquet"`); + await conn.send(`CREATE TABLE tmp AS SELECT * FROM "${ baseDir }/tpch/0_01/parquet/lineitem.parquet"`); await conn.send(`CHECKPOINT;`); await conn.close(); await db.terminate(); - const logger = new duckdb.ConsoleLogger(LogLevel.ERROR); const worker = new Worker(bundle().mainWorker!); - db = new duckdb.AsyncDuckDB(logger, worker); + db = new AsyncDuckDB(logger, worker); await db.instantiate(bundle().mainModule, bundle().pthreadWorker); await db.open({ path: 'opfs://test.db', - accessMode: duckdb.DuckDBAccessMode.READ_WRITE + accessMode: DuckDBAccessMode.READ_WRITE }); conn = await db.connect(); @@ -98,16 +107,16 @@ export function testOPFS(baseDir: string, bundle: () => duckdb.DuckDBBundle): vo it('Load Parquet file that are already with empty handler', async () => { //1. write to opfs - const parquetBuffer = await fetch(`${baseDir}/tpch/0_01/parquet/lineitem.parquet`).then(res => + const parquetBuffer = await fetch(`${ baseDir }/tpch/0_01/parquet/lineitem.parquet`).then(res => res.arrayBuffer(), ); const opfsRoot = await navigator.storage.getDirectory(); - const fileHandle = await opfsRoot.getFileHandle('test.parquet', {create: true}); + const fileHandle = await opfsRoot.getFileHandle('test.parquet', { create: true }); const writable = await fileHandle.createWritable(); await writable.write(parquetBuffer); await writable.close(); //2. handle is empty object, because worker gets a File Handle using the file name. - await db.registerFileHandle('test.parquet', null, duckdb.DuckDBDataProtocol.BROWSER_FSACCESS, true); + await db.registerFileHandle('test.parquet', null, DuckDBDataProtocol.BROWSER_FSACCESS, true); await conn.send(`CREATE TABLE lineitem1 AS SELECT * FROM read_parquet('test.parquet')`); await conn.send(`CHECKPOINT;`); @@ -122,17 +131,17 @@ export function testOPFS(baseDir: string, bundle: () => duckdb.DuckDBBundle): vo it('Load Parquet file that are already with opfs file handler in datadir', async () => { //1. write to opfs - const parquetBuffer = await fetch(`${baseDir}/tpch/0_01/parquet/lineitem.parquet`).then(res => + const parquetBuffer = await fetch(`${ baseDir }/tpch/0_01/parquet/lineitem.parquet`).then(res => res.arrayBuffer(), ); const opfsRoot = await navigator.storage.getDirectory(); - const datadir = await opfsRoot.getDirectoryHandle("datadir", {create: true}); - const fileHandle = await datadir.getFileHandle('test.parquet', {create: true}); + const datadir = await opfsRoot.getDirectoryHandle("datadir", { create: true }); + const fileHandle = await datadir.getFileHandle('test.parquet', { create: true }); const writable = await fileHandle.createWritable(); await writable.write(parquetBuffer); await writable.close(); //2. handle is opfs file handler - await db.registerFileHandle('test.parquet', fileHandle, duckdb.DuckDBDataProtocol.BROWSER_FSACCESS, true); + await db.registerFileHandle('test.parquet', fileHandle, DuckDBDataProtocol.BROWSER_FSACCESS, true); await conn.send(`CREATE TABLE lineitem1 AS SELECT * FROM read_parquet('test.parquet')`); await conn.send(`CHECKPOINT;`); @@ -146,16 +155,16 @@ export function testOPFS(baseDir: string, bundle: () => duckdb.DuckDBBundle): vo }); it('Load Parquet file that are already', async () => { - const parquetBuffer = await fetch(`${baseDir}/tpch/0_01/parquet/lineitem.parquet`).then(res => + const parquetBuffer = await fetch(`${ baseDir }/tpch/0_01/parquet/lineitem.parquet`).then(res => res.arrayBuffer(), ); const opfsRoot = await navigator.storage.getDirectory(); - const fileHandle = await opfsRoot.getFileHandle('test.parquet', {create: true}); + const fileHandle = await opfsRoot.getFileHandle('test.parquet', { create: true }); const writable = await fileHandle.createWritable(); await writable.write(parquetBuffer); await writable.close(); - await db.registerFileHandle('test.parquet', fileHandle, duckdb.DuckDBDataProtocol.BROWSER_FSACCESS, true); + await db.registerFileHandle('test.parquet', fileHandle, DuckDBDataProtocol.BROWSER_FSACCESS, true); await conn.send(`CREATE TABLE lineitem1 AS SELECT * FROM read_parquet('test.parquet')`); await conn.send(`CHECKPOINT;`); await conn.send(`CREATE TABLE lineitem2 AS SELECT * FROM read_parquet('test.parquet')`); @@ -197,9 +206,9 @@ export function testOPFS(baseDir: string, bundle: () => duckdb.DuckDBBundle): vo it('Drop File + Export as CSV to OPFS + Load CSV', async () => { const opfsRoot = await navigator.storage.getDirectory(); - const testHandle = await opfsRoot.getFileHandle('test.csv', {create: true}); - await db.registerFileHandle('test.csv', testHandle, duckdb.DuckDBDataProtocol.BROWSER_FSACCESS, true); - await conn.send(`CREATE TABLE zzz AS SELECT * FROM "${baseDir}/tpch/0_01/parquet/lineitem.parquet"`); + const testHandle = await opfsRoot.getFileHandle('test.csv', { create: true }); + await db.registerFileHandle('test.csv', testHandle, DuckDBDataProtocol.BROWSER_FSACCESS, true); + await conn.send(`CREATE TABLE zzz AS SELECT * FROM "${ baseDir }/tpch/0_01/parquet/lineitem.parquet"`); await conn.send(`COPY (SELECT * FROM zzz) TO 'test.csv'`); await conn.send(`COPY (SELECT * FROM zzz) TO 'non_existing.csv'`); await conn.close(); @@ -208,7 +217,7 @@ export function testOPFS(baseDir: string, bundle: () => duckdb.DuckDBBundle): vo await db.open({}); conn = await db.connect(); - await db.registerFileHandle('test.csv', testHandle, duckdb.DuckDBDataProtocol.BROWSER_FSACCESS, true); + await db.registerFileHandle('test.csv', testHandle, DuckDBDataProtocol.BROWSER_FSACCESS, true); const result = await conn.send(`SELECT count(*)::INTEGER as cnt FROM 'test.csv';`); const batches = []; @@ -224,14 +233,14 @@ export function testOPFS(baseDir: string, bundle: () => duckdb.DuckDBBundle): vo it('Drop Files + Export as CSV to OPFS + Load CSV', async () => { const opfsRoot = await navigator.storage.getDirectory(); - const testHandle1 = await opfsRoot.getFileHandle('test1.csv', {create: true}); - const testHandle2 = await opfsRoot.getFileHandle('test2.csv', {create: true}); - const testHandle3 = await opfsRoot.getFileHandle('test3.csv', {create: true}); - await db.registerFileHandle('test1.csv', testHandle1, duckdb.DuckDBDataProtocol.BROWSER_FSACCESS, true); - await db.registerFileHandle('test2.csv', testHandle2, duckdb.DuckDBDataProtocol.BROWSER_FSACCESS, true); - await db.registerFileHandle('test3.csv', testHandle3, duckdb.DuckDBDataProtocol.BROWSER_FSACCESS, true); - - await conn.send(`CREATE TABLE zzz AS SELECT * FROM "${baseDir}/tpch/0_01/parquet/lineitem.parquet"`); + const testHandle1 = await opfsRoot.getFileHandle('test1.csv', { create: true }); + const testHandle2 = await opfsRoot.getFileHandle('test2.csv', { create: true }); + const testHandle3 = await opfsRoot.getFileHandle('test3.csv', { create: true }); + await db.registerFileHandle('test1.csv', testHandle1, DuckDBDataProtocol.BROWSER_FSACCESS, true); + await db.registerFileHandle('test2.csv', testHandle2, DuckDBDataProtocol.BROWSER_FSACCESS, true); + await db.registerFileHandle('test3.csv', testHandle3, DuckDBDataProtocol.BROWSER_FSACCESS, true); + + await conn.send(`CREATE TABLE zzz AS SELECT * FROM "${ baseDir }/tpch/0_01/parquet/lineitem.parquet"`); await conn.send(`COPY (SELECT * FROM zzz) TO 'test1.csv'`); await conn.send(`COPY (SELECT * FROM zzz) TO 'test2.csv'`); await conn.send(`COPY (SELECT * FROM zzz) TO 'test3.csv'`); @@ -242,9 +251,9 @@ export function testOPFS(baseDir: string, bundle: () => duckdb.DuckDBBundle): vo await db.open({}); conn = await db.connect(); - await db.registerFileHandle('test1.csv', testHandle1, duckdb.DuckDBDataProtocol.BROWSER_FSACCESS, true); - await db.registerFileHandle('test2.csv', testHandle2, duckdb.DuckDBDataProtocol.BROWSER_FSACCESS, true); - await db.registerFileHandle('test3.csv', testHandle3, duckdb.DuckDBDataProtocol.BROWSER_FSACCESS, true); + await db.registerFileHandle('test1.csv', testHandle1, DuckDBDataProtocol.BROWSER_FSACCESS, true); + await db.registerFileHandle('test2.csv', testHandle2, DuckDBDataProtocol.BROWSER_FSACCESS, true); + await db.registerFileHandle('test3.csv', testHandle3, DuckDBDataProtocol.BROWSER_FSACCESS, true); { const result1 = await conn.send(`SELECT count(*)::INTEGER as cnt FROM 'test1.csv';`); @@ -280,28 +289,19 @@ export function testOPFS(baseDir: string, bundle: () => duckdb.DuckDBBundle): vo async function removeFiles() { const opfsRoot = await navigator.storage.getDirectory(); - await opfsRoot.removeEntry('test.db').catch(() => { - }); - await opfsRoot.removeEntry('test.db.wal').catch(() => { - }); - await opfsRoot.removeEntry('test.csv').catch(() => { - }); - await opfsRoot.removeEntry('test1.csv').catch(() => { - }); - await opfsRoot.removeEntry('test2.csv').catch(() => { - }); - await opfsRoot.removeEntry('test3.csv').catch(() => { - }); - await opfsRoot.removeEntry('test.parquet').catch(() => { - }); + await opfsRoot.removeEntry('test.db').catch(_ignore); + await opfsRoot.removeEntry('test.db.wal').catch(_ignore); + await opfsRoot.removeEntry('test.csv').catch(_ignore); + await opfsRoot.removeEntry('test1.csv').catch(_ignore); + await opfsRoot.removeEntry('test2.csv').catch(_ignore); + await opfsRoot.removeEntry('test3.csv').catch(_ignore); + await opfsRoot.removeEntry('test.parquet').catch(_ignore); try { const datadir = await opfsRoot.getDirectoryHandle('datadir'); - datadir.removeEntry('test.parquet').catch(() => { - }); + datadir.removeEntry('test.parquet').catch(_ignore); } catch (e) { // } - await opfsRoot.removeEntry('datadir').catch(() => { - }); + await opfsRoot.removeEntry('datadir').catch(_ignore); } } From 3a768f55ca8709f0aabb0309b05c3df3a5840b90 Mon Sep 17 00:00:00 2001 From: arkw Date: Thu, 20 Feb 2025 20:56:02 +0900 Subject: [PATCH 08/13] fix --- packages/duckdb-wasm/test/opfs.test.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/packages/duckdb-wasm/test/opfs.test.ts b/packages/duckdb-wasm/test/opfs.test.ts index 051a7ebf6..37387cfe1 100644 --- a/packages/duckdb-wasm/test/opfs.test.ts +++ b/packages/duckdb-wasm/test/opfs.test.ts @@ -14,8 +14,6 @@ export function testOPFS(baseDir: string, bundle: () => DuckDBBundle): void { let db: AsyncDuckDB; let conn: AsyncDuckDBConnection; - const _ignore: () => void = () => { - }; beforeAll(async () => { removeFiles(); @@ -305,3 +303,6 @@ export function testOPFS(baseDir: string, bundle: () => DuckDBBundle): void { await opfsRoot.removeEntry('datadir').catch(_ignore); } } + +//ignore block +const _ignore: () => void = () => {}; From d45a2343f4a764eced83a61a72e685d1246ab08a Mon Sep 17 00:00:00 2001 From: arkw Date: Wed, 19 Feb 2025 16:00:22 +0900 Subject: [PATCH 09/13] remove patch --- patches/duckdb/binary_executor.patch | 92 ---------------------------- 1 file changed, 92 deletions(-) delete mode 100644 patches/duckdb/binary_executor.patch diff --git a/patches/duckdb/binary_executor.patch b/patches/duckdb/binary_executor.patch deleted file mode 100644 index d93319cbc..000000000 --- a/patches/duckdb/binary_executor.patch +++ /dev/null @@ -1,92 +0,0 @@ -diff --git a/src/include/duckdb/common/vector_operations/binary_executor.hpp b/src/include/duckdb/common/vector_operations/binary_executor.hpp -index 55c10bb289..c5f57edabf 100644 ---- a/src/include/duckdb/common/vector_operations/binary_executor.hpp -+++ b/src/include/duckdb/common/vector_operations/binary_executor.hpp -@@ -381,6 +381,8 @@ public: - } - } - -+#define DUCKDB_SMALLER_BINARY -+ - template - static idx_t SelectFlat(Vector &left, Vector &right, const SelectionVector *sel, idx_t count, - SelectionVector *true_sel, SelectionVector *false_sel) { -@@ -417,14 +419,22 @@ public: - ldata, rdata, sel, count, combined_mask, true_sel, false_sel); - } - } -- -+#ifndef DUCKDB_SMALLER_BINARY - template -+#else -+ template -+#endif - static inline idx_t - SelectGenericLoop(const LEFT_TYPE *__restrict ldata, const RIGHT_TYPE *__restrict rdata, - const SelectionVector *__restrict lsel, const SelectionVector *__restrict rsel, - const SelectionVector *__restrict result_sel, idx_t count, ValidityMask &lvalidity, - ValidityMask &rvalidity, SelectionVector *true_sel, SelectionVector *false_sel) { - idx_t true_count = 0, false_count = 0; -+#ifdef DUCKDB_SMALLER_BINARY -+ const bool HAS_TRUE_SEL = true_sel; -+ const bool HAS_FALSE_SEL = false_sel; -+ const bool NO_NULL = false; -+#endif - for (idx_t i = 0; i < count; i++) { - auto result_idx = result_sel->get_index(i); - auto lindex = lsel->get_index(i); -@@ -452,6 +462,7 @@ public: - const SelectionVector *__restrict lsel, const SelectionVector *__restrict rsel, - const SelectionVector *__restrict result_sel, idx_t count, ValidityMask &lvalidity, - ValidityMask &rvalidity, SelectionVector *true_sel, SelectionVector *false_sel) { -+#ifndef DUCKDB_SMALLER_BINARY - if (true_sel && false_sel) { - return SelectGenericLoop( - ldata, rdata, lsel, rsel, result_sel, count, lvalidity, rvalidity, true_sel, false_sel); -@@ -463,6 +474,10 @@ public: - return SelectGenericLoop( - ldata, rdata, lsel, rsel, result_sel, count, lvalidity, rvalidity, true_sel, false_sel); - } -+#else -+ return SelectGenericLoop(ldata, rdata, lsel, rsel, result_sel, count, lvalidity, -+ rvalidity, true_sel, false_sel); -+#endif - } - - template -@@ -471,10 +486,13 @@ public: - const SelectionVector *__restrict lsel, const SelectionVector *__restrict rsel, - const SelectionVector *__restrict result_sel, idx_t count, ValidityMask &lvalidity, - ValidityMask &rvalidity, SelectionVector *true_sel, SelectionVector *false_sel) { -+#ifndef DUCKDB_SMALLER_BINARY - if (!lvalidity.AllValid() || !rvalidity.AllValid()) { - return SelectGenericLoopSelSwitch( - ldata, rdata, lsel, rsel, result_sel, count, lvalidity, rvalidity, true_sel, false_sel); -- } else { -+ } else -+#endif -+ { - return SelectGenericLoopSelSwitch( - ldata, rdata, lsel, rsel, result_sel, count, lvalidity, rvalidity, true_sel, false_sel); - } -@@ -502,6 +520,7 @@ public: - if (left.GetVectorType() == VectorType::CONSTANT_VECTOR && - right.GetVectorType() == VectorType::CONSTANT_VECTOR) { - return SelectConstant(left, right, sel, count, true_sel, false_sel); -+#ifndef DUCKDB_SMALLER_BINARY - } else if (left.GetVectorType() == VectorType::CONSTANT_VECTOR && - right.GetVectorType() == VectorType::FLAT_VECTOR) { - return SelectFlat(left, right, sel, count, true_sel, false_sel); -@@ -511,10 +530,12 @@ public: - } else if (left.GetVectorType() == VectorType::FLAT_VECTOR && - right.GetVectorType() == VectorType::FLAT_VECTOR) { - return SelectFlat(left, right, sel, count, true_sel, false_sel); -+#endif - } else { - return SelectGeneric(left, right, sel, count, true_sel, false_sel); - } - } -+#undef DUCKDB_SMALLER_BINARY - }; - - } // namespace duckdb From fb54de0052e79afd7988e8130c7e4663ef6fe871 Mon Sep 17 00:00:00 2001 From: Makito Date: Sat, 22 Feb 2025 20:17:37 +0800 Subject: [PATCH 10/13] fix: respect access mode and file open flags --- .../duckdb-wasm/src/bindings/bindings_base.ts | 10 +- .../src/bindings/bindings_interface.ts | 6 +- packages/duckdb-wasm/src/bindings/runtime.ts | 15 ++- .../src/bindings/runtime_browser.ts | 23 ++-- .../duckdb-wasm/src/bindings/runtime_node.ts | 32 ++++-- .../src/parallel/worker_dispatcher.ts | 3 +- packages/duckdb-wasm/test/index_node.ts | 2 + packages/duckdb-wasm/test/nodefs.test.ts | 71 ++++++++++++ packages/duckdb-wasm/test/opfs.test.ts | 108 +++++++++++++++--- patches/duckdb/fix_load_database.patch | 31 +++-- 10 files changed, 249 insertions(+), 52 deletions(-) create mode 100644 packages/duckdb-wasm/test/nodefs.test.ts diff --git a/packages/duckdb-wasm/src/bindings/bindings_base.ts b/packages/duckdb-wasm/src/bindings/bindings_base.ts index f395bdb10..24f0cd615 100644 --- a/packages/duckdb-wasm/src/bindings/bindings_base.ts +++ b/packages/duckdb-wasm/src/bindings/bindings_base.ts @@ -1,5 +1,5 @@ import { DuckDBModule, PThread } from './duckdb_module'; -import { DuckDBConfig } from './config'; +import { DuckDBAccessMode, DuckDBConfig } from './config'; import { Logger } from '../log'; import { InstantiationProgress } from './progress'; import { DuckDBBindings } from './bindings_interface'; @@ -469,9 +469,9 @@ export abstract class DuckDBBindingsBase implements DuckDBBindings { } dropResponseBuffers(this.mod); } - public async prepareFileHandle(fileName: string, protocol: DuckDBDataProtocol): Promise { + public async prepareFileHandle(fileName: string, protocol: DuckDBDataProtocol, accessMode?: DuckDBAccessMode): Promise { if (protocol === DuckDBDataProtocol.BROWSER_FSACCESS && this._runtime.prepareFileHandles) { - const list = await this._runtime.prepareFileHandles([fileName], DuckDBDataProtocol.BROWSER_FSACCESS); + const list = await this._runtime.prepareFileHandles([fileName], DuckDBDataProtocol.BROWSER_FSACCESS, accessMode); for (const item of list) { const { handle, path: filePath, fromCached } = item; if (!fromCached && handle.getSize()) { @@ -483,9 +483,9 @@ export abstract class DuckDBBindingsBase implements DuckDBBindings { throw new Error(`prepareFileHandle: unsupported protocol ${protocol}`); } /** Prepare a file handle that could only be acquired aschronously */ - public async prepareDBFileHandle(path: string, protocol: DuckDBDataProtocol): Promise { + public async prepareDBFileHandle(path: string, protocol: DuckDBDataProtocol, accessMode?: DuckDBAccessMode): Promise { if (protocol === DuckDBDataProtocol.BROWSER_FSACCESS && this._runtime.prepareDBFileHandle) { - const list = await this._runtime.prepareDBFileHandle(path, DuckDBDataProtocol.BROWSER_FSACCESS); + const list = await this._runtime.prepareDBFileHandle(path, DuckDBDataProtocol.BROWSER_FSACCESS, accessMode); for (const item of list) { const { handle, path: filePath, fromCached } = item; if (!fromCached && handle.getSize()) { diff --git a/packages/duckdb-wasm/src/bindings/bindings_interface.ts b/packages/duckdb-wasm/src/bindings/bindings_interface.ts index 271a42ef9..3a5e3049d 100644 --- a/packages/duckdb-wasm/src/bindings/bindings_interface.ts +++ b/packages/duckdb-wasm/src/bindings/bindings_interface.ts @@ -1,4 +1,4 @@ -import { DuckDBConfig, DuckDBConnection, DuckDBDataProtocol, FileStatistics, InstantiationProgress } from '.'; +import { DuckDBAccessMode, DuckDBConfig, DuckDBConnection, DuckDBDataProtocol, FileStatistics, InstantiationProgress } from '.'; import { CSVInsertOptions, JSONInsertOptions, ArrowInsertOptions } from './insert_options'; import { ScriptTokens } from './tokens'; import { WebFile } from './web_file'; @@ -54,8 +54,8 @@ export interface DuckDBBindings { protocol: DuckDBDataProtocol, directIO: boolean, ): Promise; - prepareFileHandle(path: string, protocol: DuckDBDataProtocol): Promise; - prepareDBFileHandle(path: string, protocol: DuckDBDataProtocol): Promise; + prepareFileHandle(path: string, protocol: DuckDBDataProtocol, accessMode?: DuckDBAccessMode): Promise; + prepareDBFileHandle(path: string, protocol: DuckDBDataProtocol, accessMode?: DuckDBAccessMode): Promise; globFiles(path: string): WebFile[]; dropFile(name: string): void; dropFiles(): void; diff --git a/packages/duckdb-wasm/src/bindings/runtime.ts b/packages/duckdb-wasm/src/bindings/runtime.ts index b720e6877..c611ca4d9 100644 --- a/packages/duckdb-wasm/src/bindings/runtime.ts +++ b/packages/duckdb-wasm/src/bindings/runtime.ts @@ -1,3 +1,4 @@ +import { DuckDBAccessMode } from './config'; import { DuckDBModule } from './duckdb_module'; import { UDFFunction } from './udf_function'; import * as udf_rt from './udf_runtime'; @@ -58,6 +59,16 @@ export enum FileFlags { FILE_FLAGS_FILE_CREATE_NEW = 1 << 4, //! Open file in append mode FILE_FLAGS_APPEND = 1 << 5, + //! Open file with restrictive permissions (600 on linux/mac) can only be used when creating, throws if file exists + FILE_FLAGS_PRIVATE = 1 << 6, + //! Return NULL if the file does not exist instead of throwing an error + FILE_FLAGS_NULL_IF_NOT_EXISTS = 1 << 7, + //! Multiple threads may perform reads and writes in parallel + FILE_FLAGS_PARALLEL_ACCESS = 1 << 8, + //! Ensure that this call creates the file, throw is file exists + FILE_FLAGS_EXCLUSIVE_CREATE = 1 << 9, + //! Return NULL if the file exist instead of throwing an error + FILE_FLAGS_NULL_IF_EXISTS = 1 << 10, } /** Configuration for the AWS S3 Filesystem */ @@ -158,8 +169,8 @@ export interface DuckDBRuntime { // Prepare a file handle that could only be acquired aschronously prepareFileHandle?: (path: string, protocol: DuckDBDataProtocol) => Promise; - prepareFileHandles?: (path: string[], protocol: DuckDBDataProtocol) => Promise; - prepareDBFileHandle?: (path: string, protocol: DuckDBDataProtocol) => Promise; + prepareFileHandles?: (path: string[], protocol: DuckDBDataProtocol, accessMode?: DuckDBAccessMode) => Promise; + prepareDBFileHandle?: (path: string, protocol: DuckDBDataProtocol, accessMode?: DuckDBAccessMode) => Promise; // Call a scalar UDF function callScalarUDF( diff --git a/packages/duckdb-wasm/src/bindings/runtime_browser.ts b/packages/duckdb-wasm/src/bindings/runtime_browser.ts index 0b4ebedca..cd187f64f 100644 --- a/packages/duckdb-wasm/src/bindings/runtime_browser.ts +++ b/packages/duckdb-wasm/src/bindings/runtime_browser.ts @@ -15,6 +15,7 @@ import { } from './runtime'; import { DuckDBModule } from './duckdb_module'; import * as udf from './udf_runtime'; +import { DuckDBAccessMode } from './config'; const OPFS_PREFIX_LEN = 'opfs://'.length; const PATH_SEP_REGEX = /\/|\\/; @@ -110,8 +111,11 @@ export const BROWSER_RUNTIME: DuckDBRuntime & { BROWSER_RUNTIME._opfsRoot = await navigator.storage.getDirectory(); } }, - /** Prepare a file handle that could only be acquired aschronously */ - async prepareFileHandles(filePaths: string[], protocol: DuckDBDataProtocol): Promise { + /** Prepare a file handle that could only be acquired asynchronously */ + async prepareFileHandles(filePaths: string[], protocol: DuckDBDataProtocol, accessMode?: DuckDBAccessMode): Promise { + // DuckDBAccessMode.UNDEFINED will be treated as READ_WRITE + // See: https://github.com/duckdb/duckdb/blob/5f5512b827df6397afd31daedb4bbdee76520019/src/main/database.cpp#L442-L444 + const isReadWrite = !accessMode || accessMode === DuckDBAccessMode.READ_WRITE; if (protocol === DuckDBDataProtocol.BROWSER_FSACCESS) { await BROWSER_RUNTIME.assignOPFSRoot(); const prepare = async (path: string): Promise => { @@ -135,13 +139,16 @@ export const BROWSER_RUNTIME: DuckDBRuntime & { } // mkdir -p for (const folder of folders) { - dirHandle = await dirHandle.getDirectoryHandle(folder, { create: true }); + dirHandle = await dirHandle.getDirectoryHandle(folder, { create: isReadWrite }); } } const fileHandle = await dirHandle.getFileHandle(fileName, { create: false }).catch(e => { if (e?.name === 'NotFoundError') { - console.debug(`File ${path} does not exists yet, creating...`); - return dirHandle.getFileHandle(fileName, { create: true }); + if (isReadWrite) { + console.debug(`File ${path} does not exists yet, creating...`); + return dirHandle.getFileHandle(fileName, { create: true }); + } + console.debug(`File ${path} does not exists, aborting as we are in read-only mode`); } throw e; }); @@ -166,11 +173,11 @@ export const BROWSER_RUNTIME: DuckDBRuntime & { } throw new Error(`Unsupported protocol ${protocol} for paths ${filePaths} with protocol ${protocol}`); }, - /** Prepare a file handle that could only be acquired aschronously */ - async prepareDBFileHandle(dbPath: string, protocol: DuckDBDataProtocol): Promise { + /** Prepare a file handle that could only be acquired asynchronously */ + async prepareDBFileHandle(dbPath: string, protocol: DuckDBDataProtocol, accessMode?: DuckDBAccessMode): Promise { if (protocol === DuckDBDataProtocol.BROWSER_FSACCESS && this.prepareFileHandles) { const filePaths = [dbPath, `${dbPath}.wal`]; - return this.prepareFileHandles(filePaths, protocol); + return this.prepareFileHandles(filePaths, protocol, accessMode); } throw new Error(`Unsupported protocol ${protocol} for path ${dbPath} with protocol ${protocol}`); }, diff --git a/packages/duckdb-wasm/src/bindings/runtime_node.ts b/packages/duckdb-wasm/src/bindings/runtime_node.ts index 2f92368bc..38cdb62af 100644 --- a/packages/duckdb-wasm/src/bindings/runtime_node.ts +++ b/packages/duckdb-wasm/src/bindings/runtime_node.ts @@ -74,16 +74,32 @@ export const NODE_RUNTIME: DuckDBRuntime & { switch (file?.dataProtocol) { // Native file case DuckDBDataProtocol.NODE_FS: { + let openFlags = fs.constants.O_RDONLY; + if (flags & FileFlags.FILE_FLAGS_WRITE) { + openFlags = fs.constants.O_RDWR; + } + if (flags & FileFlags.FILE_FLAGS_FILE_CREATE) { + openFlags |= fs.constants.O_CREAT; + } else if (flags & FileFlags.FILE_FLAGS_FILE_CREATE_NEW) { + openFlags |= fs.constants.O_TRUNC; + } let fd = NODE_RUNTIME._files?.get(file.dataUrl!); - if (fd === null || fd === undefined) { - fd = fs.openSync( - file.dataUrl!, - fs.constants.O_CREAT | fs.constants.O_RDWR, - fs.constants.S_IRUSR | fs.constants.S_IWUSR, - ); - NODE_RUNTIME._filesById?.set(file.fileId!, fd); + let fileSize = 0; + try { + if (fd === null || fd === undefined) { + fd = fs.openSync(file.dataUrl!, openFlags, fs.constants.S_IRUSR | fs.constants.S_IWUSR); + NODE_RUNTIME._filesById?.set(file.fileId!, fd); + } + fileSize = fs.fstatSync(fd).size; + } + catch (e: any) { + if (e.code === 'ENOENT' && (flags & FileFlags.FILE_FLAGS_NULL_IF_NOT_EXISTS)) { + // No-op because we intend to ignore ENOENT while the file does not exist + return 0; // nullptr + } else { + throw e; + } } - const fileSize = fs.fstatSync(fd).size; const result = mod._malloc(2 * 8); mod.HEAPF64[(result >> 3) + 0] = +fileSize; mod.HEAPF64[(result >> 3) + 1] = 0; diff --git a/packages/duckdb-wasm/src/parallel/worker_dispatcher.ts b/packages/duckdb-wasm/src/parallel/worker_dispatcher.ts index 3a5a8f295..402b83e45 100644 --- a/packages/duckdb-wasm/src/parallel/worker_dispatcher.ts +++ b/packages/duckdb-wasm/src/parallel/worker_dispatcher.ts @@ -136,8 +136,9 @@ export abstract class AsyncDuckDBDispatcher implements Logger { case WorkerRequestType.OPEN: { const path = request.data.path; + const accessMode = request.data.accessMode; if (path?.startsWith('opfs://')) { - await this._bindings.prepareDBFileHandle(path, DuckDBDataProtocol.BROWSER_FSACCESS); + await this._bindings.prepareDBFileHandle(path, DuckDBDataProtocol.BROWSER_FSACCESS, accessMode); request.data.useDirectIO = true; } this._bindings.open(request.data); diff --git a/packages/duckdb-wasm/test/index_node.ts b/packages/duckdb-wasm/test/index_node.ts index d170ac88f..d8404b603 100644 --- a/packages/duckdb-wasm/test/index_node.ts +++ b/packages/duckdb-wasm/test/index_node.ts @@ -69,6 +69,7 @@ import { testAllTypes, testAllTypesAsync } from './all_types.test'; import { testBindings, testAsyncBindings } from './bindings.test'; import { testBatchStream } from './batch_stream.test'; import { testFilesystem } from './filesystem.test'; +import { testNodeFS } from './nodefs.test'; import { testAsyncBatchStream } from './batch_stream_async.test'; import { testArrowInsert, testArrowInsertAsync } from './insert_arrow.test'; import { testJSONInsert, testJSONInsertAsync } from './insert_json.test'; @@ -92,6 +93,7 @@ testAsyncBindings(() => adb!, dataDir, duckdb.DuckDBDataProtocol.NODE_FS); testBatchStream(() => db!); testAsyncBatchStream(() => adb!); testFilesystem(() => adb!, resolveData, dataDir, duckdb.DuckDBDataProtocol.NODE_FS); +testNodeFS(() => adb!); testArrowInsert(() => db!); testArrowInsertAsync(() => adb!); testJSONInsert(() => db!); diff --git a/packages/duckdb-wasm/test/nodefs.test.ts b/packages/duckdb-wasm/test/nodefs.test.ts new file mode 100644 index 000000000..152c841b2 --- /dev/null +++ b/packages/duckdb-wasm/test/nodefs.test.ts @@ -0,0 +1,71 @@ +import * as duckdb from '../src/'; +import { tmpdir } from 'os'; +import { randomUUID } from 'crypto'; +import path from 'path'; +import { unlink } from 'fs/promises'; + +export function testNodeFS(db: () => duckdb.AsyncDuckDB): void { + const files: string[] = []; + + afterAll(async () => { + await Promise.all(files.map(file => unlink(file).catch(() => {}))); + await db().flushFiles(); + await db().dropFiles(); + }); + + describe('Node FS', () => { + it('Should not create an empty DB file in read-only mode for non-existent path', async () => { + const tmp = tmpdir(); + const filename = `duckdb_test_${randomUUID().replace(/-/g, '')}`; + files.push(path.join(tmp, filename)); + + await expectAsync( + db().open({ + path: path.join(tmp, filename), + accessMode: duckdb.DuckDBAccessMode.READ_ONLY, + }), + ).toBeRejectedWithError(/database does not exist/); + }); + + it('Should create DB file in read-write mode for non-existent path', async () => { + const tmp = tmpdir(); + const filename = `duckdb_test_${randomUUID().replace(/-/g, '')}`; + files.push(path.join(tmp, filename)); + + await expectAsync( + db().open({ + path: path.join(tmp, filename), + accessMode: duckdb.DuckDBAccessMode.READ_WRITE, + }), + ).toBeResolved(); + }); + + it('Should create an empty DB file in read-only mode for non-existent path with direct I/O', async () => { + const tmp = tmpdir(); + const filename = `duckdb_test_${randomUUID().replace(/-/g, '')}`; + files.push(path.join(tmp, filename)); + + await expectAsync( + db().open({ + path: path.join(tmp, filename), + accessMode: duckdb.DuckDBAccessMode.READ_ONLY, + useDirectIO: true, + }), + ).toBeRejectedWithError(/database does not exist/); + }); + + it('Should create DB file in read-write mode for non-existent path with direct I/O', async () => { + const tmp = tmpdir(); + const filename = `duckdb_test_${randomUUID().replace(/-/g, '')}`; + files.push(path.join(tmp, filename)); + + await expectAsync( + db().open({ + path: path.join(tmp, filename), + accessMode: duckdb.DuckDBAccessMode.READ_WRITE, + useDirectIO: true, + }), + ).toBeResolved(); + }); + }); +} diff --git a/packages/duckdb-wasm/test/opfs.test.ts b/packages/duckdb-wasm/test/opfs.test.ts index eaf1a0fcc..e3087cd26 100644 --- a/packages/duckdb-wasm/test/opfs.test.ts +++ b/packages/duckdb-wasm/test/opfs.test.ts @@ -278,30 +278,106 @@ export function testOPFS(baseDir: string, bundle: () => duckdb.DuckDBBundle): vo }); }); - async function removeFiles() { - const opfsRoot = await navigator.storage.getDirectory(); - await opfsRoot.removeEntry('test.db').catch(() => { - }); - await opfsRoot.removeEntry('test.db.wal').catch(() => { - }); - await opfsRoot.removeEntry('test.csv').catch(() => { - }); - await opfsRoot.removeEntry('test1.csv').catch(() => { + describe('Open database in OPFS', () => { + it('should not open a non-existent DB file in read-only', async () => { + const logger = new duckdb.ConsoleLogger(LogLevel.ERROR); + const worker = new Worker(bundle().mainWorker!); + const db_ = new duckdb.AsyncDuckDB(logger, worker); + await db_.instantiate(bundle().mainModule, bundle().pthreadWorker); + + await expectAsync(db_.open({ + path: 'opfs://non_existent.db', + accessMode: duckdb.DuckDBAccessMode.READ_ONLY, + })).toBeRejectedWithError(Error, /file or directory could not be found/); + + await db_.terminate(); + await worker.terminate(); + + // Files should not be found with DuckDBAccessMode.READ_ONLY + const opfsRoot = await navigator.storage.getDirectory(); + await expectAsync(opfsRoot.getFileHandle('non_existent.db', { create: false })) + .toBeRejectedWithError(Error, /file or directory could not be found/); }); - await opfsRoot.removeEntry('test2.csv').catch(() => { + + it('should not open a non-existent DB file and mkdir in read-only', async () => { + const logger = new duckdb.ConsoleLogger(LogLevel.ERROR); + const worker = new Worker(bundle().mainWorker!); + const db_ = new duckdb.AsyncDuckDB(logger, worker); + await db_.instantiate(bundle().mainModule, bundle().pthreadWorker); + + await expectAsync(db_.open({ + path: 'opfs://duckdb_test/path/to/non_existent.db', + accessMode: duckdb.DuckDBAccessMode.READ_ONLY, + })).toBeRejectedWithError(Error, /file or directory could not be found/); + + await db_.terminate(); + await worker.terminate(); }); - await opfsRoot.removeEntry('test3.csv').catch(() => { + + it('should open a non-existent DB file and mkdir in read-write', async () => { + const logger = new duckdb.ConsoleLogger(LogLevel.ERROR); + const worker = new Worker(bundle().mainWorker!); + const db_ = new duckdb.AsyncDuckDB(logger, worker); + await db_.instantiate(bundle().mainModule, bundle().pthreadWorker); + + await expectAsync(db_.open({ + path: 'opfs://duckdb_test/path/to/duck.db', + accessMode: duckdb.DuckDBAccessMode.READ_WRITE, + })).toBeResolved(); + + await db_.terminate(); + await worker.terminate(); }); - await opfsRoot.removeEntry('test.parquet').catch(() => { + + it('should open a non-existent DB file in read-write and create files', async () => { + const logger = new duckdb.ConsoleLogger(LogLevel.ERROR); + const worker = new Worker(bundle().mainWorker!); + const db_ = new duckdb.AsyncDuckDB(logger, worker); + await db_.instantiate(bundle().mainModule, bundle().pthreadWorker); + + const opfsRoot = await navigator.storage.getDirectory(); + + // Ensure files do not exist + await expectAsync(opfsRoot.getFileHandle('non_existent_2.db', { create: false })) + .toBeRejectedWithError(Error, /file or directory could not be found/); + await expectAsync(opfsRoot.getFileHandle('non_existent_2.db.wal', { create: false })) + .toBeRejectedWithError(Error, /file or directory could not be found/); + + await expectAsync(db_.open({ + path: 'opfs://non_existent_2.db', + accessMode: duckdb.DuckDBAccessMode.READ_WRITE, + })).toBeResolved(); + + await db_.terminate(); + await worker.terminate(); + + // Files should be found with DuckDBAccessMode.READ_WRITE + await expectAsync(opfsRoot.getFileHandle('non_existent_2.db', { create: false })).toBeResolved(); + await expectAsync(opfsRoot.getFileHandle('non_existent_2.db.wal', { create: false })).toBeResolved(); }); + }) + + async function removeFiles() { + const opfsRoot = await navigator.storage.getDirectory(); + await opfsRoot.removeEntry('test.db').catch(() => {}); + await opfsRoot.removeEntry('test.db.wal').catch(() => {}); + await opfsRoot.removeEntry('test.csv').catch(() => {}); + await opfsRoot.removeEntry('test1.csv').catch(() => {}); + await opfsRoot.removeEntry('test2.csv').catch(() => {}); + await opfsRoot.removeEntry('test3.csv').catch(() => {}); + await opfsRoot.removeEntry('test.parquet').catch(() => {}); try { const datadir = await opfsRoot.getDirectoryHandle('datadir'); - datadir.removeEntry('test.parquet').catch(() => { - }); + datadir.removeEntry('test.parquet').catch(() => {}); } catch (e) { // } - await opfsRoot.removeEntry('datadir').catch(() => { - }); + await opfsRoot.removeEntry('datadir').catch(() => {}); + // In case of failure caused leftovers + await opfsRoot.removeEntry('non_existent.db').catch(() => {}); + await opfsRoot.removeEntry('non_existent.db.wal').catch(() => {}); + await opfsRoot.removeEntry('non_existent_2.db').catch(() => {}); + await opfsRoot.removeEntry('non_existent_2.db.wal').catch(() => {}); + await opfsRoot.removeEntry('duckdb_test', { recursive: true }).catch(() => {}); } } diff --git a/patches/duckdb/fix_load_database.patch b/patches/duckdb/fix_load_database.patch index 5399bfc4d..7c5c6b5bc 100644 --- a/patches/duckdb/fix_load_database.patch +++ b/patches/duckdb/fix_load_database.patch @@ -1,20 +1,33 @@ diff --git a/src/storage/storage_manager.cpp b/src/storage/storage_manager.cpp -index cb6c654e5f..a6b2af3b85 100644 +index cb6c654e5f..4a8bb03de0 100644 --- a/src/storage/storage_manager.cpp +++ b/src/storage/storage_manager.cpp -@@ -160,9 +160,12 @@ void SingleFileStorageManager::LoadDatabase(StorageOptions storage_options) { +@@ -160,11 +160,23 @@ void SingleFileStorageManager::LoadDatabase(StorageOptions storage_options) { row_group_size, STANDARD_VECTOR_SIZE); } } - // Check if the database file already exists. - // Note: a file can also exist if there was a ROLLBACK on a previous transaction creating that file. - if (!read_only && !fs.FileExists(path)) { -+ auto db_file_handle = fs.OpenFile(path, FileFlags::FILE_FLAGS_READ | FileFlags::FILE_FLAGS_NULL_IF_NOT_EXISTS); -+ bool is_empty_file = db_file_handle->GetFileSize() == 0; -+ db_file_handle.reset(); +- // file does not exist and we are in read-write mode +- // create a new file + -+ // first check if the database exists -+ if (!read_only && ( !fs.FileExists(path) || ( options.use_direct_io && is_empty_file )) ) { - // file does not exist and we are in read-write mode - // create a new file ++ bool create_or_trunc = false; ++ if (!read_only) { ++ if (fs.FileExists(path)) { ++ auto db_file_handle = fs.OpenFile(path, FileFlags::FILE_FLAGS_READ | FileFlags::FILE_FLAGS_NULL_IF_NOT_EXISTS); ++ create_or_trunc = db_file_handle != nullptr ? db_file_handle->GetFileSize() == 0 : true; ++ if (db_file_handle != nullptr) { ++ db_file_handle.reset(); ++ } ++ } else { ++ create_or_trunc = true; ++ } ++ } ++ ++ if (create_or_trunc) { ++ // We are in read-write mode and the file does not exist or is empty ++ // Create a new database file or truncate the existing one + // check if a WAL file already exists + auto wal_path = GetWALPath(); From 3ad91aa37fb6fb75ff5fdf6c82d699a69db7ee95 Mon Sep 17 00:00:00 2001 From: arkw Date: Fri, 28 Feb 2025 14:00:19 +0900 Subject: [PATCH 11/13] add lib.webworker.d.ts for mode of createSyncAccessHandle --- packages/duckdb-wasm/types/lib.webworker.d.ts | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 packages/duckdb-wasm/types/lib.webworker.d.ts diff --git a/packages/duckdb-wasm/types/lib.webworker.d.ts b/packages/duckdb-wasm/types/lib.webworker.d.ts new file mode 100644 index 000000000..e49a8ae06 --- /dev/null +++ b/packages/duckdb-wasm/types/lib.webworker.d.ts @@ -0,0 +1,3 @@ +interface FileSystemFileHandle extends FileSystemHandle { + createSyncAccessHandle(mode?: 'read' | 'readwrite' | 'readwrite-unsafe'): Promise; +} \ No newline at end of file From b24a1dc7b03e12debc7f4d970e000a949a3ba70a Mon Sep 17 00:00:00 2001 From: arkw Date: Fri, 28 Feb 2025 14:25:23 +0900 Subject: [PATCH 12/13] support createSyncAccessHandle mode --- .../duckdb-wasm/src/bindings/bindings_base.ts | 16 +++---- .../src/bindings/bindings_interface.ts | 6 +-- packages/duckdb-wasm/src/bindings/config.ts | 1 + packages/duckdb-wasm/src/bindings/runtime.ts | 4 +- .../src/bindings/runtime_browser.ts | 27 ++++++++---- .../src/parallel/async_bindings.ts | 12 +++--- .../src/parallel/worker_dispatcher.ts | 9 ++-- .../src/parallel/worker_request.ts | 6 +-- packages/duckdb-wasm/test/filesystem.test.ts | 4 ++ packages/duckdb-wasm/test/index_browser.ts | 42 +++++++++---------- packages/duckdb-wasm/test/opfs.test.ts | 10 ++--- packages/duckdb-wasm/types/lib.webworker.d.ts | 14 ++++++- 12 files changed, 87 insertions(+), 64 deletions(-) diff --git a/packages/duckdb-wasm/src/bindings/bindings_base.ts b/packages/duckdb-wasm/src/bindings/bindings_base.ts index fb2c061c6..673df278a 100644 --- a/packages/duckdb-wasm/src/bindings/bindings_base.ts +++ b/packages/duckdb-wasm/src/bindings/bindings_base.ts @@ -5,8 +5,8 @@ import { InstantiationProgress } from './progress'; import { DuckDBBindings } from './bindings_interface'; import { DuckDBConnection } from './connection'; import { StatusCode } from '../status'; -import { dropResponseBuffers, DuckDBRuntime, readString, callSRet, copyBuffer, DuckDBDataProtocol } from './runtime'; -import { CSVInsertOptions, JSONInsertOptions, ArrowInsertOptions } from './insert_options'; +import { callSRet, copyBuffer, dropResponseBuffers, DuckDBDataProtocol, DuckDBRuntime, readString } from './runtime'; +import { ArrowInsertOptions, CSVInsertOptions, JSONInsertOptions } from './insert_options'; import { ScriptTokens } from './tokens'; import { FileStatistics } from './file_stats'; import { arrowToSQLField, arrowToSQLType } from '../json_typedef'; @@ -469,9 +469,9 @@ export abstract class DuckDBBindingsBase implements DuckDBBindings { } dropResponseBuffers(this.mod); } - public async prepareFileHandle(fileName: string, protocol: DuckDBDataProtocol, accessMode?: DuckDBAccessMode): Promise { + public async prepareFileHandle(fileName: string, protocol: DuckDBDataProtocol, accessMode: DuckDBAccessMode, multiWindowMode:boolean): Promise { if (protocol === DuckDBDataProtocol.BROWSER_FSACCESS && this._runtime.prepareFileHandles) { - const list = await this._runtime.prepareFileHandles([fileName], DuckDBDataProtocol.BROWSER_FSACCESS, accessMode); + const list = await this._runtime.prepareFileHandles([fileName], DuckDBDataProtocol.BROWSER_FSACCESS, accessMode, multiWindowMode); for (const item of list) { const { handle, path: filePath, fromCached } = item; if (!fromCached && handle.getSize()) { @@ -483,9 +483,9 @@ export abstract class DuckDBBindingsBase implements DuckDBBindings { throw new Error(`prepareFileHandle: unsupported protocol ${protocol}`); } /** Prepare a file handle that could only be acquired aschronously */ - public async prepareDBFileHandle(path: string, protocol: DuckDBDataProtocol, accessMode?: DuckDBAccessMode): Promise { + public async prepareDBFileHandle(path: string, protocol: DuckDBDataProtocol, accessMode: DuckDBAccessMode, multiWindowMode:boolean): Promise { if (protocol === DuckDBDataProtocol.BROWSER_FSACCESS && this._runtime.prepareDBFileHandle) { - const list = await this._runtime.prepareDBFileHandle(path, DuckDBDataProtocol.BROWSER_FSACCESS, accessMode); + const list = await this._runtime.prepareDBFileHandle(path, DuckDBDataProtocol.BROWSER_FSACCESS, accessMode, multiWindowMode); for (const item of list) { const { handle, path: filePath, fromCached } = item; if (!fromCached && handle.getSize()) { @@ -655,9 +655,9 @@ export abstract class DuckDBBindingsBase implements DuckDBBindings { return copy; } /** Enable tracking of file statistics */ - public async registerOPFSFileName(file: string): Promise { + public async registerOPFSFileName(file: string, accesssMode:DuckDBAccessMode = DuckDBAccessMode.READ_WRITE, multiWindowMode:boolean = false): Promise { if (file.startsWith("opfs://")) { - return await this.prepareFileHandle(file, DuckDBDataProtocol.BROWSER_FSACCESS); + return await this.prepareFileHandle(file, DuckDBDataProtocol.BROWSER_FSACCESS, accesssMode, multiWindowMode); } else { throw new Error("Not an OPFS file name: " + file); } diff --git a/packages/duckdb-wasm/src/bindings/bindings_interface.ts b/packages/duckdb-wasm/src/bindings/bindings_interface.ts index a9db907ce..68e9203be 100644 --- a/packages/duckdb-wasm/src/bindings/bindings_interface.ts +++ b/packages/duckdb-wasm/src/bindings/bindings_interface.ts @@ -54,15 +54,15 @@ export interface DuckDBBindings { protocol: DuckDBDataProtocol, directIO: boolean, ): Promise; - prepareFileHandle(path: string, protocol: DuckDBDataProtocol, accessMode?: DuckDBAccessMode): Promise; - prepareDBFileHandle(path: string, protocol: DuckDBDataProtocol, accessMode?: DuckDBAccessMode): Promise; + prepareFileHandle(path: string, protocol: DuckDBDataProtocol, accessMode: DuckDBAccessMode, multiWindowMode:boolean): Promise; + prepareDBFileHandle(path: string, protocol: DuckDBDataProtocol, accessMode: DuckDBAccessMode, multiWindowMode:boolean): Promise; globFiles(path: string): WebFile[]; dropFile(name: string): void; dropFiles(names?: string[]): void; flushFiles(): void; copyFileToPath(name: string, path: string): void; copyFileToBuffer(name: string): Uint8Array; - registerOPFSFileName(file: string): Promise; + registerOPFSFileName(file: string, accessMode: DuckDBAccessMode, multiWindowMode:boolean): Promise; collectFileStatistics(file: string, enable: boolean): void; exportFileStatistics(file: string): FileStatistics; } diff --git a/packages/duckdb-wasm/src/bindings/config.ts b/packages/duckdb-wasm/src/bindings/config.ts index a3013d19e..288fac353 100644 --- a/packages/duckdb-wasm/src/bindings/config.ts +++ b/packages/duckdb-wasm/src/bindings/config.ts @@ -36,6 +36,7 @@ export interface DuckDBOPFSConfig { * - "manual": Files must be manually registered and dropped. */ fileHandling?: "auto" | "manual"; + window?: "single" | "multi"; } export enum DuckDBAccessMode { diff --git a/packages/duckdb-wasm/src/bindings/runtime.ts b/packages/duckdb-wasm/src/bindings/runtime.ts index c611ca4d9..ca79e54b8 100644 --- a/packages/duckdb-wasm/src/bindings/runtime.ts +++ b/packages/duckdb-wasm/src/bindings/runtime.ts @@ -169,8 +169,8 @@ export interface DuckDBRuntime { // Prepare a file handle that could only be acquired aschronously prepareFileHandle?: (path: string, protocol: DuckDBDataProtocol) => Promise; - prepareFileHandles?: (path: string[], protocol: DuckDBDataProtocol, accessMode?: DuckDBAccessMode) => Promise; - prepareDBFileHandle?: (path: string, protocol: DuckDBDataProtocol, accessMode?: DuckDBAccessMode) => Promise; + prepareFileHandles?: (path: string[], protocol: DuckDBDataProtocol, accessMode: DuckDBAccessMode, multiWindowMode:boolean) => Promise; + prepareDBFileHandle?: (path: string, protocol: DuckDBDataProtocol, accessMode: DuckDBAccessMode, multiWindowMode:boolean) => Promise; // Call a scalar UDF function callScalarUDF( diff --git a/packages/duckdb-wasm/src/bindings/runtime_browser.ts b/packages/duckdb-wasm/src/bindings/runtime_browser.ts index cd187f64f..eee05e46f 100644 --- a/packages/duckdb-wasm/src/bindings/runtime_browser.ts +++ b/packages/duckdb-wasm/src/bindings/runtime_browser.ts @@ -142,18 +142,27 @@ export const BROWSER_RUNTIME: DuckDBRuntime & { dirHandle = await dirHandle.getDirectoryHandle(folder, { create: isReadWrite }); } } - const fileHandle = await dirHandle.getFileHandle(fileName, { create: false }).catch(e => { + let fileHandle:FileSystemFileHandle; + try { + fileHandle = await dirHandle.getFileHandle(fileName, { create: false }); + } catch (e:any) { if (e?.name === 'NotFoundError') { if (isReadWrite) { - console.debug(`File ${path} does not exists yet, creating...`); - return dirHandle.getFileHandle(fileName, { create: true }); + console.debug(`File ${ path } does not exists yet, creating...`); + fileHandle = await dirHandle.getFileHandle(fileName, { create: true }); + } else { + console.debug(`File ${ path } does not exists, aborting as we are in read-only mode`); + throw e; } - console.debug(`File ${path} does not exists, aborting as we are in read-only mode`); + } else { + throw e; } - throw e; - }); + } try { - const handle = await fileHandle.createSyncAccessHandle(); + let syncAccessHandleMode:FileSystemSyncAccessHandleMode = isReadWrite ? "readwrite" : "readwrite-unsafe"; + const handle = await fileHandle.createSyncAccessHandle({ + mode : syncAccessHandleMode + }); BROWSER_RUNTIME._preparedHandles[path] = handle; return { path, @@ -174,10 +183,10 @@ export const BROWSER_RUNTIME: DuckDBRuntime & { throw new Error(`Unsupported protocol ${protocol} for paths ${filePaths} with protocol ${protocol}`); }, /** Prepare a file handle that could only be acquired asynchronously */ - async prepareDBFileHandle(dbPath: string, protocol: DuckDBDataProtocol, accessMode?: DuckDBAccessMode): Promise { + async prepareDBFileHandle(dbPath: string, protocol: DuckDBDataProtocol, accessMode: DuckDBAccessMode, multiWindowMode:boolean): Promise { if (protocol === DuckDBDataProtocol.BROWSER_FSACCESS && this.prepareFileHandles) { const filePaths = [dbPath, `${dbPath}.wal`]; - return this.prepareFileHandles(filePaths, protocol, accessMode); + return this.prepareFileHandles(filePaths, protocol, accessMode, multiWindowMode); } throw new Error(`Unsupported protocol ${protocol} for path ${dbPath} with protocol ${protocol}`); }, diff --git a/packages/duckdb-wasm/src/parallel/async_bindings.ts b/packages/duckdb-wasm/src/parallel/async_bindings.ts index e2c9f3056..f1e8bd78f 100644 --- a/packages/duckdb-wasm/src/parallel/async_bindings.ts +++ b/packages/duckdb-wasm/src/parallel/async_bindings.ts @@ -13,7 +13,7 @@ import { AsyncDuckDBConnection } from './async_connection'; import { CSVInsertOptions, JSONInsertOptions, ArrowInsertOptions } from '../bindings/insert_options'; import { ScriptTokens } from '../bindings/tokens'; import { FileStatistics } from '../bindings/file_stats'; -import { DuckDBConfig } from '../bindings/config'; +import { DuckDBAccessMode, DuckDBConfig } from '../bindings/config'; import { InstantiationProgress } from '../bindings/progress'; import { arrowToSQLField } from '../json_typedef'; import { WebFile } from '../bindings/web_file'; @@ -592,10 +592,10 @@ export class AsyncDuckDB implements AsyncDuckDBBindings { } /** Enable file statistics */ - public async registerOPFSFileName(name: string): Promise { - const task = new WorkerTask( + public async registerOPFSFileName(name: string, accessMode?:DuckDBAccessMode, multiWindowMode?:boolean): Promise { + const task = new WorkerTask( WorkerRequestType.REGISTER_OPFS_FILE_NAME, - [name], + [name, accessMode ?? DuckDBAccessMode.READ_ONLY, multiWindowMode ?? false], ); await this.postTask(task, []); } @@ -704,8 +704,8 @@ export class AsyncDuckDB implements AsyncDuckDBBindings { const result: string[] = []; for (const file of files) { try { - await this.registerOPFSFileName(file); - result.push(file); + await this.registerOPFSFileName( file, this.config.accessMode ?? DuckDBAccessMode.READ_WRITE, this.config.opfs?.window == "multi"); + result.push( file ); } catch (e) { console.error(e); throw new Error("File Not found:" + file); diff --git a/packages/duckdb-wasm/src/parallel/worker_dispatcher.ts b/packages/duckdb-wasm/src/parallel/worker_dispatcher.ts index b8f00c2cf..4965173cf 100644 --- a/packages/duckdb-wasm/src/parallel/worker_dispatcher.ts +++ b/packages/duckdb-wasm/src/parallel/worker_dispatcher.ts @@ -1,4 +1,4 @@ -import { DuckDBBindings, DuckDBDataProtocol } from '../bindings'; +import { DuckDBAccessMode, DuckDBBindings, DuckDBDataProtocol } from '../bindings'; import { WorkerResponseVariant, WorkerRequestVariant, WorkerRequestType, WorkerResponseType } from './worker_request'; import { Logger, LogEntryVariant } from '../log'; import { InstantiationProgress } from '../bindings/progress'; @@ -136,9 +136,10 @@ export abstract class AsyncDuckDBDispatcher implements Logger { case WorkerRequestType.OPEN: { const path = request.data.path; - const accessMode = request.data.accessMode; + const accessMode = request.data.accessMode if (path?.startsWith('opfs://')) { - await this._bindings.prepareDBFileHandle(path, DuckDBDataProtocol.BROWSER_FSACCESS, accessMode); + const multiWindowMode = request.data.opfs?.window == "multi"; + await this._bindings.prepareDBFileHandle(path, DuckDBDataProtocol.BROWSER_FSACCESS, accessMode ?? DuckDBAccessMode.READ_ONLY, multiWindowMode); request.data.useDirectIO = true; } this._bindings.open(request.data); @@ -362,7 +363,7 @@ export abstract class AsyncDuckDBDispatcher implements Logger { break; case WorkerRequestType.REGISTER_OPFS_FILE_NAME: - await this._bindings.registerOPFSFileName(request.data[0]); + await this._bindings.registerOPFSFileName(request.data[0], request.data[1], request.data[2]); this.sendOK(request); break; diff --git a/packages/duckdb-wasm/src/parallel/worker_request.ts b/packages/duckdb-wasm/src/parallel/worker_request.ts index d92d13dab..1d55d3be5 100644 --- a/packages/duckdb-wasm/src/parallel/worker_request.ts +++ b/packages/duckdb-wasm/src/parallel/worker_request.ts @@ -5,7 +5,7 @@ import { FileStatistics } from '../bindings/file_stats'; import { DuckDBConfig } from '../bindings/config'; import { WebFile } from '../bindings/web_file'; import { InstantiationProgress } from '../bindings/progress'; -import { DuckDBDataProtocol } from '../bindings'; +import { DuckDBDataProtocol, DuckDBAccessMode } from '../bindings'; export type ConnectionID = number; export type StatementID = number; @@ -109,7 +109,7 @@ export type WorkerRequestVariant = | WorkerRequest | WorkerRequest | WorkerRequest - | WorkerRequest + | WorkerRequest | WorkerRequest | WorkerRequest | WorkerRequest @@ -168,7 +168,7 @@ export type WorkerResponseVariant = export type WorkerTaskVariant = | WorkerTask - | WorkerTask + | WorkerTask | WorkerTask | WorkerTask | WorkerTask diff --git a/packages/duckdb-wasm/test/filesystem.test.ts b/packages/duckdb-wasm/test/filesystem.test.ts index 8cba9ae27..ce8089852 100644 --- a/packages/duckdb-wasm/test/filesystem.test.ts +++ b/packages/duckdb-wasm/test/filesystem.test.ts @@ -237,6 +237,8 @@ export function testFilesystem( //); //expect(schema_script.trim()).toEqual(`CREATE TABLE foo(v BIGINT);`); //expect(csv_buffer.trim()).toEqual(`1\n2\n3\n4\n5`); + + await conn.query('DROP TABLE foo'); }); it('Generate Series as Parquet', async () => { @@ -264,6 +266,8 @@ export function testFilesystem( expect(content.nullCount).toEqual(0); expect(content.numRows).toEqual(5); expect(content.getChildAt(0)?.toArray()).toEqual(new Int32Array([1, 2, 3, 4, 5])); + + await conn.query('DROP TABLE foo'); }); }); diff --git a/packages/duckdb-wasm/test/index_browser.ts b/packages/duckdb-wasm/test/index_browser.ts index 1588c4fdd..e271efe6a 100644 --- a/packages/duckdb-wasm/test/index_browser.ts +++ b/packages/duckdb-wasm/test/index_browser.ts @@ -115,28 +115,28 @@ import { longQueries } from './long_queries.test'; const baseURL = window.location.origin; const dataURL = `${baseURL}/data`; -testHTTPFS(() => db!); -testHTTPFSAsync(() => adb!, resolveData, dataURL); -testUDF(() => db!); -longQueries(() => adb!); -testTableNames(() => db!); -testTableNamesAsync(() => adb!); -testRegressionAsync(() => adb!); -testAllTypes(() => db!); -testAllTypesAsync(() => adb!); -testBindings(() => db!, dataURL); -testAsyncBindings(() => adb!, dataURL, duckdb.DuckDBDataProtocol.HTTP); -testBatchStream(() => db!); -testAsyncBatchStream(() => adb!); +// testHTTPFS(() => db!); +// testHTTPFSAsync(() => adb!, resolveData, dataURL); +// testUDF(() => db!); +// longQueries(() => adb!); +// testTableNames(() => db!); +// testTableNamesAsync(() => adb!); +// testRegressionAsync(() => adb!); +// testAllTypes(() => db!); +// testAllTypesAsync(() => adb!); +// testBindings(() => db!, dataURL); +// testAsyncBindings(() => adb!, dataURL, duckdb.DuckDBDataProtocol.HTTP); +// testBatchStream(() => db!); +// testAsyncBatchStream(() => adb!); testFilesystem(() => adb!, resolveData, dataURL, duckdb.DuckDBDataProtocol.HTTP); testOPFS(dataURL, () => DUCKDB_BUNDLE!); -testArrowInsert(() => db!); -testArrowInsertAsync(() => adb!); -testJSONInsert(() => db!); -testJSONInsertAsync(() => adb!); -testCSVInsert(() => db!); -testCSVInsertAsync(() => adb!); -testTokenization(() => db!); -testTokenizationAsync(() => adb!); +// testArrowInsert(() => db!); +// testArrowInsertAsync(() => adb!); +// testJSONInsert(() => db!); +// testJSONInsertAsync(() => adb!); +// testCSVInsert(() => db!); +// testCSVInsertAsync(() => adb!); +// testTokenization(() => db!); +// testTokenizationAsync(() => adb!); //testEXCEL(() => db!); //testJSON(() => db!); diff --git a/packages/duckdb-wasm/test/opfs.test.ts b/packages/duckdb-wasm/test/opfs.test.ts index 32b593cf2..c89b2aed6 100644 --- a/packages/duckdb-wasm/test/opfs.test.ts +++ b/packages/duckdb-wasm/test/opfs.test.ts @@ -105,7 +105,7 @@ export function testOPFS(baseDir: string, bundle: () => DuckDBBundle): void { it('Load Parquet file that are already with empty handler', async () => { //1. write to opfs - const fileHandler = await getOpfsFileHandlerFromUrl({ + await getOpfsFileHandlerFromUrl({ url: `${ baseDir }/tpch/0_01/parquet/lineitem.parquet`, path: 'test.parquet' }); @@ -363,8 +363,10 @@ export function testOPFS(baseDir: string, bundle: () => DuckDBBundle): void { it('Copy CSV to OPFS + Load CSV', async () => { //1. data preparation + db.config.accessMode = DuckDBAccessMode.READ_WRITE; db.config.opfs = { - fileHandling: "auto" + fileHandling: "auto", + window:"multi" }; await conn.query(`COPY ( SELECT 32 AS value ) TO 'opfs://file.csv'`); await conn.query(`COPY ( SELECT 42 AS value ) TO 'opfs://file.csv'`); @@ -382,7 +384,6 @@ export function testOPFS(baseDir: string, bundle: () => DuckDBBundle): void { describe('Open database in OPFS', () => { it('should not open a non-existent DB file in read-only', async () => { - const logger = new ConsoleLogger(LogLevel.ERROR); const worker = new Worker(bundle().mainWorker!); const db_ = new AsyncDuckDB(logger, worker); await db_.instantiate(bundle().mainModule, bundle().pthreadWorker); @@ -402,7 +403,6 @@ export function testOPFS(baseDir: string, bundle: () => DuckDBBundle): void { }); it('should not open a non-existent DB file and mkdir in read-only', async () => { - const logger = new ConsoleLogger(LogLevel.ERROR); const worker = new Worker(bundle().mainWorker!); const db_ = new AsyncDuckDB(logger, worker); await db_.instantiate(bundle().mainModule, bundle().pthreadWorker); @@ -417,7 +417,6 @@ export function testOPFS(baseDir: string, bundle: () => DuckDBBundle): void { }); it('should open a non-existent DB file and mkdir in read-write', async () => { - const logger = new ConsoleLogger(LogLevel.ERROR); const worker = new Worker(bundle().mainWorker!); const db_ = new AsyncDuckDB(logger, worker); await db_.instantiate(bundle().mainModule, bundle().pthreadWorker); @@ -432,7 +431,6 @@ export function testOPFS(baseDir: string, bundle: () => DuckDBBundle): void { }); it('should open a non-existent DB file in read-write and create files', async () => { - const logger = new ConsoleLogger(LogLevel.ERROR); const worker = new Worker(bundle().mainWorker!); const db_ = new AsyncDuckDB(logger, worker); await db_.instantiate(bundle().mainModule, bundle().pthreadWorker); diff --git a/packages/duckdb-wasm/types/lib.webworker.d.ts b/packages/duckdb-wasm/types/lib.webworker.d.ts index e49a8ae06..d47e699d1 100644 --- a/packages/duckdb-wasm/types/lib.webworker.d.ts +++ b/packages/duckdb-wasm/types/lib.webworker.d.ts @@ -1,3 +1,13 @@ -interface FileSystemFileHandle extends FileSystemHandle { - createSyncAccessHandle(mode?: 'read' | 'readwrite' | 'readwrite-unsafe'): Promise; +type FileSystemSyncAccessHandleMode = 'readwrite' | 'read-only' | 'readwrite-unsafe'; + +interface FileSystemCreateSyncAccessHandleOptions { + mode?: FileSystemSyncAccessHandleMode +} + +interface FileSystemFileHandle { + createSyncAccessHandle(optional: FileSystemCreateSyncAccessHandleOptions = {}): Promise; +} + +interface FileSystemSyncAccessHandle { + mode: FileSystemSyncAccessHandleMode; } \ No newline at end of file From f954bef306ecfeda881d2570ed54ce1f84c5193d Mon Sep 17 00:00:00 2001 From: arkw Date: Sat, 1 Mar 2025 20:36:46 +0900 Subject: [PATCH 13/13] fix: mode name --- packages/duckdb-wasm/src/bindings/runtime_browser.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/duckdb-wasm/src/bindings/runtime_browser.ts b/packages/duckdb-wasm/src/bindings/runtime_browser.ts index eee05e46f..377e67cd8 100644 --- a/packages/duckdb-wasm/src/bindings/runtime_browser.ts +++ b/packages/duckdb-wasm/src/bindings/runtime_browser.ts @@ -145,7 +145,7 @@ export const BROWSER_RUNTIME: DuckDBRuntime & { let fileHandle:FileSystemFileHandle; try { fileHandle = await dirHandle.getFileHandle(fileName, { create: false }); - } catch (e:any) { + } catch (e: any) { if (e?.name === 'NotFoundError') { if (isReadWrite) { console.debug(`File ${ path } does not exists yet, creating...`); @@ -159,9 +159,9 @@ export const BROWSER_RUNTIME: DuckDBRuntime & { } } try { - let syncAccessHandleMode:FileSystemSyncAccessHandleMode = isReadWrite ? "readwrite" : "readwrite-unsafe"; + let mode:FileSystemSyncAccessHandleMode = isReadWrite ? "readwrite" : "readwrite-unsafe"; const handle = await fileHandle.createSyncAccessHandle({ - mode : syncAccessHandleMode + mode : mode }); BROWSER_RUNTIME._preparedHandles[path] = handle; return {