From f0a042b3fa80fb8f84b77264712eead93ec3f528 Mon Sep 17 00:00:00 2001 From: Carlo Piovesan Date: Wed, 7 May 2025 12:45:00 +0200 Subject: [PATCH 01/28] Publish on NPM only from main --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 82faf1146..c9d216edd 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -1084,7 +1084,7 @@ jobs: env: NODE_AUTH_TOKEN: ${{ secrets.NPM_PUBLISH_TOKEN }} TAG: ${{ startsWith(github.head_ref, 'v') && 'latest' || 'next' }} - if: env.NODE_AUTH_TOKEN != null + if: env.NODE_AUTH_TOKEN != null && github.ref == 'refs/heads/main' run: | echo "${TAG}" ./scripts/npm_publish_lib.sh From 423b5284f87bfc94537c7047c1dc046e8b526c1e Mon Sep 17 00:00:00 2001 From: Carlo Piovesan Date: Wed, 7 May 2025 17:39:03 +0200 Subject: [PATCH 02/28] Add patch to implement VirtualFileSystem constructor --- patches/duckdb/virtualized_file_system.patch | 77 ++++++++++++++++++++ 1 file changed, 77 insertions(+) create mode 100644 patches/duckdb/virtualized_file_system.patch diff --git a/patches/duckdb/virtualized_file_system.patch b/patches/duckdb/virtualized_file_system.patch new file mode 100644 index 000000000..40f0b47d3 --- /dev/null +++ b/patches/duckdb/virtualized_file_system.patch @@ -0,0 +1,77 @@ +diff --git a/src/common/virtual_file_system.cpp b/src/common/virtual_file_system.cpp +index 74892a4e05..04454b8895 100644 +--- a/src/common/virtual_file_system.cpp ++++ b/src/common/virtual_file_system.cpp +@@ -5,7 +5,7 @@ + + namespace duckdb { + +-VirtualFileSystem::VirtualFileSystem() : default_fs(FileSystem::CreateLocal()) { ++VirtualFileSystem::VirtualFileSystem(unique_ptr default) : default_fs(std::move(default)) { + VirtualFileSystem::RegisterSubSystem(FileCompressionType::GZIP, make_uniq()); + } + +diff --git a/src/include/duckdb/common/virtual_file_system.hpp b/src/include/duckdb/common/virtual_file_system.hpp +index 110ad04877..1c2d4c91d5 100644 +--- a/src/include/duckdb/common/virtual_file_system.hpp ++++ b/src/include/duckdb/common/virtual_file_system.hpp +@@ -17,7 +17,7 @@ namespace duckdb { + // bunch of wrappers to allow registering protocol handlers + class VirtualFileSystem : public FileSystem { + public: +- VirtualFileSystem(); ++ explicit VirtualFileSystem(unique_ptr inner_file_system); + + unique_ptr OpenFile(const string &path, FileOpenFlags flags, + optional_ptr opener = nullptr) override; +diff --git a/src/main/database.cpp b/src/main/database.cpp +index f51caaaee6..33230b68ab 100644 +--- a/src/main/database.cpp ++++ b/src/main/database.cpp +@@ -422,9 +422,9 @@ void DatabaseInstance::Configure(DBConfig &new_config, const char *database_path + } + config.extension_parameters = new_config.extension_parameters; + if (new_config.file_system) { +- config.file_system = std::move(new_config.file_system); ++ config.file_system = make_uniq(std::move(new_config.file_system)); + } else { +- config.file_system = make_uniq(); ++ config.file_system = make_uniq(FileSystem::CreateLocal()); + } + if (database_path && !config.options.enable_external_access) { + config.AddAllowedPath(database_path); +diff --git a/test/api/test_threads.cpp b/test/api/test_threads.cpp +index f63e36d41c..5effeb7963 100644 +--- a/test/api/test_threads.cpp ++++ b/test/api/test_threads.cpp +@@ -61,7 +61,7 @@ TEST_CASE("Test database maximum_threads argument", "[api]") { + // FIXME: not yet + { + DuckDB db(nullptr); +- auto file_system = make_uniq(); ++ auto file_system = make_uniq(FileSystem::CreateLocal()); + REQUIRE(db.NumberOfThreads() == DBConfig().GetSystemMaxThreads(*file_system)); + } + // but we can set another value +@@ -114,7 +114,7 @@ TEST_CASE("Test external threads", "[api]") { + REQUIRE(db.NumberOfThreads() == 13); + + con.Query("RESET threads"); +- auto file_system = make_uniq(); ++ auto file_system = make_uniq(FileSystem::CreateLocal()); + REQUIRE(config.options.maximum_threads == DBConfig().GetSystemMaxThreads(*file_system)); + REQUIRE(db.NumberOfThreads() == DBConfig().GetSystemMaxThreads(*file_system)); + } +diff --git a/test/sqlite/sqllogic_command.cpp b/test/sqlite/sqllogic_command.cpp +index 6d4fcca0c8..eda4cb0794 100644 +--- a/test/sqlite/sqllogic_command.cpp ++++ b/test/sqlite/sqllogic_command.cpp +@@ -482,7 +482,7 @@ void Statement::ExecuteInternal(ExecuteContext &context) const { + } + + void UnzipCommand::ExecuteInternal(ExecuteContext &context) const { +- VirtualFileSystem vfs; ++ VirtualFileSystem vfs(FileSystem::CreateLocal()); + + // input + FileOpenFlags in_flags(FileFlags::FILE_FLAGS_READ); From fac50413c28c58e9ab7bfbbaa24a957f1a216ba7 Mon Sep 17 00:00:00 2001 From: Carlo Piovesan Date: Wed, 7 May 2025 15:35:00 +0200 Subject: [PATCH 03/28] Actually rely on VirtualFileSystem at the duckdb-wasm level Currently via patch to duckdb/duckdb, to be upstreamed --- lib/src/webdb.cc | 3 ++- packages/duckdb-wasm/test/httpfs_test.ts | 6 ++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/lib/src/webdb.cc b/lib/src/webdb.cc index 345c9ac67..f1cdd8cde 100644 --- a/lib/src/webdb.cc +++ b/lib/src/webdb.cc @@ -36,6 +36,7 @@ #include "duckdb/common/types/data_chunk.hpp" #include "duckdb/common/types/vector.hpp" #include "duckdb/common/types/vector_buffer.hpp" +#include "duckdb/common/virtual_file_system.hpp" #include "duckdb/main/query_result.hpp" #include "duckdb/parser/expression/constant_expression.hpp" #include "duckdb/parser/parser.hpp" @@ -866,7 +867,7 @@ arrow::Status WebDB::Open(std::string_view args_json) { auto buffered_fs_ptr = buffered_fs.get(); duckdb::DBConfig db_config; - db_config.file_system = std::move(buffered_fs); + db_config.file_system = std::move(make_uniq(std::move(buffered_fs))); db_config.options.allow_unsigned_extensions = config_->allow_unsigned_extensions; db_config.options.maximum_threads = config_->maximum_threads; db_config.options.use_temporary_directory = false; diff --git a/packages/duckdb-wasm/test/httpfs_test.ts b/packages/duckdb-wasm/test/httpfs_test.ts index 4ffe2ac0b..6e2f719cd 100644 --- a/packages/duckdb-wasm/test/httpfs_test.ts +++ b/packages/duckdb-wasm/test/httpfs_test.ts @@ -257,6 +257,12 @@ export function testHTTPFSAsync( expect(BigInt(results.getChildAt(2)?.get(2))).toEqual(BigInt(9n)); }); + it('can fetch over https csv.gz', async () => { + await conn!.query( + `select * from "https://raw.githubusercontent.com/duckdb/duckdb/v1.2.2/data/csv/test_apple_financial.csv.gz";`, + ); + }); + it('can read and write csv file from S3 with correct auth credentials', async () => { let data = await resolveData('/uni/studenten.parquet'); await setAwsConfig(conn!); From 420a0fd793c4654efc1a7942c3dfcc62cf100cab Mon Sep 17 00:00:00 2001 From: Carlo Piovesan Date: Wed, 7 May 2025 19:28:34 +0200 Subject: [PATCH 04/28] fix patch --- patches/duckdb/virtualized_file_system.patch | 59 ++------------------ 1 file changed, 4 insertions(+), 55 deletions(-) diff --git a/patches/duckdb/virtualized_file_system.patch b/patches/duckdb/virtualized_file_system.patch index 40f0b47d3..028fc3709 100644 --- a/patches/duckdb/virtualized_file_system.patch +++ b/patches/duckdb/virtualized_file_system.patch @@ -1,5 +1,5 @@ diff --git a/src/common/virtual_file_system.cpp b/src/common/virtual_file_system.cpp -index 74892a4e05..04454b8895 100644 +index 74892a4e05..c3d1c4f333 100644 --- a/src/common/virtual_file_system.cpp +++ b/src/common/virtual_file_system.cpp @@ -5,7 +5,7 @@ @@ -7,12 +7,12 @@ index 74892a4e05..04454b8895 100644 namespace duckdb { -VirtualFileSystem::VirtualFileSystem() : default_fs(FileSystem::CreateLocal()) { -+VirtualFileSystem::VirtualFileSystem(unique_ptr default) : default_fs(std::move(default)) { ++VirtualFileSystem::VirtualFileSystem(unique_ptr inner) : default_fs(std::move(inner)) { VirtualFileSystem::RegisterSubSystem(FileCompressionType::GZIP, make_uniq()); } diff --git a/src/include/duckdb/common/virtual_file_system.hpp b/src/include/duckdb/common/virtual_file_system.hpp -index 110ad04877..1c2d4c91d5 100644 +index 110ad04877..30a7eb29d3 100644 --- a/src/include/duckdb/common/virtual_file_system.hpp +++ b/src/include/duckdb/common/virtual_file_system.hpp @@ -17,7 +17,7 @@ namespace duckdb { @@ -20,58 +20,7 @@ index 110ad04877..1c2d4c91d5 100644 class VirtualFileSystem : public FileSystem { public: - VirtualFileSystem(); -+ explicit VirtualFileSystem(unique_ptr inner_file_system); ++ VirtualFileSystem(unique_ptr inner_file_system = nullptr); unique_ptr OpenFile(const string &path, FileOpenFlags flags, optional_ptr opener = nullptr) override; -diff --git a/src/main/database.cpp b/src/main/database.cpp -index f51caaaee6..33230b68ab 100644 ---- a/src/main/database.cpp -+++ b/src/main/database.cpp -@@ -422,9 +422,9 @@ void DatabaseInstance::Configure(DBConfig &new_config, const char *database_path - } - config.extension_parameters = new_config.extension_parameters; - if (new_config.file_system) { -- config.file_system = std::move(new_config.file_system); -+ config.file_system = make_uniq(std::move(new_config.file_system)); - } else { -- config.file_system = make_uniq(); -+ config.file_system = make_uniq(FileSystem::CreateLocal()); - } - if (database_path && !config.options.enable_external_access) { - config.AddAllowedPath(database_path); -diff --git a/test/api/test_threads.cpp b/test/api/test_threads.cpp -index f63e36d41c..5effeb7963 100644 ---- a/test/api/test_threads.cpp -+++ b/test/api/test_threads.cpp -@@ -61,7 +61,7 @@ TEST_CASE("Test database maximum_threads argument", "[api]") { - // FIXME: not yet - { - DuckDB db(nullptr); -- auto file_system = make_uniq(); -+ auto file_system = make_uniq(FileSystem::CreateLocal()); - REQUIRE(db.NumberOfThreads() == DBConfig().GetSystemMaxThreads(*file_system)); - } - // but we can set another value -@@ -114,7 +114,7 @@ TEST_CASE("Test external threads", "[api]") { - REQUIRE(db.NumberOfThreads() == 13); - - con.Query("RESET threads"); -- auto file_system = make_uniq(); -+ auto file_system = make_uniq(FileSystem::CreateLocal()); - REQUIRE(config.options.maximum_threads == DBConfig().GetSystemMaxThreads(*file_system)); - REQUIRE(db.NumberOfThreads() == DBConfig().GetSystemMaxThreads(*file_system)); - } -diff --git a/test/sqlite/sqllogic_command.cpp b/test/sqlite/sqllogic_command.cpp -index 6d4fcca0c8..eda4cb0794 100644 ---- a/test/sqlite/sqllogic_command.cpp -+++ b/test/sqlite/sqllogic_command.cpp -@@ -482,7 +482,7 @@ void Statement::ExecuteInternal(ExecuteContext &context) const { - } - - void UnzipCommand::ExecuteInternal(ExecuteContext &context) const { -- VirtualFileSystem vfs; -+ VirtualFileSystem vfs(FileSystem::CreateLocal()); - - // input - FileOpenFlags in_flags(FileFlags::FILE_FLAGS_READ); From f3d7db78f7f0756f874d6d58643feb7856c5ec29 Mon Sep 17 00:00:00 2001 From: Carlo Piovesan Date: Wed, 7 May 2025 11:25:36 +0200 Subject: [PATCH 05/28] Wrap results of fetchQueryResults via DuckDBWasmResultsWrapper This is currently just adding infrastructure, there should be no relevant changes Note that enum added to lib/include/duckdb/web/webdb.h would evetntually need to be dupliciated, adding 256, to packages/duckdb-wasm/src/status.ts --- lib/include/duckdb/web/utils/wasm_response.h | 4 ++++ lib/include/duckdb/web/webdb.h | 16 +++++++++++++++- lib/src/utils/wasm_response.cc | 10 ++++++++++ lib/src/webdb.cc | 6 +++--- lib/src/webdb_api.cc | 2 +- .../duckdb-wasm/src/bindings/bindings_base.ts | 5 ++++- packages/duckdb-wasm/src/status.ts | 7 ++++++- 7 files changed, 43 insertions(+), 7 deletions(-) diff --git a/lib/include/duckdb/web/utils/wasm_response.h b/lib/include/duckdb/web/utils/wasm_response.h index 2cff60931..9edbca39e 100644 --- a/lib/include/duckdb/web/utils/wasm_response.h +++ b/lib/include/duckdb/web/utils/wasm_response.h @@ -8,6 +8,8 @@ namespace duckdb { namespace web { +struct DuckDBWasmResultsWrapper; + struct WASMResponse { /// The status code double statusCode = 1; @@ -35,6 +37,8 @@ class WASMResponseBuffer { /// Store the arrow status. /// Returns wheather the result was OK bool Store(WASMResponse& response, arrow::Status status); + /// Store a DuckDBWasmResultsWrapper + void Store(WASMResponse& response, DuckDBWasmResultsWrapper& value); /// Store a string void Store(WASMResponse& response, std::string value); /// Store a string view diff --git a/lib/include/duckdb/web/webdb.h b/lib/include/duckdb/web/webdb.h index f7d15841a..5421435e7 100644 --- a/lib/include/duckdb/web/webdb.h +++ b/lib/include/duckdb/web/webdb.h @@ -29,6 +29,20 @@ namespace web { struct BufferingArrowIPCStreamDecoder; +struct DuckDBWasmResultsWrapper { + // Additional ResponseStatuses to be >= 256, and mirrored to packages/duckdb-wasm/src/status.ts + // Missing mapping result in a throw, but they should eventually align (it's fine if typescript side only has a + // subset) + enum ResponseStatus : uint32_t { ARROW_BUFFER = 0, MAX_ARROW_ERROR = 255 }; + DuckDBWasmResultsWrapper(arrow::Result> res, + ResponseStatus status = ResponseStatus::ARROW_BUFFER) + : arrow_buffer(res), status(status) {} + DuckDBWasmResultsWrapper(arrow::Status res, ResponseStatus status = ResponseStatus::ARROW_BUFFER) + : arrow_buffer(res), status(status) {} + arrow::Result> arrow_buffer; + ResponseStatus status; +}; + class WebDB { public: /// A connection @@ -93,7 +107,7 @@ class WebDB { /// Cancel a pending query bool CancelPendingQuery(); /// Fetch a data chunk from a pending query - arrow::Result> FetchQueryResults(); + DuckDBWasmResultsWrapper FetchQueryResults(); /// Get table names arrow::Result GetTableNames(std::string_view text); diff --git a/lib/src/utils/wasm_response.cc b/lib/src/utils/wasm_response.cc index d5645314e..61a062cea 100644 --- a/lib/src/utils/wasm_response.cc +++ b/lib/src/utils/wasm_response.cc @@ -3,6 +3,7 @@ #include #include "arrow/buffer.h" +#include "duckdb/web/webdb.h" namespace duckdb { namespace web { @@ -26,6 +27,15 @@ bool WASMResponseBuffer::Store(WASMResponse& response, arrow::Status status) { return true; } +void WASMResponseBuffer::Store(WASMResponse& response, DuckDBWasmResultsWrapper& value) { + if (value.status == DuckDBWasmResultsWrapper::ResponseStatus::ARROW_BUFFER) { + Store(response, std::move(value.arrow_buffer)); + } else { + Clear(); + response.statusCode = value.status; + } +} + void WASMResponseBuffer::Store(WASMResponse& response, std::string value) { result_str_ = std::move(value); response.statusCode = 0; diff --git a/lib/src/webdb.cc b/lib/src/webdb.cc index f1cdd8cde..659cb4a42 100644 --- a/lib/src/webdb.cc +++ b/lib/src/webdb.cc @@ -232,12 +232,12 @@ bool WebDB::Connection::CancelPendingQuery() { } } -arrow::Result> WebDB::Connection::FetchQueryResults() { +DuckDBWasmResultsWrapper WebDB::Connection::FetchQueryResults() { try { // Fetch data if a query is active duckdb::unique_ptr chunk; if (current_query_result_ == nullptr) { - return nullptr; + return DuckDBWasmResultsWrapper{nullptr}; } // Fetch next result chunk chunk = current_query_result_->Fetch(); @@ -249,7 +249,7 @@ arrow::Result> WebDB::Connection::FetchQueryResul current_query_result_.reset(); current_schema_.reset(); current_schema_patched_.reset(); - return nullptr; + return DuckDBWasmResultsWrapper{nullptr}; } // Serialize the record batch diff --git a/lib/src/webdb_api.cc b/lib/src/webdb_api.cc index bad9c7f9e..bf3521333 100644 --- a/lib/src/webdb_api.cc +++ b/lib/src/webdb_api.cc @@ -241,7 +241,7 @@ bool duckdb_web_pending_query_cancel(ConnectionHdl connHdl, const char* script) void duckdb_web_query_fetch_results(WASMResponse* packed, ConnectionHdl connHdl) { auto c = reinterpret_cast(connHdl); auto r = c->FetchQueryResults(); - WASMResponseBuffer::Get().Store(*packed, std::move(r)); + WASMResponseBuffer::Get().Store(*packed, r); } /// Get table names void duckdb_web_get_tablenames(WASMResponse* packed, ConnectionHdl connHdl, const char* query) { diff --git a/packages/duckdb-wasm/src/bindings/bindings_base.ts b/packages/duckdb-wasm/src/bindings/bindings_base.ts index f395bdb10..08cbc2bbe 100644 --- a/packages/duckdb-wasm/src/bindings/bindings_base.ts +++ b/packages/duckdb-wasm/src/bindings/bindings_base.ts @@ -4,7 +4,7 @@ import { Logger } from '../log'; import { InstantiationProgress } from './progress'; import { DuckDBBindings } from './bindings_interface'; import { DuckDBConnection } from './connection'; -import { StatusCode } from '../status'; +import { StatusCode, IsArrowBuffer } from '../status'; import { dropResponseBuffers, DuckDBRuntime, readString, callSRet, copyBuffer, DuckDBDataProtocol } from './runtime'; import { CSVInsertOptions, JSONInsertOptions, ArrowInsertOptions } from './insert_options'; import { ScriptTokens } from './tokens'; @@ -224,6 +224,9 @@ export abstract class DuckDBBindingsBase implements DuckDBBindings { /** Fetch query results */ public fetchQueryResults(conn: number): Uint8Array { const [s, d, n] = callSRet(this.mod, 'duckdb_web_query_fetch_results', ['number'], [conn]); + if (!IsArrowBuffer(s)) { + throw new Error("Unexpected StatusCode from duckdb_web_query_fetch_results (" + s + ") and with self reported error as" + readString(this.mod, d, n)); + } if (s !== StatusCode.SUCCESS) { throw new Error(readString(this.mod, d, n)); } diff --git a/packages/duckdb-wasm/src/status.ts b/packages/duckdb-wasm/src/status.ts index 6390a5ec1..a308e1e56 100644 --- a/packages/duckdb-wasm/src/status.ts +++ b/packages/duckdb-wasm/src/status.ts @@ -1,3 +1,8 @@ export enum StatusCode { - SUCCESS = 0, + SUCCESS = 0, + MAX_ARROW_ERROR = 255, +} + +export function IsArrowBuffer(status: StatusCode): boolean { + return (status <= StatusCode.MAX_ARROW_ERROR); } From 8ee9eac110ce5827fefc610278adc5c71ab6b263 Mon Sep 17 00:00:00 2001 From: Yves Date: Thu, 1 May 2025 16:18:52 +0200 Subject: [PATCH 06/28] Make sure we `Fetch` query result when ready --- lib/src/webdb.cc | 42 +++++++++++++++++++ .../duckdb-wasm/src/bindings/bindings_base.ts | 8 +++- .../src/bindings/bindings_interface.ts | 2 +- .../duckdb-wasm/src/bindings/connection.ts | 6 ++- .../src/parallel/async_bindings.ts | 4 +- .../src/parallel/async_bindings_interface.ts | 2 +- .../src/parallel/async_connection.ts | 10 +++-- .../src/parallel/worker_dispatcher.ts | 3 +- .../src/parallel/worker_request.ts | 4 +- 9 files changed, 68 insertions(+), 13 deletions(-) diff --git a/lib/src/webdb.cc b/lib/src/webdb.cc index 659cb4a42..d9002dc4c 100644 --- a/lib/src/webdb.cc +++ b/lib/src/webdb.cc @@ -74,6 +74,8 @@ namespace web { static constexpr int64_t DEFAULT_QUERY_POLLING_INTERVAL = 100; +static uint8_t MAGIC_BYTE_RETRY = 84; + /// Create the default webdb database duckdb::unique_ptr WebDB::Create() { if constexpr (ENVIRONMENT == Environment::WEB) { @@ -239,6 +241,46 @@ DuckDBWasmResultsWrapper WebDB::Connection::FetchQueryResults() { if (current_query_result_ == nullptr) { return DuckDBWasmResultsWrapper{nullptr}; } + + if (current_query_result_->type == QueryResultType::STREAM_RESULT) { + auto& stream_result = current_query_result_->Cast(); + + auto before = std::chrono::steady_clock::now(); + uint64_t elapsed; + auto polling_interval = + webdb_.config_->query.query_polling_interval.value_or(DEFAULT_QUERY_POLLING_INTERVAL); + bool ready = false; + do { + switch (stream_result.ExecuteTask()) { + case StreamExecutionResult::EXECUTION_ERROR: + return arrow::Status{arrow::StatusCode::ExecutionError, + std::move(current_query_result_->GetError())}; + case StreamExecutionResult::EXECUTION_CANCELLED: + return arrow::Status{arrow::StatusCode::ExecutionError, + "The execution of the query was cancelled before it could finish, likely " + "caused by executing a different query"}; + case StreamExecutionResult::CHUNK_READY: + case StreamExecutionResult::EXECUTION_FINISHED: + ready = true; + break; + case StreamExecutionResult::BLOCKED: + stream_result.WaitForTask(); + return arrow::Buffer::Wrap(&MAGIC_BYTE_RETRY, 1); + case StreamExecutionResult::NO_TASKS_AVAILABLE: + return arrow::Buffer::Wrap(&MAGIC_BYTE_RETRY, 1); + case StreamExecutionResult::CHUNK_NOT_READY: + break; + } + + auto after = std::chrono::steady_clock::now(); + elapsed = std::chrono::duration_cast(after - before).count(); + } while (!ready && elapsed < polling_interval); + + if (!ready) { + return arrow::Buffer::Wrap(&MAGIC_BYTE_RETRY, 1); + } + } + // Fetch next result chunk chunk = current_query_result_->Fetch(); if (current_query_result_->HasError()) { diff --git a/packages/duckdb-wasm/src/bindings/bindings_base.ts b/packages/duckdb-wasm/src/bindings/bindings_base.ts index 08cbc2bbe..e7607af3e 100644 --- a/packages/duckdb-wasm/src/bindings/bindings_base.ts +++ b/packages/duckdb-wasm/src/bindings/bindings_base.ts @@ -222,7 +222,7 @@ export abstract class DuckDBBindingsBase implements DuckDBBindings { return this.mod.ccall('duckdb_web_pending_query_cancel', 'boolean', ['number'], [conn]); } /** Fetch query results */ - public fetchQueryResults(conn: number): Uint8Array { + public fetchQueryResults(conn: number): Uint8Array | null { const [s, d, n] = callSRet(this.mod, 'duckdb_web_query_fetch_results', ['number'], [conn]); if (!IsArrowBuffer(s)) { throw new Error("Unexpected StatusCode from duckdb_web_query_fetch_results (" + s + ") and with self reported error as" + readString(this.mod, d, n)); @@ -230,8 +230,14 @@ export abstract class DuckDBBindingsBase implements DuckDBBindings { if (s !== StatusCode.SUCCESS) { throw new Error(readString(this.mod, d, n)); } + const res = copyBuffer(this.mod, d, n); dropResponseBuffers(this.mod); + + // Special case indicating to the caller they need to retry + if (res.length === 1 && res[0] == 84) { + return null; + } return res; } /** Get table names */ diff --git a/packages/duckdb-wasm/src/bindings/bindings_interface.ts b/packages/duckdb-wasm/src/bindings/bindings_interface.ts index 271a42ef9..2920972b2 100644 --- a/packages/duckdb-wasm/src/bindings/bindings_interface.ts +++ b/packages/duckdb-wasm/src/bindings/bindings_interface.ts @@ -19,7 +19,7 @@ export interface DuckDBBindings { startPendingQuery(conn: number, text: string, allowStreamResult: boolean): Uint8Array | null; pollPendingQuery(conn: number): Uint8Array | null; cancelPendingQuery(conn: number): boolean; - fetchQueryResults(conn: number): Uint8Array; + fetchQueryResults(conn: number): Uint8Array | null; getTableNames(conn: number, text: string): string[]; createPrepared(conn: number, text: string): number; diff --git a/packages/duckdb-wasm/src/bindings/connection.ts b/packages/duckdb-wasm/src/bindings/connection.ts index 70b5f018d..fb54a3660 100644 --- a/packages/duckdb-wasm/src/bindings/connection.ts +++ b/packages/duckdb-wasm/src/bindings/connection.ts @@ -53,7 +53,6 @@ export class DuckDBConnection { //Otherwise, reject with the error reject(e); } - } }); } @@ -125,7 +124,10 @@ export class ResultStreamIterator implements Iterable { if (this._depleted) { return { done: true, value: null }; } - const bufferI8 = this.bindings.fetchQueryResults(this.conn); + let bufferI8 = null; + do { + bufferI8 = this.bindings.fetchQueryResults(this.conn); + } while (bufferI8 == null); this._depleted = bufferI8.length == 0; return { done: this._depleted, diff --git a/packages/duckdb-wasm/src/parallel/async_bindings.ts b/packages/duckdb-wasm/src/parallel/async_bindings.ts index dc8a81e53..9d8a1c2e7 100644 --- a/packages/duckdb-wasm/src/parallel/async_bindings.ts +++ b/packages/duckdb-wasm/src/parallel/async_bindings.ts @@ -442,8 +442,8 @@ export class AsyncDuckDB implements AsyncDuckDBBindings { } /** Fetch query results */ - public async fetchQueryResults(conn: ConnectionID): Promise { - const task = new WorkerTask( + public async fetchQueryResults(conn: ConnectionID): Promise { + const task = new WorkerTask( WorkerRequestType.FETCH_QUERY_RESULTS, conn, ); diff --git a/packages/duckdb-wasm/src/parallel/async_bindings_interface.ts b/packages/duckdb-wasm/src/parallel/async_bindings_interface.ts index 97ba2b191..da1e7cbf6 100644 --- a/packages/duckdb-wasm/src/parallel/async_bindings_interface.ts +++ b/packages/duckdb-wasm/src/parallel/async_bindings_interface.ts @@ -22,7 +22,7 @@ export interface AsyncDuckDBBindings { startPendingQuery(conn: number, text: string, allowStreamResult: boolean): Promise; pollPendingQuery(conn: number): Promise; cancelPendingQuery(conn: number): Promise; - fetchQueryResults(conn: number): Promise; + fetchQueryResults(conn: number): Promise; createPrepared(conn: number, text: string): Promise; closePrepared(conn: number, statement: number): Promise; diff --git a/packages/duckdb-wasm/src/parallel/async_connection.ts b/packages/duckdb-wasm/src/parallel/async_connection.ts index 783a18bd5..d4bdd8ecd 100644 --- a/packages/duckdb-wasm/src/parallel/async_connection.ts +++ b/packages/duckdb-wasm/src/parallel/async_connection.ts @@ -115,7 +115,7 @@ export class AsyncResultStreamIterator implements AsyncIterable { /** Reached end of stream? */ protected _depleted: boolean; /** In-flight */ - protected _inFlight: Promise | null; + protected _inFlight: Promise | null; constructor( protected readonly db: AsyncDuckDB, @@ -135,17 +135,21 @@ export class AsyncResultStreamIterator implements AsyncIterable { if (this._depleted) { return { done: true, value: null }; } - let buffer: Uint8Array; + let buffer: Uint8Array | null = null; if (this._inFlight != null) { buffer = await this._inFlight; this._inFlight = null; - } else { + } + + while (buffer == null) { buffer = await this.db.fetchQueryResults(this.conn); } + this._depleted = buffer.length == 0; if (!this._depleted) { this._inFlight = this.db.fetchQueryResults(this.conn); } + return { done: this._depleted, value: buffer, diff --git a/packages/duckdb-wasm/src/parallel/worker_dispatcher.ts b/packages/duckdb-wasm/src/parallel/worker_dispatcher.ts index 3a5a8f295..b637b78b4 100644 --- a/packages/duckdb-wasm/src/parallel/worker_dispatcher.ts +++ b/packages/duckdb-wasm/src/parallel/worker_dispatcher.ts @@ -279,6 +279,7 @@ export abstract class AsyncDuckDBDispatcher implements Logger { } case WorkerRequestType.FETCH_QUERY_RESULTS: { const result = this._bindings.fetchQueryResults(request.data); + const transfer = result ? [result.buffer] : []; this.postMessage( { messageId: this._nextMessageId++, @@ -286,7 +287,7 @@ export abstract class AsyncDuckDBDispatcher implements Logger { type: WorkerResponseType.QUERY_RESULT_CHUNK, data: result, }, - [result.buffer], + transfer, ); break; } diff --git a/packages/duckdb-wasm/src/parallel/worker_request.ts b/packages/duckdb-wasm/src/parallel/worker_request.ts index 38502b7b6..071510d16 100644 --- a/packages/duckdb-wasm/src/parallel/worker_request.ts +++ b/packages/duckdb-wasm/src/parallel/worker_request.ts @@ -160,7 +160,7 @@ export type WorkerResponseVariant = | WorkerResponse | WorkerResponse | WorkerResponse - | WorkerResponse + | WorkerResponse | WorkerResponse | WorkerResponse | WorkerResponse @@ -180,7 +180,7 @@ export type WorkerTaskVariant = | WorkerTask | WorkerTask | WorkerTask - | WorkerTask + | WorkerTask | WorkerTask | WorkerTask | WorkerTask From d9e0a4fa1b783ae62b66825241d147c96f4e8129 Mon Sep 17 00:00:00 2001 From: Yves Date: Fri, 9 May 2025 16:04:52 +0200 Subject: [PATCH 07/28] Use `DuckDBWasmResultsWrapper` --- lib/include/duckdb/web/webdb.h | 4 +++- lib/src/webdb.cc | 8 +++----- packages/duckdb-wasm/src/bindings/bindings_base.ts | 12 ++++++------ packages/duckdb-wasm/src/status.ts | 7 ++++++- 4 files changed, 18 insertions(+), 13 deletions(-) diff --git a/lib/include/duckdb/web/webdb.h b/lib/include/duckdb/web/webdb.h index 5421435e7..0cd847169 100644 --- a/lib/include/duckdb/web/webdb.h +++ b/lib/include/duckdb/web/webdb.h @@ -33,12 +33,14 @@ struct DuckDBWasmResultsWrapper { // Additional ResponseStatuses to be >= 256, and mirrored to packages/duckdb-wasm/src/status.ts // Missing mapping result in a throw, but they should eventually align (it's fine if typescript side only has a // subset) - enum ResponseStatus : uint32_t { ARROW_BUFFER = 0, MAX_ARROW_ERROR = 255 }; + enum ResponseStatus : uint32_t { ARROW_BUFFER = 0, MAX_ARROW_ERROR = 255, DUCKDB_WASM_RETRY = 256 }; DuckDBWasmResultsWrapper(arrow::Result> res, ResponseStatus status = ResponseStatus::ARROW_BUFFER) : arrow_buffer(res), status(status) {} DuckDBWasmResultsWrapper(arrow::Status res, ResponseStatus status = ResponseStatus::ARROW_BUFFER) : arrow_buffer(res), status(status) {} + DuckDBWasmResultsWrapper(ResponseStatus status = ResponseStatus::ARROW_BUFFER) + : DuckDBWasmResultsWrapper(nullptr, status) {} arrow::Result> arrow_buffer; ResponseStatus status; }; diff --git a/lib/src/webdb.cc b/lib/src/webdb.cc index d9002dc4c..869465657 100644 --- a/lib/src/webdb.cc +++ b/lib/src/webdb.cc @@ -74,8 +74,6 @@ namespace web { static constexpr int64_t DEFAULT_QUERY_POLLING_INTERVAL = 100; -static uint8_t MAGIC_BYTE_RETRY = 84; - /// Create the default webdb database duckdb::unique_ptr WebDB::Create() { if constexpr (ENVIRONMENT == Environment::WEB) { @@ -265,9 +263,9 @@ DuckDBWasmResultsWrapper WebDB::Connection::FetchQueryResults() { break; case StreamExecutionResult::BLOCKED: stream_result.WaitForTask(); - return arrow::Buffer::Wrap(&MAGIC_BYTE_RETRY, 1); + return DuckDBWasmResultsWrapper::ResponseStatus::DUCKDB_WASM_RETRY; case StreamExecutionResult::NO_TASKS_AVAILABLE: - return arrow::Buffer::Wrap(&MAGIC_BYTE_RETRY, 1); + return DuckDBWasmResultsWrapper::ResponseStatus::DUCKDB_WASM_RETRY; case StreamExecutionResult::CHUNK_NOT_READY: break; } @@ -277,7 +275,7 @@ DuckDBWasmResultsWrapper WebDB::Connection::FetchQueryResults() { } while (!ready && elapsed < polling_interval); if (!ready) { - return arrow::Buffer::Wrap(&MAGIC_BYTE_RETRY, 1); + return DuckDBWasmResultsWrapper::ResponseStatus::DUCKDB_WASM_RETRY; } } diff --git a/packages/duckdb-wasm/src/bindings/bindings_base.ts b/packages/duckdb-wasm/src/bindings/bindings_base.ts index e7607af3e..8e8a1aecd 100644 --- a/packages/duckdb-wasm/src/bindings/bindings_base.ts +++ b/packages/duckdb-wasm/src/bindings/bindings_base.ts @@ -4,7 +4,7 @@ import { Logger } from '../log'; import { InstantiationProgress } from './progress'; import { DuckDBBindings } from './bindings_interface'; import { DuckDBConnection } from './connection'; -import { StatusCode, IsArrowBuffer } from '../status'; +import { StatusCode, IsArrowBuffer, IsDuckDBWasmRetry } from '../status'; import { dropResponseBuffers, DuckDBRuntime, readString, callSRet, copyBuffer, DuckDBDataProtocol } from './runtime'; import { CSVInsertOptions, JSONInsertOptions, ArrowInsertOptions } from './insert_options'; import { ScriptTokens } from './tokens'; @@ -224,6 +224,11 @@ export abstract class DuckDBBindingsBase implements DuckDBBindings { /** Fetch query results */ public fetchQueryResults(conn: number): Uint8Array | null { const [s, d, n] = callSRet(this.mod, 'duckdb_web_query_fetch_results', ['number'], [conn]); + if (IsDuckDBWasmRetry(s)) { + dropResponseBuffers(this.mod); + return null; // Retry + } + if (!IsArrowBuffer(s)) { throw new Error("Unexpected StatusCode from duckdb_web_query_fetch_results (" + s + ") and with self reported error as" + readString(this.mod, d, n)); } @@ -233,11 +238,6 @@ export abstract class DuckDBBindingsBase implements DuckDBBindings { const res = copyBuffer(this.mod, d, n); dropResponseBuffers(this.mod); - - // Special case indicating to the caller they need to retry - if (res.length === 1 && res[0] == 84) { - return null; - } return res; } /** Get table names */ diff --git a/packages/duckdb-wasm/src/status.ts b/packages/duckdb-wasm/src/status.ts index a308e1e56..4ae0f8f98 100644 --- a/packages/duckdb-wasm/src/status.ts +++ b/packages/duckdb-wasm/src/status.ts @@ -1,8 +1,13 @@ export enum StatusCode { SUCCESS = 0, MAX_ARROW_ERROR = 255, + DUCKDB_WASM_RETRY = 256, } export function IsArrowBuffer(status: StatusCode): boolean { - return (status <= StatusCode.MAX_ARROW_ERROR); + return status <= StatusCode.MAX_ARROW_ERROR; +} + +export function IsDuckDBWasmRetry(status: StatusCode): boolean { + return status === StatusCode.DUCKDB_WASM_RETRY; } From 292d25204a230cc88f1af42c6316ec8fbaf94228 Mon Sep 17 00:00:00 2001 From: Carlo Piovesan Date: Mon, 12 May 2025 09:19:36 +0200 Subject: [PATCH 08/28] Add patch to rapidjson, to comply to CMake 4.0 requirements --- Makefile | 1 + patches/rapidjson/cmake_minimum_required.patch | 10 ++++++++++ 2 files changed, 11 insertions(+) create mode 100644 patches/rapidjson/cmake_minimum_required.patch diff --git a/Makefile b/Makefile index e4c96a669..37fc91c30 100644 --- a/Makefile +++ b/Makefile @@ -436,6 +436,7 @@ build/docker_ci_image: patch_duckdb: (find patches/duckdb/* -type f -name '*.patch' -print0 | xargs -0 cat | patch -p1 --forward -d submodules/duckdb) || true (find patches/arrow/* -type f -name '*.patch' -print0 | xargs -0 cat | patch -p1 --forward -d submodules/arrow) || true + (find patches/rapidjson/* -type f -name '*.patch' -print0 | xargs -0 cat | patch -p1 --forward -d submodules/rapidjson) || true apply_patches: patch_duckdb diff --git a/patches/rapidjson/cmake_minimum_required.patch b/patches/rapidjson/cmake_minimum_required.patch new file mode 100644 index 000000000..2b436915d --- /dev/null +++ b/patches/rapidjson/cmake_minimum_required.patch @@ -0,0 +1,10 @@ +diff --git a/CMakeLists.txt b/CMakeLists.txt +index dd1f173..2351bad 100644 +--- a/CMakeLists.txt ++++ b/CMakeLists.txt +@@ -1,4 +1,4 @@ +-CMAKE_MINIMUM_REQUIRED(VERSION 2.8.12) ++CMAKE_MINIMUM_REQUIRED(VERSION 2.8.12...3.29) + if(POLICY CMP0025) + # detect Apple's Clang + cmake_policy(SET CMP0025 NEW) From be508b2c87f835df3a94e7a98a049e5cbee7663c Mon Sep 17 00:00:00 2001 From: Yves Date: Mon, 12 May 2025 09:59:36 +0200 Subject: [PATCH 09/28] Add format target --- packages/duckdb-wasm/package.json | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/duckdb-wasm/package.json b/packages/duckdb-wasm/package.json index 7fba82009..9931a2401 100644 --- a/packages/duckdb-wasm/package.json +++ b/packages/duckdb-wasm/package.json @@ -62,6 +62,7 @@ "build:debug": "node bundle.mjs debug && tsc --emitDeclarationOnly", "build:release": "node bundle.mjs release && tsc --emitDeclarationOnly", "docs": "typedoc", + "format": "prettier --write \"**/*.+(js|ts)\"", "report": "node ./coverage.mjs", "test:node": "node --enable-source-maps ../../node_modules/jasmine/bin/jasmine ./dist/tests-node.cjs", "test:node:debug": "node --inspect-brk --enable-source-maps ../../node_modules/jasmine/bin/jasmine ./dist/tests-node.cjs", From 449ea101d7fdff4fad2104c7a7aab53bee6b2675 Mon Sep 17 00:00:00 2001 From: Yves Date: Mon, 12 May 2025 09:59:44 +0200 Subject: [PATCH 10/28] Format sources --- packages/duckdb-wasm/karma/s3rver/s3rver.js | 27 +-- .../duckdb-wasm/src/bindings/bindings_base.ts | 70 ++++-- .../duckdb-wasm/src/bindings/connection.ts | 6 +- packages/duckdb-wasm/src/bindings/runtime.ts | 6 +- .../src/bindings/runtime_browser.ts | 72 ++++--- .../duckdb-wasm/src/bindings/runtime_node.ts | 2 +- packages/duckdb-wasm/src/log.ts | 2 +- .../src/parallel/async_bindings.ts | 4 +- .../src/parallel/async_connection.ts | 4 +- packages/duckdb-wasm/src/platform.ts | 8 +- packages/duckdb-wasm/src/status.ts | 10 +- packages/duckdb-wasm/src/utils/s3_helper.ts | 200 +++++++++++------- packages/duckdb-wasm/test/bindings.test.ts | 6 +- packages/duckdb-wasm/test/excel.test.ts | 29 ++- packages/duckdb-wasm/test/httpfs_test.ts | 8 +- packages/duckdb-wasm/test/json.test.ts | 4 +- .../duckdb-wasm/test/long_queries.test.ts | 5 +- packages/duckdb-wasm/test/opfs.test.ts | 51 ++--- .../test/regression/github_1833.test.ts | 12 +- .../duckdb-wasm/test/string_test_helper.ts | 8 +- 20 files changed, 312 insertions(+), 222 deletions(-) diff --git a/packages/duckdb-wasm/karma/s3rver/s3rver.js b/packages/duckdb-wasm/karma/s3rver/s3rver.js index 72a8c51ec..4f31c8121 100644 --- a/packages/duckdb-wasm/karma/s3rver/s3rver.js +++ b/packages/duckdb-wasm/karma/s3rver/s3rver.js @@ -1,15 +1,16 @@ const S3rver = require('s3rver'); -const CORS_CONFIG = "\n" + - " \n" + - " *\n" + - " PUT\n" + - " GET\n" + - " HEAD\n" + - " *\n" + - " Content-Range\n" + - " \n" + - ""; +const CORS_CONFIG = + '\n' + + ' \n' + + ' *\n' + + ' PUT\n' + + ' GET\n' + + ' HEAD\n' + + ' *\n' + + ' Content-Range\n' + + ' \n' + + ''; var createS3rver = function (args, config, logger) { const log = logger.create('S3-test-server'); @@ -19,10 +20,10 @@ var createS3rver = function (args, config, logger) { address: 'localhost', silent: config.s3rver.silent, directory: './../../.tmp/s3rver', - configureBuckets: [{name: 'test-bucket', configs:[CORS_CONFIG]}] + configureBuckets: [{ name: 'test-bucket', configs: [CORS_CONFIG] }], }).run(); }; module.exports = { - 'framework:s3rver': ['factory', createS3rver] -}; \ No newline at end of file + 'framework:s3rver': ['factory', createS3rver], +}; diff --git a/packages/duckdb-wasm/src/bindings/bindings_base.ts b/packages/duckdb-wasm/src/bindings/bindings_base.ts index 8e8a1aecd..d73bfa2c2 100644 --- a/packages/duckdb-wasm/src/bindings/bindings_base.ts +++ b/packages/duckdb-wasm/src/bindings/bindings_base.ts @@ -135,10 +135,15 @@ export abstract class DuckDBBindingsBase implements DuckDBBindings { /** Tokenize a script */ public tokenize(text: string): ScriptTokens { const BUF = TEXT_ENCODER.encode(text); - const bufferPtr = this.mod._malloc(BUF.length ); - const bufferOfs = this.mod.HEAPU8.subarray(bufferPtr, bufferPtr + BUF.length ); + const bufferPtr = this.mod._malloc(BUF.length); + const bufferOfs = this.mod.HEAPU8.subarray(bufferPtr, bufferPtr + BUF.length); bufferOfs.set(BUF); - const [s, d, n] = callSRet(this.mod, 'duckdb_web_tokenize_buffer', ['number', 'number'], [bufferPtr, BUF.length]); + const [s, d, n] = callSRet( + this.mod, + 'duckdb_web_tokenize_buffer', + ['number', 'number'], + [bufferPtr, BUF.length], + ); this.mod._free(bufferPtr); if (s !== StatusCode.SUCCESS) { throw new Error(readString(this.mod, d, n)); @@ -172,7 +177,12 @@ export abstract class DuckDBBindingsBase implements DuckDBBindings { const bufferPtr = this.mod._malloc(BUF.length); const bufferOfs = this.mod.HEAPU8.subarray(bufferPtr, bufferPtr + BUF.length); bufferOfs.set(BUF); - const [s, d, n] = callSRet(this.mod, 'duckdb_web_query_run_buffer', ['number', 'number', 'number'], [conn, bufferPtr, BUF.length]); + const [s, d, n] = callSRet( + this.mod, + 'duckdb_web_query_run_buffer', + ['number', 'number', 'number'], + [conn, bufferPtr, BUF.length], + ); this.mod._free(bufferPtr); if (s !== StatusCode.SUCCESS) { throw new Error(readString(this.mod, d, n)); @@ -189,10 +199,15 @@ export abstract class DuckDBBindingsBase implements DuckDBBindings { */ public startPendingQuery(conn: number, text: string, allowStreamResult: boolean = false): Uint8Array | null { const BUF = TEXT_ENCODER.encode(text); - const bufferPtr = this.mod._malloc(BUF.length ); - const bufferOfs = this.mod.HEAPU8.subarray(bufferPtr, bufferPtr + BUF.length ); + const bufferPtr = this.mod._malloc(BUF.length); + const bufferOfs = this.mod.HEAPU8.subarray(bufferPtr, bufferPtr + BUF.length); bufferOfs.set(BUF); - const [s, d, n] = callSRet(this.mod, 'duckdb_web_pending_query_start_buffer', ['number', 'number', 'number', 'boolean'], [conn, bufferPtr, BUF.length, allowStreamResult]); + const [s, d, n] = callSRet( + this.mod, + 'duckdb_web_pending_query_start_buffer', + ['number', 'number', 'number', 'boolean'], + [conn, bufferPtr, BUF.length, allowStreamResult], + ); this.mod._free(bufferPtr); if (s !== StatusCode.SUCCESS) { throw new Error(readString(this.mod, d, n)); @@ -230,7 +245,12 @@ export abstract class DuckDBBindingsBase implements DuckDBBindings { } if (!IsArrowBuffer(s)) { - throw new Error("Unexpected StatusCode from duckdb_web_query_fetch_results (" + s + ") and with self reported error as" + readString(this.mod, d, n)); + throw new Error( + 'Unexpected StatusCode from duckdb_web_query_fetch_results (' + + s + + ') and with self reported error as' + + readString(this.mod, d, n), + ); } if (s !== StatusCode.SUCCESS) { throw new Error(readString(this.mod, d, n)); @@ -246,7 +266,12 @@ export abstract class DuckDBBindingsBase implements DuckDBBindings { const bufferPtr = this.mod._malloc(BUF.length); const bufferOfs = this.mod.HEAPU8.subarray(bufferPtr, bufferPtr + BUF.length); bufferOfs.set(BUF); - const [s, d, n] = callSRet(this.mod, 'duckdb_web_get_tablenames_buffer', ['number', 'number', 'number'], [conn, bufferPtr, BUF.length]); + const [s, d, n] = callSRet( + this.mod, + 'duckdb_web_get_tablenames_buffer', + ['number', 'number', 'number'], + [conn, bufferPtr, BUF.length], + ); this.mod._free(bufferPtr); if (s !== StatusCode.SUCCESS) { throw new Error(readString(this.mod, d, n)); @@ -306,7 +331,12 @@ export abstract class DuckDBBindingsBase implements DuckDBBindings { const bufferPtr = this.mod._malloc(BUF.length); const bufferOfs = this.mod.HEAPU8.subarray(bufferPtr, bufferPtr + BUF.length); bufferOfs.set(BUF); - const [s, d, n] = callSRet(this.mod, 'duckdb_web_prepared_create_buffer', ['number', 'number', 'number'], [conn, bufferPtr, BUF.length]); + const [s, d, n] = callSRet( + this.mod, + 'duckdb_web_prepared_create_buffer', + ['number', 'number', 'number'], + [conn, bufferPtr, BUF.length], + ); this.mod._free(bufferPtr); if (s !== StatusCode.SUCCESS) { throw new Error(readString(this.mod, d, n)); @@ -513,28 +543,28 @@ export abstract class DuckDBBindingsBase implements DuckDBBindings { directIO: boolean, ): Promise { if (protocol === DuckDBDataProtocol.BROWSER_FSACCESS) { - if( handle instanceof FileSystemSyncAccessHandle ){ + if (handle instanceof FileSystemSyncAccessHandle) { // already a handle is sync handle. - } else if( handle instanceof FileSystemFileHandle ){ + } else if (handle instanceof FileSystemFileHandle) { // handle is an async handle, should convert to sync handle const fileHandle: FileSystemFileHandle = handle as any; try { handle = (await fileHandle.createSyncAccessHandle()) as any; } catch (e: any) { - throw new Error( e.message + ":" + name ); + throw new Error(e.message + ':' + name); } - } else if( name != null ){ + } else if (name != null) { // should get sync handle from the file name. try { const opfsRoot = await navigator.storage.getDirectory(); const fileHandle = await opfsRoot.getFileHandle(name); handle = (await fileHandle.createSyncAccessHandle()) as any; } catch (e: any) { - throw new Error( e.message + ":" + name ); + throw new Error(e.message + ':' + name); } } } - return handle; + return handle; } /** Register a file object URL async */ public async registerFileHandleAsync( @@ -625,10 +655,10 @@ export abstract class DuckDBBindingsBase implements DuckDBBindings { } /** Enable tracking of file statistics */ public registerOPFSFileName(file: string): Promise { - if (file.startsWith("opfs://")) { - return this.prepareFileHandle(file, DuckDBDataProtocol.BROWSER_FSACCESS); - } else { - throw new Error("Not an OPFS file name: " + file); + if (file.startsWith('opfs://')) { + return this.prepareFileHandle(file, DuckDBDataProtocol.BROWSER_FSACCESS); + } else { + throw new Error('Not an OPFS file name: ' + file); } } public collectFileStatistics(file: string, enable: boolean): void { diff --git a/packages/duckdb-wasm/src/bindings/connection.ts b/packages/duckdb-wasm/src/bindings/connection.ts index fb54a3660..a80c12215 100644 --- a/packages/duckdb-wasm/src/bindings/connection.ts +++ b/packages/duckdb-wasm/src/bindings/connection.ts @@ -111,7 +111,11 @@ export class ResultStreamIterator implements Iterable { /** Reached end of stream? */ _depleted: boolean; - constructor(protected bindings: DuckDBBindings, protected conn: number, protected header: Uint8Array) { + constructor( + protected bindings: DuckDBBindings, + protected conn: number, + protected header: Uint8Array, + ) { this._first = true; this._depleted = false; } diff --git a/packages/duckdb-wasm/src/bindings/runtime.ts b/packages/duckdb-wasm/src/bindings/runtime.ts index f894dba46..86b27d992 100644 --- a/packages/duckdb-wasm/src/bindings/runtime.ts +++ b/packages/duckdb-wasm/src/bindings/runtime.ts @@ -140,7 +140,7 @@ export interface DuckDBRuntime { openFile(mod: DuckDBModule, fileId: number, flags: FileFlags): void; syncFile(mod: DuckDBModule, fileId: number): void; closeFile(mod: DuckDBModule, fileId: number): void; - dropFile(mod: DuckDBModule, fileNamePtr: number, fileNameLen:number): void; + dropFile(mod: DuckDBModule, fileNamePtr: number, fileNameLen: number): void; getLastFileModificationTime(mod: DuckDBModule, fileId: number): number; truncateFile(mod: DuckDBModule, fileId: number, newSize: number): void; readFile(mod: DuckDBModule, fileId: number, buffer: number, bytes: number, location: number): number; @@ -162,7 +162,7 @@ export interface DuckDBRuntime { prepareDBFileHandle?: (path: string, protocol: DuckDBDataProtocol) => Promise; // Internal API - experimental - progressUpdate(final: number, percentage: number, iteration:number): void; + progressUpdate(final: number, percentage: number, iteration: number): void; // Call a scalar UDF function callScalarUDF( @@ -184,7 +184,7 @@ export const DEFAULT_RUNTIME: DuckDBRuntime = { openFile: (_mod: DuckDBModule, _fileId: number, flags: FileFlags): void => {}, syncFile: (_mod: DuckDBModule, _fileId: number): void => {}, closeFile: (_mod: DuckDBModule, _fileId: number): void => {}, - dropFile: (_mod: DuckDBModule, _fileNamePtr: number, _fileNameLen:number): void => {}, + dropFile: (_mod: DuckDBModule, _fileNamePtr: number, _fileNameLen: number): void => {}, getLastFileModificationTime: (_mod: DuckDBModule, _fileId: number): number => { return 0; }, diff --git a/packages/duckdb-wasm/src/bindings/runtime_browser.ts b/packages/duckdb-wasm/src/bindings/runtime_browser.ts index b5069dac4..e9f88b2b4 100644 --- a/packages/duckdb-wasm/src/bindings/runtime_browser.ts +++ b/packages/duckdb-wasm/src/bindings/runtime_browser.ts @@ -1,8 +1,6 @@ -import {StatusCode} from '../status'; -import { - WorkerResponseType, -} from '../parallel/worker_request'; -import {addS3Headers, getHTTPUrl} from '../utils'; +import { StatusCode } from '../status'; +import { WorkerResponseType } from '../parallel/worker_request'; +import { addS3Headers, getHTTPUrl } from '../utils'; import { callSRet, @@ -22,7 +20,6 @@ import * as udf from './udf_runtime'; const OPFS_PREFIX_LEN = 'opfs://'.length; const PATH_SEP_REGEX = /\/|\\/; - export const BROWSER_RUNTIME: DuckDBRuntime & { _files: Map; _fileInfoCache: Map; @@ -100,7 +97,7 @@ export const BROWSER_RUNTIME: DuckDBRuntime & { if (info == null) { return null; } - BROWSER_RUNTIME._globalFileInfo = { ...info, blob: null} as DuckDBGlobalFileInfo; + BROWSER_RUNTIME._globalFileInfo = { ...info, blob: null } as DuckDBGlobalFileInfo; return BROWSER_RUNTIME._globalFileInfo; } catch (e: any) { @@ -111,7 +108,7 @@ export const BROWSER_RUNTIME: DuckDBRuntime & { async assignOPFSRoot(): Promise { if (!BROWSER_RUNTIME._opfsRoot) { BROWSER_RUNTIME._opfsRoot = await navigator.storage.getDirectory(); - } + } }, /** Prepare a file handle that could only be acquired aschronously */ async prepareFileHandles(filePaths: string[], protocol: DuckDBDataProtocol): Promise { @@ -157,7 +154,7 @@ export const BROWSER_RUNTIME: DuckDBRuntime & { fromCached: false, }; } catch (e: any) { - throw new Error(e.message + ":" + name); + throw new Error(e.message + ':' + name); } }; const result: PreparedDBFileHandle[] = []; @@ -268,7 +265,6 @@ export const BROWSER_RUNTIME: DuckDBRuntime & { mod.HEAPF64[(result >> 3) + 1] = 0; return result; } - } catch (e: any) { error = e; console.warn(`HEAD request with range header failed: ${e}`); @@ -318,13 +314,23 @@ export const BROWSER_RUNTIME: DuckDBRuntime & { } } - if (xhr.status == 206 && contentLength2 !== null && +contentLength2 == 1 && presumedLength !== null) { + if ( + xhr.status == 206 && + contentLength2 !== null && + +contentLength2 == 1 && + presumedLength !== null + ) { const result = mod._malloc(2 * 8); mod.HEAPF64[(result >> 3) + 0] = +presumedLength; mod.HEAPF64[(result >> 3) + 1] = 0; return result; } - if (xhr.status == 200 && contentLength2 !== null && contentLength !== null && +contentLength2 == +contentLength) { + if ( + xhr.status == 200 && + contentLength2 !== null && + contentLength !== null && + +contentLength2 == +contentLength + ) { console.warn(`fall back to full HTTP read for: ${file.dataUrl}`); const data = mod._malloc(xhr.response.byteLength); const src = new Uint8Array(xhr.response, 0, xhr.response.byteLength); @@ -494,24 +500,24 @@ export const BROWSER_RUNTIME: DuckDBRuntime & { closeFile: (mod: DuckDBModule, fileId: number) => { const file = BROWSER_RUNTIME.getFileInfo(mod, fileId); BROWSER_RUNTIME._fileInfoCache.delete(fileId); - try { - switch (file?.dataProtocol) { - case DuckDBDataProtocol.BUFFER: - case DuckDBDataProtocol.HTTP: - case DuckDBDataProtocol.S3: - break; - case DuckDBDataProtocol.NODE_FS: - case DuckDBDataProtocol.BROWSER_FILEREADER: - // XXX Remove from registry - return; - case DuckDBDataProtocol.BROWSER_FSACCESS: { - const handle: FileSystemSyncAccessHandle = BROWSER_RUNTIME._files?.get(file.fileName); - if (!handle) { - throw new Error(`No OPFS access handle registered with name: ${file.fileName}`); + try { + switch (file?.dataProtocol) { + case DuckDBDataProtocol.BUFFER: + case DuckDBDataProtocol.HTTP: + case DuckDBDataProtocol.S3: + break; + case DuckDBDataProtocol.NODE_FS: + case DuckDBDataProtocol.BROWSER_FILEREADER: + // XXX Remove from registry + return; + case DuckDBDataProtocol.BROWSER_FSACCESS: { + const handle: FileSystemSyncAccessHandle = BROWSER_RUNTIME._files?.get(file.fileName); + if (!handle) { + throw new Error(`No OPFS access handle registered with name: ${file.fileName}`); + } + return handle.flush(); } - return handle.flush(); } - } } catch (e: any) { console.log(e); failWith(mod, e.toString()); @@ -691,9 +697,13 @@ export const BROWSER_RUNTIME: DuckDBRuntime & { return 0; }, progressUpdate: (done: number, percentage: number, repeat: number): void => { - if (postMessage) { - postMessage({requestId: 0, type: WorkerResponseType.PROGRESS_UPDATE, data: {status: done?"completed":"in-progress", percentage: percentage, repetitions: repeat}}); - } + if (postMessage) { + postMessage({ + requestId: 0, + type: WorkerResponseType.PROGRESS_UPDATE, + data: { status: done ? 'completed' : 'in-progress', percentage: percentage, repetitions: repeat }, + }); + } }, checkDirectory: (mod: DuckDBModule, pathPtr: number, pathLen: number) => { const path = readString(mod, pathPtr, pathLen); diff --git a/packages/duckdb-wasm/src/bindings/runtime_node.ts b/packages/duckdb-wasm/src/bindings/runtime_node.ts index 5037068a2..7f331d079 100644 --- a/packages/duckdb-wasm/src/bindings/runtime_node.ts +++ b/packages/duckdb-wasm/src/bindings/runtime_node.ts @@ -127,7 +127,7 @@ export const NODE_RUNTIME: DuckDBRuntime & { } return 0; }, - dropFile: (mod: DuckDBModule, _fileNamePtr: number, _fileNameLen:number) => {}, + dropFile: (mod: DuckDBModule, _fileNamePtr: number, _fileNameLen: number) => {}, truncateFile: (mod: DuckDBModule, fileId: number, newSize: number) => { try { const file = NODE_RUNTIME.resolveFileInfo(mod, fileId); diff --git a/packages/duckdb-wasm/src/log.ts b/packages/duckdb-wasm/src/log.ts index 4a4f836ca..c705e28b3 100644 --- a/packages/duckdb-wasm/src/log.ts +++ b/packages/duckdb-wasm/src/log.ts @@ -45,7 +45,7 @@ export type ProgressEntry = { readonly status: string; readonly percentage: string; readonly repetitions: string; -} +}; /** An execution progress handler */ export type ExecutionProgressHandler = (p: ProgressEntry) => void; diff --git a/packages/duckdb-wasm/src/parallel/async_bindings.ts b/packages/duckdb-wasm/src/parallel/async_bindings.ts index 9d8a1c2e7..2e2b3dc72 100644 --- a/packages/duckdb-wasm/src/parallel/async_bindings.ts +++ b/packages/duckdb-wasm/src/parallel/async_bindings.ts @@ -129,7 +129,7 @@ export class AsyncDuckDB implements AsyncDuckDBBindings { case WorkerResponseType.PROGRESS_UPDATE: { for (const p of this._onExecutionProgress) { p(response.data); - } + } return; } case WorkerResponseType.LOG: { @@ -524,7 +524,7 @@ export class AsyncDuckDB implements AsyncDuckDBBindings { /** Register an empty file buffer. */ public async registerEmptyFileBuffer(name: string): Promise { -/* + /* const task = new WorkerTask( WorkerRequestType.REGISTER_FILE_BUFFER, [name, new Uint8Array()], diff --git a/packages/duckdb-wasm/src/parallel/async_connection.ts b/packages/duckdb-wasm/src/parallel/async_connection.ts index d4bdd8ecd..89bec591f 100644 --- a/packages/duckdb-wasm/src/parallel/async_connection.ts +++ b/packages/duckdb-wasm/src/parallel/async_connection.ts @@ -42,8 +42,8 @@ export class AsyncDuckDBConnection { }); const buffer = await this._bindings.runQuery(this._conn, text); const reader = arrow.RecordBatchReader.from(buffer); - console.assert(reader.isSync(), "Reader is not sync"); - console.assert(reader.isFile(), "Reader is not file"); + console.assert(reader.isSync(), 'Reader is not sync'); + console.assert(reader.isFile(), 'Reader is not file'); return new arrow.Table(reader as arrow.RecordBatchFileReader); } diff --git a/packages/duckdb-wasm/src/platform.ts b/packages/duckdb-wasm/src/platform.ts index 8f17c3ad6..8d175d69c 100644 --- a/packages/duckdb-wasm/src/platform.ts +++ b/packages/duckdb-wasm/src/platform.ts @@ -20,10 +20,10 @@ export const isFirefox = () => userAgent().includes('Firefox'); export const isSafari = () => /^((?!chrome|android).)*safari/i.test(userAgent()); /** Bundles have different characteristics: - * - MVP: minimum viable product (uses features from first stable version of WebAssembly standard) - * - EH: exception handling - * - COI: cross origin isolation - */ + * - MVP: minimum viable product (uses features from first stable version of WebAssembly standard) + * - EH: exception handling + * - COI: cross origin isolation + */ export interface DuckDBBundles { mvp: { mainModule: string; diff --git a/packages/duckdb-wasm/src/status.ts b/packages/duckdb-wasm/src/status.ts index 4ae0f8f98..7b0557da1 100644 --- a/packages/duckdb-wasm/src/status.ts +++ b/packages/duckdb-wasm/src/status.ts @@ -1,13 +1,13 @@ export enum StatusCode { - SUCCESS = 0, - MAX_ARROW_ERROR = 255, - DUCKDB_WASM_RETRY = 256, + SUCCESS = 0, + MAX_ARROW_ERROR = 255, + DUCKDB_WASM_RETRY = 256, } export function IsArrowBuffer(status: StatusCode): boolean { - return status <= StatusCode.MAX_ARROW_ERROR; + return status <= StatusCode.MAX_ARROW_ERROR; } export function IsDuckDBWasmRetry(status: StatusCode): boolean { - return status === StatusCode.DUCKDB_WASM_RETRY; + return status === StatusCode.DUCKDB_WASM_RETRY; } diff --git a/packages/duckdb-wasm/src/utils/s3_helper.ts b/packages/duckdb-wasm/src/utils/s3_helper.ts index aa0d908a2..72390a9e6 100644 --- a/packages/duckdb-wasm/src/utils/s3_helper.ts +++ b/packages/duckdb-wasm/src/utils/s3_helper.ts @@ -1,30 +1,30 @@ -import {S3Config} from "../bindings"; -import {sha256} from "js-sha256"; +import { S3Config } from '../bindings'; +import { sha256 } from 'js-sha256'; export interface S3Params { - url: string, - query: string, - host: string, - region: string, - service: string, - method: string, - accessKeyId: string, - secretAccessKey: string, - sessionToken: string, - dateNow: string, - datetimeNow: string + url: string; + query: string; + host: string; + region: string; + service: string; + method: string; + accessKeyId: string; + secretAccessKey: string; + sessionToken: string; + dateNow: string; + datetimeNow: string; } export interface S3PayloadParams { - contentHash: string | null, - contentType: string | null + contentHash: string | null; + contentType: string | null; } -const getHTTPHost = function (config : S3Config | undefined, url : string, bucket : string) : string { - if (config?.endpoint?.startsWith("http")) { +const getHTTPHost = function (config: S3Config | undefined, url: string, bucket: string): string { + if (config?.endpoint?.startsWith('http')) { // Endpoint is a full url, we append the bucket const httpHost = `${config?.endpoint}`; - const offset = httpHost.indexOf("://")+3; + const offset = httpHost.indexOf('://') + 3; return httpHost.substring(offset); } else if (config?.endpoint) { // Endpoint is not a full url and the https://{bucket}.{domain} format will be used @@ -33,53 +33,60 @@ const getHTTPHost = function (config : S3Config | undefined, url : string, bucke // Default aws s3 url return `${bucket}.s3.amazonaws.com`; } -} +}; -export function getS3Params (config : S3Config | undefined, url: string, method : string) : S3Params { +export function getS3Params(config: S3Config | undefined, url: string, method: string): S3Params { const parsedS3Url = parseS3Url(url); - // when using S3 path-style access, the signed URL should also include the bucket name, + // when using S3 path-style access, the signed URL should also include the bucket name, // as it is present in the HTTP URL path. // See: https://docs.aws.amazon.com/AmazonS3/latest/userguide/access-bucket-intro.html#path-style-url-ex - let path = parsedS3Url.path; + let path = parsedS3Url.path; if (isPathStyleAccess(config)) { path = `/${parsedS3Url.bucket}${path}`; } return { url: path, - query: "", + query: '', host: getHTTPHost(config, url, parsedS3Url.bucket), - region: (config?.region) ?? "", - service: "s3", + region: config?.region ?? '', + service: 's3', method: method, - accessKeyId: (config?.accessKeyId) ?? "", - secretAccessKey: (config?.secretAccessKey) ?? "", - sessionToken: (config?.sessionToken) ?? "", - dateNow: new Date().toISOString().replace(/-/g,'').split('T')[0], - datetimeNow: new Date().toISOString().replace(/-/g,'').replace(/:/g,'').split('.')[0]+ 'Z', + accessKeyId: config?.accessKeyId ?? '', + secretAccessKey: config?.secretAccessKey ?? '', + sessionToken: config?.sessionToken ?? '', + dateNow: new Date().toISOString().replace(/-/g, '').split('T')[0], + datetimeNow: new Date().toISOString().replace(/-/g, '').replace(/:/g, '').split('.')[0] + 'Z', }; } -export function uriEncode(input : string, encode_slash = false) { +export function uriEncode(input: string, encode_slash = false) { // https://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-query-string-auth.html - const hexDigit = "0123456789ABCDEF"; - let result = ""; + const hexDigit = '0123456789ABCDEF'; + let result = ''; for (let i = 0; i < input.length; i++) { - const ch : string = input[i]; - - if ((ch >= 'A' && ch <= 'Z') || (ch >= 'a' && ch <= 'z') || (ch >= '0' && ch <= '9') || ch == '_' || - ch == '-' || ch == '~' || ch == '.') { + const ch: string = input[i]; + + if ( + (ch >= 'A' && ch <= 'Z') || + (ch >= 'a' && ch <= 'z') || + (ch >= '0' && ch <= '9') || + ch == '_' || + ch == '-' || + ch == '~' || + ch == '.' + ) { result += ch; } else if (ch == '/') { if (encode_slash) { - result += "%2F"; + result += '%2F'; } else { result += ch; } } else { - result += "%"; + result += '%'; result += hexDigit[ch.charCodeAt(0) >> 4]; result += hexDigit[ch.charCodeAt(0) & 15]; } @@ -87,47 +94,57 @@ export function uriEncode(input : string, encode_slash = false) { return result; } -export function createS3Headers(params: S3Params, payloadParams : S3PayloadParams | null = null) : Map { +export function createS3Headers(params: S3Params, payloadParams: S3PayloadParams | null = null): Map { // this is the sha256 of the empty string, its useful since we have no payload for GET requests - const payloadHash = (payloadParams?.contentHash) ?? "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"; + const payloadHash = + payloadParams?.contentHash ?? 'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855'; const res = new Map(); // res.set("host", params.host) - res.set("x-amz-date", params.datetimeNow); - res.set("x-amz-content-sha256", payloadHash); + res.set('x-amz-date', params.datetimeNow); + res.set('x-amz-content-sha256', payloadHash); if (params.sessionToken) { - res.set("x-amz-security-token", params.sessionToken); + res.set('x-amz-security-token', params.sessionToken); } // construct string to sign - let signedHeaders = ""; + let signedHeaders = ''; if (payloadParams?.contentType) { - signedHeaders += "content-type;"; + signedHeaders += 'content-type;'; } - signedHeaders += "host;x-amz-content-sha256;x-amz-date"; + signedHeaders += 'host;x-amz-content-sha256;x-amz-date'; if (params.sessionToken) { - signedHeaders += ";x-amz-security-token"; + signedHeaders += ';x-amz-security-token'; } - let canonicalRequest = params.method + "\n" + uriEncode(params.url) + "\n" + params.query; + let canonicalRequest = params.method + '\n' + uriEncode(params.url) + '\n' + params.query; if (payloadParams?.contentType) { - canonicalRequest += "\ncontent-type:" + payloadParams?.contentType; + canonicalRequest += '\ncontent-type:' + payloadParams?.contentType; } - canonicalRequest += "\nhost:" + params.host + - "\nx-amz-content-sha256:" + payloadHash + "\nx-amz-date:" + params.datetimeNow; + canonicalRequest += + '\nhost:' + params.host + '\nx-amz-content-sha256:' + payloadHash + '\nx-amz-date:' + params.datetimeNow; if (params.sessionToken && params.sessionToken.length > 0) { - canonicalRequest += "\nx-amz-security-token:" + params.sessionToken; + canonicalRequest += '\nx-amz-security-token:' + params.sessionToken; } - canonicalRequest += "\n\n" + signedHeaders + "\n" + payloadHash; + canonicalRequest += '\n\n' + signedHeaders + '\n' + payloadHash; const canonicalRequestHashStr = sha256(canonicalRequest); - const stringToSign = "AWS4-HMAC-SHA256\n" + params.datetimeNow + "\n" + params.dateNow + "/" + params.region + "/" + params.service + - "/aws4_request\n" + canonicalRequestHashStr; + const stringToSign = + 'AWS4-HMAC-SHA256\n' + + params.datetimeNow + + '\n' + + params.dateNow + + '/' + + params.region + + '/' + + params.service + + '/aws4_request\n' + + canonicalRequestHashStr; // ts-ignore's because library can accept array buffer as key, but TS arg is incorrect - const signKey = "AWS4" + params.secretAccessKey; + const signKey = 'AWS4' + params.secretAccessKey; const kDate = sha256.hmac.arrayBuffer(signKey, params.dateNow); // Note, js-sha256 has a bug in the TS interface that only supports strings as keys, while we need a bytearray @@ -137,31 +154,56 @@ export function createS3Headers(params: S3Params, payloadParams : S3PayloadParam const kRegion = sha256.hmac.arrayBuffer(kDate, params.region); // eslint-disable-next-line // @ts-ignore - const kService = sha256.hmac.arrayBuffer(kRegion, params.service,); + const kService = sha256.hmac.arrayBuffer(kRegion, params.service); // eslint-disable-next-line // @ts-ignore - const signingKey = sha256.hmac.arrayBuffer(kService, "aws4_request"); + const signingKey = sha256.hmac.arrayBuffer(kService, 'aws4_request'); // eslint-disable-next-line // @ts-ignore const signature = sha256.hmac(signingKey, stringToSign); - res.set("Authorization", "AWS4-HMAC-SHA256 Credential=" + params.accessKeyId + "/" + params.dateNow + "/" + params.region + "/" + - params.service + "/aws4_request, SignedHeaders=" + signedHeaders + - ", Signature=" + signature); + res.set( + 'Authorization', + 'AWS4-HMAC-SHA256 Credential=' + + params.accessKeyId + + '/' + + params.dateNow + + '/' + + params.region + + '/' + + params.service + + '/aws4_request, SignedHeaders=' + + signedHeaders + + ', Signature=' + + signature, + ); return res; } -const createS3HeadersFromS3Config = function (config : S3Config | undefined, url : string, method : string, contentType: string | null = null, payload : Uint8Array | null = null) : Map { +const createS3HeadersFromS3Config = function ( + config: S3Config | undefined, + url: string, + method: string, + contentType: string | null = null, + payload: Uint8Array | null = null, +): Map { const params = getS3Params(config, url, method); const payloadParams = { contentType: contentType, - contentHash: payload ? sha256.hex(payload!) : null + contentHash: payload ? sha256.hex(payload!) : null, } as S3PayloadParams; return createS3Headers(params, payloadParams); -} +}; -export function addS3Headers(xhr: XMLHttpRequest, config : S3Config | undefined, url : string, method: string, contentType: string | null = null, payload : Uint8Array | null = null) { +export function addS3Headers( + xhr: XMLHttpRequest, + config: S3Config | undefined, + url: string, + method: string, + contentType: string | null = null, + payload: Uint8Array | null = null, +) { if (config?.accessKeyId || config?.sessionToken) { const headers = createS3HeadersFromS3Config(config, url, method, contentType, payload); headers.forEach((value: string, header: string) => { @@ -169,14 +211,14 @@ export function addS3Headers(xhr: XMLHttpRequest, config : S3Config | undefined, }); if (contentType) { - xhr.setRequestHeader("content-type", contentType); + xhr.setRequestHeader('content-type', contentType); } } } -export function parseS3Url (url: string) : {bucket : string, path : string} { - if (url.indexOf("s3://") != 0) { - throw new Error("URL needs to start with s3://"); +export function parseS3Url(url: string): { bucket: string; path: string } { + if (url.indexOf('s3://') != 0) { + throw new Error('URL needs to start with s3://'); } const slashPos = url.indexOf('/', 5); @@ -186,28 +228,28 @@ export function parseS3Url (url: string) : {bucket : string, path : string} { const bucket = url.substring(5, slashPos); if (!bucket) { - throw new Error("URL needs to contain a bucket name"); + throw new Error('URL needs to contain a bucket name'); } const path = url.substring(slashPos); if (!path) { - throw new Error("URL needs to contain key"); + throw new Error('URL needs to contain key'); } - return {bucket: bucket, path: path} + return { bucket: bucket, path: path }; } -function isPathStyleAccess(config : S3Config | undefined) : boolean { - if (config?.endpoint?.startsWith("http")) { - return true +function isPathStyleAccess(config: S3Config | undefined): boolean { + if (config?.endpoint?.startsWith('http')) { + return true; } - return false + return false; } -export function getHTTPUrl(config : S3Config | undefined, url : string) : string { +export function getHTTPUrl(config: S3Config | undefined, url: string): string { const parsedUrl = parseS3Url(url); if (isPathStyleAccess(config)) { // Endpoint is a full url, we append the bucket return `${config?.endpoint}/${parsedUrl.bucket}` + parsedUrl.path; } return 'https://' + getHTTPHost(config, url, parsedUrl.bucket) + parsedUrl.path; -} \ No newline at end of file +} diff --git a/packages/duckdb-wasm/test/bindings.test.ts b/packages/duckdb-wasm/test/bindings.test.ts index e016ba13e..97a4447e0 100644 --- a/packages/duckdb-wasm/test/bindings.test.ts +++ b/packages/duckdb-wasm/test/bindings.test.ts @@ -45,12 +45,10 @@ export function testBindings(db: () => duckdb.DuckDBBindings, baseURL: string): it('Platform check', async () => { await db().reset(); conn = db().connect(); - const version = conn.query<{ name: arrow.Utf8 }>( - "PRAGMA platform;", - ); + const version = conn.query<{ name: arrow.Utf8 }>('PRAGMA platform;'); const rows = version.getChildAt(0)?.toArray(); expect(rows.length).toEqual(1); - expect(rows[0].toString().substr(0,5)).toEqual("wasm_"); + expect(rows[0].toString().substr(0, 5)).toEqual('wasm_'); await db().reset(); }); }); diff --git a/packages/duckdb-wasm/test/excel.test.ts b/packages/duckdb-wasm/test/excel.test.ts index 14e266f07..d293c0e05 100644 --- a/packages/duckdb-wasm/test/excel.test.ts +++ b/packages/duckdb-wasm/test/excel.test.ts @@ -14,15 +14,30 @@ export function testEXCEL(db: () => duckdb.DuckDBBindings): void { describe('EXCEL', () => { it('sample', async () => { - expect(conn.query("SELECT text(1234567.897, 'h:mm:ss.00')",).getChildAt(0)?.toArray()).toEqual(['21:31:40.80']); - expect(conn.query("SELECT text(1234567.897, 'm/d/yyyy h:mm AM/PM')",).getChildAt(0)?.toArray()).toEqual(['2/15/5280 9:31 PM']); - expect(conn.query("SELECT text(1234567.897, 'dddd, dd of MMMM of YYYY')",).getChildAt(0)?.toArray()).toEqual(['Thursday, 15 of February of 5280']); + expect(conn.query("SELECT text(1234567.897, 'h:mm:ss.00')").getChildAt(0)?.toArray()).toEqual([ + '21:31:40.80', + ]); + expect(conn.query("SELECT text(1234567.897, 'm/d/yyyy h:mm AM/PM')").getChildAt(0)?.toArray()).toEqual([ + '2/15/5280 9:31 PM', + ]); + expect(conn.query("SELECT text(1234567.897, 'dddd, dd of MMMM of YYYY')").getChildAt(0)?.toArray()).toEqual( + ['Thursday, 15 of February of 5280'], + ); - expect(conn.query("SELECT text(1234567.897, '# ??/??')",).getChildAt(0)?.toArray()).toEqual(['1234567 61/68']); + expect(conn.query("SELECT text(1234567.897, '# ??/??')").getChildAt(0)?.toArray()).toEqual([ + '1234567 61/68', + ]); - expect(conn.query("SELECT text(12345678912, '(###) ###-####')",).getChildAt(0)?.toArray()).toEqual(['(1234) 567-8912']); - expect(conn.query("SELECT text(1234567.897, '$#,##0')",).getChildAt(0)?.toArray()).toEqual(['$1,234,568']); - expect(conn.query("SELECT excel_text(123456789123, '[<=9999999]##-####;[>9999999](###) ###-####')",).getChildAt(0)?.toArray()).toEqual(['(12345) 678-9123']); + expect(conn.query("SELECT text(12345678912, '(###) ###-####')").getChildAt(0)?.toArray()).toEqual([ + '(1234) 567-8912', + ]); + expect(conn.query("SELECT text(1234567.897, '$#,##0')").getChildAt(0)?.toArray()).toEqual(['$1,234,568']); + expect( + conn + .query("SELECT excel_text(123456789123, '[<=9999999]##-####;[>9999999](###) ###-####')") + .getChildAt(0) + ?.toArray(), + ).toEqual(['(12345) 678-9123']); }); }); } diff --git a/packages/duckdb-wasm/test/httpfs_test.ts b/packages/duckdb-wasm/test/httpfs_test.ts index 6e2f719cd..9b9617ceb 100644 --- a/packages/duckdb-wasm/test/httpfs_test.ts +++ b/packages/duckdb-wasm/test/httpfs_test.ts @@ -2,7 +2,7 @@ import * as duckdb from '../src/'; import { getS3Params, S3Params, S3PayloadParams, createS3Headers, uriEncode, getHTTPUrl } from '../src/utils'; import { AsyncDuckDBConnection, DuckDBBindings, DuckDBBindingsBase, DuckDBModule } from '../src/'; import BROWSER_RUNTIME from '../src/bindings/runtime_browser'; -import {generateLongQueryString} from "./string_test_helper"; +import { generateLongQueryString } from './string_test_helper'; // S3 config for tests const BUCKET_NAME = 'test-bucket'; @@ -312,7 +312,7 @@ export function testHTTPFSAsync( `COPY (SELECT * FROM range(1000,1010) tbl(i)) TO 's3://${BUCKET_NAME}/test_written.csv' (FORMAT 'csv');`, ); const result = await conn!.query(`SELECT * FROM "s3://${BUCKET_NAME}/test_written.csv";`); - expect(Number((result.getChildAt(0)?.get(6)))).toEqual(Number(1006)); + expect(Number(result.getChildAt(0)?.get(6))).toEqual(Number(1006)); await expectAsync( conn!.query( `COPY (SELECT * FROM range(2000,2010) tbl(i)) TO 's3://${BUCKET_NAME}/test_written.csv' (FORMAT 'csv');`, @@ -330,7 +330,7 @@ export function testHTTPFSAsync( const result = await conn!.query( `SELECT * FROM "${S3_ENDPOINT}/${BUCKET_NAME}/correct_auth_test.parquet?${queryString}";`, ); - expect(Number((result.getChildAt(0)?.get(6)))).toEqual(Number(29120)); + expect(Number(result.getChildAt(0)?.get(6))).toEqual(Number(29120)); }); it('can read csv file from URL with long query string', async () => { @@ -343,7 +343,7 @@ export function testHTTPFSAsync( const result = await conn!.query( `SELECT * FROM "${S3_ENDPOINT}/${BUCKET_NAME}/correct_auth_test.csv?${queryString}";`, ); - expect(Number((result.getChildAt(0)?.get(6)))).toEqual(Number(29120)); + expect(Number(result.getChildAt(0)?.get(6))).toEqual(Number(29120)); }); }); } diff --git a/packages/duckdb-wasm/test/json.test.ts b/packages/duckdb-wasm/test/json.test.ts index 7e7cacfb1..46d115903 100644 --- a/packages/duckdb-wasm/test/json.test.ts +++ b/packages/duckdb-wasm/test/json.test.ts @@ -14,8 +14,8 @@ export function testJSON(db: () => duckdb.DuckDBBindings): void { describe('JSON', () => { it('sample', async () => { - expect(conn.query("select to_json({n: 42})",).getChildAt(0)?.toArray()).toEqual(['{"n":42}']); - expect(conn.query("select json_object('duck', 42)",).getChildAt(0)?.toArray()).toEqual(['{"duck":42}']); + expect(conn.query('select to_json({n: 42})').getChildAt(0)?.toArray()).toEqual(['{"n":42}']); + expect(conn.query("select json_object('duck', 42)").getChildAt(0)?.toArray()).toEqual(['{"duck":42}']); }); }); } diff --git a/packages/duckdb-wasm/test/long_queries.test.ts b/packages/duckdb-wasm/test/long_queries.test.ts index 5ff02daf6..87729d022 100644 --- a/packages/duckdb-wasm/test/long_queries.test.ts +++ b/packages/duckdb-wasm/test/long_queries.test.ts @@ -26,14 +26,13 @@ export function longQueries(db: () => duckdb.AsyncDuckDB): void { let str = `with big_expr as ( select `; let i = 1; - while (str.length < 1e6) { + while (str.length < 1e6) { str += ` ` + i + ` as col_` + i + `,`; i++; } - str += ` NULL as col_NULL) select 99;` + str += ` NULL as col_NULL) select 99;`; await conn.query(str); }); }); } - diff --git a/packages/duckdb-wasm/test/opfs.test.ts b/packages/duckdb-wasm/test/opfs.test.ts index eaf1a0fcc..f4dfb4e02 100644 --- a/packages/duckdb-wasm/test/opfs.test.ts +++ b/packages/duckdb-wasm/test/opfs.test.ts @@ -1,5 +1,5 @@ import * as duckdb from '../src/'; -import {LogLevel} from '../src/'; +import { LogLevel } from '../src/'; import * as arrow from 'apache-arrow'; export function testOPFS(baseDir: string, bundle: () => duckdb.DuckDBBundle): void { @@ -29,7 +29,7 @@ export function testOPFS(baseDir: string, bundle: () => duckdb.DuckDBBundle): vo await db.instantiate(bundle().mainModule, bundle().pthreadWorker); await db.open({ path: 'opfs://test.db', - accessMode: duckdb.DuckDBAccessMode.READ_WRITE + accessMode: duckdb.DuckDBAccessMode.READ_WRITE, }); conn = await db.connect(); }); @@ -83,7 +83,7 @@ export function testOPFS(baseDir: string, bundle: () => duckdb.DuckDBBundle): vo await db.instantiate(bundle().mainModule, bundle().pthreadWorker); await db.open({ path: 'opfs://test.db', - accessMode: duckdb.DuckDBAccessMode.READ_WRITE + accessMode: duckdb.DuckDBAccessMode.READ_WRITE, }); conn = await db.connect(); @@ -102,7 +102,7 @@ export function testOPFS(baseDir: string, bundle: () => duckdb.DuckDBBundle): vo res.arrayBuffer(), ); const opfsRoot = await navigator.storage.getDirectory(); - const fileHandle = await opfsRoot.getFileHandle('test.parquet', {create: true}); + const fileHandle = await opfsRoot.getFileHandle('test.parquet', { create: true }); const writable = await fileHandle.createWritable(); await writable.write(parquetBuffer); await writable.close(); @@ -126,8 +126,8 @@ export function testOPFS(baseDir: string, bundle: () => duckdb.DuckDBBundle): vo res.arrayBuffer(), ); const opfsRoot = await navigator.storage.getDirectory(); - const datadir = await opfsRoot.getDirectoryHandle("datadir", {create: true}); - const fileHandle = await datadir.getFileHandle('test.parquet', {create: true}); + const datadir = await opfsRoot.getDirectoryHandle('datadir', { create: true }); + const fileHandle = await datadir.getFileHandle('test.parquet', { create: true }); const writable = await fileHandle.createWritable(); await writable.write(parquetBuffer); await writable.close(); @@ -150,7 +150,7 @@ export function testOPFS(baseDir: string, bundle: () => duckdb.DuckDBBundle): vo res.arrayBuffer(), ); const opfsRoot = await navigator.storage.getDirectory(); - const fileHandle = await opfsRoot.getFileHandle('test.parquet', {create: true}); + const fileHandle = await opfsRoot.getFileHandle('test.parquet', { create: true }); const writable = await fileHandle.createWritable(); await writable.write(parquetBuffer); await writable.close(); @@ -192,12 +192,11 @@ export function testOPFS(baseDir: string, bundle: () => duckdb.DuckDBBundle): vo const table3 = await new arrow.Table<{ cnt: arrow.Int }>(batches3); expect(table3.getChildAt(0)?.get(0)).toBeGreaterThan(60_000); } - }); it('Drop File + Export as CSV to OPFS + Load CSV', async () => { const opfsRoot = await navigator.storage.getDirectory(); - const testHandle = await opfsRoot.getFileHandle('test.csv', {create: true}); + const testHandle = await opfsRoot.getFileHandle('test.csv', { create: true }); await db.registerFileHandle('test.csv', testHandle, duckdb.DuckDBDataProtocol.BROWSER_FSACCESS, true); await conn.send(`CREATE TABLE zzz AS SELECT * FROM "${baseDir}/tpch/0_01/parquet/lineitem.parquet"`); await conn.send(`COPY (SELECT * FROM zzz) TO 'test.csv'`); @@ -221,12 +220,11 @@ export function testOPFS(baseDir: string, bundle: () => duckdb.DuckDBBundle): vo await db.dropFile('test.csv'); }); - it('Drop Files + Export as CSV to OPFS + Load CSV', async () => { const opfsRoot = await navigator.storage.getDirectory(); - const testHandle1 = await opfsRoot.getFileHandle('test1.csv', {create: true}); - const testHandle2 = await opfsRoot.getFileHandle('test2.csv', {create: true}); - const testHandle3 = await opfsRoot.getFileHandle('test3.csv', {create: true}); + const testHandle1 = await opfsRoot.getFileHandle('test1.csv', { create: true }); + const testHandle2 = await opfsRoot.getFileHandle('test2.csv', { create: true }); + const testHandle3 = await opfsRoot.getFileHandle('test3.csv', { create: true }); await db.registerFileHandle('test1.csv', testHandle1, duckdb.DuckDBDataProtocol.BROWSER_FSACCESS, true); await db.registerFileHandle('test2.csv', testHandle2, duckdb.DuckDBDataProtocol.BROWSER_FSACCESS, true); await db.registerFileHandle('test3.csv', testHandle3, duckdb.DuckDBDataProtocol.BROWSER_FSACCESS, true); @@ -280,28 +278,19 @@ export function testOPFS(baseDir: string, bundle: () => duckdb.DuckDBBundle): vo async function removeFiles() { const opfsRoot = await navigator.storage.getDirectory(); - await opfsRoot.removeEntry('test.db').catch(() => { - }); - await opfsRoot.removeEntry('test.db.wal').catch(() => { - }); - await opfsRoot.removeEntry('test.csv').catch(() => { - }); - await opfsRoot.removeEntry('test1.csv').catch(() => { - }); - await opfsRoot.removeEntry('test2.csv').catch(() => { - }); - await opfsRoot.removeEntry('test3.csv').catch(() => { - }); - await opfsRoot.removeEntry('test.parquet').catch(() => { - }); + await opfsRoot.removeEntry('test.db').catch(() => {}); + await opfsRoot.removeEntry('test.db.wal').catch(() => {}); + await opfsRoot.removeEntry('test.csv').catch(() => {}); + await opfsRoot.removeEntry('test1.csv').catch(() => {}); + await opfsRoot.removeEntry('test2.csv').catch(() => {}); + await opfsRoot.removeEntry('test3.csv').catch(() => {}); + await opfsRoot.removeEntry('test.parquet').catch(() => {}); try { const datadir = await opfsRoot.getDirectoryHandle('datadir'); - datadir.removeEntry('test.parquet').catch(() => { - }); + datadir.removeEntry('test.parquet').catch(() => {}); } catch (e) { // } - await opfsRoot.removeEntry('datadir').catch(() => { - }); + await opfsRoot.removeEntry('datadir').catch(() => {}); } } diff --git a/packages/duckdb-wasm/test/regression/github_1833.test.ts b/packages/duckdb-wasm/test/regression/github_1833.test.ts index 4e597ed2c..ca89e0497 100644 --- a/packages/duckdb-wasm/test/regression/github_1833.test.ts +++ b/packages/duckdb-wasm/test/regression/github_1833.test.ts @@ -14,17 +14,17 @@ export function test1833(db: () => duckdb.AsyncDuckDB): void { }); describe('GitHub issues', () => { it('1833', async () => { - await conn.query(` + await conn.query(` CREATE TABLE "Test" (value VARCHAR) `); - const stmt = await conn.prepare(` + const stmt = await conn.prepare(` INSERT INTO "Test" (value) VALUES (?) `); - await stmt.query('🦆🦆🦆🦆🦆'); - await stmt.query('goo␀se'); - await stmt.query('goo\u0000se'); - const result = await conn.query(` + await stmt.query('🦆🦆🦆🦆🦆'); + await stmt.query('goo␀se'); + await stmt.query('goo\u0000se'); + const result = await conn.query(` SELECT * FROM "Test" `); expect(result.schema.fields.length).toBe(1); diff --git a/packages/duckdb-wasm/test/string_test_helper.ts b/packages/duckdb-wasm/test/string_test_helper.ts index 96e83b823..96078f0d4 100644 --- a/packages/duckdb-wasm/test/string_test_helper.ts +++ b/packages/duckdb-wasm/test/string_test_helper.ts @@ -5,16 +5,18 @@ export function generateLongQueryString(): string { const eee = repeatCharacter('E', 256); const ggg = repeatCharacter('G', 128); - return `test=inline` + + return ( + `test=inline` + `&Test-Security-Token=${aaa}` + `&Test-Algorithm=${ccc}` + `&Test-Date=${ddd}` + `&Test-SignedHeaders=host` + `&Test-Expires=43200` + `&Test-Credential=${eee}` + - `&Test-Signature=${ggg}`; + `&Test-Signature=${ggg}` + ); } export function repeatCharacter(char: string, length: number): string { return char.repeat(length); -} \ No newline at end of file +} From 6e6bb138f0ff022e5e57fa310c4407d2aecf7324 Mon Sep 17 00:00:00 2001 From: Carlo Piovesan Date: Mon, 12 May 2025 13:13:38 +0200 Subject: [PATCH 11/28] Pass EXPORTED_FUNCTIONS as external file --- lib/CMakeLists.txt | 59 +++++++--------------------------------------- 1 file changed, 8 insertions(+), 51 deletions(-) diff --git a/lib/CMakeLists.txt b/lib/CMakeLists.txt index f7df08312..cc9450cbd 100644 --- a/lib/CMakeLists.txt +++ b/lib/CMakeLists.txt @@ -150,11 +150,17 @@ else() endif() if (DUCKDB_WASM_LOADABLE_EXTENSIONS) - set(WASM_LINK_FLAGS "${WASM_LINK_FLAGS} -s MAIN_MODULE=1 -s FILESYSTEM=1 -s ENVIRONMENT='web,node,worker' -s ALLOW_TABLE_GROWTH -lembind") + set(WASM_LINK_FLAGS "${WASM_LINK_FLAGS} -s FILESYSTEM=1 -s ENVIRONMENT='web,node,worker' -s ALLOW_TABLE_GROWTH -lembind") else() set(WASM_LINK_FLAGS "${WASM_LINK_FLAGS} -s FILESYSTEM=0 -s ENVIRONMENT='web,node,worker'") endif() +if (FINAL_BUILD) + set(WASM_LINK_FLAGS "${WASM_LINK_FLAGS} -s MAIN_MODULE=2 -s EXPORTED_FUNCTIONS='@exported_list.txt'") +else() + set(WASM_LINK_FLAGS "${WASM_LINK_FLAGS} -s MAIN_MODULE=1") +endif() + # --------------------------------------------------------------------------- # Parallelism @@ -287,56 +293,7 @@ if(EMSCRIPTEN) -s MAXIMUM_MEMORY=4GB \ -s MODULARIZE=1 \ -s EXPORT_NAME='DuckDB' \ - -s EXPORTED_FUNCTIONS='[ \ - _main, \ - _malloc, \ - _calloc, \ - _free, \ - _duckdb_web_clear_response, \ - _duckdb_web_collect_file_stats, \ - _duckdb_web_connect, \ - _duckdb_web_copy_file_to_buffer, \ - _duckdb_web_copy_file_to_path, \ - _duckdb_web_disconnect, \ - _duckdb_web_export_file_stats, \ - _duckdb_web_fail_with, \ - _duckdb_web_flush_file, \ - _duckdb_web_flush_files, \ - _duckdb_web_fs_drop_file, \ - _duckdb_web_fs_drop_files, \ - _duckdb_web_fs_get_file_info_by_id, \ - _duckdb_web_fs_get_file_info_by_name, \ - _duckdb_web_fs_glob_add_path, \ - _duckdb_web_fs_glob_file_infos, \ - _duckdb_web_fs_register_file_buffer, \ - _duckdb_web_fs_register_file_url, \ - _duckdb_web_get_feature_flags, \ - _duckdb_web_get_global_file_info, \ - _duckdb_web_get_tablenames, \ - _duckdb_web_get_tablenames_buffer, \ - _duckdb_web_get_version, \ - _duckdb_web_insert_arrow_from_ipc_stream, \ - _duckdb_web_insert_csv_from_path, \ - _duckdb_web_insert_json_from_path, \ - _duckdb_web_open, \ - _duckdb_web_pending_query_cancel, \ - _duckdb_web_pending_query_poll, \ - _duckdb_web_pending_query_start, \ - _duckdb_web_pending_query_start_buffer, \ - _duckdb_web_prepared_close, \ - _duckdb_web_prepared_create, \ - _duckdb_web_prepared_create_buffer, \ - _duckdb_web_prepared_run, \ - _duckdb_web_prepared_send, \ - _duckdb_web_query_fetch_results, \ - _duckdb_web_query_run, \ - _duckdb_web_query_run_buffer, \ - _duckdb_web_reset, \ - _duckdb_web_tokenize, \ - _duckdb_web_tokenize_buffer, \ - _duckdb_web_udf_scalar_create \ - ]' \ - -s EXPORTED_RUNTIME_METHODS='[\"ccall\", \"stackSave\", \"stackAlloc\", \"stackRestore\"]' \ + -s EXPORTED_RUNTIME_METHODS='[\"ccall\", \"stackSave\", \"stackAlloc\", \"stackRestore\", \"createDyncallWrapper\"]' \ --js-library=${CMAKE_SOURCE_DIR}/js-stubs.js") endif() From 31464a3ae6f03b410ed9ed7fdc417d3fb131f8b6 Mon Sep 17 00:00:00 2001 From: Carlo Piovesan Date: Mon, 12 May 2025 13:17:00 +0200 Subject: [PATCH 12/28] Add Makefile step to generate hacky exported list --- Makefile | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/Makefile b/Makefile index 37fc91c30..126ae8ea6 100644 --- a/Makefile +++ b/Makefile @@ -448,3 +448,14 @@ submodules: build/bootstrap: submodules yarn_install mkdir -p build touch build/bootstrap + +update_exported_list: + cd build/relsize/${TARGET} && wasm2wat duckdb_wasm.wasm --enable-all -o duckdb-wasm.wat + cd build/relsize/${TARGET} && grep export duckdb-wasm.wat | cut -d \" -f2 | sed '$d' | grep -v "^orig" | grep -v "^dynCall_" > export_list.txt + ## filter list of c++ symbols + cd build/relsize/${TARGET} && cat export_list.txt | grep "^_" | grep -v "_Unwind_" | grep -v "__syscall_shutdown" | grep -v "0\\00\\0" | grep -v "^_ZZN5arrow" | grep -v "^_ZGVZN5arrow" | grep -v "^_ZN5arrow" | sort > cpp_list + cd build/relsize/${TARGET} && sed 's/^/_/g' cpp_list > exported_list.txt + ## filter list of c symbols + cd build/relsize/${TARGET} && cat export_list.txt | grep -v "^_" | grep -v "getTempRet" | grep -v "^sched_yield" | grep -v "emscripten_wget" | grep -v "0\\00\\0" | sort > c_exported_list + # prepend '_' + cd build/relsize/${TARGET} && sed 's/^/_/g' c_exported_list >> exported_list.txt From c8ce6a5c0c59419b1ea8ef30b288bad66db13aed Mon Sep 17 00:00:00 2001 From: Carlo Piovesan Date: Mon, 12 May 2025 13:19:59 +0200 Subject: [PATCH 13/28] Conditionally on USE_GENERATED_EXPORTED_LIST, use just generated list --- lib/CMakeLists.txt | 2 +- scripts/wasm_build_lib.sh | 21 +++++++++++++++++++++ 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/lib/CMakeLists.txt b/lib/CMakeLists.txt index cc9450cbd..cbc3367bf 100644 --- a/lib/CMakeLists.txt +++ b/lib/CMakeLists.txt @@ -155,7 +155,7 @@ else() set(WASM_LINK_FLAGS "${WASM_LINK_FLAGS} -s FILESYSTEM=0 -s ENVIRONMENT='web,node,worker'") endif() -if (FINAL_BUILD) +if (USE_GENERATED_EXPORTED_LIST) set(WASM_LINK_FLAGS "${WASM_LINK_FLAGS} -s MAIN_MODULE=2 -s EXPORTED_FUNCTIONS='@exported_list.txt'") else() set(WASM_LINK_FLAGS "${WASM_LINK_FLAGS} -s MAIN_MODULE=1") diff --git a/scripts/wasm_build_lib.sh b/scripts/wasm_build_lib.sh index cf2b268ad..a3e4f43e6 100755 --- a/scripts/wasm_build_lib.sh +++ b/scripts/wasm_build_lib.sh @@ -69,6 +69,27 @@ emmake make \ -j${CORES} \ duckdb_wasm +make update_exported_list + +if [ "$USE_GENERATED_EXPORTED_LIST" == "yes" ]; then +emcmake cmake \ + -S${CPP_SOURCE_DIR} \ + -B${BUILD_DIR} \ + -DDUCKDB_WASM_VERSION=${DUCKDB_WASM_VERSION_NAME} \ + -DCMAKE_C_COMPILER_LAUNCHER=ccache \ + -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \ + -DDUCKDB_LOCATION=${DUCKDB_LOCATION} \ + -DWASM_LINK_FLAGS_EXT="${LINK_FLAGS}" \ + -DDUCKDB_EXTENSION_CONFIGS=extension_config_wasm.cmake \ + -DUSE_GENERATED_EXPORTED_LIST=1 \ + ${ADDITIONAL_FLAGS} + +emmake make \ + -C${BUILD_DIR} \ + -j${CORES} \ + duckdb_wasm +fi + npm install -g js-beautify js-beautify ${BUILD_DIR}/duckdb_wasm.js > ${BUILD_DIR}/beauty.js awk '{gsub(/get\(stubs, prop\) \{/,"get(stubs,prop) { if (prop.startsWith(\"invoke_\")) {return createDyncallWrapper(prop.substring(7));}"); print}' ${BUILD_DIR}/beauty.js > ${BUILD_DIR}/beauty2.js From 56215625192386a8dcee1859ba2b83e9124d569a Mon Sep 17 00:00:00 2001 From: Carlo Piovesan Date: Mon, 12 May 2025 13:22:17 +0200 Subject: [PATCH 14/28] Use generated export list by default --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 126ae8ea6..09baae69f 100644 --- a/Makefile +++ b/Makefile @@ -357,7 +357,7 @@ app: wasm wasmpack shell docs js_tests_release yarn workspace @duckdb/duckdb-wasm-app build:release build_loadable: - DUCKDB_PLATFORM=wasm_${TARGET} DUCKDB_WASM_LOADABLE_EXTENSIONS=1 GEN=ninja ./scripts/wasm_build_lib.sh relsize ${TARGET} + USE_GENERATED_EXPORTED_LIST=1 DUCKDB_PLATFORM=wasm_${TARGET} DUCKDB_WASM_LOADABLE_EXTENSIONS=1 GEN=ninja ./scripts/wasm_build_lib.sh relsize ${TARGET} build_loadable_unsigned: build_loadable # need to propagate the unsigned flag From 356e15ad4f0b01930956208cfa287cd955285ac9 Mon Sep 17 00:00:00 2001 From: Carlo Piovesan Date: Mon, 12 May 2025 13:52:12 +0200 Subject: [PATCH 15/28] Fixup parameter passing --- .github/workflows/main.yml | 24 ++++++++++++++++++++++++ Makefile | 2 +- scripts/wasm_build_lib.sh | 4 ++-- 3 files changed, 27 insertions(+), 3 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index c9d216edd..550514dc8 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -364,6 +364,10 @@ jobs: submodules: 'recursive' fetch-depth: 0 + - name: Install dependencies + run: | + sudo apt install wabt + - name: Prepare repository run: | make apply_patches @@ -407,6 +411,10 @@ jobs: submodules: 'recursive' fetch-depth: 0 + - name: Install dependencies + run: | + sudo apt install wabt + - name: Prepare repository run: | make apply_patches @@ -450,6 +458,10 @@ jobs: submodules: 'recursive' fetch-depth: 0 + - name: Install dependencies + run: | + sudo apt install wabt + - name: Prepare repository run: | make apply_patches @@ -494,6 +506,10 @@ jobs: submodules: 'recursive' fetch-depth: 0 + - name: Install dependencies + run: | + sudo apt install wabt + - name: Prepare repository run: | make apply_patches @@ -537,6 +553,10 @@ jobs: submodules: 'recursive' fetch-depth: 0 + - name: Install dependencies + run: | + sudo apt install wabt + - name: Prepare repository run: | make apply_patches @@ -580,6 +600,10 @@ jobs: submodules: 'recursive' fetch-depth: 0 + - name: Install dependencies + run: | + sudo apt install wabt + - name: Prepare repository run: | make apply_patches diff --git a/Makefile b/Makefile index 09baae69f..efc96932c 100644 --- a/Makefile +++ b/Makefile @@ -357,7 +357,7 @@ app: wasm wasmpack shell docs js_tests_release yarn workspace @duckdb/duckdb-wasm-app build:release build_loadable: - USE_GENERATED_EXPORTED_LIST=1 DUCKDB_PLATFORM=wasm_${TARGET} DUCKDB_WASM_LOADABLE_EXTENSIONS=1 GEN=ninja ./scripts/wasm_build_lib.sh relsize ${TARGET} + USE_GENERATED_EXPORTED_LIST=yes DUCKDB_PLATFORM=wasm_${TARGET} DUCKDB_WASM_LOADABLE_EXTENSIONS=1 GEN=ninja ./scripts/wasm_build_lib.sh relsize ${TARGET} build_loadable_unsigned: build_loadable # need to propagate the unsigned flag diff --git a/scripts/wasm_build_lib.sh b/scripts/wasm_build_lib.sh index a3e4f43e6..88b343b86 100755 --- a/scripts/wasm_build_lib.sh +++ b/scripts/wasm_build_lib.sh @@ -69,7 +69,7 @@ emmake make \ -j${CORES} \ duckdb_wasm -make update_exported_list +TARGET=${FEATURES} make update_exported_list if [ "$USE_GENERATED_EXPORTED_LIST" == "yes" ]; then emcmake cmake \ @@ -90,7 +90,7 @@ emmake make \ duckdb_wasm fi -npm install -g js-beautify +js-beautify -v || npm install -g js-beautify js-beautify ${BUILD_DIR}/duckdb_wasm.js > ${BUILD_DIR}/beauty.js awk '{gsub(/get\(stubs, prop\) \{/,"get(stubs,prop) { if (prop.startsWith(\"invoke_\")) {return createDyncallWrapper(prop.substring(7));}"); print}' ${BUILD_DIR}/beauty.js > ${BUILD_DIR}/beauty2.js awk '!(/var .*wasmExports\[/ || /var [_a-z0-9A-Z]+ = Module\[\"[_a-z0-9A-Z]+\"\] = [0-9]+;/) || /var _duckdb_web/ || /var _main/ || /var _calloc/ || /var _malloc/ || /var _free/ || /var stack/ || /var ___dl_seterr/ || /var __em/ || /var _em/ || /var _pthread/' ${BUILD_DIR}/beauty2.js > ${BUILD_DIR}/duckdb_wasm.js From 79e279e8b9616803208f6578e09aeac922f2bd7a Mon Sep 17 00:00:00 2001 From: Carlo Piovesan Date: Mon, 12 May 2025 14:17:21 +0200 Subject: [PATCH 16/28] try this --- scripts/wasm_build_lib.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/wasm_build_lib.sh b/scripts/wasm_build_lib.sh index 88b343b86..9a95b31a8 100755 --- a/scripts/wasm_build_lib.sh +++ b/scripts/wasm_build_lib.sh @@ -71,7 +71,7 @@ emmake make \ TARGET=${FEATURES} make update_exported_list -if [ "$USE_GENERATED_EXPORTED_LIST" == "yes" ]; then +if [ "${USE_GENERATED_EXPORTED_LIST}-no" == "yes" ]; then emcmake cmake \ -S${CPP_SOURCE_DIR} \ -B${BUILD_DIR} \ From 1f9d700a405f7519316b217029e7f44730835398 Mon Sep 17 00:00:00 2001 From: Carlo Piovesan Date: Mon, 12 May 2025 15:04:09 +0200 Subject: [PATCH 17/28] fix --- .github/workflows/main.yml | 6 +++--- scripts/wasm_build_lib.sh | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 550514dc8..27a39ed60 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -530,7 +530,7 @@ jobs: - name: Build Wasm module run: | - DUCKDB_PLATFORM="wasm_mvp" DUCKDB_WASM_LOADABLE_EXTENSIONS=1 GEN=ninja ./scripts/wasm_build_lib.sh relsize mvp + USE_GENERATED_EXPORTED_LIST="yes" DUCKDB_PLATFORM="wasm_mvp" DUCKDB_WASM_LOADABLE_EXTENSIONS=1 GEN=ninja ./scripts/wasm_build_lib.sh relsize mvp - name: Upload artifact uses: actions/upload-artifact@v4 @@ -577,7 +577,7 @@ jobs: - name: Build Wasm module run: | - DUCKDB_PLATFORM="wasm_eh" DUCKDB_WASM_LOADABLE_EXTENSIONS=1 GEN=ninja ./scripts/wasm_build_lib.sh relsize eh + USE_GENERATED_EXPORTED_LIST="yes" DUCKDB_PLATFORM="wasm_eh" DUCKDB_WASM_LOADABLE_EXTENSIONS=1 GEN=ninja ./scripts/wasm_build_lib.sh relsize eh - name: Upload artifact uses: actions/upload-artifact@v4 @@ -624,7 +624,7 @@ jobs: - name: Build Wasm module run: | - DUCKDB_PLATFORM="wasm_threads" DUCKDB_WASM_LOADABLE_EXTENSIONS="signed" GEN=ninja ./scripts/wasm_build_lib.sh relsize coi + USE_GENERATED_EXPORTED_LIST="yes" DUCKDB_PLATFORM="wasm_threads" DUCKDB_WASM_LOADABLE_EXTENSIONS="signed" GEN=ninja ./scripts/wasm_build_lib.sh relsize coi - name: Upload artifact uses: actions/upload-artifact@v4 diff --git a/scripts/wasm_build_lib.sh b/scripts/wasm_build_lib.sh index 9a95b31a8..fb26069f2 100755 --- a/scripts/wasm_build_lib.sh +++ b/scripts/wasm_build_lib.sh @@ -71,7 +71,7 @@ emmake make \ TARGET=${FEATURES} make update_exported_list -if [ "${USE_GENERATED_EXPORTED_LIST}-no" == "yes" ]; then +if [ "${USE_GENERATED_EXPORTED_LIST:-no}" == "yes" ]; then emcmake cmake \ -S${CPP_SOURCE_DIR} \ -B${BUILD_DIR} \ From af580b8afd59b1c6536346daba2286f4ba1cc07b Mon Sep 17 00:00:00 2001 From: Carlo Piovesan Date: Mon, 12 May 2025 15:35:24 +0200 Subject: [PATCH 18/28] invert --- scripts/wasm_build_lib.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/wasm_build_lib.sh b/scripts/wasm_build_lib.sh index fb26069f2..80a6eb5d4 100755 --- a/scripts/wasm_build_lib.sh +++ b/scripts/wasm_build_lib.sh @@ -69,7 +69,7 @@ emmake make \ -j${CORES} \ duckdb_wasm -TARGET=${FEATURES} make update_exported_list +make TARGET=${FEATURES} update_exported_list if [ "${USE_GENERATED_EXPORTED_LIST:-no}" == "yes" ]; then emcmake cmake \ From c05e26734815a9e30117c934068af6fc4a14b0dc Mon Sep 17 00:00:00 2001 From: Carlo Piovesan Date: Mon, 12 May 2025 16:06:08 +0200 Subject: [PATCH 19/28] skip not loadable --- lib/CMakeLists.txt | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/lib/CMakeLists.txt b/lib/CMakeLists.txt index cbc3367bf..16dced808 100644 --- a/lib/CMakeLists.txt +++ b/lib/CMakeLists.txt @@ -151,14 +151,14 @@ endif() if (DUCKDB_WASM_LOADABLE_EXTENSIONS) set(WASM_LINK_FLAGS "${WASM_LINK_FLAGS} -s FILESYSTEM=1 -s ENVIRONMENT='web,node,worker' -s ALLOW_TABLE_GROWTH -lembind") -else() - set(WASM_LINK_FLAGS "${WASM_LINK_FLAGS} -s FILESYSTEM=0 -s ENVIRONMENT='web,node,worker'") -endif() -if (USE_GENERATED_EXPORTED_LIST) - set(WASM_LINK_FLAGS "${WASM_LINK_FLAGS} -s MAIN_MODULE=2 -s EXPORTED_FUNCTIONS='@exported_list.txt'") + if (USE_GENERATED_EXPORTED_LIST) + set(WASM_LINK_FLAGS "${WASM_LINK_FLAGS} -s MAIN_MODULE=2 -s EXPORTED_FUNCTIONS='@exported_list.txt'") + else() + set(WASM_LINK_FLAGS "${WASM_LINK_FLAGS} -s MAIN_MODULE=1") + endif() else() - set(WASM_LINK_FLAGS "${WASM_LINK_FLAGS} -s MAIN_MODULE=1") + set(WASM_LINK_FLAGS "${WASM_LINK_FLAGS} -s FILESYSTEM=0 -s ENVIRONMENT='web,node,worker'") endif() # --------------------------------------------------------------------------- From 89e078304268b4ea3944c9a7640450019f3dc2a9 Mon Sep 17 00:00:00 2001 From: Carlo Piovesan Date: Mon, 12 May 2025 17:49:08 +0200 Subject: [PATCH 20/28] Fix parking lot --- lib/src/utils/parking_lot.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/src/utils/parking_lot.cc b/lib/src/utils/parking_lot.cc index 888272734..fd9162d3d 100644 --- a/lib/src/utils/parking_lot.cc +++ b/lib/src/utils/parking_lot.cc @@ -71,7 +71,7 @@ void ParkingLot::Park(const void* addr, std::function check, std::chrono bucket.cv.wait(lock); } } else { - auto stop = std::chrono::system_clock::now() + timeout; + auto stop = std::chrono::high_resolution_clock::now() + timeout; while (!check()) { if (bucket.cv.wait_until(lock, stop) == std::cv_status::timeout) break; } From da153d8172762121f7abb0e16c372499d268b157 Mon Sep 17 00:00:00 2001 From: Carlo Piovesan Date: Tue, 13 May 2025 08:28:51 +0200 Subject: [PATCH 21/28] Temporarily add chrono --- Makefile | 1 + 1 file changed, 1 insertion(+) diff --git a/Makefile b/Makefile index efc96932c..86f6216fb 100644 --- a/Makefile +++ b/Makefile @@ -459,3 +459,4 @@ update_exported_list: cd build/relsize/${TARGET} && cat export_list.txt | grep -v "^_" | grep -v "getTempRet" | grep -v "^sched_yield" | grep -v "emscripten_wget" | grep -v "0\\00\\0" | sort > c_exported_list # prepend '_' cd build/relsize/${TARGET} && sed 's/^/_/g' c_exported_list >> exported_list.txt + cd build/relsize/${TARGET} && echo '__ZNSt3__26chrono12system_clock9to_time_tERKNS0_10time_pointIS1_NS0_8durationIxNS_5ratioILx1ELx1000000EEEEEEE' >> exported_list.txt From ee7632c5fa8cad249be4fdd29b4210dad5d56ac0 Mon Sep 17 00:00:00 2001 From: Carlo Piovesan Date: Tue, 13 May 2025 08:29:22 +0200 Subject: [PATCH 22/28] Fixup for getTempRet0: --- lib/CMakeLists.txt | 2 +- scripts/wasm_build_lib.sh | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/CMakeLists.txt b/lib/CMakeLists.txt index 16dced808..42bce6bd6 100644 --- a/lib/CMakeLists.txt +++ b/lib/CMakeLists.txt @@ -293,7 +293,7 @@ if(EMSCRIPTEN) -s MAXIMUM_MEMORY=4GB \ -s MODULARIZE=1 \ -s EXPORT_NAME='DuckDB' \ - -s EXPORTED_RUNTIME_METHODS='[\"ccall\", \"stackSave\", \"stackAlloc\", \"stackRestore\", \"createDyncallWrapper\"]' \ + -s EXPORTED_RUNTIME_METHODS='[\"ccall\", \"stackSave\", \"stackAlloc\", \"stackRestore\", \"createDyncallWrapper\", \"getTempRet0\", \"setTempRet0\"]' \ --js-library=${CMAKE_SOURCE_DIR}/js-stubs.js") endif() diff --git a/scripts/wasm_build_lib.sh b/scripts/wasm_build_lib.sh index 80a6eb5d4..a63e08931 100755 --- a/scripts/wasm_build_lib.sh +++ b/scripts/wasm_build_lib.sh @@ -92,6 +92,8 @@ fi js-beautify -v || npm install -g js-beautify js-beautify ${BUILD_DIR}/duckdb_wasm.js > ${BUILD_DIR}/beauty.js +sed -i '' 's/case \"__table_base\"/case \"getTempRet0\": return getTempRet0; case \"__table_base\"/g' ${BUILD_DIR}/beauty.js +cp ${BUILD_DIR}/beauty.js ${BUILD_DIR}/duckdb_wasm.js awk '{gsub(/get\(stubs, prop\) \{/,"get(stubs,prop) { if (prop.startsWith(\"invoke_\")) {return createDyncallWrapper(prop.substring(7));}"); print}' ${BUILD_DIR}/beauty.js > ${BUILD_DIR}/beauty2.js awk '!(/var .*wasmExports\[/ || /var [_a-z0-9A-Z]+ = Module\[\"[_a-z0-9A-Z]+\"\] = [0-9]+;/) || /var _duckdb_web/ || /var _main/ || /var _calloc/ || /var _malloc/ || /var _free/ || /var stack/ || /var ___dl_seterr/ || /var __em/ || /var _em/ || /var _pthread/' ${BUILD_DIR}/beauty2.js > ${BUILD_DIR}/duckdb_wasm.js From a543940b352e78320a3fe97dd4aab27dfb0f4033 Mon Sep 17 00:00:00 2001 From: Carlo Piovesan Date: Tue, 13 May 2025 08:58:18 +0200 Subject: [PATCH 23/28] Fixup sed --- scripts/wasm_build_lib.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/wasm_build_lib.sh b/scripts/wasm_build_lib.sh index a63e08931..f8b5494d6 100755 --- a/scripts/wasm_build_lib.sh +++ b/scripts/wasm_build_lib.sh @@ -92,7 +92,8 @@ fi js-beautify -v || npm install -g js-beautify js-beautify ${BUILD_DIR}/duckdb_wasm.js > ${BUILD_DIR}/beauty.js -sed -i '' 's/case \"__table_base\"/case \"getTempRet0\": return getTempRet0; case \"__table_base\"/g' ${BUILD_DIR}/beauty.js +sed 's/case \"__table_base\"/case \"getTempRet0\": return getTempRet0; case \"__table_base\"/g' ${BUILD_DIR}/beauty.js > ${BUILD_DIR}/beauty_sed.js +cp ${BUILD_DIR}/beauty_sed.js ${BUILD_DIR}/beauty.js cp ${BUILD_DIR}/beauty.js ${BUILD_DIR}/duckdb_wasm.js awk '{gsub(/get\(stubs, prop\) \{/,"get(stubs,prop) { if (prop.startsWith(\"invoke_\")) {return createDyncallWrapper(prop.substring(7));}"); print}' ${BUILD_DIR}/beauty.js > ${BUILD_DIR}/beauty2.js awk '!(/var .*wasmExports\[/ || /var [_a-z0-9A-Z]+ = Module\[\"[_a-z0-9A-Z]+\"\] = [0-9]+;/) || /var _duckdb_web/ || /var _main/ || /var _calloc/ || /var _malloc/ || /var _free/ || /var stack/ || /var ___dl_seterr/ || /var __em/ || /var _em/ || /var _pthread/' ${BUILD_DIR}/beauty2.js > ${BUILD_DIR}/duckdb_wasm.js From a793af60370e9f97f5bdc61b4b5549649265cf71 Mon Sep 17 00:00:00 2001 From: Carlo Piovesan Date: Tue, 13 May 2025 09:54:37 +0200 Subject: [PATCH 24/28] Move inner --- scripts/wasm_build_lib.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/wasm_build_lib.sh b/scripts/wasm_build_lib.sh index f8b5494d6..f697badf7 100755 --- a/scripts/wasm_build_lib.sh +++ b/scripts/wasm_build_lib.sh @@ -69,9 +69,9 @@ emmake make \ -j${CORES} \ duckdb_wasm +if [ "${USE_GENERATED_EXPORTED_LIST:-no}" == "yes" ]; then make TARGET=${FEATURES} update_exported_list -if [ "${USE_GENERATED_EXPORTED_LIST:-no}" == "yes" ]; then emcmake cmake \ -S${CPP_SOURCE_DIR} \ -B${BUILD_DIR} \ From b1c34ae055e16af940bb454ba1940ab206aea15f Mon Sep 17 00:00:00 2001 From: Carlo Piovesan Date: Tue, 13 May 2025 10:09:51 +0200 Subject: [PATCH 25/28] Remove unneeded wabt install --- .github/workflows/main.yml | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 27a39ed60..040648b75 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -364,10 +364,6 @@ jobs: submodules: 'recursive' fetch-depth: 0 - - name: Install dependencies - run: | - sudo apt install wabt - - name: Prepare repository run: | make apply_patches @@ -411,10 +407,6 @@ jobs: submodules: 'recursive' fetch-depth: 0 - - name: Install dependencies - run: | - sudo apt install wabt - - name: Prepare repository run: | make apply_patches @@ -458,10 +450,6 @@ jobs: submodules: 'recursive' fetch-depth: 0 - - name: Install dependencies - run: | - sudo apt install wabt - - name: Prepare repository run: | make apply_patches From 82c134e8bb7cd65f2c0a73f7e2edcf0934c192e6 Mon Sep 17 00:00:00 2001 From: Carlo Piovesan Date: Tue, 13 May 2025 10:33:54 +0200 Subject: [PATCH 26/28] Add base_exported_list also for no-extensions case --- lib/CMakeLists.txt | 2 +- lib/base_exported_list.txt | 47 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 48 insertions(+), 1 deletion(-) create mode 100644 lib/base_exported_list.txt diff --git a/lib/CMakeLists.txt b/lib/CMakeLists.txt index 42bce6bd6..98c081337 100644 --- a/lib/CMakeLists.txt +++ b/lib/CMakeLists.txt @@ -158,7 +158,7 @@ if (DUCKDB_WASM_LOADABLE_EXTENSIONS) set(WASM_LINK_FLAGS "${WASM_LINK_FLAGS} -s MAIN_MODULE=1") endif() else() - set(WASM_LINK_FLAGS "${WASM_LINK_FLAGS} -s FILESYSTEM=0 -s ENVIRONMENT='web,node,worker'") + set(WASM_LINK_FLAGS "${WASM_LINK_FLAGS} -s FILESYSTEM=0 -s ENVIRONMENT='web,node,worker' -s EXPORTED_FUNCTIONS='@../../../lib/base_exported_list.txt'") endif() # --------------------------------------------------------------------------- diff --git a/lib/base_exported_list.txt b/lib/base_exported_list.txt new file mode 100644 index 000000000..fb149fef8 --- /dev/null +++ b/lib/base_exported_list.txt @@ -0,0 +1,47 @@ +_main +_malloc +_calloc +_free +_duckdb_web_clear_response +_duckdb_web_collect_file_stats +_duckdb_web_connect +_duckdb_web_copy_file_to_buffer +_duckdb_web_copy_file_to_path +_duckdb_web_disconnect +_duckdb_web_export_file_stats +_duckdb_web_fail_with +_duckdb_web_flush_file +_duckdb_web_flush_files +_duckdb_web_fs_drop_file +_duckdb_web_fs_drop_files +_duckdb_web_fs_get_file_info_by_id +_duckdb_web_fs_get_file_info_by_name +_duckdb_web_fs_glob_add_path +_duckdb_web_fs_glob_file_infos +_duckdb_web_fs_register_file_buffer +_duckdb_web_fs_register_file_url +_duckdb_web_get_feature_flags +_duckdb_web_get_global_file_info +_duckdb_web_get_tablenames +_duckdb_web_get_tablenames_buffer +_duckdb_web_get_version +_duckdb_web_insert_arrow_from_ipc_stream +_duckdb_web_insert_csv_from_path +_duckdb_web_insert_json_from_path +_duckdb_web_open +_duckdb_web_pending_query_cancel +_duckdb_web_pending_query_poll +_duckdb_web_pending_query_start +_duckdb_web_pending_query_start_buffer +_duckdb_web_prepared_close +_duckdb_web_prepared_create +_duckdb_web_prepared_create_buffer +_duckdb_web_prepared_run +_duckdb_web_prepared_send +_duckdb_web_query_fetch_results +_duckdb_web_query_run +_duckdb_web_query_run_buffer +_duckdb_web_reset +_duckdb_web_tokenize +_duckdb_web_tokenize_buffer +_duckdb_web_udf_scalar_create From 8a9fff01bcee7c53dcb24f6aa36c9653fb805844 Mon Sep 17 00:00:00 2001 From: Carlo Piovesan Date: Wed, 14 May 2025 15:03:05 +0200 Subject: [PATCH 27/28] Bump to duckdb 93fda3591f4298414fa362c59219c09e03f718ab --- submodules/duckdb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/submodules/duckdb b/submodules/duckdb index 57cbd519a..93fda3591 160000 --- a/submodules/duckdb +++ b/submodules/duckdb @@ -1 +1 @@ -Subproject commit 57cbd519ad94a5c5b11bde40a86bde8e074c3c91 +Subproject commit 93fda3591f4298414fa362c59219c09e03f718ab From 4e560db38dac21fbf1ecca3894ec4c5a70a0f429 Mon Sep 17 00:00:00 2001 From: Carlo Piovesan Date: Wed, 14 May 2025 15:04:16 +0200 Subject: [PATCH 28/28] Remove virtualized_file_system patch, already handled --- patches/duckdb/virtualized_file_system.patch | 26 -------------------- 1 file changed, 26 deletions(-) delete mode 100644 patches/duckdb/virtualized_file_system.patch diff --git a/patches/duckdb/virtualized_file_system.patch b/patches/duckdb/virtualized_file_system.patch deleted file mode 100644 index 028fc3709..000000000 --- a/patches/duckdb/virtualized_file_system.patch +++ /dev/null @@ -1,26 +0,0 @@ -diff --git a/src/common/virtual_file_system.cpp b/src/common/virtual_file_system.cpp -index 74892a4e05..c3d1c4f333 100644 ---- a/src/common/virtual_file_system.cpp -+++ b/src/common/virtual_file_system.cpp -@@ -5,7 +5,7 @@ - - namespace duckdb { - --VirtualFileSystem::VirtualFileSystem() : default_fs(FileSystem::CreateLocal()) { -+VirtualFileSystem::VirtualFileSystem(unique_ptr inner) : default_fs(std::move(inner)) { - VirtualFileSystem::RegisterSubSystem(FileCompressionType::GZIP, make_uniq()); - } - -diff --git a/src/include/duckdb/common/virtual_file_system.hpp b/src/include/duckdb/common/virtual_file_system.hpp -index 110ad04877..30a7eb29d3 100644 ---- a/src/include/duckdb/common/virtual_file_system.hpp -+++ b/src/include/duckdb/common/virtual_file_system.hpp -@@ -17,7 +17,7 @@ namespace duckdb { - // bunch of wrappers to allow registering protocol handlers - class VirtualFileSystem : public FileSystem { - public: -- VirtualFileSystem(); -+ VirtualFileSystem(unique_ptr inner_file_system = nullptr); - - unique_ptr OpenFile(const string &path, FileOpenFlags flags, - optional_ptr opener = nullptr) override;