Bump to last night duckdb

.github/workflows/main.yml

@@ -494,6 +494,10 @@ jobs:
                   submodules: 'recursive'
                   fetch-depth: 0
 
+            - name: Install dependencies
+              run: |
+                  sudo apt install wabt
+
             - name: Prepare repository
               run: |
                   make apply_patches
@@ -514,7 +518,7 @@ jobs:
 
             - name: Build Wasm module
               run: |
-                  DUCKDB_PLATFORM="wasm_mvp" DUCKDB_WASM_LOADABLE_EXTENSIONS=1 GEN=ninja ./scripts/wasm_build_lib.sh relsize mvp
+                  USE_GENERATED_EXPORTED_LIST="yes" DUCKDB_PLATFORM="wasm_mvp" DUCKDB_WASM_LOADABLE_EXTENSIONS=1 GEN=ninja ./scripts/wasm_build_lib.sh relsize mvp
 
             - name: Upload artifact
               uses: actions/upload-artifact@v4
@@ -537,6 +541,10 @@ jobs:
                   submodules: 'recursive'
                   fetch-depth: 0
 
+            - name: Install dependencies
+              run: |
+                  sudo apt install wabt
+
             - name: Prepare repository
               run: |
                   make apply_patches
@@ -557,7 +565,7 @@ jobs:
 
             - name: Build Wasm module
               run: |
-                  DUCKDB_PLATFORM="wasm_eh" DUCKDB_WASM_LOADABLE_EXTENSIONS=1 GEN=ninja ./scripts/wasm_build_lib.sh relsize eh
+                  USE_GENERATED_EXPORTED_LIST="yes" DUCKDB_PLATFORM="wasm_eh" DUCKDB_WASM_LOADABLE_EXTENSIONS=1 GEN=ninja ./scripts/wasm_build_lib.sh relsize eh
 
             - name: Upload artifact
               uses: actions/upload-artifact@v4
@@ -580,6 +588,10 @@ jobs:
                   submodules: 'recursive'
                   fetch-depth: 0
 
+            - name: Install dependencies
+              run: |
+                  sudo apt install wabt
+
             - name: Prepare repository
               run: |
                   make apply_patches
@@ -600,7 +612,7 @@ jobs:
 
             - name: Build Wasm module
               run: |
-                  DUCKDB_PLATFORM="wasm_threads" DUCKDB_WASM_LOADABLE_EXTENSIONS="signed" GEN=ninja ./scripts/wasm_build_lib.sh relsize coi
+                  USE_GENERATED_EXPORTED_LIST="yes" DUCKDB_PLATFORM="wasm_threads" DUCKDB_WASM_LOADABLE_EXTENSIONS="signed" GEN=ninja ./scripts/wasm_build_lib.sh relsize coi
 
             - name: Upload artifact
               uses: actions/upload-artifact@v4

Makefile

@@ -357,7 +357,7 @@ app: wasm wasmpack shell docs js_tests_release
 	yarn workspace @duckdb/duckdb-wasm-app build:release
 
 build_loadable:
-	DUCKDB_PLATFORM=wasm_${TARGET} DUCKDB_WASM_LOADABLE_EXTENSIONS=1 GEN=ninja ./scripts/wasm_build_lib.sh relsize ${TARGET}
+	USE_GENERATED_EXPORTED_LIST=yes DUCKDB_PLATFORM=wasm_${TARGET} DUCKDB_WASM_LOADABLE_EXTENSIONS=1 GEN=ninja ./scripts/wasm_build_lib.sh relsize ${TARGET}
 
 build_loadable_unsigned: build_loadable
         # need to propagate the unsigned flag
@@ -436,6 +436,7 @@ build/docker_ci_image:
 patch_duckdb:
 	(find patches/duckdb/* -type f -name '*.patch' -print0 | xargs -0 cat | patch -p1 --forward -d submodules/duckdb) || true
 	(find patches/arrow/* -type f -name '*.patch' -print0 | xargs -0 cat | patch -p1 --forward -d submodules/arrow) || true
+	(find patches/rapidjson/* -type f -name '*.patch' -print0 | xargs -0 cat | patch -p1 --forward -d submodules/rapidjson) || true
 
 apply_patches: patch_duckdb
 
@@ -447,3 +448,15 @@ submodules:
 build/bootstrap: submodules yarn_install
 	mkdir -p build
 	touch build/bootstrap
+
+update_exported_list:
+	cd build/relsize/${TARGET} && wasm2wat duckdb_wasm.wasm --enable-all -o duckdb-wasm.wat
+	cd build/relsize/${TARGET} && grep export duckdb-wasm.wat | cut -d \" -f2 | sed '$d' | grep -v "^orig" | grep -v "^dynCall_" > export_list.txt
+        ## filter list of c++ symbols
+	cd build/relsize/${TARGET} && cat export_list.txt | grep "^_" | grep -v "_Unwind_" | grep -v "__syscall_shutdown" | grep -v "0\\00\\0" | grep -v "^_ZZN5arrow" | grep -v "^_ZGVZN5arrow" | grep -v "^_ZN5arrow" | sort > cpp_list
+	cd build/relsize/${TARGET} && sed 's/^/_/g' cpp_list > exported_list.txt
+        ## filter list of c symbols
+	cd build/relsize/${TARGET} && cat export_list.txt | grep -v "^_" | grep -v "getTempRet" | grep -v "^sched_yield" | grep -v "emscripten_wget" | grep -v "0\\00\\0" | sort > c_exported_list
+	# prepend '_'
+	cd build/relsize/${TARGET} && sed 's/^/_/g' c_exported_list >> exported_list.txt
+	cd build/relsize/${TARGET} && echo '__ZNSt3__26chrono12system_clock9to_time_tERKNS0_10time_pointIS1_NS0_8durationIxNS_5ratioILx1ELx1000000EEEEEEE' >> exported_list.txt

lib/CMakeLists.txt

@@ -150,9 +150,15 @@ else()
 endif()
 
 if (DUCKDB_WASM_LOADABLE_EXTENSIONS)
-  set(WASM_LINK_FLAGS "${WASM_LINK_FLAGS} -s MAIN_MODULE=1 -s FILESYSTEM=1 -s ENVIRONMENT='web,node,worker' -s ALLOW_TABLE_GROWTH -lembind")
+  set(WASM_LINK_FLAGS "${WASM_LINK_FLAGS} -s FILESYSTEM=1 -s ENVIRONMENT='web,node,worker' -s ALLOW_TABLE_GROWTH -lembind")
+
+  if (USE_GENERATED_EXPORTED_LIST)
+    set(WASM_LINK_FLAGS "${WASM_LINK_FLAGS} -s MAIN_MODULE=2 -s EXPORTED_FUNCTIONS='@exported_list.txt'")
+  else()
+    set(WASM_LINK_FLAGS "${WASM_LINK_FLAGS} -s MAIN_MODULE=1")
+  endif()
 else()
-  set(WASM_LINK_FLAGS "${WASM_LINK_FLAGS} -s FILESYSTEM=0 -s ENVIRONMENT='web,node,worker'")
+  set(WASM_LINK_FLAGS "${WASM_LINK_FLAGS} -s FILESYSTEM=0 -s ENVIRONMENT='web,node,worker' -s EXPORTED_FUNCTIONS='@../../../lib/base_exported_list.txt'")
 endif()
 
 # ---------------------------------------------------------------------------
@@ -287,56 +293,7 @@ if(EMSCRIPTEN)
       -s MAXIMUM_MEMORY=4GB \
       -s MODULARIZE=1 \
       -s EXPORT_NAME='DuckDB' \
-      -s EXPORTED_FUNCTIONS='[ \
-          _main, \
-          _malloc, \
-          _calloc, \
-          _free, \
-          _duckdb_web_clear_response, \
-          _duckdb_web_collect_file_stats, \
-          _duckdb_web_connect, \
-          _duckdb_web_copy_file_to_buffer, \
-          _duckdb_web_copy_file_to_path, \
-          _duckdb_web_disconnect, \
-          _duckdb_web_export_file_stats, \
-          _duckdb_web_fail_with, \
-          _duckdb_web_flush_file, \
-          _duckdb_web_flush_files, \
-          _duckdb_web_fs_drop_file, \
-          _duckdb_web_fs_drop_files, \
-          _duckdb_web_fs_get_file_info_by_id, \
-          _duckdb_web_fs_get_file_info_by_name, \
-          _duckdb_web_fs_glob_add_path, \
-          _duckdb_web_fs_glob_file_infos, \
-          _duckdb_web_fs_register_file_buffer, \
-          _duckdb_web_fs_register_file_url, \
-          _duckdb_web_get_feature_flags, \
-          _duckdb_web_get_global_file_info, \
-          _duckdb_web_get_tablenames, \
-          _duckdb_web_get_tablenames_buffer, \
-          _duckdb_web_get_version, \
-          _duckdb_web_insert_arrow_from_ipc_stream, \
-          _duckdb_web_insert_csv_from_path, \
-          _duckdb_web_insert_json_from_path, \
-          _duckdb_web_open, \
-          _duckdb_web_pending_query_cancel, \
-          _duckdb_web_pending_query_poll, \
-          _duckdb_web_pending_query_start, \
-          _duckdb_web_pending_query_start_buffer, \
-          _duckdb_web_prepared_close, \
-          _duckdb_web_prepared_create, \
-          _duckdb_web_prepared_create_buffer, \
-          _duckdb_web_prepared_run, \
-          _duckdb_web_prepared_send, \
-          _duckdb_web_query_fetch_results, \
-          _duckdb_web_query_run, \
-          _duckdb_web_query_run_buffer, \
-          _duckdb_web_reset, \
-          _duckdb_web_tokenize, \
-          _duckdb_web_tokenize_buffer, \
-          _duckdb_web_udf_scalar_create \
-      ]' \
-      -s EXPORTED_RUNTIME_METHODS='[\"ccall\", \"stackSave\", \"stackAlloc\", \"stackRestore\"]' \
+      -s EXPORTED_RUNTIME_METHODS='[\"ccall\", \"stackSave\", \"stackAlloc\", \"stackRestore\", \"createDyncallWrapper\", \"getTempRet0\", \"setTempRet0\"]' \
       --js-library=${CMAKE_SOURCE_DIR}/js-stubs.js")
 
 endif()

lib/base_exported_list.txt

@@ -0,0 +1,47 @@
+_main
+_malloc
+_calloc
+_free
+_duckdb_web_clear_response
+_duckdb_web_collect_file_stats
+_duckdb_web_connect
+_duckdb_web_copy_file_to_buffer
+_duckdb_web_copy_file_to_path
+_duckdb_web_disconnect
+_duckdb_web_export_file_stats
+_duckdb_web_fail_with
+_duckdb_web_flush_file
+_duckdb_web_flush_files
+_duckdb_web_fs_drop_file
+_duckdb_web_fs_drop_files
+_duckdb_web_fs_get_file_info_by_id
+_duckdb_web_fs_get_file_info_by_name
+_duckdb_web_fs_glob_add_path
+_duckdb_web_fs_glob_file_infos
+_duckdb_web_fs_register_file_buffer
+_duckdb_web_fs_register_file_url
+_duckdb_web_get_feature_flags
+_duckdb_web_get_global_file_info
+_duckdb_web_get_tablenames
+_duckdb_web_get_tablenames_buffer
+_duckdb_web_get_version
+_duckdb_web_insert_arrow_from_ipc_stream
+_duckdb_web_insert_csv_from_path
+_duckdb_web_insert_json_from_path
+_duckdb_web_open
+_duckdb_web_pending_query_cancel
+_duckdb_web_pending_query_poll
+_duckdb_web_pending_query_start
+_duckdb_web_pending_query_start_buffer
+_duckdb_web_prepared_close
+_duckdb_web_prepared_create
+_duckdb_web_prepared_create_buffer
+_duckdb_web_prepared_run
+_duckdb_web_prepared_send
+_duckdb_web_query_fetch_results
+_duckdb_web_query_run
+_duckdb_web_query_run_buffer
+_duckdb_web_reset
+_duckdb_web_tokenize
+_duckdb_web_tokenize_buffer
+_duckdb_web_udf_scalar_create

lib/include/duckdb/web/utils/wasm_response.h

@@ -8,6 +8,8 @@
 namespace duckdb {
 namespace web {
 
+struct DuckDBWasmResultsWrapper;
+
 struct WASMResponse {
     /// The status code
     double statusCode = 1;
@@ -35,6 +37,8 @@ class WASMResponseBuffer {
     /// Store the arrow status.
     /// Returns wheather the result was OK
     bool Store(WASMResponse& response, arrow::Status status);
+    /// Store a DuckDBWasmResultsWrapper
+    void Store(WASMResponse& response, DuckDBWasmResultsWrapper& value);
     /// Store a string
     void Store(WASMResponse& response, std::string value);
     /// Store a string view

lib/include/duckdb/web/webdb.h

@@ -29,6 +29,22 @@ namespace web {
 
 struct BufferingArrowIPCStreamDecoder;
 
+struct DuckDBWasmResultsWrapper {
+    // Additional ResponseStatuses to be >= 256, and mirrored to packages/duckdb-wasm/src/status.ts
+    // Missing mapping result in a throw, but they should eventually align (it's fine if typescript side only has a
+    // subset)
+    enum ResponseStatus : uint32_t { ARROW_BUFFER = 0, MAX_ARROW_ERROR = 255, DUCKDB_WASM_RETRY = 256 };
+    DuckDBWasmResultsWrapper(arrow::Result<std::shared_ptr<arrow::Buffer>> res,
+                             ResponseStatus status = ResponseStatus::ARROW_BUFFER)
+        : arrow_buffer(res), status(status) {}
+    DuckDBWasmResultsWrapper(arrow::Status res, ResponseStatus status = ResponseStatus::ARROW_BUFFER)
+        : arrow_buffer(res), status(status) {}
+    DuckDBWasmResultsWrapper(ResponseStatus status = ResponseStatus::ARROW_BUFFER)
+        : DuckDBWasmResultsWrapper(nullptr, status) {}
+    arrow::Result<std::shared_ptr<arrow::Buffer>> arrow_buffer;
+    ResponseStatus status;
+};
+
 class WebDB {
    public:
     /// A connection
@@ -93,7 +109,7 @@ class WebDB {
         /// Cancel a pending query
         bool CancelPendingQuery();
         /// Fetch a data chunk from a pending query
-        arrow::Result<std::shared_ptr<arrow::Buffer>> FetchQueryResults();
+        DuckDBWasmResultsWrapper FetchQueryResults();
         /// Get table names
         arrow::Result<std::string> GetTableNames(std::string_view text);
 

lib/src/utils/parking_lot.cc

@@ -71,7 +71,7 @@ void ParkingLot::Park(const void* addr, std::function<bool()> check, std::chrono
             bucket.cv.wait(lock);
         }
     } else {
-        auto stop = std::chrono::system_clock::now() + timeout;
+        auto stop = std::chrono::high_resolution_clock::now() + timeout;
         while (!check()) {
             if (bucket.cv.wait_until(lock, stop) == std::cv_status::timeout) break;
         }

lib/src/utils/wasm_response.cc

@@ -3,6 +3,7 @@
 #include <cstdint>
 
 #include "arrow/buffer.h"
+#include "duckdb/web/webdb.h"
 
 namespace duckdb {
 namespace web {
@@ -26,6 +27,15 @@ bool WASMResponseBuffer::Store(WASMResponse& response, arrow::Status status) {
     return true;
 }
 
+void WASMResponseBuffer::Store(WASMResponse& response, DuckDBWasmResultsWrapper& value) {
+    if (value.status == DuckDBWasmResultsWrapper::ResponseStatus::ARROW_BUFFER) {
+        Store(response, std::move(value.arrow_buffer));
+    } else {
+        Clear();
+        response.statusCode = value.status;
+    }
+}
+
 void WASMResponseBuffer::Store(WASMResponse& response, std::string value) {
     result_str_ = std::move(value);
     response.statusCode = 0;

lib/src/webdb.cc

@@ -232,13 +232,53 @@ bool WebDB::Connection::CancelPendingQuery() {
     }
 }
 
-arrow::Result<std::shared_ptr<arrow::Buffer>> WebDB::Connection::FetchQueryResults() {
+DuckDBWasmResultsWrapper WebDB::Connection::FetchQueryResults() {
     try {
         // Fetch data if a query is active
         duckdb::unique_ptr<duckdb::DataChunk> chunk;
         if (current_query_result_ == nullptr) {
-            return nullptr;
+            return DuckDBWasmResultsWrapper{nullptr};
+        }
+
+        if (current_query_result_->type == QueryResultType::STREAM_RESULT) {
+            auto& stream_result = current_query_result_->Cast<duckdb::StreamQueryResult>();
+
+            auto before = std::chrono::steady_clock::now();
+            uint64_t elapsed;
+            auto polling_interval =
+                webdb_.config_->query.query_polling_interval.value_or(DEFAULT_QUERY_POLLING_INTERVAL);
+            bool ready = false;
+            do {
+                switch (stream_result.ExecuteTask()) {
+                    case StreamExecutionResult::EXECUTION_ERROR:
+                        return arrow::Status{arrow::StatusCode::ExecutionError,
+                                             std::move(current_query_result_->GetError())};
+                    case StreamExecutionResult::EXECUTION_CANCELLED:
+                        return arrow::Status{arrow::StatusCode::ExecutionError,
+                                             "The execution of the query was cancelled before it could finish, likely "
+                                             "caused by executing a different query"};
+                    case StreamExecutionResult::CHUNK_READY:
+                    case StreamExecutionResult::EXECUTION_FINISHED:
+                        ready = true;
+                        break;
+                    case StreamExecutionResult::BLOCKED:
+                        stream_result.WaitForTask();
+                        return DuckDBWasmResultsWrapper::ResponseStatus::DUCKDB_WASM_RETRY;
+                    case StreamExecutionResult::NO_TASKS_AVAILABLE:
+                        return DuckDBWasmResultsWrapper::ResponseStatus::DUCKDB_WASM_RETRY;
+                    case StreamExecutionResult::CHUNK_NOT_READY:
+                        break;
+                }
+
+                auto after = std::chrono::steady_clock::now();
+                elapsed = std::chrono::duration_cast<std::chrono::milliseconds>(after - before).count();
+            } while (!ready && elapsed < polling_interval);
+
+            if (!ready) {
+                return DuckDBWasmResultsWrapper::ResponseStatus::DUCKDB_WASM_RETRY;
+            }
         }
+
         // Fetch next result chunk
         chunk = current_query_result_->Fetch();
         if (current_query_result_->HasError()) {
@@ -249,7 +289,7 @@ arrow::Result<std::shared_ptr<arrow::Buffer>> WebDB::Connection::FetchQueryResul
             current_query_result_.reset();
             current_schema_.reset();
             current_schema_patched_.reset();
-            return nullptr;
+            return DuckDBWasmResultsWrapper{nullptr};
         }
 
         // Serialize the record batch

lib/src/webdb_api.cc

@@ -241,7 +241,7 @@ bool duckdb_web_pending_query_cancel(ConnectionHdl connHdl, const char* script)
 void duckdb_web_query_fetch_results(WASMResponse* packed, ConnectionHdl connHdl) {
     auto c = reinterpret_cast<WebDB::Connection*>(connHdl);
     auto r = c->FetchQueryResults();
-    WASMResponseBuffer::Get().Store(*packed, std::move(r));
+    WASMResponseBuffer::Get().Store(*packed, r);
 }
 /// Get table names
 void duckdb_web_get_tablenames(WASMResponse* packed, ConnectionHdl connHdl, const char* query) {