From 001f24bb0b0e490cadff178a4fbe39b075488bf0 Mon Sep 17 00:00:00 2001 From: Haruki Okada Date: Sat, 2 Sep 2023 23:21:34 +0900 Subject: [PATCH 01/13] Add a capability to statically link custom extension --- Makefile | 40 ++++-- lib/CMakeLists.txt | 35 +++++ lib/extension/CMakeLists.template.txt | 3 + .../generated_custom_extensions_header.hpp.in | 14 ++ lib/src/webdb.cc | 2 + packages/duckdb-wasm-app/package.json | 1 + packages/duckdb-wasm-app/src/app.tsx | 12 ++ packages/duckdb-wasm-app/webpack.common.js | 15 ++- packages/duckdb-wasm/bundle.mjs | 120 ++++++++++-------- 9 files changed, 175 insertions(+), 67 deletions(-) create mode 100644 lib/extension/CMakeLists.template.txt create mode 100644 lib/extension/generated_custom_extensions_header.hpp.in diff --git a/Makefile b/Makefile index a00b6f5d3..06287fe67 100644 --- a/Makefile +++ b/Makefile @@ -256,27 +256,43 @@ wasm_setup: set_environment check_duckdb wrapped_wasm_caches .PHONY: wasm_dev wasm_dev: wasm_setup - ${EXEC_ENVIRONMENT} ${ROOT_DIR}/scripts/wasm_build_lib.sh dev mvp - ${EXEC_ENVIRONMENT} ${ROOT_DIR}/scripts/wasm_build_lib.sh dev eh - ${EXEC_ENVIRONMENT} ${ROOT_DIR}/scripts/wasm_build_lib.sh dev coi + DUCKDB_WEB_CUSTOM_EXTENSION_DIRS=${CUSTOM_EXTENSION_DIRS} ${EXEC_ENVIRONMENT} ${ROOT_DIR}/scripts/wasm_build_lib.sh dev mvp +ifneq (${DUCKDB_SKIP_BUILD_EH}, 1) + DUCKDB_WEB_CUSTOM_EXTENSION_DIRS=${CUSTOM_EXTENSION_DIRS} ${EXEC_ENVIRONMENT} ${ROOT_DIR}/scripts/wasm_build_lib.sh dev eh +endif +ifneq (${DUCKDB_SKIP_BUILD_COI}, 1) + DUCKDB_WEB_CUSTOM_EXTENSION_DIRS=${CUSTOM_EXTENSION_DIRS} ${EXEC_ENVIRONMENT} ${ROOT_DIR}/scripts/wasm_build_lib.sh dev coi +endif .PHONY: wasm_relperf wasm_relperf: wasm_setup - ${EXEC_ENVIRONMENT} ${ROOT_DIR}/scripts/wasm_build_lib.sh relperf mvp - ${EXEC_ENVIRONMENT} ${ROOT_DIR}/scripts/wasm_build_lib.sh relperf eh - ${EXEC_ENVIRONMENT} ${ROOT_DIR}/scripts/wasm_build_lib.sh relperf coi + DUCKDB_WEB_CUSTOM_EXTENSION_DIRS=${CUSTOM_EXTENSION_DIRS} ${EXEC_ENVIRONMENT} ${ROOT_DIR}/scripts/wasm_build_lib.sh relperf mvp +ifneq (${DUCKDB_SKIP_BUILD_EH}, 1) + DUCKDB_WEB_CUSTOM_EXTENSION_DIRS=${CUSTOM_EXTENSION_DIRS} ${EXEC_ENVIRONMENT} ${ROOT_DIR}/scripts/wasm_build_lib.sh relperf eh +endif +ifneq (${DUCKDB_SKIP_BUILD_COI}, 1) + DUCKDB_WEB_CUSTOM_EXTENSION_DIRS=${CUSTOM_EXTENSION_DIRS} ${EXEC_ENVIRONMENT} ${ROOT_DIR}/scripts/wasm_build_lib.sh relperf coi +endif .PHONY: wasm_relsize wasm_relsize: wasm_setup - ${EXEC_ENVIRONMENT} ${ROOT_DIR}/scripts/wasm_build_lib.sh relsize mvp - ${EXEC_ENVIRONMENT} ${ROOT_DIR}/scripts/wasm_build_lib.sh relsize eh - ${EXEC_ENVIRONMENT} ${ROOT_DIR}/scripts/wasm_build_lib.sh relsize coi + DUCKDB_WEB_CUSTOM_EXTENSION_DIRS=${CUSTOM_EXTENSION_DIRS} ${EXEC_ENVIRONMENT} ${ROOT_DIR}/scripts/wasm_build_lib.sh relsize mvp +ifneq (${DUCKDB_SKIP_BUILD_EH}, 1) + DUCKDB_WEB_CUSTOM_EXTENSION_DIRS=${CUSTOM_EXTENSION_DIRS} ${EXEC_ENVIRONMENT} ${ROOT_DIR}/scripts/wasm_build_lib.sh relsize eh +endif +ifneq (${DUCKDB_SKIP_BUILD_COI}, 1) + DUCKDB_WEB_CUSTOM_EXTENSION_DIRS=${CUSTOM_EXTENSION_DIRS} ${EXEC_ENVIRONMENT} ${ROOT_DIR}/scripts/wasm_build_lib.sh relsize coi +endif .PHONY: wasm_debug wasm_debug: wasm_setup - ${EXEC_ENVIRONMENT} ${ROOT_DIR}/scripts/wasm_build_lib.sh debug mvp - ${EXEC_ENVIRONMENT} ${ROOT_DIR}/scripts/wasm_build_lib.sh debug eh - ${EXEC_ENVIRONMENT} ${ROOT_DIR}/scripts/wasm_build_lib.sh debug coi + DUCKDB_WEB_CUSTOM_EXTENSION_DIRS=${CUSTOM_EXTENSION_DIRS} ${EXEC_ENVIRONMENT} ${ROOT_DIR}/scripts/wasm_build_lib.sh debug mvp +ifneq (${DUCKDB_SKIP_BUILD_EH}, 1) + DUCKDB_WEB_CUSTOM_EXTENSION_DIRS=${CUSTOM_EXTENSION_DIRS} ${EXEC_ENVIRONMENT} ${ROOT_DIR}/scripts/wasm_build_lib.sh debug eh +endif +ifneq (${DUCKDB_SKIP_BUILD_COI}, 1) + DUCKDB_WEB_CUSTOM_EXTENSION_DIRS=${CUSTOM_EXTENSION_DIRS} ${EXEC_ENVIRONMENT} ${ROOT_DIR}/scripts/wasm_build_lib.sh debug coi +endif wasm: wasm_relperf diff --git a/lib/CMakeLists.txt b/lib/CMakeLists.txt index 60a692282..1929f34ef 100644 --- a/lib/CMakeLists.txt +++ b/lib/CMakeLists.txt @@ -10,6 +10,7 @@ set(CMAKE_EXPORT_COMPILE_COMMANDS ON) set(CMAKE_CXX_STANDARD 17) set(CMAKE_CXX_STANDARD_REQUIRED ON) set(ignoreMe "${DUCKDB_WASM_VERSION}") +set(DUCKDB_WEB_CUSTOM_EXTENSION_DIRS $ENV{DUCKDB_WEB_CUSTOM_EXTENSION_DIRS}) option(DUCKDB_WASM_LOADABLE_EXTENSIONS "Build with loadable extensions" OFF) option(DUCKDB_WASM_LOADABLE_EXTENSIONS_UNSIGNED "Build with loadable extensions" OFF) @@ -287,6 +288,38 @@ target_link_libraries(duckdb_web_parquet duckdb duckdb_parquet) target_link_libraries(duckdb_web_excel duckdb duckdb_excel) target_link_libraries(duckdb_web_json duckdb duckdb_json) +# --------------------------------------------------------------------------- +# Custom extensions + +function(register_custom_extension name lib) + set(DUCKDB_WEB_CUSTOM_EXTENSION_NAMES ${DUCKDB_WEB_CUSTOM_EXTENSION_NAMES} ${name}) + set(DUCKDB_WEB_CUSTOM_EXTENSION_LIBS ${DUCKDB_WEB_CUSTOM_EXTENSION_LIBS} ${lib}) +endfunction() + +set(DUCKDB_WEB_CUSTOM_EXTENSION_NAMES "") +set(DUCKDB_WEB_CUSTOM_EXTENSION_LIBS "") +if(DUCKDB_WEB_CUSTOM_EXTENSION_DIRS) + foreach(dir ${DUCKDB_WEB_CUSTOM_EXTENSION_DIRS}) + add_subdirectory(${dir}) + endforeach() +endif() + +foreach(lib ${DUCKDB_WEB_CUSTOM_EXTENSION_LIBS}) + target_link_libraries(duckdb_web ${lib}) +endforeach() + +set(GENERATED_HEADER_FILE ${PROJECT_BINARY_DIR}/codegen/include/duckdb/generated_extensions_header.hpp) +set(DUCKDB_WEB_CUSTOM_EXTENSION_INIT_FUNCTIONS "") +set(DUCKDB_WEB_CUSTOM_EXTENSION_INIT_CALLS "") + +foreach(name ${DUCKDB_WEB_CUSTOM_EXTENSION_NAMES}) + set(DUCKDB_WEB_CUSTOM_EXTENSION_INIT_FUNCTIONS "${DUCKDB_WEB_CUSTOM_EXTENSION_INIT_FUNCTIONS}extern void ${name}_init(duckdb::DuckDB *db);") + set(DUCKDB_WEB_CUSTOM_EXTENSION_INIT_CALLS "${DUCKDB_WEB_CUSTOM_EXTENSION_INIT_CALLS}${name}_init(db);") +endforeach() + +configure_file(${CMAKE_SOURCE_DIR}/extension/generated_custom_extensions_header.hpp.in ${GENERATED_HEADER_FILE}) +include_directories(${PROJECT_BINARY_DIR}/codegen/include) + # --------------------------------------------------------------------------- # Emscripten @@ -305,6 +338,8 @@ if(EMSCRIPTEN) -s WARN_ON_UNDEFINED_SYMBOLS=0 \ -s ALLOW_MEMORY_GROWTH=1 \ -s MAXIMUM_MEMORY=4GB \ + -s INITIAL_MEMORY=32MB \ + -s STACK_SIZE=16MB \ -s MODULARIZE=1 \ -s EXPORT_NAME='DuckDB' \ -s EXPORTED_FUNCTIONS='[ \ diff --git a/lib/extension/CMakeLists.template.txt b/lib/extension/CMakeLists.template.txt new file mode 100644 index 000000000..db14267af --- /dev/null +++ b/lib/extension/CMakeLists.template.txt @@ -0,0 +1,3 @@ +set(DUCKDB_WEB_CUSTOM_EXTENSION_LIB "/path/to/libhello_ext.a") + +register_custom_extension("libhello_ext") diff --git a/lib/extension/generated_custom_extensions_header.hpp.in b/lib/extension/generated_custom_extensions_header.hpp.in new file mode 100644 index 000000000..24266073d --- /dev/null +++ b/lib/extension/generated_custom_extensions_header.hpp.in @@ -0,0 +1,14 @@ +#ifndef DUCKDB_WASM_GENERATED_CUSTOM_EXTENSIONS_HEADER_HPP +#define DUCKDB_WASM_GENERATED_CUSTOM_EXTENSIONS_HEADER_HPP + +#include "duckdb/main/database.hpp" + +extern "C" { +${DUCKDB_WEB_CUSTOM_EXTENSION_INIT_FUNCTIONS} +} + +void LoadCustomExtensions(duckdb::DuckDB *db) { + ${DUCKDB_WEB_CUSTOM_EXTENSION_INIT_CALLS} +} + +#endif // DUCKDB_WASM_GENERATED_CUSTOM_EXTENSIONS_HEADER_HPP diff --git a/lib/src/webdb.cc b/lib/src/webdb.cc index 096db4bc3..430ebe002 100644 --- a/lib/src/webdb.cc +++ b/lib/src/webdb.cc @@ -62,6 +62,7 @@ #include "duckdb/web/udf.h" #include "duckdb/web/utils/debug.h" #include "duckdb/web/utils/wasm_response.h" +#include "duckdb/generated_extensions_header.hpp" #include "rapidjson/document.h" #include "rapidjson/error/en.h" #include "rapidjson/rapidjson.h" @@ -829,6 +830,7 @@ arrow::Status WebDB::Open(std::string_view args_json) { duckdb_web_json_init(db.get()); #endif #endif // WASM_LOADABLE_EXTENSIONS + LoadCustomExtensions(db.get()); RegisterCustomExtensionOptions(db); // Reset state that is specific to the old database diff --git a/packages/duckdb-wasm-app/package.json b/packages/duckdb-wasm-app/package.json index b55cb84f2..4255ff40a 100644 --- a/packages/duckdb-wasm-app/package.json +++ b/packages/duckdb-wasm-app/package.json @@ -54,6 +54,7 @@ "file-loader": "^6.2.0", "html-webpack-plugin": "^5.5.3", "jasmine-core": "^5.1.1", + "ifdef-loader": "^2.3.2", "jasmine-spec-reporter": "^7.0.0", "karma": "^6.4.2", "karma-chrome-launcher": "^3.2.0", diff --git a/packages/duckdb-wasm-app/src/app.tsx b/packages/duckdb-wasm-app/src/app.tsx index 38d4c245e..904ba4f47 100644 --- a/packages/duckdb-wasm-app/src/app.tsx +++ b/packages/duckdb-wasm-app/src/app.tsx @@ -14,18 +14,27 @@ import 'react-popper-tooltip/dist/styles.css'; import * as duckdb from '@duckdb/duckdb-wasm'; import duckdb_wasm from '@duckdb/duckdb-wasm/dist/duckdb-mvp.wasm'; +/// #if EH_BUILD_ENABLED import duckdb_wasm_eh from '@duckdb/duckdb-wasm/dist/duckdb-eh.wasm'; +/// #endif +/// #if COI_BUILD_ENABLED import duckdb_wasm_coi from '@duckdb/duckdb-wasm/dist/duckdb-coi.wasm'; +/// #endif const DUCKDB_BUNDLES: duckdb.DuckDBBundles = { mvp: { mainModule: duckdb_wasm, mainWorker: new URL('@duckdb/duckdb-wasm/dist/duckdb-browser-mvp.worker.js', import.meta.url).toString(), }, + /// #if EH_BUILD_ENABLED eh: { mainModule: duckdb_wasm_eh, mainWorker: new URL('@duckdb/duckdb-wasm/dist/duckdb-browser-eh.worker.js', import.meta.url).toString(), }, + /// #else + eh: undefined, + /// #endif + /// #if COI_BUILD_ENABLED coi: { mainModule: duckdb_wasm_coi, mainWorker: new URL('@duckdb/duckdb-wasm/dist/duckdb-browser-coi.worker.js', import.meta.url).toString(), @@ -34,6 +43,9 @@ const DUCKDB_BUNDLES: duckdb.DuckDBBundles = { import.meta.url, ).toString(), }, + /// #else + coi: undefined, + /// #endif }; const logger = new duckdb.ConsoleLogger(duckdb.LogLevel.WARNING); diff --git a/packages/duckdb-wasm-app/webpack.common.js b/packages/duckdb-wasm-app/webpack.common.js index 99fe6f52e..e6bf708ac 100644 --- a/packages/duckdb-wasm-app/webpack.common.js +++ b/packages/duckdb-wasm-app/webpack.common.js @@ -33,9 +33,20 @@ export function configure(params) { }, { test: /\.tsx?$/, - loader: 'ts-loader', exclude: /node_modules/, - options: params.tsLoaderOptions, + use: [ + { + loader: 'ts-loader', + options: params.tsLoaderOptions, + }, + { + loader: 'ifdef-loader', + options: { + EH_BUILD_ENABLED: process.env['DUCKDB_SKIP_BUILD_EH'] !== '1', + COI_BUILD_ENABLED: process.env['DUCKDB_SKIP_BUILD_COI'] !== '1', + }, + }, + ], }, { test: /\.css$/, diff --git a/packages/duckdb-wasm/bundle.mjs b/packages/duckdb-wasm/bundle.mjs index 005371cb5..663990270 100644 --- a/packages/duckdb-wasm/bundle.mjs +++ b/packages/duckdb-wasm/bundle.mjs @@ -56,6 +56,14 @@ function printErr(err) { if (err) return console.log(err); } +function ehBuildEnabled() { + return process.env['DUCKDB_SKIP_BUILD_EH'] !== '1'; +} + +function coiBuildEnabled() { + return process.env['DUCKDB_SKIP_BUILD_COI'] !== '1'; +} + // Patch broken arrow package.json // XXX Remove this hack as soon as arrow fixes the exports function patch_arrow() { @@ -179,49 +187,53 @@ fs.copyFile(path.resolve(src, 'bindings', 'duckdb-coi.wasm'), path.resolve(dist, define: { 'process.release.name': '"browser"' }, }); - console.log('[ ESBUILD ] duckdb-browser-eh.worker.js'); - await esbuild.build({ - entryPoints: ['./src/targets/duckdb-browser-eh.worker.ts'], - outfile: 'dist/duckdb-browser-eh.worker.js', - platform: 'browser', - format: 'iife', - globalName: 'duckdb', - target: TARGET_BROWSER, - bundle: true, - minify: true, - sourcemap: is_debug ? 'inline' : true, - external: EXTERNALS_WEBWORKER, - define: { 'process.release.name': '"browser"' }, - }); + if (ehBuildEnabled()) { + console.log('[ ESBUILD ] duckdb-browser-eh.worker.js'); + await esbuild.build({ + entryPoints: ['./src/targets/duckdb-browser-eh.worker.ts'], + outfile: 'dist/duckdb-browser-eh.worker.js', + platform: 'browser', + format: 'iife', + globalName: 'duckdb', + target: TARGET_BROWSER, + bundle: true, + minify: true, + sourcemap: is_debug ? 'inline' : true, + external: EXTERNALS_WEBWORKER, + define: { 'process.release.name': '"browser"' }, + }); + } - console.log('[ ESBUILD ] duckdb-browser-coi.worker.js'); - await esbuild.build({ - entryPoints: ['./src/targets/duckdb-browser-coi.worker.ts'], - outfile: 'dist/duckdb-browser-coi.worker.js', - platform: 'browser', - format: 'iife', - globalName: 'duckdb', - target: TARGET_BROWSER, - bundle: true, - minify: true, - sourcemap: is_debug ? 'inline' : true, - external: EXTERNALS_WEBWORKER, - define: { 'process.release.name': '"browser"' }, - }); + if (coiBuildEnabled()) { + console.log('[ ESBUILD ] duckdb-browser-coi.worker.js'); + await esbuild.build({ + entryPoints: ['./src/targets/duckdb-browser-coi.worker.ts'], + outfile: 'dist/duckdb-browser-coi.worker.js', + platform: 'browser', + format: 'iife', + globalName: 'duckdb', + target: TARGET_BROWSER, + bundle: true, + minify: true, + sourcemap: is_debug ? 'inline' : true, + external: EXTERNALS_WEBWORKER, + define: { 'process.release.name': '"browser"' }, + }); - console.log('[ ESBUILD ] duckdb-browser-coi.pthread.worker.js'); - await esbuild.build({ - entryPoints: ['./src/targets/duckdb-browser-coi.pthread.worker.ts'], - outfile: 'dist/duckdb-browser-coi.pthread.worker.js', - platform: 'browser', - format: 'iife', - target: TARGET_BROWSER, - bundle: true, - minify: true, - sourcemap: is_debug ? 'inline' : true, - external: EXTERNALS_WEBWORKER, - define: { 'process.release.name': '"browser"' }, - }); + console.log('[ ESBUILD ] duckdb-browser-coi.pthread.worker.js'); + await esbuild.build({ + entryPoints: ['./src/targets/duckdb-browser-coi.pthread.worker.ts'], + outfile: 'dist/duckdb-browser-coi.pthread.worker.js', + platform: 'browser', + format: 'iife', + target: TARGET_BROWSER, + bundle: true, + minify: true, + sourcemap: is_debug ? 'inline' : true, + external: EXTERNALS_WEBWORKER, + define: { 'process.release.name': '"browser"' }, + }); + } // ------------------------------- // Node bundles @@ -266,18 +278,20 @@ fs.copyFile(path.resolve(src, 'bindings', 'duckdb-coi.wasm'), path.resolve(dist, external: EXTERNALS_NODE, }); - console.log('[ ESBUILD ] duckdb-node-eh.worker.cjs'); - await esbuild.build({ - entryPoints: ['./src/targets/duckdb-node-eh.worker.ts'], - outfile: 'dist/duckdb-node-eh.worker.cjs', - platform: 'node', - format: 'cjs', - target: TARGET_NODE, - bundle: true, - minify: true, - sourcemap: is_debug ? 'inline' : true, - external: EXTERNALS_NODE, - }); + if (ehBuildEnabled()) { + console.log('[ ESBUILD ] duckdb-node-eh.worker.cjs'); + await esbuild.build({ + entryPoints: ['./src/targets/duckdb-node-eh.worker.ts'], + outfile: 'dist/duckdb-node-eh.worker.cjs', + platform: 'node', + format: 'cjs', + target: TARGET_NODE, + bundle: true, + minify: true, + sourcemap: is_debug ? 'inline' : true, + external: EXTERNALS_NODE, + }); + } // ------------------------------- // Test bundles From 52638c1a9a402ca74ce623b3283b3c1cdb179317 Mon Sep 17 00:00:00 2001 From: Haruki Okada Date: Sun, 3 Sep 2023 08:21:33 +0900 Subject: [PATCH 02/13] fix variable propagation --- Makefile | 25 +++++++++++++------------ lib/CMakeLists.txt | 12 +++++++----- 2 files changed, 20 insertions(+), 17 deletions(-) diff --git a/Makefile b/Makefile index 06287fe67..60d050022 100644 --- a/Makefile +++ b/Makefile @@ -28,6 +28,7 @@ JS_FILTER= EXTENSION_CACHE_DIR="${ROOT_DIR}/.ccache/extension" EXCEL_EXTENSION_CACHE_FILE="${EXTENSION_CACHE_DIR}/excel" JSON_EXTENSION_CACHE_FILE="${EXTENSION_CACHE_DIR}/json" +CUSTOM_EXTENSION_ENVIRONMENT=DUCKDB_WEB_CUSTOM_EXTENSION_DIRS="${CUSTOM_EXTENSION_DIRS}" cpp_lib: lib_tests @@ -256,42 +257,42 @@ wasm_setup: set_environment check_duckdb wrapped_wasm_caches .PHONY: wasm_dev wasm_dev: wasm_setup - DUCKDB_WEB_CUSTOM_EXTENSION_DIRS=${CUSTOM_EXTENSION_DIRS} ${EXEC_ENVIRONMENT} ${ROOT_DIR}/scripts/wasm_build_lib.sh dev mvp + ${CUSTOM_EXTENSION_ENVIRONMENT} ${EXEC_ENVIRONMENT} ${ROOT_DIR}/scripts/wasm_build_lib.sh dev mvp ifneq (${DUCKDB_SKIP_BUILD_EH}, 1) - DUCKDB_WEB_CUSTOM_EXTENSION_DIRS=${CUSTOM_EXTENSION_DIRS} ${EXEC_ENVIRONMENT} ${ROOT_DIR}/scripts/wasm_build_lib.sh dev eh + ${CUSTOM_EXTENSION_ENVIRONMENT} ${EXEC_ENVIRONMENT} ${ROOT_DIR}/scripts/wasm_build_lib.sh dev eh endif ifneq (${DUCKDB_SKIP_BUILD_COI}, 1) - DUCKDB_WEB_CUSTOM_EXTENSION_DIRS=${CUSTOM_EXTENSION_DIRS} ${EXEC_ENVIRONMENT} ${ROOT_DIR}/scripts/wasm_build_lib.sh dev coi + ${CUSTOM_EXTENSION_ENVIRONMENT} ${EXEC_ENVIRONMENT} ${ROOT_DIR}/scripts/wasm_build_lib.sh dev coi endif .PHONY: wasm_relperf wasm_relperf: wasm_setup - DUCKDB_WEB_CUSTOM_EXTENSION_DIRS=${CUSTOM_EXTENSION_DIRS} ${EXEC_ENVIRONMENT} ${ROOT_DIR}/scripts/wasm_build_lib.sh relperf mvp + ${CUSTOM_EXTENSION_ENVIRONMENT} ${EXEC_ENVIRONMENT} ${ROOT_DIR}/scripts/wasm_build_lib.sh relperf mvp ifneq (${DUCKDB_SKIP_BUILD_EH}, 1) - DUCKDB_WEB_CUSTOM_EXTENSION_DIRS=${CUSTOM_EXTENSION_DIRS} ${EXEC_ENVIRONMENT} ${ROOT_DIR}/scripts/wasm_build_lib.sh relperf eh + ${CUSTOM_EXTENSION_ENVIRONMENT} ${EXEC_ENVIRONMENT} ${ROOT_DIR}/scripts/wasm_build_lib.sh relperf eh endif ifneq (${DUCKDB_SKIP_BUILD_COI}, 1) - DUCKDB_WEB_CUSTOM_EXTENSION_DIRS=${CUSTOM_EXTENSION_DIRS} ${EXEC_ENVIRONMENT} ${ROOT_DIR}/scripts/wasm_build_lib.sh relperf coi + ${CUSTOM_EXTENSION_ENVIRONMENT} ${EXEC_ENVIRONMENT} ${ROOT_DIR}/scripts/wasm_build_lib.sh relperf coi endif .PHONY: wasm_relsize wasm_relsize: wasm_setup - DUCKDB_WEB_CUSTOM_EXTENSION_DIRS=${CUSTOM_EXTENSION_DIRS} ${EXEC_ENVIRONMENT} ${ROOT_DIR}/scripts/wasm_build_lib.sh relsize mvp + ${CUSTOM_EXTENSION_ENVIRONMENT} ${EXEC_ENVIRONMENT} ${ROOT_DIR}/scripts/wasm_build_lib.sh relsize mvp ifneq (${DUCKDB_SKIP_BUILD_EH}, 1) - DUCKDB_WEB_CUSTOM_EXTENSION_DIRS=${CUSTOM_EXTENSION_DIRS} ${EXEC_ENVIRONMENT} ${ROOT_DIR}/scripts/wasm_build_lib.sh relsize eh + ${CUSTOM_EXTENSION_ENVIRONMENT} ${EXEC_ENVIRONMENT} ${ROOT_DIR}/scripts/wasm_build_lib.sh relsize eh endif ifneq (${DUCKDB_SKIP_BUILD_COI}, 1) - DUCKDB_WEB_CUSTOM_EXTENSION_DIRS=${CUSTOM_EXTENSION_DIRS} ${EXEC_ENVIRONMENT} ${ROOT_DIR}/scripts/wasm_build_lib.sh relsize coi + ${CUSTOM_EXTENSION_ENVIRONMENT} ${EXEC_ENVIRONMENT} ${ROOT_DIR}/scripts/wasm_build_lib.sh relsize coi endif .PHONY: wasm_debug wasm_debug: wasm_setup - DUCKDB_WEB_CUSTOM_EXTENSION_DIRS=${CUSTOM_EXTENSION_DIRS} ${EXEC_ENVIRONMENT} ${ROOT_DIR}/scripts/wasm_build_lib.sh debug mvp + ${CUSTOM_EXTENSION_ENVIRONMENT} ${EXEC_ENVIRONMENT} ${ROOT_DIR}/scripts/wasm_build_lib.sh debug mvp ifneq (${DUCKDB_SKIP_BUILD_EH}, 1) - DUCKDB_WEB_CUSTOM_EXTENSION_DIRS=${CUSTOM_EXTENSION_DIRS} ${EXEC_ENVIRONMENT} ${ROOT_DIR}/scripts/wasm_build_lib.sh debug eh + ${CUSTOM_EXTENSION_ENVIRONMENT} ${EXEC_ENVIRONMENT} ${ROOT_DIR}/scripts/wasm_build_lib.sh debug eh endif ifneq (${DUCKDB_SKIP_BUILD_COI}, 1) - DUCKDB_WEB_CUSTOM_EXTENSION_DIRS=${CUSTOM_EXTENSION_DIRS} ${EXEC_ENVIRONMENT} ${ROOT_DIR}/scripts/wasm_build_lib.sh debug coi + ${CUSTOM_EXTENSION_ENVIRONMENT} ${EXEC_ENVIRONMENT} ${ROOT_DIR}/scripts/wasm_build_lib.sh debug coi endif wasm: wasm_relperf diff --git a/lib/CMakeLists.txt b/lib/CMakeLists.txt index 1929f34ef..a183e14be 100644 --- a/lib/CMakeLists.txt +++ b/lib/CMakeLists.txt @@ -291,16 +291,18 @@ target_link_libraries(duckdb_web_json duckdb duckdb_json) # --------------------------------------------------------------------------- # Custom extensions -function(register_custom_extension name lib) - set(DUCKDB_WEB_CUSTOM_EXTENSION_NAMES ${DUCKDB_WEB_CUSTOM_EXTENSION_NAMES} ${name}) - set(DUCKDB_WEB_CUSTOM_EXTENSION_LIBS ${DUCKDB_WEB_CUSTOM_EXTENSION_LIBS} ${lib}) -endfunction() +macro(register_custom_extension name lib) + set(DUCKDB_WEB_CUSTOM_EXTENSION_NAMES "${DUCKDB_WEB_CUSTOM_EXTENSION_NAMES};${name}" PARENT_SCOPE) + set(DUCKDB_WEB_CUSTOM_EXTENSION_LIBS "${DUCKDB_WEB_CUSTOM_EXTENSION_LIBS};${lib}" PARENT_SCOPE) +endmacro() set(DUCKDB_WEB_CUSTOM_EXTENSION_NAMES "") set(DUCKDB_WEB_CUSTOM_EXTENSION_LIBS "") + if(DUCKDB_WEB_CUSTOM_EXTENSION_DIRS) foreach(dir ${DUCKDB_WEB_CUSTOM_EXTENSION_DIRS}) - add_subdirectory(${dir}) + message(STATUS "Adding custom extension dir ${dir}") + add_subdirectory(${dir} ${CMAKE_BINARY_DIR}/${dir}) endforeach() endif() From 3ebd1ee3849ca5203b9f575b263457d567e5b024 Mon Sep 17 00:00:00 2001 From: Haruki Okada Date: Sun, 3 Sep 2023 08:50:29 +0900 Subject: [PATCH 03/13] add a comment --- lib/extension/CMakeLists.template.txt | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/lib/extension/CMakeLists.template.txt b/lib/extension/CMakeLists.template.txt index db14267af..75f12435c 100644 --- a/lib/extension/CMakeLists.template.txt +++ b/lib/extension/CMakeLists.template.txt @@ -1,3 +1,7 @@ -set(DUCKDB_WEB_CUSTOM_EXTENSION_LIB "/path/to/libhello_ext.a") +# This is a CMake file template for a custom extension. +# This file is expected to be copied to the extension directory, and specified via +# `CUSTOM_EXTENSION_DIRS` variable for duckdb-wasm Makefile. -register_custom_extension("libhello_ext") +# This CMake file is expected to call `register_custom_extension` to tell +# duckdb-wasm CMake file to link the custom extension library and call the init-function at the startup. +register_custom_extension("libhello_ext" "/path/to/libhello_ext.a") From b19b8980e337ee633830fcec4165a63c0c779664 Mon Sep 17 00:00:00 2001 From: Haruki Okada Date: Sun, 3 Sep 2023 09:38:05 +0900 Subject: [PATCH 04/13] handle bundle error --- lib/extension/CMakeLists.template.txt | 2 ++ .../duckdb-wasm/src/targets/duckdb-browser-blocking.ts | 9 ++++++--- packages/duckdb-wasm/src/targets/duckdb-node-blocking.ts | 9 ++++++--- 3 files changed, 14 insertions(+), 6 deletions(-) diff --git a/lib/extension/CMakeLists.template.txt b/lib/extension/CMakeLists.template.txt index 75f12435c..9b4ed0c05 100644 --- a/lib/extension/CMakeLists.template.txt +++ b/lib/extension/CMakeLists.template.txt @@ -2,6 +2,8 @@ # This file is expected to be copied to the extension directory, and specified via # `CUSTOM_EXTENSION_DIRS` variable for duckdb-wasm Makefile. +cmake_minimum_required(VERSION 3.10) + # This CMake file is expected to call `register_custom_extension` to tell # duckdb-wasm CMake file to link the custom extension library and call the init-function at the startup. register_custom_extension("libhello_ext" "/path/to/libhello_ext.a") diff --git a/packages/duckdb-wasm/src/targets/duckdb-browser-blocking.ts b/packages/duckdb-wasm/src/targets/duckdb-browser-blocking.ts index a82e59e51..5c342ea97 100644 --- a/packages/duckdb-wasm/src/targets/duckdb-browser-blocking.ts +++ b/packages/duckdb-wasm/src/targets/duckdb-browser-blocking.ts @@ -11,7 +11,6 @@ import { Logger } from '../log'; import { DuckDBRuntime, DuckDBBindings } from '../bindings'; import { DuckDBBundles, getPlatformFeatures } from '../platform'; import { DuckDB as DuckDBMVP } from '../bindings/bindings_browser_mvp'; -import { DuckDB as DuckDBNext } from '../bindings/bindings_browser_eh'; export async function createDuckDB( bundles: DuckDBBundles, @@ -19,9 +18,13 @@ export async function createDuckDB( runtime: DuckDBRuntime, ): Promise { const platform = await getPlatformFeatures(); + const duckdbNext = await import('../bindings/bindings_browser_eh').catch(e => { + console.log(e); + return null; + }); if (platform.wasmExceptions) { - if (bundles.eh) { - return new DuckDBNext(logger, runtime, bundles.eh!.mainModule); + if (bundles.eh && duckdbNext) { + return new duckdbNext.DuckDB(logger, runtime, bundles.eh!.mainModule); } } return new DuckDBMVP(logger, runtime, bundles.mvp.mainModule); diff --git a/packages/duckdb-wasm/src/targets/duckdb-node-blocking.ts b/packages/duckdb-wasm/src/targets/duckdb-node-blocking.ts index d31ec1fe9..a2c60d892 100644 --- a/packages/duckdb-wasm/src/targets/duckdb-node-blocking.ts +++ b/packages/duckdb-wasm/src/targets/duckdb-node-blocking.ts @@ -12,7 +12,6 @@ import { DuckDBRuntime } from '../bindings'; import { DuckDBNodeBindings } from '../bindings/bindings_node_base'; import { DuckDBBundles, getPlatformFeatures } from '../platform'; import { DuckDB as DuckDBMVP } from '../bindings/bindings_node_mvp'; -import { DuckDB as DuckDBNext } from '../bindings/bindings_node_eh'; export async function createDuckDB( bundles: DuckDBBundles, @@ -20,9 +19,13 @@ export async function createDuckDB( runtime: DuckDBRuntime, ): Promise { const platform = await getPlatformFeatures(); + const duckdbNext = await import('../bindings/bindings_node_eh').catch(e => { + console.log(e); + return null; + }); if (platform.wasmExceptions) { - if (bundles.eh) { - return new DuckDBNext(logger, runtime, bundles.eh!.mainModule); + if (bundles.eh && duckdbNext) { + return new duckdbNext.DuckDB(logger, runtime, bundles.eh!.mainModule); } } return new DuckDBMVP(logger, runtime, bundles.mvp.mainModule); From d141519d1274b1874d7f8ad105a1b82f33f40980 Mon Sep 17 00:00:00 2001 From: Haruki Okada Date: Sun, 3 Sep 2023 09:47:57 +0900 Subject: [PATCH 05/13] fix --- packages/duckdb-wasm/bundle.mjs | 3 +++ .../duckdb-wasm/src/targets/duckdb-browser-blocking.ts | 9 +++------ packages/duckdb-wasm/src/targets/duckdb-node-blocking.ts | 9 +++------ 3 files changed, 9 insertions(+), 12 deletions(-) diff --git a/packages/duckdb-wasm/bundle.mjs b/packages/duckdb-wasm/bundle.mjs index 663990270..e68a232ff 100644 --- a/packages/duckdb-wasm/bundle.mjs +++ b/packages/duckdb-wasm/bundle.mjs @@ -152,6 +152,7 @@ fs.copyFile(path.resolve(src, 'bindings', 'duckdb-coi.wasm'), path.resolve(dist, define: { 'process.release.name': '"browser"', 'process.env.NODE_ENV': '"production"', + '__EH_BUILD_ENABLED__': ehBuildEnabled(), }, }); @@ -169,6 +170,7 @@ fs.copyFile(path.resolve(src, 'bindings', 'duckdb-coi.wasm'), path.resolve(dist, define: { 'process.release.name': '"browser"', 'process.env.NODE_ENV': '"production"', + '__EH_BUILD_ENABLED__': ehBuildEnabled(), }, }); @@ -263,6 +265,7 @@ fs.copyFile(path.resolve(src, 'bindings', 'duckdb-coi.wasm'), path.resolve(dist, minify: true, sourcemap: is_debug ? 'inline' : true, external: EXTERNALS_NODE, + define: { '__EH_BUILD_ENABLED__': ehBuildEnabled() }, }); console.log('[ ESBUILD ] duckdb-node-mvp.worker.cjs'); diff --git a/packages/duckdb-wasm/src/targets/duckdb-browser-blocking.ts b/packages/duckdb-wasm/src/targets/duckdb-browser-blocking.ts index 5c342ea97..b5ec32a99 100644 --- a/packages/duckdb-wasm/src/targets/duckdb-browser-blocking.ts +++ b/packages/duckdb-wasm/src/targets/duckdb-browser-blocking.ts @@ -18,13 +18,10 @@ export async function createDuckDB( runtime: DuckDBRuntime, ): Promise { const platform = await getPlatformFeatures(); - const duckdbNext = await import('../bindings/bindings_browser_eh').catch(e => { - console.log(e); - return null; - }); if (platform.wasmExceptions) { - if (bundles.eh && duckdbNext) { - return new duckdbNext.DuckDB(logger, runtime, bundles.eh!.mainModule); + if (bundles.eh && __EH_BUILD_ENABLED__) { + const duckdb = await import('../bindings/bindings_browser_eh'); + return new duckdb.DuckDB(logger, runtime, bundles.eh!.mainModule); } } return new DuckDBMVP(logger, runtime, bundles.mvp.mainModule); diff --git a/packages/duckdb-wasm/src/targets/duckdb-node-blocking.ts b/packages/duckdb-wasm/src/targets/duckdb-node-blocking.ts index a2c60d892..d798f1a54 100644 --- a/packages/duckdb-wasm/src/targets/duckdb-node-blocking.ts +++ b/packages/duckdb-wasm/src/targets/duckdb-node-blocking.ts @@ -19,13 +19,10 @@ export async function createDuckDB( runtime: DuckDBRuntime, ): Promise { const platform = await getPlatformFeatures(); - const duckdbNext = await import('../bindings/bindings_node_eh').catch(e => { - console.log(e); - return null; - }); if (platform.wasmExceptions) { - if (bundles.eh && duckdbNext) { - return new duckdbNext.DuckDB(logger, runtime, bundles.eh!.mainModule); + if (bundles.eh && __EH_BUILD_ENABLED__) { + const duckdb = await import('../bindings/bindings_node_eh'); + return new duckdb.DuckDB(logger, runtime, bundles.eh!.mainModule); } } return new DuckDBMVP(logger, runtime, bundles.mvp.mainModule); From 3bf458dec565c597c9b47719b168d2b39b3f9609 Mon Sep 17 00:00:00 2001 From: Haruki Okada Date: Sun, 3 Sep 2023 09:51:45 +0900 Subject: [PATCH 06/13] fix --- packages/duckdb-wasm/bundle.mjs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/duckdb-wasm/bundle.mjs b/packages/duckdb-wasm/bundle.mjs index e68a232ff..eae2ad232 100644 --- a/packages/duckdb-wasm/bundle.mjs +++ b/packages/duckdb-wasm/bundle.mjs @@ -152,7 +152,7 @@ fs.copyFile(path.resolve(src, 'bindings', 'duckdb-coi.wasm'), path.resolve(dist, define: { 'process.release.name': '"browser"', 'process.env.NODE_ENV': '"production"', - '__EH_BUILD_ENABLED__': ehBuildEnabled(), + '__EH_BUILD_ENABLED__': ehBuildEnabled() ? "1": "", }, }); @@ -170,7 +170,7 @@ fs.copyFile(path.resolve(src, 'bindings', 'duckdb-coi.wasm'), path.resolve(dist, define: { 'process.release.name': '"browser"', 'process.env.NODE_ENV': '"production"', - '__EH_BUILD_ENABLED__': ehBuildEnabled(), + '__EH_BUILD_ENABLED__': ehBuildEnabled() ? "1": "", }, }); @@ -265,7 +265,7 @@ fs.copyFile(path.resolve(src, 'bindings', 'duckdb-coi.wasm'), path.resolve(dist, minify: true, sourcemap: is_debug ? 'inline' : true, external: EXTERNALS_NODE, - define: { '__EH_BUILD_ENABLED__': ehBuildEnabled() }, + define: { '__EH_BUILD_ENABLED__': ehBuildEnabled() ? "1": "" }, }); console.log('[ ESBUILD ] duckdb-node-mvp.worker.cjs'); From 541f4bcbb3c26449528b8c151ad910a02ab5278a Mon Sep 17 00:00:00 2001 From: Haruki Okada Date: Sun, 3 Sep 2023 09:53:31 +0900 Subject: [PATCH 07/13] fix --- packages/duckdb-wasm/bundle.mjs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/duckdb-wasm/bundle.mjs b/packages/duckdb-wasm/bundle.mjs index eae2ad232..9168b2af6 100644 --- a/packages/duckdb-wasm/bundle.mjs +++ b/packages/duckdb-wasm/bundle.mjs @@ -152,7 +152,7 @@ fs.copyFile(path.resolve(src, 'bindings', 'duckdb-coi.wasm'), path.resolve(dist, define: { 'process.release.name': '"browser"', 'process.env.NODE_ENV': '"production"', - '__EH_BUILD_ENABLED__': ehBuildEnabled() ? "1": "", + '__EH_BUILD_ENABLED__': ehBuildEnabled() ? '"1"': '""', }, }); @@ -170,7 +170,7 @@ fs.copyFile(path.resolve(src, 'bindings', 'duckdb-coi.wasm'), path.resolve(dist, define: { 'process.release.name': '"browser"', 'process.env.NODE_ENV': '"production"', - '__EH_BUILD_ENABLED__': ehBuildEnabled() ? "1": "", + '__EH_BUILD_ENABLED__': ehBuildEnabled() ? '"1"': '""', }, }); @@ -265,7 +265,7 @@ fs.copyFile(path.resolve(src, 'bindings', 'duckdb-coi.wasm'), path.resolve(dist, minify: true, sourcemap: is_debug ? 'inline' : true, external: EXTERNALS_NODE, - define: { '__EH_BUILD_ENABLED__': ehBuildEnabled() ? "1": "" }, + define: { '__EH_BUILD_ENABLED__': ehBuildEnabled() ? '"1"': '""' }, }); console.log('[ ESBUILD ] duckdb-node-mvp.worker.cjs'); From 2dbaaa3dc18e9f6984a015f2e91a276203e91493 Mon Sep 17 00:00:00 2001 From: Haruki Okada Date: Sun, 3 Sep 2023 10:10:46 +0900 Subject: [PATCH 08/13] fix --- packages/duckdb-wasm/bundle.mjs | 14 +++++++++++--- packages/duckdb-wasm/package.json | 3 ++- .../src/targets/duckdb-browser-blocking.ts | 6 +++--- .../src/targets/duckdb-node-blocking.ts | 6 +++--- 4 files changed, 19 insertions(+), 10 deletions(-) diff --git a/packages/duckdb-wasm/bundle.mjs b/packages/duckdb-wasm/bundle.mjs index 9168b2af6..cf5b51c10 100644 --- a/packages/duckdb-wasm/bundle.mjs +++ b/packages/duckdb-wasm/bundle.mjs @@ -109,6 +109,14 @@ fs.copyFile(path.resolve(src, 'bindings', 'duckdb-coi.wasm'), path.resolve(dist, // ------------------------------- // Browser bundles + const pluginConfig = [ + ifdefPlugin({ + variables: { + EH_BUILD_ENABLED: ehBuildEnabled(), + }, + }) + ]; + console.log('[ ESBUILD ] duckdb-browser.cjs'); await esbuild.build({ entryPoints: ['./src/targets/duckdb.ts'], @@ -152,8 +160,8 @@ fs.copyFile(path.resolve(src, 'bindings', 'duckdb-coi.wasm'), path.resolve(dist, define: { 'process.release.name': '"browser"', 'process.env.NODE_ENV': '"production"', - '__EH_BUILD_ENABLED__': ehBuildEnabled() ? '"1"': '""', }, + plugins: pluginConfig, }); console.log('[ ESBUILD ] duckdb-browser-blocking.mjs'); @@ -170,8 +178,8 @@ fs.copyFile(path.resolve(src, 'bindings', 'duckdb-coi.wasm'), path.resolve(dist, define: { 'process.release.name': '"browser"', 'process.env.NODE_ENV': '"production"', - '__EH_BUILD_ENABLED__': ehBuildEnabled() ? '"1"': '""', }, + plugins: pluginConfig, }); console.log('[ ESBUILD ] duckdb-browser-mvp.worker.js'); @@ -265,7 +273,7 @@ fs.copyFile(path.resolve(src, 'bindings', 'duckdb-coi.wasm'), path.resolve(dist, minify: true, sourcemap: is_debug ? 'inline' : true, external: EXTERNALS_NODE, - define: { '__EH_BUILD_ENABLED__': ehBuildEnabled() ? '"1"': '""' }, + plugins: pluginConfig, }); console.log('[ ESBUILD ] duckdb-node-mvp.worker.cjs'); diff --git a/packages/duckdb-wasm/package.json b/packages/duckdb-wasm/package.json index 033955c08..0f7665a48 100644 --- a/packages/duckdb-wasm/package.json +++ b/packages/duckdb-wasm/package.json @@ -56,7 +56,8 @@ "typedoc": "^0.25.0", "typescript": "^5.1.6", "wasm-feature-detect": "^1.5.1", - "web-worker": "^1.2.0" + "web-worker": "^1.2.0", + "esbuild-ifdef": "^0.2.0" }, "scripts": { "build:debug": "node bundle.mjs debug && tsc --emitDeclarationOnly", diff --git a/packages/duckdb-wasm/src/targets/duckdb-browser-blocking.ts b/packages/duckdb-wasm/src/targets/duckdb-browser-blocking.ts index b5ec32a99..a82e59e51 100644 --- a/packages/duckdb-wasm/src/targets/duckdb-browser-blocking.ts +++ b/packages/duckdb-wasm/src/targets/duckdb-browser-blocking.ts @@ -11,6 +11,7 @@ import { Logger } from '../log'; import { DuckDBRuntime, DuckDBBindings } from '../bindings'; import { DuckDBBundles, getPlatformFeatures } from '../platform'; import { DuckDB as DuckDBMVP } from '../bindings/bindings_browser_mvp'; +import { DuckDB as DuckDBNext } from '../bindings/bindings_browser_eh'; export async function createDuckDB( bundles: DuckDBBundles, @@ -19,9 +20,8 @@ export async function createDuckDB( ): Promise { const platform = await getPlatformFeatures(); if (platform.wasmExceptions) { - if (bundles.eh && __EH_BUILD_ENABLED__) { - const duckdb = await import('../bindings/bindings_browser_eh'); - return new duckdb.DuckDB(logger, runtime, bundles.eh!.mainModule); + if (bundles.eh) { + return new DuckDBNext(logger, runtime, bundles.eh!.mainModule); } } return new DuckDBMVP(logger, runtime, bundles.mvp.mainModule); diff --git a/packages/duckdb-wasm/src/targets/duckdb-node-blocking.ts b/packages/duckdb-wasm/src/targets/duckdb-node-blocking.ts index d798f1a54..d31ec1fe9 100644 --- a/packages/duckdb-wasm/src/targets/duckdb-node-blocking.ts +++ b/packages/duckdb-wasm/src/targets/duckdb-node-blocking.ts @@ -12,6 +12,7 @@ import { DuckDBRuntime } from '../bindings'; import { DuckDBNodeBindings } from '../bindings/bindings_node_base'; import { DuckDBBundles, getPlatformFeatures } from '../platform'; import { DuckDB as DuckDBMVP } from '../bindings/bindings_node_mvp'; +import { DuckDB as DuckDBNext } from '../bindings/bindings_node_eh'; export async function createDuckDB( bundles: DuckDBBundles, @@ -20,9 +21,8 @@ export async function createDuckDB( ): Promise { const platform = await getPlatformFeatures(); if (platform.wasmExceptions) { - if (bundles.eh && __EH_BUILD_ENABLED__) { - const duckdb = await import('../bindings/bindings_node_eh'); - return new duckdb.DuckDB(logger, runtime, bundles.eh!.mainModule); + if (bundles.eh) { + return new DuckDBNext(logger, runtime, bundles.eh!.mainModule); } } return new DuckDBMVP(logger, runtime, bundles.mvp.mainModule); From 29361d65a7e83f087d802e3a2b9a0ec19eeceb50 Mon Sep 17 00:00:00 2001 From: Haruki Okada Date: Sun, 3 Sep 2023 10:12:06 +0900 Subject: [PATCH 09/13] fix --- packages/duckdb-wasm/src/targets/duckdb-browser-blocking.ts | 4 ++++ packages/duckdb-wasm/src/targets/duckdb-node-blocking.ts | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/packages/duckdb-wasm/src/targets/duckdb-browser-blocking.ts b/packages/duckdb-wasm/src/targets/duckdb-browser-blocking.ts index a82e59e51..451836c0e 100644 --- a/packages/duckdb-wasm/src/targets/duckdb-browser-blocking.ts +++ b/packages/duckdb-wasm/src/targets/duckdb-browser-blocking.ts @@ -11,7 +11,9 @@ import { Logger } from '../log'; import { DuckDBRuntime, DuckDBBindings } from '../bindings'; import { DuckDBBundles, getPlatformFeatures } from '../platform'; import { DuckDB as DuckDBMVP } from '../bindings/bindings_browser_mvp'; +/// #if EH_BUILD_ENABLED import { DuckDB as DuckDBNext } from '../bindings/bindings_browser_eh'; +/// #endif export async function createDuckDB( bundles: DuckDBBundles, @@ -19,10 +21,12 @@ export async function createDuckDB( runtime: DuckDBRuntime, ): Promise { const platform = await getPlatformFeatures(); + /// #if EH_BUILD_ENABLED if (platform.wasmExceptions) { if (bundles.eh) { return new DuckDBNext(logger, runtime, bundles.eh!.mainModule); } } + /// #endif return new DuckDBMVP(logger, runtime, bundles.mvp.mainModule); } diff --git a/packages/duckdb-wasm/src/targets/duckdb-node-blocking.ts b/packages/duckdb-wasm/src/targets/duckdb-node-blocking.ts index d31ec1fe9..679fa54e4 100644 --- a/packages/duckdb-wasm/src/targets/duckdb-node-blocking.ts +++ b/packages/duckdb-wasm/src/targets/duckdb-node-blocking.ts @@ -12,7 +12,9 @@ import { DuckDBRuntime } from '../bindings'; import { DuckDBNodeBindings } from '../bindings/bindings_node_base'; import { DuckDBBundles, getPlatformFeatures } from '../platform'; import { DuckDB as DuckDBMVP } from '../bindings/bindings_node_mvp'; +/// #if EH_BUILD_ENABLED import { DuckDB as DuckDBNext } from '../bindings/bindings_node_eh'; +/// #endif export async function createDuckDB( bundles: DuckDBBundles, @@ -20,10 +22,12 @@ export async function createDuckDB( runtime: DuckDBRuntime, ): Promise { const platform = await getPlatformFeatures(); + /// #if EH_BUILD_ENABLED if (platform.wasmExceptions) { if (bundles.eh) { return new DuckDBNext(logger, runtime, bundles.eh!.mainModule); } } + /// #endif return new DuckDBMVP(logger, runtime, bundles.mvp.mainModule); } From 889d206eee3233d9b50109f405a674bfd74e8786 Mon Sep 17 00:00:00 2001 From: Haruki Okada Date: Sun, 3 Sep 2023 10:14:51 +0900 Subject: [PATCH 10/13] fix --- packages/duckdb-wasm/bundle.mjs | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/duckdb-wasm/bundle.mjs b/packages/duckdb-wasm/bundle.mjs index cf5b51c10..2f75d8128 100644 --- a/packages/duckdb-wasm/bundle.mjs +++ b/packages/duckdb-wasm/bundle.mjs @@ -4,6 +4,7 @@ import path from 'path'; import { rimrafSync } from 'rimraf'; import mkdir from 'make-dir'; import { fileURLToPath } from 'url'; +import ifdefPlugin from 'esbuild-ifdef' // ------------------------------- // Current bundling strategy From 78754c759a34950a4fa56d6653c01289488d8685 Mon Sep 17 00:00:00 2001 From: Haruki Okada Date: Sun, 3 Sep 2023 11:04:56 +0900 Subject: [PATCH 11/13] fix --- packages/duckdb-wasm/bundle.mjs | 13 ++++++------- packages/duckdb-wasm/package.json | 5 +++-- .../src/targets/duckdb-browser-blocking.ts | 8 ++++---- .../duckdb-wasm/src/targets/duckdb-node-blocking.ts | 8 ++++---- 4 files changed, 17 insertions(+), 17 deletions(-) diff --git a/packages/duckdb-wasm/bundle.mjs b/packages/duckdb-wasm/bundle.mjs index 2f75d8128..428c1ee26 100644 --- a/packages/duckdb-wasm/bundle.mjs +++ b/packages/duckdb-wasm/bundle.mjs @@ -4,7 +4,7 @@ import path from 'path'; import { rimrafSync } from 'rimraf'; import mkdir from 'make-dir'; import { fileURLToPath } from 'url'; -import ifdefPlugin from 'esbuild-ifdef' +import ifdef from 'esbuild-plugin-ifdef' // ------------------------------- // Current bundling strategy @@ -111,11 +111,7 @@ fs.copyFile(path.resolve(src, 'bindings', 'duckdb-coi.wasm'), path.resolve(dist, // Browser bundles const pluginConfig = [ - ifdefPlugin({ - variables: { - EH_BUILD_ENABLED: ehBuildEnabled(), - }, - }) + ifdef(ehBuildEnabled() ? { EH_BUILD_ENABLED: true } : {}), ]; console.log('[ ESBUILD ] duckdb-browser.cjs'); @@ -319,6 +315,7 @@ fs.copyFile(path.resolve(src, 'bindings', 'duckdb-coi.wasm'), path.resolve(dist, bundle: true, sourcemap: is_debug ? 'inline' : true, external: EXTERNALS_TEST_BROWSER, + plugins: pluginConfig, }); console.log('[ ESBUILD ] tests-node.cjs'); @@ -331,8 +328,10 @@ fs.copyFile(path.resolve(src, 'bindings', 'duckdb-coi.wasm'), path.resolve(dist, bundle: true, minify: false, sourcemap: is_debug ? 'inline' : true, - // web-worker polyfill needs to be excluded from bundling due to their dynamic require messing with bundled modules + // web-worker polyfill needs to be excluded from + // bundling due to their dynamic require messing with bundled modules external: [...EXTERNALS_NODE, 'web-worker'], + plugins: pluginConfig, }); // ------------------------------- diff --git a/packages/duckdb-wasm/package.json b/packages/duckdb-wasm/package.json index 0f7665a48..eb0321e4b 100644 --- a/packages/duckdb-wasm/package.json +++ b/packages/duckdb-wasm/package.json @@ -32,6 +32,8 @@ "@typescript-eslint/parser": "^6.7.3", "esbuild": "^0.18.13", "eslint": "^8.50.0", + "esbuild-ifdef": "^0.2.0", + "esbuild-plugin-ifdef": "^1.0.1", "eslint-plugin-jasmine": "^4.1.3", "eslint-plugin-react": "^7.33.2", "fast-glob": "^3.3.1", @@ -56,8 +58,7 @@ "typedoc": "^0.25.0", "typescript": "^5.1.6", "wasm-feature-detect": "^1.5.1", - "web-worker": "^1.2.0", - "esbuild-ifdef": "^0.2.0" + "web-worker": "^1.2.0" }, "scripts": { "build:debug": "node bundle.mjs debug && tsc --emitDeclarationOnly", diff --git a/packages/duckdb-wasm/src/targets/duckdb-browser-blocking.ts b/packages/duckdb-wasm/src/targets/duckdb-browser-blocking.ts index 451836c0e..8d660377f 100644 --- a/packages/duckdb-wasm/src/targets/duckdb-browser-blocking.ts +++ b/packages/duckdb-wasm/src/targets/duckdb-browser-blocking.ts @@ -11,9 +11,9 @@ import { Logger } from '../log'; import { DuckDBRuntime, DuckDBBindings } from '../bindings'; import { DuckDBBundles, getPlatformFeatures } from '../platform'; import { DuckDB as DuckDBMVP } from '../bindings/bindings_browser_mvp'; -/// #if EH_BUILD_ENABLED +//#ifdef EH_BUILD_ENABLED import { DuckDB as DuckDBNext } from '../bindings/bindings_browser_eh'; -/// #endif +//#endif export async function createDuckDB( bundles: DuckDBBundles, @@ -21,12 +21,12 @@ export async function createDuckDB( runtime: DuckDBRuntime, ): Promise { const platform = await getPlatformFeatures(); - /// #if EH_BUILD_ENABLED +//#ifdef EH_BUILD_ENABLED if (platform.wasmExceptions) { if (bundles.eh) { return new DuckDBNext(logger, runtime, bundles.eh!.mainModule); } } - /// #endif +//#endif return new DuckDBMVP(logger, runtime, bundles.mvp.mainModule); } diff --git a/packages/duckdb-wasm/src/targets/duckdb-node-blocking.ts b/packages/duckdb-wasm/src/targets/duckdb-node-blocking.ts index 679fa54e4..9b759382c 100644 --- a/packages/duckdb-wasm/src/targets/duckdb-node-blocking.ts +++ b/packages/duckdb-wasm/src/targets/duckdb-node-blocking.ts @@ -12,9 +12,9 @@ import { DuckDBRuntime } from '../bindings'; import { DuckDBNodeBindings } from '../bindings/bindings_node_base'; import { DuckDBBundles, getPlatformFeatures } from '../platform'; import { DuckDB as DuckDBMVP } from '../bindings/bindings_node_mvp'; -/// #if EH_BUILD_ENABLED +//#ifdef EH_BUILD_ENABLED import { DuckDB as DuckDBNext } from '../bindings/bindings_node_eh'; -/// #endif +//#endif export async function createDuckDB( bundles: DuckDBBundles, @@ -22,12 +22,12 @@ export async function createDuckDB( runtime: DuckDBRuntime, ): Promise { const platform = await getPlatformFeatures(); - /// #if EH_BUILD_ENABLED +//#ifdef EH_BUILD_ENABLED if (platform.wasmExceptions) { if (bundles.eh) { return new DuckDBNext(logger, runtime, bundles.eh!.mainModule); } } - /// #endif +//#endif return new DuckDBMVP(logger, runtime, bundles.mvp.mainModule); } From 562964cdb4406830c82d2a3f8a1dd6f752d78d38 Mon Sep 17 00:00:00 2001 From: Haruki Okada Date: Thu, 19 Oct 2023 22:13:34 +0900 Subject: [PATCH 12/13] update gitignore --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index f5d6fe310..ad5377f1a 100644 --- a/.gitignore +++ b/.gitignore @@ -28,3 +28,6 @@ compile_commands.json data/tpch/**/*.parquet .idea lib/.idea + +check_duckdb +wasm_setup From 250b885f59664fe078a707f0184eb86684a15f0a Mon Sep 17 00:00:00 2001 From: Haruki Okada Date: Thu, 19 Oct 2023 23:02:21 +0900 Subject: [PATCH 13/13] avoid unwanted yarn.lock update --- Makefile | 1 - examples/bare-browser/package.json | 2 +- examples/bare-node/package.json | 2 +- examples/esbuild-browser/package.json | 2 +- examples/esbuild-node/package.json | 2 +- packages/benchmarks/package.json | 2 +- packages/duckdb-wasm-app/package.json | 4 +- packages/duckdb-wasm-shell/package.json | 2 +- yarn.lock | 52 ++++++++++++++++++------- 9 files changed, 46 insertions(+), 23 deletions(-) diff --git a/Makefile b/Makefile index 60d050022..132753ad2 100644 --- a/Makefile +++ b/Makefile @@ -414,7 +414,6 @@ eslint: # Install all yarn packages .PHONY: yarn_install yarn_install: - yarn yarn install .PHONY: examples diff --git a/examples/bare-browser/package.json b/examples/bare-browser/package.json index 7c6322a3c..38cc9deb3 100644 --- a/examples/bare-browser/package.json +++ b/examples/bare-browser/package.json @@ -5,7 +5,7 @@ "main": "index.js", "license": "MIT", "dependencies": { - "@duckdb/duckdb-wasm": "file:../../packages/duckdb-wasm" + "@duckdb/duckdb-wasm": "1.11.0" }, "devDependencies": { "http-server": "^14.1.1" diff --git a/examples/bare-node/package.json b/examples/bare-node/package.json index 8800d9a0f..c68dfca4f 100644 --- a/examples/bare-node/package.json +++ b/examples/bare-node/package.json @@ -5,7 +5,7 @@ "main": "index.js", "license": "MIT", "dependencies": { - "@duckdb/duckdb-wasm": "file:../../packages/duckdb-wasm" + "@duckdb/duckdb-wasm": "1.11.0" }, "scripts": { "test": "node ./index.cjs" diff --git a/examples/esbuild-browser/package.json b/examples/esbuild-browser/package.json index f6f40ba69..015b497f0 100644 --- a/examples/esbuild-browser/package.json +++ b/examples/esbuild-browser/package.json @@ -5,7 +5,7 @@ "main": "index.js", "license": "MIT", "dependencies": { - "@duckdb/duckdb-wasm": "file:../../packages/duckdb-wasm" + "@duckdb/duckdb-wasm": "1.11.0" }, "devDependencies": { "esbuild": "^0.18.13", diff --git a/examples/esbuild-node/package.json b/examples/esbuild-node/package.json index d1a354639..fb884de91 100644 --- a/examples/esbuild-node/package.json +++ b/examples/esbuild-node/package.json @@ -5,7 +5,7 @@ "main": "index.js", "license": "MIT", "dependencies": { - "@duckdb/duckdb-wasm": "file:../../packages/duckdb-wasm" + "@duckdb/duckdb-wasm": "1.11.0" }, "devDependencies": { "esbuild": "^0.18.13", diff --git a/packages/benchmarks/package.json b/packages/benchmarks/package.json index dda67ff25..57c1eb3cc 100644 --- a/packages/benchmarks/package.json +++ b/packages/benchmarks/package.json @@ -4,7 +4,7 @@ "license": "MIT", "private": true, "dependencies": { - "@duckdb/duckdb-wasm": "file:../duckdb-wasm", + "@duckdb/duckdb-wasm": "1.11.0", "apache-arrow": "^13.0.0", "apache-arrow-3": "npm:apache-arrow@^3.0.0", "arquero": "^5.2.0", diff --git a/packages/duckdb-wasm-app/package.json b/packages/duckdb-wasm-app/package.json index 4255ff40a..493d7d3d8 100644 --- a/packages/duckdb-wasm-app/package.json +++ b/packages/duckdb-wasm-app/package.json @@ -18,8 +18,8 @@ "node": ">=14.13" }, "dependencies": { - "@duckdb/duckdb-wasm": "file:../duckdb-wasm", - "@duckdb/duckdb-wasm-shell": "file:../duckdb-wasm-shell", + "@duckdb/duckdb-wasm": "1.11.0", + "@duckdb/duckdb-wasm-shell": "1.11.0", "apache-arrow": "^13.0.0", "bootstrap": "^5.3.2", "classnames": "^2.3.2", diff --git a/packages/duckdb-wasm-shell/package.json b/packages/duckdb-wasm-shell/package.json index e32517918..17bde4db6 100644 --- a/packages/duckdb-wasm-shell/package.json +++ b/packages/duckdb-wasm-shell/package.json @@ -24,7 +24,7 @@ "csv" ], "dependencies": { - "@duckdb/duckdb-wasm": "file:../duckdb-wasm", + "@duckdb/duckdb-wasm": "1.11.0", "xterm": "^5.1.0", "xterm-addon-fit": "^0.7.0", "xterm-addon-web-links": "^0.9.0", diff --git a/yarn.lock b/yarn.lock index 253a6d5b6..eee5aa398 100644 --- a/yarn.lock +++ b/yarn.lock @@ -456,20 +456,6 @@ resolved "https://registry.yarnpkg.com/@discoveryjs/json-ext/-/json-ext-0.5.7.tgz#1d572bfbbe14b7704e0ba0f39b74815b84870d70" integrity sha512-dBVuXR082gk3jsFp7Rd/JI4kytwGHecnCoTtXFb7DB6CNHp4rg5k1bhg0nWdLGLnOV71lmDzGQaLMy8iPLY0pw== -"@duckdb/duckdb-wasm-shell@file:packages/duckdb-wasm-shell": - version "1.11.0" - dependencies: - "@duckdb/duckdb-wasm" "file:packages/duckdb-wasm" - xterm "^5.1.0" - xterm-addon-fit "^0.7.0" - xterm-addon-web-links "^0.9.0" - xterm-addon-webgl "^0.16.0" - -"@duckdb/duckdb-wasm@file:packages/duckdb-wasm": - version "1.11.0" - dependencies: - apache-arrow "^13.0.0" - "@emotion/is-prop-valid@^0.8.2": version "0.8.8" resolved "https://registry.yarnpkg.com/@emotion/is-prop-valid/-/is-prop-valid-0.8.8.tgz#db28b1c4368a259b60a97311d6a952d4fd01ac1a" @@ -3101,6 +3087,16 @@ es6-error@^4.0.1: resolved "https://registry.yarnpkg.com/es6-error/-/es6-error-4.1.1.tgz#9e3af407459deed47e9a91f9b885a84eb05c561d" integrity sha512-Um/+FxMr9CISWh0bi5Zv0iOD+4cFh5qLeks1qhAopKVAJw3drgKbKySikp7wGhDL0HPeaja0P5ULZrxLkniUVg== +esbuild-ifdef@^0.2.0: + version "0.2.0" + resolved "https://registry.yarnpkg.com/esbuild-ifdef/-/esbuild-ifdef-0.2.0.tgz#7f6163a0358f8d747935009fb38d042514668602" + integrity sha512-1+QJyEI3hIN5SY56MBgnfOQWqNTjfRBbkGKyTHvL44A+Q/NlKwJ79w1nZcIjHhvouM7ArngK99yiMjd6G3iC6A== + +esbuild-plugin-ifdef@^1.0.1: + version "1.0.1" + resolved "https://registry.yarnpkg.com/esbuild-plugin-ifdef/-/esbuild-plugin-ifdef-1.0.1.tgz#64f281879725231b6d1493b0f38ab1ae1ac80313" + integrity sha512-dn1sf+WGIUBp4kxN+rJRYIXrmGTwRLyhWASmhl8IKKKPfIlWFE8hloAVnI6mM5jPF3+abDvkndYe+RZDFXItbA== + esbuild@^0.18.13: version "0.18.13" resolved "https://registry.yarnpkg.com/esbuild/-/esbuild-0.18.13.tgz#59160add6c3420947fe008238140ed3480baf817" @@ -4144,6 +4140,13 @@ ieee754@^1.1.13: resolved "https://registry.yarnpkg.com/ieee754/-/ieee754-1.2.1.tgz#8eb7a10a63fff25d15a57b001586d177d1b0d352" integrity sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA== +ifdef-loader@^2.3.2: + version "2.3.2" + resolved "https://registry.yarnpkg.com/ifdef-loader/-/ifdef-loader-2.3.2.tgz#d08972f53c32e90dce6722b6efb9b3a91717cfef" + integrity sha512-kH9bHPrfIFxLpq3XEruJqSlHXch2nOljKIDRS/6MU5LDZTyHeaSWVf04wNYX+8RT+NDmeS8Vm5HwZ7akkXo8ig== + dependencies: + loader-utils "^1.1.0" + ignore@^5.2.0, ignore@^5.2.4: version "5.2.4" resolved "https://registry.yarnpkg.com/ignore/-/ignore-5.2.4.tgz#a291c0c6178ff1b960befe47fcdec301674a6324" @@ -4765,6 +4768,13 @@ json-stable-stringify-without-jsonify@^1.0.1: resolved "https://registry.yarnpkg.com/json-stable-stringify-without-jsonify/-/json-stable-stringify-without-jsonify-1.0.1.tgz#9db7b59496ad3f3cfef30a75142d2d930ad72651" integrity sha1-nbe1lJatPzz+8wp1FC0tkwrXJlE= +json5@^1.0.1: + version "1.0.2" + resolved "https://registry.yarnpkg.com/json5/-/json5-1.0.2.tgz#63d98d60f21b313b77c4d6da18bfa69d80e1d593" + integrity sha512-g1MWMLBiz8FKi1e4w0UyVL3w+iJceWAFBAaBnnGKOpNa5f8TLktkbre1+s6oICydWAm+HRUGTmI+//xv2hvXYA== + dependencies: + minimist "^1.2.0" + json5@^2.1.2: version "2.2.0" resolved "https://registry.yarnpkg.com/json5/-/json5-2.2.0.tgz#2dfefe720c6ba525d9ebd909950f0515316c89a3" @@ -5012,6 +5022,15 @@ loader-runner@^4.2.0: resolved "https://registry.yarnpkg.com/loader-runner/-/loader-runner-4.2.0.tgz#d7022380d66d14c5fb1d496b89864ebcfd478384" integrity sha512-92+huvxMvYlMzMt0iIOukcwYBFpkYJdpl2xsZ7LrlayO7E8SOv+JJUEK17B/dJIHAOLMfh2dZZ/Y18WgmGtYNw== +loader-utils@^1.1.0: + version "1.4.2" + resolved "https://registry.yarnpkg.com/loader-utils/-/loader-utils-1.4.2.tgz#29a957f3a63973883eb684f10ffd3d151fec01a3" + integrity sha512-I5d00Pd/jwMD2QCduo657+YM/6L3KZu++pmX9VFncxaxvHcru9jx1lBaFft+r4Mt2jK0Yhp41XlRAihzPxHNCg== + dependencies: + big.js "^5.2.2" + emojis-list "^3.0.0" + json5 "^1.0.1" + loader-utils@^2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/loader-utils/-/loader-utils-2.0.0.tgz#e4cace5b816d425a166b5f097e10cd12b36064b0" @@ -5283,6 +5302,11 @@ minimatch@^9.0.1, minimatch@^9.0.3: dependencies: brace-expansion "^2.0.1" +minimist@^1.2.0: + version "1.2.8" + resolved "https://registry.yarnpkg.com/minimist/-/minimist-1.2.8.tgz#c1a464e7693302e082a075cee0c057741ac4772c" + integrity sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA== + minimist@^1.2.5, minimist@^1.2.6: version "1.2.6" resolved "https://registry.yarnpkg.com/minimist/-/minimist-1.2.6.tgz#8637a5b759ea0d6e98702cfb3a9283323c93af44"