diff --git a/.buildkite/builder-docker-image.yml b/.buildkite/builder-docker-image.yml new file mode 100644 index 0000000000000..2e5b70016917d --- /dev/null +++ b/.buildkite/builder-docker-image.yml @@ -0,0 +1,30 @@ +env: + DOCKERHUB_REPO: electricsql + IMAGE_NAME: pglite-builder + +agent: + docker: true + gcp: true + +steps: + - label: '🚀 Build & publish the builder image to Docker Hub' + if: build.env("TRIGGERED_FROM_GHA") == 'true' + command: + - | + source ./pglite/.buildconfig + export PG_VERSION=$$PG_VERSION + export SDK_VERSION=$$SDK_VERSION + export IMGNAME="${DOCKERHUB_REPO}/${IMAGE_NAME}" + export IMGTAG="$${PG_VERSION}_$${SDK_VERSION}" + - echo "Building with PG_VERSION $$PG_VERSION and SDK_VERSION $$SDK_VERSION" + - | + if docker manifest inspect $${IMGNAME}:$${IMGTAG} >/dev/null 2>&1; then + echo "Image $${IMGNAME}:$${IMGTAG} already exists on Docker Hub. Exiting successfully." + exit 0 + fi + - docker buildx build --platform linux/arm64/v8,linux/amd64 --push + --build-arg PG_VERSION=$${PG_VERSION} + --build-arg SDK_VERSION=$${SDK_VERSION} + -t $${IMGNAME}:$${IMGTAG} + -t $${IMGNAME}:latest + ./pglite diff --git a/.github/workflows/build_wasm_postgres.yml b/.github/workflows/build_wasm_postgres.yml new file mode 100644 index 0000000000000..3596387eef97a --- /dev/null +++ b/.github/workflows/build_wasm_postgres.yml @@ -0,0 +1,39 @@ +name: Build postgres-pglite (aka Postgresql WASM) + +permissions: + pull-requests: write + issues: write + contents: read + +on: + workflow_call: + push: + branches: ['pglite/REL_16_STABLE'] + pull_request: + +jobs: + build: + name: Build postgres-pglite (WASM Postgres) + runs-on: ubuntu-22.04 + env: + BUILD_CONFIG_FILE: pglite/.buildconfig + PGROOT: /tmp/pglite + TOTAL_MEMORY: 128MB + CMA_MB: 16 + DEBUG: false + OBJDUMP: true + contrib: contrib + extra: extra + EXTRA_EXT: vector + + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 1 + + - name: Build postgres WASM via emsdk + working-directory: ${{ github.workspace }} + env: + PGSRC: ${{ github.workspace }} + run: | + pglite/cibuild/build-with-docker.sh diff --git a/.github/workflows/trigger_buildkite_build.yml b/.github/workflows/trigger_buildkite_build.yml new file mode 100644 index 0000000000000..1ae4895f4de09 --- /dev/null +++ b/.github/workflows/trigger_buildkite_build.yml @@ -0,0 +1,25 @@ +name: Trigger BuildKite build of Docker image for building PGlite + +on: + push: + branches: ['pglite/REL_16_STABLE'] + paths: + - pglite/.buildconfig + pull_request: + paths: + - pglite/.buildconfig + +jobs: + build: + name: Build Docker image + runs-on: ubuntu-22.04 + steps: + - name: Trigger a Buildkite Build on Push using v2.0.0 + uses: buildkite/trigger-pipeline-action@v2.0.0 + with: + buildkite_api_access_token: ${{ secrets.TRIGGER_BK_BUILD_TOKEN }} + pipeline: 'vaxine/build-pglite-builder-docker-image' + branch: ${{ github.head_ref || github.ref_name }} + commit: HEAD + message: '🚀 Triggered from a GitHub Action' + build_env_vars: '{"TRIGGERED_FROM_GHA": "true"}' diff --git a/README b/README index 6416a8cf3b467..e1cb210534d24 100644 --- a/README +++ b/README @@ -1,3 +1,12 @@ +PGlite PostgreSQL Fork +====================== + +This is a fork of PostgreSQL used by PGlite. + +For more information, please visit the PGlite website at https://pglite.dev or see the main PGlite repository at https://github.com/electric-sql/pglite. + +We use the issues tracker on the main PGlite repository to track issues with this fork. + PostgreSQL Database Management System ===================================== diff --git a/contrib/pgstattuple/pgstatindex.c b/contrib/pgstattuple/pgstatindex.c index 5c06ba6db438e..0ea74cef43f23 100644 --- a/contrib/pgstattuple/pgstatindex.c +++ b/contrib/pgstattuple/pgstatindex.c @@ -656,9 +656,9 @@ pgstathashindex(PG_FUNCTION_ARGS) stats.unused_pages++; else if (PageGetSpecialSize(page) != MAXALIGN(sizeof(HashPageOpaqueData))) - ereport(ERROR, + ereport(WARNING, (errcode(ERRCODE_INDEX_CORRUPTED), - errmsg("index \"%s\" contains corrupted page at block %u", + errmsg("# 661(FATAL block=%d): index \"%s\" contains corrupted page at block %u", blkno, RelationGetRelationName(rel), BufferGetBlockNumber(buf)))); else diff --git a/contrib/xml2/Makefile b/contrib/xml2/Makefile index 0d703fe0e8f94..926649f63d3f2 100644 --- a/contrib/xml2/Makefile +++ b/contrib/xml2/Makefile @@ -11,7 +11,7 @@ DATA = xml2--1.1.sql xml2--1.0--1.1.sql PGFILEDESC = "xml2 - XPath querying and XSLT" REGRESS = xml2 - +PG_CFLAGS=$(shell xml2-config --cflags) SHLIB_LINK += $(filter -lxslt, $(LIBS)) -lxml2 ifdef USE_PGXS diff --git a/pglite/.buildconfig b/pglite/.buildconfig new file mode 100644 index 0000000000000..7c5ca9b9badbc --- /dev/null +++ b/pglite/.buildconfig @@ -0,0 +1,4 @@ +PG_VERSION=16.4 +SDK_VERSION=3.1.74.3bi +WASI_SDK_VERSION=24.0.4 +SDKROOT=/opt/python-wasm-sdk diff --git a/pglite/.gitignore b/pglite/.gitignore new file mode 100644 index 0000000000000..c8077e173e7f6 --- /dev/null +++ b/pglite/.gitignore @@ -0,0 +1,4 @@ +patches/imports.pgcore +dist +patches/imports +patches/exports diff --git a/pglite/Dockerfile b/pglite/Dockerfile new file mode 100644 index 0000000000000..5b2f73fe2c2b8 --- /dev/null +++ b/pglite/Dockerfile @@ -0,0 +1,68 @@ +FROM ubuntu:22.04 AS build_sdk + +ARG PG_VERSION=16.4 +ARG SDK_VERSION=3.1.74.3bi +ARG DEBUG=false +ARG OBJDUMP=true + +ENV \ + PG_VERSION=$PG_VERSION \ + SDK_VERSION=$SDK_VERSION \ + SDKROOT=/opt/python-wasm-sdk \ + SYS_PYTHON=/usr/bin/python3 \ + DEBUG=$DEBUG \ + BUILDS=3.12 \ + EMFLAVOUR=3.1.74 + +WORKDIR /workspace + +# Install dependencies to build python-wasm-sdk +RUN \ + apt-get update &&\ + apt-get install -y \ + sudo patchelf git clang unzip autoconf libtool \ + libsqlite3-dev libffi-dev libssl-dev zlib1g-dev pkg-config \ + libncursesw5-dev python3 python3-pip \ + make build-essential wget lz4 bzip2 pv curl + +# Download the python-wasm-sdk source for the given version +# RUN git clone --depth 1 --branch ${SDK_VERSION} https://github.com/pygame-web/python-wasm-sdk.git +RUN git clone --depth 1 https://github.com/electric-sql/portable-sdk.git + +WORKDIR /workspace/portable-sdk/python-wasm-sdk + +# Make python-wasm-sdk +RUN chmod +x python-wasm-sdk.sh + +RUN bash -c "./python-wasm-sdk.sh" + +FROM ubuntu:22.04 + +# Install dependencies to build postgres wasm +RUN \ + apt-get update &&\ + apt-get install -y lz4 wget pv bash bzip2 python3 build-essential libreadline-dev zlib1g-dev bison flex &&\ + apt-get clean + +ARG PG_VERSION=16.4 +ARG SDK_VERSION=3.1.74.3bi +ARG DEBUG=false +ARG OBJDUMP=true + +ENV \ + PG_VERSION=$PG_VERSION \ + SDK_VERSION=$SDK_VERSION \ + SDK_ARCHIVE=python3.12-wasm-sdk-Ubuntu-22.04.tar.lz4 \ + SDKROOT=/opt/python-wasm-sdk \ + SYS_PYTHON=/usr/bin/python3 \ + PGROOT=/tmp/pglite \ + DEBUG=$DEBUG \ + OBJDUMP=$OBJDUMP + +COPY --from=build_sdk /tmp/sdk /tmp/sdk + +RUN tar --use-compress-program=lz4 -xvf /tmp/sdk/${SDK_ARCHIVE} -C / + +RUN rm -r /tmp/sdk + +WORKDIR /workspace \ No newline at end of file diff --git a/pglite/cibuild.sh b/pglite/cibuild.sh new file mode 100755 index 0000000000000..442e92f1ca4c2 --- /dev/null +++ b/pglite/cibuild.sh @@ -0,0 +1,577 @@ +#!/bin/bash + +# data transfer zone this is == (wire query size + result size ) + 2 +# expressed in EMSDK MB +export CMA_MB=${CMA_MB:-64} + +export CI=${CI:-false} + +chmod +x ./extra/*.sh cibuild/*.sh + +. .buildconfig + +export PG_VERSION SDK_VERSION WASI_SDK_VERSION SDKROOT + +export PG_VERSION=${PG_VERSION:-16.4} +# export WORKSPACE=${GITHUB_WORKSPACE:-$(pwd)} +export WORKSPACE=$(pwd) +export PGROOT=${PGROOT:-/tmp/pglite} +export WEBROOT=${WEBROOT:-/tmp/web} +export DEBUG=${DEBUG:-false} +export PGDATA=${PGROOT}/base +export PGUSER=${PGUSER:-postgres} +export PGPATCH=${WORKSPACE}/patches +export TOTAL_MEMORY=${TOTAL_MEMORY:-128MB} +export WASI=${WASI:-false} + + +# exit on error +EOE=false + +# the default is a user writeable path. +if mkdir -p ${PGROOT}/sdk +then + echo "checking for valid prefix ${PGROOT}" +else + sudo mkdir -p ${PGROOT}/sdk ${PGROOT}/bin + sudo chown $(whoami) -R ${PGROOT} +fi + +# TODO: also handle PGPASSFILE hostname:port:database:username:password +# https://www.postgresql.org/docs/devel/libpq-pgpass.html +export CRED="-U $PGUSER --pwfile=${PGROOT}/password" + +if [ -f ${PGROOT}/password ] +then + echo "not changing db password" + PGPASS=$(cat ${PGROOT}/password) +else + PGPASS=${PGPASS:-password} + echo ${PGPASS:-password} > ${PGROOT}/password +fi + +export PGPASS + + +export PG_DEBUG_HEADER="${PGROOT}/include/pg_debug.h" + + +echo " +System node/pnpm ( may interfer) : + + node : $(which node) $(which node && $(which node) -v) + PNPM : $(which pnpm) + + +" + + + +# setup compiler+node. emsdk provides node (18), recent enough for bun. +# TODO: but may need to adjust $PATH with stock emsdk. + +if ${WASI:-false} +then + echo "Wasi build (experimental)" + . /opt/python-wasm-sdk/wasm32-wasi-shell.sh + +else + if which emcc + then + echo "emcc found in PATH=$PATH" + else + . /opt/python-wasm-sdk/wasm32-bi-emscripten-shell.sh + fi + export PG_LINK=${PG_LINK:-$(which emcc)} + + echo " + + Using provided emsdk from $(which emcc) + Using PG_LINK=$PG_LINK as linker + + node : $(which node) $($(which node) -v) + PNPM : $(which pnpm) + + +" + + # custom code for node/web builds that modify pg main/tools behaviour + # this used by both node/linkweb build stages + + # pass the "kernel" contiguous memory zone size to the C compiler. + CC_PGLITE="-DCMA_MB=${CMA_MB}" + +fi + +CC_PGLITE="-DPATCH_PG_DEBUG=${PG_DEBUG_HEADER} ${CC_PGLITE}" + +export CC_PGLITE + +export PGPRELOAD="\ +--preload-file ${PGROOT}/share/postgresql@${PGROOT}/share/postgresql \ +--preload-file ${PGROOT}/lib/postgresql@${PGROOT}/lib/postgresql \ +--preload-file ${PGROOT}/password@${PGROOT}/password \ +--preload-file ${PGROOT}/PGPASSFILE@/home/web_user/.pgpass \ +--preload-file placeholder@${PGROOT}/bin/postgres \ +--preload-file placeholder@${PGROOT}/bin/initdb\ +" + +# ========================= symbol extractor ============================ + +OBJDUMP=${OBJDUMP:-true} + +if $OBJDUMP +then + if [ -f $PGROOT/bin/wasm-objdump ] + then + echo "wasm-objdump found" + else + WRAPPER=$(which wasm-objdump) + WASIFILE=$(realpath ${WRAPPER}.wasi) + if $WRAPPER -h $WASIFILE | grep -q 'file format wasm 0x1' + then + mkdir -p $PGROOT/bin/ + if cp -f $WRAPPER $WASIFILE $PGROOT/bin/ + then + echo "wasm-objdump found and working, and copied to $PGROOT/bin/" + else + OBJDUMP=false + fi + else + echo " + ERROR: $(which wasm-objdump) is not working properly ( is wasmtime ok ? ) + + " + OBJDUMP=false + fi + fi +else + echo " + + WARNING: OBJDUMP disabled, some newer or complex extensions may not load properly + + +" +fi + +if $OBJDUMP +then + mkdir -p patches/imports patches/imports.pgcore +else + echo " + + WARNING: wasm-objdump not found or OBJDUMP disabled, some extensions may not load properly + + +" +fi + +export OBJDUMP + + +# ========================= pg core configuration ============================ + +# testing postgres.js file instead of ${PGROOT}/pgopts.sh because build should not have failed. +if [ -f ${WEBROOT}/postgres.js ] +then + echo using current from ${WEBROOT} + + . ${PGROOT}/pgopts.sh + +else + + # default to web/release size optim. + + mkdir -p ${PGROOT}/include + if $DEBUG + then + export PGDEBUG="" + export CDEBUG="-g3 -O0" + export LDEBUG="-g3 -O0" + cat > ${PG_DEBUG_HEADER} << END +#ifndef I_PGDEBUG +#define I_PGDEBUG +#define WASM_USERNAME "$PGUSER" +#define PGDEBUG 1 +#define PDEBUG(string) puts(string) +#define JSDEBUG(string) {EM_ASM({ console.log(string); });} +#define ADEBUG(string) { PDEBUG(string); JSDEBUG(string) } +#endif +END + + else + export PGDEBUG="" + export CDEBUG="-g3 -O0" + export LDEBUG="-g3 -O0" + cat > ${PG_DEBUG_HEADER} << END +#ifndef I_PGDEBUG +#define I_PGDEBUG +#define WASM_USERNAME "$PGUSER" +#define PDEBUG(string) +#define JSDEBUG(string) +#define ADEBUG(string) +#define PGDEBUG 0 +#endif +END + fi + + mkdir -p ${PGROOT}/include/postgresql/server + cp ${PG_DEBUG_HEADER} ${PGROOT}/include/ + cp ${PG_DEBUG_HEADER} ${PGROOT}/include/postgresql + cp ${PG_DEBUG_HEADER} ${PGROOT}/include/postgresql/server + + # store all pg options that have impact on cmd line initdb/boot + cat > ${PGROOT}/pgopts.sh < ${PGROOT}/config.site < ${PGROOT}/etc/postgresql/locale <> ${PGROOT}/pgopts.sh + echo "export PGLITE=${PGLITE}" >> ${PGROOT}/pgopts.sh + + +fi + +# put emsdk-shared the pg extension linker from build dir in the path +# and also pg_config from the install dir. +export PATH=${WORKSPACE}/build/postgres/bin:${PGROOT}/bin:$PATH + + + +# At this stage, PG should be installed to PREFIX and ready for linking +# or building ext. + + + + +# =========================================================================== +# =========================================================================== +# EXTENSIONS +# =========================================================================== +# =========================================================================== + +if echo " $*"|grep -q " contrib" +then + # TEMP FIX for SDK + SSL_INCDIR=$EMSDK/upstream/emscripten/cache/sysroot/include/openssl + [ -f $SSL_INCDIR/evp.h ] || ln -s $PREFIX/include/openssl $SSL_INCDIR + SKIP="\ + [\ + sslinfo bool_plperl hstore_plperl hstore_plpython jsonb_plperl jsonb_plpython\ + ltree_plpython sepgsql bool_plperl start-scripts\ + ]" + + for extdir in postgresql/contrib/* + do + if [ -d "$extdir" ] + then + ext=$(echo -n $extdir|cut -d/ -f3) + if echo -n $SKIP|grep -q "$ext " + then + echo skipping extension $ext + else + echo " + + Building contrib extension : $ext : begin +" + pushd build/postgres/contrib/$ext + if PATH=$PREFIX/bin:$PATH emmake make install + then + echo " + Building contrib extension : $ext : end +" + else + echo " + + Extension $ext from $extdir failed to build + +" + exit 216 + fi + popd + python3 cibuild/pack_extension.py + + fi + fi + done + + +fi + + if echo " $*"|grep -q " extra" +then + for extra_ext in ${EXTRA_EXT:-"vector"} + do + if $CI + then + if [ -d $PREFIX/include/X11 ] + then + echo -n + else + # install EXTRA sdk + . /etc/lsb-release + DISTRIB="${DISTRIB_ID}-${DISTRIB_RELEASE}" + CIVER=${CIVER:-$DISTRIB} + SDK_URL=https://github.com/pygame-web/python-wasm-sdk-extra/releases/download/$SDK_VERSION/python3.13-emsdk-sdk-extra-${CIVER}.tar.lz4 + echo "Installing $SDK_URL" + curl -sL --retry 5 $SDK_URL | tar xvP --use-compress-program=lz4 | pv -p -l -s 15000 >/dev/null + chmod +x ./extra/*.sh + fi + fi + echo "======================= ${extra_ext} : $(pwd) ===================" + + ./extra/${extra_ext}.sh || exit 400 + + python3 cibuild/pack_extension.py + done +fi + +# =========================================================================== +# =========================================================================== +# PGLite +# =========================================================================== +# =========================================================================== + + +# run this last so all extensions files can be packaged +# those include *.control *.sql and *.so +# TODO: check if some versionned *.sql files can be omitted +# TODO: for bigger extensions than pgvector make separate packaging. + +if echo " $*"|grep " node" +then + echo "====================== node : $(pwd) ========================" + mkdir -p /tmp/sdk/ + + # remove versionned symlinks + rm ${PGROOT}/lib/lib*.so.? 2>/dev/null + + if $WASI + then + tar -cpRz ${PGROOT} > /tmp/sdk/postgres-${PG_VERSION}-wasisdk.tar.gz + else + tar -cpRz ${PGROOT} > /tmp/sdk/postgres-${PG_VERSION}-emsdk.tar.gz + fi + +fi + +# run linkweb after node build because it may remove some wasm .so used by node from fs +# as they don't need to be in MEMFS since they are fetched. + +# include current pglite source for easy local rebuild with just npm run build:js. + +if echo " $*"|grep " linkweb" +then + + # build web version + echo "========== linkweb : $(pwd) ==================" + pushd build/postgres + . $WORKSPACE/cibuild/linkweb.sh + popd +fi + + +# pglite* also use web build files, so order them last. + + +while test $# -gt 0 +do + case "$1" in + + pglite) echo "=================== pglite : $(pwd) =======================" + # TODO: SAMs NOTE - Not using this in GitHub action as it doesnt resolve pnpm correctly + # replaced with pglite-prep and pglite-bundle-sdk + + pushd ${PGLITE} + pnpm install --frozen-lockfile + + mkdir -p $PGLITE/release + rm $PGLITE/release/* 2>/dev/null + + + # copy packed extensions for dist + echo " + +__________________________ enabled extensions (dlfcn)_____________________________ +" + cp -vf ${WEBROOT}/*.tar.gz ${PGLITE}/release/ +echo " +__________________________________________________________________________________ +" + + # copy wasm web prebuilt artifacts to release folder + # TODO: get them from web for nosdk systems. + + cp -vf ${WEBROOT}/postgres.{js,data,wasm} ${PGLITE}/release/ + + # debug CI does not use pnpm/npm for building pg, so call the typescript build + # part from here + #pnpm --filter "pglite^..." build || exit 450 + pnpm run build:js || exit 476 + pnpm pack || exit 31 + packed=$(echo -n electric-sql-pglite-*.tgz) + + mv $packed /tmp/sdk/pg${PG_VERSION}-${packed} + + # for repl demo +# mkdir -p /tmp/web/pglite + + #cp -r ${PGLITE}/dist ${WEBROOT}/pglite/ + #cp -r ${PGLITE}/examples ${WEBROOT}/pglite/ + +# for dir in /tmp/web ${WEBROOT}/pglite/examples +# do +# pushd "$dir" +# cp ${PGLITE}/dist/postgres.data ./ +# popd +# done + + echo " + + + + " > ${WEBROOT}/index.html + + popd + + mkdir -p ${PGROOT}/sdk/packages/ /tmp/web/pglite /tmp/web/repl/ + cp -r $PGLITE ${PGROOT}/sdk/packages/ + + #mkdir /tmp/web/repl/dist-webcomponent -p + #cp -r ${WORKSPACE}/packages/pglite-repl/dist-webcomponent /tmp/web/repl/ + + if $CI + then + tar -cpRz ${PGROOT} > /tmp/sdk/pglite-pg${PG_VERSION}.tar.gz + + # build sdk (node) + cp /tmp/sdk/postgres-${PG_VERSION}.tar.gz ${WEBROOT}/ + + # pglite (web) + cp /tmp/sdk/pglite-pg${PG_VERSION}.tar.gz ${WEBROOT}/ + + fi + + du -hs ${WEBROOT}/* + ;; + + pglite-test) echo "================== pglite-test =========================" + export PATH=$PATH:$(pwd)/node_modules/.bin + pushd ./packages/pglite + #npm install -g concurrently playwright ava http-server pg-protocol serve tinytar buffer async-mutex 2>&1 > /dev/null + pnpm install --prefix . 2>&1 >/dev/null + pnpm run build 2>&1 >/dev/null + if pnpm exec playwright install --with-deps 2>&1 >/dev/null + then + pnpm --filter "pglite^..." test || exit 534 + pnpm test:web || pnpm test:web || pnpm test:web || exit 535 + else + echo "failed to install web-test env" + pnpm --filter "pglite^..." test || exit 538 + fi + pnpm pack + popd + ;; + + pglite-prep) echo "==================== pglite-prep ==========================" + mkdir -p $PGLITE/release + #rm $PGLITE/release/* + + # copy packed extensions + cp -vf ${WEBROOT}/*.tar.gz ${PGLITE}/release/ + cp -vf ${WEBROOT}/postgres.{js,data,wasm} $PGLITE/release/ + ;; + + postgres-pglite-dist) echo "==================== postgres-pglite-dist ==========================" + mkdir -p ${POSTGRES_PGLITE_OUT} + #rm $PGLITE/release/* + + ls ${WEBROOT} -lah + + # copy packed extensions + cp -vf ${WEBROOT}/*.tar.gz ${POSTGRES_PGLITE_OUT} + cp -vf ${WEBROOT}/postgres.{js,data,wasm} ${POSTGRES_PGLITE_OUT} + ;; + + pglite-bundle-interim) echo "================== pglite-bundle-interim ======================" + tar -cpRz ${PGLITE}/release > /tmp/sdk/pglite-interim-${PG_VERSION}.tar.gz + ;; + + demo-site) echo "==================== demo-site ==========================" + # Move all existing files to a subfolder + mkdir -p /tmp/web/x-term-repl + mv /tmp/web/* /tmp/web/x-term-repl/ + + mkdir -p /tmp/web/dist + mkdir -p /tmp/web/examples + mkdir -p /tmp/web/benchmark + + PGLITE=$(pwd)/packages/pglite + cp -r ${PGLITE}/dist/* /tmp/web/dist/ + cp -r ${PGLITE}/examples/* /tmp/web/examples/ + cp -r ${WORKSPACE}/packages/benchmark/dist/* /tmp/web/benchmark/ + ;; + esac + shift +done + + diff --git a/pglite/cibuild/build-with-docker.sh b/pglite/cibuild/build-with-docker.sh new file mode 100755 index 0000000000000..bd80b82816245 --- /dev/null +++ b/pglite/cibuild/build-with-docker.sh @@ -0,0 +1,41 @@ +#!/bin/bash +echo "======== build-with-docker.sh : $(pwd) ==========" + +# these are all the elements that can be build as part of this project +ALL="contrib extra node linkweb postgres-pglite-dist" + +# this is what we will actually build +WHAT=${*:-$ALL} + +echo "======== Building PGlite prerequisites ${WHAT} using Docker ==========" + +# trap 'echo caught interrupt and exiting;' INT + +source ./pglite/.buildconfig + +if [[ -z "$SDK_VERSION" || -z "$PG_VERSION" ]]; then + echo "Missing SDK_VERSION and PG_VERSION env vars." + echo "Source them from .buildconfig" + exit 1 +fi + +# we are using a custom emsdk to build pglite wasm +# this is available as a docker image under electricsql/pglite-builder +IMG_NAME="electricsql/pglite-builder" +IMG_TAG="${PG_VERSION}_${SDK_VERSION}" + +docker run \ + --rm \ + -e OBJDUMP=${OBJDUMP:-true} \ + -e PGSRC=/workspace/postgres-src \ + -e POSTGRES_PGLITE_OUT=/workspace/dist \ + -v ./pglite/cibuild.sh:/workspace/cibuild.sh:rw \ + -v ./pglite/.buildconfig:/workspace/.buildconfig:rw \ + -v ./pglite/extra:/workspace/extra:rw \ + -v ./pglite/cibuild:/workspace/cibuild:rw \ + -v ./pglite/patches:/workspace/patches:rw \ + -v ./pglite/tests:/workspace/tests:rw \ + -v .:/workspace/postgres-src \ + -v ./pglite/dist:/workspace/dist \ + $IMG_NAME:$IMG_TAG \ + bash /workspace/cibuild.sh $WHAT \ No newline at end of file diff --git a/pglite/cibuild/getsyms.py b/pglite/cibuild/getsyms.py new file mode 100755 index 0000000000000..8a43204854b3d --- /dev/null +++ b/pglite/cibuild/getsyms.py @@ -0,0 +1,170 @@ +#!/usr/bin/env python3 + +import sys +import os + + +def dbg(*argv, **kw): + kw.setdefault("file", sys.stderr) + return print(*argv, **kw) + + +SNIFF = "" + + +# we use output from wasi `wasm-objdump -x` run + +exports = [] +recording = False + +SECTION = sys.argv[-1].lower() + +if SECTION.startswith("export"): + BEGIN = "Export[" + END = "Start:" + SECTION = "exports" + WAY = "-> " + + with open(os.environ.get("PGDUMP", "dump.postgres"), "r") as pgdump: + for line in pgdump.readlines(): + line = line.strip(";\n\r") + if line: + print(f"_{line}") + +else: + BEGIN = "Import[" + END = "Function[" + SECTION = "imports" + WAY = "<- " + +with open(os.environ.get("OBJDUMP", "dump.wasm-objdump"), "r") as wasmdump: + for line in wasmdump.readlines(): + line = line.strip() + + if not line: + continue + if SNIFF: + if line.find(SNIFF) >= 0: + dbg(line, recording) + + if recording: + if line.startswith(END): + dbg("-" * 40) + recording = False + break + + exports.append(line) + if line[0] != "-": + dbg(line) + + else: + if line[0] == "-": + continue + + if line.startswith(BEGIN): + dbg("\n\n#", line) + recording = True + continue + dbg("skip", line) + +dbg(f"found {len(exports)} {SECTION}") + +if 1: + badchars = '"<> ' + VERBOSE = "-v" in sys.argv + REPORT = [] + for line in exports: + typ, header = line.split("] ", 1) + if typ.startswith("- func["): + typ = "def" + elif typ.startswith("- global["): + pass + typ = "var" + elif typ.startswith("- memory["): + pass + typ = "mem" + elif typ.startswith("- table["): + pass + typ = "tbl" + + if SNIFF: + if line.find(SNIFF) >= 0: + dbg( + f""" + +------------------------------------------------------------------------------------- + +{line=} +{typ=} + + +------------------------------------------------------------------------------------- + +""" + ) + + try: + if typ in ("def", "var"): + left, right = header.rsplit(WAY, 1) + left = left.strip(badchars) + right = right.strip(badchars) + # GOT.mem. GOT.func. env. + clean = False + for clean in (".", " <"): + if left.find(clean) >= 0: + _, left = left.rsplit(clean, 1) + _, right = right.rsplit(".", 1) + clean = True + break + if clean: + left = left.strip(badchars) + if right.find(".") > 0: + _, right = right.rsplit(".", 1) + right = right.strip(badchars) + + if left.find("=") > 0: + left = right + + if left.find("=") > 0: + left = "" + + if not left: + left = right + + if SNIFF: + if line.find(SNIFF) >= 0: + dbg(f"{left=} {right=} '{line}'") + + if left.find("=") > 0: + left = "" + elif left.find("::") > 0: + if VERBOSE: + raise Exception("bad export (c++)") + # continue + elif left.find(" ") > 0: + if VERBOSE: + raise Exception("bad export (space)") + continue + + if VERBOSE: + demangle = os.popen(f"c++filt {right}").read().strip() + if not left or demangle == left: + dbg(typ, right, "# right") + elif demangle != right: + dbg(typ, left, "#", demangle) + else: + dbg(typ, "LEFT=", left, "RIGHT=", demangle) + + if (right or left) not in REPORT: + REPORT.append(right or left) + except Exception as e: + dbg("ERROR", typ, header, e) + + dbg(len(REPORT), "unique") + NODUNDER = [] + + for rep in REPORT: + if rep in NODUNDER: + print(f"{rep}") + else: + print(f"_{rep}") diff --git a/pglite/cibuild/linkexport.sh b/pglite/cibuild/linkexport.sh new file mode 100755 index 0000000000000..6735cdf21d40d --- /dev/null +++ b/pglite/cibuild/linkexport.sh @@ -0,0 +1,42 @@ +# this only runs when wasm-objdump is working and OBJDUMP not set to false + +echo "============= link export : begin ===============" + +# -O0 -sEXPORT_ALL should make all symbols visible without any mangling. +COPT="-O0 -g3" \ + emcc \ + $EMCC_WEB -fPIC -sMAIN_MODULE=1 -sEXPORT_ALL -sERROR_ON_UNDEFINED_SYMBOLS -sASSERTIONS=0 \ + -DPREFIX=${PGROOT} -lnodefs.js -lidbfs.js \ + -sEXPORTED_RUNTIME_METHODS=FS,setValue,getValue,UTF8ToString,stringToNewUTF8,stringToUTF8OnStack,ccall,cwrap,callMain \ + $PGPRELOAD \ + -o postgres.html $PG_O $PG_L || exit 14 + +echo "FULL:" > ${WORKSPACE}/build/sizes.log +du -hs postgres.wasm >> ${WORKSPACE}/build/sizes.log +echo >> ${WORKSPACE}/build/sizes.log + +mkdir -p ${WORKSPACE}/patches/exports + +echo "getting wasm exports lists" +wasm-objdump -x $(realpath postgres.wasm) > ${WORKSPACE}/patches/exports/pgcore.wasm-objdump + + +pushd ${WORKSPACE} + echo "getting postgres exports lists" + cat $(find build/postgres -type f |grep /exports) \ + | grep -v ^\ local \ + | grep -v ^{\ global \ + | sort | uniq > ${WORKSPACE}/patches/exports/pgcore.exports + + echo " + Merging wasm pg core symbols and postgres exports lists + into ${WORKSPACE} patches/exports/pgcore +" + + OBJDUMP=patches/exports/pgcore.wasm-objdump \ + PGDUMP=patches/exports/pgcore.exports \ + python3 cibuild/getsyms.py exports > patches/exports/pgcore +popd + +echo "============= link export : end ===============" + diff --git a/pglite/cibuild/linkimports.sh b/pglite/cibuild/linkimports.sh new file mode 100755 index 0000000000000..0f706f2ae471e --- /dev/null +++ b/pglite/cibuild/linkimports.sh @@ -0,0 +1,95 @@ +echo "============= link imports : begin ===============" + +# TODO : make a C-API list +# _main,_getenv,_setenv,_interactive_one,_interactive_write,_interactive_read,_pg_initdb,_pg_shutdown + + +# extract own pg lib requirements + +pushd ${WORKSPACE} + > patches/imports/pgcore + for extra_pg_so in $(find $PGROOT/lib/postgresql/|grep \.so$) + do + SOBASE=patches/imports.pgcore/$(basename $extra_pg_so .so) + wasm-objdump -x $(realpath $extra_pg_so) > $SOBASE.wasm-objdump + OBJDUMP=$SOBASE.wasm-objdump \ + PGDUMP=patches/exports/pgcore.exports \ + python3 cibuild/getsyms.py imports >> patches/imports/pgcore + done +popd + +#not yet + +#_emscripten_copy_from +#_emscripten_copy_to +#_emscripten_copy_to_end + +# copyFrom,copyTo,copyToEnd + cat ${WORKSPACE}/patches/imports/* | sort | uniq > /tmp/symbols + + echo "Requesting $(wc -l /tmp/symbols) symbols from pg core for PGlite extensions" + + + + python3 < ${WORKSPACE}/patches/exports/pglite + +import sys +import os + +def dbg(*argv, **kw): + kw.setdefault('file',sys.stderr) + return print(*argv,**kw) + +with open("${WORKSPACE}/patches/exports/pgcore", "r") as file: + exports = set(map(str.strip, file.readlines())) + +with open("/tmp/symbols", "r") as file: + imports = set(map(str.strip, file.readlines())) + +matches = list( imports.intersection(exports) ) + +# ? +for sym in """ +_check_function_bodies +_clock_gettime +_CurrentMemoryContext +___cxa_throw +_error_context_stack +_getenv +_interactive_one +_interactive_read +_interactive_write +_lowerstr +_main +_main_repl +_pg_initdb +_pg_repl_raf +_pg_shutdown +_readstoplist +_searchstoplist +_setenv +_shmem_request_hook +_shmem_startup_hook +_stderr +_TopMemoryContext +""".splitlines(): + if sym and not sym in matches: + matches.append(sym) + +#__ZNSt13runtime_errorD1Ev + +matches.sort() + +for sym in matches: + print(sym) + + + +dbg(f""" +exports {len(exports)} +imports {len(imports)} +Matches : {len(matches)} +""") +END + +echo "============= link imports : end ===============" diff --git a/pglite/cibuild/linkwasi.sh b/pglite/cibuild/linkwasi.sh new file mode 100755 index 0000000000000..b1798055318f4 --- /dev/null +++ b/pglite/cibuild/linkwasi.sh @@ -0,0 +1,799 @@ +#!/bin/bash +. /opt/python-wasm-sdk/wasisdk/wasisdk_env.sh +# dlfcn emulation + +WASI_CFLAGS="-DPATCH_PG_DEBUG=/tmp/pglite/include/pg_debug.h -DPREFIX=/tmp/pglite -DPYDK=1" wasi-c -fPIC \ + -Ipostgresql/src/include \ + -Ipostgresql/src/backend \ + -c -o build/postgres/wasi_dlfcn.o \ + -Ibuild/postgres/src/include patches/wasi_dlfcn.c || exit 8 + +# -L./build/postgres/src/backend/snowball -ldict_snowball +# ./build/postgres/src/backend/snowball/dict_snowball.o +# ./build/postgres/src/backend/snowball/libdict_snowball.a + +pushd build/postgres/src/backend + +# a bit of tidy up + +PGOBJ="access/brin/brin.o \ +access/brin/brin_bloom.o \ +access/brin/brin_inclusion.o \ +access/brin/brin_minmax.o \ +access/brin/brin_minmax_multi.o \ +access/brin/brin_pageops.o \ +access/brin/brin_revmap.o \ +access/brin/brin_tuple.o \ +access/brin/brin_validate.o \ +access/brin/brin_xlog.o \ +access/common/attmap.o \ +access/common/bufmask.o \ +access/common/detoast.o \ +access/common/heaptuple.o \ +access/common/indextuple.o \ +access/common/printsimple.o \ +access/common/printtup.o \ +access/common/relation.o \ +access/common/reloptions.o \ +access/common/scankey.o \ +access/common/session.o \ +access/common/syncscan.o \ +access/common/toast_compression.o \ +access/common/toast_internals.o \ +access/common/tupconvert.o \ +access/common/tupdesc.o \ +access/gin/ginarrayproc.o \ +access/gin/ginbtree.o \ +access/gin/ginbulk.o \ +access/gin/gindatapage.o \ +access/gin/ginentrypage.o \ +access/gin/ginfast.o \ +access/gin/ginget.o \ +access/gin/gininsert.o \ +access/gin/ginlogic.o \ +access/gin/ginpostinglist.o \ +access/gin/ginscan.o \ +access/gin/ginutil.o \ +access/gin/ginvacuum.o \ +access/gin/ginvalidate.o \ +access/gin/ginxlog.o \ +access/gist/gist.o \ +access/gist/gistbuild.o \ +access/gist/gistbuildbuffers.o \ +access/gist/gistget.o \ +access/gist/gistproc.o \ +access/gist/gistscan.o \ +access/gist/gistsplit.o \ +access/gist/gistutil.o \ +access/gist/gistvacuum.o \ +access/gist/gistvalidate.o \ +access/gist/gistxlog.o \ +access/hash/hash.o \ +access/hash/hash_xlog.o \ +access/hash/hashfunc.o \ +access/hash/hashinsert.o \ +access/hash/hashovfl.o \ +access/hash/hashpage.o \ +access/hash/hashsearch.o \ +access/hash/hashsort.o \ +access/hash/hashutil.o \ +access/hash/hashvalidate.o \ +access/heap/heapam.o \ +access/heap/heapam_handler.o \ +access/heap/heapam_visibility.o \ +access/heap/heaptoast.o \ +access/heap/hio.o \ +access/heap/pruneheap.o \ +access/heap/rewriteheap.o \ +access/heap/vacuumlazy.o \ +access/heap/visibilitymap.o \ +access/index/amapi.o \ +access/index/amvalidate.o \ +access/index/genam.o \ +access/index/indexam.o \ +access/nbtree/nbtcompare.o \ +access/nbtree/nbtdedup.o \ +access/nbtree/nbtinsert.o \ +access/nbtree/nbtpage.o \ +access/nbtree/nbtree.o \ +access/nbtree/nbtsearch.o \ +access/nbtree/nbtsort.o \ +access/nbtree/nbtsplitloc.o \ +access/nbtree/nbtutils.o \ +access/nbtree/nbtvalidate.o \ +access/nbtree/nbtxlog.o \ +access/rmgrdesc/brindesc.o \ +access/rmgrdesc/clogdesc.o \ +access/rmgrdesc/committsdesc.o \ +access/rmgrdesc/dbasedesc.o \ +access/rmgrdesc/genericdesc.o \ +access/rmgrdesc/gindesc.o \ +access/rmgrdesc/gistdesc.o \ +access/rmgrdesc/hashdesc.o \ +access/rmgrdesc/heapdesc.o \ +access/rmgrdesc/logicalmsgdesc.o \ +access/rmgrdesc/mxactdesc.o \ +access/rmgrdesc/nbtdesc.o \ +access/rmgrdesc/relmapdesc.o \ +access/rmgrdesc/replorigindesc.o \ +access/rmgrdesc/rmgrdesc_utils.o \ +access/rmgrdesc/seqdesc.o \ +access/rmgrdesc/smgrdesc.o \ +access/rmgrdesc/spgdesc.o \ +access/rmgrdesc/standbydesc.o \ +access/rmgrdesc/tblspcdesc.o \ +access/rmgrdesc/xactdesc.o \ +access/rmgrdesc/xlogdesc.o \ +access/spgist/spgdoinsert.o \ +access/spgist/spginsert.o \ +access/spgist/spgkdtreeproc.o \ +access/spgist/spgproc.o \ +access/spgist/spgquadtreeproc.o \ +access/spgist/spgscan.o \ +access/spgist/spgtextproc.o \ +access/spgist/spgutils.o \ +access/spgist/spgvacuum.o \ +access/spgist/spgvalidate.o \ +access/spgist/spgxlog.o \ +access/table/table.o \ +access/table/tableam.o \ +access/table/tableamapi.o \ +access/table/toast_helper.o \ +access/tablesample/bernoulli.o \ +access/tablesample/system.o \ +access/tablesample/tablesample.o \ +access/transam/clog.o \ +access/transam/commit_ts.o \ +access/transam/generic_xlog.o \ +access/transam/multixact.o \ +access/transam/parallel.o \ +access/transam/rmgr.o \ +access/transam/slru.o \ +access/transam/subtrans.o \ +access/transam/timeline.o \ +access/transam/transam.o \ +access/transam/twophase.o \ +access/transam/twophase_rmgr.o \ +access/transam/varsup.o \ +access/transam/xact.o \ +access/transam/xlog.o \ +access/transam/xlogarchive.o \ +access/transam/xlogbackup.o \ +access/transam/xlogfuncs.o \ +access/transam/xloginsert.o \ +access/transam/xlogprefetcher.o \ +access/transam/xlogreader.o \ +access/transam/xlogrecovery.o \ +access/transam/xlogstats.o \ +access/transam/xlogutils.o \ +archive/shell_archive.o \ +backup/backup_manifest.o \ +backup/basebackup.o \ +backup/basebackup_copy.o \ +backup/basebackup_gzip.o \ +backup/basebackup_lz4.o \ +backup/basebackup_zstd.o \ +backup/basebackup_progress.o \ +backup/basebackup_server.o \ +backup/basebackup_sink.o \ +backup/basebackup_target.o \ +backup/basebackup_throttle.o \ +bootstrap/bootparse.o \ +bootstrap/bootscanner.o \ +bootstrap/bootstrap.o \ +catalog/aclchk.o \ +catalog/catalog.o \ +catalog/dependency.o \ +catalog/heap.o \ +catalog/index.o \ +catalog/indexing.o \ +catalog/namespace.o \ +catalog/objectaccess.o \ +catalog/objectaddress.o \ +catalog/partition.o \ +catalog/pg_aggregate.o \ +catalog/pg_attrdef.o \ +catalog/pg_cast.o \ +catalog/pg_class.o \ +catalog/pg_collation.o \ +catalog/pg_constraint.o \ +catalog/pg_conversion.o \ +catalog/pg_db_role_setting.o \ +catalog/pg_depend.o \ +catalog/pg_enum.o \ +catalog/pg_inherits.o \ +catalog/pg_largeobject.o \ +catalog/pg_namespace.o \ +catalog/pg_operator.o \ +catalog/pg_parameter_acl.o \ +catalog/pg_proc.o \ +catalog/pg_publication.o \ +catalog/pg_range.o \ +catalog/pg_shdepend.o \ +catalog/pg_subscription.o \ +catalog/pg_type.o \ +catalog/storage.o \ +catalog/toasting.o \ +parser/analyze.o \ +parser/gram.o \ +parser/parse_agg.o \ +parser/parse_clause.o \ +parser/parse_coerce.o \ +parser/parse_collate.o \ +parser/parse_cte.o \ +parser/parse_enr.o \ +parser/parse_expr.o \ +parser/parse_func.o \ +parser/parse_merge.o \ +parser/parse_node.o \ +parser/parse_oper.o \ +parser/parse_param.o \ +parser/parse_relation.o \ +parser/parse_target.o \ +parser/parse_type.o \ +parser/parse_utilcmd.o \ +parser/parser.o \ +parser/scan.o \ +parser/scansup.o \ +commands/aggregatecmds.o \ +commands/alter.o \ +commands/amcmds.o \ +commands/analyze.o \ +commands/async.o \ +commands/cluster.o \ +commands/collationcmds.o \ +commands/comment.o \ +commands/constraint.o \ +commands/conversioncmds.o \ +commands/copy.o \ +commands/copyfrom.o \ +commands/copyfromparse.o \ +commands/copyto.o \ +commands/createas.o \ +commands/dbcommands.o \ +commands/define.o \ +commands/discard.o \ +commands/dropcmds.o \ +commands/event_trigger.o \ +commands/explain.o \ +commands/extension.o \ +commands/foreigncmds.o \ +commands/functioncmds.o \ +commands/indexcmds.o \ +commands/lockcmds.o \ +commands/matview.o \ +commands/opclasscmds.o \ +commands/operatorcmds.o \ +commands/policy.o \ +commands/portalcmds.o \ +commands/prepare.o \ +commands/proclang.o \ +commands/publicationcmds.o \ +commands/schemacmds.o \ +commands/seclabel.o \ +commands/sequence.o \ +commands/statscmds.o \ +commands/subscriptioncmds.o \ +commands/tablecmds.o \ +commands/tablespace.o \ +commands/trigger.o \ +commands/tsearchcmds.o \ +commands/typecmds.o \ +commands/user.o \ +commands/vacuum.o \ +commands/vacuumparallel.o \ +commands/variable.o \ +commands/view.o \ +executor/execAmi.o \ +executor/execAsync.o \ +executor/execCurrent.o \ +executor/execExpr.o \ +executor/execExprInterp.o \ +executor/execGrouping.o \ +executor/execIndexing.o \ +executor/execJunk.o \ +executor/execMain.o \ +executor/execParallel.o \ +executor/execPartition.o \ +executor/execProcnode.o \ +executor/execReplication.o \ +executor/execSRF.o \ +executor/execScan.o \ +executor/execTuples.o \ +executor/execUtils.o \ +executor/functions.o \ +executor/instrument.o \ +executor/nodeAgg.o \ +executor/nodeAppend.o \ +executor/nodeBitmapAnd.o \ +executor/nodeBitmapHeapscan.o \ +executor/nodeBitmapIndexscan.o \ +executor/nodeBitmapOr.o \ +executor/nodeCtescan.o \ +executor/nodeCustom.o \ +executor/nodeForeignscan.o \ +executor/nodeFunctionscan.o \ +executor/nodeGather.o \ +executor/nodeGatherMerge.o \ +executor/nodeGroup.o \ +executor/nodeHash.o \ +executor/nodeHashjoin.o \ +executor/nodeIncrementalSort.o \ +executor/nodeIndexonlyscan.o \ +executor/nodeIndexscan.o \ +executor/nodeLimit.o \ +executor/nodeLockRows.o \ +executor/nodeMaterial.o \ +executor/nodeMemoize.o \ +executor/nodeMergeAppend.o \ +executor/nodeMergejoin.o \ +executor/nodeModifyTable.o \ +executor/nodeNamedtuplestorescan.o \ +executor/nodeNestloop.o \ +executor/nodeProjectSet.o \ +executor/nodeRecursiveunion.o \ +executor/nodeResult.o \ +executor/nodeSamplescan.o \ +executor/nodeSeqscan.o \ +executor/nodeSetOp.o \ +executor/nodeSort.o \ +executor/nodeSubplan.o \ +executor/nodeSubqueryscan.o \ +executor/nodeTableFuncscan.o \ +executor/nodeTidrangescan.o \ +executor/nodeTidscan.o \ +executor/nodeUnique.o \ +executor/nodeValuesscan.o \ +executor/nodeWindowAgg.o \ +executor/nodeWorktablescan.o \ +executor/spi.o \ +executor/tqueue.o \ +executor/tstoreReceiver.o \ +foreign/foreign.o \ +lib/binaryheap.o \ +lib/bipartite_match.o \ +lib/bloomfilter.o \ +lib/dshash.o \ +lib/hyperloglog.o \ +lib/ilist.o \ +lib/integerset.o \ +lib/knapsack.o \ +lib/pairingheap.o \ +lib/rbtree.o \ +libpq/auth-sasl.o \ +libpq/auth-scram.o \ +libpq/auth.o \ +libpq/be-fsstubs.o \ +libpq/be-secure-common.o \ +libpq/be-secure.o \ +libpq/crypt.o \ +libpq/hba.o \ +libpq/ifaddr.o \ +libpq/pqcomm.o \ +libpq/pqformat.o \ +libpq/pqmq.o \ +libpq/pqsignal.o \ +nodes/bitmapset.o \ +nodes/copyfuncs.o \ +nodes/equalfuncs.o \ +nodes/extensible.o \ +nodes/list.o \ +nodes/makefuncs.o \ +nodes/multibitmapset.o \ +nodes/nodeFuncs.o \ +nodes/nodes.o \ +nodes/outfuncs.o \ +nodes/params.o \ +nodes/print.o \ +nodes/queryjumblefuncs.o \ +nodes/read.o \ +nodes/readfuncs.o \ +nodes/tidbitmap.o \ +nodes/value.o \ +optimizer/geqo/geqo_copy.o \ +optimizer/geqo/geqo_cx.o \ +optimizer/geqo/geqo_erx.o \ +optimizer/geqo/geqo_eval.o \ +optimizer/geqo/geqo_main.o \ +optimizer/geqo/geqo_misc.o \ +optimizer/geqo/geqo_mutation.o \ +optimizer/geqo/geqo_ox1.o \ +optimizer/geqo/geqo_ox2.o \ +optimizer/geqo/geqo_pmx.o \ +optimizer/geqo/geqo_pool.o \ +optimizer/geqo/geqo_px.o \ +optimizer/geqo/geqo_random.o \ +optimizer/geqo/geqo_recombination.o \ +optimizer/geqo/geqo_selection.o \ +optimizer/path/allpaths.o \ +optimizer/path/clausesel.o \ +optimizer/path/costsize.o \ +optimizer/path/equivclass.o \ +optimizer/path/indxpath.o \ +optimizer/path/joinpath.o \ +optimizer/path/joinrels.o \ +optimizer/path/pathkeys.o \ +optimizer/path/tidpath.o \ +optimizer/plan/analyzejoins.o \ +optimizer/plan/createplan.o \ +optimizer/plan/initsplan.o \ +optimizer/plan/planagg.o \ +optimizer/plan/planmain.o \ +optimizer/plan/planner.o \ +optimizer/plan/setrefs.o \ +optimizer/plan/subselect.o \ +optimizer/prep/prepagg.o \ +optimizer/prep/prepjointree.o \ +optimizer/prep/prepqual.o \ +optimizer/prep/preptlist.o \ +optimizer/prep/prepunion.o \ +optimizer/util/appendinfo.o \ +optimizer/util/clauses.o \ +optimizer/util/inherit.o \ +optimizer/util/joininfo.o \ +optimizer/util/orclauses.o \ +optimizer/util/paramassign.o \ +optimizer/util/pathnode.o \ +optimizer/util/placeholder.o \ +optimizer/util/plancat.o \ +optimizer/util/predtest.o \ +optimizer/util/relnode.o \ +optimizer/util/restrictinfo.o \ +optimizer/util/tlist.o \ +optimizer/util/var.o \ +partitioning/partbounds.o \ +partitioning/partdesc.o \ +partitioning/partprune.o \ +port/atomics.o \ +port/pg_sema.o \ +port/pg_shmem.o \ +postmaster/autovacuum.o \ +postmaster/auxprocess.o \ +postmaster/bgworker.o \ +postmaster/bgwriter.o \ +postmaster/checkpointer.o \ +postmaster/fork_process.o \ +postmaster/interrupt.o \ +postmaster/pgarch.o \ +postmaster/postmaster.o \ +postmaster/startup.o \ +postmaster/syslogger.o \ +postmaster/walwriter.o \ +regex/regcomp.o \ +regex/regerror.o \ +regex/regexec.o \ +regex/regexport.o \ +regex/regfree.o \ +regex/regprefix.o \ +replication/logical/applyparallelworker.o \ +replication/logical/decode.o \ +replication/logical/launcher.o \ +replication/logical/logical.o \ +replication/logical/logicalfuncs.o \ +replication/logical/message.o \ +replication/logical/origin.o \ +replication/logical/proto.o \ +replication/logical/relation.o \ +replication/logical/reorderbuffer.o \ +replication/logical/snapbuild.o \ +replication/logical/tablesync.o \ +replication/logical/worker.o \ +replication/repl_gram.o \ +replication/repl_scanner.o \ +replication/slot.o \ +replication/slotfuncs.o \ +replication/syncrep.o \ +replication/syncrep_gram.o \ +replication/syncrep_scanner.o \ +replication/walreceiver.o \ +replication/walreceiverfuncs.o \ +replication/walsender.o \ +rewrite/rewriteDefine.o \ +rewrite/rewriteHandler.o \ +rewrite/rewriteManip.o \ +rewrite/rewriteRemove.o \ +rewrite/rewriteSearchCycle.o \ +rewrite/rewriteSupport.o \ +rewrite/rowsecurity.o \ +statistics/dependencies.o \ +statistics/extended_stats.o \ +statistics/mcv.o \ +statistics/mvdistinct.o \ +storage/buffer/buf_init.o \ +storage/buffer/buf_table.o \ +storage/buffer/bufmgr.o \ +storage/buffer/freelist.o \ +storage/buffer/localbuf.o \ +storage/file/buffile.o \ +storage/file/copydir.o \ +storage/file/fd.o \ +storage/file/fileset.o \ +storage/file/reinit.o \ +storage/file/sharedfileset.o \ +storage/freespace/freespace.o \ +storage/freespace/fsmpage.o \ +storage/freespace/indexfsm.o \ +storage/ipc/barrier.o \ +storage/ipc/dsm.o \ +storage/ipc/dsm_impl.o \ +storage/ipc/ipc.o \ +storage/ipc/ipci.o \ +storage/ipc/latch.o \ +storage/ipc/pmsignal.o \ +storage/ipc/procarray.o \ +storage/ipc/procsignal.o \ +storage/ipc/shm_mq.o \ +storage/ipc/shm_toc.o \ +storage/ipc/shmem.o \ +storage/ipc/signalfuncs.o \ +storage/ipc/sinval.o \ +storage/ipc/sinvaladt.o \ +storage/ipc/standby.o \ +storage/large_object/inv_api.o \ +storage/lmgr/condition_variable.o \ +storage/lmgr/deadlock.o \ +storage/lmgr/lmgr.o \ +storage/lmgr/lock.o \ +storage/lmgr/lwlock.o \ +storage/lmgr/lwlocknames.o \ +storage/lmgr/predicate.o \ +storage/lmgr/proc.o \ +storage/lmgr/s_lock.o \ +storage/lmgr/spin.o \ +storage/page/bufpage.o \ +storage/page/checksum.o \ +storage/page/itemptr.o \ +storage/smgr/md.o \ +storage/smgr/smgr.o \ +storage/sync/sync.o \ +tcop/cmdtag.o \ +tcop/dest.o \ +tcop/fastpath.o \ +tcop/postgres.o \ +tcop/pquery.o \ +tcop/utility.o \ +tsearch/dict.o \ +tsearch/dict_ispell.o \ +tsearch/dict_simple.o \ +tsearch/dict_synonym.o \ +tsearch/dict_thesaurus.o \ +tsearch/regis.o \ +tsearch/spell.o \ +tsearch/to_tsany.o \ +tsearch/ts_locale.o \ +tsearch/ts_parse.o \ +tsearch/ts_selfuncs.o \ +tsearch/ts_typanalyze.o \ +tsearch/ts_utils.o \ +tsearch/wparser.o \ +tsearch/wparser_def.o \ +utils/activity/backend_progress.o \ +utils/activity/backend_status.o \ +utils/activity/pgstat.o \ +utils/activity/pgstat_archiver.o \ +utils/activity/pgstat_bgwriter.o \ +utils/activity/pgstat_checkpointer.o \ +utils/activity/pgstat_database.o \ +utils/activity/pgstat_function.o \ +utils/activity/pgstat_io.o \ +utils/activity/pgstat_relation.o \ +utils/activity/pgstat_replslot.o \ +utils/activity/pgstat_shmem.o \ +utils/activity/pgstat_slru.o \ +utils/activity/pgstat_subscription.o \ +utils/activity/pgstat_wal.o \ +utils/activity/pgstat_xact.o \ +utils/activity/wait_event.o \ +utils/adt/acl.o \ +utils/adt/amutils.o \ +utils/adt/array_expanded.o \ +utils/adt/array_selfuncs.o \ +utils/adt/array_typanalyze.o \ +utils/adt/array_userfuncs.o \ +utils/adt/arrayfuncs.o \ +utils/adt/arraysubs.o \ +utils/adt/arrayutils.o \ +utils/adt/ascii.o \ +utils/adt/bool.o \ +utils/adt/cash.o \ +utils/adt/char.o \ +utils/adt/cryptohashfuncs.o \ +utils/adt/date.o \ +utils/adt/datetime.o \ +utils/adt/datum.o \ +utils/adt/dbsize.o \ +utils/adt/domains.o \ +utils/adt/encode.o \ +utils/adt/enum.o \ +utils/adt/expandeddatum.o \ +utils/adt/expandedrecord.o \ +utils/adt/float.o \ +utils/adt/format_type.o \ +utils/adt/formatting.o \ +utils/adt/genfile.o \ +utils/adt/geo_ops.o \ +utils/adt/geo_selfuncs.o \ +utils/adt/geo_spgist.o \ +utils/adt/hbafuncs.o \ +utils/adt/inet_cidr_ntop.o \ +utils/adt/inet_net_pton.o \ +utils/adt/int.o \ +utils/adt/int8.o \ +utils/adt/json.o \ +utils/adt/jsonb.o \ +utils/adt/jsonb_gin.o \ +utils/adt/jsonb_op.o \ +utils/adt/jsonb_util.o \ +utils/adt/jsonfuncs.o \ +utils/adt/jsonbsubs.o \ +utils/adt/jsonpath.o \ +utils/adt/jsonpath_exec.o \ +utils/adt/jsonpath_gram.o \ +utils/adt/jsonpath_scan.o \ +utils/adt/like.o \ +utils/adt/like_support.o \ +utils/adt/lockfuncs.o \ +utils/adt/mac.o \ +utils/adt/mac8.o \ +utils/adt/mcxtfuncs.o \ +utils/adt/misc.o \ +utils/adt/multirangetypes.o \ +utils/adt/multirangetypes_selfuncs.o \ +utils/adt/name.o \ +utils/adt/network.o \ +utils/adt/network_gist.o \ +utils/adt/network_selfuncs.o \ +utils/adt/network_spgist.o \ +utils/adt/numeric.o \ +utils/adt/numutils.o \ +utils/adt/oid.o \ +utils/adt/oracle_compat.o \ +utils/adt/orderedsetaggs.o \ +utils/adt/partitionfuncs.o \ +utils/adt/pg_locale.o \ +utils/adt/pg_lsn.o \ +utils/adt/pg_upgrade_support.o \ +utils/adt/pgstatfuncs.o \ +utils/adt/pseudotypes.o \ +utils/adt/quote.o \ +utils/adt/rangetypes.o \ +utils/adt/rangetypes_gist.o \ +utils/adt/rangetypes_selfuncs.o \ +utils/adt/rangetypes_spgist.o \ +utils/adt/rangetypes_typanalyze.o \ +utils/adt/regexp.o \ +utils/adt/regproc.o \ +utils/adt/ri_triggers.o \ +utils/adt/rowtypes.o \ +utils/adt/ruleutils.o \ +utils/adt/selfuncs.o \ +utils/adt/tid.o \ +utils/adt/timestamp.o \ +utils/adt/trigfuncs.o \ +utils/adt/tsginidx.o \ +utils/adt/tsgistidx.o \ +utils/adt/tsquery.o \ +utils/adt/tsquery_cleanup.o \ +utils/adt/tsquery_gist.o \ +utils/adt/tsquery_op.o \ +utils/adt/tsquery_rewrite.o \ +utils/adt/tsquery_util.o \ +utils/adt/tsrank.o \ +utils/adt/tsvector.o \ +utils/adt/tsvector_op.o \ +utils/adt/tsvector_parser.o \ +utils/adt/uuid.o \ +utils/adt/varbit.o \ +utils/adt/varchar.o \ +utils/adt/varlena.o \ +utils/adt/version.o \ +utils/adt/windowfuncs.o \ +utils/adt/xid.o \ +utils/adt/xid8funcs.o \ +utils/adt/xml.o \ +utils/cache/attoptcache.o \ +utils/cache/catcache.o \ +utils/cache/evtcache.o \ +utils/cache/inval.o \ +utils/cache/lsyscache.o \ +utils/cache/partcache.o \ +utils/cache/plancache.o \ +utils/cache/relcache.o \ +utils/cache/relfilenumbermap.o \ +utils/cache/relmapper.o \ +utils/cache/spccache.o \ +utils/cache/syscache.o \ +utils/cache/ts_cache.o \ +utils/cache/typcache.o \ +utils/error/assert.o \ +utils/error/csvlog.o \ +utils/error/elog.o \ +utils/error/jsonlog.o \ +utils/fmgr/dfmgr.o \ +utils/fmgr/fmgr.o \ +utils/fmgr/funcapi.o \ +utils/hash/dynahash.o \ +utils/hash/pg_crc.o \ +utils/init/globals.o \ +utils/init/miscinit.o \ +utils/init/postinit.o \ +utils/init/usercontext.o \ +utils/mb/conv.o \ +utils/mb/mbutils.o \ +utils/mb/stringinfo_mb.o \ +utils/mb/wstrcmp.o \ +utils/mb/wstrncmp.o \ +utils/misc/conffiles.o \ +utils/misc/guc.o \ +utils/misc/guc-file.o \ +utils/misc/guc_funcs.o \ +utils/misc/guc_tables.o \ +utils/misc/help_config.o \ +utils/misc/pg_config.o \ +utils/misc/pg_controldata.o \ +utils/misc/pg_rusage.o \ +utils/misc/ps_status.o \ +utils/misc/queryenvironment.o \ +utils/misc/rls.o \ +utils/misc/sampling.o \ +utils/misc/superuser.o \ +utils/misc/timeout.o \ +utils/misc/tzparser.o \ +utils/mmgr/alignedalloc.o \ +utils/mmgr/aset.o \ +utils/mmgr/dsa.o \ +utils/mmgr/freepage.o \ +utils/mmgr/generation.o \ +utils/mmgr/mcxt.o \ +utils/mmgr/memdebug.o \ +utils/mmgr/portalmem.o \ +utils/mmgr/slab.o \ +utils/resowner/resowner.o \ +utils/sort/logtape.o \ +utils/sort/qsort_interruptible.o \ +utils/sort/sharedtuplestore.o \ +utils/sort/sortsupport.o \ +utils/sort/tuplesort.o \ +utils/sort/tuplesortvariants.o \ +utils/sort/tuplestore.o \ +utils/time/combocid.o \ +utils/time/snapmgr.o \ +utils/fmgrtab.o \ +../../src/timezone/localtime.o \ +../../src/timezone/pgtz.o \ +../../src/timezone/strftime.o \ +jit/jit.o" + + +# put everything in one lib +llvm-ar cr ../../libpglite.a $PGOBJ + + +# just linking + +# -Wl,--no-entry -mexec-model=reactor + +$CC -o postgres \ + -fno-strict-aliasing \ + -fwrapv \ + \ + -O2 ../../src/common/libpgcommon_srv.a ../../src/port/libpgport_srv.a \ + \ + ../../libpglite.a main/main.o ../../wasi_dlfcn.o \ + \ + ../../src/backend/snowball/libdict_snowball.a \ + ../../src/pl/plpgsql/src/libplpgsql.a \ + -lz -lm -lwasi-emulated-mman -lwasi-emulated-signal -lc \ + -Wl,--export=pg_initdb \ + -Wl,--export=setup \ + -Wl,--export=loop \ + -Wl,--export=interactive_one \ + -Wl,--export=use_socketfile \ + -Wl,--export=interactive_write \ + -Wl,--export=interactive_read \ + -Wl,--global-base=33333333 + +cp -vf postgres postgres.wasi || exit 192 +#cp -vf postgres.wasi /tmp/pglite/bin/postgres.wasi + +popd diff --git a/pglite/cibuild/linkweb.sh b/pglite/cibuild/linkweb.sh new file mode 100755 index 0000000000000..6cf6c893bd3bd --- /dev/null +++ b/pglite/cibuild/linkweb.sh @@ -0,0 +1,235 @@ +#!/bin/bash +echo "============= link web : begin ===============" + +WEBROOT=${WEBROOT:-/tmp/sdk} +echo " + + + +linkweb:begin + + $(pwd) + + WEBROOT=${WEBROOT} + + CC_PGLITE=$CC_PGLITE + +" + +mkdir -p $WEBROOT + +if $WASI +then + echo "no wasi web linking yet" + exit 0 +fi + + +NOWARN="-Wno-missing-prototypes -Wno-unused-function -Wno-declaration-after-statement -Wno-incompatible-pointer-types-discards-qualifiers" + +# client lib ( eg psycopg ) for websocketed pg server +emcc $CDEBUG -shared -o ${WEBROOT}/libpgc.so \ + ./src/interfaces/libpq/libpq.a \ + ./src/port/libpgport.a \ + ./src/common/libpgcommon.a || exit 26 + +# this override completely pg server main loop for web use purpose +pushd src + rm pg_initdb.o backend/main/main.o ./backend/tcop/postgres.o ./backend/utils/init/postinit.o + + emcc -DPG_INITDB_MAIN=1 -sFORCE_FILESYSTEM -DPREFIX=${PGROOT} ${CC_PGLITE} \ + -I${PGROOT}/include -I${PGROOT}/include/postgresql/server -I${PGROOT}/include/postgresql/internal \ + -c -o ../pg_initdb.o ${PGSRC}/src/bin/initdb/initdb.c $NOWARN || exit 34 + + # + emcc -DPG_LINK_MAIN=1 -DPREFIX=${PGROOT} ${CC_PGLITE} -DPG_EC_STATIC \ + -I${PGROOT}/include -I${PGROOT}/include/postgresql/server -I${PGROOT}/include/postgresql/internal \ + -c -o ./backend/tcop/postgres.o ${PGSRC}/src/backend/tcop/postgres.c $NOWARN|| exit 39 + + EMCC_CFLAGS="${CC_PGLITE} -DPREFIX=${PGROOT} -DPG_INITDB_MAIN=1 $NOWARN" \ + emmake make backend/main/main.o backend/utils/init/postinit.o || exit 41 +popd + + +echo "========================================================" +echo -DPREFIX=${PGROOT} $CC_PGLITE +file ${WEBROOT}/libpgc.so pg_initdb.o src/backend/main/main.o src/backend/tcop/postgres.o src/backend/utils/init/postinit.o +echo "========================================================" + + +pushd src/backend + + # https://github.com/emscripten-core/emscripten/issues/12167 + # --localize-hidden + # https://github.com/llvm/llvm-project/issues/50623 + + + echo " ---------- building web test PREFIX=$PGROOT ------------" + du -hs ${WEBROOT}/libpg?.* + + PG_O="../../src/fe_utils/string_utils.o ../../src/common/logging.o \ + $(find . -type f -name "*.o" \ + | grep -v ./utils/mb/conversion_procs \ + | grep -v ./replication/pgoutput \ + | grep -v src/bin/ \ + | grep -v ./snowball/dict_snowball.o ) \ + ../../src/timezone/localtime.o \ + ../../src/timezone/pgtz.o \ + ../../src/timezone/strftime.o \ + ../../pg_initdb.o" + + PG_L="../../src/common/libpgcommon_srv.a ../../src/port/libpgport_srv.a ../.././src/interfaces/libpq/libpq.a -L$PREFIX/lib -lxml2 -lz" + # -lz for xml2 + # -sUSE_ZLIB" + + if $DEBUG + then + echo -n + fi + + export PG_L + + +# ? -sLZ4=1 -sENVIRONMENT=web +# -sSINGLE_FILE => Uncaught SyntaxError: Cannot use 'import.meta' outside a module (at postgres.html:1:6033) +# -sENVIRONMENT=web => XHR + + export EMCC_WEB="-sNO_EXIT_RUNTIME=1 -sFORCE_FILESYSTEM=1 -sENVIRONMENT=node,web" + + if ${PGES6:-true} + then + # es6 + MODULE="$LDEBUG --closure 0 -sMODULARIZE=1 -sEXPORT_ES6=1 -sEXPORT_NAME=Module" + export COPTS="-O2 -g0" + else + export COPTS="-O0 -g3" + # local debug always fast build + MODULE="-sMODULARIZE=0 -sEXPORT_ES6=0" + fi + + MODULE="$MODULE --shell-file ${WORKSPACE}/tests/repl.html" + # closure -sSIMPLE_OPTIMIZATION ? + + # ======================================================= + # size optimisations + # ======================================================= + + rm ${PGROOT}/lib/lib*.so.? 2>/dev/null + + echo "#!/bin/true" > placeholder + chmod +x placeholder + + # for ./bin + + # share/postgresql/pg_hba.conf.sample REQUIRED + # rm ${PGROOT}/share/postgresql/*.sample + + # ./lib/lib*.a => ignored + + # ./include ignored + + # timezones ? + + # encodings ? + # ./lib/postgresql/utf8_and*.so + rm ${PGROOT}/lib/postgresql/utf8_and*.so + + + # ========================================================= + + # --js-library + # cp ${WORKSPACE}/patches/library_pgfs.js ${EMSDK}/upstream/emscripten/src/library_pgfs.js + + + if $OBJDUMP + then + echo " + + Linking to : $PG_L + + +" + + # link with MAIN_MODULE=1 ( ie export all ) and extract all sym. + . ${WORKSPACE}/cibuild/linkexport.sh || exit 158 + + if [ -f ${WORKSPACE}/patches/exports/pgcore ] + then + echo "PGLite can export $(wc -l ${WORKSPACE}/patches/exports/pgcore) core symbols" + . ${WORKSPACE}/cibuild/linkimports.sh || exit 163 + + else + echo " + + _________________________________________________________ + WARNING: using cached/provided imported symbol list + _________________________________________________________ + + + " + fi + + else + echo " + +_________________________________________________________ + WARNING: using cached/provided exported symbol list +_________________________________________________________ + + +" + fi + + + cat ${WORKSPACE}/patches/exports/pglite > exports + + # min + # LINKER="-sMAIN_MODULE=2" + + # tailored + LINKER="-sMAIN_MODULE=2 -sEXPORTED_FUNCTIONS=@exports" + + # FULL + # LINKER="-sMAIN_MODULE=1 -sEXPORTED_FUNCTIONS=@exports" + + + emcc $EMCC_WEB $LINKER $MODULE \ + -sTOTAL_MEMORY=${TOTAL_MEMORY} -sSTACK_SIZE=4MB -sGLOBAL_BASE=${CMA_MB}MB \ + -fPIC -D__PYDK__=1 -DPREFIX=${PGROOT} \ + -sALLOW_TABLE_GROWTH -sALLOW_MEMORY_GROWTH -sERROR_ON_UNDEFINED_SYMBOLS -sASSERTIONS=0 \ + -lnodefs.js -lidbfs.js \ + -sEXPORTED_RUNTIME_METHODS=FS,setValue,getValue,UTF8ToString,stringToNewUTF8,stringToUTF8OnStack,ccall,cwrap,callMain \ + $PGPRELOAD \ + -o postgres.html $PG_O $PG_L || exit 200 + + cp postgres.js /tmp/ + + echo "TAILORED:" >> ${WORKSPACE}/build/sizes.log + du -hs postgres.wasm >> ${WORKSPACE}/build/sizes.log + echo >> ${WORKSPACE}/build/sizes.log + + + mkdir -p ${WEBROOT} + + cp -vf postgres.* ${WEBROOT}/ + #cp ${PGROOT}/lib/libecpg.so ${WEBROOT}/ + + for tarf in ${PGROOT}/sdk/*.tar + do + gzip -f -9 $tarf + done + + cp ${PGROOT}/sdk/*.tar.gz ${WEBROOT}/ + + cp $WORKSPACE/{tests/vtx.js,patches/tinytar.min.js} ${WEBROOT}/ + +popd + + +echo " +============= link web : end =============== + + + +" + diff --git a/pglite/cibuild/pack_extension.py b/pglite/cibuild/pack_extension.py new file mode 100644 index 0000000000000..79b26640f5c88 --- /dev/null +++ b/pglite/cibuild/pack_extension.py @@ -0,0 +1,139 @@ +# cibuild/pack_extension.py + +# TODO: use this file for merging symbols too. + +# use recorded file list in ${PGROOT}/pg.installed +# get other files into a tarball, find a .so and named everything after it + + +import asyncio +import tarfile +import os +import sys +from pathlib import Path + +class Error(Exception): + pass + +def gather(root: Path, *kw): + + for current, dirnames, filenames in os.walk(root): + rel = Path("/").joinpath(Path(current).relative_to(root)) + + # print(rel, len(dirnames), len(filenames)) + yield rel, filenames + + + +def is_extension(path:Path, fullpath:Path): + global EXTNAME, SYMBOLS, PGPATCH, PGROOT + asp = path.as_posix() + + # check .so + if asp.startswith('/lib/postgresql/'): + if path.suffix == ".so": + EXTNAME = path.stem + if os.environ.get('OBJDUMP',''): + os.system(f"wasm-objdump -x {fullpath} > {PGROOT}/dump.{EXTNAME}") + os.system(f"OBJDUMP={PGROOT}/dump.{EXTNAME} python3 cibuild/getsyms.py imports > {PGPATCH}/imports/{EXTNAME}") + with open(f"{PGPATCH}/imports/{EXTNAME}","r") as f: + SYMBOLS=f.readlines() + + return True + + # rpath + if asp.startswith('/lib/'): + return True + + if asp.startswith('/share/postgresql/extension'): + return True + + + + +async def archive(target_folder): + global INSTALLED, PACKLIST + + walked = [] + for folder, filenames in gather(target_folder): + walked.append([folder, filenames]) + + + for folder, filenames in walked: + for filename in filenames: + test = Path(folder) / Path(filename) + asp = test.as_posix() + if (PGROOT/test).is_symlink(): + print("SYMLINK:", test) + continue + if asp not in INSTALLED: + if asp.startswith('/sdk/'): + continue + + if asp.startswith('/base/'): + continue + + fp = PGROOT / asp[1:] + if fp.is_symlink(): + continue + if is_extension(test, fp): + #print(f"{EXTNAME=}", test ) + PACKLIST.append( [fp, test] ) + else: + print("custom:", test) + + +PGROOT=Path(os.environ.get('PGROOT',"/tmp/pglite")) +PGPATCH=Path(os.environ.get('PGPATCH', PGROOT)) + +INSTALLED = [] + +EXTNAME = "" +PACKLIST = [] +SYMBOLS=[] + +PREINST = "/plpgsql" +IS_PREINST = PREINST in sys.argv +for line in open(PGROOT / "pg.installed" ).readlines(): + asp = Path(line[1:].strip()).as_posix() + if IS_PREINST: + if asp.find(PREINST)>0: + continue + INSTALLED.append( asp ) + + +print("="*80) +asyncio.run( archive(PGROOT) ) +print("="*80) + +if not EXTNAME: + print("ERROR: no new installed extension found, is it builtin ?") + sys.exit(105) + +print(f""" +PG installed in : {PGROOT=} + + + {EXTNAME =} ({len(SYMBOLS)} imports) + + + +""") + +swd = os.getcwd() + +if (not IS_PREINST) and ("builtin" not in sys.argv): + if len(PACKLIST): + os.chdir(PGROOT) + with tarfile.open(PGROOT / "sdk" / f"{EXTNAME}.tar" , "w:") as tar: + for fp, fn in PACKLIST: + print(f"{EXTNAME} : {fp} => {fn}") + tar.add(fn.as_posix()[1:]) + #if "builtin" not in sys.argv: + os.remove(fp) + os.chdir(swd) + else: + print(f"Nothing found to pack for {EXTNAME}, did you 'make install' ?") +else: + print("Nothing to pack for builtin extension :", EXTNAME) + diff --git a/pglite/cibuild/pg-16.x.sh b/pglite/cibuild/pg-16.x.sh new file mode 100755 index 0000000000000..a576580d66aa7 --- /dev/null +++ b/pglite/cibuild/pg-16.x.sh @@ -0,0 +1,70 @@ +ARCHIVE=postgresql-${PG_VERSION}.tar.gz + +if echo ${PG_VERSION} | grep -q 16.5 +then + PG_PREREL=true + ARCHIVE_URL=https://github.com/postgres/postgres/archive/refs/tags/REL_16_5.tar.gz +else + PG_PREREL=false + ARCHIVE_URL=https://ftp.postgresql.org/pub/source/v${PG_VERSION}/${ARCHIVE} +fi + + +if [ -f postgresql/postgresql-${PG_VERSION}.patched ] +then + echo " + + Version ${PG_VERSION} already selected and patch stage already done + +" +else + [ -f ${ARCHIVE} ] || wget -q -c -O${ARCHIVE} ${ARCHIVE_URL} + + tar xfz ${ARCHIVE} + + if $PG_PREREL + then + ln -sf $(pwd)/postgres-REL_16_? postgresql-${PG_VERSION} + fi + + if pushd postgresql-${PG_VERSION} + then + echo + > ./src/template/emscripten + > ./src/include/port/emscripten.h + > ./src/makefiles/Makefile.emscripten + for patchdir in \ + postgresql-emscripten \ + postgresql-wasm postgresql-wasm-${PG_VERSION} \ + postgresql-pglite postgresql-pglite-${PG_VERSION} + do + if [ -d ../patches/$patchdir ] + then + cat ../patches/$patchdir/*.diff | patch -p1 || exit 24 + fi + done + touch postgresql-${PG_VERSION}.patched + popd + fi + + # either a submodule dir or a symlink. + # release only use symlink + + rm postgresql 2>/dev/null + ln -s postgresql-${PG_VERSION} postgresql + +fi + +export PGSRC=$(realpath postgresql-${PG_VERSION}) + +if [ -f ${PGROOT}/pg.installed ] +then + echo " + skipping pg build, using previous install because ${PGROOT}/pg.installed + +" +else + echo "Building $ARCHIVE (patched) from $PGSRC" + . cibuild/pgbuild.sh +fi + diff --git a/pglite/cibuild/pg-git.sh b/pglite/cibuild/pg-git.sh new file mode 100755 index 0000000000000..167b1039c1ffd --- /dev/null +++ b/pglite/cibuild/pg-git.sh @@ -0,0 +1,75 @@ +if [ -f postgresql/postgresql-${PG_VERSION}.patched ] +then + echo version already selected and patch stage already done +else + git clone --no-tags --depth 1 --single-branch --branch master https://github.com/postgres/postgres postgresql-${PG_VERSION} + + if pushd postgresql-${PG_VERSION} + then + echo + > ./src/template/emscripten + > ./src/template/wasi + > ./src/include/port/wasi.h + > ./src/makefiles/Makefile.wasi + for patchdir in \ + postgresql-debug postgresql-wasi \ + postgresql-wasm postgresql-wasm-${PG_VERSION} \ + postgresql-pglite postgresql-pglite-${PG_VERSION} + do + if [ -d ../patches/$patchdir ] + then + cat ../patches/$patchdir/*.diff | patch -p1 || exit 24 + fi + done + touch postgresql-${PG_VERSION}.patched + popd + fi + + # either a submodule dir or a symlink. + # release only use symlink + + rm postgresql 2>/dev/null + ln -s postgresql-${PG_VERSION} postgresql + +fi + +export PGSRC=$(realpath postgresql-${PG_VERSION}) + +if [ -f ${PGROOT}/pg.installed ] +then + echo "skipping pg build, using previous install from ${PGROOT}" +else + if $WASI + then + + CNF="${PGSRC}/configure FLEX=`which flex` --prefix=${PGROOT} \ + --disable-spinlocks --disable-atomics \ + --without-zlib --disable-largefile --without-llvm \ + --without-pam --disable-largefile --without-zlib --with-openssl=no \ + --without-readline --without-icu \ + ${PGDEBUG}" + +# --cache-file=${PGROOT}/config.cache.wasi + if \ + ZIC=/usr/sbin/zic \ + CC=wasi-c \ + CXX=wasi-c++ \ + CONFIG_SITE=${PGDATA}/config.site \ + $CNF \ + --host=$(arch) --target=wasm32-unknown-wasi --with-template=wasi + then + echo configure ok + make -j 1 + else + echo configure failed + exit 57 + fi + + read + + else + echo "Building $ARCHIVE (patched) from $PGSRC" + . cibuild/pgbuild.sh + fi +fi + diff --git a/pglite/cibuild/pgbuild.sh b/pglite/cibuild/pgbuild.sh new file mode 100755 index 0000000000000..25b831748dec6 --- /dev/null +++ b/pglite/cibuild/pgbuild.sh @@ -0,0 +1,364 @@ +echo " + + + +pgbuild:begin + +CC_PGLITE=$CC_PGLITE + +" + + + mkdir -p build/postgres + pushd build/postgres + + # create empty package.json to avoid emsdk node conflicts + # with root package.json of project + echo "{}" > package.json + + + if $CI + then + echo "CI : using build cache" + else + if [ -f Makefile ] + then + echo "Cleaning up previous build ..." + make distclean 2>&1 > /dev/null + fi + fi + +# TODO: --with-libxml xml2 >= 2.6.23 +# TODO: --with-libxslt add to sdk +# --disable-atomics https://github.com/WebAssembly/threads/pull/147 "Allow atomic operations on unshared memories" + + + + if ${WASI} + then + echo "WASI BUILD: turning off xml/xslt support" + XML2="" + UUID="" + BUILD=wasi + export MAIN_MODULE="-lwasi-emulated-getpid -lwasi-emulated-mman -lwasi-emulated-signal -lwasi-emulated-process-clocks" + else + # --with-libxml does not fit with --without-zlib + if $CI + then + # do not build obsolete ext xml2 on CI + XML2="--with-zlib --with-libxml" + else + XML2="--with-zlib --with-libxml --with-libxslt" + fi + UUID="--with-uuid=ossp" + BUILD=emscripten + export MAIN_MODULE="-sMAIN_MODULE=1" + fi + + + + export XML2_CONFIG=$PREFIX/bin/xml2-config + export ZIC=$(pwd)/bin/zic + + cp ${PGSRC}/./src/include/port/wasm_common.h /tmp/pglite/include/wasm_common.h + + CNF="${PGSRC}/configure FLEX=`which flex` --prefix=${PGROOT} \ + --cache-file=${PGROOT}/config.cache.${BUILD} \ + --disable-spinlocks --disable-largefile --without-llvm \ + --without-pam --disable-largefile --with-openssl=no \ + --without-readline --without-icu \ + ${UUID} ${XML2} ${PGDEBUG}" + + echo " ==== building wasm MVP:$MVP Debug=${PGDEBUG} with opts : $@ == " + + # -lwebsocket.js -sPROXY_POSIX_SOCKETS -pthread -sPROXY_TO_PTHREAD + # CONFIG_SITE=$(pwd)/config.site EMCC_CFLAGS="--oformat=html" \ + + # crash clang CFLAGS=-Wno-error=implicit-function-declaration + + + mkdir -p bin + + + [ -f /usr/bin/zic ] && cp /usr/bin/zic bin/ + if [ -f bin/zic ] + then + echo "using system zic" + GETZIC=false + else + GETZIC=true + fi + + + if $WASI + then + export EXT=wasi + cat > ${PGROOT}/config.site < bin/zic < ${PGROOT}/config.site < bin/zic < /tmp/disable-shared.log + + mkdir -p $PGROOT/bin + + cat > $PGROOT/bin/emsdk-shared <> /tmp/disable-shared.log +# shared build +\${PG_LINK:-emcc} -L${PREFIX}/lib -DPREFIX=${PGROOT} -shared -sSIDE_MODULE=1 \$@ -Wno-unused-function +END + ln -sf $PGROOT/bin/emsdk-shared bin/emsdk-shared + + + cat > $PGROOT/bin/wasi-shared <> /tmp/disable-shared.log +# shared build +echo =================================================================================== +wasi-c -L${PREFIX}/lib -DPREFIX=${PGROOT} -shared \$@ -Wno-unused-function +echo =================================================================================== +END + ln -sf $PGROOT/bin/wasi-shared bin/wasi-shared + + chmod +x bin/zic $PGROOT/bin/wasi-shared $PGROOT/bin/emsdk-shared + + # for zic and emsdk-shared/wasi-shared called from makefile + export PATH=$(pwd)/bin:$PATH + + EMCC_NODE="-sEXIT_RUNTIME=1 -DEXIT_RUNTIME -sNODERAWFS -sENVIRONMENT=node" + + # -lwebsocket.js" + # -sWEBSOCKET_SUBPROTOCOL=binary -sWEBSOCKET_URL=ws://127.0.0.1:25432" + + # -lwebsocket.js + # -sPROXY_POSIX_SOCKETS -pthread -sPROXY_TO_PTHREAD $EMCC_CFLAGS" + + # -sWASMFS + + EMCC_ENV="${EMCC_NODE} -sFORCE_FILESYSTEM=0" + EMCC_ENV="${EMCC_NODE} -sERROR_ON_UNDEFINED_SYMBOLS" + + # PREFIX only required for static initdb + EMCC_CFLAGS="-sERROR_ON_UNDEFINED_SYMBOLS=1 ${CC_PGLITE} -DPREFIX=${PGROOT} -Wno-macro-redefined -Wno-unused-function" + + WASI_CFLAGS="${CC_PGLITE} -DPREFIX=${PGROOT} -DPYDK=1 -Wno-declaration-after-statement -Wno-macro-redefined -Wno-unused-function -Wno-missing-prototypes -Wno-incompatible-pointer-types" + + ZIC=${ZIC:-$(realpath bin/zic)} + + + + if EMCC_CFLAGS="${EMCC_ENV} ${EMCC_CFLAGS}" WASI_CFLAGS="$WASI_CFLAGS" emmake make $BUILD=1 -j $(nproc) 2>&1 > /tmp/build.log + then + echo build ok + if $WASI + then + echo "will need to link postgres to extensions statically" + cp -vf src/backend/postgres src/backend/postgres.wasi + else + cp -vf src/backend/postgres src/backend/postgres.cjs + fi + + # if running a 32bits zic from current build + unset LD_PRELOAD + + if EMCC_CFLAGS="${EMCC_ENV} ${EMCC_CFLAGS}" WASI_CFLAGS="$WASI_CFLAGS" emmake make $BUILD=1 install 2>&1 > /tmp/install.log + then + echo install ok + if $WASI + then + # remove unlinked server + rm src/backend/postgres src/backend/postgres.wasi $PGROOT/bin/postgres $PGROOT/bin/postgres.wasi + pushd ../.. + chmod +x ./cibuild/linkwasi.sh + WASI_CFLAGS="$WASI_CFLAGS" ./cibuild/linkwasi.sh || exit 190 + popd + cp src/backend/postgres.wasi $PGROOT/bin/ || exit 205 + fi + + pushd ${PGROOT} + + find . -type f | grep -v plpgsql > ${PGROOT}/pg.installed + popd + + goback=$(pwd) + popd + python3 cibuild/pack_extension.py builtin + pushd $goback + + pushd ${PGROOT} + find . -type f > ${PGROOT}/pg.installed + popd + + else + cat /tmp/install.log + echo "install failed" + exit 225 + fi + else + cat /tmp/build.log + echo "build failed" + exit 230 + fi + + # wip + mv -vf ./src/bin/psql/psql.$EXT ./src/bin/pg_config/pg_config.$EXT ${PGROOT}/bin/ + mv -vf ./src/bin/pg_dump/pg_restore.$EXT ./src/bin/pg_dump/pg_dump.$EXT ./src/bin/pg_dump/pg_dumpall.$EXT ${PGROOT}/bin/ + mv -vf ./src/bin/pg_resetwal/pg_resetwal.$EXT ./src/bin/initdb/initdb.$EXT ./src/backend/postgres.$EXT ${PGROOT}/bin/ + + + python3 > ${PGROOT}/PGPASSFILE < ${PGROOT}/bin/pg_config < ${PGROOT}/postgres <&1 | grep --line-buffered -v ^var\\ Module + + # force node wasm version + cp -vf ${PGROOT}/postgres ${PGROOT}/bin/postgres + + cat > ${PGROOT}/initdb <$PGROOT/initdb.sh < /tmp/initdb.txt + +${PGROOT}/initdb --no-clean --wal-segsize=1 -g $LANG $CRED --pgdata=${PGDATA} + +mv /tmp/initdb.boot.txt \${SQL}.boot.sql +mv /tmp/initdb.single.txt \${SQL}.single.sql + +if \${CI:-false} +then + cp -vf \$SQL ${PGROOT}/\$(md5sum \$SQL|cut -c1-32).sql +fi + +# --wal-segsize=1 -> -X 1048576 + +# CKSUM_B -k --data-checksums +# 2024-04-24 05:53:28.121 GMT [42] WARNING: page verification failed, calculated checksum 5487 but expected 0 +# 2024-04-24 05:53:28.121 GMT [42] FATAL: invalid page in block 0 of relation base/1/1259 + +CMD="${PGROOT}/postgres --boot -D ${PGDATA} -d 3 $PGOPTS -X 1048576" +echo "\$CMD < \$SQL.boot.sql" +\$CMD < \$SQL.boot.sql 2>&1 \\ + | grep -v --line-buffered 'bootstrap> boot' \\ + | grep -v --line-buffered 'index' + +echo " + +\$(md5sum /tmp/initdb-\$\$.*.sql) + + boot done +" +END + + popd +echo "pgbuild:end + + + + +" + + + diff --git a/pglite/cibuild/pglite-ts.sh b/pglite/cibuild/pglite-ts.sh new file mode 100755 index 0000000000000..3fbdc4e8f65d9 --- /dev/null +++ b/pglite/cibuild/pglite-ts.sh @@ -0,0 +1,53 @@ +pushd packages/pglite + PGLITE=$(pwd) + + pnpm install --frozen-lockfile + + mkdir -p $PGLITE/release + rm $PGLITE/release/* 2>/dev/null + + # copy packed extensions + cp ${WEBROOT}/*.tar.gz ${PGLITE}/release/ + + # copy wasm web prebuilt artifacts to release folder + # TODO: get them from web for nosdk systems. + + cp ${WEBROOT}/postgres.{js,data,wasm} ${PGLITE}/release/ + + # debug CI does not use pnpm/npm for building pg, so call the typescript build + # part from here + pnpm run build:js || exit 28 + + mkdir -p /tmp/sdk + pnpm pack || exit 31 + packed=$(echo -n electric-sql-pglite-*.tgz) + + mv $packed /tmp/sdk/pg${PG_VERSION}-${packed} + + # for repl demo + mkdir -p /tmp/web/pglite + cp -r ${PGLITE}/dist /tmp/web/pglite/ + cp -r ${PGLITE}/examples /tmp/web/pglite/ + + for dir in /tmp/web /tmp/web/pglite/examples + do + pushd "$dir" + cp ${PGLITE}/dist/postgres.data ./ + popd + done + + echo " + + + + " > ${WEBROOT}/index.html + +popd + diff --git a/pglite/cibuild/repo-pg-16.x.sh b/pglite/cibuild/repo-pg-16.x.sh new file mode 100755 index 0000000000000..d994b7ae802a6 --- /dev/null +++ b/pglite/cibuild/repo-pg-16.x.sh @@ -0,0 +1,18 @@ + +PG_VERSION=16.4 +PG_PREREL=false + +# these are files that shadow original portion of pg core, with minimal changes +# to original code +# some may be included multiple time +CC_PGLITE="-DPATCH_MAIN=${PGSRC}/src/pglite-extra/pg_main.c ${CC_PGLITE}" +CC_PGLITE="-DPATCH_LOOP=${PGSRC}/src/pglite-extra/interactive_one.c ${CC_PGLITE}" +CC_PGLITE="-DPATCH_PLUGIN=${PGSRC}/src/pglite-extra/pg_plugin.h ${CC_PGLITE}" + +export CC_PGLITE + +rm postgresql 2>/dev/null +ln -s $PGSRC postgresql + +echo "Building postgresql-${PG_VERSION} in folder $PGSRC" +. cibuild/pgbuild.sh \ No newline at end of file diff --git a/pglite/cibuild/reqsym.py b/pglite/cibuild/reqsym.py new file mode 100644 index 0000000000000..d59e1851d3014 --- /dev/null +++ b/pglite/cibuild/reqsym.py @@ -0,0 +1,27 @@ +#!/usr/bin/env python3 +import sys +import os + +def dbg(*argv, **kw): + kw.setdefault('file',sys.stderr) + return print(*argv,**kw) + +if "${WORKSPACE}".endswith('{WORKSPACE}'): + with open("/data/git/pglite-16.x/patches/exports.pglite", "r") as file: + exports = set(map(str.strip, file.readlines())) +else: + with open("${WORKSPACE}/patches/exports.pglite", "r") as file: + exports = set(map(str.strip, file.readlines())) + +with open("/tmp/symbols", "r") as file: + imports = set(map(str.strip, file.readlines())) + +matches = list( imports.intersection(exports) ) +for sym in matches: + print(sym) + +dbg(f""" +exports {len(exports)} +imports {len(imports)} +Matches : {len(matches)} +""") diff --git a/pglite/extra/postgis.diff b/pglite/extra/postgis.diff new file mode 100644 index 0000000000000..a302bf5546f22 --- /dev/null +++ b/pglite/extra/postgis.diff @@ -0,0 +1,11 @@ +--- postgis-3.5.0/configure ++++ postgis-3.5.0-wasm/configure +@@ -18972,7 +18972,7 @@ + + { $as_echo "$as_me:${as_lineno-$LINENO}: checking PROJ version via compiled code" >&5 + $as_echo_n "checking PROJ version via compiled code... " >&6; } +- if test "$cross_compiling" = yes; then : ++ if test "$cross_compiling_not_a_problem" = yes; then : + { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 + $as_echo "$as_me: error: in \`$ac_pwd':" >&2;} + as_fn_error $? "cannot run test program while cross compiling diff --git a/pglite/extra/postgis.sh b/pglite/extra/postgis.sh new file mode 100755 index 0000000000000..d17bb7d4291df --- /dev/null +++ b/pglite/extra/postgis.sh @@ -0,0 +1,74 @@ + +BUILD=build +mkdir -p ${BUILD} src + + + +pushd ${BUILD} + if [ -d postgis-3.5.0 ] + then + echo -n + else + [ -f ../src/postgis-3.5.0.tar.gz ] || wget -c https://download.osgeo.org/postgis/source/postgis-3.5.0.tar.gz -O../src/postgis-3.5.0.tar.gz + tar xfz ../src/postgis-3.5.0.tar.gz + pushd postgis-3.5.0 + patch -p1 < ${WORKSPACE}/extra/postgis.diff + popd + fi +popd + +if which emcc +then + echo -n +else + reset; + . /opt/python-wasm-sdk/wasm32-bi-emscripten-shell.sh + export PGROOT=${PGROOT:-/tmp/pglite} + export PATH=${PGROOT}/bin:$PATH +fi + +pushd ${BUILD}/postgis-3.5.0 + + cat > config.site < --with-gdalconfig= + # --with-gdalconfig=$PREFIX/bin/gdal-config + CONFIG_SITE=config.site emconfigure ./configure FLEX=`which flex` \ + --without-raster --without-topology --without-address-standardizer \ + --with-gdalconfig=$PREFIX/bin/gdal-config \ + --without-gui --without-phony-revision --without-protobuf \ + --without-interrupt-tests --without-json \ + --without-libiconv --without-libiconv-prefix \ + --with-pgconfig=/tmp/pglite/bin/pg_config \ + --with-xml2config=$SDKROOT/devices/emsdk/usr/bin/xml2-config \ + --with-projdir=$SDKROOT/devices/emsdk/usr \ + --with-geosconfig=$SDKROOT/devices/emsdk/usr/bin/geos-config $@ + + # workaround iconv + mkdir -p loader/no/lib + + # or would fail on some frontend functions linking. + sed -i 's/PGSQL_FE_LDFLAGS=-L/PGSQL_FE_LDFLAGS=-O0 -g3 -sERROR_ON_UNDEFINED_SYMBOLS=0 -L/g' loader/Makefile + #DEFAULT_LIBRARY_FUNCS_TO_INCLUDE="_emscripten_memcpy_js" + EMCC_CFLAGS="-O0 -g3 -sERROR_ON_UNDEFINED_SYMBOLS=0 -Wno-unused-function -lpng -ljpeg" emmake make + # /opt/python-wasm-sdk/devices/emsdk/usr/lib/libgeos.a + rm postgis/postgis-3.s* + PATH=/tmp/pglite/bin:$PATH PG_LINK="em++ $PREFIX/lib/libgeos.a $EMSDK/upstream/emscripten/cache/sysroot/lib/wasm32-emscripten/pic/libsqlite3.a" emmake make install + rm $PGROOT/share/postgresql/extension/postgis*.sql + cp ./extensions/postgis/sql/postgis--3.5.0.sql $PGROOT/share/postgresql/extension/postgis--3.5.0.sql + rm -rf /tmp/pglite/share/postgresql/contrib + rm -rf /tmp/pglite/lib/postgresql/pgxs +popd + diff --git a/pglite/extra/vector.sh b/pglite/extra/vector.sh new file mode 100755 index 0000000000000..92fdcb36bcbd8 --- /dev/null +++ b/pglite/extra/vector.sh @@ -0,0 +1,39 @@ +#!/bin/bash + +mkdir -p build src + +pushd build + # [ -d pgvector ] || git clone --no-tags --depth 1 --single-branch --branch master https://github.com/pgvector/pgvector + + if [ -d vector ] + then + echo using local pgvector + else + [ -f ../src/pgvector.tar.gz ] || wget -c -q https://github.com/pgvector/pgvector/archive/refs/tags/v0.8.0.tar.gz -O../src/pgvector.tar.gz + tar xvfz ../src/pgvector.tar.gz + mv pgvector-?.?.? vector + fi +popd + + + +if which emcc +then + echo -n +else + reset; + . /opt/python-wasm-sdk/wasm32-bi-emscripten-shell.sh + export PGROOT=${PGROOT:-/tmp/pglite} + export PATH=${PGROOT}/bin:$PATH +fi + + +pushd build/vector + # path for wasm-shared already set to (pwd:pg build dir)/bin + # OPTFLAGS="" turns off arch optim (sse/neon). + PG_CONFIG=${PGROOT}/bin/pg_config emmake make OPTFLAGS="" install || exit 33 + cp sql/vector.sql sql/vector--0.7.3.sql ${PGROOT}/share/postgresql/extension + rm ${PGROOT}/share/postgresql/extension/vector--?.?.?--?.?.?.sql ${PGROOT}/share/postgresql/extension/vector.sql +popd + + diff --git a/pglite/tests/repl.html b/pglite/tests/repl.html new file mode 100644 index 0000000000000..08b1af8cbe1a9 --- /dev/null +++ b/pglite/tests/repl.html @@ -0,0 +1,1077 @@ + + + + + + + PG SHELL TEST + + + + + + + + + + + + + + + + + +
+
emscripten
+
Downloading...
+
+ +
+ + + +
+ + +
+ +
+ +
+ + + +#if MODULARIZE + + +#endif + + + + + diff --git a/pglite/tests/vtx.js b/pglite/tests/vtx.js new file mode 100644 index 0000000000000..ada9ccaee4ed3 --- /dev/null +++ b/pglite/tests/vtx.js @@ -0,0 +1,320 @@ +"use strict"; + +var readline = { last_cx : -1 , index : 0, history : ["help()"] } + +// two modes based on RAW_MODE: default readline emulation or vm.rawstdin + +readline.complete = function (line) { + if ( readline.history[ readline.history.length -1 ] != line ) + readline.history.push(line); + readline.index = 0; + vm.readline(line + "\n") + +} + +if (!window.Terminal) { + if (window.vm) { + var xterm_cdn + if (window.vm.config && window.vm.config.cdn) { + xterm_cdn = window.vm.config.cdn+"vt/" + console.log("Terminal+ImageAddon importing from CDN :", xterm_cdn) + } else { + xterm_cdn = xterm_cdn || "https://pygame-web.github.io/archives/vt/" + console.warn("Terminal+ImageAddon importing from fallback ", xterm_cdn) + } + + for (const css of ["xterm.css"]) { + const cssref = document.createElement('link') + cssref.setAttribute("rel", "stylesheet") + cssref.setAttribute("type", "text/css") + cssref.setAttribute("href", xterm_cdn + css) + document.getElementsByTagName("head")[0].appendChild(cssref) + } + + await import(xterm_cdn + "xterm.js") + await import(xterm_cdn + "xterm-addon-image.js") + } else { + console.error("only use import for vtx.js module") + } + +} else { + console.warn("Terminal+ImageAddon were inlined") +} + + + +export class WasmTerminal { + constructor(hostid, cols, rows, fontsize, is_fbdev, addons_list) { + this.input = '' + this.resolveInput = null + this.activeInput = true + this.inputStartCursor = null + + this.nodup = 1 + var theme = { + background: '#1a1c1f' + } + + + var transparency = false + var sback = 1000 + + if (is_fbdev) { + theme = { + foreground: '#ffffff', + background: 'rgba(0, 0, 0, 0)' + } + sback = 0 + transparency = true + } + + this.xterm = new Terminal( + { +// rendererType : "dom", + rendererType : "webgl", + experimentalCharAtlas : "webgl", + theme: theme, + allowTransparency: transparency, + allowProposedApi : true , // xterm 0.5 + sixel + scrollback: sback, + fontFamily: 'Courier-new, courier, monospace', + fontSize: (fontsize || 12), + cols: (cols || 132), + rows: (rows || 32) + } + ); + + if (typeof(Worker) !== "undefined") { + + for (const addon of (addons_list||[]) ) { + console.warn(hostid, cols, rows, addon) + const imageAddon = new ImageAddon.ImageAddon(addon.url , addon); + this.xterm.loadAddon(imageAddon); + this.sixel = function write(data) { + this.xterm.write(data) + } + } + + } else { + console.warn("No worker support, not loading xterm addons") + this.sixel = function ni() { + console.warn("SIXEL N/I") + } + } + + this.xterm.open(document.getElementById(hostid)) + + this.xterm.onKey((keyEvent) => { + // Fix for iOS Keyboard Jumping on space + if (keyEvent.key === " ") { + keyEvent.domEvent.preventDefault(); + } + + }); + + this.xterm.onData(this.handleTermData) + } + + open(container) { + this.xterm.open(container); + } + + ESC() { + for (var i=0; i < arguments.length; i++) + this.xterm.write("\x1b"+arguments[i]) + } + + handleTermData = (data) => { + + + +// TODO: check mouse Y pos for raw mode in debug mode + if (window.RAW_MODE) { + vm.rawstdin(data) + return + } + + const ord = data.charCodeAt(0); + let ofs; + + const cx = this.xterm.buffer.active.cursorX + + // TODO: Handle ANSI escape sequences + if (ord === 0x1b) { + + // Handle special characters + switch ( data.charCodeAt(1) ) { + case 0x5b: + + const cursor = readline.history.length + readline.index + var histo = ">h> " + + switch ( data.charCodeAt(2) ) { + // "?" + case 63: + const c4 = data.charCodeAt(4) + const c5 = data.charCodeAt(5) + if ((c4==54) && (c5==99)) { + // Primary Device Attribute of Sixel support : 4 + // "?6c" https://github.com/odknt/st/issues/1 + console.log("query") + + } + + + case 65: + //console.log("VT UP") + // memo cursor pos before entering histo + if (!readline.index) { + if (readline.last_cx < 0 ) { + readline.last_cx = cx + readline.buffer = this.input + } + // TODO: get current line content from XTERM + } + + if ( cursor >0 ) { + readline.index-- + histo = ">h> " +readline.history[cursor-1] + //console.log(__FILE__," histo-up :", readline.index, cursor, histo) + + this.ESC("[132D","[2K") + this.xterm.write(histo) + this.input = histo.substr(4) + } + break; + + case 66: + //console.log("VT DOWN") + if ( readline.index < 0 ) { + readline.index++ + histo = histo + readline.history[cursor] + this.ESC("[132D","[2K") + this.xterm.write(histo) + this.input = histo.substr(4) + } else { + // we are back + if (readline.last_cx >= 0) { + histo = histo + readline.buffer + readline.buffer = "" + this.ESC("[2K") + this.ESC("[132D") + this.xterm.write(histo) + this.input = histo.substr(4) + this.ESC("[132D") + this.ESC("["+readline.last_cx+"C") + //console.log(__FILE__," histo-back", readline.index, cursor, histo) + readline.last_cx = -1 + } + } + break; + + case 67: + //console.log("VT RIGHT") + break; + + case 68: + //console.log("VT LEFT") + break; + + case 60: + // vm.rawstdin(data) + break; + + default: + console.log(__FILE__,"VT unhandled ? "+data.charCodeAt(2)) + } + break + default: + + console.log(__FILE__,"VT ESC "+data.charCodeAt(1)) + } + + } else if (ord < 32 || ord === 0x7f) { + switch (data) { + case "\r": // ENTER + case "\x0a": // CTRL+J + case "\x0d": // CTRL+M + this.xterm.write('\r\n'); + readline.complete(this.input) + this.input = ''; + break; + case "\x7F": // BACKSPACE + case "\x08": // CTRL+H + case "\x04": // CTRL+D + this.handleCursorErase(true); + break; + + case "\0x03": // CTRL+C + + break + + // ^L for clearing VT but keep X pos. + case "\x0c": + const cy = this.xterm.buffer.active.cursorY + + if (cy < this.xterm.rows ) + this.ESC("[B","[J","[A") + + this.ESC("[A","[K","[1J") + + for (var i=1;i0 ) + this.ESC("["+cx+"C") + break; + + default: + switch (ord) { + case 3: + readline.complete("raise KeyboardInterrupt") + break + default : + console.log("vt:" + ord ) + } + } + } else { + this.input += data; + this.xterm.write(data) + } + } + + handleCursorErase() { + // Don't delete past the start of input + if (this.xterm.buffer.active.cursorX <= this.inputStartCursor) { + return + } + this.input = this.input.slice(0, -1) + this.xterm.write('\x1B[D') + this.xterm.write('\x1B[P') + } + + + clear() { + this.xterm.clear() + } + + // direct write + sixel(data) { + this.xterm.write(data) + } + + print(message) { + const normInput = message.replace(/[\r\n]+/g, "\n").replace(/\n/g, "\r\n") + this.xterm.write(normInput) + } + +} + + +window.WasmTerminal = WasmTerminal +window.readline = readline + + + + + diff --git a/src/Makefile.shlib b/src/Makefile.shlib index f94d59d1c5970..11767a8fd8f64 100644 --- a/src/Makefile.shlib +++ b/src/Makefile.shlib @@ -233,6 +233,33 @@ ifeq ($(SHLIB_EXPORTS),) override CXXFLAGS += $(CXXFLAGS_SL_MODULE) endif +ifeq ($(PORTNAME), emscripten) + LINK.shared = emsdk-shared + ifdef soname + # emscripten uses unversioned shared libraries + shlib = $(shlib_bare) + soname = $(shlib_bare) + endif + BUILD.exports = ( echo '{ global:'; $(AWK) '/^[^\#]/ {printf "%s;\n",$$1}' $<; echo ' local: *; };' ) >$@ + exports_file = $(SHLIB_EXPORTS:%.txt=%.list) +# ifneq (,$(exports_file)) +# LINK.shared += -Wl,--version-script=$(exports_file) +# endif +endif + +ifeq ($(PORTNAME), wasi) + LINK.shared = wasi-shared + ifdef soname + # emscripten uses unversioned shared libraries + shlib = $(shlib_bare) + soname = $(shlib_bare) + endif + BUILD.exports = ( echo '{ global:'; $(AWK) '/^[^\#]/ {printf "%s;\n",$$1}' $<; echo ' local: *; };' ) >$@ + exports_file = $(SHLIB_EXPORTS:%.txt=%.list) +# ifneq (,$(exports_file)) +# LINK.shared += -Wl,--version-script=$(exports_file) +# endif +endif ## ## BUILD @@ -248,8 +275,11 @@ all-lib: lib$(NAME).pc endif all-static-lib: $(stlib) - +ifdef wasi +all-shared-lib: all-static-lib +else all-shared-lib: $(shlib) +endif # In this rule, "touch $@" works around a problem on some platforms wherein # ar updates the library file's mod time with a value calculated to diff --git a/src/backend/Makefile b/src/backend/Makefile index 82cae98a44a1f..0304de6f4e0eb 100644 --- a/src/backend/Makefile +++ b/src/backend/Makefile @@ -37,9 +37,11 @@ LOCALOBJS += utils/probes.o endif endif -OBJS = \ +ONLYOBJS = \ $(LOCALOBJS) \ - $(SUBDIROBJS) \ + $(SUBDIROBJS) +OBJS = \ + $(ONLYOBJS) \ $(top_builddir)/src/common/libpgcommon_srv.a \ $(top_builddir)/src/port/libpgport_srv.a @@ -60,6 +62,7 @@ override LDFLAGS := $(LDFLAGS) $(LDFLAGS_EX) $(LDFLAGS_EX_BE) all: submake-libpgport submake-catalog-headers submake-utils-headers postgres $(POSTGRES_IMP) +ifneq ($(PORTNAME), emscripten) ifneq ($(PORTNAME), cygwin) ifneq ($(PORTNAME), win32) ifneq ($(PORTNAME), aix) @@ -70,6 +73,21 @@ postgres: $(OBJS) endif endif endif +endif + +ifeq ($(PORTNAME), emscripten) +AR ?= llvm-ar +LIBPGLITE ?= $(top_builddir)/libpglite.a +LIBPG = $(top_builddir)/libpostgres.a +PGLITE = $(top_builddir)/src/common/libpgcommon_srv.a $(top_builddir)/src/port/libpgport_srv.a $(LIBPG) +PGMAIN = main/main.o tcop/postgres.o +postgres: $(OBJS) + $(AR) rcs $(top_builddir)/libpgmain.a $(PGMAIN) + $(AR) rcs $(LIBPG) $(filter-out $(PGMAIN),$(call expand_subsys,$(ONLYOBJS))) + $(CC) -r -o $(top_builddir)/libpglite.o -Wl,--whole-archive $(PGLITE) + $(AR) rcs $(LIBPGLITE) $(top_builddir)/libpglite.o + COPTS="$(LOPTS)" $(CC) $(MAIN_MODULE) $(CFLAGS) $(LDFLAGS) -o $@ $(LIBPGLITE) $(top_builddir)/libpgmain.a $(LIBS) +endif ifeq ($(PORTNAME), cygwin) diff --git a/src/backend/access/nbtree/nbtutils.c b/src/backend/access/nbtree/nbtutils.c index e4528db477949..c1294fba57d75 100644 --- a/src/backend/access/nbtree/nbtutils.c +++ b/src/backend/access/nbtree/nbtutils.c @@ -2732,7 +2732,7 @@ _bt_allequalimage(Relation rel, bool debugmessage) break; } } - +#if !defined(__EMSCRIPTEN__) if (debugmessage) { if (allequalimage) @@ -2742,6 +2742,6 @@ _bt_allequalimage(Relation rel, bool debugmessage) elog(DEBUG1, "index \"%s\" cannot use deduplication", RelationGetRelationName(rel)); } - +#endif return allequalimage; } diff --git a/src/backend/access/transam/xact.c b/src/backend/access/transam/xact.c index 4a2ea4adbaf0c..a3e40843bc11a 100644 --- a/src/backend/access/transam/xact.c +++ b/src/backend/access/transam/xact.c @@ -1740,6 +1740,7 @@ RecordTransactionAbort(bool isSubXact) if (TransactionIdDidCommit(xid)) elog(PANIC, "cannot abort transaction %u, it was already committed", xid); + else elog(WARNING, "# 1743: aborting transaction %u", xid); /* * Are we using the replication origins feature? Or, in other words, are @@ -2748,7 +2749,9 @@ AbortTransaction(void) * handler. We do this fairly early in the sequence so that the timeout * infrastructure will be functional if needed while aborting. */ +#if !defined(__EMSCRIPTEN__) && !defined(__wasi__) sigprocmask(SIG_SETMASK, &UnBlockSig, NULL); +#endif /* * check the current transaction state @@ -5107,7 +5110,10 @@ AbortSubTransaction(void) * handler. We do this fairly early in the sequence so that the timeout * infrastructure will be functional if needed while aborting. */ +#if !defined(__EMSCRIPTEN__) && !defined(__wasi__) sigprocmask(SIG_SETMASK, &UnBlockSig, NULL); +#endif + /* * check the current transaction state diff --git a/src/backend/access/transam/xlogarchive.c b/src/backend/access/transam/xlogarchive.c index 524e80adb1ccd..80fb0ce0e48d6 100644 --- a/src/backend/access/transam/xlogarchive.c +++ b/src/backend/access/transam/xlogarchive.c @@ -33,6 +33,10 @@ #include "storage/ipc.h" #include "storage/lwlock.h" +#if defined(__wasi__) +#define system(cmd) system_wasi(cmd) +#endif + /* * Attempt to retrieve the specified file from off-line archival storage. * If successful, fill "path" with its complete path (note that this will be diff --git a/src/backend/bootstrap/bootstrap.c b/src/backend/bootstrap/bootstrap.c index 49e956b2c577f..daff9dfbf9dd0 100644 --- a/src/backend/bootstrap/bootstrap.c +++ b/src/backend/bootstrap/bootstrap.c @@ -199,7 +199,11 @@ CheckerModeMain(void) * to shared memory sizing, options work (or at least do not cause an error * up to shared memory creation). */ +#if !defined(__EMSCRIPTEN__) && !defined(__wasi__) void +#else +int +#endif BootstrapModeMain(int argc, char *argv[], bool check_only) { int i; @@ -353,7 +357,7 @@ BootstrapModeMain(int argc, char *argv[], bool check_only) */ if (pg_link_canary_is_frontend()) elog(ERROR, "backend is incorrectly linked to frontend functions"); - +puts("# 360: InitPostgres(boot): " __FILE__ ); InitPostgres(NULL, InvalidOid, NULL, InvalidOid, false, false, NULL); /* Initialize stuff for bootstrap-file processing */ @@ -378,7 +382,12 @@ BootstrapModeMain(int argc, char *argv[], bool check_only) /* Clean up and exit */ cleanup(); +#if !defined(__EMSCRIPTEN__) && !defined(__wasi__) proc_exit(0); +#else + puts("# 338 cleanup(boot): " __FILE__); + return 0; +#endif } diff --git a/src/backend/catalog/index.c b/src/backend/catalog/index.c index 6f1910a6e0f75..ec9e94accb19a 100644 --- a/src/backend/catalog/index.c +++ b/src/backend/catalog/index.c @@ -2995,7 +2995,7 @@ index_build(Relation heapRelation, indexInfo->ii_ParallelWorkers = plan_create_index_workers(RelationGetRelid(heapRelation), RelationGetRelid(indexRelation)); - +#if !defined(__EMSCRIPTEN__) if (indexInfo->ii_ParallelWorkers == 0) ereport(DEBUG1, (errmsg_internal("building index \"%s\" on table \"%s\" serially", @@ -3007,7 +3007,7 @@ index_build(Relation heapRelation, RelationGetRelationName(indexRelation), RelationGetRelationName(heapRelation), indexInfo->ii_ParallelWorkers))); - +#endif /* * Switch to the table owner's userid, so that any index functions are run * as that user. Also lock down security-restricted operations and diff --git a/src/backend/commands/async.c b/src/backend/commands/async.c index d6a91a9eaacc3..e8fe9300fc2d8 100644 --- a/src/backend/commands/async.c +++ b/src/backend/commands/async.c @@ -1700,8 +1700,12 @@ SignalBackends(void) * NotifyQueueLock; which is unlikely but certainly possible. So we * just log a low-level debug message if it happens. */ +#if defined(__EMSCRIPTEN__) + HandleNotifyInterrupt(); +#else if (SendProcSignal(pid, PROCSIG_NOTIFY_INTERRUPT, ids[i]) < 0) elog(DEBUG3, "could not signal backend with PID %d: %m", pid); +#endif } pfree(pids); diff --git a/src/backend/commands/collationcmds.c b/src/backend/commands/collationcmds.c index 4088481b9c7e0..e46bcb87d5d8b 100644 --- a/src/backend/commands/collationcmds.c +++ b/src/backend/commands/collationcmds.c @@ -828,14 +828,14 @@ pg_import_system_collations(PG_FUNCTION_ARGS) maxaliases = 100; aliases = (CollAliasData *) palloc(maxaliases * sizeof(CollAliasData)); naliases = 0; - locale_a_handle = OpenPipeStream("locale -a", "r"); - if (locale_a_handle == NULL) + if (locale_a_handle == NULL) { + puts("======================== ERROR ================"); ereport(ERROR, (errcode_for_file_access(), errmsg("could not execute command \"%s\": %m", "locale -a"))); - + } while (fgets(localebuf, sizeof(localebuf), locale_a_handle)) { size_t len; diff --git a/src/backend/commands/dbcommands.c b/src/backend/commands/dbcommands.c index 7295800a7d8c1..84d8e3d642ef4 100644 --- a/src/backend/commands/dbcommands.c +++ b/src/backend/commands/dbcommands.c @@ -1776,8 +1776,9 @@ dropdb(const char *dbname, bool missing_ok, bool force) RequestCheckpoint(CHECKPOINT_IMMEDIATE | CHECKPOINT_FORCE | CHECKPOINT_WAIT); /* Close all smgr fds in all backends. */ +#if !defined(__wasi__) && !defined(__EMSCRIPTEN__) WaitForProcSignalBarrier(EmitProcSignalBarrier(PROCSIGNAL_BARRIER_SMGRRELEASE)); - +#endif /* * Remove all tablespace subdirs belonging to the database. */ diff --git a/src/backend/commands/event_trigger.c b/src/backend/commands/event_trigger.c index d4b00d1a828a4..9ecba5a83226f 100644 --- a/src/backend/commands/event_trigger.c +++ b/src/backend/commands/event_trigger.c @@ -642,6 +642,8 @@ EventTriggerDDLCommandStart(Node *parsetree) List *runlist; EventTriggerData trigdata; +#if defined(__EMSCRIPTEN__) || defined(__wasi__) +#else /* * Event Triggers are completely disabled in standalone mode. There are * (at least) two reasons for this: @@ -660,6 +662,7 @@ EventTriggerDDLCommandStart(Node *parsetree) */ if (!IsUnderPostmaster) return; +#endif runlist = EventTriggerCommonSetup(parsetree, EVT_DDLCommandStart, @@ -690,12 +693,15 @@ EventTriggerDDLCommandEnd(Node *parsetree) List *runlist; EventTriggerData trigdata; +#if defined(__EMSCRIPTEN__) || defined(__wasi__) +#else /* * See EventTriggerDDLCommandStart for a discussion about why event * triggers are disabled in single user mode. */ if (!IsUnderPostmaster) return; +#endif /* * Also do nothing if our state isn't set up, which it won't be if there @@ -738,12 +744,15 @@ EventTriggerSQLDrop(Node *parsetree) List *runlist; EventTriggerData trigdata; +#if defined(__EMSCRIPTEN__) || defined(__wasi__) +#else /* * See EventTriggerDDLCommandStart for a discussion about why event * triggers are disabled in single user mode. */ if (!IsUnderPostmaster) return; +#endif /* * Use current state to determine whether this event fires at all. If @@ -809,12 +818,15 @@ EventTriggerTableRewrite(Node *parsetree, Oid tableOid, int reason) List *runlist; EventTriggerData trigdata; +#if defined(__EMSCRIPTEN__) || defined(__wasi__) +#else /* * See EventTriggerDDLCommandStart for a discussion about why event * triggers are disabled in single user mode. */ if (!IsUnderPostmaster) return; +#endif /* * Also do nothing if our state isn't set up, which it won't be if there diff --git a/src/backend/libpq/auth.c b/src/backend/libpq/auth.c index eaae83cc3a8b2..e90351221010c 100644 --- a/src/backend/libpq/auth.c +++ b/src/backend/libpq/auth.c @@ -1866,7 +1866,7 @@ auth_peer(hbaPort *port) return STATUS_ERROR; } -#ifndef WIN32 +#if !defined(WIN32) && !defined(__wasi__) errno = 0; /* clear errno before call */ pw = getpwuid(uid); if (!pw) diff --git a/src/backend/libpq/be-fsstubs.c b/src/backend/libpq/be-fsstubs.c index 230c65753205b..4f0d78f1a3c48 100644 --- a/src/backend/libpq/be-fsstubs.c +++ b/src/backend/libpq/be-fsstubs.c @@ -150,8 +150,12 @@ be_lo_close(PG_FUNCTION_ARGS) * *****************************************************************************/ +#if defined(__EMSCRIPTEN__) || defined(__wasi__) +static int +#else int -lo_read(int fd, char *buf, int len) +#endif +lo_read3(int fd, char *buf, int len) { int status; LargeObjectDesc *lobj; @@ -178,8 +182,12 @@ lo_read(int fd, char *buf, int len) return status; } +#if defined(__EMSCRIPTEN__) || defined(__wasi__) +static int +#else int -lo_write(int fd, const char *buf, int len) +#endif +lo_write3(int fd, const char *buf, int len) { int status; LargeObjectDesc *lobj; @@ -190,7 +198,7 @@ lo_write(int fd, const char *buf, int len) errmsg("invalid large-object descriptor: %d", fd))); lobj = cookies[fd]; - /* see comment in lo_read() */ + /* see comment in lo_read3() */ if ((lobj->flags & IFS_WRLOCK) == 0) ereport(ERROR, (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), @@ -365,7 +373,7 @@ be_loread(PG_FUNCTION_ARGS) len = 0; retval = (bytea *) palloc(VARHDRSZ + len); - totalread = lo_read(fd, VARDATA(retval), len); + totalread = lo_read3(fd, VARDATA(retval), len); SET_VARSIZE(retval, totalread + VARHDRSZ); PG_RETURN_BYTEA_P(retval); @@ -382,7 +390,7 @@ be_lowrite(PG_FUNCTION_ARGS) PreventCommandIfReadOnly("lowrite()"); bytestowrite = VARSIZE_ANY_EXHDR(wbuf); - totalwritten = lo_write(fd, VARDATA_ANY(wbuf), bytestowrite); + totalwritten = lo_write3(fd, VARDATA_ANY(wbuf), bytestowrite); PG_RETURN_INT32(totalwritten); } @@ -560,7 +568,7 @@ lo_truncate_internal(int32 fd, int64 len) errmsg("invalid large-object descriptor: %d", fd))); lobj = cookies[fd]; - /* see comment in lo_read() */ + /* see comment in lo_read3() */ if ((lobj->flags & IFS_WRLOCK) == 0) ereport(ERROR, (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), diff --git a/src/backend/libpq/pqcomm.c b/src/backend/libpq/pqcomm.c index 16378e8a73336..90c993f159dc8 100644 --- a/src/backend/libpq/pqcomm.c +++ b/src/backend/libpq/pqcomm.c @@ -123,8 +123,12 @@ static char *PqSendBuffer; static int PqSendBufferSize; /* Size send buffer */ static int PqSendPointer; /* Next index to store a byte in PqSendBuffer */ static int PqSendStart; /* Next index to send a byte in PqSendBuffer */ - +#if defined(__EMSCRIPTEN__) || defined(__wasi__) +static char PqRecvBuffer_static[PQ_RECV_BUFFER_SIZE]; +static char *PqRecvBuffer; +#else static char PqRecvBuffer[PQ_RECV_BUFFER_SIZE]; +#endif static int PqRecvPointer; /* Next index to read a byte from PqRecvBuffer */ static int PqRecvLength; /* End of data available in PqRecvBuffer */ @@ -136,6 +140,7 @@ static bool PqCommReadingMsg; /* in the middle of reading a message */ /* Internal functions */ + static void socket_comm_reset(void); static void socket_close(int code, Datum arg); static void socket_set_nonblocking(bool nonblocking); @@ -147,9 +152,6 @@ static void socket_putmessage_noblock(char msgtype, const char *s, size_t len); static int internal_putbytes(const char *s, size_t len); static int internal_flush(void); -static int Lock_AF_UNIX(const char *unixSocketDir, const char *unixSocketPath); -static int Setup_AF_UNIX(const char *sock_path); - static const PQcommMethods PqCommSocketMethods = { socket_comm_reset, socket_flush, @@ -159,6 +161,10 @@ static const PQcommMethods PqCommSocketMethods = { socket_putmessage_noblock }; +static int Lock_AF_UNIX(const char *unixSocketDir, const char *unixSocketPath); +static int Setup_AF_UNIX(const char *sock_path); + + const PQcommMethods *PqCommMethods = &PqCommSocketMethods; WaitEventSet *FeBeWaitSet; @@ -180,7 +186,7 @@ pq_init(void) PqSendPointer = PqSendStart = PqRecvPointer = PqRecvLength = 0; PqCommBusy = false; PqCommReadingMsg = false; - +#if !defined(__EMSCRIPTEN__) && !defined(__wasi__) /* set up process-exit hook to close the socket */ on_proc_exit(socket_close, 0); @@ -210,7 +216,12 @@ pq_init(void) MyLatch, NULL); AddWaitEventToSet(FeBeWaitSet, WL_POSTMASTER_DEATH, PGINVALID_SOCKET, NULL, NULL); - +#else + PDEBUG("# 220: FIXME: socketfile"); + #pragma message "FIXME: socketfile" + /* because we fill before starting reading message */ + PqRecvBuffer = &PqRecvBuffer_static[0]; +#endif /* * The event positions match the order we added them, but let's sanity * check them to be sure. @@ -632,7 +643,7 @@ Setup_AF_UNIX(const char *sock_path) Assert(Unix_socket_group); if (Unix_socket_group[0] != '\0') { -#ifdef WIN32 +#if defined(WIN32) || defined(__wasi__) elog(WARNING, "configuration item unix_socket_group is not supported on this platform"); #else char *endptr; @@ -1148,6 +1159,20 @@ pq_buffer_has_data(void) * This must be called before any of the pq_get* functions. * -------------------------------- */ +#if defined(I_EMSCRIPTEN) || defined(I_WASI) +EMSCRIPTEN_KEEPALIVE void +pq_recvbuf_fill(FILE* fp, int packetlen) { + fread( PqRecvBuffer, packetlen, 1, fp); + PqRecvPointer = 0; + PqRecvLength = packetlen; +#if PDEBUG + printf("# 1199: pq_recvbuf_fill cma_rsize=%d PqRecvLength=%d buf=%p reply=%p\n", cma_rsize, PqRecvLength, &PqRecvBuffer[0], &PqSendBuffer[0]); +#endif + +} +#endif +extern int cma_rsize; +static char * PqSendBuffer_save; void pq_startmsgread(void) { @@ -1159,7 +1184,29 @@ pq_startmsgread(void) ereport(FATAL, (errcode(ERRCODE_PROTOCOL_VIOLATION), errmsg("terminating connection because protocol synchronization was lost"))); +#if defined(I_EMSCRIPTEN) || defined(I_WASI) + if (!pq_buffer_has_data()) { + if (cma_rsize) { + PqRecvPointer = 0; + PqRecvLength = cma_rsize; + PqRecvBuffer = (char*)0x1; + + PqSendPointer = 0; + PqSendBuffer_save = PqSendBuffer; + PqSendBuffer = 2 + (char*)(cma_rsize); + PqSendBufferSize = (64*1024*1024) - (int)(&PqSendBuffer[0]); + } else { + PqRecvBuffer = &PqRecvBuffer_static[0]; + if (PqSendBuffer_save) + PqSendBuffer=PqSendBuffer_save; + PqSendBufferSize = PQ_SEND_BUFFER_SIZE; + } + } +#if PDEBUG + printf("# 1199: pq_startmsgread cma_rsize=%d PqRecvLength=%d buf=%p reply=%p\n", cma_rsize, PqRecvLength, &PqRecvBuffer[0], &PqSendBuffer[0]); +#endif +#endif PqCommReadingMsg = true; } @@ -1282,8 +1329,65 @@ pq_getmessage(StringInfo s, int maxlen) return 0; } +#if defined(__EMSCRIPTEN__) || defined(__wasi__) + +extern FILE* SOCKET_FILE; +extern int SOCKET_DATA; + +static int +internal_putbytes(const char *s, size_t len) { + if (PqSendPointer >= PqSendBufferSize) + { + puts("# 1329: overflow"); + } + + if (!cma_rsize) { + int wc= fwrite(s, 1, len, SOCKET_FILE); +#if 0 + fprintf(stderr,"# 1334: sockfile %d +%d top=%p %d/%d\n", SOCKET_DATA, wc, PqSendBuffer, PqSendPointer,PqSendBufferSize); +#endif + SOCKET_DATA+=wc; + } else { + size_t amount; + while (len > 0) + { + /* If buffer is full, then flush it out */ + if (PqSendPointer >= PqSendBufferSize) + { + socket_set_nonblocking(false); + if (internal_flush()) + return EOF; + } + amount = PqSendBufferSize - PqSendPointer; + if (amount > len) + amount = len; + memcpy(PqSendBuffer + PqSendPointer, s, amount); + PqSendPointer += amount; + s += amount; + len -= amount; + SOCKET_DATA+=amount; + } +#if 0 + fprintf(stderr,"# 1359: cma buffer wire: %d +%zu top=%p %d/%d\n", SOCKET_DATA, amount, PqSendBuffer, PqSendPointer,PqSendBufferSize); +#endif + } + return 0; +} +static int +socket_flush(void) { + return internal_flush(); +} +static int +internal_flush(void) { + /* no flush for raw wire */ + if (!cma_rsize) { + PqSendStart = PqSendPointer = 0; + } + return 0; +} +#else static int internal_putbytes(const char *s, size_t len) { @@ -1403,7 +1507,7 @@ internal_flush(void) PqSendStart = PqSendPointer = 0; return 0; } - +#endif /* wasm */ /* -------------------------------- * pq_flush_if_writable - flush pending output if writable without blocking * diff --git a/src/backend/main/main.c b/src/backend/main/main.c index ed11e8be7fab7..99abed7841e64 100644 --- a/src/backend/main/main.c +++ b/src/backend/main/main.c @@ -41,20 +41,26 @@ #include "utils/pg_locale.h" #include "utils/ps_status.h" - -const char *progname; static bool reached_main = false; - static void startup_hacks(const char *progname); static void init_locale(const char *categoryname, int category, const char *locale); static void help(const char *progname); static void check_root(const char *progname); - /* * Any Postgres server process begins execution here. */ +#if defined(__wasi__) +static bool +is_web_env() { + return false; +} +#include PG_MAIN_INCLUDE +#elif defined(__EMSCRIPTEN__) +#include PG_MAIN_INCLUDE +#else +const char* progname; int main(int argc, char *argv[]) { @@ -199,6 +205,7 @@ main(int argc, char *argv[]) /* the functions above should not return */ abort(); } +#endif /* wasm */ diff --git a/src/backend/port/posix_sema.c b/src/backend/port/posix_sema.c index 315ed16b197eb..f0eeb1eb5e18f 100644 --- a/src/backend/port/posix_sema.c +++ b/src/backend/port/posix_sema.c @@ -298,10 +298,16 @@ PGSemaphoreReset(PGSemaphore sema) * There's no direct API for this in POSIX, so we have to ratchet the * semaphore down to 0 with repeated trywait's. */ +#if defined(__EMSCRIPTEN__) || defined(__wasi__) + sem_trywait(PG_SEM_REF(sema)); + return; +#else for (;;) { if (sem_trywait(PG_SEM_REF(sema)) < 0) { + + if (errno == EAGAIN || errno == EDEADLK) break; /* got it down to 0 */ if (errno == EINTR) @@ -309,6 +315,7 @@ PGSemaphoreReset(PGSemaphore sema) elog(FATAL, "sem_trywait failed: %m"); } } +#endif } /* diff --git a/src/backend/port/sysv_shmem.c b/src/backend/port/sysv_shmem.c index eaba244bc9c85..4baf4734cc83c 100644 --- a/src/backend/port/sysv_shmem.c +++ b/src/backend/port/sysv_shmem.c @@ -17,8 +17,8 @@ * *------------------------------------------------------------------------- */ +#define PG_SHMEM #include "postgres.h" - #include #include #include @@ -691,12 +691,47 @@ PGShmemHeader * PGSharedMemoryCreate(Size size, PGShmemHeader **shim) { - IpcMemoryKey NextShmemSegID; - void *memAddress; + IpcMemoryKey NextShmemSegID = 0; + void *memAddress = NULL; PGShmemHeader *hdr; struct stat statbuf; Size sysvsize; +/* + puts("@\n@\n@\n@\n@\n@\n PGSharedMemoryCreate @\n@\n@\n@\n@\n@\n"); + + elog(NOTICE, "Init WASM shared memory"); + + hdr = (PGShmemHeader *) malloc(size); + hdr->creatorPID = getpid(); + hdr->magic = PGShmemMagic; + hdr->dsm_control = 0; + + + hdr->device = statbuf.st_dev; + hdr->inode = statbuf.st_ino; + + hdr->totalsize = size; + hdr->freeoffset = MAXALIGN(sizeof(PGShmemHeader)); + *shim = hdr; + + UsedShmemSegAddr = memAddress; + UsedShmemSegID = (unsigned long) NextShmemSegID; + + if (AnonymousShmem == NULL) + return hdr; + memcpy(AnonymousShmem, hdr, sizeof(PGShmemHeader)); + return (PGShmemHeader *) AnonymousShmem; +*/ + + + + + + + + + /* * We use the data directory's ID info (inode and device numbers) to * positively identify shmem segments associated with this data dir, and diff --git a/src/backend/postmaster/checkpointer.c b/src/backend/postmaster/checkpointer.c index f482f6423d1d2..2923ceac6dc90 100644 --- a/src/backend/postmaster/checkpointer.c +++ b/src/backend/postmaster/checkpointer.c @@ -936,7 +936,9 @@ RequestCheckpoint(int flags) /* * If in a standalone backend, just do it ourselves. */ +#if !defined(__wasi__) && !defined(__EMSCRIPTEN__) if (!IsPostmasterEnvironment) +#endif { /* * There's no point in doing slow checkpoints in a standalone backend, diff --git a/src/backend/postmaster/postmaster.c b/src/backend/postmaster/postmaster.c index b42aae41fce83..f98519b18c53e 100644 --- a/src/backend/postmaster/postmaster.c +++ b/src/backend/postmaster/postmaster.c @@ -422,7 +422,8 @@ static void BackendRun(Port *port) pg_attribute_noreturn(); static void ExitPostmaster(int status) pg_attribute_noreturn(); static int ServerLoop(void); static int BackendStartup(Port *port); -static int ProcessStartupPacket(Port *port, bool ssl_done, bool gss_done); +//static +int ProcessStartupPacket(Port *port, bool ssl_done, bool gss_done); static void SendNegotiateProtocolVersion(List *unrecognized_protocol_options); static void processCancelRequest(Port *port, void *pkt); static void report_fork_failure_to_client(Port *port, int errnum); @@ -1533,7 +1534,7 @@ getInstallationPaths(const char *argv0) /* Locate the postgres executable itself */ if (find_my_exec(argv0, my_exec_path) < 0) ereport(FATAL, - (errmsg("%s: could not locate my own executable path", argv0))); + (errmsg("%s:1536: could not locate my own executable path", argv0))); #ifdef EXEC_BACKEND /* Locate executable backend before we change working directory */ @@ -1947,10 +1948,21 @@ ServerLoop(void) * should make no assumption here about the order in which the client may make * requests. */ -static int -ProcessStartupPacket(Port *port, bool ssl_done, bool gss_done) + +static inline uint32 +em_bswap32(uint32 x) +{ + return + ((x << 24) & 0xff000000) | + ((x << 8) & 0x00ff0000) | + ((x >> 8) & 0x0000ff00) | + ((x >> 24) & 0x000000ff); +} + +// static int +int ProcessStartupPacket(Port *port, bool ssl_done, bool gss_done) { - int32 len; + uint32 len; char *buf; ProtocolVersion proto; MemoryContext oldcontext; @@ -1988,15 +2000,15 @@ ProcessStartupPacket(Port *port, bool ssl_done, bool gss_done) return STATUS_ERROR; } - len = pg_ntoh32(len); + //len = pg_ntoh32(len); + len = em_bswap32(len); len -= 4; - - if (len < (int32) sizeof(ProtocolVersion) || + if (len < (uint32) sizeof(ProtocolVersion) || len > MAX_STARTUP_PACKET_LENGTH) { ereport(COMMERROR, (errcode(ERRCODE_PROTOCOL_VIOLATION), - errmsg("invalid length of startup packet"))); + errmsg("# 2000 invalid length of startup packet"))); return STATUS_ERROR; } @@ -2029,7 +2041,7 @@ ProcessStartupPacket(Port *port, bool ssl_done, bool gss_done) { ereport(COMMERROR, (errcode(ERRCODE_PROTOCOL_VIOLATION), - errmsg("invalid length of startup packet"))); + errmsg("# 2033 invalid length of startup packet"))); return STATUS_ERROR; } processCancelRequest(port, buf); diff --git a/src/backend/storage/buffer/bufmgr.c b/src/backend/storage/buffer/bufmgr.c index e066a3f888fb7..f1c8b40efb5b6 100644 --- a/src/backend/storage/buffer/bufmgr.c +++ b/src/backend/storage/buffer/bufmgr.c @@ -1134,16 +1134,16 @@ ReadBuffer_common(SMgrRelation smgr, char relpersistence, ForkNumber forkNum, { ereport(WARNING, (errcode(ERRCODE_DATA_CORRUPTED), - errmsg("invalid page in block %u of relation %s; zeroing out page", + errmsg("1137: invalid page in block %u of relation %s; zeroing out page", blockNum, relpath(smgr->smgr_rlocator, forkNum)))); MemSet((char *) bufBlock, 0, BLCKSZ); } else - ereport(ERROR, + ereport(WARNING, (errcode(ERRCODE_DATA_CORRUPTED), - errmsg("invalid page in block %u of relation %s", - blockNum, + errmsg("1145(FATAL): invalid page in block %u fork=%d p=%p sz=%d of relation %s", + blockNum,forkNum,bufBlock, BLCKSZ, relpath(smgr->smgr_rlocator, forkNum)))); } } diff --git a/src/backend/storage/file/fd.c b/src/backend/storage/file/fd.c index 16b3e8f90584b..042de5f1b1340 100644 --- a/src/backend/storage/file/fd.c +++ b/src/backend/storage/file/fd.c @@ -69,7 +69,7 @@ * *------------------------------------------------------------------------- */ - +#define PG_FD #include "postgres.h" #include @@ -478,6 +478,11 @@ pg_fdatasync(int fd) void pg_flush_data(int fd, off_t offset, off_t nbytes) { +#if defined(__EMSCRIPTEN__) || defined(__wasi__) + //int res = sync_file_range(fd, offset, nbytes, SYNC_FILE_RANGE_WAIT_BEFORE | SYNC_FILE_RANGE_WRITE | SYNC_FILE_RANGE_WAIT_AFTER); + (void)fsync(fd); + // fprintf(stderr, "# pg_flush_data(int fd=%d, off_t offset=%lld, off_t nbytes=%lld res=%d\n", fd,offset,nbytes, res); +#else /* * Right now file flushing is primarily used to avoid making later * fsync()/fdatasync() calls have less impact. Thus don't trigger flushes @@ -648,6 +653,7 @@ pg_flush_data(int fd, off_t offset, off_t nbytes) return; } #endif +#endif /* wasm */ } /* @@ -660,7 +666,7 @@ pg_ftruncate(int fd, off_t length) retry: ret = ftruncate(fd, length); - +printf("# 670 pg_ftruncate(int fd=%d, off_t length=%lld)=%d\n" __FILE__, fd, length, ret); if (ret == -1 && errno == EINTR) goto retry; @@ -692,7 +698,7 @@ pg_truncate(const char *path, off_t length) retry: ret = truncate(path, length); - +printf("# 670 pg_truncate(path=%s, off_t length=%lld)=%d\n" __FILE__, path, length, ret); if (ret == -1 && errno == EINTR) goto retry; #endif @@ -2619,7 +2625,7 @@ OpenTransientFilePerm(const char *fileName, int fileFlags, mode_t fileMode) return -1; /* failure */ } - +#if !defined(__EMSCRIPTEN__) && !defined(__wasi__) /* * Routines that want to initiate a pipe stream should use OpenPipeStream * rather than plain popen(). This lets fd.c deal with freeing FDs if @@ -2679,7 +2685,7 @@ OpenPipeStream(const char *command, const char *mode) return NULL; } - +#endif /* * Free an AllocateDesc of any type. * diff --git a/src/backend/storage/ipc/ipc.c b/src/backend/storage/ipc/ipc.c index 6591b5d6a8b64..9a6abc6b8cb36 100644 --- a/src/backend/storage/ipc/ipc.c +++ b/src/backend/storage/ipc/ipc.c @@ -103,6 +103,48 @@ static int on_proc_exit_index, void proc_exit(int code) { +#if defined(__EMSCRIPTEN__) || defined(__wasi__) + if (code==66) { + fprintf(stderr,"# 108:fake shutdown\n"); + proc_exit_inprogress = true; + InterruptPending = false; + ProcDiePending = false; + QueryCancelPending = false; + InterruptHoldoffCount = 1; + CritSectionCount = 0; + + error_context_stack = NULL; + debug_query_string = NULL; + + shmem_exit_inprogress = true; + int save_before_shmem_exit_index = before_shmem_exit_index; + while (--before_shmem_exit_index >= 0) { + if (before_shmem_exit_index!=4) { + printf("# skipped shmem_exit_index=%d/%d\n", before_shmem_exit_index, save_before_shmem_exit_index); + continue; + } else + printf("# before_shmem_exit_index=%d/%d\n", before_shmem_exit_index, save_before_shmem_exit_index); + before_shmem_exit_list[before_shmem_exit_index].function(code, before_shmem_exit_list[before_shmem_exit_index].arg); + } + before_shmem_exit_index = save_before_shmem_exit_index; + puts("# dsm_backend_shutdown ?"); + // dsm_backend_shutdown(); + shmem_exit_inprogress = false; + /* + + int save_on_proc_exit_index = on_proc_exit_index; + while (--on_proc_exit_index >= 0) { + printf("# on_proc_exit_list=%d/%d\n", on_proc_exit_list, save_on_proc_exit_index); + on_proc_exit_list[on_proc_exit_index].function(code, on_proc_exit_list[on_proc_exit_index].arg); + } + on_proc_exit_index = save_on_proc_exit_index; + */ + } else { + proc_exit_inprogress = true; + fprintf(stderr,"# proc_exit(%d) ignored at 118:%s\n",code, __FILE__); + } + return; +#endif /* not safe if forked by system(), etc. */ if (MyProcPid != (int) getpid()) elog(PANIC, "proc_exit() called in child process"); @@ -152,7 +194,6 @@ proc_exit(int code) #endif elog(DEBUG3, "exit(%d)", code); - exit(code); } @@ -228,7 +269,7 @@ void shmem_exit(int code) { shmem_exit_inprogress = true; - +if (code!=66){ /* * Call before_shmem_exit callbacks. * @@ -276,7 +317,7 @@ shmem_exit(int code) on_shmem_exit_list[on_shmem_exit_index].function(code, on_shmem_exit_list[on_shmem_exit_index].arg); on_shmem_exit_index = 0; - +} shmem_exit_inprogress = false; } @@ -364,6 +405,17 @@ before_shmem_exit(pg_on_exit_callback function, Datum arg) void on_shmem_exit(pg_on_exit_callback function, Datum arg) { +#if defined(__wasi__) || defined(__EMSCRIPTEN__) + if (!atexit_callback_setup) { + PDEBUG("# 410:" __FILE__ " on_shmem_exit(pg_on_exit_callback function, Datum arg) FIRST CALL"); + if (on_shmem_exit_index >= MAX_ON_EXITS) { + PDEBUG("# 412:" __FILE__ " on_shmem_exit(pg_on_exit_callback function, Datum arg) OVERFLOW"); + } + } else { + PDEBUG("# 415:" __FILE__ " on_shmem_exit(pg_on_exit_callback function, Datum arg) STUB"); + return; + } +#endif if (on_shmem_exit_index >= MAX_ON_EXITS) ereport(FATAL, (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED), diff --git a/src/backend/storage/ipc/latch.c b/src/backend/storage/ipc/latch.c index e572677c7034c..799152a1eb006 100644 --- a/src/backend/storage/ipc/latch.c +++ b/src/backend/storage/ipc/latch.c @@ -206,6 +206,7 @@ static inline int WaitEventSetWaitBlock(WaitEventSet *set, int cur_timeout, void InitializeLatchSupport(void) { +#if !defined(__wasi__) #if defined(WAIT_USE_SELF_PIPE) int pipefd[2]; @@ -315,6 +316,7 @@ InitializeLatchSupport(void) /* Ignore SIGURG, because we'll receive it via kqueue. */ pqsignal(SIGURG, SIG_IGN); #endif +#endif /* __wasi__ */ } void diff --git a/src/backend/storage/ipc/procsignal.c b/src/backend/storage/ipc/procsignal.c index c85cb5cc18dbb..d1c0dfdc81420 100644 --- a/src/backend/storage/ipc/procsignal.c +++ b/src/backend/storage/ipc/procsignal.c @@ -420,6 +420,9 @@ WaitForProcSignalBarrier(uint64 generation) (errmsg("still waiting for backend with PID %d to accept ProcSignalBarrier", (int) slot->pss_pid))); oldval = pg_atomic_read_u64(&slot->pss_barrierGeneration); +#if defined(__EMSCRIPTEN__) || defined(__wasi__) + break; +#endif } ConditionVariableCancelSleep(); } diff --git a/src/backend/storage/ipc/signalfuncs.c b/src/backend/storage/ipc/signalfuncs.c index b595c2d691233..b29ce76edbd8b 100644 --- a/src/backend/storage/ipc/signalfuncs.c +++ b/src/backend/storage/ipc/signalfuncs.c @@ -98,7 +98,12 @@ pg_signal_backend(int pid, int sig) */ /* If we have setsid(), signal the backend's whole process group */ -#ifdef HAVE_SETSID +#if defined(__EMSCRIPTEN__) || defined(__wasi__) +# if PGDEBUG + printf("# 103: FIXME: kill(pid=%d, sig=%d) ", pid, sig); +# endif + if (0) +#elif defined(HAVE_SETSID) if (kill(-pid, sig)) #else if (kill(pid, sig)) @@ -106,7 +111,7 @@ pg_signal_backend(int pid, int sig) { /* Again, just a warning to allow loops */ ereport(WARNING, - (errmsg("could not send signal to process %d: %m", pid))); + (errmsg("# 109: could not send signal to process %d: %m", pid))); return SIGNAL_BACKEND_ERROR; } return SIGNAL_BACKEND_SUCCESS; diff --git a/src/backend/storage/lmgr/proc.c b/src/backend/storage/lmgr/proc.c index e9e445bb2166d..9b1d2b61d377e 100644 --- a/src/backend/storage/lmgr/proc.c +++ b/src/backend/storage/lmgr/proc.c @@ -306,7 +306,11 @@ InitProcess(void) elog(PANIC, "proc header uninitialized"); if (MyProc != NULL) - elog(ERROR, "you already exist"); +#if defined(__wasi__) || defined(__EMSCRIPTEN__) + elog(WARNING, "# 309: you already exist"); +#else + elog(ERROR, "# 309: you already exist"); +#endif /* Decide which list should supply our PGPROC. */ if (IsAnyAutoVacuumProcess()) @@ -519,7 +523,7 @@ InitAuxiliaryProcess(void) elog(PANIC, "proc header uninitialized"); if (MyProc != NULL) - elog(ERROR, "you already exist"); + elog(ERROR, "# 522: you already exist"); /* * We use the ProcStructLock to protect assignment and releasing of diff --git a/src/backend/tcop/postgres.c b/src/backend/tcop/postgres.c index e7d486ca2fc49..26034cf01159e 100644 --- a/src/backend/tcop/postgres.c +++ b/src/backend/tcop/postgres.c @@ -4052,7 +4052,11 @@ process_postgres_switches(int argc, char *argv[], GucContext ctx, #endif } - +#if defined(__EMSCRIPTEN__) || defined(__wasi__) +#define PG_MAIN +#include PG_MAIN_INCLUDE +#undef PG_MAIN +#else /* * PostgresSingleUserMain * Entry point for single user mode. argc/argv are the command line @@ -4948,7 +4952,7 @@ PostgresMain(const char *dbname, const char *username) } } /* end of input-reading loop */ } - +#endif /* wasm */ /* * Throw an error if we're a WAL sender process. * @@ -5067,7 +5071,7 @@ ShowUsage(const char *title) (long) user.tv_usec, (long) sys.tv_sec, (long) sys.tv_usec); -#ifndef WIN32 +#if !defined(WIN32) && !defined(__wasi__) /* * The following rusage fields are not defined by POSIX, but they're diff --git a/src/backend/tcop/utility.c b/src/backend/tcop/utility.c index e3ccf6c7f7e8b..a95d933b42ac6 100644 --- a/src/backend/tcop/utility.c +++ b/src/backend/tcop/utility.c @@ -811,7 +811,7 @@ standard_ProcessUtility(PlannedStmt *pstmt, ListenStmt *stmt = (ListenStmt *) parsetree; CheckRestrictedOperation("LISTEN"); - +#if !defined(__EMSCRIPTEN__) && !defined(__wasi__) /* * We don't allow LISTEN in background processes, as there is * no mechanism for them to collect NOTIFY messages, so they'd @@ -827,7 +827,7 @@ standard_ProcessUtility(PlannedStmt *pstmt, /* translator: %s is name of a SQL command, eg LISTEN */ errmsg("cannot execute %s within a background process", "LISTEN"))); - +#endif Async_Listen(stmt->conditionname); } break; diff --git a/src/backend/utils/adt/ruleutils.c b/src/backend/utils/adt/ruleutils.c index f01cc2521c800..f15a4c4b3014b 100644 --- a/src/backend/utils/adt/ruleutils.c +++ b/src/backend/utils/adt/ruleutils.c @@ -321,8 +321,9 @@ static SPIPlanPtr plan_getviewrule = NULL; static const char *query_getviewrule = "SELECT * FROM pg_catalog.pg_rewrite WHERE ev_class = $1 AND rulename = $2"; /* GUC parameters */ +#if !defined(__EMSCRIPTEN__) && !defined(__wasi__) bool quote_all_identifiers = false; - +#endif /* ---------- * Local functions diff --git a/src/backend/utils/error/elog.c b/src/backend/utils/error/elog.c index 7112fb00069a4..c1d67896411cf 100644 --- a/src/backend/utils/error/elog.c +++ b/src/backend/utils/error/elog.c @@ -349,12 +349,16 @@ errstart(int elevel, const char *domain) ErrorData *edata; bool output_to_server; bool output_to_client = false; +#if defined(__EMSCRIPTEN__) || defined(__wasi__) +# warning "FIXME: error levels" +#else int i; /* * Check some cases in which we want to promote an error into a more * severe error. None of this logic applies for non-error messages. */ + if (elevel >= ERROR) { /* @@ -395,7 +399,7 @@ errstart(int elevel, const char *domain) for (i = 0; i <= errordata_stack_depth; i++) elevel = Max(elevel, errordata[i].elevel); } - +#endif /* * Now decide whether we need to process this report at all; if it's * warning or less and not enabled for logging, just return false without @@ -540,7 +544,13 @@ errfinish(const char *filename, int lineno, const char *funcname) */ recursion_depth--; +#if 0 //defined(__EMSCRIPTEN__) || defined(__wasi__) + fprintf(stderr, "# 547: PG_RE_THROW(ERROR : %d) ignored\n", recursion_depth); + trap(); +#else + fprintf(stderr, "# 549: PG_RE_THROW(ERROR : %d)\n", recursion_depth); PG_RE_THROW(); +#endif } /* Emit the message to the right places */ @@ -588,7 +598,11 @@ errfinish(const char *filename, int lineno, const char *funcname) * FATAL termination. The postmaster may or may not consider this * worthy of panic, depending on which subprocess returns it. */ +#if defined(__EMSCRIPTEN__) || defined(__wasi__) + puts("# 599: proc_exit(FATAL) ignored"); +#else proc_exit(1); +#endif } if (elevel >= PANIC) @@ -698,6 +712,7 @@ errsave_finish(struct Node *context, const char *filename, int lineno, */ if (edata->elevel >= ERROR) { +puts("#712"); errfinish(filename, lineno, funcname); pg_unreachable(); } diff --git a/src/backend/utils/fmgr/dfmgr.c b/src/backend/utils/fmgr/dfmgr.c index b85d52c913c3a..867cb669595bf 100644 --- a/src/backend/utils/fmgr/dfmgr.c +++ b/src/backend/utils/fmgr/dfmgr.c @@ -121,12 +121,15 @@ load_external_function(const char *filename, const char *funcname, /* Look up the function within the library. */ retval = dlsym(lib_handle, funcname); - +#if !defined(__wasi__) if (retval == NULL && signalNotFound) ereport(ERROR, (errcode(ERRCODE_UNDEFINED_FUNCTION), errmsg("could not find function \"%s\" in file \"%s\"", funcname, fullname))); +#else + fprintf(stderr, "could not find function \"%s\" in file \"%s\" rv=%p snf=%b\n", funcname, fullname, retval, signalNotFound); +#endif pfree(fullname); return retval; diff --git a/src/backend/utils/init/miscinit.c b/src/backend/utils/init/miscinit.c index b0ca61a8d663b..a42df2020ee9c 100644 --- a/src/backend/utils/init/miscinit.c +++ b/src/backend/utils/init/miscinit.c @@ -143,7 +143,7 @@ InitPostmasterChild(void) * children, but for consistency we make all postmaster child processes do * this. */ -#ifdef HAVE_SETSID +#if defined(HAVE_SETSID) && !defined(__wasi__) if (setsid() < 0) elog(FATAL, "setsid() failed: %m"); #endif @@ -209,7 +209,7 @@ InitStandaloneProcess(const char *argv0) if (my_exec_path[0] == '\0') { if (find_my_exec(argv0, my_exec_path) < 0) - elog(FATAL, "%s: could not locate my own executable path", + elog(WARNING, "%s:212: could not locate my own executable path", argv0); } @@ -368,7 +368,7 @@ checkDataDir(void) * * XXX can we safely enable this check on Windows? */ -#if !defined(WIN32) && !defined(__CYGWIN__) +#if !defined(WIN32) && !defined(__CYGWIN__) && !defined(__EMSCRIPTEN__) && !defined(__wasi__) if (stat_buf.st_uid != geteuid()) ereport(FATAL, (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), @@ -388,7 +388,7 @@ checkDataDir(void) * be proper support for Unix-y file permissions. Need to think of a * reasonable check to apply on Windows. */ -#if !defined(WIN32) && !defined(__CYGWIN__) +#if !defined(WIN32) && !defined(__CYGWIN__) && !defined(__EMSCRIPTEN__) && !defined(__wasi__) if (stat_buf.st_mode & PG_MODE_MASK_GROUP) ereport(FATAL, (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), @@ -409,7 +409,7 @@ checkDataDir(void) * Suppress when on Windows, because there may not be proper support for * Unix-y file permissions. */ -#if !defined(WIN32) && !defined(__CYGWIN__) +#if !defined(WIN32) && !defined(__CYGWIN__) && !defined(__EMSCRIPTEN__) && !defined(__wasi__) SetDataDirectoryCreatePerm(stat_buf.st_mode); umask(pg_mode_mask); @@ -1223,7 +1223,13 @@ CreateLockFile(const char *filename, bool amPostmaster, * Think not to make the file protection weaker than 0600/0640. See * comments below. */ + +#if defined(__wasi__) +printf("# 1228: CreateLockFile(%s) w+ (forced)\n", filename); + fd = fileno(fopen(filename, "w+")); +#else fd = open(filename, O_RDWR | O_CREAT | O_EXCL, pg_file_create_mode); +#endif if (fd >= 0) break; /* Success; exit the retry loop */ diff --git a/src/backend/utils/init/postinit.c b/src/backend/utils/init/postinit.c index df4d15a50fbbb..41828cda2c711 100644 --- a/src/backend/utils/init/postinit.c +++ b/src/backend/utils/init/postinit.c @@ -13,6 +13,7 @@ * *------------------------------------------------------------------------- */ +#define PG_POSTINIT #include "postgres.h" #include @@ -714,6 +715,7 @@ BaseInit(void) * Be very careful with the order of calls in the InitPostgres function. * -------------------------------- */ + void InitPostgres(const char *in_dbname, Oid dboid, const char *username, Oid useroid, @@ -883,8 +885,18 @@ InitPostgres(const char *in_dbname, Oid dboid, } else if (!IsUnderPostmaster) { +#if defined(__EMSCRIPTEN__) || defined(__wasi__) +if (!strcmp( username , WASM_USERNAME )) { +#endif InitializeSessionUserIdStandalone(); am_superuser = true; +#if defined(__EMSCRIPTEN__) || defined(__wasi__) +} else { + //puts("# 894: switching session id"); + InitializeSessionUserId(username, InvalidOid); + am_superuser = superuser(); +} +#endif if (!ThereIsAtLeastOneRole()) ereport(WARNING, (errcode(ERRCODE_UNDEFINED_OBJECT), @@ -1239,6 +1251,32 @@ InitPostgres(const char *in_dbname, Oid dboid, CommitTransactionCommand(); } +/* ========================================================================*/ +/* +void +ReInitPostgres(const char *in_dbname, Oid dboid, + const char *username, Oid useroid, + bool load_session_libraries, + bool override_allow_connections, + char *out_dbname) +{ + puts("ReInitPostgres:Begin"); + InitPostgres(in_dbname, dboid, username, useroid, load_session_libraries, override_allow_connections, out_dbname); + puts("ReInitPostgres:End"); +} +*/ +/* ========================================================================*/ + + + + + + + + + + + /* * Process any command-line switches and any additional GUC variable * settings passed in the startup packet. @@ -1345,14 +1383,15 @@ process_settings(Oid databaseid, Oid roleid) static void ShutdownPostgres(int code, Datum arg) { +puts("# 1348: " __FILE__); /* Make sure we've killed any active transaction */ AbortOutOfAnyTransaction(); - /* * User locks are not released by transaction end, so be sure to release * them explicitly. */ LockReleaseAll(USER_LOCKMETHOD, true); +puts("# 1356: " __FILE__); } diff --git a/src/backend/utils/misc/timeout.c b/src/backend/utils/misc/timeout.c index 8ab755d363d47..f7c88fac469d7 100644 --- a/src/backend/utils/misc/timeout.c +++ b/src/backend/utils/misc/timeout.c @@ -110,10 +110,20 @@ find_active_timeout(TimeoutId id) * Insert specified timeout reason into the list of active timeouts * at the given index. */ + +bool insert_timeout_warned = false; static void insert_timeout(TimeoutId id, int index) { int i; +#if defined(__EMSCRIPTEN__) || defined(__wasi__) + if (!insert_timeout_warned) //(index<0) + { + insert_timeout_warned = true; + fprintf(stderr, "# 117(FATAL): insert_timeout(TimeoutId id=%d, int index=%d): " __FILE__ "\n", id, index); + } + return; +#endif if (index < 0 || index > num_active_timeouts) elog(FATAL, "timeout index %d out of range 0..%d", index, @@ -128,6 +138,7 @@ insert_timeout(TimeoutId id, int index) active_timeouts[index] = &all_timeouts[id]; num_active_timeouts++; + } /* @@ -209,6 +220,10 @@ enable_timeout(TimeoutId id, TimestampTz now, TimestampTz fin_time, static void schedule_alarm(TimestampTz now) { +#if defined(__wasi__) + puts("# 224: schedule_alarm(TimestampTz now)"); + (void)signal_due_at; +#else if (num_active_timeouts > 0) { struct itimerval timeval; @@ -347,6 +362,7 @@ schedule_alarm(TimestampTz now) elog(FATAL, "could not enable SIGALRM timer: %m"); } } +#endif } diff --git a/src/bin/initdb/initdb.c b/src/bin/initdb/initdb.c index 458dc1137a319..1110c80bbde07 100644 --- a/src/bin/initdb/initdb.c +++ b/src/bin/initdb/initdb.c @@ -45,7 +45,7 @@ * *------------------------------------------------------------------------- */ - +#define PG_INITDB #include "postgres_fe.h" #include @@ -81,7 +81,6 @@ #include "mb/pg_wchar.h" #include "miscadmin.h" - /* Ideally this would be in a .h file, but it hardly seems worth the trouble */ extern const char *select_default_timezone(const char *share_path); @@ -168,7 +167,11 @@ static int wal_segment_size_mb; /* internal vars */ +#if !defined(PG_MAIN) static const char *progname; +#else +# define dynamic_shared_memory_type idb_dynamic_shared_memory_type +#endif static int encodingid; static char *bki_file; static char *hba_file; @@ -752,6 +755,7 @@ cleanup_directories_atexit(void) static char * get_id(void) { +#if !defined(__EMSCRIPTEN__) && !defined(__wasi__) const char *username; #ifndef WIN32 @@ -762,10 +766,12 @@ get_id(void) exit(1); } #endif - username = get_user_name_or_exit(progname); return pg_strdup(username); +#else + return pg_strdup(getenv("PGUSER")); +#endif /* wasm */ } static char * @@ -1011,6 +1017,9 @@ set_null_conf(void) static const char * choose_dsm_implementation(void) { +#if defined(__wasi__) || defined(__EMSCRIPTEN__) + return "posix"; +#endif #if defined(HAVE_SHM_OPEN) && !defined(__sun__) int ntries = 10; pg_prng_state prng_state; @@ -1615,9 +1624,11 @@ get_su_pwd(void) if (ferror(pwf)) pg_fatal("could not read password from file \"%s\": %m", pwfilename); - else - pg_fatal("password file \"%s\" is empty", + else { + printf("password file \"%s\" is empty\n", pwfilename); + pwd1 = strdup("password"); + } } fclose(pwf); @@ -2562,8 +2573,13 @@ setup_bin_paths(const char *argv0) strlcpy(full_path, progname, sizeof(full_path)); if (ret == -1) +#if defined(__EMSCRIPTEN__) || defined(__wasi__) + printf("# WARNING: program \"%s\" is needed by %s but was not found in the same directory as \"%s\"\n", + "postgres", progname, full_path); +#else pg_fatal("program \"%s\" is needed by %s but was not found in the same directory as \"%s\"", "postgres", progname, full_path); +#endif // wasm else pg_fatal("program \"%s\" was found by \"%s\" but was not the same version as %s", "postgres", full_path, progname); @@ -2617,22 +2633,22 @@ setup_locale_encoding(void) lc_numeric, lc_time); } - +puts("# 2651"); if (!encoding) - { + { puts("# 2653"); int ctype_enc; ctype_enc = pg_get_encoding_from_locale(lc_ctype, true); - +puts("# 2657"); /* * If ctype_enc=SQL_ASCII, it's compatible with any encoding. ICU does * not support SQL_ASCII, so select UTF-8 instead. */ if (locale_provider == COLLPROVIDER_ICU && ctype_enc == PG_SQL_ASCII) ctype_enc = PG_UTF8; - +puts("# 2664"); if (ctype_enc == -1) - { + { puts("# 2666"); /* Couldn't recognize the locale's codeset */ pg_log_error("could not find suitable encoding for locale \"%s\"", lc_ctype); @@ -2641,7 +2657,7 @@ setup_locale_encoding(void) exit(1); } else if (!pg_valid_server_encoding_id(ctype_enc)) - { + { puts("# 2675"); /* * We recognized it, but it's not a legal server encoding. On * Windows, UTF-8 works with any locale, so we can fall back to @@ -2664,15 +2680,17 @@ setup_locale_encoding(void) #endif } else - { + { puts("# 2698"); encodingid = ctype_enc; printf(_("The default database encoding has accordingly been set to \"%s\".\n"), pg_encoding_to_char(encodingid)); } } - else + else { + puts("# 2705"); encodingid = get_encoding_id(encoding); - + } + puts("# 2706"); if (!check_locale_encoding(lc_ctype, encodingid) || !check_locale_encoding(lc_collate, encodingid)) exit(1); /* check_locale_encoding printed the error */ @@ -2991,7 +3009,11 @@ initialize_data_directory(void) /* Select suitable configuration settings */ set_null_conf(); +#if !defined(__EMSCRIPTEN__) && !defined(__wasi__) test_config_settings(); +#else + dynamic_shared_memory_type = choose_dsm_implementation(); +#endif // wasm /* Now create all the text config files */ setup_config(); @@ -3056,9 +3078,30 @@ initialize_data_directory(void) check_ok(); } +#if defined(PG_INITDB_MAIN) +extern void MemoryContextInit(void); + +// FIXME: get PGDATA from env. +char *argv[] = { + WASM_PREFIX "/bin/initdb", +// "--no-clean", + "--wal-segsize=1", + "-g", + "-E", "UTF8", "--locale=C.UTF-8", "--locale-provider=libc", + "-U", WASM_USERNAME, "--pwfile=" WASM_PREFIX "/password", + "--pgdata=" WASM_PREFIX "/base", + NULL +}; + +int argc = sizeof(argv) / sizeof(char*) - 1; + +EMSCRIPTEN_KEEPALIVE int +pg_initdb_main() +#else int main(int argc, char *argv[]) +#endif { static struct option long_options[] = { {"pgdata", required_argument, NULL, 'D'}, @@ -3118,10 +3161,16 @@ main(int argc, char *argv[]) */ setvbuf(stdout, NULL, PG_IOLBF, 0); +#if defined(PG_INITDB_MAIN) + progname = get_progname(argv[0]); +// printf("calling pg_initdb_main for %s\n", progname); + MemoryContextInit(); + pg_logging_init(progname); +#else pg_logging_init(argv[0]); progname = get_progname(argv[0]); +#endif set_pglocale_pgservice(argv[0], PG_TEXTDOMAIN("initdb")); - if (argc > 1) { if (strcmp(argv[1], "--help") == 0 || strcmp(argv[1], "-?") == 0) @@ -3377,11 +3426,11 @@ main(int argc, char *argv[]) set_info_version(); setup_data_file_paths(); - +puts("# 3442"); setup_locale_encoding(); - +puts("# 3444"); setup_text_search(); - +puts("# 3446"); printf("\n"); if (data_checksums) @@ -3393,9 +3442,11 @@ main(int argc, char *argv[]) get_su_pwd(); printf("\n"); - +puts("# 3458"); initialize_data_directory(); - +#if defined(PG_INITDB_MAIN) + puts("# 3461: TODO: fsync_pgdata ?"); +#else if (do_sync) { fputs(_("syncing data to disk ... "), stdout); @@ -3413,7 +3464,7 @@ main(int argc, char *argv[]) pg_log_warning_hint("You can change this by editing pg_hba.conf or using the option -A, or " "--auth-local and --auth-host, the next time you run initdb."); } - +puts("# 3480"); if (!noinstructions) { /* @@ -3448,8 +3499,7 @@ main(int argc, char *argv[]) destroyPQExpBuffer(start_db_cmd); } - - +#endif success = true; return 0; } diff --git a/src/bin/pg_config/pg_config.c b/src/bin/pg_config/pg_config.c index 70cab817ddd68..5458bb918f520 100644 --- a/src/bin/pg_config/pg_config.c +++ b/src/bin/pg_config/pg_config.c @@ -152,7 +152,9 @@ main(int argc, char **argv) if (find_my_exec(argv[0], my_exec_path) < 0) { fprintf(stderr, _("%s: could not find own program executable\n"), progname); +#if !defined(__EMSCRIPTEN__) && !defined(__wasi__) exit(1); +#endif } configdata = get_configdata(my_exec_path, &configdata_len); @@ -162,6 +164,7 @@ main(int argc, char **argv) for (i = 0; i < configdata_len; i++) printf("%s = %s\n", configdata[i].name, configdata[i].setting); exit(0); + } /* otherwise print requested items */ diff --git a/src/bin/pg_ctl/pg_ctl.c b/src/bin/pg_ctl/pg_ctl.c index fc160b0594ce8..03716840f8c0f 100644 --- a/src/bin/pg_ctl/pg_ctl.c +++ b/src/bin/pg_ctl/pg_ctl.c @@ -155,6 +155,16 @@ static bool wait_for_postmaster_stop(void); static bool wait_for_postmaster_promote(void); static bool postmaster_is_alive(pid_t pid); +#if defined(__wasi__) +#if defined(HAVE_SETSID) +#undef HAVE_SETSID +#endif + +#if defined(HAVE_GETRLIMIT) +#undef HAVE_GETRLIMIT +#endif +#endif /* __wasi__ */ + #if defined(HAVE_GETRLIMIT) static void unlimit_core_size(void); #endif @@ -493,9 +503,9 @@ start_postmaster(void) else cmd = psprintf("exec \"%s\" %s%s < \"%s\" 2>&1", exec_path, pgdata_opt, post_opts, DEVNULL); - +#if !defined(__wasi__) (void) execl("/bin/sh", "/bin/sh", "-c", cmd, (char *) NULL); - +#endif /* exec failed */ write_stderr(_("%s: could not start server: %s\n"), progname, strerror(errno)); diff --git a/src/bin/pg_dump/parallel.c b/src/bin/pg_dump/parallel.c index da0723ad38587..51ff2ab674c2e 100644 --- a/src/bin/pg_dump/parallel.c +++ b/src/bin/pg_dump/parallel.c @@ -49,7 +49,7 @@ * The pstate->te[] entry for each worker is valid when it's in WRKR_WORKING * state, and must be NULL in other states. */ - +#define PG_DUMP_PARALLEL #include "postgres_fe.h" #ifndef WIN32 @@ -445,6 +445,7 @@ ShutdownWorkersHard(ParallelState *pstate) static void WaitForTerminatingWorkers(ParallelState *pstate) { +#if !defined(__wasi__) while (!HasEveryWorkerTerminated(pstate)) { ParallelSlot *slot = NULL; @@ -504,6 +505,7 @@ WaitForTerminatingWorkers(ParallelState *pstate) slot->workerStatus = WRKR_TERMINATED; pstate->te[j] = NULL; } +#endif /* __wasi__ */ } diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c index 1694ff55f89b5..b9f206a8757e2 100644 --- a/src/bin/pg_dump/pg_dump.c +++ b/src/bin/pg_dump/pg_dump.c @@ -30,6 +30,13 @@ *------------------------------------------------------------------------- */ #include "postgres_fe.h" +#if !defined(__EMSCRIPTEN__) && !defined(__wasi__) +#ifdef quote_all_identifiers +#undef quote_all_identifiers +#endif +#define fe_utils_quote_all_identifiers quote_all_identifiers +static bool quote_all_identifiers; +#endif #include #include @@ -328,8 +335,11 @@ static void set_restrict_relation_kind(Archive *AH, const char *value); static void setupDumpWorker(Archive *AH); static TableInfo *getRootTableInfo(const TableInfo *tbinfo); static bool forcePartitionRootLoad(const TableInfo *tbinfo); - - +#if defined(__wasi__) +__attribute__((export_name("_setup"))) void setup() { + puts("WASM_SYM_EXPORTED"); +} +#endif int main(int argc, char **argv) { @@ -411,7 +421,7 @@ main(int argc, char **argv) {"lock-wait-timeout", required_argument, NULL, 2}, {"no-table-access-method", no_argument, &dopt.outputNoTableAm, 1}, {"no-tablespaces", no_argument, &dopt.outputNoTablespaces, 1}, - {"quote-all-identifiers", no_argument, "e_all_identifiers, 1}, + {"quote-all-identifiers", no_argument, &fe_utils_quote_all_identifiers, true}, {"load-via-partition-root", no_argument, &dopt.load_via_partition_root, 1}, {"role", required_argument, NULL, 3}, {"section", required_argument, NULL, 5}, @@ -435,7 +445,9 @@ main(int argc, char **argv) {NULL, 0, NULL, 0} }; - +#if defined(__wasi__) +chdir("/"); +#endif pg_logging_init(argv[0]); pg_logging_set_level(PG_LOG_WARNING); set_pglocale_pgservice(argv[0], PG_TEXTDOMAIN("pg_dump")); @@ -803,9 +815,12 @@ main(int argc, char **argv) * Open the database using the Archiver, so it knows about it. Errors mean * death. */ +puts("# 813 : " __FILE__); + //setup(); ConnectDatabase(fout, &dopt.cparams, false); +puts("# 815 : " __FILE__); setup_connection(fout, dumpencoding, dumpsnapshot, use_role); - +puts("# 817 : " __FILE__); /* * On hot standbys, never try to dump unlogged table data, since it will * just throw an error. @@ -1152,9 +1167,10 @@ setup_connection(Archive *AH, const char *dumpencoding, const char *dumpsnapshot, char *use_role) { DumpOptions *dopt = AH->dopt; +puts("# 1164 : get_connection : "__FILE__); PGconn *conn = GetConnection(AH); const char *std_strings; - +puts("# 1164 : setup_connection"); PQclear(ExecuteSqlQueryForSingleRow(AH, ALWAYS_SECURE_SEARCH_PATH_SQL)); /* @@ -1239,8 +1255,8 @@ setup_connection(Archive *AH, const char *dumpencoding, /* * Quote all identifiers, if requested. */ - if (quote_all_identifiers) - ExecuteSqlStatement(AH, "SET quote_all_identifiers = true"); + if (fe_utils_quote_all_identifiers) + ExecuteSqlStatement(AH, "SET fe_utils_quote_all_identifiers = true"); /* * Adjust row-security mode, if supported. diff --git a/src/bin/pg_dump/pg_dumpall.c b/src/bin/pg_dump/pg_dumpall.c index 591fcb07c5b16..86b8cda0c0742 100644 --- a/src/bin/pg_dump/pg_dumpall.c +++ b/src/bin/pg_dump/pg_dumpall.c @@ -14,7 +14,13 @@ */ #include "postgres_fe.h" - +#if !defined(__EMSCRIPTEN__) && !defined(__wasi__) +#ifdef quote_all_identifiers +#undef quote_all_identifiers +#endif +#define fe_utils_quote_all_identifiers quote_all_identifiers +static bool quote_all_identifiers; +#endif #include #include @@ -163,7 +169,7 @@ main(int argc, char *argv[]) {"lock-wait-timeout", required_argument, NULL, 2}, {"no-table-access-method", no_argument, &no_table_access_method, 1}, {"no-tablespaces", no_argument, &no_tablespaces, 1}, - {"quote-all-identifiers", no_argument, "e_all_identifiers, 1}, + {"quote-all-identifiers", no_argument, &fe_utils_quote_all_identifiers, true}, {"load-via-partition-root", no_argument, &load_via_partition_root, 1}, {"role", required_argument, NULL, 3}, {"use-set-session-authorization", no_argument, &use_setsessauth, 1}, @@ -434,7 +440,7 @@ main(int argc, char *argv[]) appendPQExpBufferStr(pgdumpopts, " --no-table-access-method"); if (no_tablespaces) appendPQExpBufferStr(pgdumpopts, " --no-tablespaces"); - if (quote_all_identifiers) + if (fe_utils_quote_all_identifiers) appendPQExpBufferStr(pgdumpopts, " --quote-all-identifiers"); if (load_via_partition_root) appendPQExpBufferStr(pgdumpopts, " --load-via-partition-root"); @@ -534,8 +540,8 @@ main(int argc, char *argv[]) } /* Force quoting of all identifiers if requested. */ - if (quote_all_identifiers) - executeCommand(conn, "SET quote_all_identifiers = true"); + if (fe_utils_quote_all_identifiers) + executeCommand(conn, "SET fe_utils_quote_all_identifiers = true"); fprintf(OPF, "--\n-- PostgreSQL database cluster dump\n--\n\n"); if (verbose) diff --git a/src/bin/pg_resetwal/pg_resetwal.c b/src/bin/pg_resetwal/pg_resetwal.c index ca57713f63bb4..278a0f5af4bcd 100644 --- a/src/bin/pg_resetwal/pg_resetwal.c +++ b/src/bin/pg_resetwal/pg_resetwal.c @@ -330,7 +330,7 @@ main(int argc, char *argv[]) * -- any other user won't have sufficient permissions to modify files in * the data directory. */ -#ifndef WIN32 +#if !defined(WIN32) && !defined(__EMSCRIPTEN__) && !defined(__wasi__) if (geteuid() == 0) { pg_log_error("cannot be executed by \"root\""); diff --git a/src/bin/pg_upgrade/parallel.c b/src/bin/pg_upgrade/parallel.c index eaf0f0afefa93..93346c1d000cc 100644 --- a/src/bin/pg_upgrade/parallel.c +++ b/src/bin/pg_upgrade/parallel.c @@ -277,6 +277,7 @@ win32_transfer_all_new_dbs(transfer_thread_arg *args) bool reap_child(bool wait_for_child) { +#if !defined(__wasi__) #ifndef WIN32 int work_status; pid_t child; @@ -336,6 +337,6 @@ reap_child(bool wait_for_child) /* do this after job has been removed */ parallel_jobs--; - +#endif /* __wasi__ */ return true; } diff --git a/src/bin/psql/command.c b/src/bin/psql/command.c index 6733f008fd55e..07cc083ec4475 100644 --- a/src/bin/psql/command.c +++ b/src/bin/psql/command.c @@ -5146,6 +5146,10 @@ do_shell(const char *command) static bool do_watch(PQExpBuffer query_buf, double sleep, int iter) { +#if defined(__wasi__) + pg_log_error("#5150 wasi: could not set timer"); + return false; +#else long sleep_ms = (long) (sleep * 1000); printQueryOpt myopt = pset.popt; const char *strftime_fmt; @@ -5384,6 +5388,7 @@ do_watch(PQExpBuffer query_buf, double sleep, int iter) pg_free(title); return (res >= 0); +#endif /* __wasi__ */ } /* diff --git a/src/common/exec.c b/src/common/exec.c index f209b934df781..1b7073ecca304 100644 --- a/src/common/exec.c +++ b/src/common/exec.c @@ -22,7 +22,7 @@ * This should be harmless everywhere else. */ #define _DARWIN_BETTER_REALPATH - +#define PG_EXEC #ifndef FRONTEND #include "postgres.h" #else diff --git a/src/common/logging.c b/src/common/logging.c index dab718b482ec3..eaed3139fc106 100644 --- a/src/common/logging.c +++ b/src/common/logging.c @@ -19,8 +19,9 @@ #include "common/logging.h" enum pg_log_level __pg_log_level; - +#if !defined(PG_MAIN) static const char *progname; +#endif static int log_flags; static void (*log_pre_callback) (void); diff --git a/src/fe_utils/print.c b/src/fe_utils/print.c index 7af1ccb6b562b..9a3cc047d2a5b 100644 --- a/src/fe_utils/print.c +++ b/src/fe_utils/print.c @@ -15,6 +15,7 @@ * *------------------------------------------------------------------------- */ +#define FE_UTILS_PRINT #include "postgres_fe.h" #include diff --git a/src/fe_utils/string_utils.c b/src/fe_utils/string_utils.c index 0429a72bfe245..e7c573ec763cb 100644 --- a/src/fe_utils/string_utils.c +++ b/src/fe_utils/string_utils.c @@ -23,7 +23,9 @@ static PQExpBuffer defaultGetLocalPQExpBuffer(void); /* Globals exported by this file */ -int quote_all_identifiers = 0; +bool fe_utils_quote_all_identifiers = false; + + PQExpBuffer (*getLocalPQExpBuffer) (void) = defaultGetLocalPQExpBuffer; @@ -72,7 +74,7 @@ fmtId(const char *rawid) * These checks need to match the identifier production in scan.l. Don't * use islower() etc. */ - if (quote_all_identifiers) + if (fe_utils_quote_all_identifiers) need_quotes = true; /* slightly different rules for first character */ else if (!((rawid[0] >= 'a' && rawid[0] <= 'z') || rawid[0] == '_')) diff --git a/src/include/bootstrap/bootstrap.h b/src/include/bootstrap/bootstrap.h index 2c28a52ce71fb..b7533b0f9fefe 100644 --- a/src/include/bootstrap/bootstrap.h +++ b/src/include/bootstrap/bootstrap.h @@ -31,8 +31,11 @@ extern PGDLLIMPORT Relation boot_reldesc; extern PGDLLIMPORT Form_pg_attribute attrtypes[MAXATTR]; extern PGDLLIMPORT int numattr; - +#if defined(__EMSCRIPTEN__) || defined(__wasi__) + int BootstrapModeMain(int argc, char *argv[], bool check_only); +#else extern void BootstrapModeMain(int argc, char *argv[], bool check_only) pg_attribute_noreturn(); +#endif extern void closerel(char *relname); extern void boot_openrel(char *relname); diff --git a/src/include/common/file_utils.h b/src/include/common/file_utils.h index b7efa1226d6ef..bdc0fcd388811 100644 --- a/src/include/common/file_utils.h +++ b/src/include/common/file_utils.h @@ -27,10 +27,14 @@ typedef enum PGFileType struct iovec; /* avoid including port/pg_iovec.h here */ #ifdef FRONTEND +#if !defined(fsync_fname) extern int fsync_fname(const char *fname, bool isdir); +#endif extern void fsync_pgdata(const char *pg_data, int serverVersion); extern void fsync_dir_recurse(const char *dir); +#if !defined(durable_rename) extern int durable_rename(const char *oldfile, const char *newfile); +#endif extern int fsync_parent_path(const char *fname); #endif diff --git a/src/include/common/logging.h b/src/include/common/logging.h index 99e888af93d39..c0ab0f59fdffc 100644 --- a/src/include/common/logging.h +++ b/src/include/common/logging.h @@ -91,7 +91,6 @@ void pg_logging_set_level(enum pg_log_level new_level); void pg_logging_increase_verbosity(void); void pg_logging_set_pre_callback(void (*cb) (void)); void pg_logging_set_locus_callback(void (*cb) (const char **filename, uint64 *lineno)); - void pg_log_generic(enum pg_log_level level, enum pg_log_part part, const char *pg_restrict fmt,...) pg_attribute_printf(3, 4); diff --git a/src/include/fe_utils/string_utils.h b/src/include/fe_utils/string_utils.h index e10c9090754cc..ca153cb033ed2 100644 --- a/src/include/fe_utils/string_utils.h +++ b/src/include/fe_utils/string_utils.h @@ -16,11 +16,16 @@ #ifndef STRING_UTILS_H #define STRING_UTILS_H +#if !defined(__wasi__) #include "libpq-fe.h" #include "pqexpbuffer.h" +#else +#include "../interfaces/libpq/libpq-fe.h" +#include "../interfaces/libpq/pqexpbuffer.h" +#endif /* Global variables controlling behavior of fmtId() and fmtQualifiedId() */ -extern PGDLLIMPORT int quote_all_identifiers; +extern PGDLLIMPORT bool quote_all_identifiers; extern PQExpBuffer (*getLocalPQExpBuffer) (void); /* Functions */ diff --git a/src/include/fmgr.h b/src/include/fmgr.h index b120f5e7fefac..bf33fc48873e2 100644 --- a/src/include/fmgr.h +++ b/src/include/fmgr.h @@ -505,7 +505,10 @@ PG_MAGIC_FUNCTION_NAME(void) \ return &Pg_magic_data; \ } \ extern int no_such_variable - +#if defined(__wasi__) && !defined(__EMSCRIPTEN__) +#undef PG_MODULE_MAGIC +#define PG_MODULE_MAGIC +#endif /*------------------------------------------------------------------------- * Support routines and macros for callers of fmgr-compatible functions diff --git a/src/include/libpq/be-fsstubs.h b/src/include/libpq/be-fsstubs.h index e70a6cb56cc5e..6e8b9f2e65ca6 100644 --- a/src/include/libpq/be-fsstubs.h +++ b/src/include/libpq/be-fsstubs.h @@ -19,8 +19,10 @@ * Probably these should have had the underscore-free names, * but too late now... */ -extern int lo_read(int fd, char *buf, int len); -extern int lo_write(int fd, const char *buf, int len); +#if !defined(__EMSCRIPTEN__) && !defined(__wasi__) +extern int lo_read3(int fd, char *buf, int len); +extern int lo_write3(int fd, const char *buf, int len); +#endif /* * Cleanup LOs at xact commit/abort diff --git a/src/include/port/emscripten.h b/src/include/port/emscripten.h new file mode 100644 index 0000000000000..0d1cf63abfe1b --- /dev/null +++ b/src/include/port/emscripten.h @@ -0,0 +1,50 @@ +/* src/include/port/emscripten.h */ + +#ifndef I_EMSCRIPTEN +#define I_EMSCRIPTEN + +#if !defined(__cplusplus) +#include +#endif + +#include "/tmp/pglite/include/wasm_common.h" + + +#define BOOT_END_MARK "build indices" +#define FD_BUFFER_MAX 16384 + + +/* + * and now popen will return predefined slot from a file list + * as file handle in initdb.c + */ +#if defined(PG_INITDB) && !defined(PG_MAIN) +#define popen(command, mode) pg_popen(command, mode) +#include // FILE+fprintf +extern FILE* IDB_PIPE_FP; +extern FILE* SOCKET_FILE; +extern int SOCKET_DATA; +extern int IDB_STAGE; +FILE *pg_popen(const char *command, const char *type) { + if (IDB_STAGE>1) { + fprintf(stderr,"# popen[%s]\n", command); + return stderr; + } + + if (!IDB_STAGE) { + fprintf(stderr,"# popen[%s] (BOOT)\n", command); + IDB_PIPE_FP = fopen( IDB_PIPE_BOOT, "w"); + IDB_STAGE = 1; + } else { + fprintf(stderr,"# popen[%s] (SINGLE)\n", command); + IDB_PIPE_FP = fopen( IDB_PIPE_SINGLE, "w"); + IDB_STAGE = 2; + } + + return IDB_PIPE_FP; + +} +#endif // PG_INITDB + + +#endif // I_EMSCRIPTEN diff --git a/src/include/port/pg_pthread.h b/src/include/port/pg_pthread.h index d102ce9d6f33e..6922eb423b945 100644 --- a/src/include/port/pg_pthread.h +++ b/src/include/port/pg_pthread.h @@ -12,7 +12,9 @@ #ifndef PG_PTHREAD_H #define PG_PTHREAD_H - +#if defined(__wasi__) +#define PYDK +#endif /* __wasi__ */ #include #ifndef HAVE_PTHREAD_BARRIER_WAIT @@ -20,7 +22,7 @@ #ifndef PTHREAD_BARRIER_SERIAL_THREAD #define PTHREAD_BARRIER_SERIAL_THREAD (-1) #endif - +#if !defined(__wasi__) typedef struct pg_pthread_barrier { bool sense; /* we only need a one bit phase */ @@ -29,10 +31,12 @@ typedef struct pg_pthread_barrier pthread_mutex_t mutex; pthread_cond_t cond; } pthread_barrier_t; - extern int pthread_barrier_init(pthread_barrier_t *barrier, const void *attr, int count); +#else + extern int pthread_barrier_init(pthread_barrier_t *__restrict, const pthread_barrierattr_t *__restrict, unsigned); +#endif extern int pthread_barrier_wait(pthread_barrier_t *barrier); extern int pthread_barrier_destroy(pthread_barrier_t *barrier); diff --git a/src/include/port/wasi.h b/src/include/port/wasi.h new file mode 100644 index 0000000000000..0ab7d640d64eb --- /dev/null +++ b/src/include/port/wasi.h @@ -0,0 +1,259 @@ +#ifndef I_WASI +#define I_WASI + +#if defined(HAVE_SETSID) +#undef HAVE_SETSID +#endif + +#if defined(HAVE_GETRLIMIT) +#undef HAVE_GETRLIMIT +#endif + + +#define PLATFORM_DEFAULT_SYNC_METHOD SYNC_METHOD_FDATASYNC + +#define EMSCRIPTEN_KEEPALIVE __attribute__((used)) +#define __declspec( dllimport ) __attribute__((used)) + +#define em_callback_func void +#define emscripten_set_main_loop(...) +#define emscripten_force_exit(...) +#define EM_JS(...) + +#include "/tmp/pglite/include/wasm_common.h" + + +static pid_t +fork(void) { + puts("# 31: fork -1"); + return -1; +} + + +// TODO: socket here +// ================================================================== + +#include +static int +listen(int sockfd, int backlog) { + return 0; +} + +static struct group *_Nullable +getgrnam(const char *_Nullable name) { + return NULL; +} + +static int +getsockname(int sockfd, struct sockaddr *restrict addr, socklen_t *restrict addrlen) { + return -1; +} + +static int +getaddrinfo(const char *restrict node, + const char *restrict service, + void *restrict hints, + void **restrict res) { + puts("# 60: getaddrinfo"); + return -1; +} +static void +freeaddrinfo(void *res) { + puts("# 65: freeaddrinfo"); +} + +extern ssize_t recvfrom_bc(int socket, void *buffer, size_t length, int flags, void *address, socklen_t *address_len); + +// ======== signal ======================== +#define SA_RESTART 4 +#define SIG_SETMASK 2 + +#define SIG_BLOCK 0 +#define SIG_UNBLOCK 1 + +/* A signal handler. */ +typedef void (*handler_t) (int signal); +typedef unsigned char sigset_t; +typedef void (*__sighandler_t) (int); + +struct sigaction { + __sighandler_t sa_handler; + unsigned long sa_flags; +#ifdef SA_RESTORER + __sigrestore_t sa_restorer; +#endif + sigset_t sa_mask; +}; +extern int sigemptyset(sigset_t *set); +extern int sigaction(int signum, const struct sigaction *act, struct sigaction *oldact); +extern int sigdelset (sigset_t *set, int sig); +extern int sigfillset (sigset_t *set); +extern int sigprocmask (int operation, const sigset_t *set, sigset_t *old_set); +extern int sigaddset (sigset_t *set, int sig); + +// STUBS +extern int pthread_sigmask(int how, const sigset_t *set, sigset_t *oldset); +extern int sigismember(const sigset_t *set, int signum); +extern int sigpending(sigset_t *set); +extern int sigwait(const sigset_t *restrict set, int *restrict sig); + +// ==================================================== +// unistd +extern unsigned int alarm(unsigned int seconds); + +// ==================================================== + + +#include + +static int +sigsetjmp(sigjmp_buf env, int savesigs) { +// puts("# 120: sigsetjmp"); + return 0; +} + +static void +siglongjmp(sigjmp_buf env, int val) { + puts("# 120: siglongjmp"); +} + + + +// WIP : + +#include +static uid_t +getuid(void) { + return 1000; +} + +static int +dup(int fd) { + puts("# 128: dup"); + return fd; +} +static int +dup2(int old, int new) { + puts("# 140: dup2"); + return -1; +} +static int +pipe(int fd[2]) { + puts("# 145: pipe"); + abort(); + return -1; +} + +#include +#define RLIMIT_NOFILE 7 +#define RLIMIT_STACK 3 +#define RLIM_INFINITY ((unsigned long int)(~0UL)) + +struct rlimit { + unsigned long rlim_cur; + unsigned long rlim_max; +}; +static int +getrlimit(int resource, struct rlimit *rlim) { + return -1; +} + + +static const char *gai_strerror_msg = "# 165: gai_strerror_msg"; +static const char * +gai_strerror(int errcode) { + return gai_strerror_msg; +} + + +static int +getrusage(int who, struct rusage *usage) { + return -1; +} + +// WIP: semaphores here +// ================================================================== +#include + +static int +semctl(int semid, int semnum, int cmd, ...) { + return 0; // -1; +} + +static int +semget(key_t key, int nsems, int semflg) { +#if 0 // PGDEBUG + printf("# 213: semget(key_t key = %d, int nsems=%d, int semflg=%d)\n", key, nsems, semflg); +#endif + return 1; +} + +static int +semop(int semid, struct sembuf *sops, size_t nsops) { + return 0; // -1; +} + + + +#include +#if defined(PYDK) +extern int shm_open(const char *name, int oflag, mode_t mode); +extern int shm_unlink(const char *name); +#else +static int +shm_open(const char *name, int oflag, mode_t mode) { + char tmpnam[128]; + int fd; + snprintf(tmpnam, 128, "/tmp%s", name); + fd=fileno(fopen(tmpnam, "w+")); + fprintf(stderr, "# 212: shm_open(%s) => %d\n", tmpnam, fd); + return fd; +} + +static int +shm_unlink(const char *name) { + char tmpnam[128]; + snprintf(tmpnam, 128, "/tmp%s", name); + fprintf(stderr, "# 220: shm_unlink(%s) STUB\n", tmpnam); + return remove(tmpnam); // -1 +} + +#endif + +// initdb chmod +#if defined(__wasi__) + #define chmod(...) 0 +#endif + +// popen +extern int pg_pclose(FILE *stream); +extern FILE * pg_popen(const char *command, const char *type); +#define popen(command, mode) pg_popen(command, mode) +#define pclose(stream) pg_pclose(stream) + + + +#define system(command) system_wasi(command) +extern int system_wasi(const char *command); + + + +// time.h + +static void +tzset(void) { + puts("# 241: tzset(void) STUB"); +} + +#if defined(PG_INITDB) || defined(FE_UTILS_PRINT) || defined(PG_DUMP_PARALLEL) +static void +__SIG_IGN(int param) { +} +#endif + + +extern void sock_flush(); + + +#endif // I_WASI + diff --git a/src/include/port/wasm_common.h b/src/include/port/wasm_common.h new file mode 100644 index 0000000000000..dd94ea92ea90e --- /dev/null +++ b/src/include/port/wasm_common.h @@ -0,0 +1,309 @@ +#define WAIT_USE_POLL 1 + +#define HAVE_LINUX_EIDRM_BUG +/* + * Set the default wal_sync_method to fdatasync. With recent Linux versions, + * xlogdefs.h's normal rules will prefer open_datasync, which (a) doesn't + * perform better and (b) causes outright failures on ext4 data=journal + * filesystems, because those don't support O_DIRECT. + */ +#define PLATFORM_DEFAULT_WAL_SYNC_METHOD WAL_SYNC_METHOD_FDATASYNC + +// force the name used with --single +#if !defined(WASM_USERNAME) +#define WASM_USERNAME "postgres" +#endif + +// MEMFS ok files +#define IDB_PIPE_BOOT "/tmp/initdb.boot.txt" +#define IDB_PIPE_SINGLE "/tmp/initdb.single.txt" + +/* --------------- how to configure those when installed ? ---------------- */ + +// socket emulation via file, need to go in PGDATA for nodefs mount in web mode +#define PGS_ILOCK "/tmp/pglite/base/.s.PGSQL.5432.lock.in" +#define PGS_IN "/tmp/pglite/base/.s.PGSQL.5432.in" +#define PGS_OLOCK "/tmp/pglite/base/.s.PGSQL.5432.lock.out" +#define PGS_OUT "/tmp/pglite/base/.s.PGSQL.5432.out" + + +#define PG_DEBUG_HEADER + + +#if defined(PREFIX) +#define em_xstr(s) em_str(s) +#define em_str(s) #s +# define WASM_PREFIX em_xstr(PREFIX) +# define PG_MAIN_INCLUDE em_xstr(PATCH_MAIN) +# define PG_PLUGIN_INCLUDE em_xstr(PATCH_PLUGIN +# undef PG_DEBUG_HEADER +# define PG_DEBUG_HEADER em_xstr(PATCH_PG_DEBUG) +#else +# define WASM_PREFIX "/pgdata" +# define PG_MAIN_INCLUDE "/pgdata/pg_main.c" +# define PG_PLUGIN_INCLUDE "/pgdata/pg_plugin.h" +#endif + +// #include PG_DEBUG_HEADER +#include "/tmp/pglite/include/pg_debug.h" + + + + + +// #define COPY_INTERNAL +#define COPY_OFF +#define PGDLLIMPORT +#define PG_FORCE_DISABLE_INLINE + + + +#define WASM_PGOPTS \ + "-c", "log_checkpoints=false",\ + "-c", "search_path=pg_catalog",\ + "-c", "exit_on_error=true",\ + "-c", "ignore_invalid_pages=on",\ + "-c", "temp_buffers=8MB",\ + "-c", "work_mem=4MB",\ + "-c", "fsync=on",\ + "-c", "synchronous_commit=on",\ + "-c", "wal_buffers=4MB",\ + "-c", "min_wal_size=80MB",\ + "-c", "shared_buffers=128MB" + +// we want client and server in the same lib for now. +#if defined(PG_INITDB) && defined(PG_MAIN) +extern const char *progname; +#endif + +// exported in ./src/fe_utils/string_utils.c +#include +extern PGDLLIMPORT bool fe_utils_quote_all_identifiers; + +extern int pg_char_to_encoding_private(const char *name); +extern const char *pg_encoding_to_char_private(int encoding); +extern int pg_valid_server_encoding_id_private(int encoding); + +#if defined(pg_char_to_encoding) +#undef pg_char_to_encoding +#endif +#define pg_char_to_encoding(encoding) pg_char_to_encoding_private(encoding) + +#if defined(pg_encoding_to_char) +#undef pg_encoding_to_char +#endif +#define pg_encoding_to_char(encoding) pg_encoding_to_char_private(encoding) + +#if defined(pg_valid_server_encoding_id) +#undef pg_valid_server_encoding_id +#endif +#define pg_valid_server_encoding_id(encoding) pg_valid_server_encoding_id_private(encoding) + + +/* + * 'proc_exit' is a wasi system call, so change its name everywhere. + */ + +#define proc_exit(arg) pg_proc_exit(arg) + + +/* + * popen is routed via pg_popen to stderr or a IDB_PIPE_* file + * link a pclose replacement when we are in exec.c ( PG_EXEC defined ) + */ + + +#if defined(PG_EXEC) +#define pclose(stream) pg_pclose(stream) +#include // FILE + +EMSCRIPTEN_KEEPALIVE FILE* +SOCKET_FILE = NULL; + +EMSCRIPTEN_KEEPALIVE int +SOCKET_DATA = 0; + +EMSCRIPTEN_KEEPALIVE FILE* +IDB_PIPE_FP = NULL; + +EMSCRIPTEN_KEEPALIVE int +IDB_STAGE = 0; + +int pg_pclose(FILE *stream); + +int pg_pclose(FILE *stream) { + if (IDB_STAGE==1) + fprintf(stderr,"# pg_pclose(%s) 133:" __FILE__ "\n" , IDB_PIPE_BOOT); + if (IDB_STAGE==2) + fprintf(stderr,"# pg_pclose(%s) 135:" __FILE__ "\n" , IDB_PIPE_SINGLE); + + if (IDB_PIPE_FP) { + fflush(IDB_PIPE_FP); + fclose(IDB_PIPE_FP); + IDB_PIPE_FP = NULL; + } + return 0; +} + + +#endif // PG_EXEC + + + +/* + * OpenPipeStream : another kind of pipe open in fd.c + * known to try "locale -a" from collationcmds.c when in initdb. + * + */ +#if defined(PG_FD) +#include // strlen +#include // access+F_OK +#include // FILE+fprintf + +FILE *wasm_OpenPipeStream(const char *command, const char *mode); +FILE * +wasm_OpenPipeStream(const char *command, const char *mode) { + + FILE *result = NULL; + const char *prefix = getenv("PGSYSCONFDIR"); + const char *locale = "/locale"; + char *localefile = malloc( strlen(prefix) + strlen(locale) + 1 ); + localefile = strcat(prefix,locale); +#if PGDEBUG + fprintf(stderr, "# 232:%s: OpenPipeStream(command=%s, mode=%s)\n#\tredirected to %s\n", __FILE__, command, mode, localefile); +#endif + if (localefile) { + if (access(localefile, F_OK) != 0) { + FILE *fakeloc = fopen(localefile, "w"); + { + const char* encoding = getenv("PGCLIENTENCODING"); + fprintf(fakeloc, "C\nC.%s\nPOSIX\n%s\n", encoding, encoding); + } + if (fakeloc) + fclose(fakeloc); + } + result = fopen(localefile, "r"); + free(localefile); + } + + return result; +} + +#else +# define OpenPipeStream(cmd, mode) wasm_OpenPipeStream(cmd, mode) +#endif + + + + + + +/* + * handle pg_shmem.c special case + */ + +#if defined(PG_SHMEM) +#include // print +#include // malloc +#include // SC_ +#include +#include + +/* + * Shared memory control operation. + */ + +//extern int shmctl (int __shmid, int __cmd, struct shmid_ds *__buf); + +int +shmctl (int __shmid, int __cmd, struct shmid_ds *__buf) { + printf("FIXME: int shmctl (int __shmid=%d, int __cmd=%d, struct shmid_ds *__buf=%p)\n", __shmid, __cmd, __buf); + return 0; +} + + +volatile void *FAKE_SHM ; +volatile key_t FAKE_KEY = 0; + +/* Get shared memory segment. */ +// extern int shmget (key_t __key, size_t __size, int __shmflg); +int +shmget (key_t __key, size_t __size, int __shmflg) { + printf("# FIXING: int shmget (key_t __key=%d, size_t __size=%zu, int __shmflg=%d) pagesize default=%d\n", __key, __size, __shmflg, getpagesize()); + if (!FAKE_KEY) { + FAKE_SHM = malloc(__size); + FAKE_KEY = 666; + return (int)FAKE_KEY; + } else { + printf("# ERROR: int shmget (key_t __key=%d, size_t __size=%zu, int __shmflg=%d)\n", __key, __size, __shmflg); + //abort(); + return (int)FAKE_KEY; + } + return -1; +} + +/* Attach shared memory segment. */ +// extern void *shmat (int __shmid, const void *__shmaddr, int __shmflg); +void *shmat (int __shmid, const void *__shmaddr, int __shmflg) { + printf("# FIXING: void *shmat (int __shmid=%d, const void *__shmaddr=%p, int __shmflg=%d)\n", __shmid, __shmaddr, __shmflg); + if (__shmid==666) { + return (void *)FAKE_SHM; + } else { + printf("# ERROR: void *shmat (int __shmid=%d, const void *__shmaddr=%p, int __shmflg=%d)\n", __shmid, __shmaddr, __shmflg); + abort(); + } + return NULL; +} + +/* Detach shared memory segment. */ +// extern int shmdt (const void *__shmaddr); +int +shmdt (const void *__shmaddr) { + puts("# FIXME: int shmdt (const void *__shmaddr)"); + return 0; +} + + +/* +typedef struct { + pthread_mutex_t mutex; + pthread_cond_t cond; + int count; +} sem_t; + +int sem_init(sem_t *sem, int pshared, unsigned int value){ + if(pshared > 1) { + return -1; + } + sem->count = value; + pthread_mutex_init(&sem->mutex, NULL); + pthread_cond_init(&sem->cond, NULL); + return 0; +} + +int sem_destroy(sem_t *sem){ + pthread_mutex_destroy(&sem->mutex); + pthread_cond_destroy(&sem->cond); + return 0; +} + +int sem_wait(sem_t *sem){ + pthread_mutex_lock(&sem->mutex); + while(sem->count == 0){ + pthread_cond_wait(&sem->cond, &sem->mutex); + } + sem->count--; + pthread_mutex_unlock(&sem->mutex); + return 0; +} + +int sem_post(sem_t *sem){ + pthread_mutex_lock(&sem->mutex); + sem->count++; + pthread_cond_signal(&sem->cond); + pthread_mutex_unlock(&sem->mutex); + return 0; +} +*/ + +#endif // PG_SHMEM diff --git a/src/include/storage/dsm_impl.h b/src/include/storage/dsm_impl.h index daf07bd19cd68..2e2c4d0a7d0b0 100644 --- a/src/include/storage/dsm_impl.h +++ b/src/include/storage/dsm_impl.h @@ -23,20 +23,49 @@ * Determine which dynamic shared memory implementations will be supported * on this platform, and which one will be the default. */ +#if 0 // defined(__wasi__) || defined(__EMSCRIPTEN__) + #define DEFAULT_DYNAMIC_SHARED_MEMORY_TYPE DSM_IMPL_SYSV + #define USE_DSM_SYSV + extern PGDLLIMPORT int dynamic_shared_memory_type; + extern PGDLLIMPORT int min_dynamic_shared_memory; + #define PG_DYNSHMEM_DIR "/tmp/pglite" + #define PG_DYNSHMEM_MMAP_FILE_PREFIX "mmap." + +#elif 0 // defined(__wasi__) || defined(__EMSCRIPTEN__) + #define DEFAULT_DYNAMIC_SHARED_MEMORY_TYPE DSM_IMPL_MMAP + #define USE_DSM_MMAP + extern PGDLLIMPORT int dynamic_shared_memory_type; + extern PGDLLIMPORT int min_dynamic_shared_memory; + #define PG_DYNSHMEM_DIR "/tmp/pglite" + #define PG_DYNSHMEM_MMAP_FILE_PREFIX "mmap." + +#elif defined(__wasi__) || defined(__EMSCRIPTEN__) + #define DEFAULT_DYNAMIC_SHARED_MEMORY_TYPE DSM_IMPL_POSIX + #define USE_DSM_POSIX + extern PGDLLIMPORT int dynamic_shared_memory_type; + extern PGDLLIMPORT int min_dynamic_shared_memory; + #define PG_DYNSHMEM_DIR "/tmp/pglite" + #define PG_DYNSHMEM_MMAP_FILE_PREFIX "mmap." + +#else + #ifdef WIN32 -#define USE_DSM_WINDOWS -#define DEFAULT_DYNAMIC_SHARED_MEMORY_TYPE DSM_IMPL_WINDOWS +# define USE_DSM_WINDOWS +# define DEFAULT_DYNAMIC_SHARED_MEMORY_TYPE DSM_IMPL_WINDOWS #else -#ifdef HAVE_SHM_OPEN -#define USE_DSM_POSIX -#define DEFAULT_DYNAMIC_SHARED_MEMORY_TYPE DSM_IMPL_POSIX -#endif -#define USE_DSM_SYSV -#ifndef DEFAULT_DYNAMIC_SHARED_MEMORY_TYPE -#define DEFAULT_DYNAMIC_SHARED_MEMORY_TYPE DSM_IMPL_SYSV -#endif +# ifdef HAVE_SHM_OPEN +# define USE_DSM_POSIX +# define DEFAULT_DYNAMIC_SHARED_MEMORY_TYPE DSM_IMPL_POSIX +# endif +# define USE_DSM_SYSV +# ifndef DEFAULT_DYNAMIC_SHARED_MEMORY_TYPE +# define DEFAULT_DYNAMIC_SHARED_MEMORY_TYPE DSM_IMPL_SYSV +# endif #define USE_DSM_MMAP -#endif + +#endif /* defined(__wasi__) || defined(__EMSCRIPTEN__) */ + + /* GUC. */ extern PGDLLIMPORT int dynamic_shared_memory_type; @@ -50,7 +79,7 @@ extern PGDLLIMPORT int min_dynamic_shared_memory; */ #define PG_DYNSHMEM_DIR "pg_dynshmem" #define PG_DYNSHMEM_MMAP_FILE_PREFIX "mmap." - +#endif /* A "name" for a dynamic shared memory segment. */ typedef uint32 dsm_handle; diff --git a/src/include/storage/fd.h b/src/include/storage/fd.h index 6791a406fc114..10fb748842e18 100644 --- a/src/include/storage/fd.h +++ b/src/include/storage/fd.h @@ -188,13 +188,17 @@ extern int pg_fsync_writethrough(int fd); extern int pg_fdatasync(int fd); extern void pg_flush_data(int fd, off_t offset, off_t nbytes); extern int pg_truncate(const char *path, off_t length); -extern void fsync_fname(const char *fname, bool isdir); +extern void fd_fsync_fname(const char *fname, bool isdir); extern int fsync_fname_ext(const char *fname, bool isdir, bool ignore_perm, int elevel); -extern int durable_rename(const char *oldfile, const char *newfile, int elevel); +extern int fd_durable_rename(const char *oldfile, const char *newfile, int elevel); extern int durable_unlink(const char *fname, int elevel); extern void SyncDataDirectory(void); extern int data_sync_elevel(int elevel); +#define durable_rename(oldfile, newfile, elevel) fd_durable_rename(oldfile, newfile, elevel) +#define fsync_fname(fname, isdir) fd_fsync_fname(fname, isdir) + + /* Filename components */ #define PG_TEMP_FILES_DIR "pgsql_tmp" #define PG_TEMP_FILE_PREFIX "pgsql_tmp" diff --git a/src/include/storage/ipc.h b/src/include/storage/ipc.h index 888c08b30675e..8e1c31ad9cade 100644 --- a/src/include/storage/ipc.h +++ b/src/include/storage/ipc.h @@ -64,8 +64,11 @@ typedef void (*shmem_startup_hook_type) (void); /* ipc.c */ extern PGDLLIMPORT bool proc_exit_inprogress; extern PGDLLIMPORT bool shmem_exit_inprogress; - +#if defined(__EMSCRIPTEN__) || defined(__wasi__) +extern void pg_proc_exit(int code); +#else extern void proc_exit(int code) pg_attribute_noreturn(); +#endif extern void shmem_exit(int code); extern void on_proc_exit(pg_on_exit_callback function, Datum arg); extern void on_shmem_exit(pg_on_exit_callback function, Datum arg); diff --git a/src/include/utils/elog.h b/src/include/utils/elog.h index 0292e88b4f293..217045d518072 100644 --- a/src/include/utils/elog.h +++ b/src/include/utils/elog.h @@ -137,6 +137,19 @@ struct Node; * prevents gcc from making the unreachability deduction at optlevel -O0. *---------- */ +#if defined(__EMSCRIPTEN__) || defined(__wasi__) +#define ereport_domain(elevel, domain, ...) \ + do { \ + pg_prevent_errno_in_scope(); \ + if (__builtin_constant_p(elevel) && (elevel) >= ERROR ? \ + errstart_cold(elevel, domain) : \ + errstart(elevel, domain)) \ + __VA_ARGS__, errfinish(__FILE__, __LINE__, __func__); \ + if (__builtin_constant_p(elevel) && (elevel) >= ERROR) \ + { puts("# 149:pg_unreachable():" __FILE__); pg_unreachable(); } \ + } while(0) + +#else #ifdef HAVE__BUILTIN_CONSTANT_P #define ereport_domain(elevel, domain, ...) \ do { \ @@ -159,7 +172,7 @@ struct Node; pg_unreachable(); \ } while(0) #endif /* HAVE__BUILTIN_CONSTANT_P */ - +#endif #define ereport(elevel, ...) \ ereport_domain(elevel, TEXTDOMAIN, __VA_ARGS__) diff --git a/src/include/utils/palloc.h b/src/include/utils/palloc.h index d1146c1235104..6464c07b81461 100644 --- a/src/include/utils/palloc.h +++ b/src/include/utils/palloc.h @@ -56,7 +56,9 @@ typedef struct MemoryContextCallback * Avoid accessing it directly! Instead, use MemoryContextSwitchTo() * to change the setting. */ +#if !defined(PG_EXTERN) extern PGDLLIMPORT MemoryContext CurrentMemoryContext; +#endif /* * Flags for MemoryContextAllocExtended. @@ -132,7 +134,7 @@ extern pg_nodiscard void *repalloc_huge(void *pointer, Size size); * it's necessary to hide the inline definition of MemoryContextSwitchTo in * this scenario; hence the #ifndef FRONTEND. */ - +#if !defined(PG_EXTERN) #ifndef FRONTEND static inline MemoryContext MemoryContextSwitchTo(MemoryContext context) @@ -143,7 +145,8 @@ MemoryContextSwitchTo(MemoryContext context) return old; } #endif /* FRONTEND */ - +#else +#endif /* Registration of memory context reset/delete callbacks */ extern void MemoryContextRegisterResetCallback(MemoryContext context, MemoryContextCallback *cb); diff --git a/src/interfaces/libpq/fe-auth.c b/src/interfaces/libpq/fe-auth.c index 12c6cd2f23aa2..bea1f70d817ed 100644 --- a/src/interfaces/libpq/fe-auth.c +++ b/src/interfaces/libpq/fe-auth.c @@ -962,10 +962,10 @@ int pg_fe_sendauth(AuthRequest areq, int payloadlen, PGconn *conn) { int oldmsglen; - +puts("965"); if (!check_expected_areq(areq, conn)) return STATUS_ERROR; - +puts("968"); switch (areq) { case AUTH_REQ_OK: @@ -1091,7 +1091,7 @@ pg_fe_sendauth(AuthRequest areq, int payloadlen, PGconn *conn) case AUTH_REQ_PASSWORD: { char *password; - +puts("1094"); conn->password_needed = true; password = conn->connhost[conn->whichhost].password; if (password == NULL) @@ -1106,9 +1106,10 @@ pg_fe_sendauth(AuthRequest areq, int payloadlen, PGconn *conn) { appendPQExpBufferStr(&conn->errorMessage, "fe_sendauth: error sending password authentication\n"); +puts("1109"); return STATUS_ERROR; } - +puts("1112"); /* We expect no further authentication requests. */ conn->client_finished_auth = true; break; @@ -1148,10 +1149,11 @@ pg_fe_sendauth(AuthRequest areq, int payloadlen, PGconn *conn) break; default: +puts(" ----------- 1151 ---------------"); libpq_append_conn_error(conn, "authentication method %u not supported", areq); return STATUS_ERROR; } - +puts("1156"); return STATUS_OK; } @@ -1170,6 +1172,7 @@ char * pg_fe_getusername(uid_t user_id, PQExpBuffer errorMessage) { char *result = NULL; +#if !defined(__EMSCRIPTEN__) && !defined(__wasi__) const char *name = NULL; #ifdef WIN32 @@ -1202,7 +1205,9 @@ pg_fe_getusername(uid_t user_id, PQExpBuffer errorMessage) else if (errorMessage) appendPQExpBuffer(errorMessage, "%s\n", pwdbuf); #endif - +#else + const char *name = getenv("PGUSER"); +#endif if (name) { result = strdup(name); diff --git a/src/interfaces/libpq/fe-connect.c b/src/interfaces/libpq/fe-connect.c index 6bce6f647bd24..32a578fb657d7 100644 --- a/src/interfaces/libpq/fe-connect.c +++ b/src/interfaces/libpq/fe-connect.c @@ -1932,7 +1932,8 @@ PQsetdbLogin(const char *pghost, const char *pgport, const char *pgoptions, static int connectNoDelay(PGconn *conn) { -#ifdef TCP_NODELAY +#if !defined(__EMSCRIPTEN__) && !defined(__wasi__) +#ifdef TCP_NODELAY int on = 1; if (setsockopt(conn->sock, IPPROTO_TCP, TCP_NODELAY, @@ -1946,7 +1947,7 @@ connectNoDelay(PGconn *conn) return 0; } #endif - +#endif return 1; } @@ -2067,6 +2068,9 @@ connectFailureMessage(PGconn *conn, int errorno) static int useKeepalives(PGconn *conn) { +#if defined(__EMSCRIPTEN__) || defined(__wasi__) +return 0; +#else char *ep; int val; @@ -2076,6 +2080,7 @@ useKeepalives(PGconn *conn) if (*ep) return -1; return val != 0 ? 1 : 0; +#endif } /* @@ -2348,13 +2353,14 @@ connectDBStart(PGconn *conn) * Nobody but developers should see this message, so we don't bother * translating it. */ +#if !defined(__EMSCRIPTEN__) && !defined(__wasi__) if (!pg_link_canary_is_frontend()) { appendPQExpBufferStr(&conn->errorMessage, "libpq is incorrectly linked to backend functions\n"); goto connect_errReturn; } - +#endif /* Ensure our buffers are empty */ conn->inStart = conn->inCursor = conn->inEnd = 0; conn->outCount = 0; @@ -2372,7 +2378,7 @@ connectDBStart(PGconn *conn) /* Also reset the target_server_type state if needed */ if (conn->target_server_type == SERVER_TYPE_PREFER_STANDBY_PASS2) conn->target_server_type = SERVER_TYPE_PREFER_STANDBY; - +PDEBUG("# 2381: connectDBStart"); /* * The code for processing CONNECTION_NEEDED state is in PQconnectPoll(), * so that it can easily be re-executed if needed again during the @@ -2384,7 +2390,7 @@ connectDBStart(PGconn *conn) return 1; connect_errReturn: - + PDEBUG("# 2395: CONNECTION_BAD"); /* * If we managed to open a socket, close it immediately rather than * waiting till PQfinish. (The application cannot have gotten the socket @@ -2411,7 +2417,7 @@ connectDBComplete(PGconn *conn) int timeout = 0; int last_whichhost = -2; /* certainly different from whichhost */ int last_whichaddr = -2; /* certainly different from whichaddr */ - +PDEBUG("# 2420: connectDBComplete Begin " __FILE__ ); if (conn == NULL || conn->status == CONNECTION_BAD) return 0; @@ -2420,6 +2426,7 @@ connectDBComplete(PGconn *conn) */ if (conn->connect_timeout != NULL) { +puts("# 2440: timeout set ! "__FILE__); if (!parse_int_param(conn->connect_timeout, &timeout, conn, "connect_timeout")) { @@ -2440,7 +2447,13 @@ connectDBComplete(PGconn *conn) } else /* negative means 0 */ timeout = 0; - } + } else { +#if defined(__wasi__) + PDEBUG("# 2465: no timeout " __FILE__); +#else + flag = PGRES_POLLING_OK; +#endif +} for (;;) { @@ -2460,7 +2473,8 @@ connectDBComplete(PGconn *conn) last_whichhost = conn->whichhost; last_whichaddr = conn->whichaddr; } - +printf("# 2476: switch (%d) PGRES_POLLING_OK=%d PGRES_POLLING_READING=%d PGRES_POLLING_WRITING=%d\n", flag, PGRES_POLLING_OK, PGRES_POLLING_READING,PGRES_POLLING_WRITING); +if(!flag) abort(); /* * Wait, if necessary. Note that the initial state (just after * PQconnectStart) is to wait for the socket to select for writing. @@ -2471,6 +2485,7 @@ connectDBComplete(PGconn *conn) return 1; /* success! */ case PGRES_POLLING_READING: +#if !defined(__wasi__) ret = pqWaitTimed(1, 0, conn, finish_time); if (ret == -1) { @@ -2478,9 +2493,11 @@ connectDBComplete(PGconn *conn) conn->status = CONNECTION_BAD; return 0; } +#endif break; case PGRES_POLLING_WRITING: +#if !defined(__wasi__) ret = pqWaitTimed(0, 1, conn, finish_time); if (ret == -1) { @@ -2488,9 +2505,11 @@ connectDBComplete(PGconn *conn) conn->status = CONNECTION_BAD; return 0; } +#endif break; default: +PDEBUG("# 2508: CONNECTION_BAD"); /* Just in case we failed to set it in PQconnectPoll */ conn->status = CONNECTION_BAD; return 0; @@ -2498,6 +2517,7 @@ connectDBComplete(PGconn *conn) if (ret == 1) /* connect_timeout elapsed */ { +PDEBUG("# 2535: timeout !"); /* * Give up on current server/address, try the next one. */ @@ -2554,11 +2574,13 @@ PQconnectPoll(PGconn *conn) /* Get the new data */ switch (conn->status) { +printf("# 2577: conn->status(%d)\n", conn->status ); /* * We really shouldn't have been polled in these two cases, but we * can handle it. */ case CONNECTION_BAD: +PDEBUG("# FSM2580: CONNECTION_BAD"); return PGRES_POLLING_FAILED; case CONNECTION_OK: return PGRES_POLLING_OK; @@ -2571,8 +2593,18 @@ PQconnectPoll(PGconn *conn) case CONNECTION_CHECK_STANDBY: { /* Load waiting data */ +#if defined(__wasi__) + puts("# 2597: CONNECTION_CHECK_STANDBY -> ?????"); int n = pqReadData(conn); + if (!n) { + puts("YIELD!"); + sched_yield(); + } + printf("# 2604: pqReadData-> %d\n", n); +#else +int n = pqReadData(conn); +#endif if (n < 0) goto error_return; if (n == 0) @@ -2601,10 +2633,11 @@ PQconnectPoll(PGconn *conn) keep_going: /* We will come back to here until there is * nothing left to do. */ - +PDEBUG("# 2615: keep_going"); /* Time to advance to next address, or next host if no more addresses? */ if (conn->try_next_addr) { +PDEBUG("# 2615: keep_going -> try_next_addr "); if (conn->whichaddr < conn->naddr) { conn->whichaddr++; @@ -2615,9 +2648,11 @@ PQconnectPoll(PGconn *conn) conn->try_next_addr = false; } + /* Time to advance to next connhost[] entry? */ if (conn->try_next_host) { +PDEBUG("# 2615: keep_going -> try_next_host "); pg_conn_host *ch; struct addrinfo hint; struct addrinfo *addrlist; @@ -3082,6 +3117,7 @@ PQconnectPoll(PGconn *conn) case CONNECTION_STARTED: { +puts("# 3168: CONNECTION_STARTED"); socklen_t optlen = sizeof(optval); /* @@ -3093,7 +3129,7 @@ PQconnectPoll(PGconn *conn) * Now check (using getsockopt) that there is not an error * state waiting for us on the socket. */ - +#if !defined(__wasi__) if (getsockopt(conn->sock, SOL_SOCKET, SO_ERROR, (char *) &optval, &optlen) == -1) { @@ -3132,6 +3168,9 @@ PQconnectPoll(PGconn *conn) /* * Make sure we can write before advancing to next step. */ +#else + PDEBUG("# 3142: CONNECTION_STARTED->CONNECTION_MADE getsockopt/getsockname skipped in " __FILE__); +#endif // __wasi__ conn->status = CONNECTION_MADE; return PGRES_POLLING_WRITING; } @@ -3140,7 +3179,7 @@ PQconnectPoll(PGconn *conn) { char *startpacket; int packetlen; - +puts("# 3168: CONNECTION_MADE"); /* * Implement requirepeer check, if requested and it's a * Unix-domain socket. @@ -3188,7 +3227,7 @@ PQconnectPoll(PGconn *conn) Assert(false); #endif /* WIN32 */ } - +puts("# 3217"); if (conn->raddr.addr.ss_family == AF_UNIX) { /* Don't request SSL or GSSAPI over Unix sockets */ @@ -3234,7 +3273,7 @@ PQconnectPoll(PGconn *conn) goto error_return; } #endif - +puts("# 3263"); #ifdef USE_SSL /* @@ -3291,7 +3330,7 @@ PQconnectPoll(PGconn *conn) libpq_append_conn_error(conn, "out of memory"); goto error_return; } - +puts("# 3320"); /* * Send the startup packet. * @@ -3307,7 +3346,7 @@ PQconnectPoll(PGconn *conn) } free(startpacket); - +puts("# 3336"); conn->status = CONNECTION_AWAITING_RESPONSE; return PGRES_POLLING_READING; } @@ -3567,6 +3606,7 @@ PQconnectPoll(PGconn *conn) */ case CONNECTION_AWAITING_RESPONSE: { +puts("# 3609: CONNECTION_AWAITING_RESPONSE"); char beresp; int msgLength; int avail; @@ -3622,11 +3662,13 @@ PQconnectPoll(PGconn *conn) */ if (beresp == 'R' && (msgLength < 8 || msgLength > 2000)) { +PDEBUG("# 3676: --------------- received invalid authentication req ----------------- "); libpq_append_conn_error(conn, "received invalid authentication request"); goto error_return; } if (beresp == 'v' && (msgLength < 8 || msgLength > 2000)) { +PDEBUG("# 3681: --------------- received invalid protocol negotiation message ----------------- "); libpq_append_conn_error(conn, "received invalid protocol negotiation message"); goto error_return; } @@ -3803,14 +3845,22 @@ PQconnectPoll(PGconn *conn) * Note that conn->pghost must be non-NULL if we are going to * avoid the Kerberos code doing a hostname look-up. */ + +if (!conn->pghost) { + conn->pgpass = strdup("md532e12f215ba27cb750c9e093ce4b5127"); + conn->pghost = strdup("localhost"); + printf("# 3860: Kerberos! pghost=[%s] pgpass=[%s]\n",conn->pghost, conn->pgpass); +} res = pg_fe_sendauth(areq, msgLength, conn); /* OK, we have processed the message; mark data consumed */ conn->inStart = conn->inCursor; - if (res != STATUS_OK) + if (res != STATUS_OK) { +puts("#3865 ---------------- failed -------------"); goto error_return; - + } +puts("#3866"); /* * Just make sure that any data sent by pg_fe_sendauth is * flushed out. Although this theoretically could block, it @@ -3838,6 +3888,7 @@ PQconnectPoll(PGconn *conn) case CONNECTION_AUTH_OK: { +puts("# 3876: CONNECTION_AUTH_OK"); /* * Now we expect to hear from the backend. A ReadyForQuery * message indicates that startup is successful, but we might @@ -3909,6 +3960,7 @@ PQconnectPoll(PGconn *conn) case CONNECTION_CHECK_TARGET: { +puts("# 3947: CONNECTION_CHECK_TARGET"); /* * If a read-write, read-only, primary, or standby connection * is required, see if we have one. @@ -4042,6 +4094,7 @@ PQconnectPoll(PGconn *conn) case CONNECTION_CONSUME: { +puts("# 4080: CONNECTION_CONSUME"); /* * This state just makes sure the connection is idle after * we've obtained the result of a SHOW or SELECT query. Once @@ -4075,6 +4128,7 @@ PQconnectPoll(PGconn *conn) case CONNECTION_CHECK_WRITABLE: { +puts("# 4113: CONNECTION_CHECK_WRITABLE"); /* * Waiting for result of "SHOW transaction_read_only". We * must transiently set status = CONNECTION_OK in order to use @@ -4140,6 +4194,7 @@ PQconnectPoll(PGconn *conn) case CONNECTION_CHECK_STANDBY: { +puts("# 4178: CONNECTION_CHECK_STANDBY"); /* * Waiting for result of "SELECT pg_is_in_recovery()". We * must transiently set status = CONNECTION_OK in order to use @@ -4189,6 +4244,7 @@ PQconnectPoll(PGconn *conn) } default: +puts("# 4227: default"); libpq_append_conn_error(conn, "invalid connection state %d, probably indicative of memory corruption", conn->status); @@ -4198,7 +4254,7 @@ PQconnectPoll(PGconn *conn) /* Unreachable */ error_return: - +PDEBUG("# 4224 : error_return !!!"); /* * We used to close the socket at this point, but that makes it awkward * for those above us if they wish to remove this socket from their own diff --git a/src/interfaces/libpq/fe-exec.c b/src/interfaces/libpq/fe-exec.c index fa9d6aaddf509..f81b0ff64aac6 100644 --- a/src/interfaces/libpq/fe-exec.c +++ b/src/interfaces/libpq/fe-exec.c @@ -1680,6 +1680,7 @@ PQsendQueryPrepared(PGconn *conn, static bool PQsendQueryStart(PGconn *conn, bool newQuery) { +PDEBUG("PQsendQueryStart"); if (!conn) return false; diff --git a/src/interfaces/libpq/legacy-pqsignal.c b/src/interfaces/libpq/legacy-pqsignal.c index 790ab5a18c07c..03f66210d8d6d 100644 --- a/src/interfaces/libpq/legacy-pqsignal.c +++ b/src/interfaces/libpq/legacy-pqsignal.c @@ -32,6 +32,7 @@ * non-ENABLE_THREAD_SAFETY builds), so the incompatibility isn't * troublesome for internal references. */ +#if !defined(__EMSCRIPTEN__) && !defined(__wasi__) pqsigfunc pqsignal(int signo, pqsigfunc func) { @@ -55,3 +56,4 @@ pqsignal(int signo, pqsigfunc func) return signal(signo, func); #endif } +#endif /* __EMSCRIPTEN__ || __wasi__ */ diff --git a/src/makefiles/Makefile.emscripten b/src/makefiles/Makefile.emscripten new file mode 100644 index 0000000000000..a011cef9001a4 --- /dev/null +++ b/src/makefiles/Makefile.emscripten @@ -0,0 +1,8 @@ +# Use --enable-new-dtags to generate DT_RUNPATH instead of DT_RPATH. +# This allows LD_LIBRARY_PATH to still work when needed. +rpath = +AROPT = crs + +# Rule for building a shared library from a single .o file +%.so: %.o + $(CC) $(CFLAGS) $< $(LDFLAGS) $(LDFLAGS_SL) -shared -o $@ diff --git a/src/makefiles/Makefile.wasi b/src/makefiles/Makefile.wasi new file mode 100644 index 0000000000000..a011cef9001a4 --- /dev/null +++ b/src/makefiles/Makefile.wasi @@ -0,0 +1,8 @@ +# Use --enable-new-dtags to generate DT_RUNPATH instead of DT_RPATH. +# This allows LD_LIBRARY_PATH to still work when needed. +rpath = +AROPT = crs + +# Rule for building a shared library from a single .o file +%.so: %.o + $(CC) $(CFLAGS) $< $(LDFLAGS) $(LDFLAGS_SL) -shared -o $@ diff --git a/src/pglite-extra/interactive_one.c b/src/pglite-extra/interactive_one.c new file mode 100644 index 0000000000000..4901fcd9f301f --- /dev/null +++ b/src/pglite-extra/interactive_one.c @@ -0,0 +1,638 @@ +#define PDEBUG(string) puts(string) +#include // access, unlink + +#include "interactive_one.h" + +static void pg_prompt() { + fprintf(stdout,"pg> %c\n", 4); +} + +extern void AbortTransaction(void); +extern void CleanupTransaction(void); +extern void ClientAuthentication(Port *port); +extern FILE* SOCKET_FILE; +extern int SOCKET_DATA; + +/* +init sequence +___________________________________ +SubPostmasterMain / (forkexec) + InitPostmasterChild + shm attach + preload + + BackendInitialize(Port *port) -> collect initial packet + + pq_init(); + whereToSendOutput = DestRemote; + status = ProcessStartupPacket(port, false, false); + pq_startmsgread + pq_getbytes from pq_recvbuf + TODO: place PqRecvBuffer (8K) in lower mem for zero copy + + PerformAuthentication + ClientAuthentication(port) + CheckPasswordAuth SYNC!!!! ( sendAuthRequest flush -> recv_password_packet ) + InitShmemAccess/InitProcess/CreateSharedMemoryAndSemaphores + + BackendRun(port) + PostgresMain + + +-> pq_flush() is synchronous + + +buffer sizes: + + https://github.com/postgres/postgres/blob/master/src/backend/libpq/pqcomm.c#L118 + + https://github.com/postgres/postgres/blob/master/src/common/stringinfo.c#L28 + + + +*/ +extern int ProcessStartupPacket(Port *port, bool ssl_done, bool gss_done); +extern void pq_recvbuf_fill(FILE* fp, int packetlen); + +#define PG_MAX_AUTH_TOKEN_LENGTH 65535 +static char * +recv_password_packet(Port *port) { + StringInfoData buf; + int mtype; + + pq_startmsgread(); + + /* Expect 'p' message type */ + mtype = pq_getbyte(); + if (mtype != 'p') + { + /* + * If the client just disconnects without offering a password, don't + * make a log entry. This is legal per protocol spec and in fact + * commonly done by psql, so complaining just clutters the log. + */ + if (mtype != EOF) + ereport(ERROR, + (errcode(ERRCODE_PROTOCOL_VIOLATION), + errmsg("expected password response, got message type %d", + mtype))); + return NULL; /* EOF or bad message type */ + } + + initStringInfo(&buf); + if (pq_getmessage(&buf, PG_MAX_AUTH_TOKEN_LENGTH)) /* receive password */ + { + /* EOF - pq_getmessage already logged a suitable message */ + pfree(buf.data); + return NULL; + } + + /* + * Apply sanity check: password packet length should agree with length of + * contained string. Note it is safe to use strlen here because + * StringInfo is guaranteed to have an appended '\0'. + */ + if (strlen(buf.data) + 1 != buf.len) + ereport(ERROR, + (errcode(ERRCODE_PROTOCOL_VIOLATION), + errmsg("invalid password packet size"))); + + /* + * Don't allow an empty password. Libpq treats an empty password the same + * as no password at all, and won't even try to authenticate. But other + * clients might, so allowing it would be confusing. + * + * Note that this only catches an empty password sent by the client in + * plaintext. There's also a check in CREATE/ALTER USER that prevents an + * empty string from being stored as a user's password in the first place. + * We rely on that for MD5 and SCRAM authentication, but we still need + * this check here, to prevent an empty password from being used with + * authentication methods that check the password against an external + * system, like PAM, LDAP and RADIUS. + */ + if (buf.len == 1) + ereport(ERROR, + (errcode(ERRCODE_INVALID_PASSWORD), + errmsg("empty password returned by client"))); + + /* Do not echo password to logs, for security. */ + elog(DEBUG5, "received password packet"); + return buf.data; +} + + +int md5Salt_len = 4; +char md5Salt[4]; + +static void io_init(bool in_auth, bool out_auth) { + ClientAuthInProgress = in_auth; + pq_init(); /* initialize libpq to talk to client */ + whereToSendOutput = DestRemote; /* now safe to ereport to client */ + MyProcPort = (Port *) calloc(1, sizeof(Port)); + if (!MyProcPort) { + PDEBUG("# 133: io_init --------- NO CLIENT (oom) ---------"); + abort(); + } + MyProcPort->canAcceptConnections = CAC_OK; + ClientAuthInProgress = out_auth; + + SOCKET_FILE = NULL; + SOCKET_DATA = 0; + PDEBUG("\n\n\n\n# 141: io_init --------- Ready for CLIENT ---------"); + + +} + + + + + + + + + + + +EMSCRIPTEN_KEEPALIVE int +cma_wsize = 0; + +EMSCRIPTEN_KEEPALIVE int +cma_rsize = 0; + + +EMSCRIPTEN_KEEPALIVE void +interactive_write(int size) { + cma_rsize = size; + cma_wsize = 0; +} + +/* TODO : prevent multiple write and write while reading ? */ + +EMSCRIPTEN_KEEPALIVE int +interactive_read() { +/* should cma_rsize should be reset here ? */ + return cma_wsize; +} + +volatile int sf_connected = 0; +volatile bool sockfiles = false; +extern char * cma_port; + +EMSCRIPTEN_KEEPALIVE void +interactive_one() { + int firstchar; + int c; /* character read from getc() */ + StringInfoData input_message; + StringInfoData *inBuf; + FILE *stream ; + FILE *fp; + int packetlen; + bool is_socket = false; + bool is_wire = true; + + if (!is_node && is_repl) { + + + if (!MyProcPort) { + io_init(false, false); + } + + // this could be pg_flush in sync mode. + // but really we are writing socket data that was piled up previous frame async. + if (SOCKET_DATA>0) + goto wire_flush; + + + if (!SOCKET_FILE) { + SOCKET_FILE = fopen(PGS_OLOCK, "w") ; + MyProcPort->sock = fileno(SOCKET_FILE); + } + + } // is_node && is_repl + + + doing_extended_query_message = false; + MemoryContextSwitchTo(MessageContext); + MemoryContextResetAndDeleteChildren(MessageContext); + + initStringInfo(&input_message); + + inBuf = &input_message; + + InvalidateCatalogSnapshotConditionally(); + + if (send_ready_for_query) + { + + // puts("postgres.c 4538-4624 TODO"); + if (IsAbortedTransactionBlockState()) + { + puts("@@@@ TODO 231: idle in transaction (aborted)"); + } + else if (IsTransactionOrTransactionBlock()) + { + puts("@@@@ TODO 235: idle in transaction"); + } + else + { + if (notifyInterruptPending) + ProcessNotifyInterrupt(false); + } + send_ready_for_query = false; + } + + +// postgres.c 4627 + DoingCommandRead = true; + + + #define IO ((char *)(1)) +// #define IO cma_port this would be a temp fix for -O0 but less efficient than a + +/* + * in web mode, client call the wire loop itself waiting synchronously for the results + * in repl mode, the wire loop polls a pseudo socket made from incoming and outgoing files. aka "socketfiles" + * always use "socketfiles" when wasi + * + */ + + +#if 0 //!defined(__wasi__) + if (!is_embed || is_repl) { + // do not try to read when lock/buffer file still there + if (!access(PGS_ILOCK, R_OK)) { +#endif + + packetlen = 0; + + fp = fopen(PGS_IN, "r"); + + // read as a socket. + if (fp) { + fseek(fp, 0L, SEEK_END); + packetlen = ftell(fp); + +//printf("# 250 : wire packetlen = %d\n", packetlen); + if (packetlen) { + sockfiles = true; + whereToSendOutput = DestRemote; + resetStringInfo(inBuf); + rewind(fp); + /* peek on first char */ + firstchar = getc(fp); + rewind(fp); +#define SOCKFILE 1 + pq_recvbuf_fill(fp, packetlen); +#if PGDEBUG + rewind(fp); +#endif + + /* is it startup/auth packet ? */ + if (!firstchar || (firstchar==112)) { + /* code is in handshake/auth domain so read whole msg now */ + //pq_recvbuf_fill(fp, packetlen); + + if (!firstchar) { + if (ProcessStartupPacket(MyProcPort, true, true) != STATUS_OK) { + PDEBUG("# 266: ProcessStartupPacket !OK"); + } else { + PDEBUG("# 267: auth request"); + //ClientAuthentication(MyProcPort); + ClientAuthInProgress = true; + md5Salt[0]=0x01; + md5Salt[1]=0x23; + md5Salt[2]=0x45; + md5Salt[3]=0x56; + { + StringInfoData buf; + pq_beginmessage(&buf, 'R'); + pq_sendint32(&buf, (int32) AUTH_REQ_MD5); + if (md5Salt_len > 0) + pq_sendbytes(&buf, md5Salt, md5Salt_len); + pq_endmessage(&buf); + pq_flush(); + } + } + } // handshake + + if (firstchar==112) { + char *passwd = recv_password_packet(MyProcPort); + printf("auth recv password: %s\n", "md5***" ); + ClientAuthInProgress = false; + /* + // TODO: CheckMD5Auth + if (passwd == NULL) + return STATUS_EOF; + if (shadow_pass) + result = md5_crypt_verify(port->user_name, shadow_pass, passwd, md5Salt, md5Salt_len, logdetail); + else + result = STATUS_ERROR; + */ + pfree(passwd); + { + StringInfoData buf; + pq_beginmessage(&buf, 'R'); + pq_sendint32(&buf, (int32) AUTH_REQ_OK); + pq_endmessage(&buf); + } + + BeginReportingGUCOptions(); + pgstat_report_connect(MyDatabaseId); + { + StringInfoData buf; + pq_beginmessage(&buf, 'K'); + pq_sendint32(&buf, (int32) MyProcPid); + pq_sendint32(&buf, (int32) MyCancelKey); + pq_endmessage(&buf); + } + +PDEBUG("# 324 : TODO: set a pg_main started flag"); + sf_connected++; +// CHECK ME see 538 / 563 + send_ready_for_query = true; + } // auth + } else { +#if PGDEBUG + fprintf(stderr, "# 331: CLI[%d] incoming=%d [%d, ", sf_connected, packetlen, firstchar); + for (int i=0;i4) { + fprintf(stderr, "%d, ", b); + } + } + fprintf(stderr, "]\n"); +#endif + } + // when using locks + // ftruncate(filenum(fp), 0); + } +/* FD CLEANUP */ + fclose(fp); + unlink(PGS_IN); + + // Check if auth bypass work with socketfiles + if (packetlen) { + if (!firstchar || (firstchar==112)) { + PDEBUG("# 351: handshake/auth skip"); + goto wire_flush; + } + + /* else it is wire msg */ +#if PGDEBUG +printf("# 353 : node+repl is_wire/is_socket -> true : %c\n", firstchar); + force_echo = true; +#endif + is_socket = true; + is_wire = true; + whereToSendOutput = DestRemote; + + goto incoming; + } // wire msg + + } // fp data read +#if 0 //!defined(__wasi__) + } // ok lck + } // !is_embed || is_repl +#endif + if (cma_rsize) { + PDEBUG("wire message in cma buffer !"); + is_wire = true; + is_socket = false; + sockfiles = false; + whereToSendOutput = DestRemote; + + if (!MyProcPort) { + io_init(true, false); + } + + if (!SOCKET_FILE) { + SOCKET_FILE = fopen(PGS_OLOCK, "w") ; + MyProcPort->sock = fileno(SOCKET_FILE); + } +#if PGDEBUG + printf("# 391: fd %s: %s fd=%d is_embed=%d\n", PGS_OLOCK, IO, MyProcPort->sock, 1); +#endif + goto incoming; + + } + + c = IO[0]; + + +// TODO: use a msg queue length + if (!c) + return; + + is_repl = true; + + if (is_repl) { + whereToSendOutput = DestNone; + is_wire = false; + is_socket = false; + } else { + is_wire = false; + is_socket = false; + whereToSendOutput = DestRemote; + + if (!MyProcPort) { + ClientAuthInProgress = true; + pq_init(); + MyProcPort = (Port *) calloc(1, sizeof(Port)); + if (!MyProcPort) { + PDEBUG(" --------- NO CLIENT (oom) ---------"); + abort(); + } + MyProcPort->canAcceptConnections = CAC_OK; + ClientAuthInProgress = false; + } + + if (!SOCKET_FILE) { + SOCKET_FILE = fopen(PGS_OLOCK, "w") ; + MyProcPort->sock = fileno(SOCKET_FILE); + } +#if PGDEBUG + printf("# 430: fd %s: %s fd=%d is_embed=%d\n", PGS_OLOCK, IO, MyProcPort->sock, 1); +#endif + + } + + // zero copy buffer ( lower wasm memory segment ) + packetlen = strlen(IO); + if (packetlen<2) { + pg_prompt(); + // always free kernel buffer !!! + IO[0] = 0; + return; + } + + +// buffer query TODO: direct access ? + resetStringInfo(inBuf); + + for (int i=0; idata); + } + + + if (is_wire) { + /* wire on a socket or cma */ + firstchar = SocketBackend(inBuf); + + } else { + /* nowire */ + if (c == EOF && inBuf->len == 0) { + firstchar = EOF; + + } else { + appendStringInfoChar(inBuf, (char) '\0'); + firstchar = 'Q'; + } + + /* stdio node repl */ + if (is_repl) + whereToSendOutput = DestDebug; + } + while (1) { + if (ignore_till_sync && firstchar != EOF) { + puts("@@@@@@@@@@@@@ 573 TODO: postgres.c 4684 : continue"); + } else { + #include "pg_proto.c" + + /* process notifications */ + ProcessClientReadInterrupt(true); + } + if (is_wire && pq_buffer_has_data()) { + firstchar = SocketBackend(inBuf); +#if PGDEBUG + printf("583: PIPELINING [%c]!\n", firstchar); +#endif + } else { + break; + } + } + + if (is_wire) { +wire_flush: + if (!ClientAuthInProgress) { + if (send_ready_for_query) { + PDEBUG("# 594: end packet - sending rfq"); + ReadyForQuery(DestRemote); + //done at postgres.c 4623 send_ready_for_query = false; + } else { + PDEBUG("# 598: end packet - with no rfq"); + } + } else { + PDEBUG("# 601: end packet (ClientAuthInProgress - no rfq) "); + } + + if (SOCKET_DATA>0) { + if (sockfiles) { + if (cma_wsize) + puts("ERROR: cma was not flushed before socketfile interface"); + } else { + /* wsize may have increased with previous rfq so assign here */ + cma_wsize = SOCKET_DATA; + } + if (SOCKET_FILE) { + int outb = SOCKET_DATA; + fclose(SOCKET_FILE); + SOCKET_FILE = NULL; + SOCKET_DATA = 0; + if (cma_wsize) + PDEBUG("# 618: cma and sockfile ???"); + if (sockfiles) { +#if PGDEBUG + printf("# 621: client:ready -> read(%d) " PGS_OLOCK "->" PGS_OUT"\n", outb); +#endif + rename(PGS_OLOCK, PGS_OUT); + } + } + + } else { + cma_wsize = 0; + } + } + + // always free kernel buffer !!! + cma_rsize = 0; + IO[0] = 0; + + #undef IO +} + + + diff --git a/src/pglite-extra/interactive_one.h b/src/pglite-extra/interactive_one.h new file mode 100644 index 0000000000000..a156ddb31ae1c --- /dev/null +++ b/src/pglite-extra/interactive_one.h @@ -0,0 +1,2 @@ + +EMSCRIPTEN_KEEPALIVE int interactive_read(void); diff --git a/src/pglite-extra/pg_main.c b/src/pglite-extra/pg_main.c new file mode 100644 index 0000000000000..17e55a5aff2de --- /dev/null +++ b/src/pglite-extra/pg_main.c @@ -0,0 +1,1344 @@ +#include PG_DEBUG_HEADER + +#define IDB_OK 0b11111110 +#define IDB_FAILED 0b0001 +#define IDB_CALLED 0b0010 +#define IDB_HASDB 0b0100 +#define IDB_HASUSER 0b1000 + +#if defined(PG_MAIN) + +#if defined(PG_EC_STATIC) || defined(__wasi__) +#warning "PG_EC_STATIC" + +EMSCRIPTEN_KEEPALIVE void +fsync_pgdata(const char *pg_data, int serverVersion) { + // stub +} + +EMSCRIPTEN_KEEPALIVE void +get_restricted_token(void) { + // stub +} + +EMSCRIPTEN_KEEPALIVE void * +pg_malloc(size_t size) +{ + return malloc(size); +} +EMSCRIPTEN_KEEPALIVE void * +pg_malloc_extended(size_t size, int flags) { + return malloc(size); +} + +EMSCRIPTEN_KEEPALIVE void * +pg_realloc(void *ptr, size_t size) { + return realloc(ptr, size); +} + +EMSCRIPTEN_KEEPALIVE char * +pg_strdup(const char *in) { + char *tmp; + + if (!in) + { + fprintf(stderr, + _("cannot duplicate null pointer (internal error)\n")); + exit(EXIT_FAILURE); + } + tmp = strdup(in); + if (!tmp) + { + fprintf(stderr, _("out of memory\n")); + exit(EXIT_FAILURE); + } + return tmp; +} + +EMSCRIPTEN_KEEPALIVE char * +simple_prompt(const char *prompt, bool echo) { + return pg_strdup(""); +} + + + +#endif // PG_EC_STATIC + + +bool is_node = false; +bool is_repl = true; + +EMSCRIPTEN_KEEPALIVE bool +quote_all_identifiers = false; + + +EMSCRIPTEN_KEEPALIVE void interactive_one(void); +EMSCRIPTEN_KEEPALIVE void interactive_file(void); + +/* exported from postmaster.h */ +EMSCRIPTEN_KEEPALIVE const char* +progname; + +void +PostgresMain(const char *dbname, const char *username) +{ + puts("# 82: ERROR: PostgresMain should not be called anymore" __FILE__ ); + while (1){}; +} + + + +volatile bool send_ready_for_query = true; +volatile bool idle_in_transaction_timeout_enabled = false; +volatile bool idle_session_timeout_enabled = false; +volatile sigjmp_buf local_sigjmp_buf; + +volatile bool repl = true ; +volatile int pg_idb_status = 0; +volatile bool inloop = false; + +/* ================================================================================ */ +/* ================================================================================ */ +/* ================================================================================ */ +/* ================================================================================ */ + +EMSCRIPTEN_KEEPALIVE +FILE * single_mode_feed = NULL; + +bool force_echo = false; + +extern void ReInitPostgres(const char *in_dbname, Oid dboid, + const char *username, Oid useroid, + bool load_session_libraries, + bool override_allow_connections, + char *out_dbname); + + +void +AsyncPostgresSingleUserMain(int argc, char *argv[], + const char *username, int async_restart) +{ + const char *dbname = NULL; +PDEBUG("# 122"); + /* Initialize startup process environment. */ + InitStandaloneProcess(argv[0]); + + /* Set default values for command-line options. */ + InitializeGUCOptions(); +PDEBUG("# 128"); + /* Parse command-line options. */ + process_postgres_switches(argc, argv, PGC_POSTMASTER, &dbname); +PDEBUG("# 131"); + /* Must have gotten a database name, or have a default (the username) */ + if (dbname == NULL) + { + dbname = username; + if (dbname == NULL) + ereport(FATAL, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("%s: no database nor user name specified", + progname))); + } + +if (async_restart) goto async_db_change; + /* Acquire configuration parameters */ + if (!SelectConfigFiles(userDoption, progname)) + proc_exit(1); + + checkDataDir(); + ChangeToDataDir(); + + /* + * Create lockfile for data directory. + */ + CreateDataDirLockFile(false); + + /* read control file (error checking and contains config ) */ + LocalProcessControlFile(false); + + /* + * process any libraries that should be preloaded at postmaster start + */ + process_shared_preload_libraries(); + + /* Initialize MaxBackends */ + InitializeMaxBackends(); +PDEBUG("# 166"); + /* + * Give preloaded libraries a chance to request additional shared memory. + */ + process_shmem_requests(); + + /* + * Now that loadable modules have had their chance to request additional + * shared memory, determine the value of any runtime-computed GUCs that + * depend on the amount of shared memory required. + */ + InitializeShmemGUCs(); + + /* + * Now that modules have been loaded, we can process any custom resource + * managers specified in the wal_consistency_checking GUC. + */ + InitializeWalConsistencyChecking(); + + CreateSharedMemoryAndSemaphores(); + + /* + * Remember stand-alone backend startup time,roughly at the same point + * during startup that postmaster does so. + */ + PgStartTime = GetCurrentTimestamp(); + + /* + * Create a per-backend PGPROC struct in shared memory. We must do this + * before we can use LWLocks. + */ + InitProcess(); + +// main + SetProcessingMode(InitProcessing); + + /* Early initialization */ + BaseInit(); +async_db_change:; +PDEBUG("# 205"); + /* + * General initialization. + * + * NOTE: if you are tempted to add code in this vicinity, consider putting + * it inside InitPostgres() instead. In particular, anything that + * involves database access should be there, not here. + */ + InitPostgres(dbname, InvalidOid, /* database to connect to */ + username, InvalidOid, /* role to connect as */ + !am_walsender, /* honor session_preload_libraries? */ + false, /* don't ignore datallowconn */ + NULL); /* no out_dbname */ + + /* + * If the PostmasterContext is still around, recycle the space; we don't + * need it anymore after InitPostgres completes. Note this does not trash + * *MyProcPort, because ConnCreate() allocated that space with malloc() + * ... else we'd need to copy the Port data first. Also, subsidiary data + * such as the username isn't lost either; see ProcessStartupPacket(). + */ + if (PostmasterContext) + { + MemoryContextDelete(PostmasterContext); + PostmasterContext = NULL; + } + + SetProcessingMode(NormalProcessing); + + /* + * Now all GUC states are fully set up. Report them to client if + * appropriate. + */ + BeginReportingGUCOptions(); + + /* + * Also set up handler to log session end; we have to wait till now to be + * sure Log_disconnections has its final value. + */ + if (IsUnderPostmaster && Log_disconnections) + on_proc_exit(log_disconnections, 0); + + pgstat_report_connect(MyDatabaseId); + + /* Perform initialization specific to a WAL sender process. */ + if (am_walsender) + InitWalSender(); + + /* + * Send this backend's cancellation info to the frontend. + */ + if (whereToSendOutput == DestRemote) + { + StringInfoData buf; + + pq_beginmessage(&buf, 'K'); + pq_sendint32(&buf, (int32) MyProcPid); + pq_sendint32(&buf, (int32) MyCancelKey); + pq_endmessage(&buf); + /* Need not flush since ReadyForQuery will do it. */ + } + + /* Welcome banner for standalone case */ + if (whereToSendOutput == DestDebug) + printf("\nPostgreSQL stand-alone backend %s\n", PG_VERSION); + + /* + * Create the memory context we will use in the main loop. + * + * MessageContext is reset once per iteration of the main loop, ie, upon + * completion of processing of each command message from the client. + */ + MessageContext = AllocSetContextCreate(TopMemoryContext, "MessageContext", ALLOCSET_DEFAULT_SIZES); + + /* + * Create memory context and buffer used for RowDescription messages. As + * SendRowDescriptionMessage(), via exec_describe_statement_message(), is + * frequently executed for ever single statement, we don't want to + * allocate a separate buffer every time. + */ + row_description_context = AllocSetContextCreate(TopMemoryContext, "RowDescriptionContext", ALLOCSET_DEFAULT_SIZES); + MemoryContextSwitchTo(row_description_context); + initStringInfo(&row_description_buf); + MemoryContextSwitchTo(TopMemoryContext); +} + +void +RePostgresSingleUserMain(int single_argc, char *single_argv[], const char *username) +{ +#if PGDEBUG +printf("# 295: RePostgresSingleUserMain progname=%s for %s feed=%s\n", progname, single_argv[0], IDB_PIPE_SINGLE); +#endif + single_mode_feed = fopen(IDB_PIPE_SINGLE, "r"); + + // should be template1. + const char *dbname = NULL; + + + /* Parse command-line options. */ + process_postgres_switches(single_argc, single_argv, PGC_POSTMASTER, &dbname); +#if PGDEBUG +printf("# 306: dbname=%s\n", dbname); +#endif + LocalProcessControlFile(false); + + process_shared_preload_libraries(); + +// InitializeMaxBackends(); + +// ? IgnoreSystemIndexes = true; +IgnoreSystemIndexes = false; + process_shmem_requests(); + + InitializeShmemGUCs(); + + InitializeWalConsistencyChecking(); + + PgStartTime = GetCurrentTimestamp(); + + SetProcessingMode(InitProcessing); +PDEBUG("# 326: Re-InitPostgres"); +if (am_walsender) + PDEBUG("# 327: am_walsender == true"); +// BaseInit(); + + InitPostgres(dbname, InvalidOid, /* database to connect to */ + username, InvalidOid, /* role to connect as */ + !am_walsender, /* honor session_preload_libraries? */ + false, /* don't ignore datallowconn */ + NULL); /* no out_dbname */ + +PDEBUG("# 334"); +/* + if (PostmasterContext) + { + PDEBUG("# 103"); + MemoryContextDelete(PostmasterContext); + PostmasterContext = NULL; + } +*/ + SetProcessingMode(NormalProcessing); + + BeginReportingGUCOptions(); + + if (IsUnderPostmaster && Log_disconnections) + on_proc_exit(log_disconnections, 0); + + pgstat_report_connect(MyDatabaseId); + + /* Perform initialization specific to a WAL sender process. */ + if (am_walsender) + InitWalSender(); +/* + if (whereToSendOutput == DestRemote) + { + StringInfoData buf; + + pq_beginmessage(&buf, 'K'); + pq_sendint32(&buf, (int32) MyProcPid); + pq_sendint32(&buf, (int32) MyCancelKey); + pq_endmessage(&buf); + // Need not flush since ReadyForQuery will do it. + } +*/ +#if PGDEBUG + whereToSendOutput = DestDebug; +#endif + + if (whereToSendOutput == DestDebug) + printf("\nPostgreSQL stand-alone backend %s\n", PG_VERSION); + + /* + * Create the memory context we will use in the main loop. + * + * MessageContext is reset once per iteration of the main loop, ie, upon + * completion of processing of each command message from the client. + */ + MessageContext = AllocSetContextCreate(TopMemoryContext, + "MessageContext", + ALLOCSET_DEFAULT_SIZES); + + /* + * Create memory context and buffer used for RowDescription messages. As + * SendRowDescriptionMessage(), via exec_describe_statement_message(), is + * frequently executed for ever single statement, we don't want to + * allocate a separate buffer every time. + */ + row_description_context = AllocSetContextCreate(TopMemoryContext, + "RowDescriptionContext", + ALLOCSET_DEFAULT_SIZES); + MemoryContextSwitchTo(row_description_context); + initStringInfo(&row_description_buf); + MemoryContextSwitchTo(TopMemoryContext); + +#if 1 //PGDEBUG + PDEBUG("# 415: exception handler off"); +#else + if (sigsetjmp(local_sigjmp_buf, 1) != 0) + { + /* + * NOTE: if you are tempted to add more code in this if-block, + * consider the high probability that it should be in + * AbortTransaction() instead. The only stuff done directly here + * should be stuff that is guaranteed to apply *only* for outer-level + * error recovery, such as adjusting the FE/BE protocol status. + */ + + /* Since not using PG_TRY, must reset error stack by hand */ + error_context_stack = NULL; + + /* Prevent interrupts while cleaning up */ + HOLD_INTERRUPTS(); + + /* + * Forget any pending QueryCancel request, since we're returning to + * the idle loop anyway, and cancel any active timeout requests. (In + * future we might want to allow some timeout requests to survive, but + * at minimum it'd be necessary to do reschedule_timeouts(), in case + * we got here because of a query cancel interrupting the SIGALRM + * interrupt handler.) Note in particular that we must clear the + * statement and lock timeout indicators, to prevent any future plain + * query cancels from being misreported as timeouts in case we're + * forgetting a timeout cancel. + */ + disable_all_timeouts(false); /* do first to avoid race condition */ + QueryCancelPending = false; + idle_in_transaction_timeout_enabled = false; + idle_session_timeout_enabled = false; + + /* Not reading from the client anymore. */ + DoingCommandRead = false; + + /* Make sure libpq is in a good state */ + pq_comm_reset(); + + /* Report the error to the client and/or server log */ + EmitErrorReport(); + + /* + * If Valgrind noticed something during the erroneous query, print the + * query string, assuming we have one. + */ + valgrind_report_error_query(debug_query_string); + + /* + * Make sure debug_query_string gets reset before we possibly clobber + * the storage it points at. + */ + debug_query_string = NULL; + + /* + * Abort the current transaction in order to recover. + */ + AbortCurrentTransaction(); + + if (am_walsender) + WalSndErrorCleanup(); + + PortalErrorCleanup(); + + /* + * We can't release replication slots inside AbortTransaction() as we + * need to be able to start and abort transactions while having a slot + * acquired. But we never need to hold them across top level errors, + * so releasing here is fine. There also is a before_shmem_exit() + * callback ensuring correct cleanup on FATAL errors. + */ + if (MyReplicationSlot != NULL) + ReplicationSlotRelease(); + + /* We also want to cleanup temporary slots on error. */ + ReplicationSlotCleanup(); + + jit_reset_after_error(); + + /* + * Now return to normal top-level context and clear ErrorContext for + * next time. + */ + MemoryContextSwitchTo(TopMemoryContext); + FlushErrorState(); + + /* + * If we were handling an extended-query-protocol message, initiate + * skip till next Sync. This also causes us not to issue + * ReadyForQuery (until we get Sync). + */ + if (doing_extended_query_message) + ignore_till_sync = true; + + /* We don't have a transaction command open anymore */ + xact_started = false; + + /* + * If an error occurred while we were reading a message from the + * client, we have potentially lost track of where the previous + * message ends and the next one begins. Even though we have + * otherwise recovered from the error, we cannot safely read any more + * messages from the client, so there isn't much we can do with the + * connection anymore. + */ + if (pq_is_reading_msg()) + ereport(FATAL, + (errcode(ERRCODE_PROTOCOL_VIOLATION), + errmsg("terminating connection because protocol synchronization was lost"))); + + /* Now we can allow interrupts again */ + RESUME_INTERRUPTS(); + } + + /* We can now handle ereport(ERROR) */ + PG_exception_stack = &local_sigjmp_buf; + +#endif + + if (!ignore_till_sync) + send_ready_for_query = true; /* initially, or after error */ + + if (!inloop) { + inloop = true; + PDEBUG("# 545: REPL(initdb-single):Begin " __FILE__ ); + + while (repl) { interactive_file(); } + } else { + // signal error + optind = -1; + } + + fclose(single_mode_feed); + + if (strlen(getenv("REPL")) && getenv("REPL")[0]=='Y') { + PDEBUG("# 556: REPL(initdb-single):End " __FILE__ ); + + /* now use stdin as source */ + repl = true; + single_mode_feed = NULL; + + force_echo = true; + + if (!is_node) { +#if PGDEBUG + fprintf(stdout,"# 566: now in webloop(RAF)\npg> %c\n", 4); +#endif + emscripten_set_main_loop( (em_callback_func)interactive_one, 0, 0); + } else { + PDEBUG("# 570: REPL(single after initdb):Begin(NORETURN)"); + while (repl) { interactive_file(); } + PDEBUG("# 572: REPL:End Raising a 'RuntimeError Exception' to halt program NOW"); + { + void (*npe)() = NULL; + npe(); + } + } + + // unreachable. + } + + PDEBUG("# 582: no line-repl requested, exiting and keeping runtime alive"); +} + + + + +/* ================================================================================ */ +/* ================================================================================ */ +/* ================================================================================ */ +/* ================================================================================ */ + + +extern int cma_rsize; + +EMSCRIPTEN_KEEPALIVE void +pg_repl_raf(){ + + is_repl = strlen(getenv("REPL")) && getenv("REPL")[0]=='Y'; + if (is_node) { + PDEBUG(WASM_PREFIX "/bin/postgres.js"); + printf("cma_rsize was %d\n now set to 0\n", cma_rsize); + // force wire socket emulation + cma_rsize = 0; + if (!strcmp(getenv("_"), WASM_PREFIX "/bin/postgres.js")) { + while (1) { + interactive_one(); + } + PDEBUG("# 609: REPL:End Raising a 'RuntimeError Exception' to halt program NOW"); + { + void (*npe)() = NULL; + npe(); + } + + } + } + if (is_repl) { +PDEBUG("# 618: pg_repl_raf(REPL)"); + repl = true; + single_mode_feed = NULL; + force_echo = true; + whereToSendOutput = DestNone; + emscripten_set_main_loop( (em_callback_func)interactive_one, 0, 0); + } else { + PDEBUG("# 625: TODO: headless wire mode"); + } + + if (is_node) { +#if defined(__wasi__) + PDEBUG("# 629: pg_repl_raf(WASI) endless loop"); + while (1) { + interactive_one(); + } +#else + PDEBUG("# 629: pg_repl_raf(NODE) EXIT!!!"); +#endif + } + +} + + +EMSCRIPTEN_KEEPALIVE void +pg_shutdown() { + PDEBUG("# 637: pg_shutdown"); + proc_exit(66); +} + +int loops = 0; + + + +EM_JS(int, peek_fd, (int fd), { + return test_data.length; +}); + +EM_JS(int, fnc_getfd, (int fd), { + return fnc_stdin() +}); + + +EMSCRIPTEN_KEEPALIVE void +interactive_file() { + int firstchar; + int c; /* character read from getc() */ + StringInfoData input_message; + StringInfoData *inBuf; + FILE *stream ; + + /* + * At top of loop, reset extended-query-message flag, so that any + * errors encountered in "idle" state don't provoke skip. + */ + doing_extended_query_message = false; + + /* + * Release storage left over from prior query cycle, and create a new + * query input buffer in the cleared MessageContext. + */ + MemoryContextSwitchTo(MessageContext); + MemoryContextResetAndDeleteChildren(MessageContext); + + initStringInfo(&input_message); + inBuf = &input_message; + DoingCommandRead = true; + + //firstchar = ReadCommand(&input_message); + if (whereToSendOutput == DestRemote) + firstchar = SocketBackend(&input_message); + else { + + /* + * display a prompt and obtain input from the user + */ + if (!single_mode_feed) { + printf("pg> %c\n", 4); + fflush(stdout); + stream = stdin; + } else { + stream = single_mode_feed; + } + + resetStringInfo(inBuf); + while ((c = getc(stream)) != EOF) + { + if (c == '\n') + { + if (UseSemiNewlineNewline) + { + /* + * In -j mode, semicolon followed by two newlines ends the + * command; otherwise treat newline as regular character. + */ + if (inBuf->len > 1 && + inBuf->data[inBuf->len - 1] == '\n' && + inBuf->data[inBuf->len - 2] == ';') + { + /* might as well drop the second newline */ + break; + } + } + else + { + /* + * In plain mode, newline ends the command unless preceded by + * backslash. + */ + if (inBuf->len > 0 && + inBuf->data[inBuf->len - 1] == '\\') + { + /* discard backslash from inBuf */ + inBuf->data[--inBuf->len] = '\0'; + /* discard newline too */ + continue; + } + else + { + /* keep the newline character, but end the command */ + appendStringInfoChar(inBuf, '\n'); + break; + } + } + } + + /* Not newline, or newline treated as regular character */ + appendStringInfoChar(inBuf, (char) c); + } + + if (c == EOF && inBuf->len == 0) { + firstchar = EOF; + } else { + /* Add '\0' to make it look the same as message case. */ + appendStringInfoChar(inBuf, (char) '\0'); + firstchar = 'Q'; + } + + } + + if (ignore_till_sync && firstchar != EOF) + return; + + #include "pg_proto.c" +} + +#include "./interactive_one.c" + + +void +PostgresSingleUserMain(int argc, char *argv[], + const char *username) +{ + while(1){}; +} + +#else // defined(PG_MAIN) + +extern bool is_node; +extern bool is_repl; + +extern bool quote_all_identifiers; + +#if defined(__EMSCRIPTEN__) || defined(__wasi__) +#include /* chdir */ +#include /* mkdir */ +static +void mkdirp(const char *p) { + if (!mkdir(p, 0700)) { + fprintf(stderr, "# no '%s' directory, creating one ...\n", p); + } +} +#endif /* wasm */ + + +#if defined(PG_INITDB_MAIN) || defined(__wasi__) +extern int pg_initdb_main(void); + +extern void RePostgresSingleUserMain(int single_argc, char *single_argv[], const char *username); +extern void AsyncPostgresSingleUserMain(int single_argc, char *single_argv[], const char *username, int async_restart); +extern void main_post(void); +extern void proc_exit(int code); +extern bool IsPostmasterEnvironment; + +extern volatile int pg_idb_status; +#if PGDEBUG +void print_bits(size_t const size, void const * const ptr); +void print_bits(size_t const size, void const * const ptr) +{ + unsigned char *b = (unsigned char*) ptr; + unsigned char byte; + int i, j; + + for (i = size-1; i >= 0; i--) { + for (j = 7; j >= 0; j--) { + byte = (b[i] >> j) & 1; + printf("%u", byte); + } + } + puts(""); +} +#endif // PGDEBUG +EMSCRIPTEN_KEEPALIVE int +pg_initdb() { + PDEBUG("# 1066: pg_initdb()"); + optind = 1; + int async_restart = 1; + pg_idb_status |= IDB_FAILED; + + if (!chdir(getenv("PGDATA"))){ + if (access("PG_VERSION", F_OK) == 0) { + chdir("/"); + + pg_idb_status |= IDB_HASDB; + + /* assume auth success for now */ + pg_idb_status |= IDB_HASUSER; +#if PGDEBUG + printf("# 1080: pg_initdb: db exists at : %s TODO: test for db name : %s \n", getenv("PGDATA"), getenv("PGDATABASE")); + print_bits(sizeof(pg_idb_status), &pg_idb_status); +#endif // PGDEBUG + main_post(); + async_restart = 0; + { + char *single_argv[] = { + WASM_PREFIX "/bin/postgres", + "--single", + "-d", "1", "-B", "16", "-S", "512", "-f", "siobtnmh", + "-D", getenv("PGDATA"), + "-F", "-O", "-j", + WASM_PGOPTS, + getenv("PGDATABASE"), + NULL + }; + int single_argc = sizeof(single_argv) / sizeof(char*) - 1; + optind = 1; + AsyncPostgresSingleUserMain(single_argc, single_argv, strdup(getenv("PGUSER")), async_restart); + } + + goto initdb_done; + } + chdir("/"); +#if PGDEBUG + printf("pg_initdb: no db found at : %s\n", getenv("PGDATA") ); +#endif // PGDEBUG + } +#if PGDEBUG + PDEBUG("# 1080"); + printf("# pg_initdb_main result = %d\n", pg_initdb_main() ); +#else + pg_initdb_main(); +#endif // PGDEBUG + + /* save stdin and use previous initdb output to feed boot mode */ + int saved_stdin = dup(STDIN_FILENO); + { + PDEBUG("# 1118: restarting in boot mode for initdb"); + freopen(IDB_PIPE_BOOT, "r", stdin); + + char *boot_argv[] = { + WASM_PREFIX "/bin/postgres", + "--boot", + "-D", getenv("PGDATA"), + "-d","3", + WASM_PGOPTS, + "-X", "1048576", + NULL + }; + int boot_argc = sizeof(boot_argv) / sizeof(char*) - 1; + + set_pglocale_pgservice(boot_argv[0], PG_TEXTDOMAIN("initdb")); + + optind = 1; + BootstrapModeMain(boot_argc, boot_argv, false); + fclose(stdin); +#if PGDEBUG + puts("# 886: keep " IDB_PIPE_BOOT ); +#else + remove(IDB_PIPE_BOOT); +#endif + stdin = fdopen(saved_stdin, "r"); + /* fake a shutdown to comlplete WAL/OID states */ + proc_exit(66); + } + + /* use previous initdb output to feed single mode */ + + + /* or resume a previous db */ + //IsPostmasterEnvironment = true; + if (ShmemVariableCache->nextOid < ((Oid) FirstNormalObjectId)) { +#if PGDEBUG + puts("# 891: warning oid base too low, will need to set OID range after initdb(bootstrap/single)"); +#endif + } + + { + PDEBUG("# 889: restarting in single mode for initdb"); + + char *single_argv[] = { + WASM_PREFIX "/bin/postgres", + "--single", + "-d", "1", "-B", "16", "-S", "512", "-f", "siobtnmh", + "-D", getenv("PGDATA"), + "-F", "-O", "-j", + WASM_PGOPTS, + "template1", + NULL + }; + int single_argc = sizeof(single_argv) / sizeof(char*) - 1; + optind = 1; + RePostgresSingleUserMain(single_argc, single_argv, strdup( getenv("PGUSER"))); + } + +initdb_done:; + pg_idb_status |= IDB_CALLED; + IsPostmasterEnvironment = true; + if (ShmemVariableCache->nextOid < ((Oid) FirstNormalObjectId)) { + /* IsPostmasterEnvironment is now true + these will be executed when required in varsup.c/GetNewObjectId + ShmemVariableCache->nextOid = FirstNormalObjectId; + ShmemVariableCache->oidCount = 0; + */ +#if PGDEBUG + puts("# 922: initdb done, oid base too low but OID range will be set because IsPostmasterEnvironment"); +#endif + } + + if (optind>0) { + /* RESET getopt */ + optind = 1; + /* we did not fail, clear the default failed state */ + pg_idb_status &= IDB_OK; + } else { + PDEBUG("# exiting on initdb-single error"); + // TODO raise js exception + } + return pg_idb_status; +} + + +#endif // PG_INITDB_MAIN + +#define PGDB WASM_PREFIX "/base" + +EM_JS(int, is_web_env, (), { + try { + if (window) return 1; + } catch(x) {return 0} +}); + +static void +main_pre(int argc, char *argv[]) { + + + char key[256]; + int i=0; +// extra env is always after normal args + PDEBUG("# ============= extra argv dump =================="); + { + for (;ienv dump =================="); + { + for (;i255) { + puts("buffer overrun on extra env at:"); + puts(kv); + continue; + } + if (kv[sk]=='=') { + memcpy(key, kv, sk); + key[sk] = 0; +#if PGDEBUG + printf("%s='%s'\n", &(key[0]), &(kv[sk+1])); +#endif + setenv(key, &kv[sk+1], 1); + } + } + } + } + PDEBUG("\n# ========================================="); + + argv[0] = strdup(WASM_PREFIX "/bin/postgres"); + + +#if defined(__EMSCRIPTEN__) + EM_ASM({ + Module.is_worker = (typeof WorkerGlobalScope !== 'undefined') && self instanceof WorkerGlobalScope; + Module.FD_BUFFER_MAX = $0; + Module.emscripten_copy_to = console.warn; + }, FD_BUFFER_MAX); /* ( global mem start / num fd max ) */ + + if (is_node) { + setenv("ENVIRONMENT", "node" , 1); + EM_ASM({ +#if PGDEBUG + console.warn("prerun(C-node) worker=", Module.is_worker); +#endif + Module['postMessage'] = function custom_postMessage(event) { + console.log("# 1252: onCustomMessage:",__FILE__, event); + }; + }); + + } else { + setenv("ENVIRONMENT", "web" , 1); +#if PGDEBUG + EM_ASM({ + console.warn("prerun(C-web) worker=", Module.is_worker); + }); +#endif + is_repl = true; + } + + EM_ASM({ + if (Module.is_worker) { +#if PGDEBUG + console.log("Main: running in a worker, setting onCustomMessage"); +#endif + function onCustomMessage(event) { + console.log("onCustomMessage:", event); + }; + Module['onCustomMessage'] = onCustomMessage; + } else { +#if PGDEBUG + console.log("Running in main thread, faking onCustomMessage"); +#endif + Module['postMessage'] = function custom_postMessage(event) { + switch (event.type) { + case "raw" : { + stringToUTF8( event.data, shm_rawinput, Module.FD_BUFFER_MAX); + break; + } + + case "stdin" : { + stringToUTF8( event.data, 1, Module.FD_BUFFER_MAX); + break; + } + case "rcon" : { + stringToUTF8( event.data, shm_rcon, Module.FD_BUFFER_MAX); + break; + } + default : console.warn("custom_postMessage?", event); + } + }; + //if (!window.vm) + // window.vm = Module; + }; + }); + +#endif // __EMSCRIPTEN__ + chdir("/"); + mkdirp("/tmp"); + mkdirp(WASM_PREFIX); + + // postgres does not know where to find the server configuration file. + // also we store the fake locale file there. + // postgres.js:1605 You must specify the --config-file or -D invocation option or set the PGDATA environment variable. + + /* enforce ? */ + setenv("PGSYSCONFDIR", WASM_PREFIX, 1); + setenv("PGCLIENTENCODING", "UTF8", 1); + + // default is to run a repl loop + setenv("REPL", "Y", 0); +/* + * we cannot run "locale -a" either from web or node. the file getenv("PGSYSCONFDIR") / "locale" + * serves as popen output + */ + + setenv("LC_CTYPE", "C" , 1); + + /* defaults */ + + setenv("TZ", "UTC", 0); + setenv("PGTZ", "UTC", 0); + setenv("PGUSER", WASM_USERNAME , 0); + setenv("PGDATA", PGDB , 0); + setenv("PGDATABASE", "template1" , 0); + setenv("PG_COLOR", "always", 0); + +#if PGDEBUG + puts("# ============= env dump =================="); + for (char **env = environ; *env != 0; env++) { + char *drefp = *env; + printf("# %s\n", drefp); + } + puts("# ========================================="); +#endif +} + +int g_argc; +char **g_argv; + +void main_post(); +void main_post() { + /* + * Fire up essential subsystems: error and memory management + * + * Code after this point is allowed to use elog/ereport, though + * localization of messages may not work right away, and messages won't go + * anywhere but stderr until GUC settings get loaded. + */ + MemoryContextInit(); + + /* + * Set up locale information + */ + set_pglocale_pgservice(g_argv[0], PG_TEXTDOMAIN("postgres")); + + /* + * In the postmaster, absorb the environment values for LC_COLLATE and + * LC_CTYPE. Individual backends will change these later to settings + * taken from pg_database, but the postmaster cannot do that. If we leave + * these set to "C" then message localization might not work well in the + * postmaster. + */ + init_locale("LC_COLLATE", LC_COLLATE, ""); + init_locale("LC_CTYPE", LC_CTYPE, ""); + + /* + * LC_MESSAGES will get set later during GUC option processing, but we set + * it here to allow startup error messages to be localized. + */ + #ifdef LC_MESSAGES + init_locale("LC_MESSAGES", LC_MESSAGES, ""); + #endif + + /* + * We keep these set to "C" always, except transiently in pg_locale.c; see + * that file for explanations. + */ + init_locale("LC_MONETARY", LC_MONETARY, "C"); + init_locale("LC_NUMERIC", LC_NUMERIC, "C"); + init_locale("LC_TIME", LC_TIME, "C"); + + /* + * Now that we have absorbed as much as we wish to from the locale + * environment, remove any LC_ALL setting, so that the environment + * variables installed by pg_perm_setlocale have force. + */ + unsetenv("LC_ALL"); +} +EMSCRIPTEN_KEEPALIVE void __cxa_throw(void *thrown_exception, void *tinfo, void *dest); +EMSCRIPTEN_KEEPALIVE void +__cxa_throw(void *thrown_exception, void *tinfo, void *dest) {} + +/* +EMSCRIPTEN_KEEPALIVE void * +_ZNSt13runtime_errorD1Ev(void * int32) { + return NULL; +} +*/ + + +extern void AsyncPostgresSingleUserMain(int single_argc, char *single_argv[], const char *username, int async_restart); + + +#if defined(__wasi__) + +//# include "wasi_signal.c" + +# define PG_INITDB_MAIN +# define PG_MAIN + +# define FRONTEND + #include "../postgresql/src/common/logging.c" +# undef FRONTEND + + #define icu_language_tag(loc_str) icu_language_tag_idb(loc_str) + #define icu_validate_locale(loc_str) icu_validate_locale_idb(loc_str) + #include "../postgresql/src/interfaces/libpq/pqexpbuffer.c" + #define fsync_pgdata(...) + + #include "../postgresql/src/bin/initdb/initdb.c" + + void use_socketfile(void) { + is_repl = true; + is_node = true; + } +#undef PG_INITDB_MAIN +#undef PG_MAIN +#endif // __wasi__ +EMSCRIPTEN_KEEPALIVE int main_repl(); +EMSCRIPTEN_KEEPALIVE int +main_repl() { + bool hadloop_error = false; + + whereToSendOutput = DestNone; + + if (!mkdir(PGDB, 0700)) { + /* no db : run initdb now. */ +#if PGDEBUG + fprintf(stderr, "PGDATA=%s not found, running initdb with defaults\n", PGDB ); +#endif + #if defined(PG_INITDB_MAIN) + #warning "web build" +puts("1168"); + hadloop_error = pg_initdb() & IDB_FAILED; + #else + #warning "node build" + #if defined(__wasi__) + hadloop_error = pg_initdb() & IDB_FAILED; + #endif + #endif + + } else { + // download a db case ? + mkdirp(PGDB); + + // db fixup because empty dirs are not packaged + /* + mkdirp(WASM_PREFIX "/lib"); + mkdirp(WASM_PREFIX "/lib/postgresql"); + */ + mkdirp(PGDB "/pg_wal"); + mkdirp(PGDB "/pg_wal/archive_status"); + mkdirp(PGDB "/pg_wal/summaries"); + + mkdirp(PGDB "/pg_tblspc"); + mkdirp(PGDB "/pg_snapshots"); + mkdirp(PGDB "/pg_commit_ts"); + mkdirp(PGDB "/pg_notify"); + mkdirp(PGDB "/pg_replslot"); + mkdirp(PGDB "/pg_twophase"); + + + mkdirp(PGDB "/pg_logical"); + mkdirp(PGDB "/pg_logical/snapshots"); + mkdirp(PGDB "/pg_logical/mappings"); + + } + + if (!hadloop_error) { + main_post(); + + /* + * Catch standard options before doing much else, in particular before we + * insist on not being root. + */ + if (g_argc > 1) { + if (strcmp(g_argv[1], "--help") == 0 || strcmp(g_argv[1], "-?") == 0) + { + help(progname); + exit(0); + } + if (strcmp(g_argv[1], "--version") == 0 || strcmp(g_argv[1], "-V") == 0) + { + fputs(PG_BACKEND_VERSIONSTR, stdout); + exit(0); + } + + } + + if (g_argc > 1 && strcmp(g_argv[1], "--check") == 0) { + BootstrapModeMain(g_argc, g_argv, true); + return 0; + } + + if (g_argc > 1 && strcmp(g_argv[1], "--boot") == 0) { + PDEBUG("# 1410: boot: " __FILE__ ); + BootstrapModeMain(g_argc, g_argv, false); + return 0; + } + + PDEBUG("# 1415: single: " __FILE__ ); + AsyncPostgresSingleUserMain(g_argc, g_argv, strdup(getenv("PGUSER")), 0); + } + return 0; +} + +extern void pg_repl_raf(void); + +int +main(int argc, char **argv) +{ + int ret=0; + is_node = !is_web_env(); + + main_pre(argc, argv); +#if PGDEBUG + printf("# 1249: argv0 (%s) PGUSER=%s PGDATA=%s\n PGDATABASE=%s REPL=%s\n", + argv[0], getenv("PGUSER"), getenv("PGDATA"), getenv("PGDATABASE"), getenv("REPL") ); +#endif + progname = get_progname(argv[0]); + + /* + PGDATESTYLE + TZ + PG_SHMEM_ADDR + + PGCTLTIMEOUT + PG_TEST_USE_UNIX_SOCKETS + INITDB_TEMPLATE + PSQL_HISTORY + TMPDIR + PGOPTIONS + */ + + /* + * Platform-specific startup hacks + */ + startup_hacks(progname); + + /* + * Remember the physical location of the initially given argv[] array for + * possible use by ps display. On some platforms, the argv[] storage must + * be overwritten in order to set the process title for ps. In such cases + * save_ps_display_args makes and returns a new copy of the argv[] array. + * + * save_ps_display_args may also move the environment strings to make + * extra room. Therefore this should be done as early as possible during + * startup, to avoid entanglements with code that might save a getenv() + * result pointer. + */ + argv = save_ps_display_args(argc, argv); + g_argv = argv; + g_argc = argc; + + is_repl = strlen(getenv("REPL")) && getenv("REPL")[0]=='Y'; + if (!is_repl) { + PDEBUG("# 1289: exit with live runtime (nodb)"); + return 0; + } + PDEBUG("# 1292: repl"); + // so it is repl + main_repl(); + if (is_node) { + PDEBUG("# 1296: node repl"); + pg_repl_raf(); + } + emscripten_force_exit(ret); + return ret; +} + +#endif // PG_MAIN diff --git a/src/pglite-extra/pg_plugin.h b/src/pglite-extra/pg_plugin.h new file mode 100644 index 0000000000000..d0ffeca1ea819 --- /dev/null +++ b/src/pglite-extra/pg_plugin.h @@ -0,0 +1,10 @@ +#ifndef PG_EXTERN +#define PG_EXTERN + +/* + * TODO: async fetch/compilation of a pre-selected plugin + * with emscripten_run_preload_plugins() + */ + + +#endif diff --git a/src/pglite-extra/pg_proto.c b/src/pglite-extra/pg_proto.c new file mode 100644 index 0000000000000..ff3dd2f316593 --- /dev/null +++ b/src/pglite-extra/pg_proto.c @@ -0,0 +1,274 @@ +/* + * this file is used by both interactive_file ( initdb boot/single ) + * and interactive_one() + * + */ + switch (firstchar) + { + case 'Q': /* simple query */ + { + const char *query_string; + + /* Set statement_timestamp() */ + SetCurrentStatementStartTimestamp(); + + query_string = pq_getmsgstring(&input_message); + pq_getmsgend(&input_message); + + if (am_walsender) + { + if (!exec_replication_command(query_string)) + exec_simple_query(query_string); + } + else + exec_simple_query(query_string); + + send_ready_for_query = true; + if (!single_mode_feed) + fprintf(stdout,"pg> %c\n", 4); + } + break; + + case 'P': /* parse */ + { + const char *stmt_name; + const char *query_string; + int numParams; + Oid *paramTypes = NULL; + + forbidden_in_wal_sender(firstchar); + + /* Set statement_timestamp() */ + SetCurrentStatementStartTimestamp(); + + stmt_name = pq_getmsgstring(&input_message); + query_string = pq_getmsgstring(&input_message); + numParams = pq_getmsgint(&input_message, 2); + if (numParams > 0) + { + paramTypes = palloc_array(Oid, numParams); + for (int i = 0; i < numParams; i++) + paramTypes[i] = pq_getmsgint(&input_message, 4); + } + pq_getmsgend(&input_message); + + exec_parse_message(query_string, stmt_name, + paramTypes, numParams); + + //valgrind_report_error_query(query_string); + } + break; + + case 'B': /* bind */ + forbidden_in_wal_sender(firstchar); + + /* Set statement_timestamp() */ + SetCurrentStatementStartTimestamp(); + + /* + * this message is complex enough that it seems best to put + * the field extraction out-of-line + */ + exec_bind_message(&input_message); + + /* exec_bind_message does valgrind_report_error_query */ + break; + + case 'E': /* execute */ + { + const char *portal_name; + int max_rows; + + forbidden_in_wal_sender(firstchar); + + /* Set statement_timestamp() */ + SetCurrentStatementStartTimestamp(); + + portal_name = pq_getmsgstring(&input_message); + max_rows = pq_getmsgint(&input_message, 4); + pq_getmsgend(&input_message); + + exec_execute_message(portal_name, max_rows); + + /* exec_execute_message does valgrind_report_error_query */ + } + break; + + case 'F': /* fastpath function call */ + forbidden_in_wal_sender(firstchar); + + /* Set statement_timestamp() */ + SetCurrentStatementStartTimestamp(); + + /* Report query to various monitoring facilities. */ + pgstat_report_activity(STATE_FASTPATH, NULL); + set_ps_display(""); + + /* start an xact for this function invocation */ + start_xact_command(); + + /* + * Note: we may at this point be inside an aborted + * transaction. We can't throw error for that until we've + * finished reading the function-call message, so + * HandleFunctionRequest() must check for it after doing so. + * Be careful not to do anything that assumes we're inside a + * valid transaction here. + */ + + /* switch back to message context */ + MemoryContextSwitchTo(MessageContext); + + HandleFunctionRequest(&input_message); + + /* commit the function-invocation transaction */ + finish_xact_command(); + + // valgrind_report_error_query("fastpath function call"); + + send_ready_for_query = true; + break; + + case 'C': /* close */ + { + int close_type; + const char *close_target; + + forbidden_in_wal_sender(firstchar); + + close_type = pq_getmsgbyte(&input_message); + close_target = pq_getmsgstring(&input_message); + pq_getmsgend(&input_message); + + switch (close_type) + { + case 'S': + if (close_target[0] != '\0') + DropPreparedStatement(close_target, false); + else + { + /* special-case the unnamed statement */ + drop_unnamed_stmt(); + } + break; + case 'P': + { + Portal portal; + + portal = GetPortalByName(close_target); + if (PortalIsValid(portal)) + PortalDrop(portal, false); + } + break; + default: + ereport(ERROR, + (errcode(ERRCODE_PROTOCOL_VIOLATION), + errmsg("invalid CLOSE message subtype %d", + close_type))); + break; + } + + if (whereToSendOutput == DestRemote) + pq_putemptymessage('3'); /* CloseComplete */ + + //valgrind_report_error_query("CLOSE message"); + } + break; + + case 'D': /* describe */ + { + int describe_type; + const char *describe_target; + + forbidden_in_wal_sender(firstchar); + + /* Set statement_timestamp() (needed for xact) */ + SetCurrentStatementStartTimestamp(); + + describe_type = pq_getmsgbyte(&input_message); + describe_target = pq_getmsgstring(&input_message); + pq_getmsgend(&input_message); + + switch (describe_type) + { + case 'S': + exec_describe_statement_message(describe_target); + break; + case 'P': + exec_describe_portal_message(describe_target); + break; + default: + ereport(ERROR, + (errcode(ERRCODE_PROTOCOL_VIOLATION), + errmsg("invalid DESCRIBE message subtype %d", + describe_type))); + break; + } + + // valgrind_report_error_query("DESCRIBE message"); + } + break; + + case 'H': /* flush */ + pq_getmsgend(&input_message); + if (whereToSendOutput == DestRemote) + pq_flush(); + break; + + case 'S': /* sync */ + pq_getmsgend(&input_message); + finish_xact_command(); + //valgrind_report_error_query("SYNC message"); + send_ready_for_query = true; + break; + + /* + * 'X' means that the frontend is closing down the socket. EOF + * means unexpected loss of frontend connection. Either way, + * perform normal shutdown. + */ + case EOF: + + /* for the cumulative statistics system */ + pgStatSessionEndCause = DISCONNECT_CLIENT_EOF; + + /* FALLTHROUGH */ + + case 'X': + + /* + * Reset whereToSendOutput to prevent ereport from attempting + * to send any more messages to client. + */ + if (whereToSendOutput == DestRemote) + whereToSendOutput = DestNone; + + /* + * NOTE: if you are tempted to add more code here, DON'T! + * Whatever you had in mind to do should be set up as an + * on_proc_exit or on_shmem_exit callback, instead. Otherwise + * it will fail to be called during other backend-shutdown + * scenarios. + */ +// puts("# 697:proc_exit/repl/skip"); //proc_exit(0); + repl = false; + return; + + case 'd': /* copy data */ + case 'c': /* copy done */ + case 'f': /* copy fail */ + + /* + * Accept but ignore these messages, per protocol spec; we + * probably got here because a COPY failed, and the frontend + * is still sending data. + */ + break; + + default: + ereport(FATAL, + (errcode(ERRCODE_PROTOCOL_VIOLATION), + errmsg("invalid frontend message type %d", + firstchar))); + } // end switch + diff --git a/src/pglite-extra/wasi_dlfcn.c b/src/pglite-extra/wasi_dlfcn.c new file mode 100644 index 0000000000000..44dac9e5e3cd2 --- /dev/null +++ b/src/pglite-extra/wasi_dlfcn.c @@ -0,0 +1,198 @@ +// dict helper for dlfcn tables +// TODO: use QSTR + +#include "postgres.h" + +#include "fmgr.h" + +typedef struct dict_entry_s { + const char *key; + int value; +} dict_entry_s; + +typedef struct dict_s { + int len; + int cap; + dict_entry_s *entry; +} dict_s, *dict_t; + +static int +dict_find_index(dict_t dict, const char *key) { + for (int i = 0; i < dict->len; i++) { + if (!strcmp(dict->entry[i].key, key)) { + return i; + } + } + return -1; +} + +static int +dict_find(dict_t dict, const char *key, int def) { + int idx = dict_find_index(dict, key); + return idx == -1 ? def : dict->entry[idx].value; +} + +static void +dict_add(dict_t dict, const char *key, int value) { + int idx = dict_find_index(dict, key); + if (idx != -1) { + dict->entry[idx].value = value; + return; + } + if (dict->len == dict->cap) { + dict->cap *= 2; + dict->entry = realloc(dict->entry, dict->cap * sizeof(dict_entry_s)); + } + dict->entry[dict->len].key = strdup(key); + dict->entry[dict->len].value = value; + dict->len++; +} + +static dict_t +dict_new(void) { + dict_s proto = {0, 10, malloc(10 * sizeof(dict_entry_s))}; + dict_t d = malloc(sizeof(dict_s)); + *d = proto; + return d; +} + +/* dlclose stub +static void +dict_free(dict_t dict) { + for (int i = 0; i < dict->len; i++) { + free(dict->entry[i].key); + } + free(dict->entry); + free(dict); +} +*/ + +static inline int +ends_with(const char *str, const char *suffix) +{ + if (!str || !suffix) + return 0; + size_t lenstr = strlen(str); + size_t lensuffix = strlen(suffix); + if (lensuffix > lenstr) + return 0; + return strncmp(str + lenstr - lensuffix, suffix, lensuffix) == 0; +} + +// dlfcn.h + +volatile dict_t* dltab[]; +volatile int dltab_index = 0; + +void * +sym_stub(void) { + puts("DLSYM STUB"); + return NULL; +} + +char * +dlerror(void) { + return (char *)dlerror; +} + +static const Pg_magic_struct Pg_magic_data = PG_MODULE_MAGIC_DATA; + +const Pg_magic_struct * +STUB_Pg_magic_func(void) { + return &Pg_magic_data; +} + +void STUB__PG_init(void) { +} + +void +STUB__PG_fini(void) { +} + +#define PG_FUNCTION_ARGS FunctionCallInfo fcinfo + +extern Datum dsnowball_init(PG_FUNCTION_ARGS); +extern void pg_finfo_dsnowball_init(void); +extern void pg_finfo_dsnowball_lexize(void); + +extern void pg_finfo_plpgsql_call_handler(void); +extern void pg_finfo_plpgsql_inline_handler(void); +extern void pg_finfo_plpgsql_validator(void); + +// listed in .sql +extern void plpgsql_call_handler(void); +extern void plpgsql_inline_handler(void); +extern void plpgsql_validator(void); + +extern void _PG_init(void); + + +void * +dlopen(const char *filename, int flags) { + dict_t tab = NULL; + fprintf(stderr,"void *dlopen(const char *filename = %s, int flags=%d)\n", filename, flags); + for (int i=0; i< dltab_index; i++) { + if ( dict_find_index(dltab[i], filename) > 0 ) + return (void *)i; + } + printf("dlopen: new lib '%s'\n", filename ); + if ( ends_with(filename,"/plpgsql.so") ){ + puts(" ========= CALLING _PG_init ========="); + _PG_init(); + } + + tab = dict_new(); + dict_add(tab, filename, dltab_index++ ); + dltab[dltab_index] = tab; + + return (void *)dltab_index; +} + +void * +dlsym(void *__restrict handle, const char *__restrict symbol) { + void *sym = NULL; + if ( !strcmp(symbol, "Pg_magic_func") ) { + sym = &STUB_Pg_magic_func; + goto report; + } + + if ( !strcmp(symbol, "_PG_init") ) + // sym = &STUB__PG_init; + return &STUB__PG_init; + + if ( !strcmp(symbol, "dsnowball_init") ) + //sym = &dsnowball_init; + return &dsnowball_init; + + if ( !strcmp(symbol, "pg_finfo_dsnowball_init") ) + return &pg_finfo_dsnowball_init; + + if ( !strcmp(symbol, "pg_finfo_dsnowball_lexize") ) + return &pg_finfo_dsnowball_lexize; + + if ( !strcmp(symbol, "pg_finfo_plpgsql_call_handler") ) + return &pg_finfo_plpgsql_call_handler; + + if ( !strcmp(symbol, "pg_finfo_plpgsql_inline_handler") ) + return &pg_finfo_plpgsql_inline_handler; + + if ( !strcmp(symbol, "pg_finfo_plpgsql_validator") ) + return &pg_finfo_plpgsql_validator; + + if ( !strcmp(symbol, "plpgsql_call_handler") ) + return &plpgsql_call_handler; + + if ( !strcmp(symbol, "plpgsql_inline_handler") ) + return &plpgsql_inline_handler; + + if ( !strcmp(symbol, "plpgsql_validator") ) { + sym = &plpgsql_validator; + goto report; + } + +report:; + fprintf(stderr, "void *dlsym(void *handle = %p, const char *symbol = %s) => %p\n", handle, symbol, sym); + return sym; +} + + diff --git a/src/pglite-extra/wasi_port.c b/src/pglite-extra/wasi_port.c new file mode 100644 index 0000000000000..14fd3638bbf8d --- /dev/null +++ b/src/pglite-extra/wasi_port.c @@ -0,0 +1,597 @@ + +// WIP: signal here +// ================================================================== +#include + +/* Convenience type when working with signal handlers. */ +typedef void (*sa_handler_t) (int); + +/* Return the handler of a signal, as a sa_handler_t value regardless + of its true type. The resulting function can be compared to + special values like SIG_IGN but it is not portable to call it. */ +// static inline sa_handler_t; + +/* +struct sigaction { + __sighandler_t sa_handler; + unsigned long sa_flags; +#ifdef SA_RESTORER + __sigrestore_t sa_restorer; +#endif + sigset_t sa_mask; +}; +*/ + +#ifdef SIGABRT_COMPAT +#define SIGABRT_COMPAT_MASK (1U << SIGABRT_COMPAT) +#else +#define SIGABRT_COMPAT_MASK 0 +#endif + + +/* Present to allow compilation, but unsupported by gnulib. */ +#if 0 +union sigval +{ + int sival_int; + void *sival_ptr; +}; + + +struct siginfo_t +{ + int si_signo; + int si_code; + int si_errno; + pid_t si_pid; + uid_t si_uid; + void *si_addr; + int si_status; + long si_band; + union sigval si_value; +}; + +typedef struct siginfo_t siginfo_t; + +struct sigaction_x +{ + union + { + void (*_sa_handler) (int); + /* Present to allow compilation, but unsupported by gnulib. POSIX + says that implementations may, but not must, make sa_sigaction + overlap with sa_handler, but we know of no implementation where + they do not overlap. */ + void (*_sa_sigaction) (int, siginfo_t *, void *); + } _sa_func; + sigset_t sa_mask; + /* Not all POSIX flags are supported. */ + int sa_flags; +}; +#endif // 0 + +#define sa_handler _sa_func._sa_handler +#define sa_sigaction _sa_func._sa_sigaction + +/* Unsupported flags are not present. */ +#define SA_RESETHAND 1 +#define SA_NODEFER 2 +#define SA_RESTART 4 + +#define SIG_BLOCK 0 +#define SIG_UNBLOCK 1 + + + +/* Set of current actions. If sa_handler for an entry is NULL, then + that signal is not currently handled by the sigaction handler. */ +struct sigaction volatile action_array[NSIG] /* = 0 */; + +// typedef void (*__sighandler_t) (int); + +# define _SIGSET_NWORDS (1024 / (8 * sizeof (unsigned long int))) + + +/* Set of currently blocked signals. */ +volatile sigset_t blocked_set /* = 0 */; + +/* Set of currently blocked and pending signals. */ +volatile sig_atomic_t pending_array[NSIG] /* = { 0 } */; + +/* The previous signal handlers. + Only the array elements corresponding to blocked signals are relevant. */ +volatile handler_t old_handlers[NSIG]; + + +int +sigemptyset(sigset_t *set) { + *set = 0; + return 0; +} + +int +sigaction(int signum, const struct sigaction *act, struct sigaction *oldact) { + puts("# 96: sigaction STUB"); + return 0; +} + +int +sigfillset (sigset_t *set) { + *set = ((2U << (NSIG - 1)) - 1) & ~ SIGABRT_COMPAT_MASK; + return 0; +} + +int +sigaddset (sigset_t *set, int sig) { + if (sig >= 0 && sig < NSIG) + { + #ifdef SIGABRT_COMPAT + if (sig == SIGABRT_COMPAT) + sig = SIGABRT; + #endif + + *set |= 1U << sig; + return 0; + } + else + { + errno = EINVAL; + return -1; + } +} + +int +sigdelset (sigset_t *set, int sig) { + if (sig >= 0 && sig < NSIG) + { + #ifdef SIGABRT_COMPAT + if (sig == SIGABRT_COMPAT) + sig = SIGABRT; + #endif + + *set &= ~(1U << sig); + return 0; + } + else + { + errno = EINVAL; + return -1; + } +} + +/* Signal handler that is installed for blocked signals. */ +void +blocked_handler (int sig) +{ + /* Reinstall the handler, in case the signal occurs multiple times + while blocked. There is an inherent race where an asynchronous + signal in between when the kernel uninstalled the handler and + when we reinstall it will trigger the default handler; oh + well. */ + signal (sig, blocked_handler); + if (sig >= 0 && sig < NSIG) + pending_array[sig] = 1; +} + +int +sigprocmask (int operation, const sigset_t *set, sigset_t *old_set) { + if (old_set != NULL) + *old_set = blocked_set; + + if (set != NULL) + { + sigset_t new_blocked_set; + sigset_t to_unblock; + sigset_t to_block; + + switch (operation) + { + case SIG_BLOCK: + new_blocked_set = blocked_set | *set; + break; + case SIG_SETMASK: + new_blocked_set = *set; + break; + case SIG_UNBLOCK: + new_blocked_set = blocked_set & ~*set; + break; + default: + errno = EINVAL; + return -1; + } + to_unblock = blocked_set & ~new_blocked_set; + to_block = new_blocked_set & ~blocked_set; + + if (to_block != 0) + { + int sig; + + for (sig = 0; sig < NSIG; sig++) + if ((to_block >> sig) & 1) + { + pending_array[sig] = 0; + if ((old_handlers[sig] = signal (sig, blocked_handler)) != SIG_ERR) + blocked_set |= 1U << sig; + } + } + + if (to_unblock != 0) + { + sig_atomic_t received[NSIG]; + int sig; + + for (sig = 0; sig < NSIG; sig++) + if ((to_unblock >> sig) & 1) + { + if (signal (sig, old_handlers[sig]) != blocked_handler) + /* The application changed a signal handler while the signal + was blocked, bypassing our rpl_signal replacement. + We don't support this. */ + abort (); + received[sig] = pending_array[sig]; + blocked_set &= ~(1U << sig); + pending_array[sig] = 0; + } + else + received[sig] = 0; + + for (sig = 0; sig < NSIG; sig++) + if (received[sig]) + raise (sig); + } + } + return 0; +} + + +// STUBS +int sigismember(const sigset_t *set, int signum) { + return -1; +} + +int +pthread_sigmask(int how, const sigset_t *set, sigset_t *oldset) { + return 0; +} + +int sigpending(sigset_t *set) { + return -1; +} + +int sigwait(const sigset_t *restrict set, int *restrict sig) { + return 0; +} + +unsigned int alarm(unsigned int seconds) { + return 0; +} + + +// WIP : shm +// ======================================================================================== +volatile int shm_index = 0; + +#include +void get_shm_path(char *tmpnam, const char *name) { + const char *shm = getenv("SHM"); + if (shm) { + printf("# 281 SHM=%s.%d", shm, shm_index); + snprintf(tmpnam, 128, "%s.%d", shm, shm_index++); + } else { + snprintf(tmpnam, 128, "/tmp%s", name); + } +} + +int shm_open(const char *name, int oflag, mode_t mode) { + char tmpnam[128]; + int fd; + get_shm_path(&tmpnam, name); + fd=fileno(fopen(tmpnam, "w+")); + fprintf(stderr, "# 287: shm_open(%s) => %d\n", tmpnam, fd); + return fd; +} + +int shm_unlink(const char *name) { + char tmpnam[128]; + if (getenv("SHM")) { + fprintf(stderr, "# 294: shm_unlink(%s) STUB\n", name); + return 0; + } + get_shm_path(&tmpnam, name); + return remove(tmpnam); +} + + +// popen +// ======================================================================================== + + +#include // FILE+fprintf +extern FILE* IDB_PIPE_FP; +extern FILE* SOCKET_FILE; +extern int SOCKET_DATA; +extern int IDB_STAGE; + + +static inline int +ends_with(const char *str, const char *suffix) +{ + if (!str || !suffix) + return 0; + size_t lenstr = strlen(str); + size_t lensuffix = strlen(suffix); + if (lensuffix > lenstr) + return 0; + return strncmp(str + lenstr - lensuffix, suffix, lensuffix) == 0; +} + +FILE *pg_popen(const char *command, const char *type) { + if ( ends_with(command,"-V") || (IDB_STAGE>1)) { + fprintf(stderr,"# wasi-popen[%s] STUB\n", command); + return stderr; + } + + if (!IDB_STAGE) { + fprintf(stderr,"# wasi-popen[%s] (BOOT)\n", command); + IDB_PIPE_FP = fopen( IDB_PIPE_BOOT, "w"); + IDB_STAGE = 1; + } else { + fprintf(stderr,"# wasi-popen[%s] (SINGLE)\n", command); + IDB_PIPE_FP = fopen( IDB_PIPE_SINGLE, "w"); + IDB_STAGE = 2; + } + + return IDB_PIPE_FP; +} + +int +system_wasi(const char *command) { + fprintf(stderr, "# 164: system('%s')\n", command); + return -1; +} + +// pthread.h +// ======================================================================================== + + + +int pthread_create(pthread_t *restrict thread, + const pthread_attr_t *restrict attr, + void *(*start_routine)(void *), + void *restrict arg) { + puts("# 327: pthread_create STUB"); + return 0; +} + +int pthread_join(pthread_t thread, void **retval) { + return 0; +} + +int pthread_mutex_lock(pthread_mutex_t *mutex) { + return 0; +} + +int pthread_mutex_unlock(pthread_mutex_t *mutex) { + return 0; +} + +void wait(); + +// present in share/wasi-sysroot/lib/wasm32-wasi/libwasi-emulated-signal.a(signal.o) +// void __SIG_IGN(int param) { } + + +FILE *tmpfile(void) { + return fopen(mktemp("/tmp/tmpfile"),"w"); +} + + + + + +// unix socket via file emulation using sched_yiedl for event pump. +// ================================================================================================= + +#include + +volatile int stage = 0; +volatile int fd_queue = 0; +volatile int fd_out=2; +volatile FILE *fd_FILE = NULL; + +// default fd is stderr +int socket(int domain, int type, int protocol) { +#if 0 + printf("# 404 : domain =%d type=%d proto=%d -> FORCE FD to 3 \n", domain , type, protocol); + return 3; +#else + printf("# 408 : domain =%d type=%d proto=%d\n", domain , type, protocol); +#endif + if (domain|AF_UNIX) { + fd_FILE = fopen(PGS_ILOCK, "w"); + if (fd_FILE) { + fd_out = fileno(fd_FILE); + printf("# 414: AF_UNIX sock=%d (fd_sock write) FILE=%s\n", fd_out, PGS_ILOCK); + } else { + printf("# 416: AF_UNIX ERROR OPEN (w/w+) FILE=%s\n", PGS_ILOCK); + abort(); + } + } + return fd_out; +} + +int connect(int socket, void *address, socklen_t address_len) { +#if 1 + puts("# 425: connect STUB"); + //fd_out = 3; + return 0; +#else + puts("# 429: connect EINPROGRESS"); + errno = EINPROGRESS; + return -1; +#endif +} + +ssize_t sendto(int sockfd, const void *buf, size_t len, int flags, void *dest_addr, socklen_t addrlen) { + int sent = write( fd_out, buf, len); + + printf("# 438: send/sendto(%d ?= %d )/%d sockfd=%d fno=%d fd_out=%d)\n", sent, ftell(fd_FILE), len, sockfd, fileno(fd_FILE), fd_out); + fd_queue+=sent; + return sent; +} + +ssize_t send(int sockfd, const void *buf, size_t len, int flags) { + return sendto(sockfd, buf, len, flags, NULL, 0); +} + +volatile bool web_warned = false; + +void sock_flush() { + if (fd_queue) { + printf(" -- 451 sockflush : AIO YIELD, expecting %s filled on return --\n", PGS_OUT); + if (!fd_FILE) { + if (!web_warned) { + puts("# 454: WARNING: fd_FILE not set but queue not empty, assuming web"); + web_warned = true; + } + + } else { + printf("# 459: SENT=%d/%d fd_out=%d fno=%d\n", ftell(fd_FILE), fd_queue, fd_out, fileno(fd_FILE)); + fclose(fd_FILE); + rename(PGS_ILOCK, PGS_IN); +//freopen(PGS_ILOCK, "w+", fd_FILE); + fd_FILE = fopen(PGS_ILOCK, "w"); + fd_out = fileno(fd_FILE); + printf("# 465: fd_out=%d fno=%d\n", fd_out, fileno(fd_FILE)); + } + fd_queue = 0; + sched_yield(); + return; + } + + printf(" -- 472 sockflush[%d] : NO YIELD --\n",stage); + + // limit inf loops + if (stage++ > 1024) { + puts("# 476 sock_flush : busy looping ?"); + abort(); + } +} + + +volatile int fd_current_pos = 0; +volatile int fd_filesize = 0; + + +ssize_t recvfrom_bc(int socket, void *buffer, size_t length, int flags, void *address, socklen_t *address_len) { + int busy = 0; + int rcv = -1; + sock_flush(); +/* + while (access(PGS_OUT, F_OK) != 0) { + if (!(++busy % 555111)) { + printf("# 471: FIXME: busy wait (%d) for input stream %s\n", busy, PGS_OUT); + } + if (busy>1665334) { + errno = EINTR; + return -1; + } + } +*/ + FILE *sock_in = fopen(PGS_OUT,"r"); + if (sock_in) { + if (!fd_filesize) { + fseek(sock_in, 0L, SEEK_END); + fd_filesize = ftell(sock_in); + } + fseek(sock_in, fd_current_pos, SEEK_SET); + + char *buf = buffer; + buf[0] = 0; + rcv = fread(buf, 1, length, sock_in); + + if (rcv1665334) { + errno = EINTR; + return -1; + } + } +*/ + FILE *sock_in = fopen(PGS_OUT,"r"); + if (sock_in) { + if (!fd_filesize) { + fseek(sock_in, 0L, SEEK_END); + fd_filesize = ftell(sock_in); + } + fseek(sock_in, fd_current_pos, SEEK_SET); + + char *buf = buffer; + buf[0] = 0; + rcv = fread(buf, 1, length, sock_in); + + if (rcvmutex); return 0; } +#else +int +pthread_barrier_init(pthread_barrier_t *__restrict barrier, const pthread_barrierattr_t *__restrict attr, unsigned count) { + return 0; +} + +int +pthread_barrier_wait(pthread_barrier_t *barrier) { + return 0; +} + +int +pthread_barrier_destroy(pthread_barrier_t *barrier) { + return 0; +} +#endif + diff --git a/src/template/emscripten b/src/template/emscripten new file mode 100644 index 0000000000000..f3edf627a24e0 --- /dev/null +++ b/src/template/emscripten @@ -0,0 +1,14 @@ +# src/template/emscripten + +# Prefer unnamed POSIX semaphores if available, unless user overrides choice +if test x"$PREFERRED_SEMAPHORES" = x"" ; then + PREFERRED_SEMAPHORES=UNNAMED_POSIX +fi + +# Force _GNU_SOURCE on; plperl is broken with Perl 5.8.0 otherwise +# This is also required for ppoll(2), and perhaps other things +CPPFLAGS="$CPPFLAGS -D_GNU_SOURCE" + +# Extra CFLAGS for code that will go into a shared library +CFLAGS_SL="-fPIC" + diff --git a/src/template/wasi b/src/template/wasi new file mode 100644 index 0000000000000..f3edf627a24e0 --- /dev/null +++ b/src/template/wasi @@ -0,0 +1,14 @@ +# src/template/emscripten + +# Prefer unnamed POSIX semaphores if available, unless user overrides choice +if test x"$PREFERRED_SEMAPHORES" = x"" ; then + PREFERRED_SEMAPHORES=UNNAMED_POSIX +fi + +# Force _GNU_SOURCE on; plperl is broken with Perl 5.8.0 otherwise +# This is also required for ppoll(2), and perhaps other things +CPPFLAGS="$CPPFLAGS -D_GNU_SOURCE" + +# Extra CFLAGS for code that will go into a shared library +CFLAGS_SL="-fPIC" + diff --git a/src/test/regress/pg_regress.c b/src/test/regress/pg_regress.c index 57aa0de3b7adf..bb0bb535170bd 100644 --- a/src/test/regress/pg_regress.c +++ b/src/test/regress/pg_regress.c @@ -37,6 +37,15 @@ #include "pg_regress.h" #include "portability/instr_time.h" +#if defined(__wasi__) +#if defined(HAVE_GETRLIMIT) +#undef HAVE_GETRLIMIT +#endif +#define execl(...) (-1) +#define wait(...) (INVALID_PID) +#define raise(...) +#endif /* __wasi__ */ + /* for resultmap we need a list of pairs of strings */ typedef struct _resultmap {