wip: add pg_duckdb ext

Dockerfile-17-fast

@@ -0,0 +1,210 @@
+# syntax=docker/dockerfile:1.6
+# Alpine-based slim PostgreSQL 17 image with Nix extensions
+# Fast build variant: uses a BuildKit cache mount to persist the Nix store between builds.
+# First build still downloads packages; subsequent builds skip downloads entirely.
+
+####################
+# Stage 1: Nix builder
+####################
+FROM alpine:3.21 AS nix-builder
+
+# Install dependencies for nix installer (coreutils for GNU cp, sudo for installer)
+RUN apk add --no-cache \
+    bash \
+    coreutils \
+    curl \
+    shadow \
+    sudo \
+    xz
+
+# Create users (Alpine syntax)
+RUN addgroup -S postgres && \
+    adduser -S -h /var/lib/postgresql -s /bin/bash -G postgres postgres && \
+    addgroup -S wal-g && \
+    adduser -S -s /bin/bash -G wal-g wal-g
+
+WORKDIR /nixpg
+COPY . .
+
+# Build PostgreSQL and groonga with extensions.
+#
+# --mount=type=cache persists /nix between builds on this machine so Nix does not
+# re-download or rebuild packages that are already in the store.  The cache is keyed
+# by id so it is shared across invalidated layers (e.g. when COPY . . changes).
+#
+# Because cache-mount contents are NOT committed to the image layer we copy the
+# finished store to /nix-output at the end; the production stage COPYs from there.
+#
+# Nix is installed in single-user (--no-daemon) mode so the entire store lives
+# under /nix (the cache mount) and needs no daemon process.
+RUN --mount=type=cache,id=psql17-nix-store,target=/nix,sharing=locked \
+    sh -c ' \
+        set -eu; \
+        \
+        # Write nix.conf every time (it lives in the writable layer, not the cache mount). \
+        # build-users-group must be empty so the single-user installer works as root; \
+        # sandbox=false is required inside Docker containers. \
+        mkdir -p /etc/nix; \
+        printf "build-users-group = \nsandbox = false\nextra-experimental-features = nix-command flakes\nextra-substituters = https://nix-postgres-artifacts.s3.amazonaws.com\nextra-trusted-public-keys = nix-postgres-artifacts:dGZlQOvKcNEjvT7QEAJbcV6b6uk7VF/hWMjhYleiaLI=\n" > /etc/nix/nix.conf; \
+        \
+        if [ ! -f /nix/var/nix/profiles/default/bin/nix ]; then \
+            curl -L https://releases.nixos.org/nix/nix-2.33.2/install | sh -s -- --no-daemon --no-channel-add; \
+        fi; \
+        \
+        export PATH="/nix/var/nix/profiles/default/bin:$PATH"; \
+        \
+        nix profile add path:.#psql_17_slim/bin; \
+        nix store gc; \
+        \
+        nix profile add path:.#supabase-groonga; \
+        nix store gc; \
+        \
+        # Copy the store out of the cache mount so it is committed to the image layer. \
+        mkdir -p /nix-output; \
+        cp -a /nix/. /nix-output/; \
+    ' && \
+    mkdir -p /tmp/groonga-plugins && \
+    cp -r /nix-output/var/nix/profiles/default/lib/groonga/plugins /tmp/groonga-plugins/
+
+####################
+# Stage 2: Gosu builder
+####################
+FROM alpine:3.21 AS gosu-builder
+
+ARG TARGETARCH
+ARG GOSU_VERSION=1.16
+
+RUN apk add --no-cache gnupg curl
+
+# Download and verify gosu
+RUN curl -fsSL "https://github.com/tianon/gosu/releases/download/${GOSU_VERSION}/gosu-${TARGETARCH}" -o /usr/local/bin/gosu && \
+    curl -fsSL "https://github.com/tianon/gosu/releases/download/${GOSU_VERSION}/gosu-${TARGETARCH}.asc" -o /usr/local/bin/gosu.asc && \
+    GNUPGHOME="$(mktemp -d)" && \
+    export GNUPGHOME && \
+    gpg --batch --keyserver hkps://keys.openpgp.org --recv-keys B42F6819007F00F88E364FD4036A9C25BF357DD4 && \
+    gpg --batch --verify /usr/local/bin/gosu.asc /usr/local/bin/gosu && \
+    rm -rf "$GNUPGHOME" /usr/local/bin/gosu.asc && \
+    chmod +x /usr/local/bin/gosu
+
+####################
+# Stage 3: Final production image
+####################
+FROM alpine:3.21 AS production
+
+# Install minimal runtime dependencies
+RUN apk add --no-cache \
+    bash \
+    curl \
+    shadow \
+    su-exec \
+    tzdata \
+    musl-locales \
+    musl-locales-lang \
+    && rm -rf /var/cache/apk/*
+
+# Create postgres user/group
+RUN addgroup -S postgres && \
+    adduser -S -G postgres -h /var/lib/postgresql -s /bin/bash postgres && \
+    addgroup -S wal-g && \
+    adduser -S -G wal-g -s /bin/bash wal-g && \
+    adduser postgres wal-g
+
+# Copy Nix store and profiles from builder (written to /nix-output to escape cache mount)
+COPY --from=nix-builder /nix-output /nix
+
+# Copy groonga plugins
+COPY --from=nix-builder /tmp/groonga-plugins/plugins /usr/lib/groonga/plugins
+
+# Copy gosu
+COPY --from=gosu-builder /usr/local/bin/gosu /usr/local/bin/gosu
+
+# Setup PostgreSQL directories
+RUN mkdir -p /usr/lib/postgresql/bin \
+    /usr/lib/postgresql/share/postgresql \
+    /usr/share/postgresql \
+    /var/lib/postgresql/data \
+    /var/run/postgresql \
+    && chown -R postgres:postgres /usr/lib/postgresql \
+    && chown -R postgres:postgres /var/lib/postgresql \
+    && chown -R postgres:postgres /usr/share/postgresql \
+    && chown -R postgres:postgres /var/run/postgresql
+
+# Create symbolic links for binaries
+RUN for f in /nix/var/nix/profiles/default/bin/*; do \
+        ln -sf "$f" /usr/lib/postgresql/bin/ 2>/dev/null || true; \
+        ln -sf "$f" /usr/bin/ 2>/dev/null || true; \
+    done
+
+# Create symbolic links for PostgreSQL shares
+RUN ln -sf /nix/var/nix/profiles/default/share/postgresql/* /usr/lib/postgresql/share/postgresql/ 2>/dev/null || true && \
+    ln -sf /nix/var/nix/profiles/default/share/postgresql/* /usr/share/postgresql/ 2>/dev/null || true && \
+    ln -sf /usr/lib/postgresql/share/postgresql/timezonesets /usr/share/postgresql/timezonesets 2>/dev/null || true
+
+# Set permissions
+RUN chown -R postgres:postgres /usr/lib/postgresql && \
+    chown -R postgres:postgres /usr/share/postgresql
+
+# Setup configs
+COPY --chown=postgres:postgres ansible/files/postgresql_config/postgresql.conf.j2 /etc/postgresql/postgresql.conf
+COPY --chown=postgres:postgres ansible/files/postgresql_config/pg_hba.conf.j2 /etc/postgresql/pg_hba.conf
+COPY --chown=postgres:postgres ansible/files/postgresql_config/pg_ident.conf.j2 /etc/postgresql/pg_ident.conf
+COPY --chown=postgres:postgres ansible/files/postgresql_config/conf.d /etc/postgresql-custom/conf.d
+COPY --chown=postgres:postgres ansible/files/postgresql_config/postgresql-stdout-log.conf /etc/postgresql/logging.conf
+COPY --chown=postgres:postgres ansible/files/postgresql_config/supautils.conf.j2 /etc/postgresql-custom/supautils.conf
+COPY --chown=postgres:postgres ansible/files/postgresql_extension_custom_scripts /etc/postgresql-custom/extension-custom-scripts
+COPY --chown=postgres:postgres ansible/files/pgsodium_getkey_urandom.sh.j2 /usr/lib/postgresql/bin/pgsodium_getkey.sh
+COPY --chown=postgres:postgres ansible/files/postgresql_config/custom_walg.conf /etc/postgresql-custom/wal-g.conf
+COPY --chown=postgres:postgres ansible/files/postgresql_config/custom_read_replica.conf /etc/postgresql-custom/read-replica.conf
+COPY --chown=postgres:postgres ansible/files/walg_helper_scripts/wal_fetch.sh /home/postgres/wal_fetch.sh
+COPY ansible/files/walg_helper_scripts/wal_change_ownership.sh /root/wal_change_ownership.sh
+
+# Configure PostgreSQL settings
+RUN sed -i \
+    -e "s|#unix_socket_directories = '/tmp'|unix_socket_directories = '/var/run/postgresql'|g" \
+    -e "s|#session_preload_libraries = ''|session_preload_libraries = 'supautils'|g" \
+    -e "s|#include = '/etc/postgresql-custom/supautils.conf'|include = '/etc/postgresql-custom/supautils.conf'|g" \
+    -e "s|#include = '/etc/postgresql-custom/wal-g.conf'|include = '/etc/postgresql-custom/wal-g.conf'|g" /etc/postgresql/postgresql.conf && \
+    echo "pgsodium.getkey_script= '/usr/lib/postgresql/bin/pgsodium_getkey.sh'" >> /etc/postgresql/postgresql.conf && \
+    echo "vault.getkey_script= '/usr/lib/postgresql/bin/pgsodium_getkey.sh'" >> /etc/postgresql/postgresql.conf && \
+    chown -R postgres:postgres /etc/postgresql-custom
+
+# Remove timescaledb and plv8 references (not in pg17)
+RUN sed -i 's/ timescaledb,//g;' "/etc/postgresql/postgresql.conf" && \
+    sed -i 's/db_user_namespace = off/#db_user_namespace = off/g;' "/etc/postgresql/postgresql.conf" && \
+    sed -i 's/ timescaledb,//g; s/ plv8,//g' "/etc/postgresql-custom/supautils.conf"
+
+# Include schema migrations
+COPY migrations/db /docker-entrypoint-initdb.d/
+COPY ansible/files/pgbouncer_config/pgbouncer_auth_schema.sql /docker-entrypoint-initdb.d/init-scripts/00-schema.sql
+COPY ansible/files/stat_extension.sql /docker-entrypoint-initdb.d/migrations/00-extension.sql
+
+# Add entrypoint script
+ADD --chmod=0755 \
+    https://github.com/docker-library/postgres/raw/889f9447cd2dfe21cccfbe9bb7945e3b037e02d8/17/bullseye/docker-entrypoint.sh \
+    /usr/local/bin/docker-entrypoint.sh
+
+# Setup pgsodium key script
+RUN mkdir -p /usr/share/postgresql/extension/ && \
+    ln -s /usr/lib/postgresql/bin/pgsodium_getkey.sh /usr/share/postgresql/extension/pgsodium_getkey && \
+    chmod +x /usr/lib/postgresql/bin/pgsodium_getkey.sh
+
+# Environment variables
+ENV PATH="/nix/var/nix/profiles/default/bin:/usr/lib/postgresql/bin:${PATH}"
+ENV PGDATA=/var/lib/postgresql/data
+ENV POSTGRES_HOST=/var/run/postgresql
+ENV POSTGRES_USER=supabase_admin
+ENV POSTGRES_DB=postgres
+ENV POSTGRES_INITDB_ARGS="--allow-group-access --locale-provider=icu --encoding=UTF-8 --icu-locale=en_US.UTF-8"
+ENV LANG=en_US.UTF-8
+ENV LANGUAGE=en_US:en
+ENV LC_ALL=en_US.UTF-8
+ENV GRN_PLUGINS_DIR=/usr/lib/groonga/plugins
+# Point to minimal glibc locales included in slim Nix package for initdb locale support
+ENV LOCALE_ARCHIVE=/nix/var/nix/profiles/default/lib/locale/locale-archive
+
+ENTRYPOINT ["docker-entrypoint.sh"]
+HEALTHCHECK --interval=2s --timeout=2s --retries=10 CMD pg_isready -U postgres -h localhost
+STOPSIGNAL SIGINT
+EXPOSE 5432
+
+CMD ["postgres", "-D", "/etc/postgresql"]

ansible/files/postgresql_config/postgresql.conf.j2

@@ -687,7 +687,8 @@ default_text_search_config = 'pg_catalog.english'
 #local_preload_libraries = ''
 #session_preload_libraries = ''
 
-shared_preload_libraries = 'pg_stat_statements, pgaudit, plpgsql, plpgsql_check, pg_cron, pg_net, pgsodium, timescaledb, auto_explain, pg_tle, plan_filter, supabase_vault'	# (change requires restart)
+shared_preload_libraries = 'pg_stat_statements, pgaudit, plpgsql, plpgsql_check, pg_cron, pg_net, pgsodium, timescaledb, auto_explain, pg_tle, plan_filter, supabase_vault, pg_duckdb'	# (change requires restart)
+duckdb.postgres_role = 'duckdb_role'	# (change requires restart)
 jit_provider = 'llvmjit'		# JIT library to use
 
 # - Other Defaults -

ansible/files/postgresql_config/supautils.conf.j2

@@ -1,12 +1,12 @@
 supautils.extensions_parameter_overrides = '{"pg_cron":{"schema":"pg_catalog"}}'
 supautils.policy_grants = '{"postgres":["auth.audit_log_entries","auth.flow_state","auth.identities","auth.instances","auth.mfa_amr_claims","auth.mfa_challenges","auth.mfa_factors","auth.oauth_clients","auth.one_time_tokens","auth.refresh_tokens","auth.saml_providers","auth.saml_relay_states","auth.sessions","auth.sso_domains","auth.sso_providers","auth.users","realtime.messages","realtime.subscription","storage.buckets","storage.buckets_analytics","storage.objects","storage.prefixes","storage.s3_multipart_uploads","storage.s3_multipart_uploads_parts"]}'
 supautils.drop_trigger_grants = '{"postgres":["auth.audit_log_entries","auth.flow_state","auth.identities","auth.instances","auth.mfa_amr_claims","auth.mfa_challenges","auth.mfa_factors","auth.oauth_clients","auth.one_time_tokens","auth.refresh_tokens","auth.saml_providers","auth.saml_relay_states","auth.sessions","auth.sso_domains","auth.sso_providers","auth.users","realtime.messages","realtime.subscription","storage.buckets","storage.buckets_analytics","storage.objects","storage.prefixes","storage.s3_multipart_uploads","storage.s3_multipart_uploads_parts"]}'
-# full list:                                 address_standardizer, address_standardizer_data_us, adminpack, amcheck, autoinc, bloom, btree_gin, btree_gist, citext, cube, dblink, dict_int, dict_xsyn, earthdistance, file_fdw, fuzzystrmatch, hstore, http, hypopg, index_advisor, insert_username, intagg, intarray, isn, lo, ltree, moddatetime, old_snapshot, orioledb, pageinspect, pg_buffercache, pg_cron, pg_freespacemap, pg_graphql, pg_hashids, pg_jsonschema, pg_net, pg_prewarm, pg_repack, pg_stat_monitor, pg_stat_statements, pg_surgery, pg_tle, pg_trgm, pg_visibility, pg_walinspect, pgaudit, pgcrypto, pgjwt, pgmq, pgroonga, pgroonga_database, pgrouting, pgrowlocks, pgsodium, pgstattuple, pgtap, plcoffee, pljava, plls, plpgsql, plpgsql_check, plv8, postgis, postgis_raster, postgis_sfcgal, postgis_tiger_geocoder, postgis_topology, postgres_fdw, refint, rum, seg, sslinfo, supabase_vault, supautils, tablefunc, tcn, timescaledb, tsm_system_rows, tsm_system_time, unaccent, uuid-ossp, vector, wrappers, xml2
+# full list:                                 address_standardizer, address_standardizer_data_us, adminpack, amcheck, autoinc, bloom, btree_gin, btree_gist, citext, cube, dblink, dict_int, dict_xsyn, earthdistance, file_fdw, fuzzystrmatch, hstore, http, hypopg, index_advisor, insert_username, intagg, intarray, isn, lo, ltree, moddatetime, old_snapshot, orioledb, pageinspect, pg_buffercache, pg_cron, pg_duckdb, pg_freespacemap, pg_graphql, pg_hashids, pg_jsonschema, pg_net, pg_prewarm, pg_repack, pg_stat_monitor, pg_stat_statements, pg_surgery, pg_tle, pg_trgm, pg_visibility, pg_walinspect, pgaudit, pgcrypto, pgjwt, pgmq, pgroonga, pgroonga_database, pgrouting, pgrowlocks, pgsodium, pgstattuple, pgtap, plcoffee, pljava, plls, plpgsql, plpgsql_check, plv8, postgis, postgis_raster, postgis_sfcgal, postgis_tiger_geocoder, postgis_topology, postgres_fdw, refint, rum, seg, sslinfo, supabase_vault, supautils, tablefunc, tcn, timescaledb, tsm_system_rows, tsm_system_time, unaccent, uuid-ossp, vector, wrappers, xml2
 # omitted because may be unsafe:             adminpack, amcheck, file_fdw, lo, old_snapshot, pageinspect, pg_freespacemap, pg_surgery, pg_visibility
 # omitted because deprecated:                intagg, xml2
 # omitted because doesn't require superuser: pgmq
 # omitted because protected:                 plpgsql
-supautils.privileged_extensions = 'address_standardizer, address_standardizer_data_us, autoinc, bloom, btree_gin, btree_gist, citext, cube, dblink, dict_int, dict_xsyn, earthdistance, fuzzystrmatch, hstore, http, hypopg, index_advisor, insert_username, intarray, isn, ltree, moddatetime, orioledb, pg_buffercache, pg_cron, pg_graphql, pg_hashids, pg_jsonschema, pg_net, pg_prewarm, pg_repack, pg_stat_monitor, pg_stat_statements, pg_tle, pg_trgm, pg_walinspect, pgaudit, pgcrypto, pgjwt, pgroonga, pgroonga_database, pgrouting, pgrowlocks, pgsodium, pgstattuple, pgtap, plcoffee, pljava, plls, plpgsql_check, plv8, postgis, postgis_raster, postgis_sfcgal, postgis_tiger_geocoder, postgis_topology, postgres_fdw, refint, rum, seg, sslinfo, supabase_vault, supautils, tablefunc, tcn, timescaledb, tsm_system_rows, tsm_system_time, unaccent, uuid-ossp, vector, wrappers'
+supautils.privileged_extensions = 'address_standardizer, address_standardizer_data_us, autoinc, bloom, btree_gin, btree_gist, citext, cube, dblink, dict_int, dict_xsyn, earthdistance, fuzzystrmatch, hstore, http, hypopg, index_advisor, insert_username, intarray, isn, ltree, moddatetime, orioledb, pg_buffercache, pg_cron, pg_duckdb, pg_graphql, pg_hashids, pg_jsonschema, pg_net, pg_prewarm, pg_repack, pg_stat_monitor, pg_stat_statements, pg_tle, pg_trgm, pg_walinspect, pgaudit, pgcrypto, pgjwt, pgroonga, pgroonga_database, pgrouting, pgrowlocks, pgsodium, pgstattuple, pgtap, plcoffee, pljava, plls, plpgsql_check, plv8, postgis, postgis_raster, postgis_sfcgal, postgis_tiger_geocoder, postgis_topology, postgres_fdw, refint, rum, seg, sslinfo, supabase_vault, supautils, tablefunc, tcn, timescaledb, tsm_system_rows, tsm_system_time, unaccent, uuid-ossp, vector, wrappers'
 supautils.extension_custom_scripts_path = '/etc/postgresql-custom/extension-custom-scripts'
 supautils.privileged_extensions_superuser = 'supabase_admin'
 supautils.privileged_role = 'supabase_privileged_role'

ansible/files/postgresql_extension_custom_scripts/pg_duckdb/after-create.sql

@@ -0,0 +1 @@
+grant usage on foreign data wrapper duckdb to duckdb_role;

ansible/vars.yml

@@ -10,9 +10,9 @@ postgres_major:
 
 # Full version strings for each major version
 postgres_release:
-  postgresorioledb-17: "17.6.0.061-orioledb"
-  postgres17: "17.6.1.104"
-  postgres15: "15.14.1.104"
+  postgresorioledb-17: "17.6.0.061-orioledb-duckdbv2"
+  postgres17: "17.6.1.104-duckdbv2"
+  postgres15: "15.14.1.104-duckdbv2"
 
 # Non Postgres Extensions
 pgbouncer_release: 1.25.1

migrations/db/migrations/20260309201836_pg_duckdb_grants.sql

@@ -0,0 +1,20 @@
+-- migrate:up
+
+-- Create a shared group role for DuckDB access.
+-- Both postgres (developer/admin) and service_role (runtime API) need to run
+-- DuckDB queries. We use a group role rather than cross-granting between them,
+-- which mirrors the supabase_privileged_role pattern.
+--
+-- The FDW grant (GRANT USAGE ON FOREIGN DATA WRAPPER duckdb TO duckdb_role) is
+-- handled by ansible/files/postgresql_extension_custom_scripts/pg_duckdb/after-create.sql
+-- rather than an event trigger, following the established pattern for postgres_fdw etc.
+DO $$
+BEGIN
+  IF NOT EXISTS (SELECT FROM pg_roles WHERE rolname = 'duckdb_role') THEN
+    CREATE ROLE duckdb_role;
+    GRANT duckdb_role TO postgres WITH ADMIN OPTION;
+    GRANT duckdb_role TO service_role, supabase_admin;
+  END IF;
+END $$;
+
+-- migrate:down

nix/ext/duckdb-lib.nix

@@ -0,0 +1,69 @@
+{
+  lib,
+  stdenv,
+  fetchFromGitHub,
+  cmake,
+  ninja,
+  openssl,
+  python3,
+}:
+
+stdenv.mkDerivation (finalAttrs: {
+  pname = "duckdb";
+  version = "1.5.1";
+
+  src = fetchFromGitHub {
+    owner = "duckdb";
+    repo = "duckdb";
+    rev = "v${finalAttrs.version}";
+    hash = "sha256-FygBpfhvezvUbI969Dta+vZOPt6BnSW2d5gO4I4oB2A=";
+  };
+
+  outputs = [
+    "out"
+    "lib"
+    "dev"
+  ];
+
+  # cmake installs the shared library and headers; nothing goes into $out
+  # since BUILD_SHELL=OFF (no CLI binary). We create $out explicitly so
+  # nixpkgs's multi-output setup hooks have a valid fallback for outputBin,
+  # outputDoc, outputMan, outputInfo, etc.
+  postInstall = ''
+    mkdir -p $out
+  '';
+
+  nativeBuildInputs = [
+    cmake
+    ninja
+    # python3 is required by DuckDB's cmake build scripts for code generation
+    python3
+  ];
+
+  buildInputs = [ openssl ];
+
+  cmakeFlags = [
+    # Required by pg_duckdb so that DuckDB symbols are visible when loaded
+    # by PostgreSQL's dlopen. Without this, pg_duckdb's .so cannot resolve
+    # DuckDB symbols at runtime.
+    "-DCXX_EXTRA=-fvisibility=default"
+    (lib.cmakeBool "BUILD_SHELL" false)
+    (lib.cmakeBool "BUILD_PYTHON" false)
+    (lib.cmakeBool "BUILD_UNITTESTS" false)
+    # Prevent cmake from trying to fetch anything from the internet
+    (lib.cmakeBool "FETCHCONTENT_FULLY_DISCONNECTED" true)
+    # Embed the version string so DuckDB doesn't report "unknown"
+    (lib.cmakeFeature "OVERRIDE_GIT_DESCRIBE" "v${finalAttrs.version}-0-g0000000")
+  ];
+
+  # Skip the test suite — we just want the library
+  doInstallCheck = false;
+  doCheck = false;
+
+  meta = {
+    description = "DuckDB shared library (for use by pg_duckdb)";
+    homepage = "https://duckdb.org/";
+    license = lib.licenses.mit;
+    platforms = lib.platforms.unix;
+  };
+})