diff --git a/Dockerfile-17-fast b/Dockerfile-17-fast new file mode 100644 index 0000000000..cf59360e20 --- /dev/null +++ b/Dockerfile-17-fast @@ -0,0 +1,210 @@ +# syntax=docker/dockerfile:1.6 +# Alpine-based slim PostgreSQL 17 image with Nix extensions +# Fast build variant: uses a BuildKit cache mount to persist the Nix store between builds. +# First build still downloads packages; subsequent builds skip downloads entirely. + +#################### +# Stage 1: Nix builder +#################### +FROM alpine:3.21 AS nix-builder + +# Install dependencies for nix installer (coreutils for GNU cp, sudo for installer) +RUN apk add --no-cache \ + bash \ + coreutils \ + curl \ + shadow \ + sudo \ + xz + +# Create users (Alpine syntax) +RUN addgroup -S postgres && \ + adduser -S -h /var/lib/postgresql -s /bin/bash -G postgres postgres && \ + addgroup -S wal-g && \ + adduser -S -s /bin/bash -G wal-g wal-g + +WORKDIR /nixpg +COPY . . + +# Build PostgreSQL and groonga with extensions. +# +# --mount=type=cache persists /nix between builds on this machine so Nix does not +# re-download or rebuild packages that are already in the store. The cache is keyed +# by id so it is shared across invalidated layers (e.g. when COPY . . changes). +# +# Because cache-mount contents are NOT committed to the image layer we copy the +# finished store to /nix-output at the end; the production stage COPYs from there. +# +# Nix is installed in single-user (--no-daemon) mode so the entire store lives +# under /nix (the cache mount) and needs no daemon process. +RUN --mount=type=cache,id=psql17-nix-store,target=/nix,sharing=locked \ + sh -c ' \ + set -eu; \ + \ + # Write nix.conf every time (it lives in the writable layer, not the cache mount). \ + # build-users-group must be empty so the single-user installer works as root; \ + # sandbox=false is required inside Docker containers. \ + mkdir -p /etc/nix; \ + printf "build-users-group = \nsandbox = false\nextra-experimental-features = nix-command flakes\nextra-substituters = https://nix-postgres-artifacts.s3.amazonaws.com\nextra-trusted-public-keys = nix-postgres-artifacts:dGZlQOvKcNEjvT7QEAJbcV6b6uk7VF/hWMjhYleiaLI=\n" > /etc/nix/nix.conf; \ + \ + if [ ! -f /nix/var/nix/profiles/default/bin/nix ]; then \ + curl -L https://releases.nixos.org/nix/nix-2.33.2/install | sh -s -- --no-daemon --no-channel-add; \ + fi; \ + \ + export PATH="/nix/var/nix/profiles/default/bin:$PATH"; \ + \ + nix profile add path:.#psql_17_slim/bin; \ + nix store gc; \ + \ + nix profile add path:.#supabase-groonga; \ + nix store gc; \ + \ + # Copy the store out of the cache mount so it is committed to the image layer. \ + mkdir -p /nix-output; \ + cp -a /nix/. /nix-output/; \ + ' && \ + mkdir -p /tmp/groonga-plugins && \ + cp -r /nix-output/var/nix/profiles/default/lib/groonga/plugins /tmp/groonga-plugins/ + +#################### +# Stage 2: Gosu builder +#################### +FROM alpine:3.21 AS gosu-builder + +ARG TARGETARCH +ARG GOSU_VERSION=1.16 + +RUN apk add --no-cache gnupg curl + +# Download and verify gosu +RUN curl -fsSL "https://github.com/tianon/gosu/releases/download/${GOSU_VERSION}/gosu-${TARGETARCH}" -o /usr/local/bin/gosu && \ + curl -fsSL "https://github.com/tianon/gosu/releases/download/${GOSU_VERSION}/gosu-${TARGETARCH}.asc" -o /usr/local/bin/gosu.asc && \ + GNUPGHOME="$(mktemp -d)" && \ + export GNUPGHOME && \ + gpg --batch --keyserver hkps://keys.openpgp.org --recv-keys B42F6819007F00F88E364FD4036A9C25BF357DD4 && \ + gpg --batch --verify /usr/local/bin/gosu.asc /usr/local/bin/gosu && \ + rm -rf "$GNUPGHOME" /usr/local/bin/gosu.asc && \ + chmod +x /usr/local/bin/gosu + +#################### +# Stage 3: Final production image +#################### +FROM alpine:3.21 AS production + +# Install minimal runtime dependencies +RUN apk add --no-cache \ + bash \ + curl \ + shadow \ + su-exec \ + tzdata \ + musl-locales \ + musl-locales-lang \ + && rm -rf /var/cache/apk/* + +# Create postgres user/group +RUN addgroup -S postgres && \ + adduser -S -G postgres -h /var/lib/postgresql -s /bin/bash postgres && \ + addgroup -S wal-g && \ + adduser -S -G wal-g -s /bin/bash wal-g && \ + adduser postgres wal-g + +# Copy Nix store and profiles from builder (written to /nix-output to escape cache mount) +COPY --from=nix-builder /nix-output /nix + +# Copy groonga plugins +COPY --from=nix-builder /tmp/groonga-plugins/plugins /usr/lib/groonga/plugins + +# Copy gosu +COPY --from=gosu-builder /usr/local/bin/gosu /usr/local/bin/gosu + +# Setup PostgreSQL directories +RUN mkdir -p /usr/lib/postgresql/bin \ + /usr/lib/postgresql/share/postgresql \ + /usr/share/postgresql \ + /var/lib/postgresql/data \ + /var/run/postgresql \ + && chown -R postgres:postgres /usr/lib/postgresql \ + && chown -R postgres:postgres /var/lib/postgresql \ + && chown -R postgres:postgres /usr/share/postgresql \ + && chown -R postgres:postgres /var/run/postgresql + +# Create symbolic links for binaries +RUN for f in /nix/var/nix/profiles/default/bin/*; do \ + ln -sf "$f" /usr/lib/postgresql/bin/ 2>/dev/null || true; \ + ln -sf "$f" /usr/bin/ 2>/dev/null || true; \ + done + +# Create symbolic links for PostgreSQL shares +RUN ln -sf /nix/var/nix/profiles/default/share/postgresql/* /usr/lib/postgresql/share/postgresql/ 2>/dev/null || true && \ + ln -sf /nix/var/nix/profiles/default/share/postgresql/* /usr/share/postgresql/ 2>/dev/null || true && \ + ln -sf /usr/lib/postgresql/share/postgresql/timezonesets /usr/share/postgresql/timezonesets 2>/dev/null || true + +# Set permissions +RUN chown -R postgres:postgres /usr/lib/postgresql && \ + chown -R postgres:postgres /usr/share/postgresql + +# Setup configs +COPY --chown=postgres:postgres ansible/files/postgresql_config/postgresql.conf.j2 /etc/postgresql/postgresql.conf +COPY --chown=postgres:postgres ansible/files/postgresql_config/pg_hba.conf.j2 /etc/postgresql/pg_hba.conf +COPY --chown=postgres:postgres ansible/files/postgresql_config/pg_ident.conf.j2 /etc/postgresql/pg_ident.conf +COPY --chown=postgres:postgres ansible/files/postgresql_config/conf.d /etc/postgresql-custom/conf.d +COPY --chown=postgres:postgres ansible/files/postgresql_config/postgresql-stdout-log.conf /etc/postgresql/logging.conf +COPY --chown=postgres:postgres ansible/files/postgresql_config/supautils.conf.j2 /etc/postgresql-custom/supautils.conf +COPY --chown=postgres:postgres ansible/files/postgresql_extension_custom_scripts /etc/postgresql-custom/extension-custom-scripts +COPY --chown=postgres:postgres ansible/files/pgsodium_getkey_urandom.sh.j2 /usr/lib/postgresql/bin/pgsodium_getkey.sh +COPY --chown=postgres:postgres ansible/files/postgresql_config/custom_walg.conf /etc/postgresql-custom/wal-g.conf +COPY --chown=postgres:postgres ansible/files/postgresql_config/custom_read_replica.conf /etc/postgresql-custom/read-replica.conf +COPY --chown=postgres:postgres ansible/files/walg_helper_scripts/wal_fetch.sh /home/postgres/wal_fetch.sh +COPY ansible/files/walg_helper_scripts/wal_change_ownership.sh /root/wal_change_ownership.sh + +# Configure PostgreSQL settings +RUN sed -i \ + -e "s|#unix_socket_directories = '/tmp'|unix_socket_directories = '/var/run/postgresql'|g" \ + -e "s|#session_preload_libraries = ''|session_preload_libraries = 'supautils'|g" \ + -e "s|#include = '/etc/postgresql-custom/supautils.conf'|include = '/etc/postgresql-custom/supautils.conf'|g" \ + -e "s|#include = '/etc/postgresql-custom/wal-g.conf'|include = '/etc/postgresql-custom/wal-g.conf'|g" /etc/postgresql/postgresql.conf && \ + echo "pgsodium.getkey_script= '/usr/lib/postgresql/bin/pgsodium_getkey.sh'" >> /etc/postgresql/postgresql.conf && \ + echo "vault.getkey_script= '/usr/lib/postgresql/bin/pgsodium_getkey.sh'" >> /etc/postgresql/postgresql.conf && \ + chown -R postgres:postgres /etc/postgresql-custom + +# Remove timescaledb and plv8 references (not in pg17) +RUN sed -i 's/ timescaledb,//g;' "/etc/postgresql/postgresql.conf" && \ + sed -i 's/db_user_namespace = off/#db_user_namespace = off/g;' "/etc/postgresql/postgresql.conf" && \ + sed -i 's/ timescaledb,//g; s/ plv8,//g' "/etc/postgresql-custom/supautils.conf" + +# Include schema migrations +COPY migrations/db /docker-entrypoint-initdb.d/ +COPY ansible/files/pgbouncer_config/pgbouncer_auth_schema.sql /docker-entrypoint-initdb.d/init-scripts/00-schema.sql +COPY ansible/files/stat_extension.sql /docker-entrypoint-initdb.d/migrations/00-extension.sql + +# Add entrypoint script +ADD --chmod=0755 \ + https://github.com/docker-library/postgres/raw/889f9447cd2dfe21cccfbe9bb7945e3b037e02d8/17/bullseye/docker-entrypoint.sh \ + /usr/local/bin/docker-entrypoint.sh + +# Setup pgsodium key script +RUN mkdir -p /usr/share/postgresql/extension/ && \ + ln -s /usr/lib/postgresql/bin/pgsodium_getkey.sh /usr/share/postgresql/extension/pgsodium_getkey && \ + chmod +x /usr/lib/postgresql/bin/pgsodium_getkey.sh + +# Environment variables +ENV PATH="/nix/var/nix/profiles/default/bin:/usr/lib/postgresql/bin:${PATH}" +ENV PGDATA=/var/lib/postgresql/data +ENV POSTGRES_HOST=/var/run/postgresql +ENV POSTGRES_USER=supabase_admin +ENV POSTGRES_DB=postgres +ENV POSTGRES_INITDB_ARGS="--allow-group-access --locale-provider=icu --encoding=UTF-8 --icu-locale=en_US.UTF-8" +ENV LANG=en_US.UTF-8 +ENV LANGUAGE=en_US:en +ENV LC_ALL=en_US.UTF-8 +ENV GRN_PLUGINS_DIR=/usr/lib/groonga/plugins +# Point to minimal glibc locales included in slim Nix package for initdb locale support +ENV LOCALE_ARCHIVE=/nix/var/nix/profiles/default/lib/locale/locale-archive + +ENTRYPOINT ["docker-entrypoint.sh"] +HEALTHCHECK --interval=2s --timeout=2s --retries=10 CMD pg_isready -U postgres -h localhost +STOPSIGNAL SIGINT +EXPOSE 5432 + +CMD ["postgres", "-D", "/etc/postgresql"] diff --git a/ansible/files/postgresql_config/postgresql.conf.j2 b/ansible/files/postgresql_config/postgresql.conf.j2 index 948135ef1e..74e78650fe 100644 --- a/ansible/files/postgresql_config/postgresql.conf.j2 +++ b/ansible/files/postgresql_config/postgresql.conf.j2 @@ -687,7 +687,8 @@ default_text_search_config = 'pg_catalog.english' #local_preload_libraries = '' #session_preload_libraries = '' -shared_preload_libraries = 'pg_stat_statements, pgaudit, plpgsql, plpgsql_check, pg_cron, pg_net, pgsodium, timescaledb, auto_explain, pg_tle, plan_filter, supabase_vault' # (change requires restart) +shared_preload_libraries = 'pg_stat_statements, pgaudit, plpgsql, plpgsql_check, pg_cron, pg_net, pgsodium, timescaledb, auto_explain, pg_tle, plan_filter, supabase_vault, pg_duckdb' # (change requires restart) +duckdb.postgres_role = 'duckdb_role' # (change requires restart) jit_provider = 'llvmjit' # JIT library to use # - Other Defaults - diff --git a/ansible/files/postgresql_config/supautils.conf.j2 b/ansible/files/postgresql_config/supautils.conf.j2 index 984090544f..d44c05e938 100644 --- a/ansible/files/postgresql_config/supautils.conf.j2 +++ b/ansible/files/postgresql_config/supautils.conf.j2 @@ -1,12 +1,12 @@ supautils.extensions_parameter_overrides = '{"pg_cron":{"schema":"pg_catalog"}}' supautils.policy_grants = '{"postgres":["auth.audit_log_entries","auth.flow_state","auth.identities","auth.instances","auth.mfa_amr_claims","auth.mfa_challenges","auth.mfa_factors","auth.oauth_clients","auth.one_time_tokens","auth.refresh_tokens","auth.saml_providers","auth.saml_relay_states","auth.sessions","auth.sso_domains","auth.sso_providers","auth.users","realtime.messages","realtime.subscription","storage.buckets","storage.buckets_analytics","storage.objects","storage.prefixes","storage.s3_multipart_uploads","storage.s3_multipart_uploads_parts"]}' supautils.drop_trigger_grants = '{"postgres":["auth.audit_log_entries","auth.flow_state","auth.identities","auth.instances","auth.mfa_amr_claims","auth.mfa_challenges","auth.mfa_factors","auth.oauth_clients","auth.one_time_tokens","auth.refresh_tokens","auth.saml_providers","auth.saml_relay_states","auth.sessions","auth.sso_domains","auth.sso_providers","auth.users","realtime.messages","realtime.subscription","storage.buckets","storage.buckets_analytics","storage.objects","storage.prefixes","storage.s3_multipart_uploads","storage.s3_multipart_uploads_parts"]}' -# full list: address_standardizer, address_standardizer_data_us, adminpack, amcheck, autoinc, bloom, btree_gin, btree_gist, citext, cube, dblink, dict_int, dict_xsyn, earthdistance, file_fdw, fuzzystrmatch, hstore, http, hypopg, index_advisor, insert_username, intagg, intarray, isn, lo, ltree, moddatetime, old_snapshot, orioledb, pageinspect, pg_buffercache, pg_cron, pg_freespacemap, pg_graphql, pg_hashids, pg_jsonschema, pg_net, pg_prewarm, pg_repack, pg_stat_monitor, pg_stat_statements, pg_surgery, pg_tle, pg_trgm, pg_visibility, pg_walinspect, pgaudit, pgcrypto, pgjwt, pgmq, pgroonga, pgroonga_database, pgrouting, pgrowlocks, pgsodium, pgstattuple, pgtap, plcoffee, pljava, plls, plpgsql, plpgsql_check, plv8, postgis, postgis_raster, postgis_sfcgal, postgis_tiger_geocoder, postgis_topology, postgres_fdw, refint, rum, seg, sslinfo, supabase_vault, supautils, tablefunc, tcn, timescaledb, tsm_system_rows, tsm_system_time, unaccent, uuid-ossp, vector, wrappers, xml2 +# full list: address_standardizer, address_standardizer_data_us, adminpack, amcheck, autoinc, bloom, btree_gin, btree_gist, citext, cube, dblink, dict_int, dict_xsyn, earthdistance, file_fdw, fuzzystrmatch, hstore, http, hypopg, index_advisor, insert_username, intagg, intarray, isn, lo, ltree, moddatetime, old_snapshot, orioledb, pageinspect, pg_buffercache, pg_cron, pg_duckdb, pg_freespacemap, pg_graphql, pg_hashids, pg_jsonschema, pg_net, pg_prewarm, pg_repack, pg_stat_monitor, pg_stat_statements, pg_surgery, pg_tle, pg_trgm, pg_visibility, pg_walinspect, pgaudit, pgcrypto, pgjwt, pgmq, pgroonga, pgroonga_database, pgrouting, pgrowlocks, pgsodium, pgstattuple, pgtap, plcoffee, pljava, plls, plpgsql, plpgsql_check, plv8, postgis, postgis_raster, postgis_sfcgal, postgis_tiger_geocoder, postgis_topology, postgres_fdw, refint, rum, seg, sslinfo, supabase_vault, supautils, tablefunc, tcn, timescaledb, tsm_system_rows, tsm_system_time, unaccent, uuid-ossp, vector, wrappers, xml2 # omitted because may be unsafe: adminpack, amcheck, file_fdw, lo, old_snapshot, pageinspect, pg_freespacemap, pg_surgery, pg_visibility # omitted because deprecated: intagg, xml2 # omitted because doesn't require superuser: pgmq # omitted because protected: plpgsql -supautils.privileged_extensions = 'address_standardizer, address_standardizer_data_us, autoinc, bloom, btree_gin, btree_gist, citext, cube, dblink, dict_int, dict_xsyn, earthdistance, fuzzystrmatch, hstore, http, hypopg, index_advisor, insert_username, intarray, isn, ltree, moddatetime, orioledb, pg_buffercache, pg_cron, pg_graphql, pg_hashids, pg_jsonschema, pg_net, pg_prewarm, pg_repack, pg_stat_monitor, pg_stat_statements, pg_tle, pg_trgm, pg_walinspect, pgaudit, pgcrypto, pgjwt, pgroonga, pgroonga_database, pgrouting, pgrowlocks, pgsodium, pgstattuple, pgtap, plcoffee, pljava, plls, plpgsql_check, plv8, postgis, postgis_raster, postgis_sfcgal, postgis_tiger_geocoder, postgis_topology, postgres_fdw, refint, rum, seg, sslinfo, supabase_vault, supautils, tablefunc, tcn, timescaledb, tsm_system_rows, tsm_system_time, unaccent, uuid-ossp, vector, wrappers' +supautils.privileged_extensions = 'address_standardizer, address_standardizer_data_us, autoinc, bloom, btree_gin, btree_gist, citext, cube, dblink, dict_int, dict_xsyn, earthdistance, fuzzystrmatch, hstore, http, hypopg, index_advisor, insert_username, intarray, isn, ltree, moddatetime, orioledb, pg_buffercache, pg_cron, pg_duckdb, pg_graphql, pg_hashids, pg_jsonschema, pg_net, pg_prewarm, pg_repack, pg_stat_monitor, pg_stat_statements, pg_tle, pg_trgm, pg_walinspect, pgaudit, pgcrypto, pgjwt, pgroonga, pgroonga_database, pgrouting, pgrowlocks, pgsodium, pgstattuple, pgtap, plcoffee, pljava, plls, plpgsql_check, plv8, postgis, postgis_raster, postgis_sfcgal, postgis_tiger_geocoder, postgis_topology, postgres_fdw, refint, rum, seg, sslinfo, supabase_vault, supautils, tablefunc, tcn, timescaledb, tsm_system_rows, tsm_system_time, unaccent, uuid-ossp, vector, wrappers' supautils.extension_custom_scripts_path = '/etc/postgresql-custom/extension-custom-scripts' supautils.privileged_extensions_superuser = 'supabase_admin' supautils.privileged_role = 'supabase_privileged_role' diff --git a/ansible/files/postgresql_extension_custom_scripts/pg_duckdb/after-create.sql b/ansible/files/postgresql_extension_custom_scripts/pg_duckdb/after-create.sql new file mode 100644 index 0000000000..cbd015db97 --- /dev/null +++ b/ansible/files/postgresql_extension_custom_scripts/pg_duckdb/after-create.sql @@ -0,0 +1 @@ +grant usage on foreign data wrapper duckdb to duckdb_role; diff --git a/ansible/vars.yml b/ansible/vars.yml index 8d612b970b..241a8c4530 100644 --- a/ansible/vars.yml +++ b/ansible/vars.yml @@ -10,9 +10,9 @@ postgres_major: # Full version strings for each major version postgres_release: - postgresorioledb-17: "17.6.0.061-orioledb" - postgres17: "17.6.1.104" - postgres15: "15.14.1.104" + postgresorioledb-17: "17.6.0.061-orioledb-duckdbv2" + postgres17: "17.6.1.104-duckdbv2" + postgres15: "15.14.1.104-duckdbv2" # Non Postgres Extensions pgbouncer_release: 1.25.1 diff --git a/migrations/db/migrations/20260309201836_pg_duckdb_grants.sql b/migrations/db/migrations/20260309201836_pg_duckdb_grants.sql new file mode 100644 index 0000000000..5dcacd4fdb --- /dev/null +++ b/migrations/db/migrations/20260309201836_pg_duckdb_grants.sql @@ -0,0 +1,20 @@ +-- migrate:up + +-- Create a shared group role for DuckDB access. +-- Both postgres (developer/admin) and service_role (runtime API) need to run +-- DuckDB queries. We use a group role rather than cross-granting between them, +-- which mirrors the supabase_privileged_role pattern. +-- +-- The FDW grant (GRANT USAGE ON FOREIGN DATA WRAPPER duckdb TO duckdb_role) is +-- handled by ansible/files/postgresql_extension_custom_scripts/pg_duckdb/after-create.sql +-- rather than an event trigger, following the established pattern for postgres_fdw etc. +DO $$ +BEGIN + IF NOT EXISTS (SELECT FROM pg_roles WHERE rolname = 'duckdb_role') THEN + CREATE ROLE duckdb_role; + GRANT duckdb_role TO postgres WITH ADMIN OPTION; + GRANT duckdb_role TO service_role, supabase_admin; + END IF; +END $$; + +-- migrate:down diff --git a/nix/ext/duckdb-lib.nix b/nix/ext/duckdb-lib.nix new file mode 100644 index 0000000000..b5f3a07648 --- /dev/null +++ b/nix/ext/duckdb-lib.nix @@ -0,0 +1,69 @@ +{ + lib, + stdenv, + fetchFromGitHub, + cmake, + ninja, + openssl, + python3, +}: + +stdenv.mkDerivation (finalAttrs: { + pname = "duckdb"; + version = "1.5.1"; + + src = fetchFromGitHub { + owner = "duckdb"; + repo = "duckdb"; + rev = "v${finalAttrs.version}"; + hash = "sha256-FygBpfhvezvUbI969Dta+vZOPt6BnSW2d5gO4I4oB2A="; + }; + + outputs = [ + "out" + "lib" + "dev" + ]; + + # cmake installs the shared library and headers; nothing goes into $out + # since BUILD_SHELL=OFF (no CLI binary). We create $out explicitly so + # nixpkgs's multi-output setup hooks have a valid fallback for outputBin, + # outputDoc, outputMan, outputInfo, etc. + postInstall = '' + mkdir -p $out + ''; + + nativeBuildInputs = [ + cmake + ninja + # python3 is required by DuckDB's cmake build scripts for code generation + python3 + ]; + + buildInputs = [ openssl ]; + + cmakeFlags = [ + # Required by pg_duckdb so that DuckDB symbols are visible when loaded + # by PostgreSQL's dlopen. Without this, pg_duckdb's .so cannot resolve + # DuckDB symbols at runtime. + "-DCXX_EXTRA=-fvisibility=default" + (lib.cmakeBool "BUILD_SHELL" false) + (lib.cmakeBool "BUILD_PYTHON" false) + (lib.cmakeBool "BUILD_UNITTESTS" false) + # Prevent cmake from trying to fetch anything from the internet + (lib.cmakeBool "FETCHCONTENT_FULLY_DISCONNECTED" true) + # Embed the version string so DuckDB doesn't report "unknown" + (lib.cmakeFeature "OVERRIDE_GIT_DESCRIBE" "v${finalAttrs.version}-0-g0000000") + ]; + + # Skip the test suite — we just want the library + doInstallCheck = false; + doCheck = false; + + meta = { + description = "DuckDB shared library (for use by pg_duckdb)"; + homepage = "https://duckdb.org/"; + license = lib.licenses.mit; + platforms = lib.platforms.unix; + }; +}) diff --git a/nix/ext/pg_duckdb.nix b/nix/ext/pg_duckdb.nix new file mode 100644 index 0000000000..8be22b5adb --- /dev/null +++ b/nix/ext/pg_duckdb.nix @@ -0,0 +1,128 @@ +{ + lib, + stdenv, + fetchFromGitHub, + postgresql, + buildEnv, + lz4, + patchelf, + duckdb-lib, + latestOnly ? false, +}: +let + pname = "pg_duckdb"; + version = "1.1.1"; + + drv = stdenv.mkDerivation { + inherit pname version; + + src = fetchFromGitHub { + owner = "TylerHillery"; + repo = "pg_duckdb"; + rev = "eb7af6ede232f951c78192c79109c6d0be73b7b2"; + hash = "sha256-nhnxaesR9IOZWkfUFJ5ds+2OpcstMQ9OpUvsHGqnN7Y="; + }; + + nativeBuildInputs = lib.optionals (!stdenv.isDarwin) [ patchelf ]; + + buildInputs = [ + postgresql + duckdb-lib.lib + duckdb-lib.dev + lz4 + ]; + + # No configure script — uses PGXS directly via Makefile.global + dontConfigure = true; + + postPatch = '' + # The GitHub tarball does not include the third_party/duckdb git submodule. + # pg_duckdb's Makefile expects DuckDB headers at: + # third_party/duckdb/src/include + # third_party/duckdb/third_party/re2 + # and the built library at: + # third_party/duckdb/build/release/src/libduckdb + # We satisfy all three using our pre-built duckdb-lib derivation. + + mkdir -p third_party/duckdb/src + ln -sf ${duckdb-lib.dev}/include third_party/duckdb/src/include + + # pg_duckdb adds re2 as a system include for warning suppression only; + # it does not include re2 headers directly. + mkdir -p third_party/duckdb/third_party/re2 + + # Make the pre-built libduckdb visible at the path the Makefile's linker + # flags expect: -L$(DUCKDB_BUILD_DIR)/src -lduckdb + # (DUCKDB_BUILD_DIR = third_party/duckdb/build/release) + mkdir -p third_party/duckdb/build/release/src + ln -sf ${duckdb-lib.lib}/lib/libduckdb${postgresql.dlSuffix} \ + third_party/duckdb/build/release/src/libduckdb${postgresql.dlSuffix} + + # The Makefile has two separate dependencies on .git/modules/third_party/duckdb/HEAD: + # 1. $(FULL_DUCKDB_LIB) — recipe overridden to no-op below + # 2. $(OBJS) — every .o depends on it directly (Makefile:98) to ensure + # DuckDB headers are present before compilation. + # Fake the sentinel so make considers it satisfied without running git. + mkdir -p .git/modules/third_party/duckdb + echo "ref: refs/heads/main" > .git/modules/third_party/duckdb/HEAD + + # Override the FULL_DUCKDB_LIB build recipe with a no-op. + # GNU Make uses the last-defined recipe when a target appears multiple times. + # This prevents make from invoking cmake/ninja to build DuckDB from source; + # the pre-built library is already in place via the symlink above. + printf '\n# Nix override: skip DuckDB cmake build\n$(FULL_DUCKDB_LIB):\n\t@:\n' >> Makefile + ''; + + NIX_LDFLAGS = lib.optionalString stdenv.isDarwin "-headerpad_max_install_names"; + + makeFlags = [ + "PG_CONFIG=${postgresql}/bin/pg_config" + ]; + + installPhase = '' + runHook preInstall + mkdir -p $out/{lib,share/postgresql/extension} + + install -Dm755 pg_duckdb${postgresql.dlSuffix} $out/lib/pg_duckdb${postgresql.dlSuffix} + + # Fix rpath so pg_duckdb.so finds libduckdb in the Nix store at runtime. + # PostgreSQL uses dlopen() to load extensions, so the rpath in the .so + # file must point to an absolute path where libduckdb lives. + ${lib.optionalString (!stdenv.isDarwin) '' + ${patchelf}/bin/patchelf \ + --set-rpath "${duckdb-lib.lib}/lib:${postgresql}/lib" \ + $out/lib/pg_duckdb${postgresql.dlSuffix} + ''} + ${lib.optionalString stdenv.isDarwin '' + install_name_tool \ + -add_rpath "${duckdb-lib.lib}/lib" \ + -add_rpath "${postgresql}/lib" \ + $out/lib/pg_duckdb${postgresql.dlSuffix} + ''} + + cp pg_duckdb.control $out/share/postgresql/extension/ + cp sql/pg_duckdb--*.sql $out/share/postgresql/extension/ + + runHook postInstall + ''; + + meta = { + description = "DuckDB-powered analytical queries inside PostgreSQL"; + homepage = "https://github.com/duckdb/pg_duckdb"; + platforms = postgresql.meta.platforms; + license = lib.licenses.mit; + }; + }; +in +buildEnv { + name = pname; + paths = [ drv ]; + pathsToLink = [ + "/lib" + "/share/postgresql/extension" + ]; + + passthru = { + inherit pname version latestOnly; + }; +} diff --git a/nix/packages/default.nix b/nix/packages/default.nix index 7b1a6ea546..e3a0c8d95f 100644 --- a/nix/packages/default.nix +++ b/nix/packages/default.nix @@ -40,6 +40,7 @@ build-test-ami = pkgs.callPackage ./build-test-ami.nix { packer = self'.packages.packer; }; cleanup-ami = pkgs.callPackage ./cleanup-ami.nix { }; dbmate-tool = pkgs.callPackage ./dbmate-tool.nix { inherit (self.supabase) defaults; }; + duckdb-lib = pkgs.callPackage ../ext/duckdb-lib.nix { }; docker-image-inputs = pkgs.callPackage ./docker-image-inputs.nix { psql_15_slim = self'.packages."psql_15_slim/bin"; psql_17_slim = self'.packages."psql_17_slim/bin"; diff --git a/nix/packages/postgres.nix b/nix/packages/postgres.nix index 87daccdae7..04fe533e83 100644 --- a/nix/packages/postgres.nix +++ b/nix/packages/postgres.nix @@ -50,6 +50,7 @@ ../ext/wrappers/default.nix ../ext/supautils.nix ../ext/plv8 + ../ext/pg_duckdb.nix ]; #Where we import and build the orioledb extension, we add on our custom extensions @@ -59,7 +60,11 @@ x: x != ../ext/timescaledb.nix && x != ../ext/timescaledb-2.9.1.nix && x != ../ext/plv8 ) ourExtensions; - orioledbExtensions = orioleFilteredExtensions ++ [ ../ext/orioledb.nix ]; + # pg_duckdb is excluded from orioledb because orioledb's PostgreSQL fork + # modifies the TableAM API in ways that are incompatible with pg_duckdb v1.1.1. + orioledbExtensions = (builtins.filter (x: x != ../ext/pg_duckdb.nix) orioleFilteredExtensions) ++ [ + ../ext/orioledb.nix + ]; dbExtensions17 = orioleFilteredExtensions; # CLI extensions - minimal set for Supabase CLI with migration support @@ -137,6 +142,7 @@ inherit postgresql latestOnly; switch-ext-version = extCallPackage ./switch-ext-version.nix { }; overlayfs-on-package = extCallPackage ./overlayfs-on-package.nix { }; + duckdb-lib = pkgs.callPackage ../ext/duckdb-lib.nix { }; } ); in