From fd1d4a6f4d48979ecd8a3f15ded8566149d904a3 Mon Sep 17 00:00:00 2001 From: Lucas Vieira Date: Mon, 27 Apr 2026 17:35:16 -0300 Subject: [PATCH 1/2] feat(rds): publish prebuilt fakecloud-postgres images, pull-first runtime Move the postgres image build out of every user's first CreateDBInstance and into CI, where it runs once per release tag and lands on ghcr.io/faiscadev/fakecloud-postgres:- (linux/amd64 + linux/arm64). Each release also gets a rolling : tag. - New workflow .github/workflows/docker-rds-images.yml triggers on v*/workflow_dispatch, mirroring docker.yml's per-arch build + manifest-merge pattern. workflow_dispatch lets us backfill the already-pushed v0.13.0 tag once this lands. - RdsRuntime::ensure_postgres_image is now pull-first: inspect -> pull -> local build (covers dev / unreleased / airgapped). Local-build path keeps the embedded Dockerfile + SQL. - New env knobs: FAKECLOUD_POSTGRES_REGISTRY (registry override) and FAKECLOUD_REBUILD_POSTGRES_IMAGE (force local rebuild after asset edits). - Drop the per-content-hash tag suffix and the sha2 dependency added in #802; tag is now just -, kept in sync with releases. - Unit test pins the tag formula across the default and override registry paths. --- .github/workflows/docker-rds-images.yml | 128 +++++++++++ Cargo.lock | 1 - crates/fakecloud-rds/Cargo.toml | 1 - .../fakecloud-rds/assets/postgres/Dockerfile | 6 + crates/fakecloud-rds/src/runtime.rs | 198 ++++++++++++++---- website/content/docs/services/rds.md | 17 +- website/content/local-rds.md | 2 +- 7 files changed, 304 insertions(+), 49 deletions(-) create mode 100644 .github/workflows/docker-rds-images.yml diff --git a/.github/workflows/docker-rds-images.yml b/.github/workflows/docker-rds-images.yml new file mode 100644 index 00000000..d8afc976 --- /dev/null +++ b/.github/workflows/docker-rds-images.yml @@ -0,0 +1,128 @@ +name: RDS support images + +# Publishes ghcr.io//fakecloud-postgres:- for every +# supported postgres major (13/14/15/16) on each release tag, plus a +# rolling : tag pointing at the latest release. Image content = +# postgres: + plpython3u + the aws_lambda / aws_commons extension +# files in `crates/fakecloud-rds/assets/postgres`. +# +# Mirrors the structure of docker.yml: per-arch build with +# `push-by-digest`, then a per-major merge job that creates the manifest +# list with the human-readable tags. Manual `workflow_dispatch` exists so +# we can backfill released tags after this workflow first lands. + +on: + push: + tags: ["v*"] + workflow_dispatch: + +env: + REGISTRY: ghcr.io + IMAGE_BASE: ghcr.io/${{ github.repository_owner }}/fakecloud-postgres + +jobs: + build: + strategy: + fail-fast: false + matrix: + pg_version: ["13", "14", "15", "16"] + include: + - platform: linux/amd64 + runner: ubuntu-24.04 + - platform: linux/arm64 + runner: ubuntu-24.04-arm + runs-on: ${{ matrix.runner }} + permissions: + contents: read + packages: write + + steps: + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3 + + - name: Log in to GitHub Container Registry + uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Build and push by digest + id: build + uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6 + with: + context: crates/fakecloud-rds/assets/postgres + build-args: | + PG_VERSION=${{ matrix.pg_version }} + platforms: ${{ matrix.platform }} + cache-from: type=gha,scope=postgres-${{ matrix.pg_version }}-${{ matrix.platform }} + cache-to: type=gha,scope=postgres-${{ matrix.pg_version }}-${{ matrix.platform }},mode=max + outputs: type=image,name=${{ env.IMAGE_BASE }},push-by-digest=true,name-canonical=true,push=true + + - name: Export digest + run: | + mkdir -p /tmp/digests + digest="${{ steps.build.outputs.digest }}" + touch "/tmp/digests/${digest#sha256:}" + + - name: Upload digest + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: digest-postgres-${{ matrix.pg_version }}-${{ matrix.runner }} + path: /tmp/digests/* + if-no-files-found: error + retention-days: 1 + + merge: + runs-on: ubuntu-24.04 + needs: build + permissions: + contents: read + packages: write + strategy: + fail-fast: false + matrix: + pg_version: ["13", "14", "15", "16"] + + steps: + - name: Download digests + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4 + with: + path: /tmp/digests + pattern: digest-postgres-${{ matrix.pg_version }}-* + merge-multiple: true + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3 + + - name: Log in to GitHub Container Registry + uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Extract metadata + id: meta + uses: docker/metadata-action@c299e40c65443455700f0fdfc63efafe5b349051 # v5 + with: + images: ${{ env.IMAGE_BASE }} + # Pinned tag - on every semver tag, + # rolling tag only for tag pushes (so workflow_dispatch + # on a non-tag ref is a no-op rather than overwriting : + # with a non-release build). + tags: | + type=semver,pattern=${{ matrix.pg_version }}-{{version}} + type=raw,value=${{ matrix.pg_version }},enable=${{ startsWith(github.ref, 'refs/tags/v') }} + + - name: Create manifest list and push + working-directory: /tmp/digests + run: | + docker buildx imagetools create $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \ + $(printf '${{ env.IMAGE_BASE }}@sha256:%s ' *) + + - name: Inspect image + run: | + docker buildx imagetools inspect ${{ env.IMAGE_BASE }}:${{ steps.meta.outputs.version }} diff --git a/Cargo.lock b/Cargo.lock index aaea0cda..47a37cf1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3046,7 +3046,6 @@ dependencies = [ "parking_lot", "serde", "serde_json", - "sha2 0.10.9", "tempfile", "thiserror 2.0.18", "tokio", diff --git a/crates/fakecloud-rds/Cargo.toml b/crates/fakecloud-rds/Cargo.toml index 1bf66732..a3478e36 100644 --- a/crates/fakecloud-rds/Cargo.toml +++ b/crates/fakecloud-rds/Cargo.toml @@ -25,5 +25,4 @@ tokio-postgres = { workspace = true } mysql_async = "0.34" base64 = { workspace = true } tracing = { workspace = true } -sha2 = { workspace = true } tempfile = { workspace = true } diff --git a/crates/fakecloud-rds/assets/postgres/Dockerfile b/crates/fakecloud-rds/assets/postgres/Dockerfile index 7ec37949..3b70d089 100644 --- a/crates/fakecloud-rds/assets/postgres/Dockerfile +++ b/crates/fakecloud-rds/assets/postgres/Dockerfile @@ -1,3 +1,9 @@ +# Built and pushed on each fakecloud release tag by +# .github/workflows/docker-rds-images.yml as +# ghcr.io/faiscadev/fakecloud-postgres:- +# (plus a rolling : tag). RdsRuntime::ensure_postgres_image +# tries to pull that tag first and falls back to building from this +# Dockerfile locally when the pull fails (dev / unreleased / airgapped). ARG PG_VERSION=16 FROM postgres:${PG_VERSION} ARG PG_VERSION diff --git a/crates/fakecloud-rds/src/runtime.rs b/crates/fakecloud-rds/src/runtime.rs index 5cf09d7b..099f8c66 100644 --- a/crates/fakecloud-rds/src/runtime.rs +++ b/crates/fakecloud-rds/src/runtime.rs @@ -3,7 +3,6 @@ use std::sync::Arc; use std::time::Duration; use parking_lot::RwLock; -use sha2::{Digest, Sha256}; use tokio_postgres::NoTls; const POSTGRES_DOCKERFILE: &str = include_str!("../assets/postgres/Dockerfile"); @@ -12,6 +11,14 @@ const AWS_COMMONS_SQL: &str = include_str!("../assets/postgres/aws_commons--1.0. const AWS_LAMBDA_CONTROL: &str = include_str!("../assets/postgres/aws_lambda.control"); const AWS_LAMBDA_SQL: &str = include_str!("../assets/postgres/aws_lambda--1.0.sql"); +/// Default registry that hosts the prebuilt postgres images. CI publishes +/// to `ghcr.io/faiscadev/fakecloud-postgres:-` on each +/// release tag (see `.github/workflows/docker-rds-images.yml`). +/// Override with the `FAKECLOUD_POSTGRES_REGISTRY` env var (e.g. for +/// private mirrors); set `FAKECLOUD_REBUILD_POSTGRES_IMAGE=1` to force +/// a local rebuild even when the published tag is reachable. +const DEFAULT_POSTGRES_REGISTRY: &str = "ghcr.io/faiscadev"; + #[derive(Debug, Clone)] pub struct RunningDbContainer { pub container_id: String, @@ -574,19 +581,30 @@ impl RdsRuntime { /// `CREATE EXTENSION aws_lambda CASCADE` inside any database. /// Tag includes a content hash so changes to the embedded assets /// invalidate the local cache automatically. + /// Resolve the postgres image tag for a given major version. Tries + /// (in order): in-process cache, `docker image inspect` for a copy + /// already on the daemon, `docker pull` of the prebuilt image + /// published by CI, and finally a local `docker build` from the + /// embedded Dockerfile + extension assets. The pull path is the + /// happy path for end users on tagged releases; the build path + /// covers dev / unreleased versions / airgapped setups. + /// + /// Honors: + /// - `FAKECLOUD_POSTGRES_REGISTRY` — registry prefix (default + /// `ghcr.io/faiscadev`); useful for private mirrors. + /// - `FAKECLOUD_REBUILD_POSTGRES_IMAGE` — when set to a non-empty + /// value, skip inspect + pull and force a fresh local build. + /// Use after editing the embedded Dockerfile or extension SQL. pub(crate) async fn ensure_postgres_image( &self, major_version: &str, ) -> Result { - let tag = format!( - "fakecloud-postgres:{}-{}", - major_version, - postgres_assets_hash() - ); + let tag = postgres_image_tag(major_version); - // Per-tag mutex so concurrent first-creates don't both shell out - // to `docker build` for the same image. Inner bool tracks whether - // the build has already succeeded in this process. + // Per-tag mutex so concurrent first-creates don't all shell out + // to docker. Inner bool tracks whether resolution has succeeded + // in this process (regardless of whether it landed via inspect, + // pull, or build). let lock = { let mut cache = self.image_cache.write(); cache @@ -594,26 +612,77 @@ impl RdsRuntime { .or_insert_with(|| Arc::new(tokio::sync::Mutex::new(false))) .clone() }; - let mut built = lock.lock().await; - if *built { + let mut resolved = lock.lock().await; + if *resolved { return Ok(tag); } - // Even within a single process, the image may already exist on - // the daemon from a prior run. Skip the build if `image inspect` - // succeeds. - let inspect = tokio::process::Command::new(&self.cli) - .args(["image", "inspect", &tag]) + let force_rebuild = std::env::var("FAKECLOUD_REBUILD_POSTGRES_IMAGE") + .map(|v| !v.is_empty()) + .unwrap_or(false); + + if !force_rebuild { + // Already on the daemon (prior pull, prior build, prior + // session)? Use it as-is. + if self.docker_image_exists(&tag).await { + *resolved = true; + return Ok(tag); + } + + // Try the prebuilt image published by CI. Any failure + // (404 for unreleased version, network error, auth) falls + // through to the local build branch. + if self.try_pull_image(&tag).await { + *resolved = true; + return Ok(tag); + } + } + + self.build_postgres_image_local(major_version, &tag).await?; + *resolved = true; + Ok(tag) + } + + async fn docker_image_exists(&self, tag: &str) -> bool { + tokio::process::Command::new(&self.cli) + .args(["image", "inspect", tag]) .stdout(std::process::Stdio::null()) .stderr(std::process::Stdio::null()) .status() .await - .map_err(|e| RuntimeError::ContainerStartFailed(e.to_string()))?; - if inspect.success() { - *built = true; - return Ok(tag); + .map(|status| status.success()) + .unwrap_or(false) + } + + async fn try_pull_image(&self, tag: &str) -> bool { + tracing::info!(tag = %tag, "Pulling prebuilt fakecloud-postgres image"); + let output = match tokio::process::Command::new(&self.cli) + .args(["pull", tag]) + .output() + .await + { + Ok(output) => output, + Err(e) => { + tracing::debug!(tag = %tag, error = %e, "docker pull failed to spawn"); + return false; + } + }; + if output.status.success() { + return true; } + tracing::info!( + tag = %tag, + stderr = %String::from_utf8_lossy(&output.stderr).trim(), + "Prebuilt postgres image not available, falling back to local build" + ); + false + } + async fn build_postgres_image_local( + &self, + major_version: &str, + tag: &str, + ) -> Result<(), RuntimeError> { let build_dir = tempfile::tempdir().map_err(|e| RuntimeError::ContainerStartFailed(e.to_string()))?; let assets: [(&str, &str); 5] = [ @@ -631,7 +700,7 @@ impl RdsRuntime { tracing::info!( tag = %tag, - "Building fakecloud-postgres image with aws_lambda extension (first use can take ~60s)" + "Building fakecloud-postgres image locally (first use can take ~60s)" ); let output = tokio::process::Command::new(&self.cli) @@ -640,7 +709,7 @@ impl RdsRuntime { "--build-arg", &format!("PG_VERSION={major_version}"), "-t", - &tag, + tag, ".", ]) .current_dir(build_dir.path()) @@ -656,8 +725,7 @@ impl RdsRuntime { ))); } - *built = true; - Ok(tag) + Ok(()) } pub async fn dump_database( @@ -851,24 +919,66 @@ fn detect_bridge_gateway(cli: &str) -> Option { Some(gateway) } -/// Stable hash of the postgres image build context used as a tag suffix -/// — changes to the Dockerfile or extension files invalidate cached -/// images automatically. -fn postgres_assets_hash() -> &'static str { - use std::sync::OnceLock; - static HASH: OnceLock = OnceLock::new(); - HASH.get_or_init(|| { - let mut hasher = Sha256::new(); - hasher.update(POSTGRES_DOCKERFILE.as_bytes()); - hasher.update(AWS_COMMONS_CONTROL.as_bytes()); - hasher.update(AWS_COMMONS_SQL.as_bytes()); - hasher.update(AWS_LAMBDA_CONTROL.as_bytes()); - hasher.update(AWS_LAMBDA_SQL.as_bytes()); - let digest = hasher.finalize(); - digest.iter().take(6).fold(String::new(), |mut acc, b| { - use std::fmt::Write; - let _ = write!(acc, "{:02x}", b); - acc - }) - }) +/// Build the postgres image reference for a given major version. Uses +/// `/fakecloud-postgres:-`, where +/// the registry comes from `FAKECLOUD_POSTGRES_REGISTRY` (defaults to +/// the public `ghcr.io/faiscadev`). The version pin guarantees the +/// runtime asks the daemon for the same image CI publishes for this +/// fakecloud release; mismatched assets force a local rebuild via +/// the fall-through in `ensure_postgres_image`. +fn postgres_image_tag(major_version: &str) -> String { + let registry = std::env::var("FAKECLOUD_POSTGRES_REGISTRY") + .unwrap_or_else(|_| DEFAULT_POSTGRES_REGISTRY.to_string()); + let registry = registry.trim_end_matches('/'); + format!( + "{}/fakecloud-postgres:{}-{}", + registry, + major_version, + env!("CARGO_PKG_VERSION") + ) +} + +#[cfg(test)] +mod tests { + use super::*; + + /// Single test (rather than three) so the cases run sequentially — + /// `postgres_image_tag` reads a process-global env var and parallel + /// `cargo test` workers would race over it otherwise. + #[test] + fn postgres_image_tag_resolves_registry_overrides() { + let prev = std::env::var("FAKECLOUD_POSTGRES_REGISTRY").ok(); + + std::env::remove_var("FAKECLOUD_POSTGRES_REGISTRY"); + assert_eq!( + postgres_image_tag("16"), + format!( + "ghcr.io/faiscadev/fakecloud-postgres:16-{}", + env!("CARGO_PKG_VERSION") + ) + ); + + std::env::set_var("FAKECLOUD_POSTGRES_REGISTRY", "registry.example.com/team"); + assert_eq!( + postgres_image_tag("15"), + format!( + "registry.example.com/team/fakecloud-postgres:15-{}", + env!("CARGO_PKG_VERSION") + ) + ); + + std::env::set_var("FAKECLOUD_POSTGRES_REGISTRY", "registry.example.com/team/"); + assert_eq!( + postgres_image_tag("13"), + format!( + "registry.example.com/team/fakecloud-postgres:13-{}", + env!("CARGO_PKG_VERSION") + ) + ); + + match prev { + Some(v) => std::env::set_var("FAKECLOUD_POSTGRES_REGISTRY", v), + None => std::env::remove_var("FAKECLOUD_POSTGRES_REGISTRY"), + } + } } diff --git a/website/content/docs/services/rds.md b/website/content/docs/services/rds.md index 740bc840..dd1045be 100644 --- a/website/content/docs/services/rds.md +++ b/website/content/docs/services/rds.md @@ -93,7 +93,7 @@ When you call `CreateDBInstance` for PostgreSQL/MySQL/MariaDB/Oracle/SQL Server/ | Engine | Image | Port | Wait probe | |--------|-------|------|------------| -| `postgres` | `fakecloud-postgres:-` (built locally on top of `postgres:`, adds `plpython3u` + the `aws_lambda` and `aws_commons` extensions) | 5432 | `tokio-postgres` ping | +| `postgres` | `ghcr.io/faiscadev/fakecloud-postgres:-` (prebuilt with `plpython3u` + the `aws_lambda` and `aws_commons` extensions on top of `postgres:`; falls back to a local build if the pull fails) | 5432 | `tokio-postgres` ping | | `mysql` | `mysql:` | 3306 | `mysql_async` ping | | `mariadb` | `mariadb:` | 3306 | `mysql_async` ping | | `oracle-ee` / `oracle-se2` (+`-cdb`) | `gvenzl/oracle-free:23-slim` | 1521 | log marker `DATABASE IS READY TO USE!` + TCP probe | @@ -102,10 +102,23 @@ When you call `CreateDBInstance` for PostgreSQL/MySQL/MariaDB/Oracle/SQL Server/ The Oracle / SQL Server / Db2 images are large (1-3 GB) and take 30-300 s to first-boot. fakecloud passes the engine-specific license-acceptance environment variables (`ACCEPT_EULA`, `LICENSE`) automatically. Db2 launches with `--privileged` because the container needs it to set kernel parameters during startup. +### Prebuilt PostgreSQL image + +`ghcr.io/faiscadev/fakecloud-postgres:-` is published on every fakecloud release tag (workflow: `.github/workflows/docker-rds-images.yml`) for postgres `13`, `14`, `15`, `16`, both `linux/amd64` and `linux/arm64`. Each release also gets a rolling `:` tag pointing at the latest version for that major. Resolution order at runtime: + +1. Image already on the local Docker daemon -> use it. +2. `docker pull` of the version-pinned tag -> use it. +3. Local `docker build` from the embedded `crates/fakecloud-rds/assets/postgres/` Dockerfile (covers dev / unreleased / airgapped setups). + +Override knobs (env vars, both optional): + +- `FAKECLOUD_POSTGRES_REGISTRY=registry.example.com/team` — point at a private mirror (default `ghcr.io/faiscadev`). +- `FAKECLOUD_REBUILD_POSTGRES_IMAGE=1` — skip inspect + pull and force a fresh local build. Use after editing the embedded Dockerfile or extension SQL during development. + ## Gotchas - **Requires a Docker socket.** RDS needs access to `/var/run/docker.sock` to start and stop containers. -- **First use pulls the image.** Expect a slower first run while the database image downloads. Heavy engines (Oracle/SQL Server/Db2) can pull 1-3 GB on first use. +- **First use pulls the image.** Expect a slower first run while the database image downloads. Heavy engines (Oracle/SQL Server/Db2) can pull 1-3 GB on first use. The PostgreSQL image is custom (`ghcr.io/faiscadev/fakecloud-postgres:-`) and is pulled from the registry when available; otherwise it's built locally (~60 s). - **Aurora is partially supported.** Aurora-specific features (Global Database, Serverless v2, I/O-optimized) are recorded but don't affect the real container. - **Db2 needs `--privileged`.** fakecloud sets it automatically; the host must allow privileged containers. - **Heavy-engine boot is slow.** Oracle takes 30-90 s to first-boot, Db2 30-60 s, SQL Server ~30 s. Factor this into test budgets. diff --git a/website/content/local-rds.md b/website/content/local-rds.md index 7bb99cde..1aa05a54 100644 --- a/website/content/local-rds.md +++ b/website/content/local-rds.md @@ -16,7 +16,7 @@ Point your AWS SDK at `http://localhost:4566`. Docker required because fakecloud ## Why fakecloud for RDS - **163 RDS operations** at 100% conformance — DB instances, snapshots, read replicas, parameter groups, subnet groups, engine/version discovery, tagging, upgrades. -- **Real database engines.** fakecloud pulls real PostgreSQL / MySQL / MariaDB / Oracle / SQL Server / Db2 Docker images and runs them as the RDS instance. Your SQL schema, indexes, triggers, and extensions all work because they are real engines. +- **Real database engines.** fakecloud pulls real PostgreSQL / MySQL / MariaDB / Oracle / SQL Server / Db2 Docker images and runs them as the RDS instance. Your SQL schema, indexes, triggers, and extensions all work because they are real engines. PostgreSQL uses a prebuilt `ghcr.io/faiscadev/fakecloud-postgres:` image that bakes in `plpython3u` and the AWS RDS `aws_lambda` extension. - **Endpoint works.** `DescribeDBInstances` returns a real connectable host. Your application connects with the usual PostgreSQL / MySQL / Oracle / SQL Server / Db2 driver. - **Paid on LocalStack; free here.** RDS has always been LocalStack Pro-only. - **No account, no auth token, no paid tier.** AGPL-3.0. From 9eaddd0bd5e1ef06546d1ceb3503dff1487f2298 Mon Sep 17 00:00:00 2001 From: Lucas Vieira Date: Mon, 27 Apr 2026 17:41:35 -0300 Subject: [PATCH 2/2] =?UTF-8?q?fix(workflow):=20make=20rds=20image=20build?= =?UTF-8?q?=20matrix=20a=20real=20pg=5Fversion=20=C3=97=20platform=20cross?= =?UTF-8?q?-product?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `include` alone doesn't multiply axes; it adds entries to an existing expansion or matches keys. Promote `platform` to a top-level matrix axis so we get 4 (pg majors) × 2 (arches) = 8 build jobs, each with the right runner label resolved via include. --- .github/workflows/docker-rds-images.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.github/workflows/docker-rds-images.yml b/.github/workflows/docker-rds-images.yml index d8afc976..e2f7d9c5 100644 --- a/.github/workflows/docker-rds-images.yml +++ b/.github/workflows/docker-rds-images.yml @@ -26,6 +26,13 @@ jobs: fail-fast: false matrix: pg_version: ["13", "14", "15", "16"] + platform: + - linux/amd64 + - linux/arm64 + # `include` here matches each existing platform value and adds + # the `runner` key — together with the two-axis matrix above this + # produces 4×2 = 8 jobs each carrying pg_version, platform, and + # the right runner label. include: - platform: linux/amd64 runner: ubuntu-24.04